Chatbot training v0

.vscode/launch.json

@@ -10,10 +10,14 @@
       "request": "launch",
       "program": "${workspaceFolder}/manage.py",
       "args": [
-        "runserver"
+          "runserver",
+          "7003",
       ],
+      "env": {
+        // "ENV_NAME": "production"
+      },
       "django": true,
-      "justMyCode": true
+      "justMyCode" : false
     },
     {
       "name": "Command",
@@ -22,7 +26,7 @@
       "program": "${workspaceFolder}/manage.py",
       "args": [
           "trainChatbot",
-          "Olá >>> Olá {{name}}! Eu sou a Mel, bot da operação Mobees. Prazer! Fui criada para ajudar no relacionamento e manutenção dos equipamentos, e deixar nossa Parceria em fina sintonia! 😉"
+          "Não recebi o pagamento ainda >>> Você deve ter 160 horas de exibição de anúncios nas cidades que a Mobees já está presente, até a data do Fechamento. Caso você tenha completado as 160 horas, o pagamento acontece sempre no dia 10 do mês seguinte à data do Fechamento do ciclo – até às 23:59."
       ],
       "env": {
           // "ENV_NAME": "production"

adm/constants.py

@@ -28,6 +28,8 @@ class CTS():
     GCP_QUEUES_FST = settings.GCP_QUEUES_FST
     GCP_QUEUES_FUNC = settings.GCP_QUEUES_FUNC
     GCP_DOC_AI_PROCESSOR_ID = settings.GCP_DOC_AI_PROCESSOR_ID
+
+    CHATBOT_STORAGE_PATH = 'chatbot/'
     
     ADMIN_PROJECT_ID_GCP = "mobees-prd"
     ADMIN_TOPIC_GCP = "admin"

adm/management/commands/trainChatbot.py

@@ -1,6 +1,6 @@
 from django.core.management.base import BaseCommand, CommandError
-from chatbot.main import Chatbot
-
+from chatbot.main import instance as Chatbot
+from adm.constants import CTS
 
 class Command(BaseCommand):
 
@@ -11,10 +11,10 @@ class Command(BaseCommand):
         interaction = options['interaction']
         interaction = interaction.split('>>>')
         chat = {
-            'chat': ['User: ' + interaction[0]]
+            'chat': ['Driver: ' + interaction[0]]
         }
         for content in interaction[1:]:
             text = 'Support: '+ content
             chat['chat'].append(text)
         
-        Chatbot().train(chat)
+        Chatbot.train(chat)

chatbot/main.py

@@ -5,11 +5,15 @@ import sys
 import logging
 from datetime import datetime, timedelta
 from io import BytesIO
+from django.utils.text import slugify
 
 import openai
 from llama_index import VectorStoreIndex, Document, StorageContext, load_index_from_storage
+from llama_index.prompts  import Prompt
+from langchain.prompts.chat import (ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate)
 
-from adm.constants import CTS as cts
+
+from adm.constants import CTS
 from adm.storage import MediaStorage
 from adm.services import ParameterService
 
@@ -21,7 +25,7 @@ class Chatbot():
         openai.api_key = objParameter.getParameterByKey("OPENAI_API_KEY").value
         media_storage = MediaStorage()
         self.media_storage = media_storage
-        self.docs_path = 'chatbot/docs/'
+        self.storage_path = CTS.CHATBOT_STORAGE_PATH
 
         # load index
         self.ai_storage_context = None
@@ -33,7 +37,7 @@ class Chatbot():
             # exc_type, exc_obj, exc_tb = sys.exc_info()
             # fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
             # strErro = "ERRO: %s | %s | %s | %s" % (error, exc_type, fname, exc_tb.tb_lineno)
-            logger.info('Chatbot() init: '+error)
+            logger.info('Chatbot() init: '+str(error))
             pass
         pass
 
@@ -43,29 +47,35 @@ class Chatbot():
             # use chat to build up on existing store
             # save chat as doc in storage
             if not 'id' in chat:
-                chat['id'] = datetime.utcnow().replace(microsecond=0).isoformat()
+                question = chat['chat'][0].split(':')[-1]
+                chat['id'] = slugify(question)
             if not 'type' in chat:
-                chat['type'] = 'chat'
+                chat['type'] = 'instructions'
             if not 'category' in chat:
                 chat['category'] = 'uncategorized'
 
-            blob_name = self.docs_path + 'chat-' + chat['id'] + '.json'
+            blob_name = chat['id']
+            blob_ext = '.json'
+            file_name = blob_name + blob_ext
+            blob_dir = 'instructions/'
+            blob_path = self.storage_path + blob_dir + file_name
             text = json.dumps(chat, ensure_ascii=False)
             
-            path_bucket = os.path.join(self.media_storage.location, '/', blob_name)
+            path_bucket = os.path.join(self.media_storage.location, '/', blob_path)
             self.media_storage.save(path_bucket, BytesIO(text.encode("utf-8")))
 
             docs.append({
-                'id_': chat['id'],
+                'id_': blob_name,
                 'text': text,
                 'metadata': {
-                    'filename': blob_name, 
+                    'filename': file_name, 
                     'category': chat['category']
                 }
             })
 
         else:
-            blobs = self.media_storage.list(self.docs_path)
+            blob_dir = 'docs/'
+            blobs = self.media_storage.list(self.storage_path + blob_dir)
             for blob in blobs:
                 text = blob.download_as_bytes().decode('utf-8')
                 if len(text)>0:
@@ -74,11 +84,13 @@ class Chatbot():
                     if not 'category' in obj:  
                         obj['category'] = 'uncategorized'
                     
+                    file_name = blob.name.split('/')[-1]
+
                     docs.append({
                         'id_': obj['id'],
                         'text': text,
                         'metadata': {
-                            'filename': blob.name, 
+                            'filename': file_name, 
                             'category': obj['category']
                         }
                     })
@@ -98,9 +110,37 @@ class Chatbot():
         # save index to disk
         self.ai_index.storage_context.persist()
 
-        chat_engine = self.ai_index.as_chat_engine(
-            chat_mode='react', 
-            verbose=True
-        )
+    def chat (self, question=None):
+        status = 400
+
+        pre_prompt = [
+            SystemMessagePromptTemplate.from_template(
+                "Act as an IT Support Level 1 for the company Mobees. Your name is Mel. You must answer the company's Drivers questions in Brazilian Portuguese, as they only speak this language. Mobees Drivers access support through a chatbot interface in the company's app, which is the only channel for support. If you're not sure how to answer, respond with only the following `<assignee:support>` (this will let our system know it should forward the issue to our support team and take it from there). "
+            ),
+            HumanMessagePromptTemplate.from_template(
+                "Context information is below.\n"
+                "---------------------\n"
+                "{context_str}\n"
+                "---------------------\n"
+                "Given the context information and prior knowledge, "
+                "answer the question: {query_str}\n"
+            ),
+        ]
+        prompt_template = ChatPromptTemplate.from_messages(pre_prompt)
+        text_qa_template = Prompt.from_langchain_prompt(prompt_template)
+
+        answer = self.ai_index.as_query_engine(
+            text_qa_template=text_qa_template
+        ).query(question)
+        print(answer)
+
+        status = 200
+        data = { 'answer': answer.response }
+        return status, data
+
+        # chat_engine = self.ai_index.as_chat_engine(
+        #     verbose=True
+        # )
+        # chat_engine.chat_repl()
 
-        chat_engine.chat_repl()
+instance = Chatbot()

chatbot/urls.py

@@ -1,21 +1,8 @@
-"""mobeesia URL Configuration
-
-The `urlpatterns` list routes URLs to views. For more information please see:
-    https://docs.djangoproject.com/en/3.0/topics/http/urls/
-Examples:
-Function views
-    1. Add an import:  from my_app import views
-    2. Add a URL to urlpatterns:  path('', views.home, name='home')
-Class-based views
-    1. Add an import:  from other_app.views import Home
-    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
-Including another URLconf
-    1. Import the include() function: from django.urls import include, path
-    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
-"""
-from django.contrib import admin
 from django.urls import path
+from django.conf.urls import url
+from . import views
 
 urlpatterns = [
-    path('train', admin.site.urls),
-]
+    path('train', views.ChatbotView.as_view()),
+    path('chat', views.ChatbotView.as_view()),
+]

chatbot/views.py

@@ -1,3 +1,37 @@
+from pickle import decode_long
 from django.shortcuts import render
+from rest_framework.views import APIView, Response
+from django.http import HttpResponse
+from chatbot.main import instance as Chatbot
+
+class ChatbotView (APIView):
+
+    # def get (self, request):
+    #     status, data = InsightsBdo().list(request)
+
+    #     return Response(status=status, data=data)
+
+    def post (self, request):
+        # get insight from a prompt
+        question = request.data.get('question')
+
+        status, data = Chatbot.chat(question)
+
+        return Response(status=status, data=data)
+    
+    # def put (self, request, id=None):
+    #     # set insight exec score
+    #     etlUtil = ETLData()
+    #     score = etlUtil.getRequestData(request, ['score'])
+
+    #     status, data = InsightsBdo().score(request, id, score)
+
+    #     return Response(status=status, data=data)
+    
+    # def delete (self, request, id=None):
+    #     # delete insight
+        
+    #     status, data = InsightsBdo().delete(request, id)
+
+    #     return Response(status=status, data=data)
 
-# Create your views here.

mobeesia/mobeesia.log

@@ -0,0 +1,100 @@
+ERROR | 2023-07-14 15:45:03 | Internal Server Error: /chatbot/chat
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 34, in inner
+    response = get_response(request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 115, in _get_response
+    response = self.process_exception_by_middleware(e, request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 113, in _get_response
+    response = wrapped_callback(request, *callback_args, **callback_kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
+    return view_func(*args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/generic/base.py", line 71, in view
+    return self.dispatch(request, *args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 505, in dispatch
+    response = self.handle_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 465, in handle_exception
+    self.raise_uncaught_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
+    raise exc
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 502, in dispatch
+    response = handler(request, *args, **kwargs)
+  File "/Users/donzito/Documents/Projetos/mobees/dev/mobees-ai/chatbot/views.py", line 18, in post
+    status, data = Chatbot().chat(request, question)
+TypeError: 'Chatbot' object is not callable
+ERROR | 2023-07-14 15:47:59 | Internal Server Error: /chatbot/chat
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 34, in inner
+    response = get_response(request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 115, in _get_response
+    response = self.process_exception_by_middleware(e, request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 113, in _get_response
+    response = wrapped_callback(request, *callback_args, **callback_kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
+    return view_func(*args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/generic/base.py", line 71, in view
+    return self.dispatch(request, *args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 505, in dispatch
+    response = self.handle_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 465, in handle_exception
+    self.raise_uncaught_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
+    raise exc
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 502, in dispatch
+    response = handler(request, *args, **kwargs)
+  File "/Users/donzito/Documents/Projetos/mobees/dev/mobees-ai/chatbot/views.py", line 18, in post
+    status, data = Chatbot.chat(request, question)
+TypeError: chat() takes from 1 to 2 positional arguments but 3 were given
+ERROR | 2023-07-14 15:50:02 | Internal Server Error: /chatbot/chat
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 34, in inner
+    response = get_response(request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 145, in _get_response
+    response = self.process_exception_by_middleware(e, request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 143, in _get_response
+    response = response.render()
+  File "/usr/local/lib/python3.9/site-packages/django/template/response.py", line 105, in render
+    self.content = self.rendered_content
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/response.py", line 70, in rendered_content
+    ret = renderer.render(self.data, accepted_media_type, context)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/renderers.py", line 100, in render
+    ret = json.dumps(
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/utils/json.py", line 25, in dumps
+    return json.dumps(*args, **kwargs)
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/__init__.py", line 234, in dumps
+    return cls(
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/encoder.py", line 199, in encode
+    chunks = self.iterencode(o, _one_shot=True)
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/encoder.py", line 257, in iterencode
+    return _iterencode(o, 0)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/utils/encoders.py", line 67, in default
+    return super().default(obj)
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/encoder.py", line 179, in default
+    raise TypeError(f'Object of type {o.__class__.__name__} '
+TypeError: Object of type Response is not JSON serializable
+ERROR | 2023-07-14 15:56:00 | Internal Server Error: /chatbot/chat
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 34, in inner
+    response = get_response(request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 145, in _get_response
+    response = self.process_exception_by_middleware(e, request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 143, in _get_response
+    response = response.render()
+  File "/usr/local/lib/python3.9/site-packages/django/template/response.py", line 105, in render
+    self.content = self.rendered_content
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/response.py", line 70, in rendered_content
+    ret = renderer.render(self.data, accepted_media_type, context)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/renderers.py", line 100, in render
+    ret = json.dumps(
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/utils/json.py", line 25, in dumps
+    return json.dumps(*args, **kwargs)
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/__init__.py", line 234, in dumps
+    return cls(
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/encoder.py", line 199, in encode
+    chunks = self.iterencode(o, _one_shot=True)
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/encoder.py", line 257, in iterencode
+    return _iterencode(o, 0)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/utils/encoders.py", line 67, in default
+    return super().default(obj)
+  File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/encoder.py", line 179, in default
+    raise TypeError(f'Object of type {o.__class__.__name__} '
+TypeError: Object of type Response is not JSON serializable

mobeesia/urls.py

@@ -1,7 +1,6 @@
 
-from django.conf.urls import url
-from django.conf.urls import include
+from django.urls import include, path
 
 urlpatterns = [
-    url(r'^', include('chatbot.urls')),
+    path('chatbot/', include('chatbot.urls')),
 ]

requirements.txt

@@ -30,6 +30,6 @@ google-auth-httplib2
 google-auth-oauthlib
 googleapis-common-protos
 google-cloud-storage
-llama-index==0.7.6
+llama-index==0.7.8
 llama_hub
 openai

storage/graph_store.json

@@ -0,0 +1 @@
+{"graph_dict": {}}

storage/index_store.json

@@ -0,0 +1 @@
+{"index_store/data": {"84d83981-5f1c-436e-980e-cb4d2b5bfc2f": {"__type__": "vector_store", "__data__": "{\"index_id\": \"84d83981-5f1c-436e-980e-cb4d2b5bfc2f\", \"summary\": null, \"nodes_dict\": {\"05521fae-9547-47df-91f0-7728692e5ba2\": \"05521fae-9547-47df-91f0-7728692e5ba2\", \"24d8c798-5015-4e04-b07d-7e910256f3b9\": \"24d8c798-5015-4e04-b07d-7e910256f3b9\", \"5537300b-8c48-4203-8273-a68153d1403e\": \"5537300b-8c48-4203-8273-a68153d1403e\", \"6f47f1de-ca31-44df-bcd8-8dc87ba58de2\": \"6f47f1de-ca31-44df-bcd8-8dc87ba58de2\"}, \"doc_id_dict\": {}, \"embeddings_dict\": {}}"}}}