Help content parser

.vscode/launch.json

@@ -25,8 +25,10 @@
       "request": "launch",
       "program": "${workspaceFolder}/manage.py",
       "args": [
-          "trainChatbot",
-          "Não recebi o pagamento ainda >>> Você deve ter 160 horas de exibição de anúncios nas cidades que a Mobees já está presente, até a data do Fechamento. Caso você tenha completado as 160 horas, o pagamento acontece sempre no dia 10 do mês seguinte à data do Fechamento do ciclo – até às 23:59."
+          // "parseHelp",
+          // "./content.json"
+          // "trainChatbot",
+          // "Não recebi o pagamento ainda >>> Você deve ter 160 horas de exibição de anúncios nas cidades que a Mobees já está presente, até a data do Fechamento. Caso você tenha completado as 160 horas, o pagamento acontece sempre no dia 10 do mês seguinte à data do Fechamento do ciclo – até às 23:59."
       ],
       "env": {
           // "ENV_NAME": "production"

adm/constants.py

@@ -29,7 +29,7 @@ class CTS():
     GCP_QUEUES_FUNC = settings.GCP_QUEUES_FUNC
     GCP_DOC_AI_PROCESSOR_ID = settings.GCP_DOC_AI_PROCESSOR_ID
 
-    CHATBOT_STORAGE_PATH = 'chatbot/'
+    CHATBOT_STORAGE_PATH = 'chatbot'
     
     ADMIN_PROJECT_ID_GCP = "mobees-prd"
     ADMIN_TOPIC_GCP = "admin"

adm/management/commands/parseHelp.py

@@ -0,0 +1,49 @@
+from django.core.management.base import BaseCommand, CommandError
+import json
+import tempfile
+import os
+from io import BytesIO
+
+from adm.storage import MediaStorage
+from adm.constants import CTS
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument('file', type=str, help='JSON file')
+    
+    def handle(self, *args, **options):
+        file = options['file']
+        extension = '.json'
+        if file:
+            with open(file) as json_data:
+                content = json.load(json_data) #, encoding='utf-8'
+                content = content[0]
+            
+            sections = []
+            
+            for key, section in content.items():
+                if "type" in section and section["type"]=="info":
+                    sections.append({
+                        **section,
+                        'id': key,
+                        'type': 'help',
+                        'category': key,
+                        'description': 'This is a section of the Driver`s app Help page which is the source of truth of it`s category. Drivers can access it`s content by going to the Help page (❓) and tapping on the section`s title. Each section has a list of items, which can help Drivers on a specific topic (item`s title).'
+                    })
+            
+            for section in sections:
+                filename = 'help-' + section['id']
+                json_data = json.dumps(section, ensure_ascii=False)
+                temp_dir = tempfile.gettempdir()
+                file_path = "%s%s%s%s" % (temp_dir, os.sep, filename, extension)
+                with open(file_path, 'w') as f:
+                    f.write(json_data)
+                    
+                media_storage = MediaStorage(CTS.CHATBOT_STORAGE_PATH)
+                path_bucket = os.path.join(media_storage.location, 'docs/', filename + extension)
+                if media_storage.exists(path_bucket):
+                    media_storage.delete(path_bucket)
+                media_storage.save(path_bucket, BytesIO(json_data.encode("utf-8")))
+
+            

chatbot/main.py

@@ -23,9 +23,9 @@ class Chatbot():
     def __init__(self) -> None:
         objParameter = ParameterService()
         openai.api_key = objParameter.getParameterByKey("OPENAI_API_KEY").value
-        media_storage = MediaStorage()
-        self.media_storage = media_storage
         self.storage_path = CTS.CHATBOT_STORAGE_PATH
+        media_storage = MediaStorage(self.storage_path)
+        self.media_storage = media_storage
 
         # load index
         self.ai_storage_context = None
@@ -42,6 +42,9 @@ class Chatbot():
         pass
 
     def train (self, chat=None):
+        status = 400
+        data = None
+
         docs = []
         if chat:
             # use chat to build up on existing store
@@ -58,7 +61,7 @@ class Chatbot():
             blob_ext = '.json'
             file_name = blob_name + blob_ext
             blob_dir = 'instructions/'
-            blob_path = self.storage_path + blob_dir + file_name
+            blob_path = blob_dir + file_name
             text = json.dumps(chat, ensure_ascii=False)
             
             path_bucket = os.path.join(self.media_storage.location, '/', blob_path)
@@ -75,7 +78,7 @@ class Chatbot():
 
         else:
             blob_dir = 'docs/'
-            blobs = self.media_storage.list(self.storage_path + blob_dir)
+            blobs = self.media_storage.list(self.storage_path + '/' + blob_dir)
             for blob in blobs:
                 text = blob.download_as_bytes().decode('utf-8')
                 if len(text)>0:
@@ -108,7 +111,11 @@ class Chatbot():
                 self.ai_index.insert(ai_doc)
         
         # save index to disk
-        self.ai_index.storage_context.persist()
+        if ai_documents:
+            self.ai_index.storage_context.persist()
+            status = 200
+        
+        return status, data
 
     def chat (self, question=None):
         status = 400

chatbot/views.py

@@ -12,21 +12,20 @@ class ChatbotView (APIView):
     #     return Response(status=status, data=data)
 
     def post (self, request):
-        # get insight from a prompt
+        # get anwser from a question
         question = request.data.get('question')
 
         status, data = Chatbot.chat(question)
 
         return Response(status=status, data=data)
     
-    # def put (self, request, id=None):
-    #     # set insight exec score
-    #     etlUtil = ETLData()
-    #     score = etlUtil.getRequestData(request, ['score'])
+    def put (self, request):
+        # train chatbot with docs or chat instruction
+        chat = request.data.get('chat', None)
 
-    #     status, data = InsightsBdo().score(request, id, score)
+        status, data = Chatbot.train(chat)
 
-    #     return Response(status=status, data=data)
+        return Response(status=status, data=data)
     
     # def delete (self, request, id=None):
     #     # delete insight

mobeesia/mobeesia.log

@@ -98,3 +98,55 @@ Traceback (most recent call last):
   File "/usr/local/Cellar/python@3.9/3.9.17_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/encoder.py", line 179, in default
     raise TypeError(f'Object of type {o.__class__.__name__} '
 TypeError: Object of type Response is not JSON serializable
+ERROR | 2023-07-15 10:18:13 | Internal Server Error: /chatbot/train
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 34, in inner
+    response = get_response(request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 115, in _get_response
+    response = self.process_exception_by_middleware(e, request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 113, in _get_response
+    response = wrapped_callback(request, *callback_args, **callback_kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
+    return view_func(*args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/generic/base.py", line 71, in view
+    return self.dispatch(request, *args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 505, in dispatch
+    response = self.handle_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 465, in handle_exception
+    self.raise_uncaught_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
+    raise exc
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 502, in dispatch
+    response = handler(request, *args, **kwargs)
+  File "/Users/donzito/Documents/Projetos/mobees/dev/mobees-ai/chatbot/views.py", line 18, in post
+    status, data = Chatbot.chat(question)
+  File "/Users/donzito/Documents/Projetos/mobees/dev/mobees-ai/chatbot/main.py", line 132, in chat
+    answer = self.ai_index.as_query_engine(
+  File "/usr/local/lib/python3.9/site-packages/llama_index/indices/query/base.py", line 23, in query
+    response = self._query(str_or_query_bundle)
+  File "/usr/local/lib/python3.9/site-packages/llama_index/query_engine/retriever_query_engine.py", line 143, in _query
+    CBEventType.QUERY, payload={EventPayload.QUERY_STR: query_bundle.query_str}
+AttributeError: 'NoneType' object has no attribute 'query_str'
+ERROR | 2023-07-15 10:21:20 | Internal Server Error: /chatbot/train
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 34, in inner
+    response = get_response(request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 115, in _get_response
+    response = self.process_exception_by_middleware(e, request)
+  File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 113, in _get_response
+    response = wrapped_callback(request, *callback_args, **callback_kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
+    return view_func(*args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/django/views/generic/base.py", line 71, in view
+    return self.dispatch(request, *args, **kwargs)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 505, in dispatch
+    response = self.handle_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 465, in handle_exception
+    self.raise_uncaught_exception(exc)
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
+    raise exc
+  File "/usr/local/lib/python3.9/site-packages/rest_framework/views.py", line 502, in dispatch
+    response = handler(request, *args, **kwargs)
+  File "/Users/donzito/Documents/Projetos/mobees/dev/mobees-ai/chatbot/views.py", line 26, in put
+    status, data = Chatbot.train(chat)
+TypeError: cannot unpack non-iterable NoneType object