working with multiple files + better prompt

.gitignore

@@ -1 +1,2 @@
 venv/
+Universal Sentence Encoder/

app.py

@@ -18,6 +18,7 @@ def preprocess(text):
     return text
 
 
+# converts pdf to text
 def pdf_to_text(path, start_page=1, end_page=None):
     doc = fitz.open(path)
     total_pages = doc.page_count
@@ -35,10 +36,11 @@ def pdf_to_text(path, start_page=1, end_page=None):
     doc.close()
     return text_list
 
+# one text converts a list of chunks
+def text_to_chunks(texts, word_length=150, start_page=1, file_number=1):
 
-def text_to_chunks(texts, word_length=150, start_page=1):
-    text_toks = [t.split(' ') for t in texts]
-    page_nums = []
+    filtered_texts = [''.join(char for char in text if ord(char) < 128) for text in texts]
+    text_toks = [t.split(' ') for t in filtered_texts]
     chunks = []
     
     for idx, words in enumerate(text_toks):
@@ -49,15 +51,16 @@ def text_to_chunks(texts, word_length=150, start_page=1):
                 text_toks[idx+1] = chunk + text_toks[idx+1]
                 continue
             chunk = ' '.join(chunk).strip()
-            chunk = f'[Page no. {idx+start_page}]' + ' ' + '"' + chunk + '"'
+            chunk = f'[PDF no. {file_number}] [Page no. {idx+start_page}]' + ' ' + '"' + chunk + '"'
             chunks.append(chunk)
+    # print("chunks == " + str(chunks))
     return chunks
 
 
 class SemanticSearch:
     
     def __init__(self):
-        self.use = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')
+        self.use = hub.load('./Universal Sentence Encoder/')
         self.fitted = False
     
     
@@ -91,10 +94,13 @@ class SemanticSearch:
 
 
 
-def load_recommender(path, start_page=1):
+def load_recommender(paths, start_page=1):
     global recommender
-    texts = pdf_to_text(path, start_page=start_page)
-    chunks = text_to_chunks(texts, start_page=start_page)
+    texts = []
+    chunks = []
+    for idx, path in enumerate(paths):
+        chunks += text_to_chunks(pdf_to_text(path, start_page=start_page), start_page=start_page, file_number=idx+1)
+    # print("chunks == " +  str(chunks))
     recommender.fit(chunks)
     return 'Corpus Loaded.'
 
@@ -148,13 +154,15 @@ def generate_answer(question, openAI_key, model):
     return answer
 
 
-def question_answer(chat_history, url, file, question, openAI_key, model):
+def question_answer(chat_history, url, files, question, openAI_key, model):    
     try:
+        if files == None: 
+            files = []
         if openAI_key.strip()=='':
             return '[ERROR]: Please enter your Open AI Key. Get your key here : https://platform.openai.com/account/api-keys'
-        if url.strip() == '' and file is None:
+        if url.strip() == '' and files == []:
             return '[ERROR]: Both URL and PDF is empty. Provide at least one.'
-        if url.strip() != '' and file is not None:
+        if url.strip() != '' and files is not []:
             return '[ERROR]: Both URL and PDF is provided. Please provide only one (either URL or PDF).'
         if model is None or model =='':
             return '[ERROR]: You have not selected any model. Please choose an LLM model.'
@@ -163,11 +171,17 @@ def question_answer(chat_history, url, file, question, openAI_key, model):
             download_pdf(glob_url, 'corpus.pdf')
             load_recommender('corpus.pdf')
         else:
-            old_file_name = file.name
-            file_name = file.name
-            file_name = file_name[:-12] + file_name[-4:]
-            os.rename(old_file_name, file_name)
-            load_recommender(file_name)
+            print(files)
+            filenames = []
+            for file in files:
+                old_file_name = file.name
+                file_name = file.name
+                file_name = file_name[:-12] + file_name[-4:]
+                os.rename(old_file_name, file_name)
+                filenames.append(file_name)
+            load_recommender(filenames)
+			
+                 
         if question.strip() == '':
             return '[ERROR]: Question field is empty'
         if model == "text-davinci-003" or model == "gpt-4" or model == "gpt-4-32k":
@@ -197,21 +211,24 @@ def generate_text_text_davinci_003(openAI_key,prompt, engine="text-davinci-003")
 
 def generate_answer_text_davinci_003(question,openAI_key):
     topn_chunks = recommender(question)
+    # print("topn chunks == " + str(topn_chunks))
     prompt = ""
     prompt += 'search results:\n\n'
     for c in topn_chunks:
         prompt += c + '\n\n'
         
     prompt += "Instructions: Compose a comprehensive reply to the query using the search results given. "\
-              "Cite each reference using [ Page Number] notation (every result has this number at the beginning). "\
+              "Cite each reference using [PDF Number][Page Number] notation (every result has this number at the beginning). "\
               "Citation should be done at the end of each sentence. If the search results mention multiple subjects "\
               "with the same name, create separate answers for each. Only include information found in the results and "\
               "don't add any additional information. Make sure the answer is correct and don't output false content. "\
               "If the text does not relate to the query, simply state 'Found Nothing'. Ignore outlier "\
               "search results which has nothing to do with the question. Only answer what is asked. The "\
-              "answer should be short and concise. \n\nQuery: {question}\nAnswer: "
+              "answer should be short and concise.\n\n"
     
     prompt += f"Query: {question}\nAnswer:"
+    print("prompt == " + str(prompt))
+    # print("prompt == " + str(prompt))
     answer = generate_text_text_davinci_003(openAI_key, prompt,"text-davinci-003")
     return answer
 
@@ -248,9 +265,9 @@ with gr.Blocks(css="""#chatbot { font-size: 14px; min-height: 1200; }""") as dem
             gr.Markdown(f'<p style="text-align:center">Get your Open AI API key <a href="https://platform.openai.com/account/api-keys">here</a></p>')
             with gr.Accordion("API Key"):
                 openAI_key = gr.Textbox(label='Enter your OpenAI API key here', password=True)
-                url = gr.Textbox(label='Enter PDF URL here   (Example: https://arxiv.org/pdf/1706.03762.pdf )')
+                url = gr.Textbox(label='Enter PDF URL here  (Example: https://arxiv.org/pdf/1706.03762.pdf )')
                 gr.Markdown("<center><h4>OR<h4></center>")
-                file = gr.File(label='Upload your PDF/ Research Paper / Book here', file_types=['.pdf'])
+                files = gr.File(label='Upload your PDF/ Research Paper / Book here', file_types=['.pdf'], file_count="multiple")
             question = gr.Textbox(label='Enter your question here')
             gr.Examples(
                 [[q] for q in questions],
@@ -274,11 +291,11 @@ with gr.Blocks(css="""#chatbot { font-size: 14px; min-height: 1200; }""") as dem
             chatbot = gr.Chatbot(placeholder="Chat History", label="Chat History", lines=50, elem_id="chatbot")
 
 
-#
+  
     # Bind the click event of the button to the question_answer function
     btn.click(
         question_answer,
-        inputs=[chatbot, url, file, question, openAI_key, model],
+        inputs=[chatbot, url, files, question, openAI_key, model],
         outputs=[chatbot],
     )