Update rag.py

app/rag.py

@@ -26,7 +26,7 @@ class ChatPDF:
     pdf_count = 0
 
     def __init__(self):
-        self.text_parser = SentenceSplitter(chunk_size=128, chunk_overlap=8)
+        self.text_parser = SentenceSplitter(chunk_size=1024, chunk_overlap=24)
 
         logger.info("initializing the vector store related objects")
         # client = QdrantClient(host="localhost", port=6333)
@@ -45,9 +45,9 @@ class ChatPDF:
         llm = LlamaCPP(
             model_url="https://huggingface.co/Qwen/Qwen2-0.5B-Instruct-GGUF/resolve/main/qwen2-0_5b-instruct-fp16.gguf",
             temperature=0.1,
-            max_new_tokens=256,
+            max_new_tokens=128,
             # context_window=3900, #32k
-            generate_kwargs={"max_tokens": 256, "temperature": 0.1, "top_k": 12},
+            generate_kwargs={"max_tokens": 128, "temperature": 0.1, "top_k": 3},
             # model_kwargs={"n_gpu_layers": -1},
             # messages_to_prompt=self.messages_to_prompt,
             # completion_to_prompt=self.completion_to_prompt,
@@ -103,7 +103,7 @@ class ChatPDF:
 
         self.query_engine = index.as_query_engine(
             streaming=True,
-            similarity_top_k=12,
+            similarity_top_k=3,
         )
 
     def ask(self, query: str):