using complete local code and loading llm through ctransformers.

app.py

@@ -1,23 +1,200 @@
+# import gradio as gr
+# from qdrant_client import models, QdrantClient
+# from sentence_transformers import SentenceTransformer
+# from PyPDF2 import PdfReader
+# from langchain.text_splitter import RecursiveCharacterTextSplitter
+# from langchain.callbacks.manager import CallbackManager
+# from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+# # from langchain.llms import LlamaCpp
+# from langchain.vectorstores import Qdrant
+# from qdrant_client.http import models
+# # from langchain.llms import CTransformers
+# from ctransformers import AutoModelForCausalLM
+
+
+
+
+
+# # loading the embedding model - 
+
+# encoder = SentenceTransformer('jinaai/jina-embedding-b-en-v1')
+
+# print("embedding model loaded.............................")
+# print("####################################################")
+
+# # loading the LLM 
+
+# callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
+
+# print("loading the LLM......................................")
+
+# # llm = LlamaCpp(
+# # model_path="TheBloke/Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q8_0.gguf",
+# # n_ctx=2048,
+# # f16_kv=True, # MUST set to True, otherwise you will run into problem after a couple of calls
+# # callback_manager=callback_manager,
+# # verbose=True,
+# # )
+
+# llm = AutoModelForCausalLM.from_pretrained("TheBloke/Llama-2-7B-Chat-GGUF", 
+# model_file="llama-2-7b-chat.Q8_0.gguf", 
+# model_type="llama", 
+# # config = ctransformers.hub.AutoConfig,
+# # hf = True
+# temperature = 0.2,
+# max_new_tokens = 1024,
+# stop = ['\n']
+# )
+
+
+
+# print("LLM loaded........................................")
+# print("################################################################")
+
+# def get_chunks(text):
+# text_splitter = RecursiveCharacterTextSplitter(
+# # seperator = "\n",
+# chunk_size = 500,
+# chunk_overlap = 100,
+# length_function = len,
+# )
+
+# chunks = text_splitter.split_text(text)
+# return chunks
+
+
+# pdf_path = './100 Weird Facts About the Human Body.pdf'
+
+
+# reader = PdfReader(pdf_path)
+# text = ""
+# num_of_pages = len(reader.pages)
+# for page in range(num_of_pages):
+# current_page = reader.pages[page]
+# text += current_page.extract_text()
+
+
+# chunks = get_chunks(text)
+
+# print("Chunks are ready.....................................")
+# print("######################################################")
+
+# qdrant = QdrantClient(path = "./db")
+# print("db created................................................")
+# print("#####################################################################")
+
+# qdrant.recreate_collection(
+# collection_name="my_facts",
+# vectors_config=models.VectorParams(
+# size=encoder.get_sentence_embedding_dimension(), # Vector size is defined by used model
+# distance=models.Distance.COSINE,
+# ),
+# )
+
+# print("Collection created........................................")
+# print("#########################################################")
+
+
+
+# li = []
+# for i in range(len(chunks)):
+# li.append(i)
+ 
+# dic = zip(li, chunks)
+# dic= dict(dic)
+
+# qdrant.upload_records(
+# collection_name="my_facts",
+# records=[
+# models.Record(
+# id=idx,
+# vector=encoder.encode(dic[idx]).tolist(),
+# payload= {dic[idx][:5] : dic[idx]}
+# ) for idx in dic.keys()
+# ],
+# )
+
+# print("Records uploaded........................................")
+# print("###########################################################")
+
+# def chat(question):
+# # question = input("ask question from pdf.....")
+
+
+# hits = qdrant.search(
+# collection_name="my_facts",
+# query_vector=encoder.encode(question).tolist(),
+# limit=3
+# )
+# context = []
+# for hit in hits:
+# context.append(list(hit.payload.values())[0])
+ 
+# context = context[0] + context[1] + context[2]
+
+# system_prompt = """You are a helpful assistant, you will use the provided context to answer user questions.
+# Read the given context before answering questions and think step by step. If you can not answer a user question based on 
+# the provided context, inform the user. Do not use any other information for answering user. Provide a detailed answer to the question."""
+
+
+# B_INST, E_INST = "[INST]", "[/INST]"
+
+# B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
+
+# SYSTEM_PROMPT = B_SYS + system_prompt + E_SYS
+
+# instruction = f""" 
+# Context: {context}
+# User: {question}"""
+
+# prompt_template = B_INST + SYSTEM_PROMPT + instruction + E_INST
+
+# result = llm(prompt_template)
+# return result 
+
+# gr.Interface(
+# fn = chat,
+# inputs = gr.Textbox(lines = 10, placeholder = "Enter your question here 👉"),
+# outputs = gr.Textbox(lines = 10, placeholder = "Your answer will be here soon 🚀"),
+# title="Q&N with PDF 👩🏻‍💻📓✍🏻💡",
+# description="This app facilitates a conversation with PDFs available on https://www.delo.si/assets/media/other/20110728/100%20Weird%20Facts%20About%20the%20Human%20Body.pdf💡",
+# theme="soft",
+# examples=["Hello", "what is the speed of human nerve impulses?"],
+# # cache_examples=True,
+# ).launch() 
+
+
 import gradio as gr
+from threading import Thread
+from queue import SimpleQueue
+from typing import Any, Dict, List, Union
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.schema import LLMResult
 from qdrant_client import models, QdrantClient
 from sentence_transformers import SentenceTransformer
 from PyPDF2 import PdfReader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+from qdrant_client.models import PointStruct
+import os
 from langchain.callbacks.manager import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
-# from langchain.llms import LlamaCpp
+# from qdrant_client import QdrantClient
+# from langchain import VectorDBQA - This is obsolete
+from langchain.chains import RetrievalQA 
+from langchain.llms import LlamaCpp
+# from PyPDF2 import PdfReader
 from langchain.vectorstores import Qdrant
+# from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings import HuggingFaceBgeEmbeddings
+from transformers import AutoModel
 from qdrant_client.http import models
-# from langchain.llms import CTransformers
+# from sentence_transformers import SentenceTransformer
+from langchain.prompts import PromptTemplate
 from ctransformers import AutoModelForCausalLM
 
-
-
-
-
 # loading the embedding model - 
 
-encoder = SentenceTransformer('jinaai/jina-embedding-b-en-v1')
+encoder = SentenceTransformer("all-MiniLM-L6-v2")
 
 print("embedding model loaded.............................")
 print("####################################################")
@@ -29,7 +206,9 @@ callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
 print("loading the LLM......................................")
 
 # llm = LlamaCpp(
-# model_path="TheBloke/Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q8_0.gguf",
+# model_path="/home/devangpagare/llm/models/llama-2-7b-chat.Q3_K_S.gguf",
+# # n_gpu_layers=n_gpu_layers,
+# # n_batch=n_batch,
 # n_ctx=2048,
 # f16_kv=True, # MUST set to True, otherwise you will run into problem after a couple of calls
 # callback_manager=callback_manager,
@@ -37,17 +216,16 @@ print("loading the LLM......................................")
 # )
 
 llm = AutoModelForCausalLM.from_pretrained("TheBloke/Llama-2-7B-Chat-GGUF", 
- model_file="llama-2-7b-chat.Q8_0.gguf", 
+ model_file="llama-2-7b-chat.Q3_K_S.gguf", 
  model_type="llama", 
  # config = ctransformers.hub.AutoConfig,
  # hf = True
  temperature = 0.2,
- max_new_tokens = 1024,
- stop = ['\n']
+ # max_new_tokens = 1024,
+ # stop = ['\n']
  )
 
 
-
 print("LLM loaded........................................")
 print("################################################################")
 
@@ -75,7 +253,7 @@ for page in range(num_of_pages):
 
 
 chunks = get_chunks(text)
-
+print(chunks)
 print("Chunks are ready.....................................")
 print("######################################################")
 
@@ -95,11 +273,11 @@ print("Collection created........................................")
 print("#########################################################")
 
 
-
+# starting a list of same size as chunks
 li = []
 for i in range(len(chunks)):
  li.append(i)
- 
+# concantinating the li and chunks to create a dcitionary 
 dic = zip(li, chunks)
 dic= dict(dic)
 
@@ -110,6 +288,8 @@ qdrant.upload_records(
  id=idx,
  vector=encoder.encode(dic[idx]).tolist(),
  payload= {dic[idx][:5] : dic[idx]}
+## payload is always suppose to be a dictionary with both keys and values as strings. To do this, I used first 5 chars of
+## every value as key to make the payload.
  ) for idx in dic.keys()
  ],
 )
@@ -128,8 +308,11 @@ def chat(question):
  )
  context = []
  for hit in hits:
+ # print(hit.payload, "score:", hit.score)
  context.append(list(hit.payload.values())[0])
- 
+ # context += str(hit.payload[hit.payload.values()[:5]])
+ # print("##################################################################")
+
  context = context[0] + context[1] + context[2]
 
  system_prompt = """You are a helpful assistant, you will use the provided context to answer user questions.
@@ -162,3 +345,5 @@ gr.Interface(
  examples=["Hello", "what is the speed of human nerve impulses?"],
  # cache_examples=True,
 ).launch() 
+
+