reranking and polishing

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+carotid_embeddings_sentence_transformers_061123.csv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,66 +1,32 @@
 import openai
 import numpy as np
 import pandas as pd
-from sentence_transformers import SentenceTransformer
+from sentence_transformers import SentenceTransformer, CrossEncoder
 import re
 import gradio as gr
 import json
-import requests
+import ast
+from llm import OpenAILLM
+from qa import QuestionAnswerer
 
-# Calculate the cosine similarity
-def cos_sim(vector1, vector2):
-   cosine_similarity = np.dot(vector1, vector2) / (np.linalg.norm(vector1) * np.linalg.norm(vector2))
-   return cosine_similarity
+df = pd.read_csv("/Users/mariusvach/Code/python/leitlinien_chatbot/carotis_chatbot/carotid_embeddings_sentence_transformers_061123.csv")
+df["embeddings"] = df.embeddings.apply(ast.literal_eval)
 
-def sim_search(df, query, n=3, dot=False):
-   embedding = model.encode(query)
-   if dot: 
-      df['similarities'] = df.embeddings.apply(lambda x: x@embedding)
-      print("using dot product")
-   else:
-      df['similarities'] = df.embeddings.apply(lambda x: cos_sim(x, embedding))
-      print("using cosine similarity")
-   res = df.sort_values('similarities', ascending=False).head(n)
-   return res
-
-def create_prompt(context, question):
-    return f"""
-    Context information is below.
-    ---------------------
-    {context}
-    ---------------------
-    Given the context information and not prior knowledge, answer the query.
-    Query: {question}
-    Answer: \
-    """
-
-
-def answer_question(question, model="gpt-3.5-turbo",n=3):
-    r = sim_search(df, question,n=n)
-    context = "\n\n".join(r.chunks)
-    prompt = create_prompt(context, question)
-    response = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant answering questions in german. You answer only in german. If you do not know an answer you say it. You do not fabricate answers."},
-            {"role": "user", "content": prompt},
-        ]
-    )
-    return response.choices[0].message.content
-
-df = pd.read_csv("carotid_embeddings_sentence_transformers.csv")
-df["embeddings"] = df.embeddings.apply(json.loads)
-model = SentenceTransformer('thenlper/gte-base')
+qa = QuestionAnswerer(df, SentenceTransformer('thenlper/gte-base'), OpenAILLM('gpt-3.5-turbo-16k'), CrossEncoder('cross-encoder/ms-marco-TinyBERT-L-2-v2'))
 
 def gradio_answer(input):
-    return answer_question(input, n=2)
+    return qa.answer_question(input, n=5, )
 
 desc_string = """
-In dieser Demo kannst du einer KI Fragen zum Inhalt der ['S3-Leitlinie Diagnostik, Therapie und Nachsorge der extracraniellen Carotisstenose'](https://register.awmf.org/de/leitlinien/detail/004-028) stellen. Ein paar Beispiele findest du unten. 
+In dieser Demo kannst du einer KI Fragen zum Inhalt der ['S3-Leitlinie Diagnostik, Therapie und Nachsorge der extracraniellen Carotisstenose'](https://register.awmf.org/de/leitlinien/detail/004-028) stellen. Ein paar Beispiel-Fragen findest du unten. 
+
+## Wie funktioniert das?
 
-### Wie funktioniert das?
+1. Die Frage wird durch ein neuronalen Netzwerk in eine Vektor-Repräsentation ('Embedding') umgewandelt. 
+2. Die Ähnlichkeit des 'Frage-Vektors' wird mit den genauso erzeugten Vektoren von Textpassagen der Leitlinie verglichen. 
+3. Ein ‚Large Language Model (LLM)‘ beantwortet nun mit Hilfe der ähnlichsten Textpassagen die gestellte Frage. 
 
-Mit Hilfe eines neuronalen Netzwerkes werden die Fragen in eine Vektor-Repräsentation ('Embedding') umgewandelt. Dann wird die Ähnlichkeit des 'Frage-Vektors' mit genauso erstellten Vektoren von Textpassagen der Leitlinie berechnet. Die Textpassagen, die am ähnlichsten sind, werden dann einem 'Large Language Model (LLM)' zur Beantwortung der Frage als Context gegeben. Diese Technik heißt ['retrieval-augmented Generation (RAG)'](https://research.ibm.com/blog/retrieval-augmented-generation-RAG).
+Diese Technik heißt [‚Retrieval-augmented Generation (RAG)'](https://research.ibm.com/blog/retrieval-augmented-generation-RAG). Hierdurch kann verhindert werden, dass LLMs Fakten einfach erfinden.
 """
 
 demo = gr.Interface(
@@ -70,9 +36,11 @@ demo = gr.Interface(
    title="🧠 Q&A S3 Leitlinie Carotisstenose", 
    description=desc_string,
    examples=[
-      "In welchen Intervallen ist eine Nachuntersuchung nach Carotis-Stent (CAS) angezeigt?",
+      "Welche Sensitivität hatte die Transcranielle Doppler-Sonographie (TCD) bei der Detektion eines perioperativen Schlaganfalls?",
       "Ist eine ambulante Therapie der Carotisstenose mittels CEA oder CAS möglich und sinnvoll?",
-      "Was sollte man als Bradykardie-Therapie bei Nachdilatation eines Stents einsetzen?"]
+      "Wie viele zerebrale Ischämien in Deutschland werden durch >50%ige Stenosen oder Verschlüsse der extracraniellen A. carotis verursacht?",
+      "Welche Symptome können durch Stenosen der A. carotis verursacht werden?"
+      ]
 )
     
 demo.launch()

carotid_embeddings_sentence_transformers_061123.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c16207bb8fbb5f55fdf33300e3d75043d36e8fb562854ebd585c20f2f595b0c0
+size 10699905

llm.py

@@ -0,0 +1,27 @@
+import openai
+import os
+
+openai.api_key = os.environ["OPENAI_API_KEY"]
+
+class BaseLLM:
+    def __init__(self, model):
+        self.model = model
+    
+    def get_response(self, system_prompt, query):
+        raise NotImplementedError
+    
+
+class OpenAILLM(BaseLLM):
+    def __init__(self, model):
+        self.model = model
+    
+    def get_response(self, system_prompt, query, **kwargs):
+        response = openai.ChatCompletion.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": query},
+            ],
+            **kwargs,
+        )
+        return response.choices[0].message.content

qa.py

@@ -0,0 +1,91 @@
+import numpy as np
+import pandas as pd
+from rank_bm25 import BM25Okapi
+
+from llm import OpenAILLM
+
+def cosine_similarity(vector1, vector2):
+    return np.dot(vector1, vector2) / (np.linalg.norm(vector1) * np.linalg.norm(vector2))
+
+class QuestionAnswerer:
+    def __init__(self, docs, embedding_model, llm=OpenAILLM('gpt-3.5-turbo'), cross_encoder=None):
+        self.docs = docs
+        self.bm25 = BM25Okapi([c.split(" ") for c in self.docs.chunks.values[1:]])
+        self.embedding_model = embedding_model
+        self.llm = llm
+        self.cross_encoder = cross_encoder
+
+    def sim_search(self, query, n=10, use_hyde=False, use_dot_product=False):
+        if use_hyde:
+            generated_doc = self._get_generated_doc(query)
+            print("generated document (hyde): \n", generated_doc)
+            embedding = self.embedding_model.encode(generated_doc)
+        else: 
+            embedding = self.embedding_model.encode(query)
+
+        if use_dot_product: 
+            similarities = self.docs['embeddings'].apply(lambda x: np.dot(x, embedding))
+        else:
+            similarities = self.docs['embeddings'].apply(lambda x: cosine_similarity(x, embedding))
+
+        self.docs['similarities'] = similarities
+        return self.docs.sort_values('similarities', ascending=False).head(n)
+    
+    def sim_search_rerank(self, query, n=10, sim_search_n=100, **kwargs):
+            search_results = self.sim_search(query, n=sim_search_n, use_hyde=False, **kwargs)
+            reranked_results = self.rerank(search_results, query)
+            return reranked_results[:n]
+    
+    def sim_search_bm25(self, query, n=10):
+            tokenized_query = query.split(" ")
+            doc_scores = self.bm25.get_scores(tokenized_query)
+            self.docs['bm25'] = np.insert(doc_scores, 0, 0) #hack because I have to remove the first item, because I cannot split it
+            result = self.docs.sort_values('bm25', ascending=False)[:n]
+            return result
+
+    def _create_prompt(self, context, question):
+        return f"""
+        Context information is below.
+        ---------------------
+        {context}
+        ---------------------
+        Given the context information and not prior knowledge, answer the query.
+        Query: {question}
+        Answer: \
+        """
+
+    def _get_generated_doc(self, question):
+        prompt = f"""Write a guideline section in German answering the question below
+        ---------------------
+        Question: {question}
+        ---------------------
+        Answer: \
+        """
+        system_prompt = "You are an experienced radiologist answering medical questions. You answer only in German."
+        return self.llm.get_response(system_prompt, prompt)
+    
+    
+    def rerank(self, docs, query): 
+        inp = [[query, doc.chunks] for doc in docs.itertuples()]
+        cross_scores = self.cross_encoder.predict(inp) if self.cross_encoder else []
+        docs['cross_score'] = cross_scores
+        return docs.sort_values('cross_score', ascending=False)
+    
+    def answer_question(self, question, n=3, use_hyde=False, use_reranker=False, use_bm25=False):
+        if use_reranker and use_hyde:
+            print('reranking together with hyde is not supported yet')
+        if use_reranker: 
+            search_results = self.sim_search_rerank(question, n=n)
+        if use_bm25:
+            search_results = self.sim_search_bm25(question, n=n)
+        else:
+            search_results = self.sim_search(question, n=n, use_hyde=use_hyde)
+        
+        context = "\n\n".join(search_results['chunks'])
+
+        prompt = self._create_prompt(context, question)
+
+        system_prompt = "You are a helpful assistant answering questions in German. You answer only in German. If you do not know an answer you say it. You do not fabricate answers."
+
+        return self.llm.get_response(system_prompt, prompt)
+