added gradio app

.gitignore

@@ -1 +1,2 @@
-__pycache__/
+__pycache__/
+.chroma/

app.py

@@ -0,0 +1,172 @@
+from langchain.embeddings import TensorflowHubEmbeddings
+from langchain.chains import ConversationalRetrievalChain
+from langchain.vectorstores import Chroma
+from langchain.llms import OpenAI
+from tqdm import tqdm
+import pandas as pd
+import gradio as gr
+import openai
+import praw
+import os
+import re
+
+reddit = None
+bot = None
+chat_history = []
+
+def set_openai_key(key):
+
+    if key == "":
+        key = os.environ.get("OPENAI_API_KEY")
+
+    openai.api_key = key
+
+def set_reddit_keys(client_id, client_secret, user_agent):
+
+    global reddit
+
+    # If any of the keys are empty, use the environment variables
+    if [client_id, client_secret, user_agent] == ["", "", ""]:
+        client_id = os.environ.get("REDDIT_CLIENT_ID")
+        client_secret = os.environ.get("REDDIT_CLIENT_SECRET")
+        user_agent = os.environ.get("REDDIT_USER_AGENT")
+
+    reddit = praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)
+
+def generate_topics(query, model="gpt-3.5-turbo"):
+
+    messages = [
+        {"role": "user", "content": f"Take this query '{query}' and return a list of 10 simple to understand topics (4 words or less) to input in Search so it returns good results."},
+    ]
+
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=messages,
+        temperature=0
+    )
+
+    response_message = response["choices"][0]["message"]["content"]
+
+    topics = re.sub(r'^\d+\.\s*', '', response_message, flags=re.MULTILINE).split("\n")
+
+    # Post-processing GPT output
+
+    topics = [topic.strip() for topic in topics]
+
+    topics = [topic[1:-1] if (topic.startswith('"') and topic.endswith('"')) or (topic.startswith("'") and topic.endswith("'")) else topic for topic in topics]
+
+    topics = [re.sub(r'[^a-zA-Z0-9\s]', ' ', topic) for topic in topics]
+
+    return topics
+
+def get_relevant_comments(topics):
+
+    global reddit
+
+    comments = []
+
+    for topic in tqdm(topics):
+        for post in reddit.subreddit("all").search(
+        topic, limit=10):
+            
+            post.comment_limit = 20
+            post.comment_sort = "top"
+
+            # Top level comments only
+            post.comments.replace_more(limit=0)
+
+            for comment in post.comments:
+                author = comment.author.name if comment.author else '[deleted]'
+                comments.append([post.id, comment.id, post.subreddit.display_name, post.title, author, comment.body])
+
+    comments = pd.DataFrame(comments,columns=['source', 'comment_id', 'subreddit', 'title', 'author', 'text'])
+
+    # Drop empty texts or ["deleted"] texts
+    comments = comments[comments['text'].str.len() > 0]
+    comments = comments[comments['text'] != "[deleted]"]
+
+    # Drop comments with None authors
+    comments = comments[comments['author'] != "AutoModerator"]
+
+    # Drop duplicate ids
+    comments = comments.drop_duplicates(subset=['source'])
+
+    return comments
+
+def construct_retriever(comments, k=20):
+
+    # Convert comments dataframe to a dictionary
+    comments = comments.to_dict('records')
+
+    # Convert comments["text"] to a list of strings
+    texts = [comment["title"] + " " + comment["text"] + " " + comment["subreddit"] for comment in comments]
+
+    db = Chroma.from_texts(texts, TensorflowHubEmbeddings(model_url="https://tfhub.dev/google/universal-sentence-encoder/4"), metadatas=[{"source": comment["source"], "comment_id": comment["comment_id"], "author": comment["author"]} for comment in comments])
+
+    retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": k})
+
+    return retriever
+
+def construct_bot(retriever):
+    bot = ConversationalRetrievalChain.from_llm(OpenAI(temperature=0), retriever, return_source_documents=True)
+    return bot
+
+def get_response(query, chat_history):
+    response = bot({"question": query, "chat_history": chat_history})
+    return response
+
+def restart():
+    
+    global chat_history
+    global bot
+
+    chat_history = []
+    bot = None
+
+    print("Chat history and bot knowledge has been cleared!")
+
+    return None
+
+def main(query):
+
+    global chat_history
+    global bot
+
+    if chat_history == []:
+        print("Bot knowledge has not been initialised yet! Generating topics...")
+        topics = generate_topics(query)
+
+        print("Fetching relevant comments...")
+        comments = get_relevant_comments(topics)
+
+        print("Embedding relevant comments...")
+        retriever = construct_retriever(comments)
+
+        print("Educating bot...")
+        bot = construct_bot(retriever)
+
+        print("Bot has been constructed and is ready to use!")
+
+    response = get_response(query, chat_history)
+
+    answer, source_documents = response["answer"], response["source_documents"]
+
+    print(source_documents)
+
+    chat_history.append((query, answer))
+
+    return "", chat_history
+
+# Testing only!
+set_openai_key("")
+set_reddit_keys("", "", "")
+
+with gr.Blocks() as demo:
+    chat_bot = gr.Chatbot()
+    query = gr.Textbox()
+    clear = gr.Button("Clear")
+
+    query.submit(main, [query], [query, chat_bot])
+    clear.click(restart, None, chat_bot, queue=False)
+
+demo.launch()

main.ipynb

@@ -2,10 +2,11 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 258,
+   "execution_count": 83,
    "metadata": {},
    "outputs": [],
    "source": [
+    "from tqdm import tqdm\n",
     "import pandas as pd\n",
     "import openai\n",
     "import praw\n",
@@ -18,24 +19,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 259,
+   "execution_count": 84,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain.vectorstores import Chroma\n",
-    "from langchain.docstore.document import Document\n",
-    "from langchain.prompts import PromptTemplate\n",
-    "from langchain.indexes.vectorstore import VectorstoreIndexCreator\n",
-    "from langchain.chains.qa_with_sources import load_qa_with_sources_chain\n",
     "from langchain.chains import ConversationalRetrievalChain\n",
-    "from langchain.llms import OpenAI"
+    "from langchain.llms import OpenAI\n",
+    "from langchain.embeddings import TensorflowHubEmbeddings"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 260,
+   "execution_count": 85,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -56,12 +52,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 261,
+   "execution_count": 86,
    "metadata": {},
    "outputs": [],
    "source": [
     "query = '''\n",
-    "Where can I find a good gluten-free pizza in Stockholm?\n",
+    "I got laid off last week. How should I go about finding a new job?\n",
     "'''"
    ]
   },
@@ -75,7 +71,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 262,
+   "execution_count": 87,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -95,40 +91,44 @@
     "\n",
     "    topics = re.sub(r'^\\d+\\.\\s*', '', response_message, flags=re.MULTILINE).split(\"\\n\")\n",
     "\n",
+    "    # Post-processing GPT output\n",
+    "\n",
+    "    topics = [topic.strip() for topic in topics]\n",
+    "\n",
+    "    topics = [topic[1:-1] if (topic.startswith('\"') and topic.endswith('\"')) or (topic.startswith(\"'\") and topic.endswith(\"'\")) else topic for topic in topics]\n",
+    "\n",
+    "    topics = [re.sub(r'[^a-zA-Z0-9\\s]', ' ', topic) for topic in topics]\n",
+    "\n",
     "    return topics"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 263,
+   "execution_count": 88,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['Gluten free pizza Stockholm',\n",
-       " 'Best gluten free pizza',\n",
-       " 'Pizza without gluten',\n",
-       " 'Stockholm pizza options',\n",
-       " 'Gluten free restaurants Stockholm',\n",
-       " 'Pizza delivery gluten free',\n",
-       " 'Stockholm pizza places',\n",
-       " 'Gluten free pizza toppings',\n",
-       " 'Pizza crust gluten free',\n",
-       " 'Vegan gluten free pizza']"
+       "['Job search tips',\n",
+       " 'Resume writing advice',\n",
+       " 'Networking strategies',\n",
+       " 'Interview preparation tips',\n",
+       " 'Online job boards',\n",
+       " 'Career counseling services',\n",
+       " 'Job fairs near me',\n",
+       " 'Freelance opportunities',\n",
+       " 'Remote work options',\n",
+       " 'Industry specific job listings']"
       ]
      },
-     "execution_count": 263,
+     "execution_count": 88,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "topics = generate_topics(query)\n",
-    "topics = [topic.strip() for topic in topics]\n",
-    "topics = [topic[1:-1] if (topic.startswith('\"') and topic.endswith('\"')) or (topic.startswith(\"'\") and topic.endswith(\"'\")) else topic for topic in topics]\n",
-    "\n",
-    "topics = [re.sub(r'[^a-zA-Z0-9\\s]', ' ', topic) for topic in topics]\n",
     "\n",
     "topics"
    ]
@@ -138,42 +138,62 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Relevant Subreddit Retrieval"
+    "## Relevant Comments Retrieval"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 264,
+   "execution_count": 89,
    "metadata": {},
    "outputs": [],
    "source": [
-    "posts = []\n",
+    "def get_relevant_subreddits(topics):\n",
+    "    comments = []\n",
     "\n",
-    "for topic in topics:\n",
-    "    for post in reddit.subreddit(\"all\").search(\n",
-    "    topic, limit=10):\n",
+    "    for topic in tqdm(topics):\n",
+    "        for post in reddit.subreddit(\"all\").search(\n",
+    "        topic, limit=10):\n",
+    "            \n",
+    "            post.comment_limit = 20\n",
+    "            post.comment_sort = \"top\"\n",
     "\n",
-    "        post.comments.replace_more(limit=3)\n",
+    "            # Top level comments only\n",
+    "            post.comments.replace_more(limit=0)\n",
     "\n",
-    "        # TODO: A check can be added here once we have enough comments for a post. This should drastically reduce the number of requests to the API and time spent.\n",
-    "        for comment in post.comments.list():\n",
-    "            posts.append([comment.id, post.id, post.title, post.author, comment.body])\n",
+    "            for comment in post.comments:\n",
+    "                author = comment.author.name if comment.author else '[deleted]'\n",
+    "                comments.append([post.id, comment.id, post.subreddit.display_name, post.title, author, comment.body])\n",
     "\n",
-    "posts = pd.DataFrame(posts,columns=['source', 'post_id', 'title', 'author', 'text'])"
+    "    comments = pd.DataFrame(comments,columns=['source', 'comment_id', 'subreddit', 'title', 'author', 'text'])\n",
+    "\n",
+    "    # Drop empty texts or [\"deleted\"] texts\n",
+    "    comments = comments[comments['text'].str.len() > 0]\n",
+    "    comments = comments[comments['text'] != \"[deleted]\"]\n",
+    "\n",
+    "    # Drop comments with None authors\n",
+    "    comments = comments[comments['author'] != \"AutoModerator\"]\n",
+    "\n",
+    "    # Drop duplicate ids\n",
+    "    comments = comments.drop_duplicates(subset=['source'])\n",
+    "\n",
+    "    return comments"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 265,
+   "execution_count": 90,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 10/10 [00:41<00:00,  4.13s/it]\n"
+     ]
+    }
+   ],
    "source": [
-    "# Drop empty texts or [\"deleted\"] texts\n",
-    "posts = posts[posts['text'].str.len() > 0]\n",
-    "posts = posts[posts['text'] != \"[deleted]\"]\n",
-    "\n",
-    "# Drop duplicate ids\n",
-    "posts = posts.drop_duplicates(subset=['source'])"
+    "comments = get_relevant_subreddits(topics)"
    ]
   },
   {
@@ -186,26 +206,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 266,
+   "execution_count": 91,
    "metadata": {},
    "outputs": [],
    "source": [
-    "posts = posts.to_dict('records')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 267,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Convert posts[\"text\"] to a list of strings\n",
-    "texts = [post[\"title\"] + \" \" + post[\"text\"] for post in posts]"
+    "def construct_retriever(comments, k=10):\n",
+    "\n",
+    "    # Convert comments dataframe to a dictionary\n",
+    "    comments = comments.to_dict('records')\n",
+    "\n",
+    "    # Convert comments[\"text\"] to a list of strings\n",
+    "    texts = [comment[\"title\"] + \" \" + comment[\"text\"] for comment in comments]\n",
+    "\n",
+    "    db = Chroma.from_texts(texts, TensorflowHubEmbeddings(model_url=\"https://tfhub.dev/google/universal-sentence-encoder/4\"), metadatas=[{\"source\": comment[\"source\"], \"comment_id\": comment[\"comment_id\"], \"author\": comment[\"author\"]} for comment in comments])\n",
+    "\n",
+    "    retriever = db.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": k})\n",
+    "\n",
+    "    return retriever"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 268,
+   "execution_count": 92,
    "metadata": {},
    "outputs": [
     {
@@ -217,49 +239,51 @@
     }
    ],
    "source": [
-    "db = Chroma.from_texts(texts, OpenAIEmbeddings(), metadatas=[{\"source\": post[\"source\"], \"post_id\": post[\"post_id\"], \"author\": post[\"author\"].name} for post in posts])"
+    "retriever = construct_retriever(comments)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 269,
+   "execution_count": 93,
    "metadata": {},
    "outputs": [],
    "source": [
-    "retriever = db.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": 10})"
+    "def construct_qa(retriever):\n",
+    "    qa = ConversationalRetrievalChain.from_llm(OpenAI(temperature=0), retriever, return_source_documents=True)\n",
+    "    return qa"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 270,
+   "execution_count": 95,
    "metadata": {},
    "outputs": [],
    "source": [
-    "qa = ConversationalRetrievalChain.from_llm(OpenAI(temperature=0), retriever, return_source_documents=True)"
+    "chat_history = []\n",
+    "qa = construct_qa(retriever)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 291,
+   "execution_count": 96,
    "metadata": {},
    "outputs": [],
    "source": [
-    "chat_history = []\n",
     "result = qa({\"question\": query, \"chat_history\": chat_history})"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 292,
+   "execution_count": 97,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "' Meno Male. Super nice Napoletana style Italian pizza, they have a few restaurants throughout Stockholm.'"
+       "\" Start by updating your resume and LinkedIn profile to reflect your current skills and experience. Then, look for job postings on job boards like Indeed and LinkedIn, as well as industry-specific job listings. You can also reach out to people you know professionally, such as former colleagues or mentors, to see if they know of any job opportunities. Finally, consider using a staffing agency to help you find a job that's a great fit. Good luck!\""
       ]
      },
-     "execution_count": 292,
+     "execution_count": 97,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -270,25 +294,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 293,
+   "execution_count": 98,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Document(page_content='Gluten free pizza in Stockholm? Any pizzeria', metadata={'source': 'irirsw2', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Crispy pizza', metadata={'source': 'iritand', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Thanks!', metadata={'source': 'irimidh', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Quick search:\\n\\n[https://www.findmeglutenfree.com/se/stockholm/pizza](https://www.findmeglutenfree.com/se/stockholm/pizza)', metadata={'source': 'jef789l', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
-       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Tack!', metadata={'source': 'jeg12d4', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Most places have gluten free pizza. Only Giro and Meno Male have good gluten free pizza.', metadata={'source': 'irm0583', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Meno Male. Super nice Napoletana style Italian pizza, they have a few restaurants throughout Stockholm', metadata={'source': 'irilvan', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Stockholm is like the capital of gluten intolerance so I think you will be fine in most (central) places', metadata={'source': 'irioh1a', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? This is the right answer. Gluten-avoidants that I know like menomale.', metadata={'source': 'irjufjn', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) A lot of pizza places offer this if you ask for it. Check with your local pizza place and you might be lucky. Usually boring prefab bread tho.', metadata={'source': 'jef91ea', 'post_id': '127mh90', 'author': 'TheRoyalStork'})]"
+       "[Document(page_content='Request: job search tips WAITING FOR THE RIGHT JOB WITH DECENT PAY > QUICK JOB', metadata={'source': '122mdcc', 'comment_id': 'jdv57nv', 'author': 'notenoughbeds'}),\n",
+       " Document(page_content='Where to look for jobs? Online job boards? LinkedIn\\n\\nIndeed\\n\\nIf you need a... beginners job. Craigslist has a work section', metadata={'source': 'uj35l8', 'comment_id': 'i7gbuat', 'author': 'No-Statement-3019'}),\n",
+       " Document(page_content='Job search tips in Canada You’re selling yourself way too cheap. Look for a senior position and then people will want to hire you more too', metadata={'source': '1156zsa', 'comment_id': 'j91mo9r', 'author': 'pxpxy'}),\n",
+       " Document(page_content='Job search tips Just apply, covid has made super easy for RTs to get jobs', metadata={'source': '11uekx5', 'comment_id': 'jcnvfnx', 'author': 'Crass_Cameron'}),\n",
+       " Document(page_content='Looking for freelance opportunities >\\tWhere would be the best place to look?\\n\\nPeople you’ve worked with professionally before. Either for jobs for them or for them to refer you to people they know.', metadata={'source': '11q0h1u', 'comment_id': 'jc0w8rp', 'author': 'dataguy24'}),\n",
+       " Document(page_content='Did Career Counseling services help you land a job after you graduated? I found it to be helpful in polishing my resume', metadata={'source': 'xbql4p', 'comment_id': 'io1141n', 'author': 'avo_cado'}),\n",
+       " Document(page_content=\"Does anyone have any good job search tips? I recommend keeping an up to date LinkedIn profile that indicates you're actively searching for roles. I've also had luck with Indeed.\\n\\nDepending on your field, I recommend a staffing agency.  They can vouch for you and place you with a company that's a great fit.\", metadata={'source': '134wnu5', 'comment_id': 'jih9zzp', 'author': 'Carolinablue87'}),\n",
+       " Document(page_content='How did you find your job? Did someone you know tell you about it? Job listing aggregator (ie indeed, simplyhired)? Industry specific job listings? Somehow else? strong institutional connection between the employer and my grad school', metadata={'source': '88vquy', 'comment_id': 'dwnknrk', 'author': 'lollersauce914'}),\n",
+       " Document(page_content='I (17 F) am really confused about my career. Is there any way I can get career counseling services for free? Any resources or tips would be appreciated too! What are you confused about? I have no qualifications, just a working professional of 8 years. \\nDoes your work have a subreddit or do you have a mentor at work that you could reach out to?', metadata={'source': '10jv7c4', 'comment_id': 'j5mtxb7', 'author': 'bhop02'}),\n",
+       " Document(page_content=\"Debating whether or not to pay for access to sites that have specific job listings - any successes or tales of caution? I never had to look for my jobs through those sites. I don't think they can let the companies post job positions exclusively on their sites only. You can always find the same job post somewhere else if you keep trying. I hope this is helpful, good luck!\", metadata={'source': '11kppd0', 'comment_id': 'jb8qvjo', 'author': 'Whitney-Sweet'})]"
       ]
      },
-     "execution_count": 293,
+     "execution_count": 98,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -296,66 +320,6 @@
    "source": [
     "result[\"source_documents\"]"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 297,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "new_query = \"Are most pizzas in Stockholm gluten-free?\"\n",
-    "\n",
-    "new_result = qa({\"question\": query + new_query, \"chat_history\": result[\"chat_history\"]})"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 298,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "' Most places have gluten free pizza. Only Giro and Meno Male have good gluten free pizza. Pretty much all pizzerias serve GF pizza, but most use store-bought pizza bases. Stockholm is like the capital of gluten intolerance so I think you will be fine in most (central) places. Meno Male is a great option for Napoletana style Italian pizza, they have a few restaurants throughout Stockholm.'"
-      ]
-     },
-     "execution_count": 298,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "new_result[\"answer\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 299,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[Document(page_content='Gluten free pizza in Stockholm? Any pizzeria', metadata={'source': 'irirsw2', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Thanks!', metadata={'source': 'irimidh', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Crispy pizza', metadata={'source': 'iritand', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Most places have gluten free pizza. Only Giro and Meno Male have good gluten free pizza.', metadata={'source': 'irm0583', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Quick search:\\n\\n[https://www.findmeglutenfree.com/se/stockholm/pizza](https://www.findmeglutenfree.com/se/stockholm/pizza)', metadata={'source': 'jef789l', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
-       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Tack!', metadata={'source': 'jeg12d4', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Pretty much all pizzerias serve GF pizza, but most use store-bought pizza bases.', metadata={'source': 'irjap32', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Stockholm is like the capital of gluten intolerance so I think you will be fine in most (central) places', metadata={'source': 'irioh1a', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Meno Male. Super nice Napoletana style Italian pizza, they have a few restaurants throughout Stockholm', metadata={'source': 'irilvan', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
-       " Document(page_content='Gluten free pizza in Stockholm? Has anyone here with a gluten intolerance diagnosis eaten a gf pizza from a pizzeria? Was it without issues? I imagine that all the flour handling, the oven and the spade would contaminate any gf pizza with gluten.', metadata={'source': 'irjmdif', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'})]"
-      ]
-     },
-     "execution_count": 299,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "new_result[\"source_documents\"]"
-   ]
   }
  ],
  "metadata": {

semantic_search.py

@@ -1,39 +0,0 @@
-from sklearn.neighbors import NearestNeighbors
-import tensorflow_hub as hub
-import numpy as np
-
-class SemanticSearch:
-    
-    def __init__(self):
-        self.use = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')
-        self.fitted = False
-    
-    
-    def fit(self, data, batch=1000, n_neighbors=5):
-        print(f"Fitting with n={n_neighbors}...")
-        self.data = data
-        self.embeddings = self.get_text_embedding(data, batch=batch)
-        n_neighbors = min(n_neighbors, len(self.embeddings))
-        self.nn = NearestNeighbors(n_neighbors=n_neighbors)
-        self.nn.fit(self.embeddings)
-        self.fitted = True
-    
-    
-    def __call__(self, text, return_data=True):
-        inp_emb = self.use([text])
-        distances, neighbors = self.nn.kneighbors(inp_emb, return_distance=True)
-        
-        if return_data:
-            return [self.data[i] for i in neighbors[0]], distances
-        else:
-            return neighbors[0], distances
-    
-    
-    def get_text_embedding(self, texts, batch=1000):
-        embeddings = []
-        for i in range(0, len(texts), batch):
-            text_batch = texts[i:(i+batch)]
-            emb_batch = self.use(text_batch)
-            embeddings.append(emb_batch)
-        embeddings = np.vstack(embeddings)
-        return embeddings