getting closer to completion!

README.md

@@ -6,7 +6,6 @@ AskRedditGPT is a tool that takes in a query, sends it over to Reddit, and retur
 
 1. Take in query $q$ from user.
 2. Get $N$ topics from $q$ using GPT.
-3. Determine $S$, which is a set of subreddits best-suited to answer $N$ topics.
-4. Search $q \in S$.
-5. Retrieve a set of segments that can answer $q$.
-6. Summarise segments using GPT and return answer to user.
+3. Determine $C$, which is a set of comments best-suited to answer $N$ topics.
+4. Search $q \in C$.
+5. Use GPT to return an answer to user.

main.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 94,
+   "execution_count": 258,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -18,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 95,
+   "execution_count": 259,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -29,12 +29,13 @@
     "from langchain.prompts import PromptTemplate\n",
     "from langchain.indexes.vectorstore import VectorstoreIndexCreator\n",
     "from langchain.chains.qa_with_sources import load_qa_with_sources_chain\n",
+    "from langchain.chains import ConversationalRetrievalChain\n",
     "from langchain.llms import OpenAI"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 96,
+   "execution_count": 260,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -45,6 +46,25 @@
     "reddit = praw.Reddit(client_id=REDDIT_CLIENT_ID, client_secret=REDDIT_CLIENT_SECRET, user_agent=REDDIT_USER_AGENT)"
    ]
   },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 261,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = '''\n",
+    "Where can I find a good gluten-free pizza in Stockholm?\n",
+    "'''"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -55,14 +75,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 97,
+   "execution_count": 262,
    "metadata": {},
    "outputs": [],
    "source": [
     "def generate_topics(query, model=\"gpt-3.5-turbo\"):\n",
     "\n",
     "    messages = [\n",
-    "        {\"role\": \"user\", \"content\": f\"Take this query '{query}' and return a list of 10 simple to understand topics (3 words or less) to input in Search so it returns good results.\"},\n",
+    "        {\"role\": \"user\", \"content\": f\"Take this query '{query}' and return a list of 10 simple to understand topics (4 words or less) to input in Search so it returns good results.\"},\n",
     "    ]\n",
     "\n",
     "    response = openai.ChatCompletion.create(\n",
@@ -80,31 +100,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 108,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "query = \"Are we in a recession right now?\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 109,
+   "execution_count": 263,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['Current economic status', 'Recession indicators', 'Unemployment rates', 'GDP growth rate', 'Consumer spending trends', 'Stock market performance', 'Federal Reserve actions', 'Economic stimulus packages', 'Business closures impact', 'Housing market trends']\n"
-     ]
+     "data": {
+      "text/plain": [
+       "['Gluten free pizza Stockholm',\n",
+       " 'Best gluten free pizza',\n",
+       " 'Pizza without gluten',\n",
+       " 'Stockholm pizza options',\n",
+       " 'Gluten free restaurants Stockholm',\n",
+       " 'Pizza delivery gluten free',\n",
+       " 'Stockholm pizza places',\n",
+       " 'Gluten free pizza toppings',\n",
+       " 'Pizza crust gluten free',\n",
+       " 'Vegan gluten free pizza']"
+      ]
+     },
+     "execution_count": 263,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "topics = generate_topics(query)\n",
     "topics = [topic.strip() for topic in topics]\n",
     "topics = [topic[1:-1] if (topic.startswith('\"') and topic.endswith('\"')) or (topic.startswith(\"'\") and topic.endswith(\"'\")) else topic for topic in topics]\n",
-    "print(topics)"
+    "\n",
+    "topics = [re.sub(r'[^a-zA-Z0-9\\s]', ' ', topic) for topic in topics]\n",
+    "\n",
+    "topics"
    ]
   },
   {
@@ -117,32 +143,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 110,
+   "execution_count": 264,
    "metadata": {},
    "outputs": [],
    "source": [
     "posts = []\n",
-    "comments = []\n",
     "\n",
     "for topic in topics:\n",
     "    for post in reddit.subreddit(\"all\").search(\n",
-    "    topic, limit=5):\n",
-    "        posts.append([post.id, post.title, post.subreddit, post.selftext])\n",
-    "        post.comments.replace_more(limit=1)\n",
+    "    topic, limit=10):\n",
     "\n",
+    "        post.comments.replace_more(limit=3)\n",
+    "\n",
+    "        # TODO: A check can be added here once we have enough comments for a post. This should drastically reduce the number of requests to the API and time spent.\n",
     "        for comment in post.comments.list():\n",
-    "            posts.append([post.id, post.title, post.subreddit, comment.body])\n",
+    "            posts.append([comment.id, post.id, post.title, post.author, comment.body])\n",
     "\n",
-    "posts = pd.DataFrame(posts,columns=['source', 'title', 'subreddit', 'text'])"
+    "posts = pd.DataFrame(posts,columns=['source', 'post_id', 'title', 'author', 'text'])"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 111,
+   "execution_count": 265,
    "metadata": {},
    "outputs": [],
    "source": [
-    "posts[\"subreddit\"] = posts[\"subreddit\"].apply(lambda x: x.display_name)\n"
+    "# Drop empty texts or [\"deleted\"] texts\n",
+    "posts = posts[posts['text'].str.len() > 0]\n",
+    "posts = posts[posts['text'] != \"[deleted]\"]\n",
+    "\n",
+    "# Drop duplicate ids\n",
+    "posts = posts.drop_duplicates(subset=['source'])"
    ]
   },
   {
@@ -155,158 +186,175 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 112,
+   "execution_count": 266,
    "metadata": {},
    "outputs": [],
    "source": [
-    "text = posts[\"text\"].tolist()\n",
-    "text = \" \".join(text)"
+    "posts = posts.to_dict('records')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 113,
+   "execution_count": 267,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Convert posts[\"text\"] to a list of strings\n",
+    "texts = [post[\"title\"] + \" \" + post[\"text\"] for post in posts]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 268,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Created a chunk of size 1635, which is longer than the specified 1000\n",
-      "Created a chunk of size 1298, which is longer than the specified 1000\n",
-      "Created a chunk of size 1109, which is longer than the specified 1000\n",
-      "Created a chunk of size 2072, which is longer than the specified 1000\n",
-      "Created a chunk of size 1498, which is longer than the specified 1000\n",
-      "Created a chunk of size 1419, which is longer than the specified 1000\n",
-      "Created a chunk of size 1127, which is longer than the specified 1000\n",
-      "Created a chunk of size 1576, which is longer than the specified 1000\n",
-      "Created a chunk of size 1314, which is longer than the specified 1000\n",
-      "Created a chunk of size 2563, which is longer than the specified 1000\n",
-      "Created a chunk of size 1287, which is longer than the specified 1000\n",
-      "Created a chunk of size 1649, which is longer than the specified 1000\n",
-      "Created a chunk of size 1616, which is longer than the specified 1000\n",
-      "Created a chunk of size 1573, which is longer than the specified 1000\n",
-      "Created a chunk of size 1024, which is longer than the specified 1000\n",
-      "Created a chunk of size 1395, which is longer than the specified 1000\n",
-      "Created a chunk of size 1712, which is longer than the specified 1000\n",
-      "Created a chunk of size 1175, which is longer than the specified 1000\n",
-      "Created a chunk of size 3872, which is longer than the specified 1000\n",
-      "Created a chunk of size 1098, which is longer than the specified 1000\n",
-      "Created a chunk of size 1429, which is longer than the specified 1000\n",
-      "Created a chunk of size 1002, which is longer than the specified 1000\n",
-      "Created a chunk of size 2241, which is longer than the specified 1000\n",
-      "Created a chunk of size 1923, which is longer than the specified 1000\n",
-      "Created a chunk of size 1716, which is longer than the specified 1000\n",
-      "Created a chunk of size 2563, which is longer than the specified 1000\n",
-      "Created a chunk of size 1221, which is longer than the specified 1000\n",
-      "Created a chunk of size 2449, which is longer than the specified 1000\n",
-      "Created a chunk of size 1321, which is longer than the specified 1000\n",
-      "Created a chunk of size 1302, which is longer than the specified 1000\n",
-      "Created a chunk of size 2182, which is longer than the specified 1000\n",
-      "Created a chunk of size 1027, which is longer than the specified 1000\n",
-      "Created a chunk of size 1156, which is longer than the specified 1000\n",
-      "Created a chunk of size 7334, which is longer than the specified 1000\n",
-      "Created a chunk of size 1849, which is longer than the specified 1000\n",
-      "Created a chunk of size 2829, which is longer than the specified 1000\n",
-      "Created a chunk of size 1567, which is longer than the specified 1000\n",
-      "Created a chunk of size 1245, which is longer than the specified 1000\n",
-      "Created a chunk of size 1299, which is longer than the specified 1000\n",
-      "Created a chunk of size 1003, which is longer than the specified 1000\n",
-      "Created a chunk of size 1327, which is longer than the specified 1000\n",
-      "Created a chunk of size 2079, which is longer than the specified 1000\n",
-      "Created a chunk of size 2780, which is longer than the specified 1000\n",
-      "Created a chunk of size 1522, which is longer than the specified 1000\n",
-      "Created a chunk of size 1766, which is longer than the specified 1000\n",
-      "Created a chunk of size 1079, which is longer than the specified 1000\n",
-      "Created a chunk of size 1080, which is longer than the specified 1000\n",
-      "Created a chunk of size 1755, which is longer than the specified 1000\n",
-      "Created a chunk of size 1232, which is longer than the specified 1000\n",
-      "Created a chunk of size 1279, which is longer than the specified 1000\n",
-      "Created a chunk of size 3189, which is longer than the specified 1000\n",
-      "Created a chunk of size 1549, which is longer than the specified 1000\n",
-      "Created a chunk of size 1124, which is longer than the specified 1000\n",
-      "Created a chunk of size 1033, which is longer than the specified 1000\n",
-      "Created a chunk of size 1676, which is longer than the specified 1000\n",
-      "Created a chunk of size 1011, which is longer than the specified 1000\n",
-      "Created a chunk of size 1723, which is longer than the specified 1000\n"
+      "WARNING:chromadb:Using embedded DuckDB without persistence: data will be transient\n"
      ]
     }
    ],
    "source": [
-    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
-    "texts = text_splitter.split_text(text)\n",
-    "\n",
-    "embeddings = OpenAIEmbeddings()"
+    "db = Chroma.from_texts(texts, OpenAIEmbeddings(), metadatas=[{\"source\": post[\"source\"], \"post_id\": post[\"post_id\"], \"author\": post[\"author\"].name} for post in posts])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 269,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retriever = db.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": 10})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 270,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "qa = ConversationalRetrievalChain.from_llm(OpenAI(temperature=0), retriever, return_source_documents=True)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 114,
+   "execution_count": 291,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chat_history = []\n",
+    "result = qa({\"question\": query, \"chat_history\": chat_history})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 292,
    "metadata": {},
    "outputs": [
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Using embedded DuckDB without persistence: data will be transient\n"
-     ]
+     "data": {
+      "text/plain": [
+       "' Meno Male. Super nice Napoletana style Italian pizza, they have a few restaurants throughout Stockholm.'"
+      ]
+     },
+     "execution_count": 292,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result[\"answer\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 293,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='Gluten free pizza in Stockholm? Any pizzeria', metadata={'source': 'irirsw2', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Crispy pizza', metadata={'source': 'iritand', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Thanks!', metadata={'source': 'irimidh', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Quick search:\\n\\n[https://www.findmeglutenfree.com/se/stockholm/pizza](https://www.findmeglutenfree.com/se/stockholm/pizza)', metadata={'source': 'jef789l', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
+       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Tack!', metadata={'source': 'jeg12d4', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Most places have gluten free pizza. Only Giro and Meno Male have good gluten free pizza.', metadata={'source': 'irm0583', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Meno Male. Super nice Napoletana style Italian pizza, they have a few restaurants throughout Stockholm', metadata={'source': 'irilvan', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Stockholm is like the capital of gluten intolerance so I think you will be fine in most (central) places', metadata={'source': 'irioh1a', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? This is the right answer. Gluten-avoidants that I know like menomale.', metadata={'source': 'irjufjn', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) A lot of pizza places offer this if you ask for it. Check with your local pizza place and you might be lucky. Usually boring prefab bread tho.', metadata={'source': 'jef91ea', 'post_id': '127mh90', 'author': 'TheRoyalStork'})]"
+      ]
+     },
+     "execution_count": 293,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
-    "docsearch = Chroma.from_texts(texts, embeddings, metadatas=[{\"source\": str(i)} for i in range(len(texts))]).as_retriever()"
+    "result[\"source_documents\"]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 115,
+   "execution_count": 297,
    "metadata": {},
    "outputs": [],
    "source": [
-    "docs = docsearch.get_relevant_documents(query)"
+    "new_query = \"Are most pizzas in Stockholm gluten-free?\"\n",
+    "\n",
+    "new_result = qa({\"question\": query + new_query, \"chat_history\": result[\"chat_history\"]})"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 116,
+   "execution_count": 298,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Document(page_content=\"A recession is always just around the corner, meanwhile the S&P has been making higher lows since October 2022, unemployed in Canada is 5.0%, and GDP hasn’t been negative for two quarters. It is likely to happen. And economists will have to work fast to figure out a solution. You need demand to sustain supply. And without jobs there isn't demand, nor tax revenue.\\n\\nSo short term: pain is possible.\\n\\nMedium term: we might benefit from doing less work and getting paid the same.\", metadata={'source': '39'}),\n",
-       " Document(page_content=\"Don't listen to people who say GDP is the only thing that matters for recessions.  The NBER's definition -- which hasn't changed -- requires economic downturn across the economy in a broad sense.  We probably can't have a recession and very low unemployment at the same time.  So if we have a recession, we will have layoffs, too. [deleted] Most layoffs I've seen headlines for are in tech. While tech is a huge sector, it is important to look at the market as a whole. Job growth and job demand is still high. There are many many different indicators people look at. Housing prices is another. The difference between the bond curve. 3 month compared to the 1 year is another common one. When that inverts it means the economy isn't in a great place. It's so hard to predict which is why people constantly say to dollar cost average. Continuously buy and don't try to time anything. This time unemployment rate will be affected.\", metadata={'source': '13'}),\n",
-       " Document(page_content='High tech layoffs often triggers layoff in retail as affected people no longer have the means to spend. Restaurant closing. Meta has to write off $0.67 billion dollars or more next qtr as 1 time charge.  Tweeter let go 50 pct employees so SF is already slow business wise. Housing industry is going to get hit this winter and even spring.   Be honest with ourselves we have been in recession most of the year.  Not too many mfg jobs are in the US so you do not see that many job losses.\\n\\nAfter Xmas I imagine companies like mail order delivery will slow down also. Your pocketbook. Negative GDP, inflation, decrease in purchases, growing unemployment... decrease of consumer spending; and increase of umemployment Honestly fear of a recession is the best indicator. People and companies start holding their money close to be safe and it becomes a self fulfilling prophecy. Most reliable indicator is when people are yelling for higher prices when their asset already doubled, tripled in value.', metadata={'source': '14'}),\n",
-       " Document(page_content='?? Agreed. Investors are jumping at the slightest bit of good news. But the job market is still strong, Fed wont stop raising rates until that changes. Not to mention the fact that oil is going up again. Which was a big factor in how we got to 8% - 9% inflation in the first place. Gas is almost $6 in my area now. More than it has ever been where I live. We are at the beginning of a lot of pain. Not the middle or the end. Recession is here, and it will deepen and broaden until everyone finally sees it. I mean australia raised under expectations but I dont think  that matter I think the jump was - Australia only bumped by 25 basis pts and for some reason the market is in love with the fed pivot idea, jobs openings reduced something like 10%, Bond yields went down as well.  But no, nothing fundamentally changed. The bank of England has started QE >didn\\'t interest rates hike again?\\n\\nYou \"call a trap\" but aren\\'t entirely sure if interest rates were hiked (they were)?', metadata={'source': '179'})]"
+       "' Most places have gluten free pizza. Only Giro and Meno Male have good gluten free pizza. Pretty much all pizzerias serve GF pizza, but most use store-bought pizza bases. Stockholm is like the capital of gluten intolerance so I think you will be fine in most (central) places. Meno Male is a great option for Napoletana style Italian pizza, they have a few restaurants throughout Stockholm.'"
       ]
      },
-     "execution_count": 116,
+     "execution_count": 298,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "docs"
+    "new_result[\"answer\"]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 117,
+   "execution_count": 299,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "' It is likely that we are in a recession right now.\\nSOURCES: 39, 13, 14, 179'"
+       "[Document(page_content='Gluten free pizza in Stockholm? Any pizzeria', metadata={'source': 'irirsw2', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Thanks!', metadata={'source': 'irimidh', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Crispy pizza', metadata={'source': 'iritand', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Most places have gluten free pizza. Only Giro and Meno Male have good gluten free pizza.', metadata={'source': 'irm0583', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Quick search:\\n\\n[https://www.findmeglutenfree.com/se/stockholm/pizza](https://www.findmeglutenfree.com/se/stockholm/pizza)', metadata={'source': 'jef789l', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
+       " Document(page_content='Var kan jag köpa glutenfri kebab pizza?? (Stockholm) Tack!', metadata={'source': 'jeg12d4', 'post_id': '127mh90', 'author': 'TheRoyalStork'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Pretty much all pizzerias serve GF pizza, but most use store-bought pizza bases.', metadata={'source': 'irjap32', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Stockholm is like the capital of gluten intolerance so I think you will be fine in most (central) places', metadata={'source': 'irioh1a', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Meno Male. Super nice Napoletana style Italian pizza, they have a few restaurants throughout Stockholm', metadata={'source': 'irilvan', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'}),\n",
+       " Document(page_content='Gluten free pizza in Stockholm? Has anyone here with a gluten intolerance diagnosis eaten a gf pizza from a pizzeria? Was it without issues? I imagine that all the flour handling, the oven and the spade would contaminate any gf pizza with gluten.', metadata={'source': 'irjmdif', 'post_id': 'xyti59', 'author': 'Head-Commission-8222'})]"
       ]
      },
-     "execution_count": 117,
+     "execution_count": 299,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "chain = load_qa_with_sources_chain(OpenAI(temperature=0), chain_type=\"stuff\")\n",
-    "chain.run(input_documents=docs, question=query)"
+    "new_result[\"source_documents\"]"
    ]
   }
  ],