Rag changes

.gitignore

@@ -1,3 +1,4 @@
+### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -159,4 +160,14 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json

app.py

@@ -11,7 +11,13 @@ from utils.prompts import (
     evidence_based_prompt,
     sample_questions,
 )
+import openai
 
+from config import STUDY_FILES, OPENAI_API_KEY
+
+openai.api_key = OPENAI_API_KEY
+
+# Cache for RAG pipelines
 rag_cache = {}
 
 

config.py

@@ -1,5 +1,9 @@
 import os
 
+from dotenv import load_dotenv
+
+load_dotenv()
+
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 
 STUDY_FILES = {

data/ebola_virus_zotero_items.json

@@ -11,7 +11,7 @@
       "Marco Tuccori"
     ],
     "doi": "",
-    "year": null,
+    "date": "2020",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -28,7 +28,7 @@
       "Sina Bavari"
     ],
     "doi": "10.1146/annurev-pharmtox-010716-105055",
-    "year": null,
+    "date": "2017-01-06",
     "item_type": "journalArticle",
     "url": "https://www.annualreviews.org/doi/10.1146/annurev-pharmtox-010716-105055"
   },
@@ -48,7 +48,7 @@
       "Xiangguo Qiu"
     ],
     "doi": "10.1016/j.molmed.2017.07.002",
-    "year": null,
+    "date": "09/2017",
     "item_type": "journalArticle",
     "url": "https://linkinghub.elsevier.com/retrieve/pii/S1471491417301090"
   },
@@ -66,7 +66,7 @@
       "Robert L. Gottlieb"
     ],
     "doi": "10.1038/s41577-021-00542-x",
-    "year": null,
+    "date": "06/2021",
     "item_type": "journalArticle",
     "url": "https://www.nature.com/articles/s41577-021-00542-x"
   },
@@ -85,7 +85,7 @@
       "M. P. Grobusch"
     ],
     "doi": "10.1080/13543784.2016.1240785",
-    "year": null,
+    "date": "2016-11-01",
     "item_type": "journalArticle",
     "url": "https://www.tandfonline.com/doi/full/10.1080/13543784.2016.1240785"
   },
@@ -99,7 +99,7 @@
       "Luciana Borio"
     ],
     "doi": "10.1128/microbiolspec.EI10-0014-2016",
-    "year": null,
+    "date": "2016-05-06",
     "item_type": "journalArticle",
     "url": "https://journals.asm.org/doi/10.1128/microbiolspec.EI10-0014-2016"
   },
@@ -117,7 +117,7 @@
       "Larry Zeitlin"
     ],
     "doi": "10.3233/HAB-150284",
-    "year": null,
+    "date": "2015-12-23",
     "item_type": "journalArticle",
     "url": "https://www.medra.org/servlet/aliasResolver?alias=iospress&doi=10.3233/HAB-150284"
   },
@@ -130,7 +130,7 @@
       "Olivier Garraud"
     ],
     "doi": "10.1016/j.transci.2016.12.014",
-    "year": null,
+    "date": "02/2017",
     "item_type": "journalArticle",
     "url": "https://linkinghub.elsevier.com/retrieve/pii/S1473050216302002"
   },
@@ -144,7 +144,7 @@
       "K Karunamoorthi"
     ],
     "doi": "",
-    "year": null,
+    "date": "",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -162,7 +162,7 @@
       "Larry Zeitlin"
     ],
     "doi": "10.3233/HAB-150284",
-    "year": null,
+    "date": "2015-12-23",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -176,7 +176,7 @@
       "Aaron A. R. Tobian"
     ],
     "doi": "10.1111/trf.12913",
-    "year": null,
+    "date": "2014-12",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -194,7 +194,7 @@
       "Robert L. Gottlieb"
     ],
     "doi": "10.1038/s41577-021-00542-x",
-    "year": null,
+    "date": "2021-06",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -210,7 +210,7 @@
       "Marco Tuccori"
     ],
     "doi": "10.1128/CMR.00072-20",
-    "year": null,
+    "date": "2020-09-16",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -223,7 +223,7 @@
       "Olivier Garraud"
     ],
     "doi": "10.1016/j.transci.2016.12.014",
-    "year": null,
+    "date": "2017-02",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -237,7 +237,7 @@
       "Luciana Borio"
     ],
     "doi": "10.1128/microbiolspec.EI10-0014-2016",
-    "year": null,
+    "date": "2016-06",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -257,7 +257,7 @@
       "Xiangguo Qiu"
     ],
     "doi": "10.1016/j.molmed.2017.07.002",
-    "year": null,
+    "date": "2017-09",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -274,7 +274,7 @@
       "Sina Bavari"
     ],
     "doi": "10.1146/annurev-pharmtox-010716-105055",
-    "year": null,
+    "date": "2017-01-06",
     "item_type": "journalArticle",
     "url": ""
   }

data/gene_xpert_zotero_items.json

@@ -13,7 +13,7 @@
       "Hairong Huang"
     ],
     "doi": "10.21037/jtd.2018.02.60",
-    "year": null,
+    "date": "3/2018",
     "item_type": "journalArticle",
     "url": "http://jtd.amegroups.com/article/view/19447/15515"
   },
@@ -33,7 +33,7 @@
       "David W. Dowdy"
     ],
     "doi": "10.1097/QAI.0000000000000712",
-    "year": null,
+    "date": "2015-09-1",
     "item_type": "journalArticle",
     "url": "https://journals.lww.com/00126334-201509010-00011"
   },
@@ -52,7 +52,7 @@
       "Sydney Rosen"
     ],
     "doi": "10.1111/j.1365-3156.2012.03028.x",
-    "year": null,
+    "date": "09/2012",
     "item_type": "journalArticle",
     "url": "https://onlinelibrary.wiley.com/doi/10.1111/j.1365-3156.2012.03028.x"
   },
@@ -75,7 +75,7 @@
       "Keertan Dheda"
     ],
     "doi": "10.1183/09031936.00145511",
-    "year": null,
+    "date": "07/2012",
     "item_type": "journalArticle",
     "url": "http://erj.ersjournals.com/lookup/doi/10.1183/09031936.00145511"
   },
@@ -94,7 +94,7 @@
       "Achilles Katamba"
     ],
     "doi": "10.1186/s12913-016-1804-9",
-    "year": null,
+    "date": "12/2016",
     "item_type": "journalArticle",
     "url": "http://bmchealthservres.biomedcentral.com/articles/10.1186/s12913-016-1804-9"
   },
@@ -115,7 +115,7 @@
       "A. H. Van'T Hoog"
     ],
     "doi": "10.5588/ijtld.16.0496",
-    "year": null,
+    "date": "2017-04-01",
     "item_type": "journalArticle",
     "url": "http://www.ingentaconnect.com/content/10.5588/ijtld.16.0496"
   },
@@ -138,7 +138,7 @@
       "Julio Croda"
     ],
     "doi": "10.1093/cid/ciaa135",
-    "year": null,
+    "date": "2021-03-01",
     "item_type": "journalArticle",
     "url": "https://academic.oup.com/cid/article/72/5/771/5736588"
   },
@@ -154,7 +154,7 @@
       "Anete Trajman"
     ],
     "doi": "10.1590/s1806-37562015000004524",
-    "year": null,
+    "date": "12/2015",
     "item_type": "journalArticle",
     "url": "http://www.scielo.br/scielo.php?script=sci_arttext&pid=S1806-37132015000600536&lng=en&tlng=en"
   },
@@ -181,7 +181,7 @@
       "Keertan Dheda"
     ],
     "doi": "10.1016/S2214-109X(19)30164-0",
-    "year": null,
+    "date": "06/2019",
     "item_type": "journalArticle",
     "url": "https://linkinghub.elsevier.com/retrieve/pii/S2214109X19301640"
   },
@@ -206,7 +206,7 @@
       "Luis E Cuevas"
     ],
     "doi": "10.1136/bmjgh-2021-007592",
-    "year": null,
+    "date": "02/2022",
     "item_type": "journalArticle",
     "url": "https://gh.bmj.com/lookup/doi/10.1136/bmjgh-2021-007592"
   },
@@ -222,7 +222,7 @@
       "Elvira Richter"
     ],
     "doi": "10.1183/13993003.01333-2015",
-    "year": null,
+    "date": "02/2016",
     "item_type": "journalArticle",
     "url": "http://erj.ersjournals.com/lookup/doi/10.1183/13993003.01333-2015"
   },
@@ -244,7 +244,7 @@
       "Susan Cleary"
     ],
     "doi": "10.1371/journal.pone.0251547",
-    "year": null,
+    "date": "2021-5-14",
     "item_type": "journalArticle",
     "url": "https://dx.plos.org/10.1371/journal.pone.0251547"
   },
@@ -260,7 +260,7 @@
       "M. Shah"
     ],
     "doi": "10.5588/ijtld.13.0095",
-    "year": null,
+    "date": "2013-10-01",
     "item_type": "journalArticle",
     "url": "http://openurl.ingenta.com/content/xref?genre=article&issn=1027-3719&volume=17&issue=10&spage=1328"
   },
@@ -285,7 +285,7 @@
       "Jacob Creswell"
     ],
     "doi": "10.3201/eid2703.204090",
-    "year": null,
+    "date": "03/2021",
     "item_type": "journalArticle",
     "url": "https://wwwnc.cdc.gov/eid/article/27/3/20-4090_article.htm"
   },
@@ -300,7 +300,7 @@
       "A. Trajman"
     ],
     "doi": "10.5588/ijtld.13.0637",
-    "year": null,
+    "date": "2014-05-01",
     "item_type": "journalArticle",
     "url": "http://openurl.ingenta.com/content/xref?genre=article&issn=1027-3719&volume=18&issue=5&spage=547"
   },
@@ -317,7 +317,7 @@
       "Deborah K. Glencross"
     ],
     "doi": "10.4102/ajlm.v10i1.1229",
-    "year": null,
+    "date": "2021-11-30",
     "item_type": "journalArticle",
     "url": "http://www.ajlmonline.org/index.php/AJLM/article/view/1229"
   },
@@ -337,7 +337,7 @@
       "Maria Claudia Vater"
     ],
     "doi": "10.1590/0037-8682-0082-2018",
-    "year": null,
+    "date": "10/2018",
     "item_type": "journalArticle",
     "url": "http://www.scielo.br/scielo.php?script=sci_arttext&pid=S0037-86822018000500631&tlng=en"
   },
@@ -361,7 +361,7 @@
       "Luis E. Cuevas"
     ],
     "doi": "10.1128/JCM.00864-15",
-    "year": null,
+    "date": "08/2015",
     "item_type": "journalArticle",
     "url": "https://journals.asm.org/doi/10.1128/JCM.00864-15"
   },
@@ -386,7 +386,7 @@
       "Corinne S. Merle"
     ],
     "doi": "10.1371/journal.pone.0264206",
-    "year": null,
+    "date": "2022-2-22",
     "item_type": "journalArticle",
     "url": "https://dx.plos.org/10.1371/journal.pone.0264206"
   },
@@ -406,7 +406,7 @@
       "Prathap Tharyan"
     ],
     "doi": "10.1371/journal.pone.0205233",
-    "year": null,
+    "date": "2018-10-29",
     "item_type": "journalArticle",
     "url": "https://dx.plos.org/10.1371/journal.pone.0205233"
   },
@@ -423,7 +423,7 @@
       "Susan E Dorman"
     ],
     "doi": "10.1186/1471-2334-13-352",
-    "year": null,
+    "date": "12/2013",
     "item_type": "journalArticle",
     "url": "https://bmcinfectdis.biomedcentral.com/articles/10.1186/1471-2334-13-352"
   },
@@ -450,7 +450,7 @@
       "Frank Cobelens"
     ],
     "doi": "10.1371/journal.pmed.1001120",
-    "year": null,
+    "date": "2011-11-8",
     "item_type": "journalArticle",
     "url": "https://dx.plos.org/10.1371/journal.pmed.1001120"
   },
@@ -468,7 +468,7 @@
       "Achilles Katamba"
     ],
     "doi": "10.1371/journal.pone.0122574",
-    "year": null,
+    "date": "2015-4-1",
     "item_type": "journalArticle",
     "url": "https://dx.plos.org/10.1371/journal.pone.0122574"
   },
@@ -486,7 +486,7 @@
       "Brittany Moore"
     ],
     "doi": "10.1007/s40258-018-0397-3",
-    "year": null,
+    "date": "8/2018",
     "item_type": "journalArticle",
     "url": "http://link.springer.com/10.1007/s40258-018-0397-3"
   },
@@ -502,7 +502,7 @@
       "Charoen Chuchottaworn"
     ],
     "doi": "10.1016/j.vhri.2019.09.010",
-    "year": null,
+    "date": "05/2020",
     "item_type": "journalArticle",
     "url": "https://linkinghub.elsevier.com/retrieve/pii/S221210992030008X"
   },
@@ -519,7 +519,7 @@
       "Kevin Schwartzman"
     ],
     "doi": "10.1371/journal.pone.0150119",
-    "year": null,
+    "date": "2016-3-18",
     "item_type": "journalArticle",
     "url": "https://dx.plos.org/10.1371/journal.pone.0150119"
   },
@@ -533,7 +533,7 @@
       "Alemayehu Hailu"
     ],
     "doi": "10.1371/journal.pone.0259056",
-    "year": null,
+    "date": "2021-10-25",
     "item_type": "journalArticle",
     "url": "https://dx.plos.org/10.1371/journal.pone.0259056"
   },
@@ -555,7 +555,7 @@
       "Esther Turunga"
     ],
     "doi": "10.1097/QAI.0000000000002371",
-    "year": null,
+    "date": "2020-07-1",
     "item_type": "journalArticle",
     "url": "https://journals.lww.com/10.1097/QAI.0000000000002371"
   },
@@ -575,7 +575,7 @@
       "Jos\u00e9 Mar\u00eda Ten\u00edas-Burillo"
     ],
     "doi": "10.1016/j.eimc.2016.06.009",
-    "year": null,
+    "date": "08/2017",
     "item_type": "journalArticle",
     "url": "https://linkinghub.elsevier.com/retrieve/pii/S0213005X16301550"
   },
@@ -595,7 +595,7 @@
       "Jos\u00e9 Mar\u00eda Ten\u00edas-Burillo"
     ],
     "doi": "10.1016/j.eimc.2016.06.009",
-    "year": null,
+    "date": "08/2017",
     "item_type": "journalArticle",
     "url": "https://linkinghub.elsevier.com/retrieve/pii/S0213005X16301550"
   },
@@ -611,7 +611,7 @@
       "Nelson L.S. Lee"
     ],
     "doi": "10.1016/j.jinf.2014.12.015",
-    "year": null,
+    "date": "04/2015",
     "item_type": "journalArticle",
     "url": "https://linkinghub.elsevier.com/retrieve/pii/S016344531500002X"
   }

data/vaccine_coverage_zotero_items.json

@@ -17,7 +17,7 @@
       "Rania A Tohme"
     ],
     "doi": "10.15585/mmwr.mm7229a2",
-    "year": null,
+    "date": "2023",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -31,7 +31,7 @@
       "Edgar Mugema Mulogo"
     ],
     "doi": "10.21522/TIJPH.2013.09.03.Art019",
-    "year": null,
+    "date": "2021",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -45,7 +45,7 @@
       "Edgar Mugema Mulogo"
     ],
     "doi": "10.21522/TIJPH.2013.09.04.Art008",
-    "year": null,
+    "date": "2021",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -89,7 +89,7 @@
       "G. Shapira"
     ],
     "doi": "10.1371/journal.pmed.1004070",
-    "year": null,
+    "date": "2022",
     "item_type": "journalArticle",
     "url": "https://www.scopus.com/inward/record.uri?eid=2-s2.0-85137126764&doi=10.1371%2fjournal.pmed.1004070&partnerID=40&md5=32f1cd887f5eb46121eb1e48abfcdaaf"
   },
@@ -111,7 +111,7 @@
       "Yahaya Gavamukulya"
     ],
     "doi": "10.1016/j.dib.2019.104269",
-    "year": null,
+    "date": "2019",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -137,7 +137,7 @@
       "Kirsty Le Doare"
     ],
     "doi": "10.1136/bmjgh-2021-006102",
-    "year": null,
+    "date": "2021",
     "item_type": "journalArticle",
     "url": "https://search.ebscohost.com/login.aspx?direct=true&db=cmedm&AN=34452941&site=ehost-live&scope=site"
   },
@@ -152,7 +152,7 @@
       "Tsai-Ching Hsu"
     ],
     "doi": "",
-    "year": null,
+    "date": "2020",
     "item_type": "journalArticle",
     "url": "https://search.ebscohost.com/login.aspx?direct=true&db=a9h&AN=148721705&site=ehost-live&scope=site"
   },
@@ -176,7 +176,7 @@
       "E. M. Mulogo"
     ],
     "doi": "10.1186/s12879-022-07579-w",
-    "year": null,
+    "date": "2022",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -192,7 +192,7 @@
       "D. Kajungu"
     ],
     "doi": "10.1186/s12913-023-09875-w",
-    "year": null,
+    "date": "2023",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -210,7 +210,7 @@
       "C. Banura"
     ],
     "doi": "10.1186/s12889-022-13113-z",
-    "year": null,
+    "date": "2022",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -242,7 +242,7 @@
       "N. Fadl"
     ],
     "doi": "10.1007/s10900-023-01261-1",
-    "year": null,
+    "date": "2024",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -264,7 +264,7 @@
       "Y. Gavamukulya"
     ],
     "doi": "10.1016/j.dib.2019.104269",
-    "year": null,
+    "date": "2019",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -286,7 +286,7 @@
       "Y. Gavamukulya"
     ],
     "doi": "10.9734/ijtdh/2019/v39i330209",
-    "year": null,
+    "date": "2019",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -303,7 +303,7 @@
       "H. W. Reynolds"
     ],
     "doi": "10.3390/vaccines11030647",
-    "year": null,
+    "date": "2023",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -320,7 +320,7 @@
       "E. Sacks"
     ],
     "doi": "10.1093/heapol/czaa099",
-    "year": null,
+    "date": "2020",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -341,7 +341,7 @@
       "L. Berman"
     ],
     "doi": "10.3390/vaccines11020375",
-    "year": null,
+    "date": "2023",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -359,7 +359,7 @@
       "P. Waiswa"
     ],
     "doi": "10.1186/s12913-021-06554-6",
-    "year": null,
+    "date": "2021",
     "item_type": "journalArticle",
     "url": ""
   },
@@ -378,7 +378,7 @@
       "F. Guillen-Grima"
     ],
     "doi": "10.3390/vaccines11061103",
-    "year": null,
+    "date": "2023",
     "item_type": "journalArticle",
     "url": ""
   }

rag/rag_pipeline.py

@@ -35,7 +35,7 @@ class RAGPipeline:
                 metadata = {
                     "title": doc_data.get("title"),
                     "authors": doc_data.get("authors", []),
-                    "year": doc_data.get("year"),
+                    "year": doc_data.get("date"),
                     "doi": doc_data.get("doi"),
                 }
 

requirements.txt

@@ -3,4 +3,5 @@ gradio
 llama-index
 openai
 pandas
-pydantic
+pydantic
+python-dotenv

sample_queries.md

@@ -0,0 +1,2 @@
+1. For each of the studies on vaccine coverage, could you create a table that lists the findings from the vaccine coverage studies,  incorporating variables such as STUDYID, TITLE, DOIs, YEAR?
+