clean

.github/workflows

@@ -1,20 +0,0 @@
-name: Sync to Hugging Face hub
-on:
-  push:
-    branches: [main]
-
-  # to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  sync-to-hub:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Push to hub
-        env:
-          HF_TOKEN: ${{ secrets.token }}
-        run: git push https://mehradans92:[email protected]/spaces/mehradans92/SPACE_NAME main

README.md

@@ -1,6 +1,5 @@
 ---
 title: Decode Elm
-python_version: 3.8.16
 emoji: 📚
 colorFrom: green
 colorTo: pink

app.py

@@ -1,14 +1,18 @@
-import streamlit as st  # Web App
+import streamlit as st  #Web App
+import urllib
+from lxml import html
+import requests
+import re
 import os
+from stqdm import stqdm
+import time
+import shutil
 from PIL import Image
-from utils import *
-import asyncio
 
 import pickle
 docs = None
 api_key = ' '
 
-
 st.set_page_config(layout="wide")
 
 image = Image.open('arxiv_decode.png')
@@ -16,114 +20,251 @@ st.image(image, width=1000)
 
 #title
 st.title("Answering questions from scientific papers")
-st.markdown("##### This tool will allow you to ask questions and get answers based on scientific papers. It uses OpenAI's GPT models, and you must have your own API key. Each query is about 10k tokens, which costs about only $0.20 on your own API key, charged by OpenAI.")
-st.markdown("##### Current version searches on different pre-print servers including [arXiv](https://arxiv.org), [chemRxiv](https://chemrxiv.org/engage/chemrxiv/public-dashboard), [bioRxiv](https://www.biorxiv.org/) and [medRxiv](https://www.medrxiv.org/). 🚧Under development🚧")
+st.markdown("##### This tool will allow you to ask questions and get answers based on scientific papers. It uses OpenAI's GPT models, and you must have your own API key. Each query is about 10k tokens, which costs about only $0.20 on your own API key which is charged by OpenAI.")
+st.markdown("##### Current version searches on [ArXiv](https://arxiv.org) papers only. 🚧Under development🚧")
 st.markdown("Used libraries:\n * [PaperQA](https://github.com/whitead/paper-qa) \n* [langchain](https://github.com/hwchase17/langchain)")
-st.markdown("See this [tweet](https://twitter.com/MehradAnsari/status/1627649959204888576) for a demo.")
-
 
 api_key_url = 'https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key'
 
 api_key = st.text_input('OpenAI API Key',
          placeholder='sk-...', 
          help=f"['What is that?']({api_key_url})",
-         type="password",
-         value = '')     
+         type="password")     
 
 os.environ["OPENAI_API_KEY"] = f"{api_key}" # 
-# if len(api_key) != 51:
-#     st.warning('Please enter a valid OpenAI API key.', icon="⚠️")
+if len(api_key) != 51:
+    st.warning('Please enter a valid OpenAI API key.', icon="⚠️")
+
+
+
+def call_arXiv_API(search_query, search_by='all', sort_by='relevance', max_results='10', folder_name='arxiv-dl'):
+    '''
+      Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
+      <entry>\n    
+      <id>http://arxiv.org/abs/2008.04584v2</id>\n    
+      <updated>2021-05-11T12:00:24Z</updated>\n    
+      <published>2020-08-11T08:47:06Z</published>\n    
+      <title>Bayesian Selective Inference: Non-informative Priors</title>\n    
+      <summary>  We discuss Bayesian inference for parameters selected using the data. First,\nwe provide a critical analysis of the existing positions in the literature\nregarding the correct Bayesian approach under selection. Second, we propose two\ntypes of non-informative priors for selection models. These priors may be\nemployed to produce a posterior distribution in the absence of prior\ninformation as well as to provide well-calibrated frequentist inference for the\nselected parameter. We test the proposed priors empirically in several\nscenarios.\n</summary>\n    
+      <author>\n      <name>Daniel G. Rasines</name>\n    </author>\n    <author>\n      <name>G. Alastair Young</name>\n    </author>\n    
+      <arxiv:comment xmlns:arxiv="http://arxiv.org/schemas/atom">24 pages, 7 figures</arxiv:comment>\n    
+      <link href="http://arxiv.org/abs/2008.04584v2" rel="alternate" type="text/html"/>\n    
+      <link title="pdf" href="http://arxiv.org/pdf/2008.04584v2" rel="related" type="application/pdf"/>\n    
+      <arxiv:primary_category xmlns:arxiv="http://arxiv.org/schemas/atom" term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
+      <category term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
+      <category term="stat.TH" scheme="http://arxiv.org/schemas/atom"/>\n  
+      </entry>\n  
+    '''
+
+    # Remove space in seach query
+    search_query=search_query.strip().replace(" ", "+")
+    # Call arXiv API
+    arXiv_url=f'http://export.arxiv.org/api/query?search_query={search_by}:{search_query}&sortBy={sort_by}&start=0&max_results={max_results}'
+    with urllib.request.urlopen(arXiv_url) as url:
+        s = url.read()
+    
+    # Parse the xml data
+    root = html.fromstring(s)
+    # Fetch relevant pdf information
+    pdf_entries = root.xpath("entry")
+
+    pdf_titles   = []
+    pdf_authors  = []
+    pdf_urls     = []
+    pdf_categories = []
+    folder_names = []
+    pdf_citation = []
+    pdf_years = []
+
+    for i, pdf in enumerate(pdf_entries):
+      # print(pdf.xpath('updated/text()')[0][:4])
+      # xpath return a list with every ocurrence of the html path. Since we're getting each entry individually, we'll take the first element to avoid an unecessary list
+      pdf_titles.append(re.sub('[^a-zA-Z0-9]', ' ', pdf.xpath("title/text()")[0]))
+      pdf_authors.append(pdf.xpath("author/name/text()"))
+      pdf_urls.append(pdf.xpath("link[@title='pdf']/@href")[0])
+      pdf_categories.append(pdf.xpath("category/@term"))
+      folder_names.append(folder_name)
+      pdf_years.append(pdf.xpath('updated/text()')[0][:4])
+      pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. arXiv [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")
+
+      
+
+    pdf_info=list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
+    
+    # Check number of available files
+    # print('Requesting {max_results} files'.format(max_results=max_results))
+    if len(pdf_urls)<int(max_results):
+        matching_pdf_num=len(pdf_urls)
+        # print('Only {matching_pdf_num} files available'.format(matching_pdf_num=matching_pdf_num))
+    return pdf_info, pdf_citation
+
+
+def download_pdf(pdf_info):
+   
+    # if len(os.listdir(f'./{folder_name}') ) != 0:
+            # check folder is empty to avoid using papers from old runs:
+            # os.remove(f'./{folder_name}/*')
+    all_reference_text = []
+    for i,p in enumerate(stqdm(pdf_info, desc='Searching and downloading papers')):
+
+        pdf_title=p[0]
+        pdf_url=p[1]
+        pdf_author=p[2]
+        pdf_category=p[3]
+        folder_name=p[4]
+        pdf_citation=p[5]
+        r = requests.get(pdf_url, allow_redirects=True)
+        if  i == 0:
+            if not os.path.exists(f'{folder_name}'):
+                os.makedirs(f"{folder_name}")
+            else:
+                shutil.rmtree(f'{folder_name}') 
+                os.makedirs(f"{folder_name}")
+        with open(f'{folder_name}/{pdf_title}.pdf', 'wb') as currP:
+            currP.write(r.content)
+        if i == 0:
+            st.markdown("###### Papers found:")
+        st.markdown(f"{i+1}. {pdf_citation}")
+        time.sleep(0.15)
+        all_reference_text.append(f"{i+1}. {pdf_citation}\n")
+    if 'all_reference_text' not in st.session_state:
+        st.session_state.key = 'all_reference_text'
+    st.session_state['all_reference_text'] = ' '.join(all_reference_text)
+
+    # print(all_reference_text)
+    
 
 
 max_results_current = 5
 max_results = max_results_current
-def search_click_callback(search_query, max_results, XRxiv_servers=[]):
+# pdf_info = ''
+# pdf_citation = ''
+def search_click_callback(search_query, max_results):
     global pdf_info, pdf_citation
-    search_engines = XRxivQuery(search_query, max_results, XRxiv_servers=XRxiv_servers)
-    pdf_info = search_engines.call_API()
-    search_engines.download_pdf()
-
+    pdf_info, pdf_citation = call_arXiv_API(f'{search_query}', max_results=max_results)
+    download_pdf(pdf_info)
     return pdf_info
 
+
+
+
 with st.form(key='columns_in_form', clear_on_submit = False):
-    c1, c2 = st.columns([5, 0.8])
+    c1, c2 = st.columns([8,1])
     with c1:
         search_query = st.text_input("Input search query here:", placeholder='Keywords for most relevant search...', value=''
-                                       )
+                                       )#search_query, max_results_current))
 
     with c2:
-        max_results = st.number_input("Max papers", value=max_results_current)
+        max_results = st.text_input("Max papers", value=max_results_current)
         max_results_current = max_results_current
-    st.markdown('Pre-print server')
-    checks = st.columns(4)
-    with checks[0]:
-        ArXiv_check = st.checkbox('arXiv')
-    with checks[1]:
-        ChemArXiv_check = st.checkbox('chemRxiv')
-    with checks[2]:
-        BioArXiv_check = st.checkbox('bioRxiv')
-    with checks[3]:
-        MedrXiv_check = st.checkbox('medRxiv')  
-
     searchButton = st.form_submit_button(label = 'Search')
+    #         search_click(search_query, max_results_default)
 
 if searchButton:
-    # checking which pre-print servers selected
-    XRxiv_servers = []
-    if ArXiv_check:
-        XRxiv_servers.append('rxiv')
-    if ChemArXiv_check:
-        XRxiv_servers.append('chemrxiv')
-    if BioArXiv_check:
-        XRxiv_servers.append('biorxiv')
-    if MedrXiv_check:
-        XRxiv_servers.append('medrxiv')
     global pdf_info
-    pdf_info =  search_click_callback(search_query, max_results, XRxiv_servers=XRxiv_servers)
+    pdf_info =  search_click_callback(search_query, max_results)
     if 'pdf_info' not in st.session_state:
         st.session_state.key = 'pdf_info'
     st.session_state['pdf_info'] = pdf_info
+    # print(f'This is PDF info from search:{pdf_info}')
+
+
+# def tokenize_callback():
+
+    # return docs
 
+# tokenization_form = st.form(key='tokenization-form')
+# tokenization_form.markdown(f"Happy with your paper search results? ")
+# toknizeButton = tokenization_form.form_submit_button(label = "Yes! Let's tokenize.", on_click=tokenize_callback())
+# tokenization_form.markdown("If not, change keywords and search again. [This step costs!](https://openai.com/api/pricing/)")
 
-def answer_callback(question_query, word_count):
+
+
+# submitButton = form.form_submit_button('Submit')
+# with st.form(key='tokenization_form', clear_on_submit = False):
+#     st.markdown(f"Happy with your paper search results? If not, change keywords and search again. [This step costs!](https://openai.com/api/pricing/)")
+#     # st.text_input("Input search query here:", placeholder='Keywords for most relevant search...'
+#     #                                )#search_query, max_results_current))
+#     toknizeButton = st.form_submit_button(label = "Yes! Let's tokenize.")
+
+# if toknizeButton:
+#     tokenize_callback()
+
+# tokenize_callback()
+
+
+
+
+def answer_callback(question_query):
     import paperqa
     global docs
+    # global pdf_info
+    progress_text = "Please wait..."
+    # my_bar = st.progress(0, text = progress_text)
+    st.info('Please wait...', icon="🔥")
     if docs is None:
+        # my_bar.progress(0.2, "Please wait...")
         pdf_info = st.session_state['pdf_info']
+        # print('buliding docs')
         docs = paperqa.Docs()
-        pdf_paths = [f"{p[4]}/{p[0].replace(':','').replace('/','').replace('.','')}.pdf" for p in pdf_info]
+        pdf_paths = [f"{p[4]}/{p[0]}.pdf" for p in pdf_info]
         pdf_citations = [p[5] for p in pdf_info]
         print(list(zip(pdf_paths, pdf_citations)))
+
         for d, c in zip(pdf_paths, pdf_citations):
+            # print(d,c)
             docs.add(d, c)
-    docs._build_texts_index()
-    answer = docs.query(question_query,  length_prompt=f'use {word_count:d} words')
-    st.success('Voila! 😃')
+    # docs._build_faiss_index()
+    answer = docs.query(question_query)
+    # print(answer.formatted_answer)
+    # my_bar.progress(1.0, "Done!")
+    st.success('Voila!')
     return answer.formatted_answer
 
-with st.form(key='question_form', clear_on_submit = False):
-    c1, c2 = st.columns([6, 2])
-    with c1:
-        question_query = st.text_input("What do you wanna know from these papers?", placeholder='Input questions here...',
+
+
+form = st.form(key='question_form')
+question_query = form.text_input("What do you wanna know from these papers?", placeholder='Input questions here...',
                 value='')
-    with c2:
-        word_count = st.slider("Suggested number of words in your answer?", 30, 300, 100)
-    submitButton = st.form_submit_button('Submit')
+submitButton = form.form_submit_button('Submit')
 
 if submitButton:
     with st.expander("Found papers:", expanded=True):
         st.write(f"{st.session_state['all_reference_text']}")
-    with st.spinner('⏳ Please wait...'):
-        start = time.time()
-        final_answer = answer_callback(question_query, word_count)
-        length_answer = len(final_answer)
-        st.text_area("Answer:", final_answer, height=max(length_answer//4, 100))
-        end = time.time()
-        clock_time = end - start
-        with st.empty():
-            st.write(f"✔️ Task completed in {clock_time:.2f} seconds.")   
+    st.text_area("Answer:", answer_callback(question_query), height=600)
+
+# with st.form(key='question_form', clear_on_submit = False):
+#     question_query = st.text_input("What do you wanna know from these papers?", placeholder='Input questions here')
+#     # st.text_input("Input search query here:", placeholder='Keywords for most relevant search...'
+#     #                                )#search_query, max_results_current))
+#     submitButton = form.form_submit_button(label = "Submit", on_click=answer_callback(question_query))
+
+
+# Simulation-based inference bayesian model selection
+
+
+
+
+
+# test = "<ul> \
+#         <li>List item here</li> \
+#         <li>List item here</li> \
+#         <li>List item here</li> \
+#         <li>List item here</li> \
+#         </ul>"
+# test = "'''It was the best of times, it was the worst of times, it was \
+#             the age of wisdom, it was the age of foolishness, it was \
+#             the epoch of belief, it was the epoch of incredulity, it \
+#             was the season of Light, it was the season of Darkness, it\
+#             was the spring of hope, it was the winter of despair, (...)'''"
 
+# citation_text = st.text_area('Papers found:',test, height=300) # f'{pdf_citation}'
 
 
+# for i, cite in enumerate(pdf_citation):
+#     st.markdown(f'{i+1}. {cite}')
+#     time.sleep(1)
 
 
+# def make_clickable('link',text):
+#     return f'<a target="_blank" href="{link}">{text}'

requirements.txt

@@ -2,6 +2,4 @@ streamlit
 urllib3
 lxml
 stqdm
-paper-qa
-bs4
-altair<5
+paper-qa

test/test.py

@@ -1,39 +0,0 @@
-import unittest
-import sys
-sys.path.append('../')
-from utils import *
-import os
-import shutil
-
-class Utils(unittest.TestCase):
-    def test_arXiv_API(self):
-        search_query = 'Tools for Landscape Analysis of Optimisation Problems in Procedural Content Generation for Games'
-        pdf_info = "('Tools for Landscape Analysis of Optimisation Problems in Procedural   Content Generation for Games', 'http://arxiv.org/pdf/2302.08479v1', ['Vanessa Volz', 'Boris Naujoks', 'Pascal Kerschke', 'Tea Tusar'], ['cs.AI'], 'docs', 'Vanessa Volz, Boris Naujoks, Pascal Kerschke, Tea Tusar, Tools for Landscape Analysis of Optimisation Problems in Procedural   Content Generation for Games. arXiv [cs.AI] (2023), (available at http://arxiv.org/pdf/2302.08479v1).')"
-        max_results = 1
-        XRxiv_servers = ['rxiv']
-        search_engines = XRxivQuery(search_query, max_results, XRxiv_servers=XRxiv_servers)
-        test_pdf_info = search_engines.call_API()
-        self.assertEqual(pdf_info, str(test_pdf_info[0]))
-
-    def test_download_pdf(self):
-        search_query = 'Serverless Applications: Why, When, and How?'
-        max_results = 1
-        XRxiv_servers = ['rxiv']
-        search_engines = XRxivQuery(search_query, max_results, XRxiv_servers=XRxiv_servers)
-        test_pdf_info = search_engines.call_API()
-        search_engines.download_pdf()
-        dowloaded_dir = 'docs/Serverless Applications  Why  When  and How .pdf'
-        self.assertTrue(os.path.exists(dowloaded_dir))
-        shutil.rmtree(f'docs/')
-
-    def test_distibute_max_papers(self):
-        XRxiv_servers = ['rxiv', 'medrxiv']
-        max_results = 10
-        max_papers_in_server = distibute_max_papers(max_results, XRxiv_servers)
-        self.assertEqual(max_results, np.sum(max_papers_in_server))
-        self.assertEqual(max_papers_in_server[2], 0)
-        self.assertGreater(max_papers_in_server[0],0)
-        self.assertGreater(max_papers_in_server[3],0)
-        
-if __name__ == '__main__':
-    unittest.main()

utils.py

@@ -1,228 +0,0 @@
-import urllib
-import streamlit as st 
-import requests
-import re
-from stqdm import stqdm
-import os
-import shutil
-import time
-from bs4 import BeautifulSoup as bs
-from datetime import datetime
-from random import uniform as rand
-import json
-import numpy as np
-
-
-
-
-class XRxivQuery:
-    def __init__(self, search_query, max_results, folder_name='docs', XRxiv_servers = [], search_by='all', sort_by='relevance'):
-       self.search_query = search_query
-       self.max_results = max_results
-       self.folder_name = folder_name
-       self.XRxiv_servers = XRxiv_servers
-       self.search_by = search_by
-       self.sort_by = sort_by
-       self.all_pdf_info = []
-       self.all_pdf_citation = []
-
-    def call_API(self):
-        search_query = self.search_query.strip().replace(" ", "+").split('+')#.replace(", ", "+").replace(",", "+")#.split('+')
-        max_papers_in_server = distibute_max_papers(self.max_results, self.XRxiv_servers)
-        if 'rxiv' in self.XRxiv_servers:
-            '''
-            Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
-            <entry>\n    
-            <id>http://arxiv.org/abs/2008.04584v2</id>\n    
-            <updated>2021-05-11T12:00:24Z</updated>\n    
-            <published>2020-08-11T08:47:06Z</published>\n    
-            <title>Bayesian Selective Inference: Non-informative Priors</title>\n    
-            <summary>  We discuss Bayesian inference for parameters selected using the data. First,\nwe provide a critical analysis of the existing positions in the literature\nregarding the correct Bayesian approach under selection. Second, we propose two\ntypes of non-informative priors for selection models. These priors may be\nemployed to produce a posterior distribution in the absence of prior\ninformation as well as to provide well-calibrated frequentist inference for the\nselected parameter. We test the proposed priors empirically in several\nscenarios.\n</summary>\n    
-            <author>\n      <name>Daniel G. Rasines</name>\n    </author>\n    <author>\n      <name>G. Alastair Young</name>\n    </author>\n    
-            <arxiv:comment xmlns:arxiv="http://arxiv.org/schemas/atom">24 pages, 7 figures</arxiv:comment>\n    
-            <link href="http://arxiv.org/abs/2008.04584v2" rel="alternate" type="text/html"/>\n    
-            <link title="pdf" href="http://arxiv.org/pdf/2008.04584v2" rel="related" type="application/pdf"/>\n    
-            <arxiv:primary_category xmlns:arxiv="http://arxiv.org/schemas/atom" term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
-            <category term="math.ST" scheme="http://arxiv.org/schemas/atom"/>\n    
-            <category term="stat.TH" scheme="http://arxiv.org/schemas/atom"/>\n  
-            </entry>\n  
-            '''
-            # Call arXiv API
-            journal = 'arXiv'
-            max_rxiv_papers = max_papers_in_server[0]
-            arXiv_url=f'http://export.arxiv.org/api/query?search_query={self.search_by}:{"+".join(search_query)}&sortBy={self.sort_by}&start=0&max_results={max_rxiv_papers}'
-            with urllib.request.urlopen(arXiv_url) as url:
-                s = url.read()
-            
-            # Parse the xml data
-            from lxml import html
-            root = html.fromstring(s)
-            # Fetch relevant pdf information
-            pdf_entries = root.xpath("entry")
-            pdf_titles   = []
-            pdf_authors  = []
-            pdf_urls     = []
-            pdf_categories = []
-            folder_names = []
-            pdf_citation = []
-            pdf_years = []
-            for i, pdf in enumerate(pdf_entries):
-                pdf_titles.append(re.sub('[^a-zA-Z0-9]', ' ', pdf.xpath("title/text()")[0]))
-                pdf_authors.append(pdf.xpath("author/name/text()"))
-                pdf_urls.append(pdf.xpath("link[@title='pdf']/@href")[0])
-                pdf_categories.append(pdf.xpath("category/@term"))
-                folder_names.append(self.folder_name)
-                pdf_years.append(pdf.xpath('updated/text()')[0][:4])
-                pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. {journal} [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")
-            pdf_info = list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
-            self.all_pdf_info.append(pdf_info)
-
-        if 'chemrxiv' in self.XRxiv_servers:
-            '''
-            See https://chemrxiv.org/engage/chemrxiv/public-api/documentation#tag/public-apiv1items/operation/getPublicapiV1Items
-
-            '''
-            # Call chemrxiv API
-            journal = 'chemRxiv'
-            max_chemrxiv_papers = max_papers_in_server[1]
-            chemrxiv_url = f'https://chemrxiv.org/engage/chemrxiv/public-api/v1/items?term="{"%20".join(search_query)}"&sort=RELEVANT_DESC&limit={max_chemrxiv_papers}'
-            req = urllib.request.Request(
-                    url=chemrxiv_url, 
-                    headers={'User-Agent': 'Mozilla/5.0'}
-                )
-            s = urllib.request.urlopen(req).read()
-            jsonResponse = json.loads(s.decode('utf-8'))
-            pdf_titles   = []
-            pdf_authors  = []
-            pdf_urls     = []
-            pdf_categories = []
-            folder_names = []
-            pdf_citation = []
-            pdf_years = []
-            for i,d in enumerate(jsonResponse['itemHits']):
-                pdf_titles.append(d['item']['title'].replace("\n", ""))
-                authors_dict = d['item']['authors'] 
-                pdf_authors.append([n['firstName']+' '+ n['lastName'] for n in authors_dict])
-                pdf_urls.append('https://chemrxiv.org/engage/chemrxiv/article-details/'+ str(d['item']['id']))
-                pdf_categories.append(journal)
-                folder_names.append(self.folder_name)
-                pdf_years.append(d['item']['statusDate'][:4])
-                pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. {journal} [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")
-                # overwriting url cause chermRxiv sucks!
-                pdf_urls[i] = d['item']['asset']['original']['url']
-            pdf_info = list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
-            self.all_pdf_info.append(pdf_info)   
-
-
-        if 'biorxiv' in self.XRxiv_servers or 'medrxiv' in self.XRxiv_servers:
-            '''
-            Scraps the biorxiv and medrxiv's html to get data from each entry in a search. Entries has the following formatting:
-            <li class="first last odd search-result result-jcode-medrxiv search-result-highwire-citation">
-            <div class="highwire-article-citation highwire-citation-type-highwire-article node" data-apath="/medrxiv/early/2021/02/18/2021.02.12.21251663.atom" data-pisa="medrxiv;2021.02.12.21251663v1" data-pisa-master="medrxiv;2021.02.12.21251663" id="node-medrxivearly202102182021021221251663atom1512875027"><div class="highwire-cite highwire-cite-highwire-article highwire-citation-biorxiv-article-pap-list clearfix">
-            <span class="highwire-cite-title">
-            <a class="highwire-cite-linked-title" data-hide-link-title="0" data-icon-position="" href="http://medrxiv.org/content/early/2021/02/18/2021.02.12.21251663">
-            <span class="highwire-cite-title">ClinGen Variant Curation Interface: A Variant Classification Platform for the Application of Evidence Criteria from ACMG/AMP Guidelines</span></a> </span>
-            <div class="highwire-cite-authors"><span class="highwire-citation-authors">
-            <span class="highwire-citation-author first" data-delta="0"><span class="nlm-given-names">Christine G.</span> <span class="nlm-surname">Preston</span></span>,
-            <span class="highwire-citation-author" data-delta="1"><span class="nlm-given-names">Matt W.</span> <span class="nlm-surname">Wright</span></span>, 
-            <span class="highwire-citation-author" data-delta="2"><span class="nlm-given-names">Rao</span> <span class="nlm-surname">Madhavrao</span></span>, 
-            <div class="highwire-cite-metadata"><span class="highwire-cite-metadata-journal highwire-cite-metadata">medRxiv </span>
-            <span class="highwire-cite-metadata-pages highwire-cite-metadata">2021.02.12.21251663; </span><span class="highwire-cite-metadata-doi highwire-cite-metadata">
-            <span class="doi_label">doi:</span> https://doi.org/10.1101/2021.02.12.21251663 </span></div>
-            <div class="highwire-cite-extras"><div class="hw-make-citation" data-encoded-apath=";medrxiv;early;2021;02;18;2021.02.12.21251663.atom" data-seqnum="0" id="hw-make-citation-0">
-            <a class="link-save-citation-save use-ajax hw-link-save-unsave-catation link-icon" href="/highwire-save-citation/saveapath/%3Bmedrxiv%3Bearly%3B2021%3B02%3B18%3B2021.02.12.21251663.atom/nojs/0" id="link-save-citation-toggle-0" title="Save">
-            <span class="icon-plus"></span> <span class="title">Add to Selected Citations</span></a></div></div>
-            </div>
-            </div></li> 
-            </entry>\n  
-            '''
-            if 'biorxiv' in self.XRxiv_servers and 'medrxiv' not in self.XRxiv_servers:
-                # print('Searching biorxiv\n')
-                max_biorxiv_papers = max_papers_in_server[2]
-                journals_str = f'%20jcode%3Abiorxiv'
-            if 'biorxiv' not in self.XRxiv_servers and 'medrxiv' in self.XRxiv_servers:
-                # print('Searching medrxiv\n')
-                max_biorxiv_papers = max_papers_in_server[3]
-                journals_str = f'%20jcode%3Amedrxiv'
-            if 'biorxiv' in self.XRxiv_servers and 'medrxiv' in self.XRxiv_servers:
-                # print('Searching both biorxiv and medrxiv\n')
-                max_biorxiv_papers = max_papers_in_server[3]+ max_papers_in_server[2] # birxiv and medrxiv are together.
-                journals_str = f'%20jcode%3Abiorxiv%7C%7Cmedrxiv'
-            
-            subject_str = ('%20').join(self.search_query[0].split())
-            for subject in search_query[1:]:
-                subject_str = subject_str + '%252B' + ('%20').join(subject.split())
-            
-            current_dateTime = datetime.now()
-            today = str(current_dateTime)[:10]
-            start_day = '2013-01-01'
-            arXiv_url = f'https://www.biorxiv.org/search/'
-            arXiv_url += subject_str + journals_str + f'%20limit_from%3A2{start_day}%20limit_to%3A{today}%20numresults%3A{max_biorxiv_papers}%20sort%3Arelevance-rank%20format_result%3Astandard'
-
-            url_response = requests.post(arXiv_url)
-            html = bs(url_response.text, features='html.parser')
-            pdf_entries = html.find_all(attrs={'class': 'search-result'})
-            pdf_titles   = []
-            pdf_authors  = []
-            pdf_urls     = []
-            pdf_categories = []
-            folder_names = []
-            pdf_citation = []
-            pdf_years = []
-            for i, pdf in enumerate(pdf_entries):
-                pdf_titles.append(pdf.find('span', attrs={'class': 'highwire-cite-title'}).text.strip())
-                pdf_authors.append(pdf.find('span', attrs={'class': 'highwire-citation-authors'}).text.strip().split(', '))
-                pdf_url = pdf.find('a', href=True)['href']
-                if pdf_url[:4] != 'http':
-                    pdf_url = f'http://www.biorxiv.org'+ pdf_url
-                pdf_urls.append(pdf_url)
-                pdf_categories.append(pdf.find('span', attrs={'class': 'highwire-cite-metadata-journal highwire-cite-metadata'}).text.strip())
-                folder_names.append(self.folder_name)
-                pdf_years.append(pdf.find('span', attrs={'class': 'highwire-cite-metadata-pages highwire-cite-metadata'}).text.strip()[:4])
-                pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. {pdf_categories[i]} ({pdf_years[i]}), (available at {pdf_urls[i]}).")
-
-            pdf_info = list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
-            self.all_pdf_info.append(pdf_info)
-
-        self.all_pdf_info = [item for sublist in self.all_pdf_info for item in sublist]
-        return self.all_pdf_info
-
-    def download_pdf(self):
-        all_reference_text = []
-        for i,p in enumerate(stqdm(self.all_pdf_info, desc='🔍 Searching and downloading papers')):
-            pdf_title=p[0]
-            pdf_category=p[3]
-            pdf_url=p[1]
-            if pdf_category in ['medRxiv', 'bioRxiv']:
-                pdf_url += '.full.pdf'
-            pdf_file_name=p[0].replace(':','').replace('/','').replace('.','').replace('\n','')
-            folder_name=p[4]
-            pdf_citation=p[5]
-            r = requests.get(pdf_url, allow_redirects=True)
-            if  i == 0:
-                if not os.path.exists(f'{folder_name}'):
-                    os.makedirs(f"{folder_name}")
-                else:
-                    shutil.rmtree(f'{folder_name}') 
-                    os.makedirs(f"{folder_name}")
-            with open(f'{folder_name}/{pdf_file_name}.pdf', 'wb') as f:
-                f.write(r.content)
-            if i == 0:
-                st.markdown("###### Papers found:")
-            st.markdown(f"{i+1}. {pdf_citation}")
-            time.sleep(0.15)
-            all_reference_text.append(f"{i+1}. {pdf_citation}\n")
-        if 'all_reference_text' not in st.session_state:
-            st.session_state.key = 'all_reference_text'
-        st.session_state['all_reference_text'] = ' '.join(all_reference_text)
-
-
-                        
-def distibute_max_papers(max_results, XRxiv_servers):
-    fixed_length = len(XRxiv_servers)
-    sample = np.random.multinomial(max_results - fixed_length, np.ones(fixed_length)/fixed_length, size=1)[0] + 1
-    max_papers_in_server = np.zeros(4, dtype=int)
-    all_servers = ['rxiv', 'chemrxiv', 'biorxiv', 'medrxiv']
-    for i,s in enumerate(XRxiv_servers):
-        max_papers_in_server[all_servers.index(s)] = int(sample[i])
-    return  max_papers_in_server