add utils and the main script

app.py

@@ -0,0 +1,94 @@
+import logging
+import os
+import gradio as gr
+import pandas as pd
+from utils import get_zotero_ids, get_arxiv_papers, get_hf_embeddings, upload_to_pinecone, get_new_papers, recommend_papers
+
+HF_API_KEY = os.getenv('HF_API_KEY')
+PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
+INDEX_NAME = os.getenv('INDEX_NAME')
+NAMESPACE_NAME = os.getenv('NAMESPACE_NAME')
+
+def category_radio(cat):
+ if cat == 'Computer Vision and Pattern Recognition':
+ return 'cs.CV'
+ elif cat == 'Computation and Language':
+ return 'cs.CL'
+ elif cat == 'Artificial Intelligence':
+ return 'cs.AI'
+ elif cat == 'Robotics':
+ return 'cs.RO'
+
+def comment_radio(com):
+ if com == 'CVPR':
+ return 'CVPR'
+ else:
+ return None
+ 
+def recommend_link(recs):
+ return recs
+
+with gr.Blocks() as demo:
+
+ zotero_api_key = gr.Textbox(label="Zotero API Key")
+
+ zotero_library_id = gr.Textbox(label="Zotero Library ID")
+
+ zotero_tag = gr.Textbox(label="Zotero Tag")
+
+ arxiv_category_name = gr.State([])
+ radio_arxiv_category_name = gr.Radio(['Computer Vision and Pattern Recognition', 'Computation and Language', 'Artificial Intelligence', 'Robotics'], label="ArXiv Category Query")
+ radio_arxiv_category_name.change(fn = category_radio, inputs= radio_arxiv_category_name, outputs= arxiv_category_name)
+
+ arxiv_comment_query = gr.State([])
+ radio_arxiv_comment_query = gr.Radio(['CVPR', 'None'], label="ArXiv Comment Query")
+ radio_arxiv_comment_query.change(fn = comment_radio, inputs= radio_arxiv_comment_query, outputs= arxiv_comment_query)
+
+ threshold = gr.Slider(minimum= 0.70, maximum= 0.99, label="Similarity Score Threshold")
+
+ init_output = gr.Textbox(label="Project Initialization Result")
+
+ rec_output = gr.Markdown(label = "Recommended Papers")
+
+ init_btn = gr.Button("Initialize")
+
+ rec_btn = gr.Button("Recommend")
+
+ @init_btn.click(inputs= [zotero_api_key, zotero_library_id, zotero_tag], outputs= [init_output])
+ def init(zotero_api_key, zotero_library_id, zotero_tag, hf_api_key = HF_API_KEY, pinecone_api_key = PINECONE_API_KEY, index_name = INDEX_NAME, namespace_name = NAMESPACE_NAME):
+
+ logging.basicConfig(filename= '/mnt/c/Users/ankit/Desktop/Portfolio/Paper-Recommendation-System/logs/logfile.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+ logging.info("Project Initialization Script Started (Serverless)")
+ 
+ ids = get_zotero_ids(zotero_api_key, zotero_library_id, zotero_tag)
+
+ df = get_arxiv_papers(ids)
+
+ embeddings, dim = get_hf_embeddings(hf_api_key, df)
+
+ feedback = upload_to_pinecone(pinecone_api_key, index_name, namespace_name, embeddings, dim, df)
+
+ logging.info(feedback)
+ if feedback is dict:
+ return f"Retrieved {len(ids)} papers from Zotero. Successfully upserted {feedback['upserted_count']} embeddings in {namespace_name} namespace."
+ else :
+ return feedback
+ 
+ @rec_btn.click(inputs= [arxiv_category_name, arxiv_comment_query, threshold], outputs= [rec_output])
+ def recs(arxiv_category_name, arxiv_comment_query, threshold, hf_api_key = HF_API_KEY, pinecone_api_key = PINECONE_API_KEY, index_name = INDEX_NAME, namespace_name = NAMESPACE_NAME):
+ logging.info("Weekly Script Started (Serverless)")
+
+ df = get_arxiv_papers(category= arxiv_category_name, comment= arxiv_comment_query)
+
+ df = get_new_papers(df)
+
+ if not isinstance(df, pd.DataFrame):
+ return df
+ 
+ embeddings, _ = get_hf_embeddings(hf_api_key, df)
+
+ results = recommend_papers(pinecone_api_key, index_name, namespace_name, embeddings, df, threshold)
+
+ return results
+
+demo.launch(share = True)

utils.py

@@ -0,0 +1,132 @@
+import pandas as pd
+import arxiv
+import requests
+from pinecone import Pinecone, ServerlessSpec
+import logging
+import os
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+os.chdir(script_dir) 
+
+def get_zotero_ids(api_key, library_id, tag):
+
+ base_url = 'https://api.zotero.org'
+ suffix = '/users/'+ library_id +'/items?tag='+ tag
+
+ header = {'Authorization': 'Bearer '+ api_key}
+ request = requests.get(base_url + suffix, headers= header)
+ 
+ return [data['data']['archiveID'].replace('arXiv:', '') for data in request.json()]
+
+def get_arxiv_papers(ids = None, category = None, comment = None):
+
+ logging.getLogger('arxiv').setLevel(logging.WARNING)
+
+ client = arxiv.Client()
+
+ if category is None:
+ search = arxiv.Search(
+ id_list= ids,
+ max_results= len(ids),
+ )
+ else :
+ if comment is None:
+ custom_query = f'cat:{category}'
+ else:
+ custom_query = f'cat:{category} AND co:{comment}'
+
+ search = arxiv.Search(
+ query = custom_query,
+ max_results= 15,
+ sort_by= arxiv.SortCriterion.SubmittedDate
+ )
+ if ids is None and category is None:
+ raise ValueError('not a valid query')
+
+ df = pd.DataFrame({'Title': [result.title for result in client.results(search)],
+ 'Abstract': [result.summary.replace('\n', ' ') for result in client.results(search)],
+ 'Date': [result.published.date().strftime('%Y-%m-%d') for result in client.results(search)],
+ 'id': [result.entry_id for result in client.results(search)]})
+ 
+ if ids:
+ df.to_csv('arxiv-scrape.csv', index = False)
+ return df
+
+def get_hf_embeddings(api_key, df):
+
+ title_abs = [title + '[SEP]' + abstract for title,abstract in zip(df['Title'], df['Abstract'])]
+
+ API_URL = "https://api-inference.huggingface.co/models/malteos/scincl"
+ headers = {"Authorization": f"Bearer {api_key}"}
+
+ response = requests.post(API_URL, headers=headers, json={"inputs": title_abs, "wait_for_model": False})
+ print(str(response.status_code) + 'This part needs an update, causing KeyError 0')
+ if response.status_code == 503:
+ response = requests.post(API_URL, headers=headers, json={"inputs": title_abs, "wait_for_model": True})
+
+ embeddings = response.json()
+
+ return embeddings, len(embeddings[0])
+
+
+def upload_to_pinecone(api_key, index, namespace, embeddings, dim, df):
+ input = [{'id': df['id'][i], 'values': embeddings[i]} for i in range(len(embeddings))]
+
+ pc = Pinecone(api_key = api_key)
+ if index in pc.list_indexes().names():
+ while True:
+ logging.warning(f'Index name : {index} already exists.')
+ return f'Index name : {index} already exists'
+ 
+ pc.create_index(
+ name=index,
+ dimension=dim,
+ metric="cosine",
+ spec=ServerlessSpec(
+ cloud='aws', 
+ region='us-east-1'
+ ) 
+ ) 
+
+ index = pc.Index(index)
+ return index.upsert(vectors=input, namespace=namespace)
+
+
+def get_new_papers(df):
+ df_main = pd.read_csv('arxiv-scrape.csv')
+ df.reset_index(inplace=True)
+ df.drop(columns=['index'], inplace=True)
+ union_df = df.merge(df_main, how='left', indicator=True)
+ df = union_df[union_df['_merge'] == 'left_only'].drop(columns=['_merge'])
+ if df.empty:
+ return 'No New Papers Found'
+ else:
+ df_main = pd.concat([df_main, df], ignore_index= True)
+ df_main.drop_duplicates(inplace= True)
+ df_main.to_csv('arxiv-scrape.csv', index = False)
+ return df
+
+def recommend_papers(api_key, index, namespace, embeddings, df, threshold):
+
+ pc = Pinecone(api_key = api_key)
+ if index in pc.list_indexes().names():
+ index = pc.Index(index)
+ else:
+ raise ValueError(f"{index} doesnt exist. Project isnt initialized properly")
+ 
+ results = []
+ score_threshold = threshold
+ for i,embedding in enumerate(embeddings):
+ query = embedding
+ result = index.query(namespace=namespace,vector=query,top_k=3,include_values=False)
+ sum_score = sum(match['score'] for match in result['matches'])
+ if sum_score > score_threshold:
+ results.append(f"Paper-URL : [{df['id'][i]}]({df['id'][i]}) with score: {sum_score / 3} <br />")
+
+ if results:
+ return '\n'.join(results)
+ else:
+ return 'No Interesting Paper'
+
+
+