Merge pull request #18 from ido777/cosmetic_changes

.env.template

@@ -0,0 +1,14 @@
+## ArxivDigest- environment seting 
+
+
+##################################################################################################
+# NOTE: This is an important note! 
+# Do not edit and commit your .env.template with your personal keys, it might reveal you API keys
+# copy this file to .env and only then edit your .env file not this file 
+##################################################################################################
+OPENAI_API_KEY=your_api_key
+
+##  EMAIL SETTINGS
+SENDGRID_API_KEY=your_api_key
+FROM_EMAIL=your_email
+TO_EMAIL=your_email

.github/workflows/daily_pipeline.yaml

@@ -23,7 +23,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r src/requirements.txt
+        pip install -r requirements.txt
     - name: Generate Digest 
       run: |
         python src/action.py

README.md

@@ -83,7 +83,12 @@ To locally run the same UI as the Huggign Face space:
  
 1. Install the requirements in `src/requirements.txt` as well as `gradio`.
 2. Run `python src/app.py` and go to the local URL. From there you will be able to preview the papers from today, as well as the generated digests.
+3. If you want to run the action locally you can copy .env.template to .env and then set the environment variables in the .env file.
+- This file may be hidden by default in some operating systems due to the dot prefix. To reveal hidden files, follow the instructions for your 
+- The .env file is one of the files in .gitignore, this means that git does not track it and it will not be uploaded to the repository, accidentally.
+- For this reason you should not edit the original .env.template and put the keys or your email address in the original file, since the .template.env is tracked by git and editing it might cause you to commit it with your API keys.
 
+> **WARNING:** This is an important note! Do not edit and commit  your .env.template with your personal key, or email, it might reveal your  personal data such as API keys and email.
 
 ## ✅ Roadmap
 

src/requirements.txt → requirements.txt

@@ -1,7 +1,8 @@
+PyYAML==6.0
 beautifulsoup4==4.12.2
-tqdm==4.65.0
+numpy==1.25.0
+openai==0.27.8
+python-dotenv==1.0.0
 pytz==2023.3
-numpy==1.24.2
-openai==0.27.4
 sendgrid==6.10.0
-pyyaml==6.00
+tqdm==4.65.0

src/action.py

@@ -6,12 +6,12 @@ from datetime import date
 import argparse
 import yaml
 import os
-
+from dotenv import load_dotenv
+import openai
 from relevancy import generate_relevance_score, process_subject_fields
 from download_new_papers import get_papers
 
 
-
 # Hackathon quality code. Don't judge too harshly.
 # Feel free to submit pull requests to improve the code.
 
@@ -23,7 +23,7 @@ topics = {
     "Quantitative Finance": "q-fin",
     "Statistics": "stat",
     "Electrical Engineering and Systems Science": "eess",
-    "Economics": "econ"
+    "Economics": "econ",
 }
 
 physics_topics = {
@@ -39,32 +39,185 @@ physics_topics = {
     "Nuclear Experiment": "nucl-ex",
     "Nuclear Theory": "nucl-th",
     "Physics": "physics",
-    "Quantum Physics": "quant-ph"
+    "Quantum Physics": "quant-ph",
 }
 
 
 # TODO: surely theres a better way
 category_map = {
-    "Astrophysics": ["Astrophysics of Galaxies", "Cosmology and Nongalactic Astrophysics", "Earth and Planetary Astrophysics", "High Energy Astrophysical Phenomena", "Instrumentation and Methods for Astrophysics", "Solar and Stellar Astrophysics"],
-    "Condensed Matter": ["Disordered Systems and Neural Networks", "Materials Science", "Mesoscale and Nanoscale Physics", "Other Condensed Matter", "Quantum Gases", "Soft Condensed Matter", "Statistical Mechanics", "Strongly Correlated Electrons", "Superconductivity"],
+    "Astrophysics": [
+        "Astrophysics of Galaxies",
+        "Cosmology and Nongalactic Astrophysics",
+        "Earth and Planetary Astrophysics",
+        "High Energy Astrophysical Phenomena",
+        "Instrumentation and Methods for Astrophysics",
+        "Solar and Stellar Astrophysics",
+    ],
+    "Condensed Matter": [
+        "Disordered Systems and Neural Networks",
+        "Materials Science",
+        "Mesoscale and Nanoscale Physics",
+        "Other Condensed Matter",
+        "Quantum Gases",
+        "Soft Condensed Matter",
+        "Statistical Mechanics",
+        "Strongly Correlated Electrons",
+        "Superconductivity",
+    ],
     "General Relativity and Quantum Cosmology": ["None"],
     "High Energy Physics - Experiment": ["None"],
     "High Energy Physics - Lattice": ["None"],
     "High Energy Physics - Phenomenology": ["None"],
     "High Energy Physics - Theory": ["None"],
     "Mathematical Physics": ["None"],
-    "Nonlinear Sciences": ["Adaptation and Self-Organizing Systems", "Cellular Automata and Lattice Gases", "Chaotic Dynamics", "Exactly Solvable and Integrable Systems", "Pattern Formation and Solitons"],
+    "Nonlinear Sciences": [
+        "Adaptation and Self-Organizing Systems",
+        "Cellular Automata and Lattice Gases",
+        "Chaotic Dynamics",
+        "Exactly Solvable and Integrable Systems",
+        "Pattern Formation and Solitons",
+    ],
     "Nuclear Experiment": ["None"],
     "Nuclear Theory": ["None"],
-    "Physics": ["Accelerator Physics", "Applied Physics", "Atmospheric and Oceanic Physics", "Atomic and Molecular Clusters", "Atomic Physics", "Biological Physics", "Chemical Physics", "Classical Physics", "Computational Physics", "Data Analysis, Statistics and Probability", "Fluid Dynamics", "General Physics", "Geophysics", "History and Philosophy of Physics", "Instrumentation and Detectors", "Medical Physics", "Optics", "Physics and Society", "Physics Education", "Plasma Physics", "Popular Physics", "Space Physics"],
+    "Physics": [
+        "Accelerator Physics",
+        "Applied Physics",
+        "Atmospheric and Oceanic Physics",
+        "Atomic and Molecular Clusters",
+        "Atomic Physics",
+        "Biological Physics",
+        "Chemical Physics",
+        "Classical Physics",
+        "Computational Physics",
+        "Data Analysis, Statistics and Probability",
+        "Fluid Dynamics",
+        "General Physics",
+        "Geophysics",
+        "History and Philosophy of Physics",
+        "Instrumentation and Detectors",
+        "Medical Physics",
+        "Optics",
+        "Physics and Society",
+        "Physics Education",
+        "Plasma Physics",
+        "Popular Physics",
+        "Space Physics",
+    ],
     "Quantum Physics": ["None"],
-    "Mathematics": ["Algebraic Geometry", "Algebraic Topology", "Analysis of PDEs", "Category Theory", "Classical Analysis and ODEs", "Combinatorics", "Commutative Algebra", "Complex Variables", "Differential Geometry", "Dynamical Systems", "Functional Analysis", "General Mathematics", "General Topology", "Geometric Topology", "Group Theory", "History and Overview", "Information Theory", "K-Theory and Homology", "Logic", "Mathematical Physics", "Metric Geometry", "Number Theory", "Numerical Analysis", "Operator Algebras", "Optimization and Control", "Probability", "Quantum Algebra", "Representation Theory", "Rings and Algebras", "Spectral Theory", "Statistics Theory", "Symplectic Geometry"],
-    "Computer Science": ["Artificial Intelligence", "Computation and Language", "Computational Complexity", "Computational Engineering, Finance, and Science", "Computational Geometry", "Computer Science and Game Theory", "Computer Vision and Pattern Recognition", "Computers and Society", "Cryptography and Security", "Data Structures and Algorithms", "Databases", "Digital Libraries", "Discrete Mathematics", "Distributed, Parallel, and Cluster Computing", "Emerging Technologies", "Formal Languages and Automata Theory", "General Literature", "Graphics", "Hardware Architecture", "Human-Computer Interaction", "Information Retrieval", "Information Theory", "Logic in Computer Science", "Machine Learning", "Mathematical Software", "Multiagent Systems", "Multimedia", "Networking and Internet Architecture", "Neural and Evolutionary Computing", "Numerical Analysis", "Operating Systems", "Other Computer Science", "Performance", "Programming Languages", "Robotics", "Social and Information Networks", "Software Engineering", "Sound", "Symbolic Computation", "Systems and Control"],
-    "Quantitative Biology": ["Biomolecules", "Cell Behavior", "Genomics", "Molecular Networks", "Neurons and Cognition", "Other Quantitative Biology", "Populations and Evolution", "Quantitative Methods", "Subcellular Processes", "Tissues and Organs"],
-    "Quantitative Finance": ["Computational Finance", "Economics", "General Finance", "Mathematical Finance", "Portfolio Management", "Pricing of Securities", "Risk Management", "Statistical Finance", "Trading and Market Microstructure"],
-    "Statistics": ["Applications", "Computation", "Machine Learning", "Methodology", "Other Statistics", "Statistics Theory"],
-    "Electrical Engineering and Systems Science": ["Audio and Speech Processing", "Image and Video Processing", "Signal Processing", "Systems and Control"],
-    "Economics": ["Econometrics", "General Economics", "Theoretical Economics"]
+    "Mathematics": [
+        "Algebraic Geometry",
+        "Algebraic Topology",
+        "Analysis of PDEs",
+        "Category Theory",
+        "Classical Analysis and ODEs",
+        "Combinatorics",
+        "Commutative Algebra",
+        "Complex Variables",
+        "Differential Geometry",
+        "Dynamical Systems",
+        "Functional Analysis",
+        "General Mathematics",
+        "General Topology",
+        "Geometric Topology",
+        "Group Theory",
+        "History and Overview",
+        "Information Theory",
+        "K-Theory and Homology",
+        "Logic",
+        "Mathematical Physics",
+        "Metric Geometry",
+        "Number Theory",
+        "Numerical Analysis",
+        "Operator Algebras",
+        "Optimization and Control",
+        "Probability",
+        "Quantum Algebra",
+        "Representation Theory",
+        "Rings and Algebras",
+        "Spectral Theory",
+        "Statistics Theory",
+        "Symplectic Geometry",
+    ],
+    "Computer Science": [
+        "Artificial Intelligence",
+        "Computation and Language",
+        "Computational Complexity",
+        "Computational Engineering, Finance, and Science",
+        "Computational Geometry",
+        "Computer Science and Game Theory",
+        "Computer Vision and Pattern Recognition",
+        "Computers and Society",
+        "Cryptography and Security",
+        "Data Structures and Algorithms",
+        "Databases",
+        "Digital Libraries",
+        "Discrete Mathematics",
+        "Distributed, Parallel, and Cluster Computing",
+        "Emerging Technologies",
+        "Formal Languages and Automata Theory",
+        "General Literature",
+        "Graphics",
+        "Hardware Architecture",
+        "Human-Computer Interaction",
+        "Information Retrieval",
+        "Information Theory",
+        "Logic in Computer Science",
+        "Machine Learning",
+        "Mathematical Software",
+        "Multiagent Systems",
+        "Multimedia",
+        "Networking and Internet Architecture",
+        "Neural and Evolutionary Computing",
+        "Numerical Analysis",
+        "Operating Systems",
+        "Other Computer Science",
+        "Performance",
+        "Programming Languages",
+        "Robotics",
+        "Social and Information Networks",
+        "Software Engineering",
+        "Sound",
+        "Symbolic Computation",
+        "Systems and Control",
+    ],
+    "Quantitative Biology": [
+        "Biomolecules",
+        "Cell Behavior",
+        "Genomics",
+        "Molecular Networks",
+        "Neurons and Cognition",
+        "Other Quantitative Biology",
+        "Populations and Evolution",
+        "Quantitative Methods",
+        "Subcellular Processes",
+        "Tissues and Organs",
+    ],
+    "Quantitative Finance": [
+        "Computational Finance",
+        "Economics",
+        "General Finance",
+        "Mathematical Finance",
+        "Portfolio Management",
+        "Pricing of Securities",
+        "Risk Management",
+        "Statistical Finance",
+        "Trading and Market Microstructure",
+    ],
+    "Statistics": [
+        "Applications",
+        "Computation",
+        "Machine Learning",
+        "Methodology",
+        "Other Statistics",
+        "Statistics Theory",
+    ],
+    "Electrical Engineering and Systems Science": [
+        "Audio and Speech Processing",
+        "Image and Video Processing",
+        "Signal Processing",
+        "Systems and Control",
+    ],
+    "Economics": ["Econometrics", "General Economics", "Theoretical Economics"],
 }
 
 
@@ -83,8 +236,10 @@ def generate_body(topic, categories, interest, threshold):
                 raise RuntimeError(f"{category} is not a category of {topic}")
         papers = get_papers(abbr)
         papers = [
-            t for t in papers
-            if bool(set(process_subject_fields(t['subjects'])) & set(categories))]
+            t
+            for t in papers
+            if bool(set(process_subject_fields(t["subjects"])) & set(categories))
+        ]
     else:
         papers = get_papers(abbr)
     if interest:
@@ -92,27 +247,43 @@ def generate_body(topic, categories, interest, threshold):
             papers,
             query={"interest": interest},
             threshold_score=threshold,
-            num_paper_in_prompt=16)
+            num_paper_in_prompt=16,
+        )
         body = "<br><br>".join(
-            [f'Title: <a href="{paper["main_page"]}">{paper["title"]}</a><br>Authors: {paper["authors"]}<br>Score: {paper["Relevancy score"]}<br>Reason: {paper["Reasons for match"]}'
-             for paper in relevancy])
+            [
+                f'Title: <a href="{paper["main_page"]}">{paper["title"]}</a><br>Authors: {paper["authors"]}<br>Score: {paper["Relevancy score"]}<br>Reason: {paper["Reasons for match"]}'
+                for paper in relevancy
+            ]
+        )
         if hallucination:
-            body = "Warning: the model hallucinated some papers. We have tried to remove them, but the scores may not be accurate.<br><br>" + body
+            body = (
+                "Warning: the model hallucinated some papers. We have tried to remove them, but the scores may not be accurate.<br><br>"
+                + body
+            )
     else:
         body = "<br><br>".join(
-            [f'Title: <a href="{paper["main_page"]}">{paper["title"]}</a><br>Authors: {paper["authors"]}'
-             for paper in papers])
+            [
+                f'Title: <a href="{paper["main_page"]}">{paper["title"]}</a><br>Authors: {paper["authors"]}'
+                for paper in papers
+            ]
+        )
     return body
 
 
 if __name__ == "__main__":
+    # Load the .env file.
+    load_dotenv()
     parser = argparse.ArgumentParser()
-    parser.add_argument("--config", help="yaml config file to use", default="config.yaml")
+    parser.add_argument(
+        "--config", help="yaml config file to use", default="config.yaml"
+    )
     args = parser.parse_args()
     with open(args.config, "r") as f:
         config = yaml.safe_load(f)
+
     if "OPENAI_API_KEY" not in os.environ:
         raise RuntimeError("No openai api key found")
+    openai.api_key = os.environ.get("OPENAI_API_KEY")
 
     topic = config["topic"]
     categories = config["categories"]
@@ -123,8 +294,8 @@ if __name__ == "__main__":
     body = generate_body(topic, categories, interest, threshold)
     with open("digest.html", "w") as f:
         f.write(body)
-    if os.environ.get('SENDGRID_API_KEY', None):
-        sg = SendGridAPIClient(api_key=os.environ.get('SENDGRID_API_KEY'))
+    if os.environ.get("SENDGRID_API_KEY", None):
+        sg = SendGridAPIClient(api_key=os.environ.get("SENDGRID_API_KEY"))
         from_email = Email(from_email)  # Change to your verified sender
         to_email = To(to_email)
         subject = date.today().strftime("Personalized arXiv Digest, %d %b %Y")