AWS S3 Integration

.ipynb_checkpoints/app-checkpoint.py

@@ -2,20 +2,39 @@ import os
 import openai
 import gradio as gr
 import pdfplumber
+import boto3
 from llama_index.core import Document, VectorStoreIndex, Settings
 from llama_index.llms.openai import OpenAI
 from llama_index.embeddings.openai import OpenAIEmbedding
 from llama_index.core.postprocessor import MetadataReplacementPostProcessor
 from llama_index.core.node_parser import SentenceWindowNodeParser
 
+from dotenv import load_dotenv
+load_dotenv("config.env")
+
 # Set your OpenAI API key here
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 openai.api_key = OPENAI_API_KEY
 
-# Get the current working directory and join with 'resumes'
+# AWS S3 setup
+s3_bucket_name = "sagemaker-studio-gm4vm5dimae"
+s3_client = boto3.client('s3')
+
+# Directory to store downloaded PDFs
 resume_path = 'resumes'
-if not os.path.exists(resume_path):
-    raise ValueError(f"Directory 'resumes' not found")
+os.makedirs(resume_path, exist_ok=True)
+
+# Function to download PDFs from S3
+def download_pdfs_from_s3(bucket_name, local_path):
+    objects = s3_client.list_objects_v2(Bucket=bucket_name)
+    for obj in objects.get('Contents', []):
+        file_name = obj['Key']
+        local_file_path = os.path.join(local_path, file_name)
+        s3_client.download_file(bucket_name, file_name, local_file_path)
+        print(f"Downloaded {file_name} to {local_file_path}")
+
+# Download PDFs
+download_pdfs_from_s3(s3_bucket_name, resume_path)
 
 # Function to load PDFs using pdfplumber
 def load_pdfs_with_pdfplumber(directory):

.ipynb_checkpoints/requirements-checkpoint.txt

@@ -1 +1,6 @@
-huggingface_hub==0.22.2
+gradio
+pdfplumber
+openai
+llama_index
+boto3
+

app.py

@@ -2,6 +2,7 @@ import os
 import openai
 import gradio as gr
 import pdfplumber
+import boto3
 from llama_index.core import Document, VectorStoreIndex, Settings
 from llama_index.llms.openai import OpenAI
 from llama_index.embeddings.openai import OpenAIEmbedding
@@ -15,10 +16,25 @@ load_dotenv("config.env")
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 openai.api_key = OPENAI_API_KEY
 
-# Get the current working directory and join with 'resumes'
+# AWS S3 setup
+s3_bucket_name = "sagemaker-studio-gm4vm5dimae"
+s3_client = boto3.client('s3')
+
+# Directory to store downloaded PDFs
 resume_path = 'resumes'
-if not os.path.exists(resume_path):
-    raise ValueError(f"Directory 'resumes' not found")
+os.makedirs(resume_path, exist_ok=True)
+
+# Function to download PDFs from S3
+def download_pdfs_from_s3(bucket_name, local_path):
+    objects = s3_client.list_objects_v2(Bucket=bucket_name)
+    for obj in objects.get('Contents', []):
+        file_name = obj['Key']
+        local_file_path = os.path.join(local_path, file_name)
+        s3_client.download_file(bucket_name, file_name, local_file_path)
+        print(f"Downloaded {file_name} to {local_file_path}")
+
+# Download PDFs
+download_pdfs_from_s3(s3_bucket_name, resume_path)
 
 # Function to load PDFs using pdfplumber
 def load_pdfs_with_pdfplumber(directory):

requirements.txt

@@ -2,3 +2,5 @@ gradio
 pdfplumber
 openai
 llama_index
+boto3
+