Upload app.py

app.py

@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+"""app.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1deINvEblsMkv9h0gJzuGB4uSamW0DMX5
+"""
+
+pip install streamlit transformers gdown torch pandas numpy
+
+import warnings
+warnings.filterwarnings('ignore')
+
+import streamlit as st
+import pandas as pd
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+from transformers import AutoTokenizer, AutoModel
+import torch
+import gdown
+from pathlib import Path
+from datetime import datetime
+import json
+import torch.cuda
+
+# Configure GPU if available
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+# Initialize session state
+if 'history' not in st.session_state:
+    st.session_state.history = []
+if 'feedback' not in st.session_state:
+    st.session_state.feedback = {}
+
+# Step 1: Optimized Model Loading
+@st.cache_resource
+def load_model_and_tokenizer():
+    """
+    Optimized model loading with GPU support and model quantization
+    """
+    model_name = "Salesforce/codet5-small"
+
+    # Load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    # Load model with optimizations
+    model = AutoModel.from_pretrained(
+        model_name,
+        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+        low_cpu_mem_usage=True
+    )
+
+    # Move model to GPU if available
+    model = model.to(device)
+
+    # Set to evaluation mode
+    model.eval()
+
+    return tokenizer, model
+
+# Step 2: Optimized Dataset Loading
+@st.cache_resource
+def load_data():
+    """
+    Load and prepare dataset with progress tracking
+    """
+    Path("data").mkdir(exist_ok=True)
+    dataset_path = "/content/drive/MyDrive/practice_ml/filtered_dataset.parquet"
+
+    if not Path(dataset_path).exists():
+        with st.spinner('Downloading dataset... This might take a few minutes...'):
+            url = "https://drive.google.com/drive/folders/1dphd3vDKV46GwWKW5uo-MBl0GWGyCWUs?usp=drive_link"
+            gdown.download(url, dataset_path, quiet=False)
+
+    data = pd.read_parquet(dataset_path)
+    data['text'] = data['docstring'].fillna('') + ' ' + data['summary'].fillna('')
+    return data
+
+# Step 3: Optimized Embedding Generation
+@st.cache_data
+def generate_embedding(_model, tokenizer, text):
+    """
+    Generate embeddings with optimized batch processing
+    """
+    inputs = tokenizer(
+        text,
+        return_tensors="pt",
+        padding=True,
+        truncation=True,
+        max_length=512
+    ).to(device)
+
+    with torch.no_grad():
+        outputs = _model.encoder(**inputs)
+    return outputs.last_hidden_state.mean(dim=1).squeeze().cpu().numpy()
+
+def generate_case_study(repo_data):
+    """
+    Generate a concise case study brief from repository data
+    """
+    template = f"""
+    **Project Overview**: {repo_data['summary'][:50]}...
+
+    **Key Features**:
+    - Repository contains production-ready {repo_data['path'].split('/')[-1]} implementation
+    - {repo_data['docstring'][:50]}...
+
+    **Potential Applications**: This repository can be utilized for projects requiring {repo_data['summary'].split()[0:5]}...
+
+    **Implementation Complexity**: {'Medium' if len(repo_data['docstring']) > 500 else 'Low'}
+
+    **Integration Potential**: {'High' if 'api' in repo_data['text'].lower() or 'interface' in repo_data['text'].lower() else 'Medium'}
+    """
+    return template[:150] + "..."
+
+def save_feedback(repo_id, feedback_type):
+    """
+    Save user feedback for a repository
+    """
+    if repo_id not in st.session_state.feedback:
+        st.session_state.feedback[repo_id] = {'likes': 0, 'dislikes': 0}
+    st.session_state.feedback[repo_id][feedback_type] += 1
+
+# Main App
+st.title("Enhanced Repository Recommender System 🚀")
+
+# Sidebar for History and Stats
+with st.sidebar:
+    st.header("📊 Search History")
+    if st.session_state.history:
+        for idx, item in enumerate(st.session_state.history[-5:]):  # Show last 5 searches
+            with st.expander(f"Search {len(st.session_state.history)-idx}: {item['query'][:30]}..."):
+                st.write(f"Time: {item['timestamp']}")
+                st.write(f"Results: {len(item['results'])} repositories")
+                if st.button("Rerun this search", key=f"rerun_{idx}"):
+                    st.session_state.rerun_query = item['query']
+    else:
+        st.write("No search history yet")
+
+    st.header("📈 Usage Statistics")
+    st.write(f"Total Searches: {len(st.session_state.history)}")
+    if st.session_state.feedback:
+        total_likes = sum(f['likes'] for f in st.session_state.feedback.values())
+        total_dislikes = sum(f['dislikes'] for f in st.session_state.feedback.values())
+        st.write(f"Total Likes: {total_likes}")
+        st.write(f"Total Dislikes: {total_dislikes}")
+
+# Load resources
+@st.cache_resource
+def initialize_resources():
+    data = load_data()
+    tokenizer, model = load_model_and_tokenizer()
+    return data, tokenizer, model
+
+data, tokenizer, model = initialize_resources()
+
+# Main interface
+user_query = st.text_area(
+    "Describe your project:",
+    height=150,
+    placeholder="Example: I need a machine learning project for customer churn prediction..."
+)
+
+# Search button and filters
+col1, col2 = st.columns([2, 1])
+with col1:
+    search_button = st.button("🔍 Search Repositories", type="primary")
+with col2:
+    top_n = st.selectbox("Number of results:", [3, 5, 10], index=1)
+
+if search_button and user_query:
+    with st.spinner("Finding relevant repositories..."):
+        # Generate query embedding and get recommendations
+        query_embedding = generate_embedding(model, tokenizer, user_query)
+        data['similarity'] = data['embedding'].apply(
+            lambda x: cosine_similarity([query_embedding], [x])[0][0]
+        )
+        recommendations = data.nlargest(top_n, 'similarity')
+
+        # Save to history
+        st.session_state.history.append({
+            'query': user_query,
+            'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            'results': recommendations['repo'].tolist()
+        })
+
+        # Display recommendations
+        st.markdown("### 🎯 Top Recommendations")
+        for idx, row in recommendations.iterrows():
+            with st.expander(f"Repository {idx + 1}: {row['repo']}", expanded=True):
+                # Repository details
+                col1, col2 = st.columns([2, 1])
+                with col1:
+                    st.markdown(f"**URL:** [View Repository]({row['url']})")
+                    st.markdown(f"**Path:** `{row['path']}`")
+                with col2:
+                    st.metric("Match Score", f"{row['similarity']:.2%}")
+
+                    # Feedback buttons
+                    feedback_col1, feedback_col2 = st.columns(2)
+                    with feedback_col1:
+                        if st.button("👍", key=f"like_{idx}"):
+                            save_feedback(row['repo'], 'likes')
+                            st.success("Thanks for your feedback!")
+                    with feedback_col2:
+                        if st.button("👎", key=f"dislike_{idx}"):
+                            save_feedback(row['repo'], 'dislikes')
+                            st.success("Thanks for your feedback!")
+
+                # Case Study Tab
+                with st.expander("📑 Case Study Brief"):
+                    st.markdown(generate_case_study(row))
+
+                # Documentation Tab
+                if row['docstring']:
+                    with st.expander("📚 Documentation"):
+                        st.markdown(row['docstring'])
+
+# Footer
+st.markdown("---")
+st.markdown(
+    """
+    Made with 🤖 using CodeT5 and Streamlit |
+    GPU Status: {'🟢 Enabled' if torch.cuda.is_available() else '🔴 Disabled'} |
+    Model: CodeT5-Small
+    """
+)