app.py

import streamlit as st
from datasets import load_dataset
import pandas as pd
from transformers import pipeline
import time

# Constants
universities_url = "https://www.4icu.org/top-universities-world/"

# Load datasets with caching to optimize performance
@st.cache_resource
def load_datasets():
    ds_jobs = load_dataset("lukebarousse/data_jobs")
    ds_courses = load_dataset("azrai99/coursera-course-dataset")
    ds_custom_courses = pd.read_csv("final_cleaned_merged_coursera_courses.csv")
    ds_custom_jobs = pd.read_csv("merged_data_science_jobs.csv")
    ds_custom_universities = pd.read_csv("merged_university_data_cleaned (1).csv")
    return ds_jobs, ds_courses, ds_custom_courses, ds_custom_jobs, ds_custom_universities

ds_jobs, ds_courses, ds_custom_courses, ds_custom_jobs, ds_custom_universities = load_datasets()

# Initialize the pipeline with caching, using an accessible model like 'google/flan-t5-large'
@st.cache_resource
def load_pipeline():
    return pipeline("text2text-generation", model="google/flan-t5-large")

qa_pipeline = load_pipeline()

# Streamlit App Interface
st.title("Career Counseling Application")
st.subheader("Build Your Profile and Discover Tailored Career Recommendations")

# Sidebar for Profile Setup
st.sidebar.header("Profile Setup")
educational_background = st.sidebar.text_input("Educational Background (e.g., Degree, Major)")
interests = st.sidebar.text_input("Interests (e.g., AI, Data Science, Engineering)")
tech_skills = st.sidebar.text_area("Technical Skills (e.g., Python, SQL, Machine Learning)")
soft_skills = st.sidebar.text_area("Soft Skills (e.g., Communication, Teamwork)")

# Save profile data for session-based recommendations
if st.sidebar.button("Save Profile"):
    with st.spinner('Saving your profile...'):
        time.sleep(2)  # Simulate processing time
        st.session_state.profile_data = {
            "educational_background": educational_background,
            "interests": interests,
            "tech_skills": tech_skills,
            "soft_skills": soft_skills
        }
        st.sidebar.success("Profile saved successfully!")

# Intelligent Q&A Section
st.header("Intelligent Q&A")
question = st.text_input("Ask a career-related question:")
if question:
    with st.spinner('Processing your question...'):
        answer = qa_pipeline(question)[0]["generated_text"]
        time.sleep(2)  # Simulate processing time
        st.write("Answer:", answer)

# Career and Job Recommendations Section
st.header("Job Recommendations")
if "profile_data" in st.session_state:
    with st.spinner('Generating job recommendations...'):
        time.sleep(2)  # Simulate processing time
        job_recommendations = []

        # Find jobs from ds_jobs
        for job in ds_jobs["train"]:
            job_title = job.get("job_title_short", "Unknown Job Title")
            job_skills = job.get("job_skills", "") or ""
            if any(skill.lower() in job_skills.lower() for skill in st.session_state.profile_data["tech_skills"].split(",")):
                job_recommendations.append(job_title)

        # Find jobs from ds_custom_jobs
        for _, job in ds_custom_jobs.iterrows():
            job_title = job.get("job_title", "Unknown Job Title")
            job_skills = job.get("skills", "") or ""
            if any(skill.lower() in job_skills.lower() for skill in st.session_state.profile_data["tech_skills"].split(",")):
                job_recommendations.append(job_title)

        # Remove duplicates and keep the unique job titles
        job_recommendations = list(set(job_recommendations))

        if job_recommendations:
            st.subheader("Based on your profile, here are some potential job roles:")
            for job in job_recommendations[:5]:  # Limit to top 5 job recommendations
                st.write("- ", job)
        else:
            st.write("No specific job recommendations found matching your profile. Here are some general recommendations:")
            for job in ["Data Analyst", "Software Engineer", "Project Manager", "Research Scientist", "Business Analyst"][:5]:
                st.write("- ", job)

# Course Suggestions Section
st.header("Recommended Courses")
if "profile_data" in st.session_state:
    with st.spinner('Finding courses related to your profile...'):
        time.sleep(2)  # Simulate processing time
        course_recommendations = []

        # Find relevant courses in ds_courses
        for course in ds_courses["train"]:
            if any(interest.lower() in course.get("Course Name", "").lower() for interest in st.session_state.profile_data["interests"].split(",")):
                course_recommendations.append({
                    "name": course.get("Course Name", "Unknown Course Title"),
                    "url": course.get("Links", "#")
                })

        # Find relevant courses in ds_custom_courses
        for _, row in ds_custom_courses.iterrows():
            if any(interest.lower() in row["Course Name"].lower() for interest in st.session_state.profile_data["interests"].split(",")):
                course_recommendations.append({
                    "name": row["Course Name"],
                    "url": row.get("Links", "#")
                })

        # Remove duplicates from course recommendations by converting to a set of tuples and back to a list
        course_recommendations = list({(course["name"], course["url"]) for course in course_recommendations})

        # If there are fewer than 5 exact matches, add nearly related courses
        if len(course_recommendations) < 5:
            for course in ds_courses["train"]:
                if len(course_recommendations) >= 5:
                    break
                if any(skill.lower() in course.get("Course Name", "").lower() for skill in st.session_state.profile_data["tech_skills"].split(",")):
                    course_recommendations.append((course.get("Course Name", "Unknown Course Title"), course.get("Links", "#")))

            for _, row in ds_custom_courses.iterrows():
                if len(course_recommendations) >= 5:
                    break
                if any(skill.lower() in row["Course Name"].lower() for skill in st.session_state.profile_data["tech_skills"].split(",")):
                    course_recommendations.append((row["Course Name"], row.get("Links", "#")))

            # Remove duplicates again after adding nearly related courses
            course_recommendations = list({(name, url) for name, url in course_recommendations})

        if course_recommendations:
            st.write("Here are the top 5 courses related to your interests:")
            for course in course_recommendations[:5]:  # Limit to top 5 course recommendations
                st.write(f"- [{course[0]}]({course[1]})")

# University Recommendations Section
st.header("Top Universities")
st.write("For further education, you can explore the top universities worldwide:")
st.write(f"[View Top Universities Rankings]({universities_url})")

# Conclusiona
st.write("Thank you for using the Career Counseling Application!")