app.py

import torch
import time
import numpy as np
import pandas as pd
import evaluate
import gradio as gr
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from sklearn.metrics import accuracy_score, classification_report
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
from dataclasses import dataclass, field
from typing import List, Optional

# Load Accuracy and F1-Score Metrics
accuracy_metric = evaluate.load("accuracy")
f1_metric = evaluate.load("f1")

# Define Model Paths
MODEL_PATHS = {
    "MindBERT": "DrSyedFaizan/mindBERT",
    "BERT-base": "bert-base-uncased",
    "RoBERTa": "roberta-base",
    "DistilBERT": "distilbert-base-uncased"
}

# Load Reddit Mental Health Dataset
def load_reddit_data(file_path):
    df = pd.read_csv(file_path)
    df = df.dropna(subset=["text", "label"])  # Ensure no missing values in relevant columns
    return df

# Preprocess Dataset
def preprocess_data(df, sample_size=100):
    df_sample = df.sample(n=sample_size, random_state=42)  # Sample a subset
    test_texts = df_sample["text"].tolist()
    test_labels = df_sample["label"].tolist()
    return test_texts, test_labels

# Function to evaluate models
def evaluate_models(dataset_path):
    df = load_reddit_data(dataset_path)
    test_texts, test_labels = preprocess_data(df)
    results = []
    
    model_metadata = {
        "MindBERT": {"model_type": "BERT", "precision": "float16", "params": 0.11, "license": "MIT"},
        "BERT-base": {"model_type": "BERT", "precision": "float16", "params": 0.11, "license": "Apache-2.0"},
        "RoBERTa": {"model_type": "RoBERTa", "precision": "float16", "params": 0.125, "license": "MIT"},
        "DistilBERT": {"model_type": "DistilBERT", "precision": "float16", "params": 0.067, "license": "Apache-2.0"}
    }
    
    for model_name, model_path in MODEL_PATHS.items():
        print(f"Evaluating {model_name}...")
        tokenizer = AutoTokenizer.from_pretrained(model_path)
        model = AutoModelForSequenceClassification.from_pretrained(model_path)
        model.eval()
        
        inputs = tokenizer(test_texts, padding=True, truncation=True, return_tensors="pt")
        
        start_time = time.time()
        with torch.no_grad():
            outputs = model(**inputs)
            logits = outputs.logits
            predictions = torch.argmax(logits, dim=1).numpy()
        end_time = time.time()
        
        accuracy = accuracy_score(test_labels, predictions)
        f1_score = f1_metric.compute(predictions=predictions, references=test_labels, average="macro")["f1"]
        inference_time = round(end_time - start_time, 4)
        
        result = {
            "model": model_name,
            "model_type": model_metadata[model_name]["model_type"],
            "precision": model_metadata[model_name]["precision"],
            "params": model_metadata[model_name]["params"],
            "accuracy": round(accuracy, 4),
            "f1_score": round(f1_score, 4),
            "inference_time": inference_time,
            "license": model_metadata[model_name]["license"]
        }
        results.append(result)
    
    return pd.DataFrame(results)

# Load and evaluate
DATASET_PATH = "path/to/reddit_mental_health.csv"
df_results = evaluate_models(DATASET_PATH)

# Display results
df_results

# Initialize leaderboard with custom columns
def init_leaderboard(dataframe):
    if dataframe is None or dataframe.empty:
        raise ValueError("Leaderboard DataFrame is empty or None.")
    
    columns = fields(ModelEvalColumn)
    
    return Leaderboard(
        value=dataframe,
        datatype=[c.type for c in columns],
        select_columns=SelectColumns(
            default_selection=[c.name for c in columns if c.displayed_by_default],
            cant_deselect=[c.name for c in columns if c.never_hidden],
            label="Select Columns to Display:",
        ),
        search_columns=["model", "license"],
        hide_columns=[c.name for c in columns if c.hidden],
        filter_columns=[
            ColumnFilter("model_type", type="checkboxgroup", label="Model types"),
            ColumnFilter("precision", type="checkboxgroup", label="Precision"),
            ColumnFilter(
                "params",
                type="slider",
                min=0.01,
                max=0.5,
                label="Select the number of parameters (B)",
            ),
        ],
        interactive=False,
    )

# Custom CSS similar to the original
custom_css = """
.markdown-text {
    padding: 0 20px;
}
.tab-buttons button.selected {
    background-color: #FF9C00 !important;
    color: white !important;
}
"""

# Create Gradio Interface
demo = gr.Blocks(css=custom_css)

with demo:
    gr.HTML("<h1>Mental Health Model Evaluation Benchmark</h1>")
    gr.Markdown("This benchmark evaluates various transformer models on mental health classification tasks.", elem_classes="markdown-text")
    
    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 Model Benchmark", elem_id="model-benchmark-tab", id=0):
            # Get evaluation results
            df_results = evaluate_models()
            leaderboard = init_leaderboard(df_results)
        
        with gr.TabItem("📝 About", elem_id="about-tab", id=1):
            gr.Markdown("""
            ## About This Benchmark
            
            This leaderboard compares various transformer models on mental health text classification tasks.
            The benchmark uses a test set from Reddit Mental Health datasets with examples covering anxiety, 
            depression, bipolar disorder, suicidal ideation, stress, and normal emotional states.
            
            Models are evaluated on:
            - Accuracy
            - F1-Score (Macro)
            - Inference Time
            
            ### Model Types
            - BERT-based models
            - RoBERTa models
            - DistilBERT models
            - Specialized mental health models (MindBERT)
            """, elem_classes="markdown-text")
            
        with gr.TabItem("🚀 Submit Model", elem_id="submit-tab", id=2):
            gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
            
            with gr.Row():
                with gr.Column():
                    model_name_textbox = gr.Textbox(label="Model name")
                    model_path_textbox = gr.Textbox(label="Model path (HF repo ID)")
                    model_type = gr.Dropdown(
                        choices=["BERT", "RoBERTa", "DistilBERT", "GPT", "T5", "Other"],
                        label="Model type",
                        multiselect=False,
                        value=None,
                        interactive=True,
                    )
                    
                with gr.Column():
                    precision = gr.Dropdown(
                        choices=["float16", "float32", "int8", "int4"],
                        label="Precision",
                        multiselect=False,
                        value="float16",
                        interactive=True,
                    )
                    params = gr.Number(label="Parameters (billions)", value=0.11)
                    license = gr.Textbox(label="License", value="Apache-2.0")
                    
            submit_button = gr.Button("Submit Model for Evaluation")
            submission_result = gr.Markdown()
            
            # This would typically connect to a submission system
            def handle_submission(model_name, model_path, model_type, precision, params, license):
                return f"Model {model_name} successfully submitted for evaluation. It will appear in the leaderboard once processing is complete."
            
            submit_button.click(
                handle_submission,
                [model_name_textbox, model_path_textbox, model_type, precision, params, license],
                submission_result,
            )

    with gr.Row():
        with gr.Accordion("📙 Citation", open=False):
            citation_text = """
            @misc{mental-health-model-benchmark,
              author = {Syed Faizan},
              title = {Mental Health Model Benchmark},
              year = {2025},
              publisher = {GitHub},
              url = {https://github.com/SYEDFAIZAN1987/mindBERT}
            }
            """
            citation_button = gr.Textbox(
                value=citation_text,
                label="Citation",
                lines=10,
                elem_id="citation-button",
                show_copy_button=True,
            )

demo.launch()