import torch import time import numpy as np import pandas as pd import evaluate import gradio as gr import re import csv from transformers import AutoModelForSequenceClassification, AutoTokenizer from sklearn.metrics import accuracy_score from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns from dataclasses import dataclass from typing import List # Load Accuracy and F1-Score Metrics accuracy_metric = evaluate.load("accuracy") f1_metric = evaluate.load("f1") # Define Model Paths MODEL_PATHS = { "MindBERT": "DrSyedFaizan/mindBERT", "BERT-base": "bert-base-uncased", "RoBERTa": "roberta-base", "DistilBERT": "distilbert-base-uncased" } # Label Mapping LABEL_MAPPING = { 0: "Stress", 1: "Depression", 2: "Bipolar disorder", 3: "Personality disorder", 4: "Anxiety" } # Function to clean text using regular expressions def clean_text(text): text = text.lower() text = re.sub(r'http\S+', '', text) # Remove URLs text = re.sub(r'\s+', ' ', text) # Remove excessive whitespace text = re.sub(r'[^a-zA-Z0-9 ]', '', text) # Remove special characters return text.strip() # Load and preprocess Reddit Mental Health Dataset def load_reddit_data(file_path, sample_size=100): df = pd.read_csv(file_path, sep=",", encoding="utf-8", quotechar='"', on_bad_lines="skip", engine="python") df.columns = df.columns.str.strip() # Remove extra spaces from column names print("Columns in dataset:", df.columns) # Debugging check if "text" not in df.columns or "target" not in df.columns: raise ValueError("Dataset does not contain required 'text' and 'target' columns.") df = df.dropna(subset=["text", "target"]) # Ensure required columns exist df["text"] = df["text"].apply(clean_text) # Clean text column df_sample = df.sample(n=sample_size, random_state=42) # Sample a subset test_texts = df_sample["text"].tolist() test_labels = df_sample["target"].tolist() return test_texts, test_labels # Function to evaluate models def evaluate_models(dataset_path): test_texts, test_labels = load_reddit_data(dataset_path) results = [] model_metadata = { "MindBERT": {"model_type": "BERT", "precision": "float16", "params": 0.11, "license": "MIT"}, "BERT-base": {"model_type": "BERT", "precision": "float16", "params": 0.11, "license": "Apache-2.0"}, "RoBERTa": {"model_type": "RoBERTa", "precision": "float16", "params": 0.125, "license": "MIT"}, "DistilBERT": {"model_type": "DistilBERT", "precision": "float16", "params": 0.067, "license": "Apache-2.0"} } for model_name, model_path in MODEL_PATHS.items(): print(f"Evaluating {model_name}...") tokenizer = AutoTokenizer.from_pretrained(model_path) model = AutoModelForSequenceClassification.from_pretrained(model_path) model.eval() inputs = tokenizer(test_texts, padding=True, truncation=True, return_tensors="pt") start_time = time.time() with torch.no_grad(): outputs = model(**inputs) logits = outputs.logits predictions = torch.argmax(logits, dim=1).numpy() end_time = time.time() accuracy = accuracy_score(test_labels, predictions) f1_score = f1_metric.compute(predictions=predictions, references=test_labels, average="macro")["f1"] inference_time = round(end_time - start_time, 4) result = { "model": model_name, "model_type": model_metadata[model_name]["model_type"], "precision": model_metadata[model_name]["precision"], "params": model_metadata[model_name]["params"], "accuracy": round(accuracy, 4), "f1_score": round(f1_score, 4), "inference_time": inference_time, "license": model_metadata[model_name]["license"] } results.append(result) return pd.DataFrame(results) # Load and evaluate DATASET_PATH = "https://huggingface.co/spaces/DrSyedFaizan/mindBERTevaluation/blob/main/rmhd.csv" df_results = evaluate_models(DATASET_PATH) # Initialize leaderboard with custom columns def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") columns = fields(ModelEvalColumn) return Leaderboard( value=dataframe, datatype=[c.type for c in columns], select_columns=SelectColumns( default_selection=[c.name for c in columns if c.displayed_by_default], cant_deselect=[c.name for c in columns if c.never_hidden], label="Select Columns to Display:", ), search_columns=["model", "license"], hide_columns=[c.name for c in columns if c.hidden], filter_columns=[ ColumnFilter("model_type", type="checkboxgroup", label="Model types"), ColumnFilter("precision", type="checkboxgroup", label="Precision"), ColumnFilter( "params", type="slider", min=0.01, max=0.5, label="Select the number of parameters (B)", ), ], interactive=False, ) # Custom CSS similar to the original custom_css = """ .markdown-text { padding: 0 20px; } .tab-buttons button.selected { background-color: #FF9C00 !important; color: white !important; } """ # Create Gradio Interface demo = gr.Blocks(css=custom_css) with demo: gr.HTML("

Mental Health Model Evaluation Benchmark

") gr.Markdown("This benchmark evaluates various transformer models on mental health classification tasks.", elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 Model Benchmark", elem_id="model-benchmark-tab", id=0): # Get evaluation results df_results = evaluate_models() leaderboard = init_leaderboard(df_results) with gr.TabItem("📝 About", elem_id="about-tab", id=1): gr.Markdown(""" ## About This Benchmark This leaderboard compares various transformer models on mental health text classification tasks. The benchmark uses a test set from Reddit Mental Health datasets with examples covering anxiety, depression, bipolar disorder, suicidal ideation, stress, and normal emotional states. Models are evaluated on: - Accuracy - F1-Score (Macro) - Inference Time ### Model Types - BERT-based models - RoBERTa models - DistilBERT models - Specialized mental health models (MindBERT) """, elem_classes="markdown-text") with gr.TabItem("🚀 Submit Model", elem_id="submit-tab", id=2): gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text") with gr.Row(): with gr.Column(): model_name_textbox = gr.Textbox(label="Model name") model_path_textbox = gr.Textbox(label="Model path (HF repo ID)") model_type = gr.Dropdown( choices=["BERT", "RoBERTa", "DistilBERT", "GPT", "T5", "Other"], label="Model type", multiselect=False, value=None, interactive=True, ) with gr.Column(): precision = gr.Dropdown( choices=["float16", "float32", "int8", "int4"], label="Precision", multiselect=False, value="float16", interactive=True, ) params = gr.Number(label="Parameters (billions)", value=0.11) license = gr.Textbox(label="License", value="Apache-2.0") submit_button = gr.Button("Submit Model for Evaluation") submission_result = gr.Markdown() # This would typically connect to a submission system def handle_submission(model_name, model_path, model_type, precision, params, license): return f"Model {model_name} successfully submitted for evaluation. It will appear in the leaderboard once processing is complete." submit_button.click( handle_submission, [model_name_textbox, model_path_textbox, model_type, precision, params, license], submission_result, ) with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_text = """ @misc{mental-health-model-benchmark, author = {Syed Faizan}, title = {Mental Health Model Benchmark}, year = {2025}, publisher = {GitHub}, url = {https://github.com/SYEDFAIZAN1987/mindBERT} } """ citation_button = gr.Textbox( value=citation_text, label="Citation", lines=10, elem_id="citation-button", show_copy_button=True, ) demo.launch()