- {`from transformers import AutoConfig, AutoModel, AutoTokenizer
-
-config = AutoConfig.from_pretrained("your-username/your-model", revision="main")
-model = AutoModel.from_pretrained("your-username/your-model", revision="main")
-tokenizer = AutoTokenizer.from_pretrained("your-username/your-model", revision="main")`}
-
- {citation}
- {error}
" + + +def styled_warning(warn): + return f"{warn}
" + + +def styled_message(message): + return f"{message}
" + + +def has_no_nan_values(df, columns): + return df[columns].notna().all(axis=1) + + +def has_nan_values(df, columns): + return df[columns].isna().any(axis=1) diff --git a/src/display/utils.py b/src/display/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bea63472f7759dd0cf9cc8599f0dd12ad8e1dd33 --- /dev/null +++ b/src/display/utils.py @@ -0,0 +1,268 @@ +from dataclasses import dataclass, make_dataclass +from datasets import load_dataset +from enum import Enum +import json +import logging +from datetime import datetime +import pandas as pd + +from src.envs import MAINTAINERS_HIGHLIGHT_REPO + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + +dataset = load_dataset(MAINTAINERS_HIGHLIGHT_REPO) +curated_authors = dataset["train"][0]["CURATED_SET"] + +# Convert ISO 8601 dates to datetime objects for comparison +def parse_iso8601_datetime(date_str): + if date_str.endswith('Z'): + date_str = date_str[:-1] + '+00:00' + return datetime.fromisoformat(date_str) + +def parse_datetime(datetime_str): + formats = [ + "%Y-%m-%dT%H-%M-%S.%f", # Format with dashes + "%Y-%m-%dT%H:%M:%S.%f", # Standard format with colons + "%Y-%m-%dT%H %M %S.%f", # Spaces as separator + ] + + for fmt in formats: + try: + return datetime.strptime(datetime_str, fmt) + except ValueError: + continue + # in rare cases set unix start time for files with incorrect time (legacy files) + logging.error(f"No valid date format found for: {datetime_str}") + return datetime(1970, 1, 1) + + +def load_json_data(file_path): + """Safely load JSON data from a file.""" + try: + with open(file_path, "r") as file: + return json.load(file) + except json.JSONDecodeError: + print(f"Error reading JSON from {file_path}") + return None # Or raise an exception + + +def fields(raw_class): + return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] + + +@dataclass +class Task: + benchmark: str + metric: str + col_name: str + + +class Tasks(Enum): + ifeval = Task("leaderboard_ifeval", "strict_acc,none", "IFEval") + ifeval_raw = Task("leaderboard_ifeval", "strict_acc,none", "IFEval Raw") + + bbh = Task("leaderboard_bbh", "acc_norm,none", "BBH") + bbh_raw = Task("leaderboard_bbh", "acc_norm,none", "BBH Raw") + + math = Task("leaderboard_math_hard", "exact_match,none", "MATH Lvl 5") + math_raw = Task("leaderboard_math_hard", "exact_match,none", "MATH Lvl 5 Raw") + + gpqa = Task("leaderboard_gpqa", "acc_norm,none", "GPQA") + gpqa_raw = Task("leaderboard_gpqa", "acc_norm,none", "GPQA Raw") + + musr = Task("leaderboard_musr", "acc_norm,none", "MUSR") + musr_raw = Task("leaderboard_musr", "acc_norm,none", "MUSR Raw") + + mmlu_pro = Task("leaderboard_mmlu_pro", "acc,none", "MMLU-PRO") + mmlu_pro_raw = Task("leaderboard_mmlu_pro", "acc,none", "MMLU-PRO Raw") + + +# These classes are for user facing column names, +# to avoid having to change them all around the code +# when a modif is needed +@dataclass(frozen=True) +class ColumnContent: + name: str + type: str + displayed_by_default: bool + hidden: bool = False + never_hidden: bool = False + dummy: bool = False + + +auto_eval_column_dict = [] +# Init +auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)]) +auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)]) +# Scores +auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]) +for task in Tasks: + displayed_by_default = not task.name.endswith("_raw") + auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", displayed_by_default=displayed_by_default)]) +# Model information +auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)]) +auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]) +auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]) +auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)]) +auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Not_Merged", "bool", False)]) +auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)]) +auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]) +auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]) +auto_eval_column_dict.append( + ["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)] +) +auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]) +auto_eval_column_dict.append(["not_flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)]) +auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)]) + +auto_eval_column_dict.append(["submission_date", ColumnContent, ColumnContent("Submission Date", "bool", False, hidden=False)]) +auto_eval_column_dict.append(["upload_to_hub", ColumnContent, ColumnContent("Upload To Hub Date", "bool", False, hidden=False)]) + +auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Chat Template", "bool", False)]) +auto_eval_column_dict.append(["maintainers_highlight", ColumnContent, ColumnContent("Maintainer's Highlight", "bool", False, hidden=True)]) + +# fullname structure:Baseline
", +# AutoEvalColumn.revision.name: "N/A", +# AutoEvalColumn.precision.name: None, +# AutoEvalColumn.merged.name: False, +# AutoEvalColumn.average.name: 31.0, +# AutoEvalColumn.arc.name: 25.0, +# AutoEvalColumn.hellaswag.name: 25.0, +# AutoEvalColumn.mmlu.name: 25.0, +# AutoEvalColumn.truthfulqa.name: 25.0, +# AutoEvalColumn.winogrande.name: 50.0, +# AutoEvalColumn.gsm8k.name: 0.21, +# AutoEvalColumn.fullname.name: "baseline", +# AutoEvalColumn.model_type.name: "", +# AutoEvalColumn.not_flagged.name: False, +# } + +# Average ⬆️ human baseline is 0.897 (source: averaging human baselines below) +# ARC human baseline is 0.80 (source: https://lab42.global/arc/) +# HellaSwag human baseline is 0.95 (source: https://deepgram.com/learn/hellaswag-llm-benchmark-guide) +# MMLU human baseline is 0.898 (source: https://openreview.net/forum?id=d7KBjmI3GmQ) +# TruthfulQA human baseline is 0.94(source: https://arxiv.org/pdf/2109.07958.pdf) +# Winogrande: https://leaderboard.allenai.org/winogrande/submissions/public +# GSM8K: paper +# Define the human baselines +# human_baseline_row = { +# AutoEvalColumn.model.name: "Human performance
", +# AutoEvalColumn.revision.name: "N/A", +# AutoEvalColumn.precision.name: None, +# AutoEvalColumn.average.name: 92.75, +# AutoEvalColumn.merged.name: False, +# AutoEvalColumn.arc.name: 80.0, +# AutoEvalColumn.hellaswag.name: 95.0, +# AutoEvalColumn.mmlu.name: 89.8, +# AutoEvalColumn.truthfulqa.name: 94.0, +# AutoEvalColumn.winogrande.name: 94.0, +# AutoEvalColumn.gsm8k.name: 100, +# AutoEvalColumn.fullname.name: "human_baseline", +# AutoEvalColumn.model_type.name: "", +# AutoEvalColumn.not_flagged.name: False, +# } + + +@dataclass +class ModelDetails: + name: str + symbol: str = "" # emoji, only for the model type + + +class ModelType(Enum): + PT = ModelDetails(name="🟢 pretrained", symbol="🟢") + CPT = ModelDetails(name="🟩 continuously pretrained", symbol="🟩") + FT = ModelDetails(name="🔶 fine-tuned on domain-specific datasets", symbol="🔶") + chat = ModelDetails(name="💬 chat models (RLHF, DPO, IFT, ...)", symbol="💬") + merges = ModelDetails(name="🤝 base merges and moerges", symbol="🤝") + Unknown = ModelDetails(name="❓ other", symbol="❓") + + def to_str(self, separator=" "): + return f"{self.value.symbol}{separator}{self.value.name}" + + @staticmethod + def from_str(m_type): + if any([k for k in m_type if k in ["fine-tuned","🔶", "finetuned"]]): + return ModelType.FT + if "continuously pretrained" in m_type or "🟩" in m_type: + return ModelType.CPT + if "pretrained" in m_type or "🟢" in m_type: + return ModelType.PT + if any([k in m_type for k in ["instruction-tuned", "RL-tuned", "chat", "🟦", "⭕", "💬"]]): + return ModelType.chat + if "merge" in m_type or "🤝" in m_type: + return ModelType.merges + return ModelType.Unknown + + +class WeightType(Enum): + Adapter = ModelDetails("Adapter") + Original = ModelDetails("Original") + Delta = ModelDetails("Delta") + + +class Precision(Enum): + float16 = ModelDetails("float16") + bfloat16 = ModelDetails("bfloat16") + qt_8bit = ModelDetails("8bit") + qt_4bit = ModelDetails("4bit") + qt_GPTQ = ModelDetails("GPTQ") + Unknown = ModelDetails("?") + + @staticmethod + def from_str(precision): + if precision in ["torch.float16", "float16"]: + return Precision.float16 + if precision in ["torch.bfloat16", "bfloat16"]: + return Precision.bfloat16 + if precision in ["8bit"]: + return Precision.qt_8bit + if precision in ["4bit"]: + return Precision.qt_4bit + if precision in ["GPTQ", "None"]: + return Precision.qt_GPTQ + return Precision.Unknown + + +# Column selection +COLS = [c.name for c in fields(AutoEvalColumn)] +TYPES = [c.type for c in fields(AutoEvalColumn)] + +EVAL_COLS = [c.name for c in fields(EvalQueueColumn)] +EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)] + +BENCHMARK_COLS = [t.value.col_name for t in Tasks] + +NUMERIC_INTERVALS = { + "?": pd.Interval(-1, 0, closed="right"), + "~1.5": pd.Interval(0, 2, closed="right"), + "~3": pd.Interval(2, 4, closed="right"), + "~7": pd.Interval(4, 9, closed="right"), + "~13": pd.Interval(9, 20, closed="right"), + "~35": pd.Interval(20, 45, closed="right"), + "~60": pd.Interval(45, 70, closed="right"), + "70+": pd.Interval(70, 10000, closed="right"), +} diff --git a/src/envs.py b/src/envs.py new file mode 100644 index 0000000000000000000000000000000000000000..79cafa6698489dd430352932f3923814850e2a7d --- /dev/null +++ b/src/envs.py @@ -0,0 +1,33 @@ +import os +from huggingface_hub import HfApi + +# clone / pull the lmeh eval data +HF_TOKEN = os.environ.get("HF_TOKEN", None) + +REPO_ID = "open-llm-leaderboard/open_llm_leaderboard" +QUEUE_REPO = "open-llm-leaderboard/requests" +AGGREGATED_REPO = "open-llm-leaderboard/contents" +VOTES_REPO = "open-llm-leaderboard/votes" +MAINTAINERS_HIGHLIGHT_REPO = "open-llm-leaderboard/maintainers-highlight" + +HF_HOME = os.getenv("HF_HOME", ".") + +# Check HF_HOME write access +print(f"Initial HF_HOME set to: {HF_HOME}") + +if not os.access(HF_HOME, os.W_OK): + print(f"No write access to HF_HOME: {HF_HOME}. Resetting to current directory.") + HF_HOME = "." + os.environ["HF_HOME"] = HF_HOME +else: + print("Write access confirmed for HF_HOME") + +VOTES_PATH = os.path.join(HF_HOME, "model-votes") +EVAL_REQUESTS_PATH = os.path.join(HF_HOME, "eval-queue") + +# Rate limit variables +RATE_LIMIT_PERIOD = 7 +RATE_LIMIT_QUOTA = 5 +HAS_HIGHER_RATE_LIMIT = [] + +API = HfApi(token=HF_TOKEN) diff --git a/src/leaderboard/filter_models.py b/src/leaderboard/filter_models.py new file mode 100644 index 0000000000000000000000000000000000000000..2d3d396d324d9aa86de3a35befffc27e6170f8e0 --- /dev/null +++ b/src/leaderboard/filter_models.py @@ -0,0 +1,75 @@ +from src.display.formatting import model_hyperlink +from src.display.utils import AutoEvalColumn + + +# Models which have been flagged by users as being problematic for a reason or another +# (Model name to forum discussion link) +# None for the v2 so far! +FLAGGED_MODELS = {} + +# Models which have been requested by orgs to not be submitted on the leaderboard +DO_NOT_SUBMIT_MODELS = [ + "Voicelab/trurl-2-13b", # trained on MMLU + "TigerResearch/tigerbot-70b-chat", # per authors request + "TigerResearch/tigerbot-70b-chat-v2", # per authors request + "TigerResearch/tigerbot-70b-chat-v4-4k", # per authors request +] + + +def flag_models(leaderboard_data: list[dict]): + """Flags models based on external criteria or flagged status.""" + for model_data in leaderboard_data: + # Skip flagging if maintainers_highlight is True + if model_data.get(AutoEvalColumn.maintainers_highlight.name, False): + model_data[AutoEvalColumn.not_flagged.name] = True + continue + + # If a model is not flagged, use its "fullname" as a key + if model_data[AutoEvalColumn.not_flagged.name]: + flag_key = model_data[AutoEvalColumn.fullname.name] + else: + flag_key = None + + # Reverse the logic: Check for non-flagged models instead + if flag_key in FLAGGED_MODELS: + issue_num = FLAGGED_MODELS[flag_key].split("/")[-1] + issue_link = model_hyperlink( + FLAGGED_MODELS[flag_key], + f"See discussion #{issue_num}", + ) + model_data[AutoEvalColumn.model.name] = ( + f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}" + ) + model_data[AutoEvalColumn.not_flagged.name] = False + else: + model_data[AutoEvalColumn.not_flagged.name] = True + + +def remove_forbidden_models(leaderboard_data: list[dict]): + """Removes models from the leaderboard based on the DO_NOT_SUBMIT list.""" + indices_to_remove = [] + for ix, model in enumerate(leaderboard_data): + if model[AutoEvalColumn.fullname.name] in DO_NOT_SUBMIT_MODELS: + indices_to_remove.append(ix) + + # Remove the models from the list + for ix in reversed(indices_to_remove): + leaderboard_data.pop(ix) + return leaderboard_data + +""" +def remove_forbidden_models(leaderboard_data): + #Removes models from the leaderboard based on the DO_NOT_SUBMIT list. + indices_to_remove = [] + for ix, row in leaderboard_data.iterrows(): + if row[AutoEvalColumn.fullname.name] in DO_NOT_SUBMIT_MODELS: + indices_to_remove.append(ix) + + # Remove the models from the list + return leaderboard_data.drop(indices_to_remove) +""" + + +def filter_models_flags(leaderboard_data: list[dict]): + leaderboard_data = remove_forbidden_models(leaderboard_data) + flag_models(leaderboard_data) diff --git a/src/populate.py b/src/populate.py new file mode 100644 index 0000000000000000000000000000000000000000..41e5f8ce6ff2782071696d1b53648008584a5a65 --- /dev/null +++ b/src/populate.py @@ -0,0 +1,54 @@ +import pathlib +import pandas as pd +from datasets import Dataset +from src.display.formatting import has_no_nan_values, make_clickable_model +from src.display.utils import AutoEvalColumn, EvalQueueColumn +from src.leaderboard.filter_models import filter_models_flags +from src.display.utils import load_json_data + + +def _process_model_data(entry, model_name_key="model", revision_key="revision"): + """Enrich model data with clickable links and revisions.""" + entry[EvalQueueColumn.model_name.name] = entry.get(model_name_key, "") + entry[EvalQueueColumn.model_link.name] = make_clickable_model(entry.get(model_name_key, "")) + entry[EvalQueueColumn.revision.name] = entry.get(revision_key, "main") + return entry + + +def get_evaluation_queue_df(save_path, cols): + """Generate dataframes for pending, running, and finished evaluation entries.""" + save_path = pathlib.Path(save_path) + all_evals = [] + + for path in save_path.rglob("*.json"): + data = load_json_data(path) + if data: + all_evals.append(_process_model_data(data)) + + # Organizing data by status + status_map = { + "PENDING": ["PENDING", "RERUN"], + "RUNNING": ["RUNNING"], + "FINISHED": ["FINISHED", "PENDING_NEW_EVAL"], + } + status_dfs = {status: [] for status in status_map} + for eval_data in all_evals: + for status, extra_statuses in status_map.items(): + if eval_data["status"] in extra_statuses: + status_dfs[status].append(eval_data) + + return tuple(pd.DataFrame(status_dfs[status], columns=cols) for status in ["FINISHED", "RUNNING", "PENDING"]) + + +def get_leaderboard_df(leaderboard_dataset: Dataset, cols: list, benchmark_cols: list): + """Retrieve and process leaderboard data.""" + all_data_json = leaderboard_dataset.to_dict() + num_items = leaderboard_dataset.num_rows + all_data_json_list = [{k: all_data_json[k][ix] for k in all_data_json.keys()} for ix in range(num_items)] + filter_models_flags(all_data_json_list) + + df = pd.DataFrame.from_records(all_data_json_list) + df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False) + df = df[cols].round(decimals=2) + df = df[has_no_nan_values(df, benchmark_cols)] + return df \ No newline at end of file diff --git a/src/submission/check_validity.py b/src/submission/check_validity.py new file mode 100644 index 0000000000000000000000000000000000000000..0eeb01891dd19042136430b89897a24a02bf7489 --- /dev/null +++ b/src/submission/check_validity.py @@ -0,0 +1,209 @@ +import json +import os +import re +import logging +from collections import defaultdict +from datetime import datetime, timedelta, timezone + +import huggingface_hub +from huggingface_hub import ModelCard +from huggingface_hub.hf_api import ModelInfo, get_safetensors_metadata, parse_safetensors_file_metadata +from transformers import AutoConfig, AutoTokenizer + +from src.display.utils import parse_iso8601_datetime, curated_authors +from src.envs import HAS_HIGHER_RATE_LIMIT + + +# ht to @Wauplin, thank you for the snippet! +# See https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard/discussions/317 +def check_model_card(repo_id: str) -> tuple[bool, str]: + # Returns operation status, and error message + try: + card = ModelCard.load(repo_id) + except huggingface_hub.utils.EntryNotFoundError: + return False, "Please add a model card to your model to explain how you trained/fine-tuned it.", None + + # Enforce license metadata + if card.data.license is None and not ("license_name" in card.data and "license_link" in card.data): + return ( + False, + ( + "License not found. Please add a license to your model card using the `license` metadata or a" + " `license_name`/`license_link` pair." + ), + None, + ) + + # Enforce card content + if len(card.text) < 200: + return False, "Please add a description to your model card, it is too short.", None + + return True, "", card + + +def is_model_on_hub( + model_name: str, revision: str, token: str | None = None, trust_remote_code: bool = False, test_tokenizer: bool = False, +) -> tuple[bool, str, AutoConfig]: + try: + config = AutoConfig.from_pretrained( + model_name, revision=revision, trust_remote_code=trust_remote_code, token=token, force_download=True) + if test_tokenizer: + try: + AutoTokenizer.from_pretrained( + model_name, revision=revision, trust_remote_code=trust_remote_code, token=token, + ) + except ValueError as e: + return (False, f"uses a tokenizer which is not in a transformers release: {e}", None) + except Exception: + return ( + False, + "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", + None, + ) + except Exception: + return ( + False, + "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", + None, + ) + return True, None, config + + except ValueError: + return ( + False, + "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.", + None, + ) + + except Exception as e: + if "You are trying to access a gated repo." in str(e): + return True, "uses a gated model.", None + return False, f"was not found or misconfigured on the hub! Error raised was {e.args[0]}", None + + +def get_model_size(model_info: ModelInfo, precision: str, base_model: str| None) -> tuple[float | None, str]: + size_pattern = re.compile(r"(\d+\.)?\d+(b|m)") + safetensors = None + adapter_safetensors = None + # hack way to check that model is adapter + is_adapter = "adapter_config.json" in (s.rfilename for s in model_info.siblings) + + try: + if is_adapter: + if not base_model: + return None, "Adapter model submission detected. Please ensure the base model information is provided." + + adapter_safetensors = parse_safetensors_file_metadata(model_info.id, "adapter_model.safetensors") + safetensors = get_safetensors_metadata(base_model) + else: + safetensors = get_safetensors_metadata(model_info.id) + except Exception as e: + logging.warning(f"Failed to get safetensors metadata for model {model_info.id}: {e!s}") + + if safetensors is not None: + model_size = sum(safetensors.parameter_count.values()) + if adapter_safetensors is not None: + model_size += sum(safetensors.parameter_count.values()) + model_size = round(model_size / 1e9, 3) + else: + try: + size_match = re.search(size_pattern, model_info.id.lower()) + if size_match: + model_size = size_match.group(0) + model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3) + else: + return None, "Unknown model size" + except AttributeError: + logging.warning(f"Unable to parse model size from ID: {model_info.id}") + return None, "Unknown model size" + + size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.id.lower()) else 1 + model_size = size_factor * model_size + + return model_size, "" + +def get_model_arch(model_info: ModelInfo): + return model_info.config.get("architectures", "Unknown") + + +def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota): + # No limit for curated authors + if org_or_user in curated_authors: + return True, "" + # Increase quota first if user has higher limits + if org_or_user in HAS_HIGHER_RATE_LIMIT: + rate_limit_quota *= 2 + + if org_or_user not in users_to_submission_dates: + return True, "" + + submission_dates = sorted(users_to_submission_dates[org_or_user]) + time_limit = datetime.now(timezone.utc) - timedelta(days=rate_limit_period) + + submissions_after_timelimit = [ + parse_iso8601_datetime(d) for d in submission_dates + if parse_iso8601_datetime(d) > time_limit + ] + + num_models_submitted_in_period = len(submissions_after_timelimit) + + # Use >= to correctly enforce the rate limit + if num_models_submitted_in_period >= rate_limit_quota: + error_msg = f"Organisation or user `{org_or_user}` already has {num_models_submitted_in_period} model requests submitted in the last {rate_limit_period} days.\n" + error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗" + return False, error_msg + + return True, "" + + +def already_submitted_models(requested_models_dir: str) -> set[str]: + depth = 1 + file_names = [] + users_to_submission_dates = defaultdict(list) + + for root, _, files in os.walk(requested_models_dir): + current_depth = root.count(os.sep) - requested_models_dir.count(os.sep) + if current_depth == depth: + for file in files: + if not file.endswith(".json"): + continue + with open(os.path.join(root, file), "r") as f: + info = json.load(f) + file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}") + + # Select organisation + if info["model"].count("/") == 0 or "submitted_time" not in info: + continue + organisation, _ = info["model"].split("/") + users_to_submission_dates[organisation].append(info["submitted_time"]) + + return set(file_names), users_to_submission_dates + + +def get_model_tags(model_card, model: str): + is_merge_from_metadata = False + is_moe_from_metadata = False + + tags = [] + if model_card is None: + return tags + if model_card.data.tags: + is_merge_from_metadata = any( + [tag in model_card.data.tags for tag in ["merge", "moerge", "mergekit", "lazymergekit"]] + ) + is_moe_from_metadata = any([tag in model_card.data.tags for tag in ["moe", "moerge"]]) + + is_merge_from_model_card = any( + keyword in model_card.text.lower() for keyword in ["merged model", "merge model", "moerge"] + ) + if is_merge_from_model_card or is_merge_from_metadata: + tags.append("merge") + is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in ["moe", "mixtral"]) + # Hardcoding because of gating problem + if "Qwen/Qwen1.5-32B" in model: + is_moe_from_model_card = False + is_moe_from_name = "moe" in model.lower().replace("/", "-").replace("_", "-").split("-") + if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata: + tags.append("moe") + + return tags diff --git a/src/submission/submit.py b/src/submission/submit.py new file mode 100644 index 0000000000000000000000000000000000000000..8fad0900d65256b09295e5a6d95d543f129af5db --- /dev/null +++ b/src/submission/submit.py @@ -0,0 +1,220 @@ +import json +import os +import gradio as gr +from datetime import datetime, timezone + +from dataclasses import dataclass +from transformers import AutoConfig + +from src.display.formatting import styled_error, styled_message, styled_warning +from src.envs import ( + API, + EVAL_REQUESTS_PATH, + HF_TOKEN, + QUEUE_REPO, + RATE_LIMIT_PERIOD, + RATE_LIMIT_QUOTA, + VOTES_REPO, + VOTES_PATH, +) +from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS +from src.submission.check_validity import ( + already_submitted_models, + check_model_card, + get_model_size, + is_model_on_hub, + user_submission_permission, +) + +from src.voting.vote_system import VoteManager + +REQUESTED_MODELS = None +USERS_TO_SUBMISSION_DATES = None + +vote_manager = VoteManager(VOTES_PATH, EVAL_REQUESTS_PATH, VOTES_REPO) + +@dataclass +class ModelSizeChecker: + model: str + precision: str + model_size_in_b: float + + def get_precision_factor(self): + if self.precision in ["float16", "bfloat16"]: + return 1 + elif self.precision == "8bit": + return 2 + elif self.precision == "4bit": + return 4 + elif self.precision == "GPTQ": + config = AutoConfig.from_pretrained(self.model) + num_bits = int(config.quantization_config["bits"]) + bits_to_precision_factor = {2: 8, 3: 6, 4: 4, 8: 2} + return bits_to_precision_factor.get(num_bits, 1) + else: + raise Exception(f"Unknown precision {self.precision}.") + + def can_evaluate(self): + precision_factor = self.get_precision_factor() + return self.model_size_in_b <= 140 * precision_factor + +def add_new_eval( + model: str, + base_model: str, + revision: str, + precision: str, + weight_type: str, + model_type: str, + use_chat_template: bool, + profile: gr.OAuthProfile | None, + requested_models: set[str] = None, + users_to_submission_dates: dict[str, list[str]] = None, +): + # Login required + if profile is None: + return styled_error("Hub Login Required") + + # Name of the actual user who sent the request + username = profile.username + + # Initialize the requested_models and users_to_submission_dates variables + # If the caller did not provide these values, fetch them from the EVAL_REQUESTS_PATH + if requested_models is None or users_to_submission_dates is None: + requested_models, users_to_submission_dates = already_submitted_models(EVAL_REQUESTS_PATH) + + org_or_user = "" + model_path = model + if "/" in model: + org_or_user = model.split("/")[0] + model_path = model.split("/")[1] + + precision = precision.split(" ")[0] + current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + if model_type is None or model_type == "": + return styled_error("Please select a model type.") + + # Is the user rate limited? + if org_or_user != "": + user_can_submit, error_msg = user_submission_permission( + org_or_user, users_to_submission_dates, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA + ) + if not user_can_submit: + return styled_error(error_msg) + + # Did the model authors forbid its submission to the leaderboard? + if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS: + return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.") + + # Does the model actually exist? + if revision == "": + revision = "main" + try: + model_info = API.model_info(repo_id=model, revision=revision) + except Exception as e: + return styled_error("Could not get your model information. Please fill it up properly.") + + model_key = f"{model}_{model_info.sha}_{precision}" + if model_key in requested_models: + return styled_error(f"The model '{model}' with revision '{model_info.sha}' and precision '{precision}' has already been submitted.") + + # Check model size early + model_size, error_text = get_model_size(model_info=model_info, precision=precision, base_model=base_model) + if model_size is None: + return styled_error(error_text) + + # First check: Absolute size limit for float16 and bfloat16 + if precision in ["float16", "bfloat16"] and model_size > 100: + return styled_error(f"Sadly, models larger than 100B parameters cannot be submitted in {precision} precision at this time. " + f"Your model size: {model_size:.2f}B parameters.") + + # Second check: Precision-adjusted size limit for 8bit, 4bit, and GPTQ + if precision in ["8bit", "4bit", "GPTQ"]: + size_checker = ModelSizeChecker(model=model, precision=precision, model_size_in_b=model_size) + + if not size_checker.can_evaluate(): + precision_factor = size_checker.get_precision_factor() + max_size = 140 * precision_factor + return styled_error(f"Sadly, models this big ({model_size:.2f}B parameters) cannot be evaluated automatically " + f"at the moment on our cluster. The maximum size for {precision} precision is {max_size:.2f}B parameters.") + + architecture = "?" + # Is the model on the hub? + if weight_type in ["Delta", "Adapter"]: + base_model_on_hub, error, _ = is_model_on_hub( + model_name=base_model, revision="main", token=HF_TOKEN, test_tokenizer=True + ) + if not base_model_on_hub: + return styled_error(f'Base model "{base_model}" {error}') + if not weight_type == "Adapter": + model_on_hub, error, model_config = is_model_on_hub(model_name=model, revision=model_info.sha, test_tokenizer=True) + if not model_on_hub or model_config is None: + return styled_error(f'Model "{model}" {error}') + if model_config is not None: + architectures = getattr(model_config, "architectures", None) + if architectures: + architecture = ";".join(architectures) + # Were the model card and license filled? + try: + model_info.cardData["license"] + except Exception: + return styled_error("Please select a license for your model") + + modelcard_OK, error_msg, model_card = check_model_card(model) + if not modelcard_OK: + return styled_error(error_msg) + + # Seems good, creating the eval + print("Adding new eval") + eval_entry = { + "model": model, + "base_model": base_model, + "revision": model_info.sha, # force to use the exact model commit + "precision": precision, + "params": model_size, + "architectures": architecture, + "weight_type": weight_type, + "status": "PENDING", + "submitted_time": current_time, + "model_type": model_type, + "job_id": -1, + "job_start_time": None, + "use_chat_template": use_chat_template, + "sender": username + } + + print("Creating eval file") + OUT_DIR = f"{EVAL_REQUESTS_PATH}/{org_or_user}" + os.makedirs(OUT_DIR, exist_ok=True) + out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json" + + with open(out_path, "w") as f: + f.write(json.dumps(eval_entry)) + + print("Uploading eval file") + print(eval_entry) + API.upload_file( + path_or_fileobj=out_path, + path_in_repo=out_path.split("eval-queue/")[1], + repo_id=QUEUE_REPO, + repo_type="dataset", + commit_message=f"Add {model} to eval queue", + ) + + # Remove the local file + os.remove(out_path) + + # Always add a vote for the submitted model + vote_manager.add_vote( + selected_model=model, + pending_models_df=None, + profile=profile + ) + print(f"Automatically added a vote for {model} submitted by {username}") + + # Upload votes to the repository + vote_manager.upload_votes() + + return styled_message( + "Your request and vote has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list." + ) \ No newline at end of file diff --git a/src/tools/create_request_file.py b/src/tools/create_request_file.py new file mode 100644 index 0000000000000000000000000000000000000000..c1aea64f3fdd25fa5ea087cf02709359c5b5892d --- /dev/null +++ b/src/tools/create_request_file.py @@ -0,0 +1,92 @@ +import json +import os +import pprint +from datetime import datetime, timezone + +import click +from colorama import Fore +from huggingface_hub import HfApi, snapshot_download + +from src.display.utils import ModelType, WeightType +from src.submission.check_validity import get_model_size + +EVAL_REQUESTS_PATH = "eval-queue" +QUEUE_REPO = "open-llm-leaderboard/requests" + +precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ") +model_types = [e.name for e in ModelType] +weight_types = [e.name for e in WeightType] + + +def main(): + api = HfApi() + current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset") + + model_name = click.prompt("Enter model name") + revision = click.prompt("Enter revision", default="main") + precision = click.prompt("Enter precision", default="float16", type=click.Choice(precisions)) + model_type = click.prompt("Enter model type", type=click.Choice(model_types)) + weight_type = click.prompt("Enter weight type", default="Original", type=click.Choice(weight_types)) + base_model = click.prompt("Enter base model", default="") + status = click.prompt("Enter status", default="FINISHED") + + try: + model_info = api.model_info(repo_id=model_name, revision=revision) + except Exception as e: + print(f"{Fore.RED}Could not find model info for {model_name} on the Hub\n{e}{Fore.RESET}") + return 1 + + model_size = get_model_size(model_info=model_info, precision=precision) + + try: + license = model_info.cardData["license"] + except Exception: + license = "?" + + eval_entry = { + "model": model_name, + "base_model": base_model, + "revision": model_info.sha, # force to use the exact model commit + "private": False, + "precision": precision, + "weight_type": weight_type, + "status": status, + "submitted_time": current_time, + "model_type": model_type, + "likes": model_info.likes, + "params": model_size, + "license": license, + } + + user_name = "" + model_path = model_name + if "/" in model_name: + user_name = model_name.split("/")[0] + model_path = model_name.split("/")[1] + + pprint.pprint(eval_entry) + + if click.confirm("Do you want to continue? This request file will be pushed to the hub"): + click.echo("continuing...") + + out_dir = f"{EVAL_REQUESTS_PATH}/{user_name}" + os.makedirs(out_dir, exist_ok=True) + out_path = f"{out_dir}/{model_path}_eval_request_{False}_{precision}_{weight_type}.json" + + with open(out_path, "w") as f: + f.write(json.dumps(eval_entry)) + + api.upload_file( + path_or_fileobj=out_path, + path_in_repo=out_path.split(f"{EVAL_REQUESTS_PATH}/")[1], + repo_id=QUEUE_REPO, + repo_type="dataset", + commit_message=f"Add {model_name} to eval queue", + ) + else: + click.echo("aborting...") + + +if __name__ == "__main__": + main() diff --git a/src/tools/model_backlinks.py b/src/tools/model_backlinks.py new file mode 100644 index 0000000000000000000000000000000000000000..c1afa53b0549a15d7f384621e3c0655aad67f67e --- /dev/null +++ b/src/tools/model_backlinks.py @@ -0,0 +1,1309 @@ +models = [ + "uni-tianyan/Uni-TianYan", + "fangloveskari/ORCA_LLaMA_70B_QLoRA", + "garage-bAInd/Platypus2-70B-instruct", + "upstage/Llama-2-70b-instruct-v2", + "fangloveskari/Platypus_QLoRA_LLaMA_70b", + "yeontaek/llama-2-70B-ensemble-v5", + "TheBloke/Genz-70b-GPTQ", + "TheBloke/Platypus2-70B-Instruct-GPTQ", + "psmathur/model_007", + "yeontaek/llama-2-70B-ensemble-v4", + "psmathur/orca_mini_v3_70b", + "ehartford/Samantha-1.11-70b", + "MayaPH/GodziLLa2-70B", + "psmathur/model_007_v2", + "chargoddard/MelangeA-70b", + "ehartford/Samantha-1.1-70b", + "psmathur/model_009", + "upstage/Llama-2-70b-instruct", + "yeontaek/llama-2-70B-ensemble-v7", + "yeontaek/llama-2-70B-ensemble-v6", + "chargoddard/MelangeB-70b", + "yeontaek/llama-2-70B-ensemble-v3", + "chargoddard/MelangeC-70b", + "garage-bAInd/Camel-Platypus2-70B", + "yeontaek/llama-2-70B-ensemble-v2", + "garage-bAInd/Camel-Platypus2-70B", + "migtissera/Synthia-70B-v1.2", + "v2ray/LLaMA-2-Wizard-70B-QLoRA", + "quantumaikr/llama-2-70b-fb16-orca-chat-10k", + "v2ray/LLaMA-2-Wizard-70B-QLoRA", + "stabilityai/StableBeluga2", + "quantumaikr/llama-2-70b-fb16-guanaco-1k", + "garage-bAInd/Camel-Platypus2-70B", + "migtissera/Synthia-70B-v1.1", + "migtissera/Synthia-70B", + "psmathur/model_101", + "augtoma/qCammel70", + "augtoma/qCammel-70", + "augtoma/qCammel-70v1", + "augtoma/qCammel-70x", + "augtoma/qCammel-70-x", + "jondurbin/airoboros-l2-70b-gpt4-1.4.1", + "dfurman/llama-2-70b-dolphin-peft", + "jondurbin/airoboros-l2-70b-2.1", + "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16", + "quantumaikr/QuantumLM-llama2-70B-Korean-LoRA", + "quantumaikr/quantumairk-llama-2-70B-instruct", + "psmathur/model_420", + "psmathur/model_51", + "garage-bAInd/Camel-Platypus2-70B", + "TheBloke/Airoboros-L2-70B-2.1-GPTQ", + "OpenAssistant/llama2-70b-oasst-sft-v10", + "garage-bAInd/Platypus2-70B", + "liuxiang886/llama2-70B-qlora-gpt4", + "upstage/llama-65b-instruct", + "quantumaikr/llama-2-70b-fb16-korean", + "NousResearch/Nous-Hermes-Llama2-70b", + "v2ray/LLaMA-2-Jannie-70B-QLoRA", + "jondurbin/airoboros-l2-70b-gpt4-m2.0", + "jondurbin/airoboros-l2-70b-gpt4-m2.0", + "OpenAssistant/llama2-70b-oasst-sft-v10", + "yeontaek/llama-2-70B-ensemble-v8", + "jondurbin/airoboros-l2-70b-gpt4-2.0", + "jarradh/llama2_70b_chat_uncensored", + "WizardLM/WizardMath-70B-V1.0", + "jordiclive/Llama-2-70b-oasst-1-200", + "WizardLM/WizardMath-70B-V1.0", + "jondurbin/airoboros-l2-70b-gpt4-2.0", + "OpenLemur/lemur-70b-chat-v1", + "tiiuae/falcon-180B", + "tiiuae/falcon-180B", + "stabilityai/StableBeluga1-Delta", + "psmathur/model_42_70b", + "psmathur/test_42_70b", + "TheBloke/fiction.live-Kimiko-V2-70B-fp16", + "tiiuae/falcon-180B", + "WizardLM/WizardMath-70B-V1.0", + "tiiuae/falcon-180B-chat", + "jondurbin/airoboros-l2-70b-gpt4-2.0", + "ehartford/samantha-1.1-llama-33b", + "ajibawa-2023/scarlett-33b", + "ddobokki/Llama-2-70b-orca-200k", + "TheBloke/gpt4-alpaca-lora_mlp-65B-HF", + "tiiuae/falcon-180B-chat", + "tiiuae/falcon-180B-chat", + "tiiuae/falcon-180B", + "TheBloke/Lemur-70B-Chat-v1-GPTQ", + "NousResearch/Nous-Puffin-70B", + "WizardLM/WizardLM-70B-V1.0", + "WizardLM/WizardMath-70B-V1.0", + "meta-llama/Llama-2-70b-hf", + "TheBloke/Llama-2-70B-fp16", + "Weyaxi/llama-2-alpacagpt4-1000step", + "WizardLM/WizardLM-70B-V1.0", + "simsim314/WizardLM-70B-V1.0-HF", + "simsim314/WizardLM-70B-V1.0-HF", + "WizardLM/WizardLM-70B-V1.0", + "openbmb/UltraLM-65b", + "psmathur/model_420_preview", + "WizardLM/WizardLM-70B-V1.0", + "simsim314/WizardLM-70B-V1.0-HF", + "OpenBuddy/openbuddy-llama2-70b-v10.1-bf16", + "upstage/llama-30b-instruct-2048", + "jondurbin/airoboros-65b-gpt4-1.2", + "TheBloke/guanaco-65B-HF", + "jondurbin/airoboros-65b-gpt4-1.3", + "meta-llama/Llama-2-70b-chat-hf", + "ValiantLabs/ShiningValiant", + "Faradaylab/Aria-70B", + "lilloukas/GPlatty-30B", + "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16", + "jondurbin/airoboros-65b-gpt4-1.4-peft", + "jondurbin/airoboros-65b-gpt4-1.4", + "jondurbin/airoboros-65b-gpt4-2.0", + "TheBloke/WizardLM-70B-V1.0-GPTQ", + "TheBloke/WizardLM-70B-V1.0-GPTQ", + "ariellee/SuperPlatty-30B", + "jondurbin/airoboros-65b-gpt4-1.4", + "jondurbin/airoboros-65b-gpt4-2.0", + "yeontaek/llama-2-70b-IA3-guanaco", + "CalderaAI/30B-Lazarus", + "Aspik101/trurl-2-13b-pl-instruct_unload", + "ehartford/WizardLM-33B-V1.0-Uncensored", + "ehartford/WizardLM-33B-V1.0-Uncensored", + "OpenBuddy/openbuddy-llama-65b-v8-bf16", + "Aspik101/llama-30b-instruct-2048-PL-lora", + "h2oai/h2ogpt-research-oasst1-llama-65b", + "Aspik101/llama-30b-instruct-2048-PL-lora", + "CalderaAI/30B-Epsilon", + "Aspik101/llama-30b-2048-instruct-PL-lora_unload", + "jondurbin/airoboros-65b-gpt4-m2.0", + "jondurbin/airoboros-65b-gpt4-m2.0", + "Aeala/Alpaca-elina-65b", + "TheBloke/robin-65b-v2-fp16", + "TheBloke/gpt4-alpaca-lora-30b-HF", + "TheBloke/Llama-2-70B-chat-GPTQ", + "upstage/llama-30b-instruct", + "OpenLemur/lemur-70b-v1", + "lmsys/vicuna-33b-v1.3", + "ausboss/llama-30b-supercot", + "ai-business/Luban-13B", + "Henk717/airochronos-33B", + "lmsys/vicuna-33b-v1.3", + "Henk717/airochronos-33B", + "bavest/fin-llama-33b-merged", + "jondurbin/airoboros-33b-gpt4-1.4", + "YeungNLP/firefly-llama-30b", + "Aspik101/30B-Lazarus-instruct-PL-lora_unload", + "uukuguy/speechless-llama2-luban-orca-platypus-13b", + "xxyyy123/test_merge_p_ov1_w0.66_w0.5_n1", + "jondurbin/airoboros-33b-gpt4-1.2", + "TheBloke/alpaca-lora-65B-HF", + "bofenghuang/vigogne-33b-instruct", + "yeontaek/llama-2-13B-ensemble-v5", + "garage-bAInd/Platypus-30B", + "Open-Orca/OpenOrca-Platypus2-13B", + "kajdun/viwaai-30b_v4", + "lilloukas/Platypus-30B", + "Open-Orca/OpenOrca-Platypus2-13B", + "Henk717/chronoboros-33B", + "jondurbin/airoboros-33b-2.1", + "HiTZ/alpaca-lora-65b-en-pt-es-ca", + "quantumaikr/QuantumLM-70B-hf", + "uukuguy/speechless-llama2-13b", + "uukuguy/speechless-llama2-hermes-orca-platypus-13b", + "openaccess-ai-collective/manticore-30b-chat-pyg-alpha", + "LLMs/WizardLM-30B-V1.0", + "TheBloke/WizardLM-30B-fp16", + "openaccess-ai-collective/hippogriff-30b-chat", + "concedo/Vicuzard-30B-Uncensored", + "TFLai/OpenOrca-Platypus2-13B-QLoRA-0.80-epoch", + "huggingface/llama-65b", + "huggyllama/llama-65b", + "gaodrew/gaodrew-llama-30b-instruct-2048-Open-Platypus-100steps", + "uukuguy/speechless-llama2-hermes-orca-platypus-wizardlm-13b", + "Sao10K/Mythical-Destroyer-V2-L2-13B", + "camel-ai/CAMEL-33B-Combined-Data", + "dsvv-cair/alpaca-cleaned-llama-30b-bf16", + "MetaIX/GPT4-X-Alpasta-30b", + "garage-bAInd/Stable-Platypus2-13B", + "TFLai/Luban-Platypus2-13B-QLora-0.80-epoch", + "TheBloke/OpenOrca-Platypus2-13B-GPTQ", + "IkariDev/Athena-tmp", + "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16", + "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16", + "Open-Orca/OpenOrcaxOpenChat-Preview2-13B", + "psmathur/model_007_13b_v2", + "Aspik101/Vicuzard-30B-Uncensored-instruct-PL-lora_unload", + "jondurbin/airoboros-33b-gpt4-m2.0", + "Sao10K/Mythical-Destroyer-L2-13B", + "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16", + "ehartford/Wizard-Vicuna-30B-Uncensored", + "TFLai/Nova-13B", + "TheBloke/robin-33B-v2-fp16", + "totally-not-an-llm/PuddleJumper-13b", + "Aeala/VicUnlocked-alpaca-30b", + "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf", + "jondurbin/airoboros-33b-gpt4", + "jondurbin/airoboros-33b-gpt4-m2.0", + "tiiuae/falcon-40b-instruct", + "psmathur/orca_mini_v3_13b", + "Aeala/GPT4-x-AlpacaDente-30b", + "MayaPH/GodziLLa-30B", + "jondurbin/airoboros-33b-gpt4-m2.0", + "TFLai/SpeechlessV1-Nova-13B", + "yeontaek/llama-2-13B-ensemble-v4", + "ajibawa-2023/carl-33b", + "jondurbin/airoboros-33b-gpt4-2.0", + "TFLai/Stable-Platypus2-13B-QLoRA-0.80-epoch", + "jondurbin/airoboros-33b-gpt4-1.3", + "TehVenom/oasst-sft-6-llama-33b-xor-MERGED-16bit", + "TFLai/OrcaMini-Platypus2-13B-QLoRA-0.80-epoch", + "jondurbin/airoboros-33b-gpt4-2.0", + "chargoddard/Chronorctypus-Limarobormes-13b", + "jondurbin/airoboros-33b-gpt4-1.3", + "Open-Orca/OpenOrca-Platypus2-13B", + "FelixChao/vicuna-33b-coder", + "FelixChao/vicuna-33b-coder", + "Gryphe/MythoMix-L2-13b", + "Aeala/Enterredaas-33b", + "yeontaek/llama-2-13B-ensemble-v1", + "TFLai/OpenOrcaPlatypus2-Platypus2-13B-QLora-0.80-epoch", + "TFLai/Ensemble5-Platypus2-13B-QLora-0.80-epoch", + "yeontaek/llama-2-13B-ensemble-v3", + "TFLai/MythoMix-Platypus2-13B-QLoRA-0.80-epoch", + "yihan6324/llama2-13b-instructmining-40k-sharegpt", + "timdettmers/guanaco-33b-merged", + "TFLai/EnsembleV5-Nova-13B", + "circulus/Llama-2-13b-orca-v1", + "Undi95/ReMM-SLERP-L2-13B", + "Gryphe/MythoMax-L2-13b", + "stabilityai/StableBeluga-13B", + "circulus/Llama-2-13b-orca-v1", + "ehartford/WizardLM-30B-Uncensored", + "The-Face-Of-Goonery/huginnv1.2", + "TheBloke/OpenOrcaxOpenChat-Preview2-13B-GPTQ", + "Sao10K/Stheno-L2-13B", + "bofenghuang/vigogne-2-13b-instruct", + "The-Face-Of-Goonery/Huginn-13b-FP16", + "grimpep/L2-MythoMax22b-instruct-Falseblock", + "TFLai/Nous-Hermes-Platypus2-13B-QLoRA-0.80-epoch", + "yeontaek/Platypus2xOpenOrca-13B-IA3-v4", + "yeontaek/Platypus2xOpenOrca-13B-IA3", + "yeontaek/Platypus2xOpenOrca-13B-IA3-ensemble", + "Open-Orca/LlongOrca-13B-16k", + "Sao10K/Stheno-Inverted-L2-13B", + "garage-bAInd/Camel-Platypus2-13B", + "digitous/Alpacino30b", + "NousResearch/Nous-Hermes-Llama2-13b", + "yeontaek/Platypus2xOpenOrca-13B-IA3-v3", + "TFLai/MythicalDestroyerV2-Platypus2-13B-QLora-0.80-epoch", + "TheBloke/VicUnlocked-30B-LoRA-HF", + "Undi95/Nous-Hermes-13B-Code", + "The-Face-Of-Goonery/Chronos-Beluga-v2-13bfp16", + "NousResearch/Nous-Hermes-Llama2-13b", + "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b", + "TheBloke/Wizard-Vicuna-30B-Uncensored-GPTQ", + "Open-Orca/OpenOrcaxOpenChat-Preview2-13B", + "Austism/chronos-hermes-13b-v2", + "yeontaek/Platypus2xOpenOrca-13B-IA3-v2.1", + "yeontaek/Platypus2xOpenOrca-13B-IA3-v2", + "Gryphe/MythoLogic-L2-13b", + "augtoma/qCammel-13", + "YeungNLP/firefly-llama2-13b-v1.2", + "Aspik101/StableBeluga-13B-instruct-PL-lora_unload", + "andreaskoepf/llama2-13b-megacode2_min100", + "rombodawg/LosslessMegaCoder-llama2-13b-mini", + "yulan-team/YuLan-Chat-2-13b-fp16", + "elinas/chronos-33b", + "YeungNLP/firefly-llama2-13b", + "Sao10K/Medusa-13b", + "OptimalScale/robin-65b-v2-delta", + "minlik/chinese-alpaca-33b-merged", + "OpenAssistant/llama2-13b-megacode2-oasst", + "TheBloke/OpenAssistant-SFT-7-Llama-30B-HF", + "Undi95/UndiMix-v1-13b", + "ehartford/Samantha-1.11-13b", + "beaugogh/Llama2-13b-sharegpt4", + "Aeala/GPT4-x-AlpacaDente2-30b", + "luffycodes/nash-vicuna-13b-v1dot5-ep2-w-rag-w-simple", + "WizardLM/WizardLM-13B-V1.1", + "uukuguy/speechless-orca-platypus-coig-lite-2k-0.6e-13b", + "huggyllama/llama-30b", + "Undi95/ReMM-L2-13B-PIPPA", + "Undi95/ReMM-L2-13B", + "gaodrew/gaodrew-gorgonzola-13b", + "lmsys/vicuna-13b-v1.5", + "yeontaek/Platypus2xOpenOrca-13B-LoRa", + "Yhyu13/llama-30B-hf-openassitant", + "huggingface/llama-30b", + "lmsys/vicuna-13b-v1.5", + "TFLai/Athena-Platypus2-13B-QLora-0.80-epoch", + "TheBloke/dromedary-65b-lora-HF", + "yeontaek/llama-2-13b-Beluga-QLoRA", + "The-Face-Of-Goonery/Huginn-13b-V4", + "The-Face-Of-Goonery/Huginn-13b-v4.5", + "The-Face-Of-Goonery/Huginn-v3-13b", + "tiiuae/falcon-40b", + "WhoTookMyAmogusNickname/NewHope_HF_not_official", + "gaodrew/OpenOrca-Platypus2-13B-thera-1250", + "SLAM-group/NewHope", + "garage-bAInd/Platypus2-13B", + "migtissera/Synthia-13B", + "elinas/chronos-13b-v2", + "mosaicml/mpt-30b-chat", + "CHIH-HUNG/llama-2-13b-OpenOrca_5w", + "uukuguy/speechless-hermes-coig-lite-13b", + "TheBloke/tulu-30B-fp16", + "uukuguy/speechless-hermes-coig-lite-13b", + "xDAN-AI/xDAN_13b_l2_lora", + "lmsys/vicuna-13b-v1.5-16k", + "openchat/openchat_v3.1", + "CHIH-HUNG/llama-2-13b-dolphin_5w", + "Aspik101/vicuna-13b-v1.5-PL-lora_unload", + "Undi95/MLewd-L2-13B", + "ehartford/minotaur-llama2-13b-qlora", + "kajdun/iubaris-13b-v3", + "TFLai/Limarp-Platypus2-13B-QLoRA-0.80-epoch", + "openchat/openchat_v3.1", + "uukuguy/speechless-orca-platypus-coig-lite-4k-0.6e-13b", + "ziqingyang/chinese-alpaca-2-13b", + "TFLai/Airboros2.1-Platypus2-13B-QLora-0.80-epoch", + "yeontaek/llama-2-13b-Guanaco-QLoRA", + "lmsys/vicuna-13b-v1.5-16k", + "ehartford/based-30b", + "kingbri/airolima-chronos-grad-l2-13B", + "openchat/openchat_v3.2", + "uukuguy/speechless-orca-platypus-coig-lite-4k-0.5e-13b", + "yeontaek/Platypus2-13B-LoRa", + "kingbri/chronolima-airo-grad-l2-13B", + "openchat/openchat_v3.2", + "TFLai/PuddleJumper-Platypus2-13B-QLoRA-0.80-epoch", + "shareAI/llama2-13b-Chinese-chat", + "ehartford/WizardLM-1.0-Uncensored-Llama2-13b", + "Aspik101/Redmond-Puffin-13B-instruct-PL-lora_unload", + "yeontaek/llama-2-13B-ensemble-v6", + "WizardLM/WizardLM-13B-V1.2", + "TheBloke/WizardLM-13B-V1.1-GPTQ", + "bhenrym14/airophin-13b-pntk-16k-fp16", + "ehartford/WizardLM-1.0-Uncensored-Llama2-13b", + "Mikael110/llama-2-13b-guanaco-fp16", + "yeontaek/airoboros-2.1-llama-2-13B-QLoRa", + "CalderaAI/13B-Legerdemain-L2", + "grimpep/llama2-22b-wizard_vicuna", + "grimpep/llama2-22B-GPLATTY", + "bhenrym14/airophin-13b-pntk-16k-fp16", + "yeontaek/llama-2-13b-QLoRA", + "OpenAssistant/llama2-13b-orca-8k-3319", + "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16", + "duliadotio/dulia-13b-8k-alpha", + "Undi95/LewdEngine", + "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16", + "CHIH-HUNG/llama-2-13b-open_orca_20w", + "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16", + "FlagAlpha/Llama2-Chinese-13b-Chat", + "LLMs/WizardLM-13B-V1.0", + "chansung/gpt4-alpaca-lora-13b-decapoda-1024", + "TheBloke/wizardLM-13B-1.0-fp16", + "digitous/13B-Chimera", + "yeontaek/Platypus2xOpenOrcaxGuanaco-13B-LoRa", + "jondurbin/airoboros-l2-13b-2.1", + "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b", + "TheBloke/UltraLM-13B-fp16", + "openaccess-ai-collective/minotaur-13b-fixed", + "NousResearch/Redmond-Puffin-13B", + "KoboldAI/LLaMA2-13B-Holomax", + "Lajonbot/WizardLM-13B-V1.2-PL-lora_unload", + "yeontaek/Platypus2-13B-LoRa-v2", + "TheBloke/airoboros-13B-HF", + "jondurbin/airoboros-13b", + "jjaaaww/posi_13b", + "CoolWP/llama-2-13b-guanaco-fp16", + "yeontaek/Platypus2-13B-QLoRa", + "h2oai/h2ogpt-research-oig-oasst1-512-30b", + "dfurman/llama-2-13b-guanaco-peft", + "NousResearch/Redmond-Puffin-13B", + "pe-nlp/llama-2-13b-platypus-vicuna-wizard", + "CHIH-HUNG/llama-2-13b-dolphin_20w", + "NousResearch/Nous-Hermes-13b", + "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEconsE4", + "ehartford/Wizard-Vicuna-13B-Uncensored", + "TheBloke/Wizard-Vicuna-13B-Uncensored-HF", + "openchat/openchat_v3.2_super", + "bhenrym14/airophin-v2-13b-PI-8k-fp16", + "openaccess-ai-collective/manticore-13b", + "The-Face-Of-Goonery/Huginn-22b-Prototype", + "jphme/Llama-2-13b-chat-german", + "grimpep/llama2-28B-Airo03", + "TheBloke/Kimiko-v2-13B-fp16", + "FPHam/Free_Sydney_13b_HF", + "lmsys/vicuna-13b-v1.3", + "FelixChao/llama2-13b-math1.1", + "CalderaAI/13B-BlueMethod", + "meta-llama/Llama-2-13b-chat-hf", + "deepse/CodeUp-Llama-2-13b-chat-hf", + "WizardLM/WizardMath-13B-V1.0", + "WizardLM/WizardMath-13B-V1.0", + "HyperbeeAI/Tulpar-7b-v0", + "xxyyy123/test_qkvo_adptor", + "xxyyy123/mc_data_30k_from_platpus_orca_7b_10k_v1_lora_qkvo_rank14_v2", + "openchat/openchat_v2_w", + "FelixChao/llama2-13b-math1.1", + "psmathur/orca_mini_v3_7b", + "TehVenom/Metharme-13b-Merged", + "xxyyy123/10k_v1_lora_qkvo_rank14_v3", + "OpenAssistant/llama2-13b-orca-v2-8k-3166", + "openaccess-ai-collective/wizard-mega-13b", + "jondurbin/airoboros-13b-gpt4-1.4", + "jondurbin/airoboros-13b-gpt4-1.4-fp16", + "Monero/Manticore-13b-Chat-Pyg-Guanaco", + "FelixChao/llama2-13b-math1.2", + "chargoddard/platypus-2-22b-relora", + "FelixChao/llama2-13b-math1.2", + "Gryphe/MythoBoros-13b", + "CalderaAI/13B-Ouroboros", + "OpenAssistant/llama2-13b-orca-v2-8k-3166", + "heegyu/LIMA2-13b-hf", + "digitous/13B-HyperMantis", + "Gryphe/MythoLogic-13b", + "TheBloke/Airoboros-L2-13B-2.1-GPTQ", + "chargoddard/platypus2-22b-relora", + "openchat/openchat_v2", + "yeontaek/Platypus2-13B-IA3", + "stabilityai/StableBeluga-7B", + "circulus/Llama-2-7b-orca-v1", + "budecosystem/genz-13b-v2", + "TheBloke/gpt4-x-vicuna-13B-HF", + "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEcons", + "zarakiquemparte/zarafusionex-1.1-l2-7b", + "Lajonbot/tableBeluga-7B-instruct-pl-lora_unload", + "jondurbin/airoboros-13b-gpt4", + "gaodrew/gaodrew-gorgonzola-13b", + "jondurbin/airoboros-13b-gpt4-1.1", + "TheBloke/gpt4-alpaca-lora-13B-HF", + "zarakiquemparte/zarablendex-vq-l2-7b", + "openaccess-ai-collective/manticore-13b-chat-pyg", + "Lajonbot/Llama-2-13b-hf-instruct-pl-lora_unload", + "NobodyExistsOnTheInternet/PuffedLIMA13bQLORA", + "xxyyy123/10k_v1_lora_qkvo_rank28_v2", + "jondurbin/airoboros-l2-13b-gpt4-1.4.1", + "dhmeltzer/Llama-2-13b-hf-eli5-wiki-1024_r_64_alpha_16", + "NobodyExistsOnTheInternet/PuffedConvo13bLoraE4", + "yihan6324/llama2-7b-instructmining-40k-sharegpt", + "CHIH-HUNG/llama-2-13b-Open_Platypus_and_ccp_2.6w", + "Aeala/GPT4-x-Alpasta-13b", + "psmathur/orca_mini_v2_13b", + "YeungNLP/firefly-llama-13b", + "psmathur/orca_mini_v2_13b", + "zarakiquemparte/zarafusionix-l2-7b", + "yihan6324/llama2-7b-instructmining-60k-sharegpt", + "yihan6324/llama-2-7b-instructmining-60k-sharegpt", + "layoric/llama-2-13b-code-alpaca", + "bofenghuang/vigogne-13b-instruct", + "Lajonbot/vicuna-13b-v1.3-PL-lora_unload", + "lvkaokao/llama2-7b-hf-chat-lora-v3", + "ehartford/dolphin-llama-13b", + "YeungNLP/firefly-llama-13b-v1.2", + "TheBloke/Kimiko-13B-fp16", + "kevinpro/Vicuna-13B-CoT", + "eachadea/vicuna-13b-1.1", + "pillowtalks-ai/delta13b", + "TheBloke/vicuna-13B-1.1-HF", + "TheBloke/Vicuna-13B-CoT-fp16", + "lmsys/vicuna-13b-delta-v1.1", + "lmsys/vicuna-13b-v1.1", + "xxyyy123/20k_v1_lora_qkvo_rank14_v2", + "TheBloke/guanaco-13B-HF", + "TheBloke/vicuna-13b-v1.3.0-GPTQ", + "edor/Stable-Platypus2-mini-7B", + "totally-not-an-llm/EverythingLM-13b-V2-16k", + "zarakiquemparte/zaraxe-l2-7b", + "beaugogh/Llama2-7b-openorca-mc-v2", + "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16", + "quantumaikr/QuantumLM", + "jondurbin/airoboros-13b-gpt4-1.2", + "TheBloke/robin-13B-v2-fp16", + "TFLai/llama-2-13b-4bit-alpaca-gpt4", + "yihan6324/llama2-7b-instructmining-orca-40k", + "dvruette/oasst-llama-13b-2-epochs", + "Open-Orca/LlongOrca-7B-16k", + "Aspik101/Nous-Hermes-13b-pl-lora_unload", + "ehartford/Samantha-1.11-CodeLlama-34b", + "nkpz/llama2-22b-chat-wizard-uncensored", + "bofenghuang/vigogne-13b-chat", + "beaugogh/Llama2-7b-openorca-mc-v1", + "OptimalScale/robin-13b-v2-delta", + "pe-nlp/llama-2-13b-vicuna-wizard", + "chargoddard/llama2-22b", + "gywy/llama2-13b-chinese-v1", + "frank098/Wizard-Vicuna-13B-juniper", + "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset", + "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-gate_up_down_proj", + "eachadea/vicuna-13b", + "yihan6324/llama2-7b-instructmining-orca-90k", + "chargoddard/llama2-22b-blocktriangular", + "luffycodes/mcq-vicuna-13b-v1.5", + "Yhyu13/chimera-inst-chat-13b-hf", + "luffycodes/mcq-vicuna-13b-v1.5", + "chargoddard/ypotryll-22b-epoch2-qlora", + "totally-not-an-llm/EverythingLM-13b-16k", + "luffycodes/mcq-hal-vicuna-13b-v1.5", + "openaccess-ai-collective/minotaur-13b", + "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset", + "chargoddard/llama2-22b-blocktriangular", + "TFLai/Platypus2-13B-QLoRA-0.80-epoch", + "meta-llama/Llama-2-13b-hf", + "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-gate_up_down_proj", + "luffycodes/mcq-hal-vicuna-13b-v1.5", + "TheBloke/Llama-2-13B-fp16", + "TaylorAI/Flash-Llama-13B", + "shareAI/bimoGPT-llama2-13b", + "wahaha1987/llama_13b_sharegpt94k_fastchat", + "openchat/openchat_8192", + "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-q_k_v_o_proj", + "dvruette/llama-13b-pretrained-sft-do2", + "CHIH-HUNG/llama-2-13b-alpaca-test", + "OpenBuddy/openbuddy-llama2-13b-v11.1-bf16", + "CHIH-HUNG/llama-2-13b-FINETUNE2_TEST_2.2w", + "project-baize/baize-v2-13b", + "jondurbin/airoboros-l2-13b-gpt4-m2.0", + "yeontaek/Platypus2xOpenOrca-13B-LoRa-v2", + "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w", + "xzuyn/Alpacino-SuperCOT-13B", + "jondurbin/airoboros-l2-13b-gpt4-2.0", + "aiplanet/effi-13b", + "clibrain/Llama-2-13b-ft-instruct-es", + "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w", + "bofenghuang/vigogne-2-7b-instruct", + "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-q_k_v_o_proj", + "bofenghuang/vigogne-2-7b-chat", + "aiplanet/effi-13b", + "haonan-li/bactrian-x-llama-13b-merged", + "beaugogh/Llama2-7b-sharegpt4", + "HWERI/Llama2-7b-sharegpt4", + "jondurbin/airoboros-13b-gpt4-1.3", + "jondurbin/airoboros-c34b-2.1", + "junelee/wizard-vicuna-13b", + "TheBloke/wizard-vicuna-13B-HF", + "Open-Orca/OpenOrca-Preview1-13B", + "TheBloke/h2ogpt-oasst1-512-30B-HF", + "TheBloke/Llama-2-13B-GPTQ", + "camel-ai/CAMEL-13B-Combined-Data", + "lmsys/vicuna-7b-v1.5", + "lmsys/vicuna-7b-v1.5-16k", + "lmsys/vicuna-7b-v1.5", + "ausboss/llama-13b-supercot", + "TheBloke/tulu-13B-fp16", + "NousResearch/Nous-Hermes-llama-2-7b", + "jlevin/guanaco-13b-llama-2", + "lmsys/vicuna-7b-v1.5-16k", + "dvruette/llama-13b-pretrained", + "nkpz/llama2-22b-daydreamer-v3", + "dvruette/llama-13b-pretrained-dropout", + "jondurbin/airoboros-l2-13b-2.1", + "LLMs/Stable-Vicuna-13B", + "64bits/LexPodLM-13B", + "lizhuang144/llama_mirror_13b_v1.0", + "TheBloke/stable-vicuna-13B-HF", + "zarakiquemparte/zaraxls-l2-7b", + "TheBloke/Llama-2-13B-GPTQ", + "Kiddyz/testlm-3", + "migtissera/Synthia-7B", + "zarakiquemparte/zarablend-l2-7b", + "mosaicml/mpt-30b-instruct", + "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged", + "vonjack/Qwen-LLaMAfied-HFTok-7B-Chat", + "l3utterfly/llama2-7b-layla", + "Lajonbot/vicuna-7b-v1.5-PL-lora_unload", + "heegyu/LIMA-13b-hf", + "frank098/WizardLM_13B_juniper", + "ashercn97/manatee-7b", + "chavinlo/gpt4-x-alpaca", + "PocketDoc/Dans-PersonalityEngine-13b", + "ehartford/WizardLM-1.0-Uncensored-CodeLlama-34b", + "digitous/Alpacino13b", + "edor/Hermes-Platypus2-mini-7B", + "lvkaokao/llama2-7b-hf-chat-lora-v2", + "Kiddyz/testlm-1-1", + "Kiddyz/testlm", + "Kiddyz/testlm-1", + "Kiddyz/testlm2", + "radm/Philosophy-Platypus2-13b", + "aiplanet/effi-13b", + "Harshvir/Llama-2-7B-physics", + "YeungNLP/firefly-ziya-13b", + "LinkSoul/Chinese-Llama-2-7b", + "PeanutJar/LLaMa-2-PeanutButter_v10-7B", + "OpenBuddy/openbuddy-llama2-13b-v11-bf16", + "StudentLLM/Alpagasus-2-13B-QLoRA-pipeline", + "meta-llama/Llama-2-13b-hf", + "WizardLM/WizardCoder-Python-34B-V1.0", + "dvruette/llama-13b-pretrained-sft-epoch-1", + "camel-ai/CAMEL-13B-Role-Playing-Data", + "ziqingyang/chinese-llama-2-13b", + "rombodawg/LosslessMegaCoder-llama2-7b-mini", + "TheBloke/koala-13B-HF", + "lmsys/vicuna-7b-delta-v1.1", + "eachadea/vicuna-7b-1.1", + "Ejafa/vicuna_7B_vanilla_1.1", + "lvkaokao/llama2-7b-hf-chat-lora", + "OpenBuddy/openbuddy-atom-13b-v9-bf16", + "Norquinal/llama-2-7b-claude-chat-rp", + "Danielbrdz/Barcenas-7b", + "heegyu/WizardVicuna2-13b-hf", + "meta-llama/Llama-2-7b-chat-hf", + "PeanutJar/LLaMa-2-PeanutButter_v14-7B", + "PeanutJar/LLaMa-2-PeanutButter_v4-7B", + "davzoku/cria-llama2-7b-v1.3", + "OpenBuddy/openbuddy-atom-13b-v9-bf16", + "lvkaokao/llama2-7b-hf-instruction-lora", + "Tap-M/Luna-AI-Llama2-Uncensored", + "ehartford/Samantha-1.11-7b", + "WizardLM/WizardCoder-Python-34B-V1.0", + "TheBloke/Manticore-13B-Chat-Pyg-Guanaco-SuperHOT-8K-GPTQ", + "Mikael110/llama-2-7b-guanaco-fp16", + "garage-bAInd/Platypus2-7B", + "PeanutJar/LLaMa-2-PeanutButter_v18_B-7B", + "mosaicml/mpt-30b", + "garage-bAInd/Platypus2-7B", + "huggingface/llama-13b", + "dvruette/oasst-llama-13b-1000-steps", + "jordiclive/gpt4all-alpaca-oa-codealpaca-lora-13b", + "huggyllama/llama-13b", + "Voicelab/trurl-2-7b", + "TFLai/llama-13b-4bit-alpaca", + "gywy/llama2-13b-chinese-v2", + "lmsys/longchat-13b-16k", + "Aspik101/trurl-2-7b-pl-instruct_unload", + "WizardLM/WizardMath-7B-V1.0", + "Norquinal/llama-2-7b-claude-chat", + "TheTravellingEngineer/llama2-7b-chat-hf-dpo", + "open-llm-leaderboard/starchat-beta", + "joehuangx/spatial-vicuna-7b-v1.5-LoRA", + "conceptofmind/LLongMA-2-13b-16k", + "tianyil1/denas-llama2", + "lmsys/vicuna-7b-v1.3", + "conceptofmind/LLongMA-2-13b-16k", + "openchat/opencoderplus", + "ajibawa-2023/scarlett-7b", + "dhmeltzer/llama-7b-SFT_eli5_wiki65k_1024_r_64_alpha_16_merged", + "psyche/kollama2-7b-v2", + "heegyu/LIMA2-7b-hf", + "dhmeltzer/llama-7b-SFT-qlora-eli5-wiki_DPO_ds_RM_top_2_1024_r_64_alpha_16", + "abhishek/llama2guanacotest", + "jondurbin/airoboros-l2-7b-2.1", + "llama-anon/instruct-13b", + "FelixChao/vicuna-7B-physics", + "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload", + "shibing624/chinese-alpaca-plus-13b-hf", + "davzoku/cria-llama2-7b-v1.3_peft", + "quantumaikr/llama-2-7b-hf-guanaco-1k", + "togethercomputer/Llama-2-7B-32K-Instruct", + "sia-ai/llama-2-7b-1-percent-open-orca-1000-steps-v0", + "TheTravellingEngineer/llama2-7b-hf-guanaco", + "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload", + "jondurbin/airoboros-l2-7b-gpt4-1.4.1", + "wahaha1987/llama_7b_sharegpt94k_fastchat", + "FelixChao/vicuna-7B-chemical", + "TinyPixel/llama2-7b-oa", + "chaoyi-wu/MedLLaMA_13B", + "edor/Platypus2-mini-7B", + "RoversX/llama-2-7b-hf-small-shards-Samantha-V1-SFT", + "venkycs/llama-v2-7b-32kC-Security", + "psyche/kollama2-7b", + "Fredithefish/Guanaco-7B-Uncensored", + "TheTravellingEngineer/llama2-7b-chat-hf-guanaco", + "ehartford/WizardLM-13B-Uncensored", + "PocketDoc/Dans-CreepingSenseOfDoom", + "wenge-research/yayi-7b-llama2", + "georgesung/llama2_7b_chat_uncensored", + "TinyPixel/llama2-7b-instruct", + "quantumaikr/QuantumLM-7B", + "xzuyn/MedicWizard-7B", + "wenge-research/yayi-7b-llama2", + "TinyPixel/lima-test", + "elyza/ELYZA-japanese-Llama-2-7b-instruct", + "lgaalves/llama-2-7b-hf_open-platypus", + "ziqingyang/chinese-alpaca-2-7b", + "TehVenom/Pygmalion-Vicuna-1.1-7b", + "meta-llama/Llama-2-7b-hf", + "bongchoi/test-llama2-7b", + "TaylorAI/Flash-Llama-7B", + "TheTravellingEngineer/llama2-7b-chat-hf-v2", + "TheTravellingEngineer/llama2-7b-chat-hf-v4", + "kashif/stack-llama-2", + "PeanutJar/LLaMa-2-PeanutButter_v18_A-7B", + "ToolBench/ToolLLaMA-7b-LoRA", + "Monero/WizardLM-13b-OpenAssistant-Uncensored", + "TheTravellingEngineer/llama2-7b-chat-hf-v2", + "TheTravellingEngineer/llama2-7b-chat-hf-v4", + "mrm8488/llama-2-coder-7b", + "elyza/ELYZA-japanese-Llama-2-7b-fast-instruct", + "clibrain/Llama-2-7b-ft-instruct-es", + "medalpaca/medalpaca-7b", + "TheBloke/tulu-7B-fp16", + "OpenBuddy/openbuddy-openllama-13b-v7-fp16", + "TaylorAI/FLAN-Llama-7B-2_Llama2-7B-Flash_868_full_model", + "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload", + "jondurbin/airoboros-l2-7b-gpt4-2.0", + "dhmeltzer/llama-7b-SFT_ds_eli5_1024_r_64_alpha_16_merged", + "GOAT-AI/GOAT-7B-Community", + "AtomEchoAI/AtomGPT_56k", + "julianweng/Llama-2-7b-chat-orcah", + "TehVenom/Pygmalion-13b-Merged", + "jondurbin/airoboros-7b-gpt4-1.1", + "dhmeltzer/llama-7b-SFT_ds_wiki65k_1024_r_64_alpha_16_merged", + "bofenghuang/vigogne-7b-chat", + "lmsys/longchat-7b-v1.5-32k", + "jondurbin/airoboros-l2-7b-gpt4-m2.0", + "synapsoft/Llama-2-7b-chat-hf-flan2022-1.2M", + "jondurbin/airoboros-7b-gpt4-1.4", + "Charlie911/vicuna-7b-v1.5-lora-mctaco", + "yihan6324/instructmining-platypus-15k", + "meta-llama/Llama-2-7b-hf", + "TheTravellingEngineer/llama2-7b-chat-hf-v3", + "quantumaikr/KoreanLM-hf", + "openthaigpt/openthaigpt-1.0.0-alpha-7b-chat-ckpt-hf", + "TheBloke/Llama-2-7B-GPTQ", + "TheBloke/Llama-2-7B-GPTQ", + "LLMs/AlpacaGPT4-7B-elina", + "ehartford/Wizard-Vicuna-7B-Uncensored", + "TheBloke/Wizard-Vicuna-7B-Uncensored-HF", + "TheTravellingEngineer/llama2-7b-chat-hf-v3", + "golaxy/gowizardlm", + "ehartford/dolphin-llama2-7b", + "CHIH-HUNG/llama-2-7b-dolphin_10w-test", + "mncai/chatdoctor", + "psyche/kollama2-7b-v3", + "jondurbin/airoboros-7b-gpt4", + "jondurbin/airoboros-7b", + "TheBloke/airoboros-7b-gpt4-fp16", + "mosaicml/mpt-7b-8k-chat", + "elyza/ELYZA-japanese-Llama-2-7b", + "bofenghuang/vigogne-7b-instruct", + "jxhong/CAlign-alpaca-7b", + "golaxy/goims", + "jondurbin/airoboros-7b-gpt4-1.2", + "jphme/orca_mini_v2_ger_7b", + "psmathur/orca_mini_v2_7b", + "notstoic/PygmalionCoT-7b", + "golaxy/gogpt2-13b", + "golaxy/gogpt2-13b-chat", + "togethercomputer/LLaMA-2-7B-32K", + "TheBloke/wizardLM-7B-HF", + "keyfan/vicuna-chinese-replication-v1.1", + "golaxy/gogpt2-7b", + "aiplanet/effi-7b", + "arver/llama7b-qlora", + "titan087/OpenLlama13B-Guanaco", + "chavinlo/alpaca-native", + "project-baize/baize-healthcare-lora-7B", + "AlpinDale/pygmalion-instruct", + "openlm-research/open_llama_13b", + "jondurbin/airoboros-7b-gpt4-1.3", + "elyza/ELYZA-japanese-Llama-2-7b-fast", + "jondurbin/airoboros-gpt-3.5-turbo-100k-7b", + "uukuguy/speechless-codellama-orca-13b", + "bigcode/starcoderplus", + "TheBloke/guanaco-7B-HF", + "Neko-Institute-of-Science/metharme-7b", + "TigerResearch/tigerbot-7b-base", + "golaxy/gogpt-7b", + "togethercomputer/LLaMA-2-7B-32K", + "yhyhy3/open_llama_7b_v2_med_instruct", + "ajibawa-2023/carl-7b", + "stabilityai/stablelm-base-alpha-7b-v2", + "conceptofmind/LLongMA-2-7b-16k", + "TehVenom/Pygmalion_AlpacaLora-7b", + "jondurbin/airoboros-7b-gpt4-1.4.1-qlora", + "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard", + "ausboss/llama7b-wizardlm-unfiltered", + "project-baize/baize-v2-7b", + "LMFlow/Robin-v2", + "HanningZhang/Robin-v2", + "LMFlow/Robin-7b-v2", + "OptimalScale/robin-7b-v2-delta", + "uukuguy/speechless-codellama-platypus-13b", + "jerryjalapeno/nart-100k-7b", + "wenge-research/yayi-13b-llama2", + "fireballoon/baichuan-vicuna-chinese-7b", + "jlevin/guanaco-unchained-llama-2-7b", + "csitfun/llama-7b-logicot", + "DevaMalla/llama7b_alpaca_1gpu_bf16", + "WeOpenML/PandaLM-Alpaca-7B-v1", + "illuin/test-custom-llama", + "yeontaek/WizardCoder-Python-13B-LoRa", + "ashercn97/giraffe-7b", + "mosaicml/mpt-7b-chat", + "abhishek/autotrain-llama-alpaca-peft-52508123785", + "Neko-Institute-of-Science/pygmalion-7b", + "TFLai/llama-7b-4bit-alpaca", + "huggingface/llama-7b", + "TheBloke/Planner-7B-fp16", + "shibing624/chinese-llama-plus-13b-hf", + "AGI-inc/lora_moe_7b_baseline", + "DevaMalla/llama-base-7b", + "AGI-inc/lora_moe_7b", + "togethercomputer/GPT-JT-6B-v0", + "ehartford/WizardLM-7B-Uncensored", + "shibing624/chinese-alpaca-plus-7b-hf", + "beomi/llama-2-ko-7b", + "mosaicml/mpt-7b-8k-instruct", + "Enno-Ai/ennodata-7b", + "mosaicml/mpt-7b-instruct", + "facebook/opt-iml-max-30b", + "WeOpenML/Alpaca-7B-v1", + "TheBloke/Project-Baize-v2-7B-GPTQ", + "codellama/CodeLlama-13b-Instruct-hf", + "TheBloke/CodeLlama-13B-Instruct-fp16", + "facebook/galactica-30b", + "FreedomIntelligence/phoenix-inst-chat-7b", + "openlm-research/open_llama_7b_v2", + "GeorgiaTechResearchInstitute/galpaca-30b", + "THUDM/chatglm2-6b", + "togethercomputer/GPT-JT-6B-v1", + "TheBloke/koala-7B-HF", + "nathan0/mpt_delta_tuned_model_v3", + "nathan0/mpt_delta_tuned_model_v2", + "GeorgiaTechResearchInstitute/galpaca-30b", + "JosephusCheung/Guanaco", + "shareAI/CodeLLaMA-chat-13b-Chinese", + "TigerResearch/tigerbot-7b-sft", + "Writer/InstructPalmyra-20b", + "OpenAssistant/codellama-13b-oasst-sft-v10", + "bigscience/bloomz-7b1-mt", + "nathan0/mpt_delta_tuned_model_v3", + "VMware/open-llama-7b-open-instruct", + "baichuan-inc/Baichuan-7B", + "anas-awadalla/mpt-7b", + "mosaicml/mpt-7b", + "bigscience/bloomz-7b1", + "ziqingyang/chinese-llama-2-7b", + "OpenAssistant/codellama-13b-oasst-sft-v10", + "wenge-research/yayi-7b", + "tiiuae/falcon-7b", + "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1", + "togethercomputer/RedPajama-INCITE-7B-Instruct", + "TheBloke/landmark-attention-llama7b-fp16", + "togethercomputer/GPT-JT-Moderation-6B", + "h2oai/h2ogpt-gm-oasst1-en-1024-20b", + "dvruette/gpt-neox-20b-full-precision", + "TehVenom/Moderator-Chan_GPT-JT-6b", + "dvruette/oasst-gpt-neox-20b-1000-steps", + "AlekseyKorshuk/pygmalion-6b-vicuna-chatml", + "facebook/opt-66b", + "Salesforce/codegen-16B-nl", + "Vmware/open-llama-7b-v2-open-instruct", + "mosaicml/mpt-7b-storywriter", + "acrastt/Marx-3B-V2", + "openlm-research/open_llama_7b", + "Fredithefish/ReasonixPajama-3B-HF", + "togethercomputer/GPT-NeoXT-Chat-Base-20B", + "psmathur/orca_mini_13b", + "RWKV/rwkv-raven-14b", + "h2oai/h2ogpt-oasst1-512-20b", + "acrastt/Marx-3B", + "klosax/open_llama_13b_600bt_preview", + "synapsoft/Llama-2-7b-hf-flan2022-1.2M", + "OpenAssistant/oasst-sft-1-pythia-12b", + "golaxy/gogpt-7b-bloom", + "Writer/palmyra-large", + "psmathur/orca_mini_7b", + "dvruette/oasst-pythia-12b-6000-steps", + "NousResearch/CodeLlama-13b-hf", + "codellama/CodeLlama-13b-hf", + "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b", + "VMware/open-llama-0.7T-7B-open-instruct-v1.1", + "dvruette/oasst-pythia-12b-flash-attn-5000-steps", + "dvruette/oasst-gpt-neox-20b-3000-steps", + "RobbeD/OpenLlama-Platypus-3B", + "facebook/opt-30b", + "acrastt/Puma-3B", + "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5", + "dvruette/oasst-pythia-12b-pretrained-sft", + "digitous/GPT-R", + "acrastt/Griffin-3B", + "togethercomputer/RedPajama-INCITE-Base-7B-v0.1", + "togethercomputer/RedPajama-INCITE-7B-Base", + "CobraMamba/mamba-gpt-3b-v3", + "Danielbrdz/CodeBarcenas-7b", + "l3utterfly/open-llama-3b-v2-layla", + "CobraMamba/mamba-gpt-3b-v2", + "OpenAssistant/pythia-12b-sft-v8-7k-steps", + "KoboldAI/GPT-NeoX-20B-Erebus", + "RobbeD/Orca-Platypus-3B", + "h2oai/h2ogpt-gm-oasst1-en-1024-12b", + "OpenAssistant/pythia-12b-sft-v8-2.5k-steps", + "AlekseyKorshuk/chatml-pyg-v1", + "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1", + "togethercomputer/RedPajama-INCITE-7B-Chat", + "digitous/Javelin-R", + "dvruette/oasst-pythia-12b-reference", + "EleutherAI/gpt-neox-20b", + "KoboldAI/fairseq-dense-13B", + "OpenAssistant/pythia-12b-sft-v8-rlhf-2k-steps", + "codellama/CodeLlama-7b-Instruct-hf", + "digitous/Javelin-GPTJ", + "KoboldAI/GPT-NeoX-20B-Skein", + "digitous/Javalion-R", + "h2oai/h2ogpt-oasst1-512-12b", + "acrastt/Bean-3B", + "KoboldAI/GPT-J-6B-Skein", + "nomic-ai/gpt4all-j", + "databricks/dolly-v2-12b", + "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2", + "databricks/dolly-v2-7b", + "Aspik101/WizardVicuna-Uncensored-3B-instruct-PL-lora_unload", + "digitous/Adventien-GPTJ", + "openlm-research/open_llama_3b_v2", + "RWKV/rwkv-4-14b-pile", + "Lazycuber/Janemalion-6B", + "OpenAssistant/pythia-12b-pre-v8-12.5k-steps", + "digitous/Janin-R", + "kfkas/Llama-2-ko-7b-Chat", + "heegyu/WizardVicuna-Uncensored-3B-0719", + "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt", + "TaylorAI/Flash-Llama-3B", + "kfkas/Llama-2-ko-7b-Chat", + "digitous/Skegma-GPTJ", + "digitous/Javalion-GPTJ", + "Pirr/pythia-13b-deduped-green_devil", + "TehVenom/PPO_Shygmalion-V8p4_Dev-6b", + "dvruette/oasst-pythia-6.9b-4000-steps", + "heegyu/WizardVicuna-3B-0719", + "psmathur/orca_mini_3b", + "OpenAssistant/galactica-6.7b-finetuned", + "frank098/orca_mini_3b_juniper", + "PygmalionAI/pygmalion-6b", + "TehVenom/PPO_Pygway-V8p4_Dev-6b", + "TFLai/gpt-neox-20b-4bit-alpaca", + "Corianas/gpt-j-6B-Dolly", + "TehVenom/Dolly_Shygmalion-6b", + "digitous/Janin-GPTJ", + "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4", + "EleutherAI/gpt-j-6b", + "KoboldAI/GPT-J-6B-Shinen", + "TehVenom/Dolly_Malion-6b", + "TehVenom/ChanMalion", + "Salesforce/codegen-6B-nl", + "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4", + "KoboldAI/GPT-J-6B-Janeway", + "togethercomputer/RedPajama-INCITE-Chat-3B-v1", + "togethercomputer/Pythia-Chat-Base-7B", + "heegyu/RedTulu-Uncensored-3B-0719", + "KoboldAI/PPO_Pygway-6b-Mix", + "KoboldAI/OPT-13B-Erebus", + "KoboldAI/fairseq-dense-6.7B", + "EleutherAI/pythia-12b-deduped", + "pszemraj/pythia-6.9b-HC3", + "Fredithefish/Guanaco-3B-Uncensored-v2", + "facebook/opt-13b", + "TehVenom/GPT-J-Pyg_PPO-6B", + "EleutherAI/pythia-6.9b-deduped", + "Devio/test-1400", + "Fredithefish/Guanaco-3B-Uncensored", + "codellama/CodeLlama-7b-hf", + "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1", + "Fredithefish/ScarletPajama-3B-HF", + "KoboldAI/OPT-13B-Nerybus-Mix", + "YeungNLP/firefly-bloom-7b1", + "DanielSc4/RedPajama-INCITE-Chat-3B-v1-RL-LoRA-8bit-test1", + "klosax/open_llama_7b_400bt_preview", + "KoboldAI/OPT-13B-Nerys-v2", + "TehVenom/PPO_Shygmalion-6b", + "amazon/LightGPT", + "KnutJaegersberg/black_goo_recipe_c", + "NousResearch/CodeLlama-7b-hf", + "togethercomputer/RedPajama-INCITE-Instruct-3B-v1", + "heegyu/WizardVicuna-open-llama-3b-v2", + "bigscience/bloom-7b1", + "Devio/test-22B", + "RWKV/rwkv-raven-7b", + "hakurei/instruct-12b", + "CobraMamba/mamba-gpt-3b", + "KnutJaegersberg/black_goo_recipe_a", + "acrastt/OmegLLaMA-3B", + "codellama/CodeLlama-7b-Instruct-hf", + "h2oai/h2ogpt-oig-oasst1-512-6_9b", + "KoboldAI/OPT-6.7B-Erebus", + "facebook/opt-6.7b", + "KnutJaegersberg/black_goo_recipe_d", + "KnutJaegersberg/LLongMA-3b-LIMA", + "KnutJaegersberg/black_goo_recipe_b", + "KoboldAI/OPT-6.7B-Nerybus-Mix", + "health360/Healix-3B", + "EleutherAI/pythia-12b", + "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K", + "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k", + "h2oai/h2ogpt-oig-oasst1-256-6_9b", + "ikala/bloom-zh-3b-chat", + "Taekyoon/llama2-ko-7b-test", + "anhnv125/pygmalion-6b-roleplay", + "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4", + "KoboldAI/OPT-6B-nerys-v2", + "Lazycuber/pyg-instruct-wizardlm", + "Devio/testC", + "KoboldAI/OPT-30B-Erebus", + "Fredithefish/CrimsonPajama", + "togethercomputer/RedPajama-INCITE-Base-3B-v1", + "bigscience/bloomz-3b", + "conceptofmind/Open-LLongMA-3b", + "RWKV/rwkv-4-7b-pile", + "openlm-research/open_llama_3b", + "ewof/koishi-instruct-3b", + "DanielSc4/RedPajama-INCITE-Chat-3B-v1-FT-LoRA-8bit-test1", + "cerebras/Cerebras-GPT-13B", + "EleutherAI/pythia-6.7b", + "aisquared/chopt-2_7b", + "Azure99/blossom-v1-3b", + "PSanni/Deer-3b", + "bertin-project/bertin-gpt-j-6B-alpaca", + "OpenBuddy/openbuddy-openllama-3b-v10-bf16", + "KoboldAI/fairseq-dense-2.7B", + "ehartford/CodeLlama-34b-Instruct-hf", + "codellama/CodeLlama-34b-Instruct-hf", + "TheBloke/CodeLlama-34B-Instruct-fp16", + "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2", + "openlm-research/open_llama_7b_700bt_preview", + "NbAiLab/nb-gpt-j-6B-alpaca", + "KoboldAI/OPT-2.7B-Erebus", + "Writer/camel-5b-hf", + "EleutherAI/pythia-2.7b", + "facebook/xglm-7.5B", + "EleutherAI/pythia-2.8b-deduped", + "klosax/open_llama_3b_350bt_preview", + "klosax/openllama-3b-350bt", + "KoboldAI/OPT-2.7B-Nerybus-Mix", + "KoboldAI/GPT-J-6B-Adventure", + "cerebras/Cerebras-GPT-6.7B", + "TFLai/pythia-2.8b-4bit-alpaca", + "facebook/opt-2.7b", + "KoboldAI/OPT-2.7B-Nerys-v2", + "bigscience/bloom-3b", + "Devio/test100", + "RWKV/rwkv-raven-3b", + "Azure99/blossom-v2-3b", + "codellama/CodeLlama-34b-Python-hf", + "bhenrym14/airoboros-33b-gpt4-1.4.1-PI-8192-fp16", + "EleutherAI/gpt-neo-2.7B", + "danielhanchen/open_llama_3b_600bt_preview", + "open-llm-leaderboard/starchat-alpha", + "pythainlp/wangchanglm-7.5B-sft-en-sharded", + "beaugogh/pythia-1.4b-deduped-sharegpt", + "HWERI/pythia-1.4b-deduped-sharegpt", + "OpenAssistant/stablelm-7b-sft-v7-epoch-3", + "codellama/CodeLlama-7b-Python-hf", + "aisquared/chopt-1_3b", + "PygmalionAI/metharme-1.3b", + "Linly-AI/Chinese-LLaMA-2-13B-hf", + "chargoddard/llama-2-34b-uncode", + "RWKV/rwkv-4-3b-pile", + "pythainlp/wangchanglm-7.5B-sft-enth", + "MBZUAI/LaMini-GPT-1.5B", + "Writer/palmyra-base", + "KoboldAI/fairseq-dense-1.3B", + "EleutherAI/pythia-1.4b-deduped", + "MBZUAI/lamini-neo-1.3b", + "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt", + "sartmis1/starcoder-finetune-openapi", + "MayaPH/opt-flan-iml-6.7b", + "facebook/xglm-4.5B", + "WizardLM/WizardCoder-15B-V1.0", + "facebook/opt-iml-max-1.3b", + "stabilityai/stablelm-tuned-alpha-7b", + "aisquared/dlite-v2-1_5b", + "stabilityai/stablelm-base-alpha-7b", + "sartmis1/starcoder-finetune-selfinstruct", + "lizhuang144/starcoder_mirror", + "bigcode/starcoder", + "TheBloke/CodeLlama-34B-Python-fp16", + "open-llm-leaderboard/bloomz-1b7-4bit-alpaca-auto-eval-adapter-applied", + "ehartford/CodeLlama-34b-Python-hf", + "codellama/CodeLlama-7b-Python-hf", + "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct", + "LoupGarou/WizardCoder-Guanaco-15B-V1.0", + "golaxy/gogpt-3b-bloom", + "EleutherAI/pythia-1.3b", + "codellama/CodeLlama-13b-Python-hf", + "hakurei/lotus-12B", + "NYTK/PULI-GPTrio", + "facebook/opt-1.3b", + "TheBloke/CodeLlama-13B-Python-fp16", + "codellama/CodeLlama-13b-Python-hf", + "RWKV/rwkv-raven-1b5", + "PygmalionAI/pygmalion-2.7b", + "bigscience/bloom-1b7", + "gpt2-xl", + "LoupGarou/WizardCoder-Guanaco-15B-V1.1", + "RWKV/rwkv-4-1b5-pile", + "codellama/CodeLlama-34b-hf", + "NousResearch/CodeLlama-34b-hf", + "rinna/bilingual-gpt-neox-4b-8k", + "lxe/Cerebras-GPT-2.7B-Alpaca-SP", + "cerebras/Cerebras-GPT-2.7B", + "jzjiao/opt-1.3b-rlhf", + "EleutherAI/gpt-neo-1.3B", + "aisquared/dlite-v1-1_5b", + "Corianas/Quokka_2.7b", + "MrNJK/gpt2-xl-sft", + "facebook/galactica-1.3b", + "aisquared/dlite-v2-774m", + "EleutherAI/pythia-1b-deduped", + "Kunhao/pile-7b-250b-tokens", + "w601sxs/b1ade-1b", + "rinna/bilingual-gpt-neox-4b", + "shaohang/SparseOPT-1.3B", + "shaohang/Sparse0.5_OPT-1.3", + "EleutherAI/polyglot-ko-12.8b", + "Salesforce/codegen-6B-multi", + "bigscience/bloom-1b1", + "TFLai/gpt-neo-1.3B-4bit-alpaca", + "FabbriSimo01/Bloom_1b_Quantized", + "MBZUAI/LaMini-GPT-774M", + "Locutusque/gpt2-large-conversational", + "Devio/test-3b", + "stabilityai/stablelm-tuned-alpha-3b", + "PygmalionAI/pygmalion-1.3b", + "KoboldAI/fairseq-dense-355M", + "Rachneet/gpt2-xl-alpaca", + "gpt2-large", + "Mikivis/gpt2-large-lora-sft", + "stabilityai/stablelm-base-alpha-3b", + "gpt2-medium", + "Kunhao/pile-7b", + "aisquared/dlite-v1-774m", + "aisquared/dlite-v2-355m", + "YeungNLP/firefly-bloom-2b6-v2", + "KnutJaegersberg/gpt-2-xl-EvolInstruct", + "KnutJaegersberg/galactica-orca-wizardlm-1.3b", + "cerebras/Cerebras-GPT-1.3B", + "FabbriSimo01/Cerebras_1.3b_Quantized", + "facebook/xglm-1.7B", + "EleutherAI/pythia-410m-deduped", + "TheBloke/GPlatty-30B-SuperHOT-8K-fp16", + "DataLinguistic/DataLinguistic-34B-V1.0", + "Corianas/Quokka_1.3b", + "TheTravellingEngineer/bloom-560m-RLHF-v2", + "Corianas/1.3b", + "RWKV/rwkv-4-430m-pile", + "porkorbeef/Llama-2-13b-sf", + "xhyi/PT_GPTNEO350_ATG", + "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ", + "bigscience/bloomz-560m", + "TheBloke/medalpaca-13B-GPTQ-4bit", + "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16", + "aisquared/dlite-v1-355m", + "uukuguy/speechless-codellama-orca-airoboros-13b-0.10e", + "yhyhy3/med-orca-instruct-33b", + "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16", + "TheTravellingEngineer/bloom-1b1-RLHF", + "MBZUAI/lamini-cerebras-1.3b", + "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1", + "TheBloke/WizardLM-7B-uncensored-GPTQ", + "TheBloke/EverythingLM-13B-16K-GPTQ", + "quantumaikr/open_llama_7b_hf", + "TheBloke/chronos-wizardlm-uc-scot-st-13B-GPTQ", + "TheBloke/WizardLM-30B-Uncensored-GPTQ", + "IDEA-CCNL/Ziya-LLaMA-13B-v1", + "Phind/Phind-CodeLlama-34B-v1", + "robowaifudev/megatron-gpt2-345m", + "MayaPH/GodziLLa-30B-instruct", + "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16", + "uukuguy/speechless-codellama-orca-platypus-13b-0.10e", + "doas/test2", + "BreadAi/PM_modelV2", + "bigcode/santacoder", + "TheBloke/wizard-vicuna-13B-GPTQ", + "porkorbeef/Llama-2-13b", + "TehVenom/DiffMerge-DollyGPT-Pygmalion", + "PygmalionAI/pygmalion-350m", + "TheBloke/orca_mini_v3_7B-GPTQ", + "TheBloke/WizardLM-Uncensored-SuperCOT-StoryTelling-30B-GPTQ", + "TheBloke/WizardLM-30B-GPTQ", + "bigscience/bloom-560m", + "TFLai/gpt2-turkish-uncased", + "TheBloke/guanaco-33B-GPTQ", + "TheBloke/openchat_v2_openorca_preview-GPTQ", + "porkorbeef/Llama-2-13b-public", + "TheBloke/LongChat-13B-GPTQ", + "yhyhy3/med-orca-instruct-33b", + "TheBloke/airoboros-33B-gpt4-1-4-SuperHOT-8K-fp16", + "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16", + "MayaPH/FinOPT-Franklin", + "TheBloke/WizardLM-33B-V1.0-Uncensored-GPTQ", + "TheBloke/Project-Baize-v2-13B-GPTQ", + "malhajar/Platypus2-70B-instruct-4bit-gptq", + "KoboldAI/OPT-350M-Erebus", + "rishiraj/bloom-560m-guanaco", + "Panchovix/WizardLM-33B-V1.0-Uncensored-SuperHOT-8k", + "doas/test5", + "vicgalle/alpaca-7b", + "beomi/KoAlpaca-Polyglot-5.8B", + "Phind/Phind-CodeLlama-34B-Python-v1", + "timdettmers/guanaco-65b-merged", + "TheBloke/wizard-mega-13B-GPTQ", + "MayaPH/GodziLLa-30B-plus", + "TheBloke/Platypus-30B-SuperHOT-8K-fp16", + "facebook/opt-350m", + "KoboldAI/OPT-350M-Nerys-v2", + "TheBloke/robin-33B-v2-GPTQ", + "jaspercatapang/Echidna-30B", + "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16", + "marcchew/test1", + "Harshvir/LaMini-Neo-1.3B-Mental-Health_lora", + "golaxy/gogpt-560m", + "TheBloke/orca_mini_13B-GPTQ", + "Panchovix/airoboros-33b-gpt4-1.2-SuperHOT-8k", + "Aspik101/tulu-7b-instruct-pl-lora_unload", + "Phind/Phind-CodeLlama-34B-v2", + "BreadAi/MusePy-1-2", + "cerebras/Cerebras-GPT-590M", + "microsoft/CodeGPT-small-py", + "victor123/WizardLM-13B-1.0", + "OptimalScale/robin-65b-v2-delta", + "voidful/changpt-bart", + "FabbriSimo01/GPT_Large_Quantized", + "MayaPH/FinOPT-Lincoln", + "KoboldAI/fairseq-dense-125M", + "SebastianSchramm/Cerebras-GPT-111M-instruction", + "TheTravellingEngineer/bloom-560m-RLHF", + "breadlicker45/dough-instruct-base-001", + "WizardLM/WizardLM-30B-V1.0", + "WizardLM/WizardLM-30B-V1.0", + "WizardLM/WizardLM-30B-V1.0", + "TaylorAI/Flash-Llama-30M-20001", + "porkorbeef/Llama-2-13b-12_153950", + "huggingtweets/bladeecity-jerma985", + "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct", + "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16", + "microsoft/DialoGPT-small", + "Corianas/590m", + "facebook/xglm-564M", + "EleutherAI/gpt-neo-125m", + "EleutherAI/pythia-160m-deduped", + "klosax/pythia-160m-deduped-step92k-193bt", + "MBZUAI/lamini-neo-125m", + "bigcode/tiny_starcoder_py", + "concedo/OPT-19M-ChatSalad", + "anton-l/gpt-j-tiny-random", + "grantprice/Cerebras-GPT-590M-finetuned-DND", + "deepnight-research/zsc-text", + "WangZeJun/bloom-820m-chat", + "cerebras/Cerebras-GPT-256M", + "ai-forever/rugpt3large_based_on_gpt2", + "alibidaran/medical_transcription_generator", + "Deci/DeciCoder-1b", + "microsoft/DialoGPT-medium", + "ogimgio/gpt-neo-125m-neurallinguisticpioneers", + "open-llm-leaderboard/bloom-560m-4bit-alpaca-auto-eval-adapter-applied", + "BreadAi/gpt-YA-1-1_160M", + "microsoft/DialoGPT-large", + "facebook/opt-125m", + "huggingtweets/jerma985", + "Locutusque/gpt2-conversational-or-qa", + "concedo/Pythia-70M-ChatSalad", + "roneneldan/TinyStories-1M", + "BreadAi/DiscordPy", + "bigcode/gpt_bigcode-santacoder", + "Tincando/fiction_story_generator", + "klosax/pythia-70m-deduped-step44k-92bt", + "Quake24/easyTermsSummerizer", + "BreadAi/gpt-YA-1-1_70M", + "EleutherAI/pythia-160m", + "euclaise/gpt-neox-122m-minipile-digits", + "MBZUAI/lamini-cerebras-590m", + "nicholasKluge/Aira-124M", + "MayaPH/FinOPT-Washington", + "cyberagent/open-calm-large", + "BreadAi/StoryPy", + "EleutherAI/pythia-70m", + "BreadAi/gpt-Youtube", + "roneneldan/TinyStories-33M", + "EleutherAI/pythia-70m-deduped", + "lgaalves/gpt2_guanaco-dolly-platypus", + "Corianas/Quokka_590m", + "lgaalves/gpt2_platypus-dolly-guanaco", + "cyberagent/open-calm-7b", + "RWKV/rwkv-4-169m-pile", + "gpt2", + "roneneldan/TinyStories-28M", + "lgaalves/gpt2_open-platypus", + "gpt2", + "SaylorTwift/gpt2_test", + "roneneldan/TinyStories-3M", + "nthngdy/pythia-owt2-70m-50k", + "Corianas/256_5epoch", + "roneneldan/TinyStories-8M", + "lgaalves/gpt2-dolly", + "nthngdy/pythia-owt2-70m-100k", + "aisquared/dlite-v2-124m", + "mncai/SGPT-1.3B-insurance-epoch10", + "huggingtweets/gladosystem", + "abhiramtirumala/DialoGPT-sarcastic-medium", + "MBZUAI/lamini-cerebras-256m", + "cerebras/Cerebras-GPT-111M", + "uberkie/metharme-1.3b-finetuned", + "MBZUAI/lamini-cerebras-111m", + "psyche/kogpt", + "Corianas/Quokka_256m", + "vicgalle/gpt2-alpaca-gpt4", + "aisquared/dlite-v1-124m", + "Mikivis/xuanxuan", + "MBZUAI/LaMini-GPT-124M", + "vicgalle/gpt2-alpaca", + "huashiyiqike/testmodel", + "Corianas/111m", + "baseline", +] diff --git a/src/voting/vote_system.py b/src/voting/vote_system.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab75738159cc377a480c566393f872d615da898 --- /dev/null +++ b/src/voting/vote_system.py @@ -0,0 +1,155 @@ +import json +import logging +import pathlib +import pandas as pd +import gradio as gr +import schedule +import time +from datetime import datetime, timezone +from src.display.utils import EvalQueueColumn + +from src.envs import API + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class VoteManager: + def __init__(self, votes_path, eval_requests_path, repo_id): + self.votes_path = votes_path + self.eval_requests_path = eval_requests_path + self.repo_id = repo_id + self.vote_dataset = self.read_vote_dataset() + self.vote_check_set = self.make_check_set(self.vote_dataset) + self.votes_to_upload = [] + + def init_vote_dataset(self): + self.vote_dataset = self.read_vote_dataset() + self.vote_check_set = self.make_check_set(self.vote_dataset) + + def read_vote_dataset(self): + result = [] + votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl" + if votes_file.exists(): + with open(votes_file, "r") as f: + for line in f: + data = json.loads(line.strip()) + result.append(data) + result = pd.DataFrame(result) + return result + + def make_check_set(self, vote_dataset: pd.DataFrame): + result = list() + for row in vote_dataset.itertuples(index=False, name='vote'): + result.append((row.model, row.revision, row.username)) + return set(result) + + def get_model_revision(self, selected_model: str) -> str: + """Fetch the revision for the given model from the request files.""" + for user_folder in pathlib.Path(self.eval_requests_path).iterdir(): + if user_folder.is_dir(): + for file in user_folder.glob("*.json"): + with open(file, "r") as f: + data = json.load(f) + if data.get("model") == selected_model: + return data.get("revision", "main") + return "main" + + def create_request_vote_df(self, pending_models_df: gr.Dataframe): + if pending_models_df.empty or "model_name" not in pending_models_df.columns: + return pending_models_df + + self.vote_dataset = self.read_vote_dataset() + vote_counts = self.vote_dataset.groupby(['model', 'revision']).size().reset_index(name='vote_count') + + pending_models_df_votes = pd.merge( + pending_models_df, + vote_counts, + left_on=["model_name", 'revision'], + right_on=['model', 'revision'], + how='left' + ) + # Filling empty votes + pending_models_df_votes['vote_count'] = pending_models_df_votes['vote_count'].fillna(0) + pending_models_df_votes = pending_models_df_votes.sort_values(by=["vote_count", "model_name"], ascending=[False, True]) + # Removing useless columns + pending_models_df_votes = pending_models_df_votes.drop(["model_name", "model"], axis=1) + return pending_models_df_votes + + # Function to be called when a user votes for a model + def add_vote( + self, + selected_model: str, + pending_models_df: gr.Dataframe | None, + profile: gr.OAuthProfile | None + ): + logger.debug(f"Type of list before usage: {type(list)}") + # model_name, revision, user_id, timestamp + if selected_model in ["str", ""]: + gr.Warning("No model selected") + return + + if profile is None: + gr.Warning("Hub Login required") + return + + vote_username = profile.username + model_revision = self.get_model_revision(selected_model) + + # tuple (immutable) for checking than already voted for model + check_tuple = (selected_model, model_revision, vote_username) + if check_tuple in self.vote_check_set: + gr.Warning("Already voted for this model") + return + + current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + vote_obj = { + "model": selected_model, + "revision": model_revision, + "username": vote_username, + "timestamp": current_time + } + + # Append the vote to the JSONL file + try: + votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl" + with open(votes_file, "a") as f: + f.write(json.dumps(vote_obj) + "\n") + logger.info(f"Vote added locally: {vote_obj}") + + self.votes_to_upload.append(vote_obj) + except Exception as e: + logger.error(f"Failed to write vote to file: {e}") + gr.Warning("Failed to record vote. Please try again") + return + + self.vote_check_set.add(check_tuple) + gr.Info(f"Voted for {selected_model}") + + if pending_models_df is None: + return + + return self.create_request_vote_df(pending_models_df) + + def upload_votes(self): + if self.votes_to_upload: + votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl" + try: + with open(votes_file, "rb") as f: + API.upload_file( + path_or_fileobj=f, + path_in_repo="votes_data.jsonl", + repo_id=self.repo_id, + repo_type="dataset", + commit_message="Updating votes_data.jsonl with new votes", + ) + logger.info("Votes uploaded to votes repository") + self.votes_to_upload.clear() + except Exception as e: + logger.error(f"Failed to upload votes to repository: {e}") + +def run_scheduler(vote_manager): + while True: + schedule.run_pending() + time.sleep(1) diff --git a/tests/submission/test_user_submission_permission.py b/tests/submission/test_user_submission_permission.py new file mode 100644 index 0000000000000000000000000000000000000000..2f459f7b7b86282a922f6c22e4f6a79daec1bcde --- /dev/null +++ b/tests/submission/test_user_submission_permission.py @@ -0,0 +1,98 @@ +import unittest +from unittest.mock import patch +from datetime import datetime, timedelta, timezone + +from src.submission.check_validity import user_submission_permission +from src.envs import RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA + +class TestUserSubmissionPermission(unittest.TestCase): + + def setUp(self): + self.user_name = "test_user" + self.rate_limit_period = RATE_LIMIT_PERIOD + self.rate_limit_quota = RATE_LIMIT_QUOTA + self.fixed_now = datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc) + # Submission dates that simulate various test cases + self.users_to_submission_dates = { + "test_user": [ + (self.fixed_now - timedelta(days=1)).isoformat(), + (self.fixed_now - timedelta(days=2)).isoformat(), + (self.fixed_now - timedelta(days=3)).isoformat(), + (self.fixed_now - timedelta(days=4)).isoformat(), + ] + } + + @staticmethod + def fixed_datetime_now(tz=None): + return datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc) + + @patch('src.submission.check_validity.datetime') + def test_user_below_quota(self, mock_datetime): + mock_datetime.now.side_effect = self.fixed_datetime_now + mock_datetime.fromisoformat = datetime.fromisoformat + allowed, message = user_submission_permission( + self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota + ) + self.assertTrue(allowed) + + @patch('src.submission.check_validity.datetime') + def test_user_at_quota(self, mock_datetime): + mock_datetime.now.side_effect = self.fixed_datetime_now + mock_datetime.fromisoformat = datetime.fromisoformat + + # Add one more submission to reach the quota + self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat()) + + allowed, message = user_submission_permission( + self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota + ) + self.assertFalse(allowed) + expected_message = ( + f"Organisation or user `{self.user_name}` already has {self.rate_limit_quota} model requests submitted " + f"in the last {self.rate_limit_period} days.\n" + "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗" + ) + self.assertEqual(message, expected_message) + + @patch('src.submission.check_validity.datetime') + def test_user_above_quota(self, mock_datetime): + mock_datetime.now.side_effect = self.fixed_datetime_now + mock_datetime.fromisoformat = datetime.fromisoformat + # Add more than quota submissions + for _ in range(self.rate_limit_quota + 1): + self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat()) + allowed, message = user_submission_permission( + self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota + ) + self.assertFalse(allowed) + + def test_user_no_previous_submissions(self): + allowed, message = user_submission_permission( + "new_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota + ) + self.assertTrue(allowed) + + @patch('src.submission.check_validity.HAS_HIGHER_RATE_LIMIT', ["specific_user"]) + @patch('src.submission.check_validity.datetime') + def test_user_higher_rate_limit(self, mock_datetime): + mock_datetime.now.side_effect = self.fixed_datetime_now + mock_datetime.fromisoformat = datetime.fromisoformat + self.users_to_submission_dates["specific_user"] = [self.fixed_now.isoformat()] * (self.rate_limit_quota + 1) + allowed, message = user_submission_permission( + "specific_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota + ) + self.assertTrue(allowed) + + @patch('src.submission.check_validity.datetime') + def test_submission_just_outside_window(self, mock_datetime): + mock_datetime.now.side_effect = self.fixed_datetime_now + mock_datetime.fromisoformat = datetime.fromisoformat + old_submission = (self.fixed_now - timedelta(days=self.rate_limit_period, seconds=1)).isoformat() + self.users_to_submission_dates["test_user"] = [old_submission] + allowed, message = user_submission_permission( + self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota + ) + self.assertTrue(allowed) + +if __name__ == '__main__': + unittest.main()