Spaces:
Runtime error
Runtime error
| import os | |
| import time | |
| from pathlib import Path | |
| import pandas as pd | |
| import streamlit as st | |
| import yaml | |
| from datasets import get_dataset_config_names | |
| from dotenv import load_dotenv | |
| from huggingface_hub import list_datasets | |
| from evaluation import filter_evaluated_models | |
| from utils import ( | |
| AUTOTRAIN_TASK_TO_HUB_TASK, | |
| commit_evaluation_log, | |
| create_autotrain_project_name, | |
| format_col_mapping, | |
| get_compatible_models, | |
| get_config_metadata, | |
| get_dataset_card_url, | |
| get_key, | |
| get_metadata, | |
| http_get, | |
| http_post, | |
| ) | |
| if Path(".env").is_file(): | |
| load_dotenv(".env") | |
| HF_TOKEN = os.getenv("HF_TOKEN") | |
| AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME") | |
| AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API") | |
| DATASETS_PREVIEW_API = os.getenv("DATASETS_PREVIEW_API") | |
| # Put image tasks on top | |
| TASK_TO_ID = { | |
| "image_binary_classification": 17, | |
| "image_multi_class_classification": 18, | |
| "binary_classification": 1, | |
| "multi_class_classification": 2, | |
| "natural_language_inference": 22, | |
| "entity_extraction": 4, | |
| "extractive_question_answering": 5, | |
| "translation": 6, | |
| "summarization": 8, | |
| "text_zero_shot_classification": 23, | |
| } | |
| TASK_TO_DEFAULT_METRICS = { | |
| "binary_classification": ["f1", "precision", "recall", "auc", "accuracy"], | |
| "multi_class_classification": [ | |
| "f1", | |
| "precision", | |
| "recall", | |
| "accuracy", | |
| ], | |
| "natural_language_inference": ["f1", "precision", "recall", "auc", "accuracy"], | |
| "entity_extraction": ["precision", "recall", "f1", "accuracy"], | |
| "extractive_question_answering": ["f1", "exact_match"], | |
| "translation": ["sacrebleu"], | |
| "summarization": ["rouge1", "rouge2", "rougeL", "rougeLsum"], | |
| "image_binary_classification": ["f1", "precision", "recall", "auc", "accuracy"], | |
| "image_multi_class_classification": [ | |
| "f1", | |
| "precision", | |
| "recall", | |
| "accuracy", | |
| ], | |
| "text_zero_shot_classification": ["accuracy", "loss"], | |
| } | |
| AUTOTRAIN_TASK_TO_LANG = { | |
| "translation": "en2de", | |
| "image_binary_classification": "unk", | |
| "image_multi_class_classification": "unk", | |
| } | |
| AUTOTRAIN_MACHINE = {"text_zero_shot_classification": "r5.16x"} | |
| SUPPORTED_TASKS = list(TASK_TO_ID.keys()) | |
| # Extracted from utils.get_supported_metrics | |
| # Hardcoded for now due to speed / caching constraints | |
| SUPPORTED_METRICS = [ | |
| "accuracy", | |
| "bertscore", | |
| "bleu", | |
| "cer", | |
| "chrf", | |
| "code_eval", | |
| "comet", | |
| "competition_math", | |
| "coval", | |
| "cuad", | |
| "exact_match", | |
| "f1", | |
| "frugalscore", | |
| "google_bleu", | |
| "mae", | |
| "mahalanobis", | |
| "matthews_correlation", | |
| "mean_iou", | |
| "meteor", | |
| "mse", | |
| "pearsonr", | |
| "perplexity", | |
| "precision", | |
| "recall", | |
| "roc_auc", | |
| "rouge", | |
| "sacrebleu", | |
| "sari", | |
| "seqeval", | |
| "spearmanr", | |
| "squad", | |
| "squad_v2", | |
| "ter", | |
| "trec_eval", | |
| "wer", | |
| "wiki_split", | |
| "xnli", | |
| "angelina-wang/directional_bias_amplification", | |
| "jordyvl/ece", | |
| "lvwerra/ai4code", | |
| "lvwerra/amex", | |
| ] | |
| ####### | |
| # APP # | |
| ####### | |
| st.title("Evaluation on the Hub") | |
| st.warning( | |
| "**⚠️ This project has been archived. If you want to evaluate LLMs, checkout [this collection](https://huggingface.co/collections/clefourrier/llm-leaderboards-and-benchmarks-✨-64f99d2e11e92ca5568a7cce) of leaderboards.**" | |
| ) | |
| st.markdown( | |
| """ | |
| Welcome to Hugging Face's automatic model evaluator 👋! | |
| This application allows you to evaluate 🤗 Transformers | |
| [models](https://huggingface.co/models?library=transformers&sort=downloads) | |
| across a wide variety of [datasets](https://huggingface.co/datasets) on the | |
| Hub. Please select the dataset and configuration below. The results of your | |
| evaluation will be displayed on the [public | |
| leaderboards](https://huggingface.co/spaces/autoevaluate/leaderboards). For | |
| more details, check out out our [blog | |
| post](https://huggingface.co/blog/eval-on-the-hub). | |
| """ | |
| ) | |
| # all_datasets = [d.id for d in list_datasets()] | |
| # query_params = st.experimental_get_query_params() | |
| # if "first_query_params" not in st.session_state: | |
| # st.session_state.first_query_params = query_params | |
| # first_query_params = st.session_state.first_query_params | |
| # default_dataset = all_datasets[0] | |
| # if "dataset" in first_query_params: | |
| # if len(first_query_params["dataset"]) > 0 and first_query_params["dataset"][0] in all_datasets: | |
| # default_dataset = first_query_params["dataset"][0] | |
| # selected_dataset = st.selectbox( | |
| # "Select a dataset", | |
| # all_datasets, | |
| # index=all_datasets.index(default_dataset), | |
| # help="""Datasets with metadata can be evaluated with 1-click. Configure an evaluation job to add \ | |
| # new metadata to a dataset card.""", | |
| # ) | |
| # st.experimental_set_query_params(**{"dataset": [selected_dataset]}) | |
| # # Check if selected dataset can be streamed | |
| # is_valid_dataset = http_get( | |
| # path="/is-valid", | |
| # domain=DATASETS_PREVIEW_API, | |
| # params={"dataset": selected_dataset}, | |
| # ).json() | |
| # if is_valid_dataset["viewer"] is False and is_valid_dataset["preview"] is False: | |
| # st.error( | |
| # """The dataset you selected is not currently supported. Open a \ | |
| # [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) for support.""" | |
| # ) | |
| # metadata = get_metadata(selected_dataset, token=HF_TOKEN) | |
| # print(f"INFO -- Dataset metadata: {metadata}") | |
| # if metadata is None: | |
| # st.warning("No evaluation metadata found. Please configure the evaluation job below.") | |
| # with st.expander("Advanced configuration"): | |
| # # Select task | |
| # selected_task = st.selectbox( | |
| # "Select a task", | |
| # SUPPORTED_TASKS, | |
| # index=SUPPORTED_TASKS.index(metadata[0]["task_id"]) if metadata is not None else 0, | |
| # help="""Don't see your favourite task here? Open a \ | |
| # [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) to request it!""", | |
| # ) | |
| # # Select config | |
| # configs = get_dataset_config_names(selected_dataset) | |
| # selected_config = st.selectbox( | |
| # "Select a config", | |
| # configs, | |
| # help="""Some datasets contain several sub-datasets, known as _configurations_. \ | |
| # Select one to evaluate your models on. \ | |
| # See the [docs](https://huggingface.co/docs/datasets/master/en/load_hub#configurations) for more details. | |
| # """, | |
| # ) | |
| # # Some datasets have multiple metadata (one per config), so we grab the one associated with the selected config | |
| # config_metadata = get_config_metadata(selected_config, metadata) | |
| # print(f"INFO -- Config metadata: {config_metadata}") | |
| # # Select splits | |
| # splits_resp = http_get( | |
| # path="/splits", | |
| # domain=DATASETS_PREVIEW_API, | |
| # params={"dataset": selected_dataset}, | |
| # ) | |
| # if splits_resp.status_code == 200: | |
| # split_names = [] | |
| # all_splits = splits_resp.json() | |
| # for split in all_splits["splits"]: | |
| # if split["config"] == selected_config: | |
| # split_names.append(split["split"]) | |
| # if config_metadata is not None: | |
| # eval_split = config_metadata["splits"].get("eval_split", None) | |
| # else: | |
| # eval_split = None | |
| # selected_split = st.selectbox( | |
| # "Select a split", | |
| # split_names, | |
| # index=split_names.index(eval_split) if eval_split is not None else 0, | |
| # help="Be wary when evaluating models on the `train` split.", | |
| # ) | |
| # # Select columns | |
| # rows_resp = http_get( | |
| # path="/first-rows", | |
| # domain=DATASETS_PREVIEW_API, | |
| # params={ | |
| # "dataset": selected_dataset, | |
| # "config": selected_config, | |
| # "split": selected_split, | |
| # }, | |
| # ).json() | |
| # col_names = list(pd.json_normalize(rows_resp["rows"][0]["row"]).columns) | |
| # st.markdown("**Map your dataset columns**") | |
| # st.markdown( | |
| # """The model evaluator uses a standardised set of column names for the input examples and labels. \ | |
| # Please define the mapping between your dataset columns (right) and the standardised column names (left).""" | |
| # ) | |
| # col1, col2 = st.columns(2) | |
| # # TODO: find a better way to layout these items | |
| # # TODO: need graceful way of handling dataset <--> task mismatch for datasets with metadata | |
| # col_mapping = {} | |
| # if selected_task in ["binary_classification", "multi_class_classification"]: | |
| # with col1: | |
| # st.markdown("`text` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`target` column") | |
| # with col2: | |
| # text_col = st.selectbox( | |
| # "This column should contain the text to be classified", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "text")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # target_col = st.selectbox( | |
| # "This column should contain the labels associated with the text", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "target")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[text_col] = "text" | |
| # col_mapping[target_col] = "target" | |
| # elif selected_task == "text_zero_shot_classification": | |
| # with col1: | |
| # st.markdown("`text` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`classes` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`target` column") | |
| # with col2: | |
| # text_col = st.selectbox( | |
| # "This column should contain the text to be classified", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "text")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # classes_col = st.selectbox( | |
| # "This column should contain the classes associated with the text", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "classes")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # target_col = st.selectbox( | |
| # "This column should contain the index of the correct class", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "target")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[text_col] = "text" | |
| # col_mapping[classes_col] = "classes" | |
| # col_mapping[target_col] = "target" | |
| # if selected_task in ["natural_language_inference"]: | |
| # config_metadata = get_config_metadata(selected_config, metadata) | |
| # with col1: | |
| # st.markdown("`text1` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`text2` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`target` column") | |
| # with col2: | |
| # text1_col = st.selectbox( | |
| # "This column should contain the first text passage to be classified", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "text1")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # text2_col = st.selectbox( | |
| # "This column should contain the second text passage to be classified", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "text2")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # target_col = st.selectbox( | |
| # "This column should contain the labels associated with the text", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "target")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[text1_col] = "text1" | |
| # col_mapping[text2_col] = "text2" | |
| # col_mapping[target_col] = "target" | |
| # elif selected_task == "entity_extraction": | |
| # with col1: | |
| # st.markdown("`tokens` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`tags` column") | |
| # with col2: | |
| # tokens_col = st.selectbox( | |
| # "This column should contain the array of tokens to be classified", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "tokens")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # tags_col = st.selectbox( | |
| # "This column should contain the labels associated with each part of the text", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "tags")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[tokens_col] = "tokens" | |
| # col_mapping[tags_col] = "tags" | |
| # elif selected_task == "translation": | |
| # with col1: | |
| # st.markdown("`source` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`target` column") | |
| # with col2: | |
| # text_col = st.selectbox( | |
| # "This column should contain the text to be translated", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "source")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # target_col = st.selectbox( | |
| # "This column should contain the target translation", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "target")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[text_col] = "source" | |
| # col_mapping[target_col] = "target" | |
| # elif selected_task == "summarization": | |
| # with col1: | |
| # st.markdown("`text` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`target` column") | |
| # with col2: | |
| # text_col = st.selectbox( | |
| # "This column should contain the text to be summarized", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "text")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # target_col = st.selectbox( | |
| # "This column should contain the target summary", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "target")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[text_col] = "text" | |
| # col_mapping[target_col] = "target" | |
| # elif selected_task == "extractive_question_answering": | |
| # if config_metadata is not None: | |
| # col_mapping = config_metadata["col_mapping"] | |
| # # Hub YAML parser converts periods to hyphens, so we remap them here | |
| # col_mapping = format_col_mapping(col_mapping) | |
| # with col1: | |
| # st.markdown("`context` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`question` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`answers.text` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`answers.answer_start` column") | |
| # with col2: | |
| # context_col = st.selectbox( | |
| # "This column should contain the question's context", | |
| # col_names, | |
| # index=col_names.index(get_key(col_mapping, "context")) if config_metadata is not None else 0, | |
| # ) | |
| # question_col = st.selectbox( | |
| # "This column should contain the question to be answered, given the context", | |
| # col_names, | |
| # index=col_names.index(get_key(col_mapping, "question")) if config_metadata is not None else 0, | |
| # ) | |
| # answers_text_col = st.selectbox( | |
| # "This column should contain example answers to the question, extracted from the context", | |
| # col_names, | |
| # index=col_names.index(get_key(col_mapping, "answers.text")) if config_metadata is not None else 0, | |
| # ) | |
| # answers_start_col = st.selectbox( | |
| # "This column should contain the indices in the context of the first character of each `answers.text`", | |
| # col_names, | |
| # index=col_names.index(get_key(col_mapping, "answers.answer_start")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[context_col] = "context" | |
| # col_mapping[question_col] = "question" | |
| # col_mapping[answers_text_col] = "answers.text" | |
| # col_mapping[answers_start_col] = "answers.answer_start" | |
| # elif selected_task in ["image_binary_classification", "image_multi_class_classification"]: | |
| # with col1: | |
| # st.markdown("`image` column") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.text("") | |
| # st.markdown("`target` column") | |
| # with col2: | |
| # image_col = st.selectbox( | |
| # "This column should contain the images to be classified", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "image")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # target_col = st.selectbox( | |
| # "This column should contain the labels associated with the images", | |
| # col_names, | |
| # index=col_names.index(get_key(config_metadata["col_mapping"], "target")) | |
| # if config_metadata is not None | |
| # else 0, | |
| # ) | |
| # col_mapping[image_col] = "image" | |
| # col_mapping[target_col] = "target" | |
| # # Select metrics | |
| # st.markdown("**Select metrics**") | |
| # st.markdown("The following metrics will be computed") | |
| # html_string = " ".join( | |
| # [ | |
| # '<div style="padding-right:5px;padding-left:5px;padding-top:5px;padding-bottom:5px;float:left">' | |
| # + '<div style="background-color:#D3D3D3;border-radius:5px;display:inline-block;padding-right:5px;' | |
| # + 'padding-left:5px;color:white">' | |
| # + metric | |
| # + "</div></div>" | |
| # for metric in TASK_TO_DEFAULT_METRICS[selected_task] | |
| # ] | |
| # ) | |
| # st.markdown(html_string, unsafe_allow_html=True) | |
| # selected_metrics = st.multiselect( | |
| # "(Optional) Select additional metrics", | |
| # sorted(list(set(SUPPORTED_METRICS) - set(TASK_TO_DEFAULT_METRICS[selected_task]))), | |
| # help="""User-selected metrics will be computed with their default arguments. \ | |
| # For example, `f1` will report results for binary labels. \ | |
| # Check out the [available metrics](https://huggingface.co/metrics) for more details.""", | |
| # ) | |
| # with st.form(key="form"): | |
| # compatible_models = get_compatible_models(selected_task, [selected_dataset]) | |
| # selected_models = st.multiselect( | |
| # "Select the models you wish to evaluate", | |
| # compatible_models, | |
| # help="""Don't see your favourite model in this list? Add the dataset and task it was trained on to the \ | |
| # [model card metadata.](https://huggingface.co/docs/hub/models-cards#model-card-metadata)""", | |
| # ) | |
| # print("INFO -- Selected models before filter:", selected_models) | |
| # hf_username = st.text_input("Enter your 🤗 Hub username to be notified when the evaluation is finished") | |
| # submit_button = st.form_submit_button("Evaluate models 🚀") | |
| # if submit_button: | |
| # if len(hf_username) == 0: | |
| # st.warning("No 🤗 Hub username provided! Please enter your username and try again.") | |
| # elif len(selected_models) == 0: | |
| # st.warning("⚠️ No models were selected for evaluation! Please select at least one model and try again.") | |
| # elif len(selected_models) > 10: | |
| # st.warning("Only 10 models can be evaluated at once. Please select fewer models and try again.") | |
| # else: | |
| # # Filter out previously evaluated models | |
| # selected_models = filter_evaluated_models( | |
| # selected_models, | |
| # selected_task, | |
| # selected_dataset, | |
| # selected_config, | |
| # selected_split, | |
| # selected_metrics, | |
| # ) | |
| # print("INFO -- Selected models after filter:", selected_models) | |
| # if len(selected_models) > 0: | |
| # project_payload = { | |
| # "username": AUTOTRAIN_USERNAME, | |
| # "proj_name": create_autotrain_project_name(selected_dataset, selected_config), | |
| # "task": TASK_TO_ID[selected_task], | |
| # "config": { | |
| # "language": AUTOTRAIN_TASK_TO_LANG[selected_task] | |
| # if selected_task in AUTOTRAIN_TASK_TO_LANG | |
| # else "en", | |
| # "max_models": 5, | |
| # "instance": { | |
| # "provider": "sagemaker" if selected_task in AUTOTRAIN_MACHINE.keys() else "ovh", | |
| # "instance_type": AUTOTRAIN_MACHINE[selected_task] | |
| # if selected_task in AUTOTRAIN_MACHINE.keys() | |
| # else "p3", | |
| # "max_runtime_seconds": 172800, | |
| # "num_instances": 1, | |
| # "disk_size_gb": 200, | |
| # }, | |
| # "evaluation": { | |
| # "metrics": selected_metrics, | |
| # "models": selected_models, | |
| # "hf_username": hf_username, | |
| # }, | |
| # }, | |
| # } | |
| # print(f"INFO -- Payload: {project_payload}") | |
| # project_json_resp = http_post( | |
| # path="/projects/create", | |
| # payload=project_payload, | |
| # token=HF_TOKEN, | |
| # domain=AUTOTRAIN_BACKEND_API, | |
| # ).json() | |
| # print(f"INFO -- Project creation response: {project_json_resp}") | |
| # if project_json_resp["created"]: | |
| # data_payload = { | |
| # "split": 4, # use "auto" split choice in AutoTrain | |
| # "col_mapping": col_mapping, | |
| # "load_config": {"max_size_bytes": 0, "shuffle": False}, | |
| # "dataset_id": selected_dataset, | |
| # "dataset_config": selected_config, | |
| # "dataset_split": selected_split, | |
| # } | |
| # data_json_resp = http_post( | |
| # path=f"/projects/{project_json_resp['id']}/data/dataset", | |
| # payload=data_payload, | |
| # token=HF_TOKEN, | |
| # domain=AUTOTRAIN_BACKEND_API, | |
| # ).json() | |
| # print(f"INFO -- Dataset creation response: {data_json_resp}") | |
| # if data_json_resp["download_status"] == 1: | |
| # train_json_resp = http_post( | |
| # path=f"/projects/{project_json_resp['id']}/data/start_processing", | |
| # token=HF_TOKEN, | |
| # domain=AUTOTRAIN_BACKEND_API, | |
| # ).json() | |
| # # For local development we process and approve projects on-the-fly | |
| # if "localhost" in AUTOTRAIN_BACKEND_API: | |
| # with st.spinner("⏳ Waiting for data processing to complete ..."): | |
| # is_data_processing_success = False | |
| # while is_data_processing_success is not True: | |
| # project_status = http_get( | |
| # path=f"/projects/{project_json_resp['id']}", | |
| # token=HF_TOKEN, | |
| # domain=AUTOTRAIN_BACKEND_API, | |
| # ).json() | |
| # if project_status["status"] == 3: | |
| # is_data_processing_success = True | |
| # time.sleep(10) | |
| # # Approve training job | |
| # train_job_resp = http_post( | |
| # path=f"/projects/{project_json_resp['id']}/start_training", | |
| # token=HF_TOKEN, | |
| # domain=AUTOTRAIN_BACKEND_API, | |
| # ).json() | |
| # st.success("✅ Data processing and project approval complete - go forth and evaluate!") | |
| # else: | |
| # # Prod/staging submissions are evaluated in a cron job via run_evaluation_jobs.py | |
| # print(f"INFO -- AutoTrain job response: {train_json_resp}") | |
| # if train_json_resp["success"]: | |
| # train_eval_index = { | |
| # "train-eval-index": [ | |
| # { | |
| # "config": selected_config, | |
| # "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task], | |
| # "task_id": selected_task, | |
| # "splits": {"eval_split": selected_split}, | |
| # "col_mapping": col_mapping, | |
| # } | |
| # ] | |
| # } | |
| # selected_metadata = yaml.dump(train_eval_index, sort_keys=False) | |
| # dataset_card_url = get_dataset_card_url(selected_dataset) | |
| # st.success("✅ Successfully submitted evaluation job!") | |
| # st.markdown( | |
| # f""" | |
| # Evaluation can take up to 1 hour to complete, so grab a ☕️ or 🍵 while you wait: | |
| # * 🔔 A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications. | |
| # * 📊 Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged. | |
| # * 🥱 Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations: | |
| # """ # noqa | |
| # ) | |
| # st.markdown( | |
| # f""" | |
| # ```yaml | |
| # {selected_metadata} | |
| # """ | |
| # ) | |
| # print("INFO -- Pushing evaluation job logs to the Hub") | |
| # evaluation_log = {} | |
| # evaluation_log["project_id"] = project_json_resp["id"] | |
| # evaluation_log["autotrain_env"] = ( | |
| # "staging" if "staging" in AUTOTRAIN_BACKEND_API else "prod" | |
| # ) | |
| # evaluation_log["payload"] = project_payload | |
| # evaluation_log["project_creation_response"] = project_json_resp | |
| # evaluation_log["dataset_creation_response"] = data_json_resp | |
| # evaluation_log["autotrain_job_response"] = train_json_resp | |
| # commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN) | |
| # else: | |
| # st.error("🙈 Oh no, there was an error submitting your evaluation job!") | |
| # else: | |
| # st.warning("⚠️ No models left to evaluate! Please select other models and try again.") | |