Ranks

app.py

@@ -18,13 +18,14 @@ import gradio as gr
 
 from huggingface_hub import HfApi, snapshot_download
 
+from compare_significance import check_significance, SUPPORTED_METRICS
+from model_compare import ModelCompare
 
 JSON_DATASET_DIR = Path("../json_dataset")
 JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
 
 JSON_DATASET_PATH = JSON_DATASET_DIR / f"train-{uuid4()}.json"
 
-
 api = HfApi()
 
 ORG= "CZLC"
@@ -38,28 +39,81 @@ DATASET_VERSIONS = ['dev-set-1', 'dev-set-2']
 
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
-
-
 class LeaderboardServer:
     def __init__(self, server_address):
         self.server_address = server_address
         self.repo_type = "dataset"
-        self.local_leaderboard = snapshot_download(self.server_address,repo_type=self.repo_type, token=HF_TOKEN,local_dir = "./")
-        print(self.local_leaderboard)
+        self.local_leaderboard = snapshot_download(self.server_address, repo_type=self.repo_type, token=HF_TOKEN,local_dir = "./")
+        self.submisssion_id_to_file = {} # Map submission ids to file paths    
+    
     def on_submit(self):
         self.local_leaderboard = snapshot_download(self.server_address,repo_type=self.repo_type, token=HF_TOKEN,local_dir = "./")
 
     def get_leaderboard(self):
         results = []
-        print(os.listdir(self.local_leaderboard))
+        
+        new_results = []
+        submission_ids = set()
+                
+        # pre-computed ranks  
+        with open(os.path.join(self.local_leaderboard, "metadata", "ranks.json")) as ranks_file:
+            ranks = json.load(ranks_file)      
+        model_compare = ModelCompare()
+        ranks = model_compare.get_tasks_ranks(ranks)
+        
+        # Models data
         for submission in glob.glob(os.path.join(self.local_leaderboard, "data") + "/*.json"):
             data = json.load(open(submission))
             submission_id = data["metadata"]["model_description"]
-            local_results = {group: data["results"][group]['acc'] for group in data['results']}
+            
+            if submission_id in submission_ids:
+                continue
+            submission_ids.add(submission_id)
+            
+            self.submisssion_id_to_file[submission_id] = submission
+       
+            
+            local_results = {task: list(task_ranks).index(submission_id)+1 for task, task_ranks in ranks.items()}
             local_results["submission_id"] = submission_id
             results.append(local_results)
         dataframe = pd.DataFrame.from_records(results)
+        # Reorder to have the id (model description) first 
+        df_order = ["submission_id"] + [col for col in dataframe.columns if col != "submission_id"]
+        dataframe = dataframe[df_order] 
         return dataframe
+    
+    def compute_ranks(self):
+        ''' Compute rankings on every submit '''
+        
+        self.get_leaderboard()
+
+        ids = list(self.submisssion_id_to_file.keys())
+        rankings = {id: {} for id in ids}
+
+        for a_idx in range(len(ids)):
+            for b_idx in range(a_idx+1, len(ids)):
+                modelA_id = ids[a_idx]
+                modelB_id = ids[b_idx]
+                res = self.compare_models(modelA_id, modelB_id)
+                rankings[modelA_id][modelB_id] =  {
+                    task: data["significant"] for task,data in res.items()
+                }
+                rankings[modelB_id][modelA_id] =  {
+                    task: not data["significant"] for task,data in res.items()
+                }
+                
+        return rankings
+        
+    
+    def compare_models(self, modelA, modelB):
+        modelA_path = self.submisssion_id_to_file.get(modelA)
+        modelB_path = self.submisssion_id_to_file.get(modelB)
+        return check_significance(modelA_path, modelB_path)
+        
+        
+    def get_rankings(self):
+        # TODO retrieve saved rankings for models on tasks
+        pass
 
     def save_json(self,file, submission_name) -> None:
         filename = os.path.basename(file)
@@ -72,11 +126,7 @@ class LeaderboardServer:
         )
 
 
-
-
 leaderboard_server =  LeaderboardServer(REPO)
-
-
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
 
@@ -86,6 +136,8 @@ MAX_SUBMISSIONS_PER_24H = 2
 # CHALLENGE_NAME = 'NOTSOFAR1'
 
 
+
+
 # if __name__ == '__main__':
 with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility: hidden}") as main):
     app_state = gr.State({})
@@ -136,9 +188,9 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
             return gr.Tabs(selected=first_tab_name), populate_leaderboard(first_tab_name, None)
 
 
-        with gr.Tab('Leaderboards') as leaderboards_tab:
-            with gr.Row():
-                gr.Markdown(LEADERBOARD_TAB_TITLE_MARKDOWN)
+        with gr.Tab('Leaderboard') as leaderboards_tab:
+            # with gr.Row():
+            #     gr.Markdown(LEADERBOARD_TAB_TITLE_MARKDOWN)
             # with gr.Row():
             #     with gr.Column():
             #         dataset_version_drop = gr.Dropdown(choices=DATASET_VERSIONS, multiselect=False,
@@ -150,17 +202,35 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
             #         gr.Markdown('')  # Empty column for spacing
             #     with gr.Column():
             #         gr.Markdown('')  # Empty column for spacing
-            with gr.Row():
-                with gr.Tabs() as leaderboards_tabs:
-                    leaderboard_tables_list = []
-                    for leaderboard_idx, leaderboard_type in enumerate(LEADERBOARD_TYPES):
-                        l_tab = create_leaderboard_tab(leaderboard_type, leaderboard_idx, None)
-                        leaderboard_tables_list.append(l_tab)
+            # with gr.Row():
+            #     with gr.Tabs() as leaderboards_tabs:
+            #         leaderboard_tables_list = []
+            #         for leaderboard_idx, leaderboard_type in enumerate(LEADERBOARD_TYPES):
+            #             l_tab = create_leaderboard_tab(leaderboard_type, leaderboard_idx, None)
+            #             leaderboard_tables_list.append(l_tab)
+            
+            # change the table based on the selected model
+            def on_dropdown_change(model_detail):
+                leaderboard = leaderboard_server.get_leaderboard()
+                return leaderboard[leaderboard["submission_id"] == model_detail]
+
+            results_table = gr.DataFrame(leaderboard_server.get_leaderboard(), interactive=False, label=None, visible=True)
+            model_detail = gr.Dropdown(choices=list(leaderboard_server.get_leaderboard()["submission_id"]), label="Select model", interactive=True)
+            model_detail_button = gr.Button("Show model detail", interactive=True)
+            model_detail_button.click(
+                fn=on_dropdown_change,
+                inputs=[model_detail],
+                outputs=[results_table]
+            )
+        
+            # results_table.select(fn=on_dropdown_change, inputs=[model_detail], outputs=[results_table])
 
             # dataset_version_drop.select(fn=on_dropdown_change, inputs=[dataset_version_drop],
             #                             outputs=[leaderboards_tabs, leaderboard_tables_list[0]])
 
+   
 
+        ##################
         # Submission Tab #
         ##################
         with gr.Tab('Submission'):
@@ -217,30 +287,57 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css="footer {visibility
                     # leaderboard_tab.render()
                     return gr.update(value='Submit', interactive=True)
 
-                gr.Markdown(SUBMISSION_TAB_TITLE_MARKDOWN)
+                def show_leaderboard():
+                    gr.Info("Loding leaderboard...")
+                    return leaderboard_server.get_leaderboard()
+
+                gr.Markdown(
+                    """
+                    # Model submission
+                    Model can be compared with other models and submitted\n
+                    Click **Compare results** to compare your model with other models in the leaderboard\n
+                    Click **Submit results** to submit your model to the leaderboard
+                    (Comparison by itself is not a submission)
+                    """
+                )
+                
                 submission_team_name_tb = gr.Textbox(label='Team Name')
-                submission_file_path = gr.File(label='Upload your results', type='filepath')
-                submission_type_radio = gr.Radio(label='Submission Track', choices=LEADERBOARD_TYPES)
+                # submission_type_radio = gr.Radio(label='Submission Track', choices=LEADERBOARD_TYPES)
+                with gr.Row():
+                    description_tb = gr.Textbox(label='Description', type='text')
+                    link_to_model_tb = gr.Textbox(label='Link to model', type='text')
+
                 with gr.Row():
                     hf_token_tb = gr.Textbox(label='Token', type='password')
                     submissions_24h_txt = gr.Textbox(label='Submissions 24h', value='')
-                description_tb = gr.Textbox(label='Description', type='text')
-                submission_btn = gr.Button(value='Submit', interactive=True)
-
+         
+                submission_file_path = gr.File(label='Upload your results', type='filepath')
+                compare_results_button = gr.DataFrame(show_leaderboard(), interactive=False, label=None, visible=True)
+                
+                # Button that triggers shows the current leaderboard
+                show_results_button = gr.Button("Compare results", interactive=True)
+                show_results_button.click(
+                    fn=show_leaderboard,
+                    outputs=[compare_results_button]
+                )
+                
+                submission_btn = gr.Button(value='Submit results', interactive=True)
                 submission_btn.click(
                     fn=on_submit_pressed,
                     outputs=[submission_btn]
                 ).then(
                     fn=process_submission,
-                    inputs=[submission_team_name_tb, submission_file_path,
-                            submission_type_radio, description_tb, app_state]
+                    inputs=[submission_team_name_tb, submission_file_path, description_tb, app_state]
                 ).then(
                     fn=on_submit_done,
                     outputs=[submission_btn]
-                ).then(
-                    fn=on_dropdown_change,
-                                    outputs=[leaderboards_tabs, leaderboard_tables_list[0]]
                 )
+                
+                # .then(
+                #     fn=on_dropdown_change,
+                #                     outputs=[leaderboards_tabs, leaderboard_tables_list[0]]
+                # )
+                      
 
         # # My Submissions Tab #
         # ######################

compare_significance.py

@@ -0,0 +1,231 @@
+import argparse
+import json
+from collections import defaultdict
+from typing import Sequence
+
+import numpy
+import numpy as np
+from scipy.stats import ttest_ind, ttest_rel
+from sklearn.metrics import roc_curve, auc
+from tqdm import tqdm
+
+# from leaderboard import SUPPORTED_METRICS
+
+SUPPORTED_METRICS = [
+    "avg_mcauroc",  # for classification tasks
+    "em",  # for QA tasks
+    "acc",  # for multichoice tasks
+    "rouge",  # for summarization tasks
+    "ppl",  # for language modeling tasks
+]
+
+
+def _get_CMs(i, probabilities, references, thresholds):
+    confusion_matrices = []
+    for threshold in thresholds[i]:
+        TP = 0
+        FP = 0
+        TN = 0
+        FN = 0
+        for j in range(len(probabilities)):
+            if probabilities[j][i] >= threshold:
+                if references[j] == i:
+                    TP += 1
+                else:
+                    FP += 1
+            else:
+                if references[j] == i:
+                    FN += 1
+                else:
+                    TN += 1
+        cm = {"TP": TP, "FP": FP, "TN": TN, "FN": FN, "threshold": threshold, "class": i}
+        confusion_matrices.append(cm)
+
+    return confusion_matrices
+
+
+def compute_significance_accuracy(predsA, referencesA, predsB, referencesB):
+    # following https://github.com/rtmdrr/testSignificanceNLP/blob/c7302d015538944364b622eb860dd9fbee6d50ec/testSignificance.py#L164C32-L165C24
+    # Calculate the T-test on TWO RELATED samples of scores, a and b. for one sided test we multiply p-value by half
+    scores_A  = [1 if pred == ref else 0 for pred, ref in zip(predsA, referencesA)]
+    scores_B  = [1 if pred == ref else 0 for pred, ref in zip(predsB, referencesB)]
+    t, p = ttest_rel(scores_A, scores_B)
+    # correct for one-tailed test
+    p_value = p / 2
+    delta = np.mean(scores_A) - np.mean(scores_B)
+    return p_value, delta
+
+def compute_significance_em(predsA, referencesA, predsB, referencesB):
+    pass
+def compute_significance_rouge(predsA, referencesA, predsB, referencesB):
+    # TODO: MDocekal
+    # Use bootstrapping
+    # https://github.com/rtmdrr/testSignificanceNLP/blob/c7302d015538944364b622eb860dd9fbee6d50ec/testSignificance.py#L89
+    pass
+def compute_significance_ppl(predsA, referencesA, predsB, referencesB):
+    # TODO: MDocekal
+    # Use bootstrapping
+    # https://github.com/rtmdrr/testSignificanceNLP/blob/c7302d015538944364b622eb860dd9fbee6d50ec/testSignificance.py#L89
+    pass
+def compute_significance_avg_mcauroc(probsA: Sequence[Sequence[float]], referencesA: Sequence[int],
+                                     probsB: Sequence[Sequence[float]], referencesB: Sequence[int]):
+    # compute MC-AUC for model A
+    model_A_scores = get_mc_auc_samples(probsA, referencesA, Nsamples=1_000)
+    model_B_scores = get_mc_auc_samples(probsB, referencesB, Nsamples=1_000)
+
+    # one-tailed test
+    p_value = ((model_A_scores[:, np.newaxis] <= model_B_scores[np.newaxis, :]).sum()
+               / (len(model_A_scores) * len(model_B_scores)))
+
+    delta = np.mean(model_A_scores) - np.mean(model_B_scores)
+    return p_value, delta
+
+
+def get_mc_auc_samples(probs, references, Nsamples=1_000_000):
+    n_classes = list(range(len(probs[0])))
+    fpr = dict()
+    thresholds = dict()
+    # compute AUC for every class
+    auc_scores_per_class = []
+    for i in range(len(n_classes)):
+        # for i-th class vs all others
+        fpr[i], _, thresholds[i] = roc_curve(y_true=[1 if x == n_classes[i] else 0 for x in references],
+                                             y_score=[prob[i] for prob in probs])
+
+        confusion_matrices = _get_CMs(i, probs, references, thresholds)
+
+        λ = 1.0  # <- Flat prior
+        # λ = 0.5  # <- Jeffrey's prior
+
+        # sample variates for every threshold
+        tpr_variates_for_each_fpr = []
+        for k in range(len(thresholds[i])):
+            tpr_variates_for_each_fpr.append(
+                numpy.random.beta(confusion_matrices[k]["TP"] + λ, confusion_matrices[k]["FN"] + λ, Nsamples))
+
+        # fprs x tpr_variates
+        tpr_variates_for_each_fpr = np.array(tpr_variates_for_each_fpr)
+
+        # now pick 1 variate for each fpr, and compute AUC
+        auc_scores = []
+        for tpr_variates in tqdm(tpr_variates_for_each_fpr.T,
+                                 desc=f"Computing AUCs for class {i + 1}/{len(n_classes)}"):
+            auc_score = auc(fpr[i], tpr_variates)
+            # if numpy.isnan(auc_score):
+            #     auc_score = 0
+            auc_scores.append(auc_score)
+        auc_scores_per_class.append(auc_scores)
+
+    auc_scores_per_class = np.array(auc_scores_per_class)
+    mcauc_scores = np.mean(auc_scores_per_class, axis=0)
+    return mcauc_scores
+
+
+def read_json(file_path):
+    data = defaultdict(list)
+    with open(file_path, "r") as f:
+        fc = json.load(f)
+    for task, results in fc["predictions"].items():
+        # determine the metric
+        metric = None
+        for key in SUPPORTED_METRICS:
+            if key in results[0]:
+                metric = key
+                break
+        if metric is None:
+            raise ValueError(f"Unsupported metric in {file_path}")
+
+        if metric == "avg_mcauroc":
+            local_data = [line[metric] for line in fc["predictions"][task]]
+            unzipped_list = list(zip(*local_data))
+            golds = unzipped_list[0]
+            probs = unzipped_list[1]
+            data[task] = (golds, probs), metric
+    return data, fc["metadata"]
+
+
+def check_significance_task(fileA, fileB, task, significance_level=0.05):
+    
+    dataA, metadataA = read_json(fileA)
+    dataB, metadataB = read_json(fileB)
+    
+    print("DEBUG",fileA, task,  dataA[task])
+
+    decisions = dict()
+    
+    metricA = dataA[task][1]
+    metricB = dataB[task][1]
+    assert metricA == metricB
+    assert len(dataA[task]) == len(dataB[task])
+
+    if metricA == "avg_mcauroc":
+        p_value, delta = compute_significance_avg_mcauroc(probsA=dataA[task][0][1], referencesA=dataA[task][0][0],
+                                                            probsB=dataB[task][0][1], referencesB=dataB[task][0][0])
+
+    elif metricA == "acc":
+        p_value, delta = compute_significance_accuracy(predsA=dataA[task][0][1], referencesA=dataA[task][0][0],
+                                                        predsB=dataB[task][0][1], referencesB=dataB[task][0][0])
+    elif metricA == "em":
+        raise NotImplementedError("Exact match is not supported yet.")
+    elif metricA == "rouge":
+        raise NotImplementedError("Rouge is not supported yet.")
+    elif metricA == "ppl":
+        raise NotImplementedError("Perplexity is not supported yet.")
+    else:
+        raise ValueError(f"Unsupported metric {metricA}")
+    decisions[task] = {
+        "significant": not (p_value > significance_level),
+        "p_value": p_value,
+        "delta": delta,
+    }
+    return decisions
+
+def check_significance(fileA, fileB, significance_level=0.05):
+    dataA, metadataA = read_json(fileA)
+    dataB, metadataB = read_json(fileB)
+    
+    decisions = dict()
+    for task in dataA.keys():
+        metricA = dataA[task][1]
+        metricB = dataB[task][1]
+        assert metricA == metricB
+        assert len(dataA[task]) == len(dataB[task])
+
+        if metricA == "avg_mcauroc":
+            p_value, delta = compute_significance_avg_mcauroc(probsA=dataA[task][0][1], referencesA=dataA[task][0][0],
+                                                              probsB=dataB[task][0][1], referencesB=dataB[task][0][0])
+
+        elif metricA == "acc":
+            p_value, delta = compute_significance_accuracy(predsA=dataA[task][0][1], referencesA=dataA[task][0][0],
+                                                           predsB=dataB[task][0][1], referencesB=dataB[task][0][0])
+        elif metricA == "em":
+            raise NotImplementedError("Exact match is not supported yet.")
+        elif metricA == "rouge":
+            raise NotImplementedError("Rouge is not supported yet.")
+        elif metricA == "ppl":
+            raise NotImplementedError("Perplexity is not supported yet.")
+        else:
+            raise ValueError(f"Unsupported metric {metricA}")
+        decisions[task] = {
+            "significant": not (p_value > significance_level),
+            "p_value": p_value,
+            "delta": delta,
+        }
+    return decisions
+
+
+def main():
+    parser = argparse.ArgumentParser(description="One-tailed test if model A improves over model B.")
+    parser.add_argument("--modelA", help="ModelA JSONL file from lm harness.")
+    parser.add_argument("--modelB", help="ModelB JSONL file from lm harness.")
+    parser.add_argument("--significance_level", type=float, default=0.05, help="Significance level (e.g., 0.05)")
+    args = parser.parse_args()
+
+    result = check_significance(args.modelA, args.modelB, args.significance_level)
+    print(json.dumps(result, indent=2))
+
+# harness already returns stderr estimate for sampling distribution
+# see https://github.com/EleutherAI/lm-evaluation-harness/blob/6433bd3fe3033d302b22cdcd53af237e9039ef29/lm_eval/api/metrics.py#L213
+
+if __name__ == "__main__":
+    main()

model_compare.py

@@ -0,0 +1,62 @@
+
+from functools import cmp_to_key
+from compare_significance import check_significance
+
+class ModelCompare():
+
+    TASKS =  ["propaganda_demonizace",
+                "propaganda_vina",
+                "propaganda_relativizace",
+                "propaganda_argumentace",
+                "propaganda_lokace",
+                "propaganda_nazor",
+                "propaganda_emoce",
+                "propaganda_fabulace",
+                "propaganda_nalepkovani",
+                "propaganda_zamereni",
+                "propaganda_zanr",
+                "propaganda_rusko",
+                "propaganda_strach",
+                "benczechmark_sentiment"]
+
+    def __init__(self, ranks:dict=None):
+        self.ranks = ranks
+
+    def compare_models(self, modelA_id, modelB_id):
+        if not self.ranks:
+            raise Exception("Missing model rankings")
+        
+        res = self.ranks[modelA_id][modelB_id][self.current_task]
+        if res == True:   
+            return 1
+        elif res == False:
+            return -1
+        else:
+            return -1
+
+
+    def get_tasks_ranks(self, ranks:dict) -> dict:
+        '''Order models based on the significance improvement'''
+        
+        self.ranks = ranks
+        
+        tasks_ranks = {}
+        
+        models = ranks.keys()
+        for task in self.TASKS:
+            self.current_task = task
+            tasks_ranks[task] = sorted(models, key=cmp_to_key(self.compare_models))
+        return tasks_ranks
+        
+    
+# models = {
+# model1 : {
+#         task1 : order_idx
+#         task2 : order_idx
+#         task3 : order_idx
+#     }
+# }
+    
+    
+    
+    

requirements.txt

@@ -4,4 +4,7 @@ azure-cosmos
 huggingface_hub
 requests
 Pyarrow
-tabulate
+tabulate
+scipy
+numpy
+scikit-learn