update leaderboard

serve/leaderboard.py

@@ -95,7 +95,7 @@ def load_leaderboard_table_csv(filename, add_hyperlink=True):
     df = df.drop(df[df["Key"].isnull()].index)
     for col in df.columns:
         if "Elo rating" in col:
-            print(col, df[col], type(df[col]), df[col] is not np.NaN)
+            # print(col, df[col], type(df[col]), df[col] is not np.NaN)
             df[col] = df[col].apply(lambda x: int(x) if (x != "-" and pd.notna(x)) else np.NaN)
         
         if add_hyperlink and col == "Model":
@@ -174,10 +174,10 @@ def get_arena_table(arena_dfs, model_table_df):
         # elo rating
         num_battles = 0
         for dim in arena_dfs.keys():
-            try:
-                print(arena_dfs[dim].loc[model_name])
-            except:
-                continue
+            # try:
+            #     print(arena_dfs[dim].loc[model_name])
+            # except:
+            #     continue
             row.append(round(arena_dfs[dim].loc[model_name]["rating"], 2))
             upper_diff = round(arena_dfs[dim].loc[model_name]["rating_q975"] - arena_dfs[dim].loc[model_name]["rating"])
             lower_diff = round(arena_dfs[dim].loc[model_name]["rating"] - arena_dfs[dim].loc[model_name]["rating_q025"])
@@ -215,25 +215,26 @@ def make_arena_leaderboard_md(elo_results):
         total_models = len(arena_df)
 
     leaderboard_md = f"""
-
-
-Total #models: **{total_models}**(anonymous). Total #votes: **{int(total_votes)}**. Last updated: {last_updated}. \n
-(Note: Only anonymous votes are considered here.)
-
+Total #models: **{total_models}**. \n
+Total #votes: **{int(total_votes)}** (Anonymous Votes only). \n
+Last updated: {last_updated}. \n
 Contribute the votes 🗳️ at [3DGen-Arena](https://huggingface.co/spaces/ZhangYuhan/3DGen-Arena)! 
-
 """
     return leaderboard_md
 
 def make_full_leaderboard_md(elo_results):
-    arena_df = elo_results["leaderboard_table_df"]
-    last_updated = elo_results["last_updated_datetime"]
-    total_votes = sum(arena_df["num_battles"]) // 2
-    total_models = len(arena_df)
+    total_votes = 0
+    for dim in elo_results.keys():
+        arena_df = elo_results[dim]["leaderboard_table_df"]
+        last_updated = elo_results[dim]["last_updated_datetime"]
+        total_votes += sum(arena_df["num_battles"].fillna(0)) // 2
+        total_models = len(arena_df)
 
     leaderboard_md = f"""
-Total #models: **{total_models}**(full:anonymous+open). Total #votes: **{total_votes}**. Last updated: {last_updated}.
-
+Total #models: **{total_models}**. \n
+Total #votes: **{int(total_votes)}** (Anonymous + Named Votes). \n
+Last updated: {last_updated}.\n
+Contribute the votes 🗳️ at [3DGen-Arena](https://huggingface.co/spaces/ZhangYuhan/3DGen-Arena)! 
 """
     return leaderboard_md
 
@@ -251,7 +252,7 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
     else:
         with open(elo_results_file, "rb") as fin:
             elo_results = pickle.load(fin)
-
+            
         # print(elo_results)
         # print(elo_results.keys())
         anony_elo_results, full_elo_results = {}, {}
@@ -266,6 +267,8 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
             p2[dim] = anony_elo_results[dim]["battle_count_heatmap"]
             p3[dim] = anony_elo_results[dim]["bootstrap_elo_rating"]
             p4[dim] = anony_elo_results[dim]["average_win_rate_bar"]
+            print(anony_arena_dfs[dim])
+            print(full_arena_dfs[dim])
         
         md = make_leaderboard_md(anony_elo_results)
         
@@ -273,10 +276,11 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
 
     if leaderboard_table_file:
         model_table_df = load_leaderboard_table_csv(leaderboard_table_file)
+        model_table_df_full = load_leaderboard_table_csv(str(leaderboard_table_file).replace('.csv', '_full.csv'))
         with gr.Tabs() as tabs:
             # arena table
             arena_table_vals = get_arena_table(anony_arena_dfs, model_table_df)
-            with gr.Tab("Arena Elo", id=0):
+            with gr.Tab("Anony. Arena", id=0):
                 md = make_arena_leaderboard_md(anony_elo_results)
                 gr.Markdown(md, elem_id="leaderboard_markdown")
                 gr.Dataframe(
@@ -308,25 +312,29 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                     column_widths=[50, 200, 100, 100, 100, 100, 100, 100, 100],
                     wrap=True,
                 )
-            # with gr.Tab("Full Leaderboard", id=1):
-            #     md = make_full_leaderboard_md(full_elo_results)
-            #     gr.Markdown(md, elem_id="leaderboard_markdown")
-            #     full_table_vals = get_full_table(anony_arena_df, full_arena_df, model_table_df)
-            #     gr.Dataframe(
-            #         headers=[
-            #             "🤖 Model",
-            #             "⭐ Arena Elo (anony)",
-            #             "⭐ Arena Elo (full)",
-            #             "Organization",
-            #             "License",
-            #         ],
-            #         datatype=["markdown", "number", "number", "str", "str"],
-            #         value=full_table_vals,
-            #         elem_id="full_leaderboard_dataframe",
-            #         column_widths=[200, 100, 100, 100, 150, 150],
-            #         height=700,
-            #         wrap=True,
-            #     )
+            with gr.Tab("Full Arena", id=1):
+                md = make_full_leaderboard_md(full_elo_results)
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                full_table_vals = get_arena_table(full_arena_dfs, model_table_df_full)
+                gr.Dataframe(
+                    headers=["Rank", "🤖 Model"] + [f"📈 {dim} Elo" for dim in anony_arena_dfs.keys()] + ["⭐ Avg. Arena Elo Ranking", "📮 Votes"],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number"
+                    ],
+                    value=full_table_vals,
+                    elem_id="full_leaderboard_dataframe",
+                    column_widths=[50, 200, 100, 100, 100, 100, 100, 100, 100],
+                    height=700,
+                    wrap=True,
+                )
         if not show_plot:
             gr.Markdown(
                 """ ## We are still collecting more votes on more models. The ranking will be updated very fruquently. Please stay tuned!