update

app.py

@@ -21,6 +21,7 @@ from huggingface_hub import HfApi
 # from datasets import Dataset, load_dataset, concatenate_datasets
 import os, uuid 
 from utils_display import model_info
+from tqdm import tqdm
 
 # get the last updated time from the elo_ranks.all.jsonl file
 LAST_UPDATED = None 
@@ -43,14 +44,19 @@ dataset = datasets.load_dataset("DongfuJiang/WildFeedback", "feedbacks", split='
 import random
 random.seed(42)
 np.random.seed(42)
-def sample_an_feedback():
-    feedback = dataset[np.random.randint(0, len(dataset))]
-
-    two_model_responses = random.sample(feedback['responses'], 2)
-    model_response_1 = two_model_responses[0]
-    model_response_2 = two_model_responses[1]
+def sample_an_feedback(selected_models):
+    shuffled_dataset = dataset.shuffle(seed=42)
+    feedback = None
+    for example in tqdm(shuffled_dataset, total=len(shuffled_dataset), desc="Searching for valid examples"):
+        example_model_responses = example['responses']
+        valid_model_responses = [model_response for model_response in example_model_responses if model_response['model'] in selected_models]
+        if len(valid_model_responses) >= 2:
+            feedback = example
+            model_response_1, model_response_2 = random.sample(valid_model_responses, 2)
+            break
+    if not feedback:
+        return gr.Exit("No valid examples found. Please select other models.")
     
-
     plan_history = {
         "user": [
             feedback['query'],
@@ -79,16 +85,18 @@ def sample_an_feedback():
         "task_type": feedback['source'],
         "plan_history": plan_history,
         "ground_history": ground_history,
-        "pred": str(model_response_1['feedback']['processed']['score']) if model_response_1['feedback']['processed'] else "A",
-        "answer": str(model_response_2['feedback']['processed']['score']) if model_response_2['feedback']['processed'] else "A",
-        "correctness": "Correct",
+        # "pred": str(model_response_1['feedback']['processed']['score']) if model_response_1['feedback']['processed'] else "A",
+        # "answer": str(model_response_2['feedback']['processed']['score']) if model_response_2['feedback']['processed'] else "A",
+        "pred": str(model_response_1['model']),
+        "answer": str(model_response_2['model']),
+        "correctness": "GPT-4o-mini",
         "image": "file/data_dir/test_images/000000341196.jpg"
     }
     return result_dict
   
 def display_chat_history(model_selections):
     # eval_item = sample_an_eval_result(eval_results, model_selections)
-    eval_item = sample_an_feedback()
+    eval_item = sample_an_feedback(model_selections)
     print("---" * 10)
     for key, value in eval_item.items():
         print(f"{key}: {value}")
@@ -176,7 +184,7 @@ def build_demo(TYPES):
                             # clear the selected_models
                             clear_button.click(lambda: {selected_models: {"value": [], "__type__": "update"}}, inputs=[], outputs=[selected_models])
                 
-                with gr.Row():
+                with gr.Row(visible=False):
                     with gr.Column(scale=1.5):
                         with gr.Accordion("📝 Task Description", open=True, elem_classes="accordion-label"):
                             task = gr.Markdown("", elem_classes="markdown-text-tiny")
@@ -208,30 +216,30 @@ def build_demo(TYPES):
                 with gr.Row():
                     with gr.Column():
                         # with gr.Accordion("🙋 Prediction", open=True, elem_classes="accordion-label"):
-                        with gr.Accordion("Feedback Score (A)", open=True, elem_classes="accordion-label"):
+                        with gr.Accordion("Model A Name", open=True, elem_classes="accordion-label"):
                             prediction = gr.Markdown("", elem_classes="markdown-text-tiny")
                             prediction.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
                     
                     with gr.Column():
                         # with gr.Accordion("🔑 Ground-Truth Answer", open=True, elem_classes="accordion-label"):
-                        with gr.Accordion("Feedback Score (B)", open=True, elem_classes="accordion-label"):
+                        with gr.Accordion("Model B Name", open=True, elem_classes="accordion-label"):
                             gold_answer = gr.HTML("", elem_id="markdown-text-tiny")
                             gold_answer.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
 
-                    with gr.Column():
-                        with gr.Accordion("Correctness", open=True, elem_classes="accordion-label"):
+                    with gr.Column(visible=True):
+                        with gr.Accordion("Feedback Model Name", open=True, elem_classes="accordion-label"):
                             correctness = gr.HTML("", elem_id="markdown-text-tiny")
                             correctness.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
                 
                 # Display chat history when button is clicked
                 btn_show_history.click(fn=display_chat_history, inputs=[selected_models], outputs=[task, Chatbot_Common_Plan, Chatbot_Common_Ground, task_metadata, prediction, gold_answer, correctness, image])
 
-            with gr.TabItem("📮 About Us", elem_id="od-benchmark-tab-table", id=3):
+            with gr.TabItem("📮 About Us", elem_id="od-benchmark-tab-table", id=3, visible=False):
                 gr.Markdown(ABOUT_MD, elem_classes="markdown-text")
         gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text-small")
         
         with gr.Row():
-            with gr.Accordion("📙 Citation", open=False, elem_classes="accordion-label"):
+            with gr.Accordion("📙 Citation", open=False, elem_classes="accordion-label", visible=False):
                 gr.Textbox(
                     value=CITATION_TEXT, 
                     lines=7,
@@ -262,7 +270,14 @@ if __name__ == "__main__":
    
     
     # available_models = sorted(list(set(list(original_df["model name "])))) 
-    available_models = list(model_info.keys())
+    # available_models = list(model_info.keys())
+
+    available_models = set()
+    for example in dataset:
+        for model_response in example['responses']:
+            available_models.add(model_response['model'])
+    available_models = sorted(list(available_models))
+
     # remove the rows where the model name is not in the available_models
     original_df = original_df[original_df["model name "].isin(available_models)]
     ablation_df = ablation_df[ablation_df["model name "].isin(available_models)]