update

app.py

@@ -16,6 +16,7 @@ from datetime import datetime, timezone
 from data_utils import load_eval_results, sample_an_eval_result, apply_length_penalty, post_processing, add_winrates, add_winrates_tasks
 # from gradio.themes.utils import colors, fonts, sizes
 from themes import Seafoam
+import datasets
 from huggingface_hub import HfApi
 # from datasets import Dataset, load_dataset, concatenate_datasets
 import os, uuid 
@@ -37,9 +38,63 @@ eval_results = load_eval_results()
  
 available_models = [] # to be filled in later
 
+dataset = datasets.load_dataset("DongfuJiang/WildFeedback", "feedbacks", split='train')
+
+import random
+random.seed(42)
+np.random.seed(42)
+def sample_an_feedback():
+    feedback = dataset[np.random.randint(0, len(dataset))]
+
+    two_model_responses = random.sample(feedback['responses'], 2)
+    model_response_1 = two_model_responses[0]
+    model_response_2 = two_model_responses[1]
+    
+
+    plan_history = {
+        "user": [
+            feedback['query'],
+            "Please give the feedback (query GPT-4o-mini)"
+        ],
+        "assistant": [
+            model_response_1['response'],
+            model_response_2['feedback']['raw']
+        ]
+    }
+
+    ground_history = {
+        "user": [
+            feedback['query'],
+            "Please give the feedback (query GPT-4o-mini)"
+        ],
+        "assistant": [
+            model_response_2['response'],
+            model_response_2['feedback']['raw']
+        ]
+    }
+
+    result_dict = {
+        "session_id": feedback['id'],
+        "task": feedback['source'],
+        "task_type": feedback['source'],
+        "plan_history": plan_history,
+        "ground_history": ground_history,
+        "pred": str(model_response_1['feedback']['processed']['score']) if model_response_1['feedback']['processed'] else "A",
+        "answer": str(model_response_2['feedback']['processed']['score']) if model_response_2['feedback']['processed'] else "A",
+        "correctness": "Correct",
+        "image": "file/data_dir/test_images/000000341196.jpg"
+    }
+    return result_dict
   
 def display_chat_history(model_selections):
-    eval_item = sample_an_eval_result(eval_results, model_selections)
+    # eval_item = sample_an_eval_result(eval_results, model_selections)
+    eval_item = sample_an_feedback()
+    print("---" * 10)
+    for key, value in eval_item.items():
+        print(f"{key}: {value}")
+    print("---" * 10)
+    
+    # eval_item = sample_an_feedback()
     session_id = eval_item["session_id"]
     task = eval_item["task"]
     task_type = eval_item["task_type"]
@@ -61,6 +116,13 @@ def display_chat_history(model_selections):
     chats_ground = [(chats_ground[i], chats_ground[i+1]) for i in range(0, len(chats_ground), 2)]
     task_metadata = f"- 🆔: `{session_id}` \n- **Task category**: {task_type}"
     
+    print(f"Task: {task}")
+    print(f"Plan History: {chats_plan}")
+    print(f"Ground History: {chats_ground}")
+    print(f"Task Metadata: {task_metadata}")
+    print(f"Prediction: {prediction}")
+    print(f"Gold Answer: {gold_answer}")
+    print(f"Correctness: {correctness}")
     if image_path != "":
         image = f'<div style="text-align: center;"> <img src="{image_path}" style="height: 250px;"> </div>'
         return task, chats_plan, chats_ground, task_metadata, prediction, gold_answer, correctness, image
@@ -133,22 +195,26 @@ def build_demo(TYPES):
 
                 with gr.Row():
                     with gr.Column(scale=1.1):
-                        gr.Markdown("## 📢 Plan Module Process History w/ <span style='background-color: #FDFDBA;'>Execution Module Results</span>", elem_classes="accordion-label")
+                        # gr.Markdown("## 📢 Plan Module Process History w/ <span style='background-color: #FDFDBA;'>Execution Module Results</span>", elem_classes="accordion-label")
+                        gr.Markdown("## 📢 Model A's response and feedback", elem_classes="accordion-label")
                         Chatbot_Common_Plan = gr.Chatbot(avatar_images=["human_icon.jpeg", "ai_icon.png"], height=1000, container=False, label="Common Plan History", likeable=False, show_share_button=False, show_label=True, elem_classes="chat-common", layout="bubble")
                         Chatbot_Common_Plan.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
                     with gr.Column(scale=1):
-                        gr.Markdown("## 📢 Ground Module Process History", elem_classes="accordion-label")
+                        # gr.Markdown("## 📢 Ground Module Process History", elem_classes="accordion-label")
+                        gr.Markdown("## 📢 Model B's response and feedback", elem_classes="accordion-label")
                         Chatbot_Common_Ground = gr.Chatbot(avatar_images=["human_icon.jpeg", "ai_icon.png"], height=1000, container=False, label="Common Ground History", likeable=False, show_share_button=False, show_label=True, elem_classes="chat-common", layout="bubble")
                         Chatbot_Common_Ground.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
 
                 with gr.Row():
                     with gr.Column():
-                        with gr.Accordion("🙋 Prediction", open=True, elem_classes="accordion-label"):
+                        # with gr.Accordion("🙋 Prediction", open=True, elem_classes="accordion-label"):
+                        with gr.Accordion("Feedback Score (A)", open=True, elem_classes="accordion-label"):
                             prediction = gr.Markdown("", elem_classes="markdown-text-tiny")
                             prediction.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
                     
                     with gr.Column():
-                        with gr.Accordion("🔑 Ground-Truth Answer", open=True, elem_classes="accordion-label"):
+                        # with gr.Accordion("🔑 Ground-Truth Answer", open=True, elem_classes="accordion-label"):
+                        with gr.Accordion("Feedback Score (B)", open=True, elem_classes="accordion-label"):
                             gold_answer = gr.HTML("", elem_id="markdown-text-tiny")
                             gold_answer.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)