update

pages/arena.py

@@ -40,7 +40,10 @@ Once the streaming is complete, you can choose the best response.\u2764\ufe0f"""
             with gr.Row():
                 columns = [gr.Textbox(label=f"Prompt {i+1}", lines=10) for i in range(len(prompts))]
             
-            content_list = [prompt + '\n{' + data_textbox.value + '}\n\nsummary:' for prompt in prompts]
+            random.shuffle(prompts)
+            prompts = prompts[:3]
+            
+            content_list = [prompt['prompt'] + '\n{' + data_textbox.value + '}\n\nsummary:' for prompt in prompts]
             model = get_model_batch_generation("Qwen/Qwen2-1.5B-Instruct")
 
             def start_streaming():

pages/leaderboard.py

@@ -1,16 +1,22 @@
+import json
+
 import gradio as gr
 import pandas as pd
 
 from pages.summarization_playground import custom_css
 
-# Sample data for the leaderboard
+with open("prompt/prompt.json", "r") as file:
+    json_data = file.read()
+    prompts = json.loads(json_data)# Sample data for the leaderboard
+
+winning_rate = [prompt['metric']['winning_number'] for prompt in prompts]
+winning_rate = [num / sum(winning_rate) for num in winning_rate]
 data = {
-    'Rank': [1, 2, 3, 4, 5],
-    'Methods': ['METHOD1_PLACEHOLDER', 'METHOD2_PLACEHOLDER', 'METHOD3_PLACEHOLDER', 'METHOD4_PLACEHOLDER', 'METHOD5_PLACEHOLDER'],
-    'METRIC1_PLACEHOLDER Score': [1287, 1272, 1267, 1262, 1258],
-    'METRIC2_PLACEHOLDER Score': [56905, 24913, 42981, 49828, 55567],
-    'METRIC3_PLACEHOLDER Score': [3423, 3423, 2152, 4353, 2342],
-    'Authors': ['AUTHOR1_PLACEHOLDER', 'AUTHOR2_PLACEHOLDER', 'AUTHOR3_PLACEHOLDER', 'AUTHOR4_PLACEHOLDER', 'AUTHOR5_PLACEHOLDER'],
+    'Rank': [i+1 for i in range(len(prompts))],
+    'Methods': [prompt['id'] for prompt in prompts],
+    'Rouge Score': [prompt['metric']['Rouge'] for prompt in prompts],
+    'Winning Rate': winning_rate,
+    'Authors': [prompt['author'] for prompt in prompts]
 }
 
 df = pd.DataFrame(data)
@@ -43,7 +49,7 @@ def create_leaderboard():
 
         sort_by = gr.Dropdown(list(df.columns), label="Sort by", value="Rank")
 
-        gr.Markdown("**Performance**\n\n**methods**: 4,   **questions**: 150")
+        gr.Markdown("**Performance**\n\n**methods**: 5,   **questions**: 15")
 
         leaderboard = gr.HTML(update_leaderboard("Rank"), elem_id="leaderboard")
 

prompt/prompt.ipynb

@@ -2,18 +2,155 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
     "import json\n",
     "\n",
     "prompts = [\n",
-    "    \"\"\"Please summarize the following conversation by highlighting the key points and main topics discussed. Include any important conclusions or decisions made during the conversation.\n",
-    "Conversation:\"\"\",\n",
-    "    \"\"\"Generate a concise summary of the conversation below. Focus on the main arguments, the flow of the discussion, and any significant outcomes or agreements reached. Make sure to capture the essence of the dialogue without including extraneous details.\n",
-    "Conversation:\"\"\",\n",
-    "    \"\"\"Provide a brief overview of the conversation provided. Summarize the main ideas exchanged, the context of the discussion, and any resolutions or actions decided. Ensure the summary is clear and easy to understand for someone who wasn't part of the conversation\"\"\"\n",
+    "    {\n",
+    "        \"id\": \"id_0\",\n",
+    "        \"prompt\": \"\"\"You are a proficient assistant tasked with summarizing medical dialogue between a doctor and a patient. Provide a careful and accurate summary based solely on the conversation. Each word is important, and missing any crucial information could have serious consequences. Use only the information from the conversation, and if something is unclear or cannot be accurately summarized, indicate it with \"N/A.\" Follow a step-by-step approach to ensure completeness .\"\"\",\n",
+    "        \"author\": \"Shunxi Wu\",\n",
+    "        \"metric\": {\n",
+    "            \"Rouge\": 0.14,\n",
+    "            \"winning_number\": 10\n",
+    "        }\n",
+    "    },\n",
+    "    {\n",
+    "        \"id\": \"id_1\",\n",
+    "        \"prompt\": \"\"\"You are a proficient assistant that helps summarize medical dialogue after the communication between the doctor and the patient, Think step by step and strictly follow the form of the example provided, structure the summary into:\n",
+    "1.CHIEF COMPLAINT:\n",
+    "<leave blank row>\n",
+    "<fill in> \n",
+    "<leave blank row>\n",
+    "2. HISTORY OF PRESENT ILLNESS: \n",
+    "<leave blank row>\n",
+    "<fill in> \n",
+    "<leave blank row>\n",
+    "3. MEDICAL HISTORY: \n",
+    "<leave blank row>\n",
+    "<fill in> \n",
+    "<leave blank row>\n",
+    "4. PHYSICAL EXAM: \n",
+    "<leave blank row>\n",
+    "<fill in> \n",
+    "<leave blank row>\n",
+    "5. RESULTS: \n",
+    "<leave blank row>\n",
+    "<fill in> \n",
+    "<leave blank row>\n",
+    "6. ASSESSMENT: \n",
+    "<leave blank row>\n",
+    "<fill in> \n",
+    "<leave blank row>\n",
+    "\n",
+    "You are autoregressive and carefully provide summarization based on the conversation. Since you are autoregressive, each token of the conversation should be considered important, and you cannot lose any important information. \n",
+    "Use only the information from the conversation, and if something is unclear or cannot be accurately summarized, indicate it with \"\"N/A.\"\" Follow a step-by-step approach to ensure completeness and adhere strictly to the provided example format.\"\"\",\n",
+    "        \"author\": \"Shunxi Wu\",\n",
+    "        \"metric\": {\n",
+    "            \"Rouge\": 0.14,\n",
+    "            \"winning_number\": 12\n",
+    "        }\n",
+    "    },\n",
+    "    {\n",
+    "        \"id\": \"id_2\",\n",
+    "        \"prompt\": \"Summarize this conversation. Tell me about the Patient's history, current condition, and treatment plan\",\n",
+    "        \"author\": \"Taru Mishra\",\n",
+    "        \"metric\": {\n",
+    "            \"Rouge\": 0.15,\n",
+    "            \"winning_number\": 3\n",
+    "        }\n",
+    "    },\n",
+    "    {\n",
+    "        \"id\": \"id_3\",\n",
+    "        \"prompt\": \"\"\"Here is a dialogue:\n",
+    "DR: You mentioned having a cough for 2 days and a\n",
+    "fever since last night along with being short of breath.\n",
+    "Is that correct?\n",
+    "PT: yes , correct\n",
+    "DR: I appreciate your concern for preventing spread. Do\n",
+    "you feel like you are unable to move around as usual?\n",
+    "PT: I’m definitely weaker and low energy the fever\n",
+    "went down to 99 this morning\n",
+    "DR: Have you taken any medications or tried anything\n",
+    "else to help you with your symptoms?\n",
+    "PT: lots of fluids and vitamin c. lozenges to minimize\n",
+    "coughing\n",
+    "DR: do you have any medical conditions or have you\n",
+    "been on any medications\n",
+    "PT: no, none\n",
+    "DR: alright. When you had a fever, did you take\n",
+    "medicine like tylenol to bring the fever down?\n",
+    "PT: I didn’t\n",
+    "First, important details from the text are extracted. The important facts that the patient states are that they had a cough for 2 days, a fever since last night, shortness of breath, weaker and low energy, fever going down to 99, lots of fluid and vitamin c taken to minimize coughing, no medical conditions, and didn't take medicine like tylenol to bring the fever down.\n",
+    "Next, they are synthesized into a summary with the important details directly copied word-for-word from the text. In this case, we copy \"\"cough for 2 days\"\", \"\"a fever since last night\"\", \"\"short of breath\"\", \"\"weaker and low energy\"\", \"\"fever went down to 99 this morning\"\", \"\"lots of fluids and vitamin c lozenges to minimize coughing\"\", and \"\"take medicine like tylenol to bring the fever down\"\".  We add connecting words, and we arrive at the summary:\n",
+    "The patient mentioned having a cough for 2 days and a fever since last night with being short of breath. The patient is weaker and low energy, and their fever went down to 99 this morning. The patient took lots of fluids and vitamin c lozenges to minimize coughing. The patient does not have any medical conditions or medications. The patient didn't take medicine like tylenol to bring the fever down.\n",
+    "Here is a dialogue, apply the same process but only output the summary:\"\"\",\n",
+    "        \"author\": \"Eric Chen\",\n",
+    "        \"metric\": {\n",
+    "            \"Rouge\": 0.11,\n",
+    "            \"winning_number\": 8\n",
+    "        }\n",
+    "    },\n",
+    "    {\n",
+    "        \"id\": \"id_4\",\n",
+    "        \"prompt\": \"\"\"Here is a dialogue:\n",
+    "DR: You mentioned having a cough for 2 days and a\n",
+    "fever since last night along with being short of breath.\n",
+    "Is that correct?\n",
+    "PT: yes , correct\n",
+    "DR: I appreciate your concern for preventing spread. Do\n",
+    "you feel like you are unable to move around as usual?\n",
+    "PT: I’m definitely weaker and low energy the fever\n",
+    "went down to 99 this morning\n",
+    "DR: Have you taken any medications or tried anything\n",
+    "else to help you with your symptoms?\n",
+    "PT: lots of fluids and vitamin c. lozenges to minimize\n",
+    "coughing\n",
+    "DR: do you have any medical conditions or have you\n",
+    "been on any medications\n",
+    "PT: no, none\n",
+    "DR: alright. When you had a fever, did you take\n",
+    "medicine like tylenol to bring the fever down?\n",
+    "PT: I didn’t\n",
+    "DR: ok alright let me do a quick physical exam now with a thermometer, it seems like your temperature is 98 now\n",
+    "PT: okay \n",
+    "DR: it seems like your infection is going away, just make sure to keep resting\n",
+    "PT: okay\n",
+    "First, important details from the text are extracted. The important facts that the patient states are that they had a cough for 2 days, a fever since last night, shortness of breath, weaker and low energy, fever going down to 99, lots of fluid and vitamin c taken to minimize coughing, no medical conditions, and didn't take medicine like tylenol to bring the fever down.\n",
+    "Next, they are synthesized into a summary with the important details directly copied word-for-word from the text. In this case, we copy \"\"cough for 2 days\"\", \"\"a fever since last night\"\", \"\"short of breath\"\", \"\"weaker and low energy\"\", \"\"fever went down to 99 this morning\"\", \"\"lots of fluids and vitamin c lozenges to minimize coughing\"\", \"\"take medicine like tylenol to bring the fever down\"\", \"\"thermometer\"\", \"\"temperature is 98 now\"\", \"\"infection is going away\"\", and \"\"make sure to keep resting\"\".\n",
+    "Then, we sort these details into the categories that they are relevant to. There are five categories, CHIEF COMPLAINT, HISTORY OF PRESENT ILLNESS, PHYSICAL EXAM, RESULTS, and ASSESSMENT. Details are split if they belong to multiple categories. Sorted they are CHIEF COMPLAINT: cough, fever, HISTORY OF PRESENT ILLNESS: cough for 2 days, a fever since last night, short of breath, weaker and low energy, fever went to 99 this morning, lots of fluids and vitamin c lozenges to minimize coughing, didn't take tylenol to bring the fever down, PHYSICAL EXAM: thermometer, RESULTS: temperature is 98, ASSESSMENT: infection is going away, make sure to keep resting.\n",
+    "Finally, we add connecting words, and add the relevant section headers and we arrive at the summary:\n",
+    "\n",
+    "CHIEF COMPLAINT\n",
+    "\n",
+    "The patient has a cough and a fever.\n",
+    "\n",
+    "HISTORY OF PRESENT ILLNESS\n",
+    "\n",
+    "The patient has had a cough for 2 days, a fever since last night, and is weaker, low energy, and short of breath. The patient's fever went down to 99 this morning. The patient has taken lots of fluids and vitamin c lozenges to minimize coughing. The patient didn't take tylenol to bring the fever down.\n",
+    "\n",
+    "PHYSICAL EXAM\n",
+    "\n",
+    "The doctor used a thermometer to test the patient's temperature.\n",
+    "\n",
+    "RESULTS\n",
+    "\n",
+    "The patient's temperature is 98 degrees.\n",
+    "\n",
+    "ASSESSMENT\n",
+    "\n",
+    "The patient's infection is going away and the patient needs to rest.\n",
+    "\n",
+    "Here is a dialogue, apply the same process but only output the summary:\"\"\",\n",
+    "        \"author\": \"Eric Chen\",\n",
+    "        \"metric\": {\n",
+    "            \"Rouge\": 0.2,\n",
+    "            \"winning_number\": 13\n",
+    "        }\n",
+    "    },\n",
     "]"
    ]
   },

prompt/prompt.json

@@ -1 +1 @@
-["Please summarize the following conversation by highlighting the key points and main topics discussed. Include any important conclusions or decisions made during the conversation.\nConversation:", "Generate a concise summary of the conversation below. Focus on the main arguments, the flow of the discussion, and any significant outcomes or agreements reached. Make sure to capture the essence of the dialogue without including extraneous details.\nConversation:", "Provide a brief overview of the conversation provided. Summarize the main ideas exchanged, the context of the discussion, and any resolutions or actions decided. Ensure the summary is clear and easy to understand for someone who wasn't part of the conversation"]
+[{"id": "id_0", "prompt": "You are a proficient assistant tasked with summarizing medical dialogue between a doctor and a patient. Provide a careful and accurate summary based solely on the conversation. Each word is important, and missing any crucial information could have serious consequences. Use only the information from the conversation, and if something is unclear or cannot be accurately summarized, indicate it with \"N/A.\" Follow a step-by-step approach to ensure completeness .", "author": "Shunxi Wu", "metric": {"Rouge": 0.14, "winning_number": 10}}, {"id": "id_1", "prompt": "You are a proficient assistant that helps summarize medical dialogue after the communication between the doctor and the patient, Think step by step and strictly follow the form of the example provided, structure the summary into:\n1.CHIEF COMPLAINT:\n<leave blank row>\n<fill in> \n<leave blank row>\n2. HISTORY OF PRESENT ILLNESS: \n<leave blank row>\n<fill in> \n<leave blank row>\n3. MEDICAL HISTORY: \n<leave blank row>\n<fill in> \n<leave blank row>\n4. PHYSICAL EXAM: \n<leave blank row>\n<fill in> \n<leave blank row>\n5. RESULTS: \n<leave blank row>\n<fill in> \n<leave blank row>\n6. ASSESSMENT: \n<leave blank row>\n<fill in> \n<leave blank row>\n\nYou are autoregressive and carefully provide summarization based on the conversation. Since you are autoregressive, each token of the conversation should be considered important, and you cannot lose any important information. \nUse only the information from the conversation, and if something is unclear or cannot be accurately summarized, indicate it with \"\"N/A.\"\" Follow a step-by-step approach to ensure completeness and adhere strictly to the provided example format.", "author": "Shunxi Wu", "metric": {"Rouge": 0.14, "winning_number": 12}}, {"id": "id_2", "prompt": "Summarize this conversation. Tell me about the Patient's history, current condition, and treatment plan", "author": "Taru Mishra", "metric": {"Rouge": 0.15, "winning_number": 3}}, {"id": "id_3", "prompt": "Here is a dialogue:\nDR: You mentioned having a cough for 2 days and a\nfever since last night along with being short of breath.\nIs that correct?\nPT: yes , correct\nDR: I appreciate your concern for preventing spread. Do\nyou feel like you are unable to move around as usual?\nPT: I\u2019m definitely weaker and low energy the fever\nwent down to 99 this morning\nDR: Have you taken any medications or tried anything\nelse to help you with your symptoms?\nPT: lots of fluids and vitamin c. lozenges to minimize\ncoughing\nDR: do you have any medical conditions or have you\nbeen on any medications\nPT: no, none\nDR: alright. When you had a fever, did you take\nmedicine like tylenol to bring the fever down?\nPT: I didn\u2019t\nFirst, important details from the text are extracted. The important facts that the patient states are that they had a cough for 2 days, a fever since last night, shortness of breath, weaker and low energy, fever going down to 99, lots of fluid and vitamin c taken to minimize coughing, no medical conditions, and didn't take medicine like tylenol to bring the fever down.\nNext, they are synthesized into a summary with the important details directly copied word-for-word from the text. In this case, we copy \"\"cough for 2 days\"\", \"\"a fever since last night\"\", \"\"short of breath\"\", \"\"weaker and low energy\"\", \"\"fever went down to 99 this morning\"\", \"\"lots of fluids and vitamin c lozenges to minimize coughing\"\", and \"\"take medicine like tylenol to bring the fever down\"\".  We add connecting words, and we arrive at the summary:\nThe patient mentioned having a cough for 2 days and a fever since last night with being short of breath. The patient is weaker and low energy, and their fever went down to 99 this morning. The patient took lots of fluids and vitamin c lozenges to minimize coughing. The patient does not have any medical conditions or medications. The patient didn't take medicine like tylenol to bring the fever down.\nHere is a dialogue, apply the same process but only output the summary:", "author": "Eric Chen", "metric": {"Rouge": 0.11, "winning_number": 8}}, {"id": "id_4", "prompt": "Here is a dialogue:\nDR: You mentioned having a cough for 2 days and a\nfever since last night along with being short of breath.\nIs that correct?\nPT: yes , correct\nDR: I appreciate your concern for preventing spread. Do\nyou feel like you are unable to move around as usual?\nPT: I\u2019m definitely weaker and low energy the fever\nwent down to 99 this morning\nDR: Have you taken any medications or tried anything\nelse to help you with your symptoms?\nPT: lots of fluids and vitamin c. lozenges to minimize\ncoughing\nDR: do you have any medical conditions or have you\nbeen on any medications\nPT: no, none\nDR: alright. When you had a fever, did you take\nmedicine like tylenol to bring the fever down?\nPT: I didn\u2019t\nDR: ok alright let me do a quick physical exam now with a thermometer, it seems like your temperature is 98 now\nPT: okay \nDR: it seems like your infection is going away, just make sure to keep resting\nPT: okay\nFirst, important details from the text are extracted. The important facts that the patient states are that they had a cough for 2 days, a fever since last night, shortness of breath, weaker and low energy, fever going down to 99, lots of fluid and vitamin c taken to minimize coughing, no medical conditions, and didn't take medicine like tylenol to bring the fever down.\nNext, they are synthesized into a summary with the important details directly copied word-for-word from the text. In this case, we copy \"\"cough for 2 days\"\", \"\"a fever since last night\"\", \"\"short of breath\"\", \"\"weaker and low energy\"\", \"\"fever went down to 99 this morning\"\", \"\"lots of fluids and vitamin c lozenges to minimize coughing\"\", \"\"take medicine like tylenol to bring the fever down\"\", \"\"thermometer\"\", \"\"temperature is 98 now\"\", \"\"infection is going away\"\", and \"\"make sure to keep resting\"\".\nThen, we sort these details into the categories that they are relevant to. There are five categories, CHIEF COMPLAINT, HISTORY OF PRESENT ILLNESS, PHYSICAL EXAM, RESULTS, and ASSESSMENT. Details are split if they belong to multiple categories. Sorted they are CHIEF COMPLAINT: cough, fever, HISTORY OF PRESENT ILLNESS: cough for 2 days, a fever since last night, short of breath, weaker and low energy, fever went to 99 this morning, lots of fluids and vitamin c lozenges to minimize coughing, didn't take tylenol to bring the fever down, PHYSICAL EXAM: thermometer, RESULTS: temperature is 98, ASSESSMENT: infection is going away, make sure to keep resting.\nFinally, we add connecting words, and add the relevant section headers and we arrive at the summary:\n\nCHIEF COMPLAINT\n\nThe patient has a cough and a fever.\n\nHISTORY OF PRESENT ILLNESS\n\nThe patient has had a cough for 2 days, a fever since last night, and is weaker, low energy, and short of breath. The patient's fever went down to 99 this morning. The patient has taken lots of fluids and vitamin c lozenges to minimize coughing. The patient didn't take tylenol to bring the fever down.\n\nPHYSICAL EXAM\n\nThe doctor used a thermometer to test the patient's temperature.\n\nRESULTS\n\nThe patient's temperature is 98 degrees.\n\nASSESSMENT\n\nThe patient's infection is going away and the patient needs to rest.\n\nHere is a dialogue, apply the same process but only output the summary:", "author": "Eric Chen", "metric": {"Rouge": 0.2, "winning_number": 13}}]