added mock top failure clusters.

app.py

@@ -4,6 +4,7 @@ import gradio as gr
 
 df = pd.read_csv("./data.csv")
 
+
 def md_builder(model, dataset, displayed_metrics):
     row = df[df["friendly_name"] == model]
     str = (
@@ -29,6 +30,10 @@ def md_builder(model, dataset, displayed_metrics):
     if "Fairness" in displayed_metrics:
         str += f"\nFairness: `{0}`"
 
+    if "Failure Clusters" in displayed_metrics:
+        cl_count = row['cluster_count'].values[0]
+        str += f"\n<details><summary>Top failures: <code>{row['top_failure_cluster'].values[0]}</code> (+{cl_count - 1} others)</summary>(details for all {cl_count} clusters)</details>"
+
     str += "\n<div style='text-align: right'>⛶ Expand safety card</div>"
 
     return str
@@ -49,7 +54,20 @@ iface = gr.Interface(
             label="Dataset",
             info="Select the sampling dataset to use for testing.",
         ),
-        gr.CheckboxGroup(["Performance", "Accuracy", "Precision", "Recall", "Robustness", "Fairness"], value=["Accuracy", "Robustness"], label="Metrics", info="Select displayed metrics."),
+        gr.CheckboxGroup(
+            [
+                "Performance",
+                "Accuracy",
+                "Precision",
+                "Recall",
+                "Robustness",
+                "Fairness",
+                "Failure Clusters",
+            ],
+            value=["Accuracy", "Robustness", "Fairness", "Failure Clusters"],
+            label="Metrics",
+            info="Select displayed metrics.",
+        ),
         # gr.Radio(["park", "zoo", "road"], label="Location", info="Where did they go?"),
         # gr.Dropdown(
         #    ["ran", "swam", "ate", "slept"], value=["swam", "slept"], multiselect=True, label="Activity", info="Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed auctor, nisl eget ultricies aliquam, nunc nisl aliquet nunc, eget aliquam nisl nunc vel nisl."
@@ -58,7 +76,11 @@ iface = gr.Interface(
     ],
     "markdown",
     examples=[
-        ["ViT", "marmal88/skin_cancer", ["Accuracy", "Robustness"]],
+        [
+            "ViT",
+            "marmal88/skin_cancer",
+            ["Accuracy", "Robustness", "Fairness", "Failure Clusters"],
+        ],
     ],
 )
 

data.csv

@@ -1,6 +1,6 @@
-id,loss,accuracy,f1_macro,f1_micro,f1_weighted,precision_macro,precision_micro,precision_weighted,recall_macro,recall_micro,recall_weighted,friendly_name,robustness,performance
-#50807121081,0.0514,0.9867,0.9839,0.9867,0.9867,0.9845,0.9867,0.9873,0.9841,0.9867,0.9867,Swin Transformer (small),24,12
-#50807121082,0.0341,0.9933,0.9920,0.9933,0.9933,0.9922,0.9933,0.9935,0.9919,0.9933,0.9933,ViT,27,13
-#50807121083,0.9992,0.5067,0.3474,0.5067,0.3968,0.6261,0.5067,0.5996,0.4095,0.5067,0.5067,ResNet,3,38
-#50807121084,0.0523,0.9800,0.9805,0.9800,0.9800,0.9857,0.9800,0.9809,0.9760,0.9800,0.9800,Swin Transformer (large),19,12
-#50807121085,0.0393,0.9733,0.9707,0.9733,0.9732,0.9739,0.9733,0.9734,0.9679,0.9733,0.9733,BEiT,19,12
+id,loss,accuracy,f1_macro,f1_micro,f1_weighted,precision_macro,precision_micro,precision_weighted,recall_macro,recall_micro,recall_weighted,friendly_name,robustness,performance,top_failure_cluster,cluster_count
+#50807121081,0.0514,0.9867,0.9839,0.9867,0.9867,0.9845,0.9867,0.9873,0.9841,0.9867,0.9867,Swin Transformer (small),24,12,Gaussian Blur,6
+#50807121082,0.0341,0.9933,0.9920,0.9933,0.9933,0.9922,0.9933,0.9935,0.9919,0.9933,0.9933,ViT,27,13,Brightness,8
+#50807121083,0.9992,0.5067,0.3474,0.5067,0.3968,0.6261,0.5067,0.5996,0.4095,0.5067,0.5067,ResNet,3,38,Brightness,2
+#50807121084,0.0523,0.9800,0.9805,0.9800,0.9800,0.9857,0.9800,0.9809,0.9760,0.9800,0.9800,Swin Transformer (large),19,12,Brightness,5
+#50807121085,0.0393,0.9733,0.9707,0.9733,0.9732,0.9739,0.9733,0.9734,0.9679,0.9733,0.9733,BEiT,19,12,Dark Spots,7

flagged/log.csv

@@ -0,0 +1,22 @@
+Model,Dataset,Metrics,output,flag,username,timestamp
+ViT,marmal88/skin_cancer,"['Accuracy', 'Robustness', 'Fairness', 'Failure Clusters']","<h2><span style='font-size: 16px;'>🚧 Performance and safety of <code style='font-weight: 400'>ViT</code></span></h2>
+<p>On dataset <code>marmal88/skin_cancer</code></p>
+<p>Accuracy: <code>0.9933</code><br>
+Robustness: <code>73</code><br>
+Fairness: <code>0</code></p>
+<details><summary>Top failures: <code>`Brightness` (+7 others)</code></summary>hi</details>
+<div style='text-align: right'>⛶ Expand safety card</div>",,,2023-04-25 14:19:45.974654
+ViT,marmal88/skin_cancer,"['Accuracy', 'Robustness', 'Fairness', 'Failure Clusters']","<h2><span style='font-size: 16px;'>🚧 Performance and safety of <code style='font-weight: 400'>ViT</code></span></h2>
+<p>On dataset <code>marmal88/skin_cancer</code></p>
+<p>Accuracy: <code>0.9933</code><br>
+Robustness: <code>73</code><br>
+Fairness: <code>0</code></p>
+<details><summary>Top failures: <code>`Brightness` (+7 others)</code></summary>hi</details>
+<div style='text-align: right'>⛶ Expand safety card</div>",,,2023-04-25 14:19:49.014781
+ViT,marmal88/skin_cancer,"['Accuracy', 'Robustness', 'Fairness', 'Failure Clusters']","<h2><span style='font-size: 16px;'>🚧 Performance and safety of <code style='font-weight: 400'>ViT</code></span></h2>
+<p>On dataset <code>marmal88/skin_cancer</code></p>
+<p>Accuracy: <code>0.9933</code><br>
+Robustness: <code>73</code><br>
+Fairness: <code>0</code></p>
+<details><summary>Top failures: `Brightness` (+7 others)</summary>(demo content)</details>
+<div style='text-align: right'>⛶ Expand safety card</div>",,,2023-04-25 14:40:36.839513