add rpi5 data and HW comparison tab

app.py

@@ -1,12 +1,15 @@
 import gradio as gr
 
-from plotting import create_yolobench_plots, get_pareto_table
+from plotting import create_yolobench_plots, get_pareto_table, create_comparison_plot
 from utils import DEEPLITE_DARK_BLUE_GRADIO
 
+
 def get_hw_description(hw_name):
     HW_URLS = {
         'Jetson Nano (GPU, ONNX Runtime, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/JetsonNano_DataSheet_DS09366001v1.1.pdf',
         'Raspberry Pi 4 Model B (CPU, TFLite, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/raspberry-pi-4-datasheet.pdf',
+        'Raspberry Pi 4 Model B (CPU, ONNX Runtime, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/raspberry-pi-4-datasheet.pdf',
+        'Raspberry Pi 5 Model B (CPU, ONNX Runtime, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Assets/Hardware%20Product%20Assets/raspberry-pi-5-product-brief.pdf',
         'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/Intel_ARK_SpecificationsChart_2023_10_11.pdf',
         'Khadas VIM3 (NPU, INT16)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/khadas_vim3_specs.pdf',
         'Orange Pi 5 (NPU, FP16)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/OrangePi_5_RK3588S_User%20Manual_v1.5.pdf',
@@ -15,18 +18,17 @@ def get_hw_description(hw_name):
 
     hw_url = HW_URLS[hw_name]
     DESC = f"""
-    
     🔸 <span style="font-size:16px">Click </span>[<span style="font-size:16px">here</span>]({hw_url})<span style="font-size:16px"> for more information on the selected hardware platform.</span>
-    
     🔸 <span style="font-size:16px">Refer to the [Deeplite Torch Zoo](https://github.com/Deeplite/deeplite-torch-zoo/tree/develop/results/yolobench) for details about latency measurement experiments.</span>
-    
     """
     return DESC
 
 
-with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
-               css="table { width: 100%; }", analytics_enabled=True) as demo:
-
+with gr.Blocks(
+    theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
+    css="table { width: 100%; }",
+    analytics_enabled=True,
+) as demo:
     gr.HTML(
         """
         <div align="center">
@@ -73,7 +75,7 @@ with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
 
     with gr.Row():
         gr.Markdown(
-       """
+            """
         <span style="font-size:16px">
 
         🚀 <b>YOLOBench</b> 🚀 is a latency-accuracy benchmark of popular single-stage detectors from the YOLO series. Major highlights of this work are:
@@ -93,65 +95,118 @@ with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
         """
         )
 
-    with gr.Row(equal_height=True):
-        with gr.Column():
-            hardware_name = gr.Dropdown(
-                choices=[
-                    'Jetson Nano (GPU, ONNX Runtime, FP32)',
-                    'Raspberry Pi 4 Model B (CPU, TFLite, FP32)',
-                    'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)',
-                    'Khadas VIM3 (NPU, INT16)',
-                    'Orange Pi 5 (NPU, FP16)',
-                    'NVIDIA A40 (GPU, TensorRT, FP32)',
-                ],
-                value='Jetson Nano (GPU, ONNX Runtime, FP32)',
-                label='Hardware',
-            )
-        with gr.Column():
-            dataset_name = gr.Dropdown(
-                choices=['COCO', 'PASCAL VOC', 'SKU-110K', 'WIDERFACE'],
-                value='COCO',
-                label='Dataset',
-            )
-
-    with gr.Row(equal_height=True):
-        with gr.Column():
-            hardware_desc = gr.Markdown(get_hw_description(hardware_name.value))
-
-        with gr.Column():
-            metric_name = gr.Radio(
-                ['[email protected]:0.95', '[email protected]', 'Precision', 'Recall'],
-                value='mAP@0.5:0.95',
-                label='Accuracy metric to plot',
-            )
-
-    with gr.Row(equal_height=True):
-        with gr.Column():
-            gr.Markdown("""
-                        <span style="font-size:16px">
-
-                        🚀 <span style="font-weight:bold">Want to add your own hardware benchmarks to YOLOBench?</span> 🚀
-                        
-                        Contact us [here](https://info.deeplite.ai/add_yolobench_data) for your benchmarking kit and we'll set you up!
-
-                        </span>
-                        """)
-
-        with gr.Column():
-            vis_options = gr.CheckboxGroup(
-                [
-                 'Model family',
-                 'Highlight Pareto',
-                 'Show Pareto only',
-                 'Log x-axis'
-                ],
-                value=['Model family',],
-                label='Visualization options',
-            )
-
-
-    with gr.Row():
-        upper_panel_fig = gr.Plot(show_label=False)
+    with gr.Tab("YOLO model comparision"):
+        with gr.Row(equal_height=True):
+            with gr.Column():
+                hardware_name = gr.Dropdown(
+                    choices=[
+                        'Jetson Nano (GPU, ONNX Runtime, FP32)',
+                        'Raspberry Pi 4 Model B (CPU, TFLite, FP32)',
+                        'Raspberry Pi 4 Model B (CPU, ONNX Runtime, FP32)',
+                        'Raspberry Pi 5 Model B (CPU, ONNX Runtime, FP32)',
+                        'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)',
+                        'Khadas VIM3 (NPU, INT16)',
+                        'Orange Pi 5 (NPU, FP16)',
+                        'NVIDIA A40 (GPU, TensorRT, FP32)',
+                    ],
+                    value='Jetson Nano (GPU, ONNX Runtime, FP32)',
+                    label='Hardware target',
+                )
+            with gr.Column():
+                dataset_name = gr.Dropdown(
+                    choices=['COCO', 'PASCAL VOC', 'SKU-110K', 'WIDERFACE'],
+                    value='COCO',
+                    label='Dataset',
+                )
+
+        with gr.Row(equal_height=True):
+            with gr.Column():
+                hardware_desc = gr.Markdown(get_hw_description(hardware_name.value))
+
+            with gr.Column():
+                metric_name = gr.Radio(
+                    ['[email protected]:0.95', '[email protected]', 'Precision', 'Recall'],
+                    value='[email protected]:0.95',
+                    label='Accuracy metric to plot',
+                )
+
+        with gr.Row(equal_height=True):
+            with gr.Column():
+                gr.Markdown(
+                    """
+                            <span style="font-size:16px">
+
+                            🚀 <span style="font-weight:bold">Want to add your own hardware benchmarks to YOLOBench?</span> 🚀
+                            Contact us [here](https://info.deeplite.ai/add_yolobench_data) for your benchmarking kit and we'll set you up!
+
+                            </span>
+                            """
+                )
+
+            with gr.Column():
+                vis_options = gr.CheckboxGroup(
+                    [
+                        'Model family',
+                        'Highlight Pareto',
+                        'Show Pareto only',
+                        'Log x-axis',
+                    ],
+                    value=[
+                        'Model family',
+                    ],
+                    label='Visualization options',
+                )
+
+        with gr.Row(equal_height=True):
+            upper_panel_fig = gr.Plot(show_label=False)
+
+    with gr.Tab("Hardware platform comparison"):
+        with gr.Row(equal_height=True):
+            with gr.Column():
+                comp_hw = gr.Dropdown(
+                    [
+                        'Jetson Nano (GPU, ONNX Runtime, FP32)',
+                        'Raspberry Pi 4 Model B (CPU, TFLite, FP32)',
+                        'Raspberry Pi 4 Model B (CPU, ONNX Runtime, FP32)',
+                        'Raspberry Pi 5 Model B (CPU, ONNX Runtime, FP32)',
+                        'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)',
+                        'Khadas VIM3 (NPU, INT16)',
+                        'Orange Pi 5 (NPU, FP16)',
+                        'NVIDIA A40 (GPU, TensorRT, FP32)',
+                    ],
+                    value=[
+                        'Jetson Nano (GPU, ONNX Runtime, FP32)',
+                        'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)',
+                    ],
+                    label='Hardware',
+                    multiselect=True,
+                )
+            with gr.Column():
+                comp_data = gr.Dropdown(
+                    choices=['COCO', 'PASCAL VOC', 'SKU-110K', 'WIDERFACE'],
+                    value='COCO',
+                    label='Dataset',
+                )
+
+        with gr.Row(equal_height=True):
+            with gr.Column():
+                comp_metric = gr.Radio(
+                    ['[email protected]:0.95', '[email protected]', 'Precision', 'Recall'],
+                    value='[email protected]:0.95',
+                    label='Accuracy metric to plot',
+                )
+
+            with gr.Column():
+                comp_vis_opt = gr.CheckboxGroup(
+                    ['Log x-axis', 'Remove datapoint markers'],
+                    value=[
+                        'Log x-axis',
+                    ],
+                    label='Visualization options',
+                )
+
+        with gr.Row(equal_height=True):
+            comp_plot = gr.Plot(show_label=False)
 
     gr.Markdown(
         """
@@ -211,7 +266,7 @@ with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
     table_mode = gr.Radio(
         ['Show top-10 models', 'Show all'],
         value='Show top-10 models',
-        label='Pareto model table'
+        label='Pareto model table',
     )
 
     with gr.Row():
@@ -231,9 +286,10 @@ with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
         }
         ```
         """
-        )
+    )
 
     inputs = [dataset_name, hardware_name, metric_name, vis_options, table_mode]
+    inputs_comparison = [comp_data, comp_hw, comp_metric, comp_vis_opt]
 
     # plot by default (VOC, Raspi4)
     demo.load(
@@ -242,6 +298,12 @@ with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
         outputs=[upper_panel_fig, pareto_table],
     )
 
+    demo.load(
+        fn=create_comparison_plot,
+        inputs=inputs_comparison,
+        outputs=[comp_plot],
+    )
+
     demo.load(
         fn=get_pareto_table,
         inputs=[dataset_name, hardware_name, metric_name],
@@ -286,6 +348,26 @@ with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
         outputs=[hardware_desc],
     )
 
+    comp_data.change(
+        fn=create_comparison_plot,
+        inputs=inputs_comparison,
+        outputs=[comp_plot],
+    )
+    comp_hw.change(
+        fn=create_comparison_plot,
+        inputs=inputs_comparison,
+        outputs=[comp_plot],
+    )
+    comp_metric.change(
+        fn=create_comparison_plot,
+        inputs=inputs_comparison,
+        outputs=[comp_plot],
+    )
+    comp_vis_opt.change(
+        fn=create_comparison_plot,
+        inputs=inputs_comparison,
+        outputs=[comp_plot],
+    )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()

plotting.py

@@ -6,7 +6,6 @@ from utils import DEEPLITE_LIGHT_BLUE_HEX, load_yolobench_data
 
 df, pareto_indices = load_yolobench_data()
 
-
 METRIC_NAME_MAPPING = {
     '[email protected]': 'mAP_0.5',
     '[email protected]:0.95': 'mAP_0.5:0.95',
@@ -19,6 +18,8 @@ METRIC_KEYS_TO_NAMES = {v: k for k, v in METRIC_NAME_MAPPING.items()}
 
 LATENCY_KEYS = {
     'Raspberry Pi 4 Model B (CPU, TFLite, FP32)': 'raspi4_tflite_latency',
+    'Raspberry Pi 4 Model B (CPU, ONNX Runtime, FP32)': 'pi4_ort_latency',
+    'Raspberry Pi 5 Model B (CPU, ONNX Runtime, FP32)': 'pi5_ort_latency',
     'Jetson Nano (GPU, ONNX Runtime, FP32)': 'nano_gpu_latency',
     'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)': 'openvino_latency',
     'Khadas VIM3 (NPU, INT16)': 'vim3_latency',
@@ -39,15 +40,18 @@ DATASET_TAGS_TO_NAMES = {v: k for k, v in DATASET_TAGS.items()}
 
 
 def get_scatter_plot(
-        dataset_tag,
-        metric_tag,
-        latency_key,
-        model_family_coloring=True,
-        add_pareto_frontier=False,
-        plot_pareto_only=False,
-        log_axis=False,
-    ):
-    fig_opts, layout_opts = {'opacity': 0.5, 'color_discrete_sequence': [DEEPLITE_LIGHT_BLUE_HEX]}, {}
+    dataset_tag,
+    metric_tag,
+    latency_key,
+    model_family_coloring=True,
+    add_pareto_frontier=False,
+    plot_pareto_only=False,
+    log_axis=False,
+):
+    fig_opts, layout_opts = {
+        'opacity': 0.5,
+        'color_discrete_sequence': [DEEPLITE_LIGHT_BLUE_HEX],
+    }, {}
     if model_family_coloring:
         fig_opts = {
             'color': 'model_family',
@@ -89,7 +93,14 @@ def get_scatter_plot(
 
     fig.update_layout(
         height=600,
-        modebar_remove=['lasso', 'autoscale', 'zoomin', 'zoomout', 'select2d', 'select'],
+        modebar_remove=[
+            'lasso',
+            'autoscale',
+            'zoomin',
+            'zoomout',
+            'select2d',
+            'select',
+        ],
         xaxis_title=f'{LATENCY_KEYS_TO_NAMES[latency_key]} latency, ms',
         yaxis_title=f"{METRIC_KEYS_TO_NAMES[metric_tag]}",
         xaxis=dict(
@@ -105,22 +116,140 @@ def get_scatter_plot(
         hoverlabel=dict(
             # bgcolor="white",
             font_size=14,
-            font_family='Source Sans Pro'
+            font_family='Source Sans Pro',
         ),
         **layout_opts,
     )
     if add_pareto_frontier:
         fig = pareto_frontier_layer(fig, dataset_tag, metric_tag, latency_key)
+    fig.update_layout(autosize=True)
+    return fig
+
+
+def get_comparison_plot(
+    dataset_tag,
+    metric_tag,
+    latency_keys,
+    log_axis=False,
+    remove_marker=False,
+):
+    if len(latency_keys) == 0:
+        layout_opts = {
+            "annotations": [
+                {
+                    "text": "Please select at least 2 hardware targets to compare!",
+                    "showarrow": False,
+                    "font": {"size": 28},
+                }
+            ]
+        }
+    else:
+        layout_opts = {
+            'legend': dict(
+                title='Hardware targets selected<br>(click to toggle)',
+            )
+        }
+
+    fig = go.Figure(
+        layout=go.Layout(
+            title=go.layout.Title(
+                text=f'{METRIC_KEYS_TO_NAMES[metric_tag]}-latency Pareto Optimal Models'
+            )
+        )
+    )
+
+    if remove_marker:
+        mode = 'lines'
+    else:
+        mode = 'lines+markers'
+
+    for latency_key in latency_keys:
+        metric_key = f'{metric_tag}_{dataset_tag}'
+        frontier = pareto_indices[metric_key][latency_key]
+        df_pareto = df.iloc[frontier, :]
+        model_name = df_pareto['model_name']
+        fig.add_trace(
+            go.Scatter(
+                x=df_pareto[latency_key],
+                y=df_pareto[f'{metric_tag}_{dataset_tag}'],
+                name=LATENCY_KEYS_TO_NAMES[latency_key],
+                mode=mode,
+                customdata=model_name,
+                hovertemplate='<b>model name:%{customdata}</b><br>latency:%{x:.2f} <br>metric: %{y:.2f} ',
+            )
+        )
+
+    if log_axis:
+        fig.update_xaxes(type='log')
+        x_axis_title = f'Inference latency, ms (BS=1, log scale)'
+    else:
+        x_axis_title = f'Inference latency, ms (BS=1)'
+
+    fig.update_layout(
+        height=600,
+        plot_bgcolor='white',
+        modebar_remove=[
+            'lasso',
+            'autoscale',
+            'zoomin',
+            'zoomout',
+            'select2d',
+            'select',
+        ],
+        xaxis_title=x_axis_title,
+        yaxis_title=f"{METRIC_KEYS_TO_NAMES[metric_tag]}",
+        xaxis=dict(
+            rangeslider=dict(
+                visible=True,
+                bgcolor=DEEPLITE_LIGHT_BLUE_HEX,
+                thickness=0.02,
+            ),
+        ),
+        yaxis=dict(
+            fixedrange=False,
+        ),
+        hoverlabel=dict(
+            # bgcolor="white",
+            font_size=14,
+            font_family='Source Sans Pro',
+        ),
+        **layout_opts,
+    )
+    fig.update_layout(autosize=True, template="plotly_white")
+
+    return fig
+
+
+def pareto_frontier_layer(
+    fig,
+    dataset_tag,
+    metric_tag,
+    latency_key,
+):
+    metric_key = f'{metric_tag}_{dataset_tag}'
+    frontier = pareto_indices[metric_key][latency_key]
+    fig.add_trace(
+        go.Scatter(
+            x=df.iloc[frontier, :][latency_key],
+            y=df.iloc[frontier, :][metric_key],
+            mode='lines+markers',
+            opacity=0.5,
+            line=go.scatter.Line(color='grey'),
+            showlegend=False,
+            name=metric_key,
+            marker=dict(symbol=['circle']),
+        )
+    )
     return fig
 
 
 def create_yolobench_plots(
-        dataset_name,
-        hardware_name,
-        metric_name,
-        vis_options,
-        table_mode,
-    ):
+    dataset_name,
+    hardware_name,
+    metric_name,
+    vis_options,
+    table_mode,
+):
     model_family_coloring = 'Model family' in vis_options
     add_pareto_frontier = 'Highlight Pareto' in vis_options
     plot_pareto_only = 'Show Pareto only' in vis_options
@@ -141,11 +270,11 @@ def create_yolobench_plots(
 
 
 def pareto_frontier_layer(
-        fig,
-        dataset_tag,
-        metric_tag,
-        latency_key,
-    ):
+    fig,
+    dataset_tag,
+    metric_tag,
+    latency_key,
+):
     metric_key = f'{metric_tag}_{dataset_tag}'
     frontier = pareto_indices[metric_key][latency_key]
     fig.add_trace(
@@ -162,8 +291,29 @@ def pareto_frontier_layer(
     return fig
 
 
+def create_comparison_plot(dataset_name, hardware_list, metric_name, vis_options):
+    log_axis = 'Log x-axis' in vis_options
+    remove_marker = 'Remove datapoint markers' in vis_options
+
+    latency_keys = []
+    for hardware_name in hardware_list:
+        latency_keys.append(LATENCY_KEYS[hardware_name])
+
+    fig = get_comparison_plot(
+        DATASET_TAGS[dataset_name],
+        METRIC_NAME_MAPPING[metric_name],
+        latency_keys,
+        log_axis,
+        remove_marker,
+    )
+    return fig
+
+
 def get_pareto_table(
-    dataset_name, hardware_name, metric_name, expand_table=False,
+    dataset_name,
+    hardware_name,
+    metric_name,
+    expand_table=False,
 ):
     dataset_tag = DATASET_TAGS[dataset_name]
     metric_tag = METRIC_NAME_MAPPING[metric_name]
@@ -175,32 +325,44 @@ def get_pareto_table(
 
     frontier = pareto_indices[metric_key][latency_key]
     table_df = df.iloc[frontier, :][['model_name', metric_key, latency_key]]
-    table_df['Input resolution (px)'] = table_df['model_name'].apply(lambda name: name.split('_')[-1])
-    table_df['Model name'] = table_df['model_name'].apply(lambda name: name.split('_')[0])
+    table_df['Input resolution (px)'] = table_df['model_name'].apply(
+        lambda name: name.split('_')[-1]
+    )
+    table_df['Model name'] = table_df['model_name'].apply(
+        lambda name: name.split('_')[0]
+    )
     table_df[metric_key_final] = table_df[metric_key].apply(lambda val: round(val, 3))
     table_df[latency_key_final] = table_df[latency_key].apply(lambda val: round(val, 2))
 
     def make_clickable(url, name):
         return f'<a href="{url}">{name}</a>'
 
-
     if dataset_name == 'COCO':
         table_df['Download link'] = table_df['model_name'].apply(
             lambda name: f'https://download.deeplite.ai/zoo/models/YOLOBench/{name.split("_")[0]}_640.pt'
         )
-        table_df['Download link'] = table_df.apply(lambda x: make_clickable(x['Download link'], 'Weights download'), axis=1)
+        table_df['Download link'] = table_df.apply(
+            lambda x: make_clickable(x['Download link'], 'Weights download'), axis=1
+        )
     else:
-        table_df['Download link'] = table_df['model_name'].apply(lambda s: 'Coming soon')
-
+        table_df['Download link'] = table_df['model_name'].apply(
+            lambda s: 'Coming soon'
+        )
 
-    table_df = table_df[['Model name', 'Input resolution (px)',
-                         metric_key_final, latency_key_final, 'Download link']].sort_values(by=metric_key_final, ascending=False)
+    table_df = table_df[
+        [
+            'Model name',
+            'Input resolution (px)',
+            metric_key_final,
+            latency_key_final,
+            'Download link',
+        ]
+    ].sort_values(by=metric_key_final, ascending=False)
     if not expand_table:
         table_df = table_df.iloc[:10, :]
 
     table_df = table_df.to_html(
-        classes='table',
-        escape=False, render_links=True, index=False
+        classes='table', escape=False, render_links=True, index=False
     )
 
     return table_df