refactor code for using configs better

app.py

@@ -1,55 +1,69 @@
+import os
+
 import streamlit as st
 
-from utils import get_dataset_config_names, get_sequence_names, pretrained_models_dict, full_ref_metrics, no_ref_metrics
+from utils import get_configs, get_display_names
 
 # st.header("EVREAL  - Event-based Video Reconstruction Evaluation and Analysis Library")
 #
 # paper_link = "https://arxiv.org/abs/2305.00434"
 # code_link = "https://github.com/ercanburak/EVREAL"
 # page_link = "https://ercanburak.github.io/evreal.html"
+# instructions_video = "https://www.youtube.com/watch?v="
 #
 # st.markdown("Paper: " + paper_link, unsafe_allow_html=True)
 # st.markdown("Code: " + paper_link, unsafe_allow_html=True)
 # st.markdown("Page: " + paper_link, unsafe_allow_html=True)
+# st.markdown("Please see this video for instructions on how to use this tool: " + instructions_video, unsafe_allow_html=True)
 
 st.title("Result Analysis Tool")
 
-datasets = get_dataset_config_names("data")
-dataset_sequences_dict = {}
-for dataset in datasets:
-    dataset_sequences_dict[dataset] = get_sequence_names("data", dataset)
-datasets = list(dataset_sequences_dict.keys())
+dataset_cfg_path = os.path.join("cfg", "dataset")
+model_cfg_path = os.path.join("cfg", "model")
+metric_cfg_path = os.path.join("cfg", "metric")
+viz_cfg_path = os.path.join("cfg", "viz")
+
+datasets = get_configs(dataset_cfg_path)
+models = get_configs(model_cfg_path)
+metrics = get_configs(metric_cfg_path)
+vizs = get_configs(viz_cfg_path)
 
-pretrained_models = list(pretrained_models_dict.keys())
+dataset_display_names = get_display_names(datasets)
+model_display_names = get_display_names(models)
+metric_display_names = get_display_names(metrics)
+viz_display_names = get_display_names(vizs)
 
-selected_pretrained_model_names = st.multiselect(
-     'Select multiple methods to compare',
-     pretrained_models)
+assert len(set(dataset_display_names)) == len(dataset_display_names), "Dataset display names are not unique"
+assert len(set(model_display_names)) == len(model_display_names), "Model display names are not unique"
+assert len(set(metric_display_names)) == len(metric_display_names), "Metric display names are not unique"
+assert len(set(viz_display_names)) == len(viz_display_names), "Viz display names are not unique"
 
-col1, col2, col3 = st.columns(3)
+selected_model_names = st.multiselect('Select multiple methods to compare', model_display_names)
+selected_models = [model for model in models if model['display_name'] in selected_model_names]
+
+col1, col2 = st.columns(2)
 
-# A selectbox on the sidebar, for selecting dataset:
 with col1:
-    selected_dataset = st.selectbox(
-        'Select dataset',
-        options=datasets
-    )
+    selected_dataset_name = st.selectbox('Select dataset', options=dataset_display_names)
+    selected_dataset = [dataset for dataset in datasets if dataset['display_name'] == selected_dataset_name][0]
 
-# A selectbox on the sidebar, for selecting sequence:
 with col2:
-    selected_sequence = st.selectbox(
-        'Select sequence',
-        options=dataset_sequences_dict[selected_dataset]
-    )
-
-metric_names = no_ref_metrics if selected_dataset in ["ECD_FAST", "MVSEC_NIGHT", "HDR"] else full_ref_metrics
-metric_names = [name.upper() for name in metric_names]
+    selected_sequence = st.selectbox('Select sequence', options=selected_dataset["sequences"].keys())
 
-selected_metric_names = st.multiselect('Select metrics to display', metric_names)
+usable_metrics = [metric for metric in metrics if metric['no_ref'] == selected_dataset['no_ref']]
+usable_metric_display_names = get_display_names(usable_metrics)
 
-other_visualizations = ["ground truth frames", "event_rates", "event_images", "histograms"]
+selected_metric_names = st.multiselect('Select metrics to display', usable_metric_display_names)
+selected_metrics = [metric for metric in usable_metrics if metric['display_name'] in selected_metric_names]
 
-selected_viz = st.multiselect('Select other visualizations to display', other_visualizations)
+selected_viz = st.multiselect('Select other visualizations to display', viz_display_names)
+selected_visualizations = [viz for viz in vizs if viz['display_name'] in selected_viz]
 
 if not st.button('Get Results'):
-    st.stop()
+    st.stop()
+
+
+# num_gt_column = 1 if len(x.intersection(y)) > 0 else 0
+#
+# num_columns = len(selected_pretrained_model_names) + 1
+

utils.py

@@ -5,40 +5,34 @@ from pathlib import Path
 from collections import OrderedDict
 
 
-pretrained_models_dict = {'E2VID': "0000_000012", 'FireNet': "0000_000020",
-                          'E2VID+': "0000_000000", 'FireNet+': "0000_000002",
-                          'ET-Net': "0000_000030", 'SSL_E2VID': "0000_000040",
-                          'SPADE_E2VID': "0000_000050"}
-
-full_ref_metrics = ['mse', 'ssim', 'lpips']
-no_ref_metrics = ['brisque', 'niqe', 'maniqa']
-
-
 def read_json(fname):
     fname = Path(fname)
     with fname.open('rt') as handle:
         return json.load(handle, object_hook=OrderedDict)
 
 
-def get_filenames(base_path, filename_pattern):
-    file_names = []
+def get_configs(base_path, filename_pattern="*.json"):
+    configs = []
     glob_pattern = os.path.join(base_path, filename_pattern)
-    file_paths = glob.glob(glob_pattern)
+    file_paths = sorted(glob.glob(glob_pattern))
     for file_path in file_paths:
         file_name = Path(file_path).stem
-        file_names.append(file_name)
-    return file_names
-
-
-def get_dataset_config_names(data_configs_path):
-    data_configs_filename_pattern = "*.json"
-    return get_filenames(data_configs_path, data_configs_filename_pattern)
-
-
-def get_sequence_names(data_configs_path, dataset_name):
-    sequences = []
-    dataset_file_name = os.path.join(data_configs_path, dataset_name + ".json")
-    dataset_config = read_json(dataset_file_name)
-    for sequence in dataset_config['sequences']:
-        sequences.append(sequence)
-    return sequences
+        config = read_json(file_path)
+        config['name'] = file_name
+        configs.append(config)
+    return configs
+
+
+def get_display_names(configs):
+    display_names = []
+    for config in configs:
+        display_names.append(config['display_name'])
+    return display_names
+
+# def get_sequence_names(data_configs_path, dataset_name):
+#     sequences = []
+#     dataset_file_name = os.path.join(data_configs_path, dataset_name + ".json")
+#     dataset_config = read_json(dataset_file_name)
+#     for sequence in dataset_config['sequences']:
+#         sequences.append(sequence)
+#     return sequences