Dynamically load sidebar and add closed average

app.py

@@ -4,36 +4,30 @@ import streamlit as st
 
 from st_pages import get_nav_from_toml, add_page_title
 
-from app.backend.constant import LEADERBOARD_MAP
+from app.backend.app_init_func import LI_CSS, init_leaderboard, init_pages
 from app.backend.data_engine import DataEngine
 
-# init pages
-with open("app/ui/pages/data_page.py", "r", encoding="utf-8") as f:
-    data_page = f.read()
-for leaderboard, group_names in LEADERBOARD_MAP.items():
-
-    for group_name in group_names:
-        path = os.path.join("app/ui/pages", f"{group_name}.py")
-        with open(path, "w", encoding="utf-8") as f:
-            f.write(data_page.replace("$group_name$", group_name)
-                    )
-
 # init global data engine
 data_engine = DataEngine()
+
 st.session_state["data_engine"] = data_engine
 st.set_page_config(layout="wide")
 
+# init leaderboard and pages
+leaderboard_change, page_change = init_leaderboard()
+
+init_pages(leaderboard_change, page_change)
+
 # load page tree
 nav = get_nav_from_toml(
     "app/ui/pages_sections.toml"
 )
 
 # Add custom CSS
-st.markdown("""
+css = "\n".join(LI_CSS)
+st.markdown(f"""
     <style>
-        ul[data-testid="stSidebarNavItems"] li:not(:first-child) {
-            text-indent: 2rem;
-        }
+        {css}
     </style>
     """
             , unsafe_allow_html=True)

app/backend/app_init_func.py

@@ -0,0 +1,84 @@
+import streamlit as st
+import os
+
+from app.backend.constant import LEADERBOARD_ICON_MAP
+
+LEADERBOARD_MAP = {}
+LI_CSS = []
+PAGE_SECTIONS = []
+
+
+def init_leaderboard():
+    data_engine = st.session_state["data_engine"]
+    leaderboard_map = {}
+    page_sections = []
+    li_css = []
+    sort_id = 0
+    leaderboard_change = False
+    page_change = False
+
+    for dataset in data_engine.datasets:
+        sort_id += 1
+        leaderboard = dataset["leaderboard"]
+        name = dataset["name"]
+
+        leaderboard_section = f"{leaderboard.capitalize()} Leaderboard"
+        if leaderboard_section not in leaderboard_map:
+            leaderboard_map[leaderboard_section] = []
+        if name.lower() == leaderboard.lower():
+            leaderboard_map[leaderboard_section].append((name, 0))
+        else:
+            leaderboard_map[leaderboard_section].append((name, sort_id))
+            li_css.append(f"""
+            ul[data-testid="stSidebarNavItems"] li:nth-child({sort_id}) {{
+            text-indent: 2rem;
+        }}
+            """)
+        page_name = leaderboard_section if name.lower() == leaderboard.lower() else name.capitalize()
+        page_sections.append(f"""
+        [[pages]]
+        path = "app/ui/pages/{name}.py"
+        name = "{page_name}"
+        icon = "{LEADERBOARD_ICON_MAP.get(page_name, "")}"
+        """)
+
+    # ensure leaderboard is first
+    for k, v in leaderboard_map.items():
+        v.sort(key=lambda x: x[1])
+
+    if leaderboard_map != LEADERBOARD_MAP:
+        LEADERBOARD_MAP.update(leaderboard_map)
+        leaderboard_change = True
+    if page_sections != PAGE_SECTIONS:
+        PAGE_SECTIONS.clear()
+        PAGE_SECTIONS.extend(page_sections)
+        page_change = True
+    if li_css != LI_CSS:
+        LI_CSS.clear()
+        LI_CSS.extend(li_css)
+
+    return leaderboard_change, page_change
+
+
+def init_pages(leaderboard_change, page_change):
+    # init pages
+    if leaderboard_change:
+        with open("app/ui/pages/data_page.py", "r", encoding="utf-8") as f:
+            data_page = f.read()
+        for leaderboard, group_names in LEADERBOARD_MAP.items():
+
+            for group_name in group_names:
+                path = os.path.join("app/ui/pages", f"{group_name[0]}.py")
+                with open(path, "w", encoding="utf-8") as f:
+                    f.write(data_page.replace("$group_name$", group_name[0])
+                            )
+    if page_change:
+        with open("app/ui/pages_sections.toml", "w", encoding="utf-8") as f:
+            f.write("\n".join(PAGE_SECTIONS))
+
+
+if __name__ == '__main__':
+    init_leaderboard()
+    init_pages()
+    print("\n".join(PAGE_SECTIONS))
+    print("\n".join(LI_CSS))

app/backend/constant.py

@@ -67,17 +67,12 @@ class Similarity(Enum):
     EUCLIDEAN = "euclidean"
 
 
-LEADERBOARD_MAP = {
-    "Text": [
-        "text",
-        "law",
-        "multilingual",
-        "german",
-        "code",
-        "tech",
-
-    ],
-    "Multimodal": [
-
-    ]
+LEADERBOARD_ICON_MAP = {
+    "Text Leaderboard":"📚",
+    "Law":"⚖️",
+    "Multilingual":"🌎",
+    "German":"🇩🇪",
+    "Code":"💻",
+    "Tech":"🛠️",
+
 }

app/backend/data_page.py

@@ -7,6 +7,8 @@ from st_aggrid import AgGrid, JsCode, ColumnsAutoSizeMode
 
 import streamlit as st
 
+from app.backend.app_init_func import LEADERBOARD_MAP
+
 COLUMNS = ['model_name',
            'embd_dtype', 'embd_dim', 'num_params', 'max_tokens', 'similarity',
            'query_instruct', 'corpus_instruct', 'reference'
@@ -16,6 +18,26 @@ HEADER_STYLE = {'fontSize': '18px'}
 CELL_STYLE = {'fontSize': '18px'}
 
 
+def is_section(group_name):
+
+    for k, v in LEADERBOARD_MAP.items():
+        leaderboard_name = v[0][0]
+
+        if group_name == leaderboard_name:
+            return True
+    return False
+
+
+def get_closed_dataset():
+    data_engine = st.session_state["data_engine"]
+    closed_list = []
+    results = data_engine.results
+    for result in results:
+        if result.get("is_closed"):
+            closed_list.append(result.get("dataset_name"))
+    return closed_list
+
+
 def render_page(group_name):
     # Add theme color and grid styles
     st.markdown("""
@@ -88,7 +110,7 @@ def render_page(group_name):
     # get columns
     column_list = []
     avg_column = None
-    if group_name == "text":
+    if is_section(group_name):
         avg_columns = []
         for column in df.columns:
 
@@ -111,11 +133,16 @@ def render_page(group_name):
                 avg_column = new_column
 
     dataset_list = []
-    if group_name != "text":
-        for dataset_dict in data_engine.datasets:
-            if dataset_dict["name"] == group_name:
-                dataset_list = dataset_dict["datasets"]
+
+    for dataset_dict in data_engine.datasets:
+        if dataset_dict["name"] == group_name:
+            dataset_list = dataset_dict["datasets"]
+    if not is_section(group_name):
         column_list.extend(dataset_list)
+    closed_list = get_closed_dataset()
+    close_avg_list = list(set(dataset_list) & set(closed_list))
+    df["Closed average"] = df[close_avg_list].mean(axis=1)
+    column_list.append("Closed average")
 
     df = df[COLUMNS + column_list].sort_values(by=avg_column, ascending=False)
 
@@ -148,17 +175,26 @@ def render_page(group_name):
              'headerStyle': HEADER_STYLE,
              'cellStyle': CELL_STYLE,
              'suppressSizeToFit': True},
+
+            {'headerName': 'Closed average',
+             'field': 'Closed average',
+             'headerStyle': HEADER_STYLE,
+             'cellStyle': CELL_STYLE,
+             'suppressSizeToFit': True},
+
             {
                 'headerName': 'Data Type',
                 'field': 'embd_dtype',
                 'headerStyle': HEADER_STYLE,
                 'cellStyle': CELL_STYLE,
+                'suppressSizeToFit': True,
             },
             {
                 'headerName': 'Embd Dim',
                 'field': 'embd_dim',
                 'headerStyle': HEADER_STYLE,
                 'cellStyle': CELL_STYLE,
+                'suppressSizeToFit': True,
             },
             {
                 'headerName': 'Model Size (# of Parameters)',
@@ -166,12 +202,14 @@ def render_page(group_name):
                 'cellDataType': 'number',
                 'headerStyle': HEADER_STYLE,
                 'cellStyle': CELL_STYLE,
+                'suppressSizeToFit': True,
             },
             {
                 'headerName': 'Context Length',
                 'field': 'max_tokens',
                 'headerStyle': HEADER_STYLE,
                 'cellStyle': CELL_STYLE,
+                'suppressSizeToFit': True,
             },
             {
                 'headerName': 'Query Instruction',
@@ -193,7 +231,7 @@ def render_page(group_name):
                'field': column,
                'headerStyle': HEADER_STYLE,
                'cellStyle': CELL_STYLE,
-               'suppressSizeToFit': True} for column in column_list if column != avg_column]
+               'suppressSizeToFit': True} for column in column_list if column not in (avg_column, "Closed average")]
         ],
         'defaultColDef': {
             'filter': True,

app/ui/pages_sections.toml

@@ -1,35 +0,0 @@
-[[pages]]
-path = "app/ui/pages/text.py"
-name = "Text Leaderboard"
-icon = "📚"
-#is_section = true
-
-#[[pages]]
-#path = "app/ui/pages/text.py"
-#name = "Overall"
-#icon = "🏆"
-
-[[pages]]
-path = "app/ui/pages/law.py"
-name = "Law"
-icon = "⚖️"
-
-[[pages]]
-path = "app/ui/pages/multilingual.py"
-name = "Multilingual"
-icon = "🌎"
-
-[[pages]]
-path = "app/ui/pages/german.py"
-name = "German"
-icon = "🇩🇪"
-
-[[pages]]
-path = "app/ui/pages/code.py"
-name = "Code"
-icon = "💻"
-
-[[pages]]
-path = "app/ui/pages/tech.py"
-name = "Tech"
-icon = "🛠️"

utils/http_utils.py

@@ -3,3 +3,5 @@ import requests
 
 def get(url: str, params: str = None, verify: bool = False):
     return requests.get(url, params, verify=verify)
+
+