Repository Migration (#1)

- mod - Repository Migration (a9340d426936185e42ddec717a929b4781341490)


Co-authored-by: Y <[email protected]>

app.py

@@ -0,0 +1,46 @@
+import os
+
+import streamlit as st
+
+from st_pages import get_nav_from_toml, add_page_title
+
+from app.backend.app_init_func import LI_CSS, init_leaderboard, init_pages
+from app.backend.data_engine import DataEngine
+
+# init global data engine
+data_engine = DataEngine()
+
+st.session_state["data_engine"] = data_engine
+st.set_page_config(layout="wide")
+
+# init leaderboard and pages
+leaderboard_change, page_change = init_leaderboard()
+
+init_pages(leaderboard_change, page_change)
+
+# load page tree
+nav = get_nav_from_toml(
+    "app/ui/pages_sections.toml"
+)
+
+# Add custom CSS
+css = "\n".join(LI_CSS)
+st.markdown(f"""
+    <style>
+
+        div[data-testid="stToolbar"] {{visibility: hidden; height: 0px;}}
+
+        footer {{visibility: hidden;}}
+    </style>
+
+    <style>
+        {css}
+    </style>
+    """
+            , unsafe_allow_html=True)
+
+pg = st.navigation(nav)
+
+# add_page_title(pg)
+
+pg.run()

app/__init__.py

@@ -0,0 +1 @@
+

app/backend/__init__.py

@@ -0,0 +1 @@
+

app/backend/app_init_func.py

@@ -0,0 +1,84 @@
+import streamlit as st
+import os
+
+from app.backend.constant import LEADERBOARD_ICON_MAP
+
+LEADERBOARD_MAP = {}
+LI_CSS = []
+PAGE_SECTIONS = []
+
+
+def init_leaderboard():
+    data_engine = st.session_state["data_engine"]
+    leaderboard_map = {}
+    page_sections = []
+    li_css = []
+    sort_id = 0
+    leaderboard_change = False
+    page_change = False
+
+    for dataset in data_engine.datasets:
+        sort_id += 1
+        leaderboard = dataset["leaderboard"]
+        name = dataset["name"]
+
+        leaderboard_section = f"{leaderboard.capitalize()} Leaderboard"
+        if leaderboard_section not in leaderboard_map:
+            leaderboard_map[leaderboard_section] = []
+        if name.lower() == leaderboard.lower():
+            leaderboard_map[leaderboard_section].append((name, 0))
+        else:
+            leaderboard_map[leaderboard_section].append((name, sort_id))
+            li_css.append(f"""
+            ul[data-testid="stSidebarNavItems"] li:nth-child({sort_id}) {{
+            text-indent: 2rem;
+        }}
+            """)
+        page_name = leaderboard_section if name.lower() == leaderboard.lower() else name.capitalize()
+        page_sections.append(f"""
+        [[pages]]
+        path = "app/ui/pages/{name}.py"
+        name = "{page_name}"
+        icon = "{LEADERBOARD_ICON_MAP.get(page_name, "")}"
+        """)
+
+    # ensure leaderboard is first
+    for k, v in leaderboard_map.items():
+        v.sort(key=lambda x: x[1])
+
+    if leaderboard_map != LEADERBOARD_MAP:
+        LEADERBOARD_MAP.update(leaderboard_map)
+        leaderboard_change = True
+    if page_sections != PAGE_SECTIONS:
+        PAGE_SECTIONS.clear()
+        PAGE_SECTIONS.extend(page_sections)
+        page_change = True
+    if li_css != LI_CSS:
+        LI_CSS.clear()
+        LI_CSS.extend(li_css)
+
+    return leaderboard_change, page_change
+
+
+def init_pages(leaderboard_change, page_change):
+    # init pages
+    if leaderboard_change:
+        with open("app/ui/pages/data_page.py", "r", encoding="utf-8") as f:
+            data_page = f.read()
+        for leaderboard, group_names in LEADERBOARD_MAP.items():
+
+            for group_name in group_names:
+                path = os.path.join("app/ui/pages", f"{group_name[0]}.py")
+                with open(path, "w", encoding="utf-8") as f:
+                    f.write(data_page.replace("$group_name$", group_name[0])
+                            )
+    if page_change:
+        with open("app/ui/pages_sections.toml", "w", encoding="utf-8") as f:
+            f.write("\n".join(PAGE_SECTIONS))
+
+
+if __name__ == '__main__':
+    init_leaderboard()
+    init_pages()
+    print("\n".join(PAGE_SECTIONS))
+    print("\n".join(LI_CSS))

app/backend/constant.py

@@ -0,0 +1,89 @@
+from enum import Enum
+
+
+class Navigation(Enum):
+    TEXT_LEADERBOARD = "Text Leaderboard"
+    MULTIMODAL_LEADERBOARD = "Multimodal Leaderboard"
+
+
+class TaskCategory(Enum):
+    LAW = "Law"
+    CODE = "Code"
+    CONVERSATIONAL = "Conversational"
+    TECH = "Tech"
+    LONG_CONTEXT = "Long-context"
+    MULTILINGUAL = "Multilingual"
+
+
+class ModelProvider(Enum):
+    OPENAI = "OpenAI"
+    VOYAGEAI = "VoyageAI"
+    COHERE = "Cohere"
+    OTHERS = "Others"
+
+
+class EvaluationMetric(Enum):
+    NDCG_1 = "NDCG@1"
+    NDCG_3 = "NDCG@3"
+    NDCG_5 = "NDCG@5"
+    NDCG_10 = "NDCG@10"
+    NDCG_20 = "NDCG@20"
+    NDCG_50 = "NDCG@50"
+    NDCG_100 = "NDCG@100"
+    RECALL_1 = "RECALL@1"
+    RECALL_3 = "RECALL@3"
+    RECALL_5 = "RECALL@5"
+    RECALL_10 = "RECALL@10"
+    RECALL_20 = "RECALL@20"
+    RECALL_50 = "RECALL@50"
+    RECALL_100 = "RECALL@100"
+    PRECISION_1 = "PRECISION@1"
+    PRECISION_3 = "PRECISION@3"
+    PRECISION_5 = "PRECISION@5"
+    PRECISION_10 = "PRECISION@10"
+    PRECISION_20 = "PRECISION@20"
+    PRECISION_50 = "PRECISION@50"
+    PRECISION_100 = "PRECISION@100"
+
+
+class EmbdDtype(Enum):
+    ALL = "all"
+    FLOAT_32 = "float32"
+    INT_8 = "int8"
+    BINARY = "binary"
+
+
+class EmbdDim(Enum):
+    OP1 = "<=1k"
+    OP2 = "1k-2k"
+    OP3 = "2k-5k"
+    OP4 = ">=5k"
+
+
+class Similarity(Enum):
+    ALL = "all"
+    COSINE = "cosine"
+    DOT = "dot"
+    EUCLIDEAN = "euclidean"
+
+
+LEADERBOARD_ICON_MAP = {
+    "Text Leaderboard": "📚",
+    "Law": "⚖️",
+    "Multilingual": "🌎",
+    "German": "🇩🇪",
+    "Code": "💻",
+    "Tech": "🛠️",
+    "Legal": "📜",
+    "English": "🇬🇧",
+    "Healthcare": "🏥",
+    "Finance": "💰",
+    "French": "🇫🇷",
+
+}
+
+
+USERNAME = "embedding-benchmark"
+SPACENAME = "RTEB"
+# https://{UserName}-{SpaceName}.hf.space/
+BASE_URL = f"https://{USERNAME}-{SPACENAME}.hf.space/"

app/backend/data_engine.py

@@ -0,0 +1,144 @@
+"""
+Data service provider
+"""
+import json
+from typing import List
+
+import pandas as pd
+
+from utils.cache_decorator import cache_df_with_custom_key, cache_dict_with_custom_key
+from utils.http_utils import get
+
+COLUMNS = ['model_name',
+           'embd_dtype', 'embd_dim', 'num_params', 'max_tokens', 'similarity',
+           'query_instruct', 'corpus_instruct',
+
+           ]
+COLUMNS_TYPES = ["markdown",
+                 'str', 'str', 'number', 'number', 'str',
+                 'str', 'str',
+
+                 ]
+
+GIT_URL = "https://raw.githubusercontent.com/embedding-benchmark/rteb/refs/heads/main/results/"
+DATASET_URL = f"{GIT_URL}datasets.json"
+MODEL_URL = f"{GIT_URL}models.json"
+RESULT_URL = f"{GIT_URL}results.json"
+
+
+class DataEngine:
+
+    def __init__(self):
+        self.df = self.init_dataframe()
+
+    @property
+    @cache_dict_with_custom_key("models")
+    def models(self):
+        """
+        Get models data
+        """
+        res = get(MODEL_URL)
+        if res.status_code == 200:
+            return res.json()
+        return {}
+
+    @property
+    @cache_dict_with_custom_key("datasets")
+    def datasets(self):
+        """
+        Get tasks data
+        """
+        res = get(DATASET_URL)
+        if res.status_code == 200:
+            return res.json()
+        return {}
+
+    @property
+    @cache_dict_with_custom_key("results")
+    def results(self):
+        """
+        Get results data
+        """
+        res = get(RESULT_URL)
+        if res.status_code == 200:
+            return res.json()
+        return {}
+
+    def init_dataframe(self):
+        """
+        Initialize DataFrame
+        """
+        d = {"hello": [123], "world": [456]}
+        return pd.DataFrame(d)
+
+    @cache_df_with_custom_key("json_result")
+    def jsons_to_df(self):
+
+        results_list = self.results
+        df_results_list = []
+        for result_dict in results_list:
+            dataset_name = result_dict["dataset_name"]
+            df_result_row = pd.DataFrame(result_dict["results"])
+            df_result_row["dataset_name"] = dataset_name
+            df_results_list.append(df_result_row)
+        df_result = pd.concat(df_results_list)
+
+        df_result = df_result[["model_name", "dataset_name", "ndcg_at_10", "embd_dim", "embd_dtype"]]
+
+        df_result["ndcg_at_10"] = (df_result["ndcg_at_10"] * 100).round(2)
+
+        df_datasets_list = []
+        for item in self.datasets:
+            dataset_names = item["datasets"]
+            df_dataset_row = pd.DataFrame(
+                {
+                    "group_name": [item["name"] for _ in range(len(dataset_names))],
+                    "dataset_name": dataset_names,
+                    "leaderboard": [item["leaderboard"] for _ in range(len(dataset_names))]
+                }
+            )
+            df_datasets_list.append(df_dataset_row)
+        df_dataset = pd.concat(df_datasets_list).drop_duplicates()
+
+        models_list = self.models
+
+        df_model = pd.DataFrame(models_list)
+
+        df = pd.merge(df_result, df_dataset, on=["dataset_name"], how="inner")
+        # df = pd.merge(df, df_model, on=["model_name"], how="inner")
+
+        # dataset_num_map = {}
+        # grouped_dataset_count = df.groupby(["group_name"]).agg({
+        #     "dataset_name": "nunique"
+        # }).reset_index()
+        #
+        # for _, row in grouped_dataset_count.iterrows():
+        #     dataset_num_map[row["group_name"]] = row["dataset_name"]
+
+        grouped_model = df.groupby(["model_name", "group_name", "embd_dim", "embd_dtype"]).agg({
+            "ndcg_at_10": "mean",
+        }).reset_index()
+
+        pivot = grouped_model.pivot(index=["model_name", "embd_dim", "embd_dtype"], columns="group_name",
+                                    values=["ndcg_at_10"]).round(2)
+
+        # Rename columns
+        pivot.columns = list(
+            map(lambda x: f"{x[1].capitalize()} Average" if x[1] != 'text' else f"Average", pivot.columns))
+
+        pivot_dataset = df_result.pivot(index=["model_name", "embd_dim", "embd_dtype"], columns="dataset_name", values="ndcg_at_10")
+
+        df = pd.merge(df_model, pivot, on=["model_name", "embd_dim", "embd_dtype"])
+        df = pd.merge(df, pivot_dataset, on=["model_name", "embd_dim", "embd_dtype"])
+
+        if df.empty:
+            return pd.DataFrame(columns=COLUMNS + ["reference"])
+        return df
+
+    def filter_df(self, group_name: str):
+        """
+        filter_by_providers
+        """
+        df = self.jsons_to_df()
+
+        return df[df["group_name"] == group_name][COLUMNS][:]

app/backend/data_page.py

@@ -0,0 +1,400 @@
+# -*- coding: utf-8 -*-
+# @Date     : 2025/2/5 16:26
+# @Author   : q275343119
+# @File     : data_page.py
+# @Description:
+import io
+
+from st_aggrid import AgGrid, JsCode, ColumnsAutoSizeMode
+import streamlit as st
+from utils.st_copy_to_clipboard import st_copy_to_clipboard
+from streamlit_theme import st_theme
+
+from app.backend.app_init_func import LEADERBOARD_MAP
+from app.backend.constant import LEADERBOARD_ICON_MAP, BASE_URL
+from app.backend.json_util import compress_msgpack, decompress_msgpack
+
+COLUMNS = ['model_name',
+           'embd_dtype', 'embd_dim', 'num_params', 'max_tokens', 'similarity',
+           'query_instruct', 'corpus_instruct', 'reference'
+
+           ]
+HEADER_STYLE = {'fontSize': '18px'}
+CELL_STYLE = {'fontSize': '18px'}
+
+
+def is_section(group_name):
+    for k, v in LEADERBOARD_MAP.items():
+        leaderboard_name = v[0][0]
+
+        if group_name == leaderboard_name:
+            return True
+    return False
+
+
+def get_closed_dataset():
+    data_engine = st.session_state["data_engine"]
+    closed_list = []
+    results = data_engine.results
+    for result in results:
+        if result.get("is_closed"):
+            closed_list.append(result.get("dataset_name"))
+    return closed_list
+
+
+def convert_df_to_csv(df):
+    output = io.StringIO()
+    df.to_csv(output, index=False)
+    return output.getvalue()
+
+
+def get_column_state():
+    """
+    get column state from url
+    """
+    query_params = st.query_params.get("grid_state", None)
+    if query_params:
+        grid_state = decompress_msgpack(query_params)
+        st.session_state.grid_state = grid_state
+        return True
+    return None
+
+
+def render_page(group_name):
+    grid_state = st.session_state.get("grid_state", {})
+    get_column_state()
+
+    # Add theme color and grid styles
+    st.title("Retrieval Embedding Benchmark (RTEB)")
+    st.markdown("""
+        <style>
+            :root {
+                --theme-color: rgb(129, 150, 64);
+                --theme-color-light: rgba(129, 150, 64, 0.2);
+            }
+
+            /* AG Grid specific overrides */
+            .ag-theme-alpine {
+                --ag-selected-row-background-color: var(--theme-color-light) !important;
+                --ag-row-hover-color: var(--theme-color-light) !important;
+                --ag-selected-tab-color: var(--theme-color) !important;
+                --ag-range-selection-border-color: var(--theme-color) !important;
+                --ag-range-selection-background-color: var(--theme-color-light) !important;
+            }
+
+            .ag-row-hover {
+                background-color: var(--theme-color-light) !important;
+            }
+
+            .ag-row-selected {
+                background-color: var(--theme-color-light) !important;
+            }
+
+            .ag-row-focus {
+                background-color: var(--theme-color-light) !important;
+            }
+
+            .ag-cell-focus {
+                border-color: var(--theme-color) !important;
+            }
+
+            /* Keep existing styles */
+            .center-text {
+                text-align: center;
+                color: var(--theme-color);
+            }
+            .center-image {
+                display: block;
+                margin-left: auto;
+                margin-right: auto;
+            }
+            h2 {
+                color: var(--theme-color) !important;
+            }
+            .ag-header-cell {
+                background-color: var(--theme-color) !important;
+                color: white !important;
+            }
+            a {
+                color: var(--theme-color) !important;
+            }
+            a:hover {
+                color: rgba(129, 150, 64, 0.8) !important;
+            }
+            /* Download Button */
+            button[data-testid="stBaseButton-secondary"] {
+                float: right;
+    
+            }
+            /* Toast On The Top*/
+            div[data-testid="stToastContainer"] {
+                position: fixed !important;
+                z-index: 2147483647 !important;
+            }
+                        
+        </style>
+        
+    """, unsafe_allow_html=True)
+
+    #  logo
+    # st.markdown('<img src="https://www.voyageai.com/logo.svg" class="center-image" width="200">', unsafe_allow_html=True)
+    title = f'<h2 class="center-text">{LEADERBOARD_ICON_MAP.get(group_name.capitalize(), "")} {group_name.capitalize()}</h2>'
+    if is_section(group_name):
+        title = f'<h2 class="center-text">{LEADERBOARD_ICON_MAP.get(group_name.capitalize() + " Leaderboard", "")} {group_name.capitalize() + " Leaderboard"}</h2>'
+    # title
+    st.markdown(title, unsafe_allow_html=True)
+
+    data_engine = st.session_state["data_engine"]
+
+    df = data_engine.jsons_to_df().copy()
+
+    csv = convert_df_to_csv(df)
+    file_name = f"{group_name.capitalize()} Leaderboard" if is_section(group_name) else group_name.capitalize()
+    st.download_button(
+        label="Download CSV",
+        data=csv,
+        file_name=f"{file_name}.csv",
+        mime="text/csv",
+        icon=":material/download:",
+    )
+
+    # get columns
+    column_list = []
+    avg_column = None
+    if is_section(group_name):
+        avg_columns = []
+        for column in df.columns:
+
+            if column.startswith("Average"):
+                avg_columns.insert(0, column)
+                continue
+            if "Average" in column:
+                avg_columns.append(column)
+                continue
+        avg_column = avg_columns[0]
+        column_list.extend(avg_columns)
+    else:
+        for column in df.columns:
+
+            if column.startswith(group_name.capitalize() + " "):
+                avg_column = column
+
+                column_list.append(avg_column)
+
+    dataset_list = []
+
+    for dataset_dict in data_engine.datasets:
+        if dataset_dict["name"] == group_name:
+            dataset_list = dataset_dict["datasets"]
+    if not is_section(group_name):
+        column_list.extend(dataset_list)
+    closed_list = get_closed_dataset()
+    close_avg_list = list(set(dataset_list) & set(closed_list))
+    df["Closed average"] = df[close_avg_list].mean(axis=1).round(2)
+    column_list.append("Closed average")
+
+    open_avg_list = list(set(dataset_list) - set(closed_list))
+    df["Open average"] = df[open_avg_list].mean(axis=1).round(2)
+    column_list.append("Open average")
+
+    df = df[COLUMNS + column_list].sort_values(by=avg_column, ascending=False)
+
+    # rename avg column  name
+    if not is_section(group_name):
+        new_column = avg_column.replace(group_name.capitalize(), "").strip()
+        df.rename(columns={avg_column: new_column}, inplace=True)
+        column_list.remove(avg_column)
+        avg_column = new_column
+
+    # setting column config
+    grid_options = {
+        'columnDefs': [
+            {
+                'headerName': 'Model Name',
+                'field': 'model_name',
+                'pinned': 'left',
+                'sortable': False,
+                'headerStyle': HEADER_STYLE,
+                'cellStyle': CELL_STYLE,
+                "tooltipValueGetter": JsCode(
+                    """function(p) {return p.value}"""
+                ),
+                "width": 250,
+                'cellRenderer': JsCode("""class CustomHTML {
+                    init(params) {
+                        const link = params.data.reference;
+                        this.eGui = document.createElement('div');
+                        this.eGui.innerHTML = link ? 
+                            `<a href="${link}" class="a-cell" target="_blank">${params.value} </a>` : 
+                            params.value;
+                    }
+
+                    getGui() {
+                        return this.eGui;
+                    }
+                }"""),
+                'suppressSizeToFit': True
+
+            },
+            {'headerName': "Overall Score",
+             'field': avg_column,
+             'headerStyle': HEADER_STYLE,
+             'cellStyle': CELL_STYLE,
+             # 'suppressSizeToFit': True
+             },
+
+            # Add Open average column definition
+            {'headerName': 'Open Average',
+             'field': 'Open average',
+             'headerStyle': HEADER_STYLE,
+             'cellStyle': CELL_STYLE,
+             # 'suppressSizeToFit': True
+             },
+
+            {'headerName': 'Closed Average',
+             'field': 'Closed average',
+             'headerStyle': HEADER_STYLE,
+             'cellStyle': CELL_STYLE,
+             # 'suppressSizeToFit': True
+             },
+
+            {
+                'headerName': 'Embd Dtype',
+                'field': 'embd_dtype',
+                'headerStyle': HEADER_STYLE,
+                'cellStyle': CELL_STYLE,
+                # 'suppressSizeToFit': True,
+            },
+            {
+                'headerName': 'Embd Dim',
+                'field': 'embd_dim',
+                'headerStyle': HEADER_STYLE,
+                'cellStyle': CELL_STYLE,
+                # 'suppressSizeToFit': True,
+            },
+            {
+                'headerName': 'Number of Parameters',
+                'field': 'num_params',
+                'cellDataType': 'number',
+                "colId": "num_params",
+                'headerStyle': HEADER_STYLE,
+                'cellStyle': CELL_STYLE,
+                'valueFormatter': JsCode(
+                    """function(params) {
+                    const num = params.value;
+                    if (num >= 1e9) return (num / 1e9).toFixed(2) + "B";
+                    if (num >= 1e6) return (num / 1e6).toFixed(2) + "M";
+                    if (num >= 1e3) return (num / 1e3).toFixed(2) + "K";
+                    return num;
+                    }"""
+                ),
+                "width": 120,
+                # 'suppressSizeToFit': True,
+            },
+            {
+                'headerName': 'Context Length',
+                'field': 'max_tokens',
+                'headerStyle': HEADER_STYLE,
+                'cellStyle': CELL_STYLE,
+                # 'suppressSizeToFit': True,
+            },
+
+            *[{'headerName': column if "Average" not in column else column.replace("Average", "").strip().capitalize(),
+               'field': column,
+               'headerStyle': HEADER_STYLE,
+               'cellStyle': CELL_STYLE,
+               "headerTooltip": column if "Average" not in column else column.replace("Average",
+                                                                                      "").strip().capitalize()
+               # 'suppressSizeToFit': True
+               } for column in column_list if
+              column not in (avg_column, "Closed average", "Open average")]
+        ],
+        'defaultColDef': {
+            'filter': True,
+            'sortable': True,
+            'resizable': True,
+            'headerClass': "multi-line-header",
+            'autoHeaderHeight': True,
+            'width': 105
+        },
+
+        "autoSizeStrategy": {
+            "type": 'fitCellContents',
+            "colIds": [column for column in column_list if column not in (avg_column, "Closed average", "Open average")]
+        },
+        "tooltipShowDelay": 500,
+        "initialState": grid_state,
+    }
+
+    custom_css = {
+        # Model Name Cell
+        ".a-cell": {
+            "display": "inline-block",
+            "white-space": "nowrap",
+            "overflow": "hidden",
+            "text-overflow": "ellipsis",
+            "width": "100%",
+            "min-width": "0"
+        },
+        # Header
+        ".multi-line-header": {
+            "text-overflow": "clip",
+            "overflow": "visible",
+            "white-space": "normal",
+            "height": "auto",
+            "font-family": 'Arial',
+            "font-size": "14px",
+            "font-weight": "bold",
+            "padding": "10px",
+            "text-align": "left",
+        }
+        ,
+        # Filter Options and Input
+        ".ag-theme-streamlit .ag-popup": {
+            "font-family": 'Arial',
+            "font-size": "14px",
+
+        }
+        , ".ag-picker-field-display": {
+            "font-family": 'Arial',
+            "font-size": "14px",
+
+        },
+        ".ag-input-field-input .ag-text-field-input": {
+            "font-family": 'Arial',
+            "font-size": "14px",
+
+        }
+
+    }
+
+    grid = AgGrid(
+        df,
+        enable_enterprise_modules=False,
+        gridOptions=grid_options,
+        allow_unsafe_jscode=True,
+        columns_auto_size_mode=ColumnsAutoSizeMode.FIT_ALL_COLUMNS_TO_VIEW,
+        theme="streamlit",
+        custom_css=custom_css,
+        update_on=["stateUpdated"],
+    )
+
+    @st.dialog("URL")
+    def share_url():
+        state = grid.grid_state
+        if state:
+            share_link = f'{BASE_URL.replace("_", "-")}{group_name}/?grid_state={compress_msgpack(state)}'
+        else:
+            share_link = f'{BASE_URL.replace("_", "-")}{group_name}'
+        st.write(share_link)
+        theme = st_theme()
+        if theme:
+            theme = theme.get("base")
+        else:
+            theme = "light"
+        st_copy_to_clipboard(share_link, before_copy_label='📋Push to copy', after_copy_label='✅Text copied!',theme=theme)
+
+    share_btn = st.button("Share this page", icon=":material/share:")
+
+    if share_btn:
+        share_url()

app/backend/json_util.py

@@ -0,0 +1,15 @@
+import msgpack
+import gzip
+import base64
+
+
+def compress_msgpack(data):
+    packed = msgpack.packb(data)
+    compressed = gzip.compress(packed)
+    return base64.urlsafe_b64encode(compressed).decode('utf-8')
+
+
+def decompress_msgpack(compressed_str):
+    compressed = base64.urlsafe_b64decode(compressed_str)
+    unpacked = gzip.decompress(compressed)
+    return msgpack.unpackb(unpacked, raw=False)

app/ui/pages/data_page.py

@@ -0,0 +1,3 @@
+from app.backend.data_page import render_page
+
+render_page("$group_name$")

requirements.txt

@@ -0,0 +1,6 @@
+streamlit==1.41.1
+streamlit-aggrid==1.0.5
+st-pages==1.0.1
+msgpack==1.1.0
+zstandard==0.23.0
+st-theme==1.2.3

utils/cache_decorator.py

@@ -0,0 +1,54 @@
+import time
+from functools import wraps
+import pandas as pd
+
+CACHE = {}
+TTL = 3600
+
+
+def cache_df_with_custom_key(cache_key: str):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if cache_key in CACHE and CACHE[cache_key].get("expiry") - time.time() < TTL:
+                return CACHE[cache_key]["data"]
+
+            result: pd.DataFrame = func(*args, **kwargs)
+            if result is not None and not result.empty:
+                d = {"expiry": time.time(), "data": result}
+                CACHE[cache_key] = d
+                return result
+
+            CACHE[cache_key]["expiry"] += TTL
+            return CACHE[cache_key]["data"]
+
+        return wrapper
+
+    return decorator
+
+
+def cache_dict_with_custom_key(cache_key: str):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if cache_key in CACHE and time.time() - CACHE[cache_key].get("expiry") < TTL:
+                return CACHE[cache_key]["data"]
+
+            result: dict = func(*args, **kwargs)
+            if result:
+                d = {"expiry": time.time(), "data": result}
+                CACHE[cache_key] = d
+                return result
+
+            CACHE[cache_key]["expiry"] += TTL
+            return CACHE[cache_key]["data"]
+
+        return wrapper
+
+    return decorator
+
+
+if __name__ == '__main__':
+    a = time.time()
+    time.sleep(5)
+    print(time.time() - a)

utils/http_utils.py

@@ -0,0 +1,7 @@
+import requests
+
+
+def get(url: str, params: str = None, verify: bool = False):
+    return requests.get(url, params, verify=verify)
+
+

utils/st_copy_to_clipboard/__init__.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+from typing import Optional
+
+import streamlit as st
+import streamlit.components.v1 as components
+
+# Tell streamlit that there is a component called streamlit_copy_to_clipboard,
+# and that the code to display that component is in the "frontend" folder
+frontend_dir = (Path(__file__).parent / "frontend").absolute()
+_component_func = components.declare_component(
+    "streamlit_copy_to_clipboard", path=str(frontend_dir)
+)
+
+
+def st_copy_to_clipboard(
+    text: str,
+    before_copy_label: str = "📋",
+    after_copy_label: str = "✅",
+    show_text: bool = False,
+    key: Optional[str] = None,
+    theme: str = 'light', # default theme is 'light'
+    
+):
+    """
+    Streamlit component to copy text to clipboard.
+
+    Parameters
+    ----------
+    text : str
+        The text to be copied to the clipboard.
+    before_copy_label : str
+        Label of the button before text is copied.
+    after_copy_label : str
+        Label of the button after text is copied.
+    show_text: bool
+        If True, show text right before the button and make it clickable as well
+    key : str or None
+        An optional key that uniquely identifies the component.
+    theme: str
+        Set the current theme for the button.
+    """
+    component_value = _component_func(
+        key=key,
+        text=text,
+        before_copy_label=before_copy_label,
+        after_copy_label=after_copy_label,
+        show_text=show_text,
+        theme=theme,
+    )
+
+    return component_value
+
+
+def main():
+    st.write("## Example")
+    text = st.text_input("Enter text to copy to clipboard", value="Hello World")
+    st_copy_to_clipboard(text)
+    st_copy_to_clipboard(text, before_copy_label='📋Push to copy', after_copy_label='✅Text copied!')
+    st_copy_to_clipboard(text, before_copy_label='Push to copy', after_copy_label='Text copied!', show_text=True)
+    st_copy_to_clipboard(text, before_copy_label='Push to copy', after_copy_label='copied!', show_text=True, theme="dark")
+
+
+
+
+if __name__ == "__main__":
+    main()

utils/st_copy_to_clipboard/frontend/index.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta charset="UTF-8" />
+		<meta http-equiv="X-UA-Compatible" content="IE=edge" />
+		<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+		<title>st-copy-to-clipboard</title>
+		<script src="./streamlit-component-lib.js"></script>
+		<script src="./main.js"></script>
+		<link rel="stylesheet" href="./style.css" />
+	</head>
+
+	<body>
+		<div id="root">
+			<button id="text-element" class="st-copy-to-clipboard-btn"></button>
+			<button id="copy-button" class="st-copy-to-clipboard-btn">📋</button>
+		</div>
+	</body>
+</html>

utils/st_copy_to_clipboard/frontend/main.js

@@ -0,0 +1,61 @@
+// The `Streamlit` object exists because our html file includes
+// `streamlit-component-lib.js`.
+// If you get an error about "Streamlit" not being defined, that
+// means you're missing that file.
+
+function sendValue(value) {
+	Streamlit.setComponentValue(value);
+}
+
+/**
+ * The component's render function. This will be called immediately after
+ * the component is initially loaded, and then again every time the
+ * component gets new data from Python.
+ */
+function onRender(event) {
+	// Only run the render code the first time the component is loaded.
+	if (!window.rendered) {
+		const { text, before_copy_label, after_copy_label, show_text, theme } = event.detail.args;
+
+		const container = document.querySelector('#container');
+		const button = document.querySelector('#copy-button');
+		const textElement = document.querySelector('#text-element');
+
+		if (theme == 'dark') {
+			button.style.border = '1px solid rgba(250, 250, 250, 0.2)';
+			button.style.color = 'white';
+		}
+
+		button.textContent = before_copy_label; // Set initial label
+
+		// Show text if show_text is true
+		if (show_text) {
+			textElement.textContent = text;
+			textElement.style.display = 'inline';
+		} else {
+			textElement.style.display = 'none';
+		}
+
+		function copyToClipboard() {
+			navigator.clipboard.writeText(text);
+
+			button.textContent = after_copy_label; // Change label after copying
+
+			setTimeout(() => {
+				if (!button) return;
+				button.textContent = before_copy_label; // Revert to original label after 1 second
+			}, 1000);
+		}
+		button.addEventListener('click', copyToClipboard);
+		textElement.addEventListener('click', copyToClipboard);
+
+		window.rendered = true;
+	}
+}
+
+// Render the component whenever python send a "render event"
+Streamlit.events.addEventListener(Streamlit.RENDER_EVENT, onRender);
+// Tell Streamlit that the component is ready to receive events
+Streamlit.setComponentReady();
+// Render with the correct height, if this is a fixed-height component
+Streamlit.setFrameHeight(100);

utils/st_copy_to_clipboard/frontend/streamlit-component-lib.js

@@ -0,0 +1,36 @@
+// Borrowed minimalistic Streamlit API from Thiago
+// https://discuss.streamlit.io/t/code-snippet-create-components-without-any-frontend-tooling-no-react-babel-webpack-etc/13064
+function sendMessageToStreamlitClient(type, data) {
+	console.log(type, data);
+	const outData = Object.assign(
+		{
+			isStreamlitMessage: true,
+			type: type,
+		},
+		data
+	);
+	window.parent.postMessage(outData, '*');
+}
+
+const Streamlit = {
+	setComponentReady: function () {
+		sendMessageToStreamlitClient('streamlit:componentReady', { apiVersion: 1 });
+	},
+	setFrameHeight: function (height) {
+		sendMessageToStreamlitClient('streamlit:setFrameHeight', { height: height });
+	},
+	setComponentValue: function (value) {
+		sendMessageToStreamlitClient('streamlit:setComponentValue', { value: value });
+	},
+	RENDER_EVENT: 'streamlit:render',
+	events: {
+		addEventListener: function (type, callback) {
+			window.addEventListener('message', function (event) {
+				if (event.data.type === type) {
+					event.detail = event.data;
+					callback(event);
+				}
+			});
+		},
+	},
+};

utils/st_copy_to_clipboard/frontend/style.css

@@ -0,0 +1,31 @@
+.st-copy-to-clipboard-btn {
+	display: inline-flex;
+	-moz-box-align: center;
+	align-items: center;
+	-moz-box-pack: center;
+	justify-content: center;
+	font-weight: 400;
+	padding: 0.25rem 0.75rem;
+	border-radius: 0.5rem;
+	min-height: 38.4px;
+	margin: 0px;
+	line-height: 1.6;
+	color: inherit;
+	width: auto;
+	user-select: none;
+	background-color: transparent; /* set bgcolor to transparent to adjust to any theme */
+	border: 1px solid rgba(49, 51, 63, 0.2);
+	cursor: pointer;
+	float: right;
+}
+
+.st-copy-to-clipboard-btn:hover {
+	border-color: rgb(255, 75, 75);
+	color: rgb(255, 75, 75);
+}
+
+.st-copy-to-clipboard-btn:active {
+	border-color: rgb(255, 75, 75);
+	background-color: rgb(255, 75, 75);
+	color: rgb(255, 255, 255);
+}