Spaces:
Running
Running
BenchmarkBot
commited on
Commit
โข
483e3a1
1
Parent(s):
e89d633
added example config
Browse files- app.py +2 -0
- src/assets/text_content.py +46 -1
app.py
CHANGED
@@ -9,6 +9,7 @@ from src.assets.text_content import (
|
|
9 |
TITLE,
|
10 |
INTRODUCTION_TEXT,
|
11 |
ABOUT_TEXT,
|
|
|
12 |
CITATION_BUTTON_LABEL,
|
13 |
CITATION_BUTTON_TEXT,
|
14 |
)
|
@@ -335,6 +336,7 @@ with demo:
|
|
335 |
|
336 |
with gr.TabItem("About ๐", id=3):
|
337 |
gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
|
|
|
338 |
|
339 |
demo.load(
|
340 |
change_tab,
|
|
|
9 |
TITLE,
|
10 |
INTRODUCTION_TEXT,
|
11 |
ABOUT_TEXT,
|
12 |
+
EXAMPLE_CONFIG_TEXT,
|
13 |
CITATION_BUTTON_LABEL,
|
14 |
CITATION_BUTTON_TEXT,
|
15 |
)
|
|
|
336 |
|
337 |
with gr.TabItem("About ๐", id=3):
|
338 |
gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
|
339 |
+
gr.Markdown(EXAMPLE_CONFIG_TEXT, elem_classes="descriptive-text")
|
340 |
|
341 |
demo.load(
|
342 |
change_tab,
|
src/assets/text_content.py
CHANGED
@@ -15,10 +15,55 @@ ABOUT_TEXT = """<h3>About the ๐ค Open LLM-Perf Leaderboard ๐๏ธ</h3>
|
|
15 |
<li>Peak memory is measured in MB during the first forward pass of the LLM (no warmup).</li>
|
16 |
<li>Each pair of (Model Type, Weight Class) is represented by the best scored model. This LLM is the one used for all the hardware/backend/optimization experiments.</li>
|
17 |
<li>Score is the average evaluation score obtained from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">๐ค Open LLM Leaderboard</a>.</li>
|
18 |
-
<li>Ranking is based on the euclidean distance from the "Perfect LLM" (i.e. 0 latency and 100% accuracy).</li>
|
19 |
</ul>
|
20 |
"""
|
21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
22 |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
|
23 |
CITATION_BUTTON_TEXT = r"""@misc{open-llm-perf-leaderboard,
|
24 |
author = {Ilyas Moutawwakil, Rรฉgis Pierrard},
|
|
|
15 |
<li>Peak memory is measured in MB during the first forward pass of the LLM (no warmup).</li>
|
16 |
<li>Each pair of (Model Type, Weight Class) is represented by the best scored model. This LLM is the one used for all the hardware/backend/optimization experiments.</li>
|
17 |
<li>Score is the average evaluation score obtained from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">๐ค Open LLM Leaderboard</a>.</li>
|
18 |
+
<li>Ranking is based on a composite metric which is the euclidean distance from the "Perfect LLM" (i.e. 0 latency and 100% accuracy).</li>
|
19 |
</ul>
|
20 |
"""
|
21 |
|
22 |
+
EXAMPLE_CONFIG_TEXT = """
|
23 |
+
Here's an example of the configuration file used to benchmark the models with Optimum-Benchmark:
|
24 |
+
```yaml
|
25 |
+
defaults:
|
26 |
+
- backend: pytorch # default backend
|
27 |
+
- benchmark: inference # default benchmark
|
28 |
+
- experiment # inheriting from experiment config
|
29 |
+
- _self_ # for hydra 1.1 compatibility
|
30 |
+
- override hydra/job_logging: colorlog # colorful logging
|
31 |
+
- override hydra/hydra_logging: colorlog # colorful logging
|
32 |
+
|
33 |
+
hydra:
|
34 |
+
run:
|
35 |
+
dir: llm-experiments/{experiment_name}
|
36 |
+
job:
|
37 |
+
chdir: true
|
38 |
+
|
39 |
+
experiment_name: {experiment_name}
|
40 |
+
|
41 |
+
model: {model}
|
42 |
+
hub_kwargs:
|
43 |
+
revision: {revision}
|
44 |
+
trust_remote_code: {trust_remote_code}
|
45 |
+
|
46 |
+
device: cuda
|
47 |
+
|
48 |
+
backend:
|
49 |
+
no_weights: true
|
50 |
+
delete_cache: true
|
51 |
+
torch_dtype: float16
|
52 |
+
load_in_4bit: true
|
53 |
+
bettertransformer: true
|
54 |
+
|
55 |
+
benchmark:
|
56 |
+
memory: true
|
57 |
+
|
58 |
+
input_shapes:
|
59 |
+
batch_size: 1
|
60 |
+
sequence_length: 1
|
61 |
+
|
62 |
+
new_tokens: 1000
|
63 |
+
```
|
64 |
+
"""
|
65 |
+
|
66 |
+
|
67 |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
|
68 |
CITATION_BUTTON_TEXT = r"""@misc{open-llm-perf-leaderboard,
|
69 |
author = {Ilyas Moutawwakil, Rรฉgis Pierrard},
|