Maharshi Gor
commited on
Commit
Β·
4330c26
1
Parent(s):
d29aa65
Pipeline cost display
Browse files- shared/workflows +1 -1
- src/components/quizbowl/bonus.py +58 -1
- src/components/quizbowl/tossup.py +58 -2
shared/workflows
CHANGED
@@ -1 +1 @@
|
|
1 |
-
Subproject commit
|
|
|
1 |
+
Subproject commit 7c8503081f2d663ad96c2e61f59dae1cc37f1cc3
|
src/components/quizbowl/bonus.py
CHANGED
@@ -13,9 +13,10 @@ from components.model_pipeline.model_pipeline import PipelineInterface, Pipeline
|
|
13 |
from components.typed_dicts import PipelineStateDict
|
14 |
from display.formatting import styled_error
|
15 |
from shared.workflows import factory
|
16 |
-
from shared.workflows.metrics import evaluate_prediction
|
17 |
from shared.workflows.qb_agents import QuizBowlBonusAgent
|
18 |
from shared.workflows.runners import run_and_eval_bonus_dataset, run_and_evaluate_bonus
|
|
|
19 |
from submission import submit
|
20 |
|
21 |
from . import populate
|
@@ -125,6 +126,7 @@ class BonusInterface:
|
|
125 |
|
126 |
self.question_display = gr.HTML(label="Question", elem_id="bonus-question-display")
|
127 |
self.error_display = gr.HTML(label="Error", elem_id="bonus-error-display", visible=False)
|
|
|
128 |
self.results_table = gr.DataFrame(
|
129 |
label="Model Outputs",
|
130 |
value=pd.DataFrame(columns=["Part", "Correct?", "Confidence", "Prediction", "Explanation"]),
|
@@ -185,6 +187,55 @@ class BonusInterface:
|
|
185 |
error_msg = styled_error(f"Error loading pipeline: {str(e)}")
|
186 |
return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
|
187 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
188 |
# ------------------------------------- Agent Functions -----------------------------------------------------------
|
189 |
|
190 |
def single_run(
|
@@ -349,6 +400,12 @@ class BonusInterface:
|
|
349 |
],
|
350 |
)
|
351 |
|
|
|
|
|
|
|
|
|
|
|
|
|
352 |
self.eval_btn.click(
|
353 |
fn=self.evaluate,
|
354 |
inputs=[self.pipeline_state],
|
|
|
13 |
from components.typed_dicts import PipelineStateDict
|
14 |
from display.formatting import styled_error
|
15 |
from shared.workflows import factory
|
16 |
+
from shared.workflows.metrics import evaluate_prediction, workflow_metrics
|
17 |
from shared.workflows.qb_agents import QuizBowlBonusAgent
|
18 |
from shared.workflows.runners import run_and_eval_bonus_dataset, run_and_evaluate_bonus
|
19 |
+
from shared.workflows.structs import Workflow
|
20 |
from submission import submit
|
21 |
|
22 |
from . import populate
|
|
|
126 |
|
127 |
self.question_display = gr.HTML(label="Question", elem_id="bonus-question-display")
|
128 |
self.error_display = gr.HTML(label="Error", elem_id="bonus-error-display", visible=False)
|
129 |
+
self.model_cost_display = gr.HTML(label="Pipeline Cost", value="", visible=False)
|
130 |
self.results_table = gr.DataFrame(
|
131 |
label="Model Outputs",
|
132 |
value=pd.DataFrame(columns=["Part", "Correct?", "Confidence", "Prediction", "Explanation"]),
|
|
|
187 |
error_msg = styled_error(f"Error loading pipeline: {str(e)}")
|
188 |
return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
|
189 |
|
190 |
+
def refresh_model_cost(self, pipeline_state: PipelineStateDict):
|
191 |
+
"""Refresh the model cost display with a styled HTML card and emoji."""
|
192 |
+
workflow = Workflow(**pipeline_state["workflow"])
|
193 |
+
cost = workflow_metrics.compute_workflow_cost(workflow)["cost"]
|
194 |
+
# Define color and emoji based on cost range
|
195 |
+
if cost < 1:
|
196 |
+
color = "#d4edda" # light green
|
197 |
+
emoji = "π’"
|
198 |
+
label = "Ultra Light"
|
199 |
+
elif cost < 2:
|
200 |
+
color = "#cce5ff" # light blue
|
201 |
+
emoji = "π΅"
|
202 |
+
label = "Lightweight"
|
203 |
+
elif cost < 3:
|
204 |
+
color = "#fff3cd" # light yellow
|
205 |
+
emoji = "π‘"
|
206 |
+
label = "Moderate"
|
207 |
+
elif cost < 4:
|
208 |
+
color = "#ffe5b4" # light orange
|
209 |
+
emoji = "π "
|
210 |
+
label = "Expensive"
|
211 |
+
else:
|
212 |
+
color = "#f8d7da" # light red
|
213 |
+
emoji = "π΄"
|
214 |
+
label = "Very Expensive"
|
215 |
+
|
216 |
+
html = f"""
|
217 |
+
<div style="
|
218 |
+
background: {color};
|
219 |
+
border-radius: 10px;
|
220 |
+
padding: 1em 1.5em;
|
221 |
+
margin: 0.5em 0;
|
222 |
+
display: flex;
|
223 |
+
align-items: center;
|
224 |
+
box-shadow: 0 2px 8px rgba(0,0,0,0.04);
|
225 |
+
font-size: 1.1em;
|
226 |
+
font-weight: 500;
|
227 |
+
">
|
228 |
+
<span style="font-size:1.5em; margin-right: 0.7em;">{emoji}</span>
|
229 |
+
<span>
|
230 |
+
<span style="font-weight:600; color: #333;">Pipeline Cost:</span>
|
231 |
+
<span style="font-size:1.2em; color: #333;">${cost:.2f}</span>
|
232 |
+
<span style="font-size:0.85em; color: #555;">per 1k question prefixes</span>
|
233 |
+
<span style="margin-left:0.5em; color: #555;">({label})</span>
|
234 |
+
</span>
|
235 |
+
</div>
|
236 |
+
"""
|
237 |
+
return gr.update(visible=True, value=html)
|
238 |
+
|
239 |
# ------------------------------------- Agent Functions -----------------------------------------------------------
|
240 |
|
241 |
def single_run(
|
|
|
400 |
],
|
401 |
)
|
402 |
|
403 |
+
self.pipeline_state.change(
|
404 |
+
fn=self.refresh_model_cost,
|
405 |
+
inputs=[self.pipeline_state],
|
406 |
+
outputs=[self.model_cost_display],
|
407 |
+
)
|
408 |
+
|
409 |
self.eval_btn.click(
|
410 |
fn=self.evaluate,
|
411 |
inputs=[self.pipeline_state],
|
src/components/quizbowl/tossup.py
CHANGED
@@ -13,10 +13,11 @@ from components.model_pipeline.tossup_pipeline import TossupPipelineInterface, T
|
|
13 |
from components.typed_dicts import TossupInterfaceDefaults, TossupPipelineStateDict
|
14 |
from display.formatting import styled_error
|
15 |
from shared.workflows import factory
|
16 |
-
from shared.workflows.metrics import evaluate_prediction
|
17 |
from shared.workflows.metrics.qb_metrics import prepare_tossup_results_df
|
18 |
from shared.workflows.qb_agents import QuizBowlTossupAgent, TossupResult
|
19 |
from shared.workflows.runners import run_and_eval_tossup_dataset, run_and_evaluate_tossup
|
|
|
20 |
from submission import submit
|
21 |
|
22 |
from . import populate
|
@@ -164,6 +165,7 @@ class TossupInterface:
|
|
164 |
self.run_btn = gr.Button("Run on Tossup Question", variant="secondary")
|
165 |
self.question_display = gr.HTML(label="Question", elem_id="tossup-question-display")
|
166 |
self.error_display = gr.HTML(label="Error", elem_id="tossup-error-display", visible=False)
|
|
|
167 |
with gr.Row():
|
168 |
self.confidence_plot = gr.Plot(
|
169 |
label="Buzz Confidence",
|
@@ -175,7 +177,6 @@ class TossupInterface:
|
|
175 |
value=pd.DataFrame(columns=["Token Position", "Correct?", "Confidence", "Prediction"]),
|
176 |
visible=False,
|
177 |
)
|
178 |
-
|
179 |
self.eval_btn, self.model_name_input, self.description_input, self.submit_btn, self.submit_status = (
|
180 |
commons.get_model_submission_accordion(self.app)
|
181 |
)
|
@@ -230,6 +231,55 @@ class TossupInterface:
|
|
230 |
error_msg = styled_error(f"Error loading pipeline: {str(e)}")
|
231 |
return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
|
232 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
233 |
# ------------------------------------- Agent Functions -----------------------------------------------------------
|
234 |
|
235 |
def single_run(
|
@@ -391,6 +441,12 @@ class TossupInterface:
|
|
391 |
],
|
392 |
)
|
393 |
|
|
|
|
|
|
|
|
|
|
|
|
|
394 |
self.eval_btn.click(
|
395 |
fn=self.evaluate,
|
396 |
inputs=[self.pipeline_state],
|
|
|
13 |
from components.typed_dicts import TossupInterfaceDefaults, TossupPipelineStateDict
|
14 |
from display.formatting import styled_error
|
15 |
from shared.workflows import factory
|
16 |
+
from shared.workflows.metrics import evaluate_prediction, workflow_metrics
|
17 |
from shared.workflows.metrics.qb_metrics import prepare_tossup_results_df
|
18 |
from shared.workflows.qb_agents import QuizBowlTossupAgent, TossupResult
|
19 |
from shared.workflows.runners import run_and_eval_tossup_dataset, run_and_evaluate_tossup
|
20 |
+
from shared.workflows.structs import TossupWorkflow
|
21 |
from submission import submit
|
22 |
|
23 |
from . import populate
|
|
|
165 |
self.run_btn = gr.Button("Run on Tossup Question", variant="secondary")
|
166 |
self.question_display = gr.HTML(label="Question", elem_id="tossup-question-display")
|
167 |
self.error_display = gr.HTML(label="Error", elem_id="tossup-error-display", visible=False)
|
168 |
+
self.model_cost_display = gr.HTML(label="Pipeline Cost", value="", visible=False)
|
169 |
with gr.Row():
|
170 |
self.confidence_plot = gr.Plot(
|
171 |
label="Buzz Confidence",
|
|
|
177 |
value=pd.DataFrame(columns=["Token Position", "Correct?", "Confidence", "Prediction"]),
|
178 |
visible=False,
|
179 |
)
|
|
|
180 |
self.eval_btn, self.model_name_input, self.description_input, self.submit_btn, self.submit_status = (
|
181 |
commons.get_model_submission_accordion(self.app)
|
182 |
)
|
|
|
231 |
error_msg = styled_error(f"Error loading pipeline: {str(e)}")
|
232 |
return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
|
233 |
|
234 |
+
def refresh_model_cost(self, pipeline_state: TossupPipelineStateDict):
|
235 |
+
"""Refresh the model cost display with a styled HTML card and emoji."""
|
236 |
+
workflow = TossupWorkflow(**pipeline_state["workflow"])
|
237 |
+
cost = workflow_metrics.compute_workflow_cost(workflow)["cost"]
|
238 |
+
# Define color and emoji based on cost range
|
239 |
+
if cost < 1:
|
240 |
+
color = "#d4edda" # light green
|
241 |
+
emoji = "π’"
|
242 |
+
label = "Ultra Light"
|
243 |
+
elif cost < 2:
|
244 |
+
color = "#cce5ff" # light blue
|
245 |
+
emoji = "π΅"
|
246 |
+
label = "Lightweight"
|
247 |
+
elif cost < 3:
|
248 |
+
color = "#fff3cd" # light yellow
|
249 |
+
emoji = "π‘"
|
250 |
+
label = "Moderate"
|
251 |
+
elif cost < 4:
|
252 |
+
color = "#ffe5b4" # light orange
|
253 |
+
emoji = "π "
|
254 |
+
label = "Expensive"
|
255 |
+
else:
|
256 |
+
color = "#f8d7da" # light red
|
257 |
+
emoji = "π΄"
|
258 |
+
label = "Very Expensive"
|
259 |
+
|
260 |
+
html = f"""
|
261 |
+
<div style="
|
262 |
+
background: {color};
|
263 |
+
border-radius: 10px;
|
264 |
+
padding: 1em 1.5em;
|
265 |
+
margin: 0.5em 0;
|
266 |
+
display: flex;
|
267 |
+
align-items: center;
|
268 |
+
box-shadow: 0 2px 8px rgba(0,0,0,0.04);
|
269 |
+
font-size: 1.1em;
|
270 |
+
font-weight: 500;
|
271 |
+
">
|
272 |
+
<span style="font-size:1.5em; margin-right: 0.7em;">{emoji}</span>
|
273 |
+
<span>
|
274 |
+
<span style="font-weight:600; color: #333;">Pipeline Cost:</span>
|
275 |
+
<span style="font-size:1.2em; color: #333;">${cost:.2f}</span>
|
276 |
+
<span style="font-size:0.85em; color: #555;">per 1k question prefixes</span>
|
277 |
+
<span style="margin-left:0.5em; color: #555;">({label})</span>
|
278 |
+
</span>
|
279 |
+
</div>
|
280 |
+
"""
|
281 |
+
return gr.update(visible=True, value=html)
|
282 |
+
|
283 |
# ------------------------------------- Agent Functions -----------------------------------------------------------
|
284 |
|
285 |
def single_run(
|
|
|
441 |
],
|
442 |
)
|
443 |
|
444 |
+
self.pipeline_state.change(
|
445 |
+
fn=self.refresh_model_cost,
|
446 |
+
inputs=[self.pipeline_state],
|
447 |
+
outputs=[self.model_cost_display],
|
448 |
+
)
|
449 |
+
|
450 |
self.eval_btn.click(
|
451 |
fn=self.evaluate,
|
452 |
inputs=[self.pipeline_state],
|