Maharshi Gor commited on
Commit
4330c26
Β·
1 Parent(s): d29aa65

Pipeline cost display

Browse files
shared/workflows CHANGED
@@ -1 +1 @@
1
- Subproject commit c5da5d36708afbade2a2175391562dea7354be2c
 
1
+ Subproject commit 7c8503081f2d663ad96c2e61f59dae1cc37f1cc3
src/components/quizbowl/bonus.py CHANGED
@@ -13,9 +13,10 @@ from components.model_pipeline.model_pipeline import PipelineInterface, Pipeline
13
  from components.typed_dicts import PipelineStateDict
14
  from display.formatting import styled_error
15
  from shared.workflows import factory
16
- from shared.workflows.metrics import evaluate_prediction
17
  from shared.workflows.qb_agents import QuizBowlBonusAgent
18
  from shared.workflows.runners import run_and_eval_bonus_dataset, run_and_evaluate_bonus
 
19
  from submission import submit
20
 
21
  from . import populate
@@ -125,6 +126,7 @@ class BonusInterface:
125
 
126
  self.question_display = gr.HTML(label="Question", elem_id="bonus-question-display")
127
  self.error_display = gr.HTML(label="Error", elem_id="bonus-error-display", visible=False)
 
128
  self.results_table = gr.DataFrame(
129
  label="Model Outputs",
130
  value=pd.DataFrame(columns=["Part", "Correct?", "Confidence", "Prediction", "Explanation"]),
@@ -185,6 +187,55 @@ class BonusInterface:
185
  error_msg = styled_error(f"Error loading pipeline: {str(e)}")
186
  return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
187
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
188
  # ------------------------------------- Agent Functions -----------------------------------------------------------
189
 
190
  def single_run(
@@ -349,6 +400,12 @@ class BonusInterface:
349
  ],
350
  )
351
 
 
 
 
 
 
 
352
  self.eval_btn.click(
353
  fn=self.evaluate,
354
  inputs=[self.pipeline_state],
 
13
  from components.typed_dicts import PipelineStateDict
14
  from display.formatting import styled_error
15
  from shared.workflows import factory
16
+ from shared.workflows.metrics import evaluate_prediction, workflow_metrics
17
  from shared.workflows.qb_agents import QuizBowlBonusAgent
18
  from shared.workflows.runners import run_and_eval_bonus_dataset, run_and_evaluate_bonus
19
+ from shared.workflows.structs import Workflow
20
  from submission import submit
21
 
22
  from . import populate
 
126
 
127
  self.question_display = gr.HTML(label="Question", elem_id="bonus-question-display")
128
  self.error_display = gr.HTML(label="Error", elem_id="bonus-error-display", visible=False)
129
+ self.model_cost_display = gr.HTML(label="Pipeline Cost", value="", visible=False)
130
  self.results_table = gr.DataFrame(
131
  label="Model Outputs",
132
  value=pd.DataFrame(columns=["Part", "Correct?", "Confidence", "Prediction", "Explanation"]),
 
187
  error_msg = styled_error(f"Error loading pipeline: {str(e)}")
188
  return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
189
 
190
+ def refresh_model_cost(self, pipeline_state: PipelineStateDict):
191
+ """Refresh the model cost display with a styled HTML card and emoji."""
192
+ workflow = Workflow(**pipeline_state["workflow"])
193
+ cost = workflow_metrics.compute_workflow_cost(workflow)["cost"]
194
+ # Define color and emoji based on cost range
195
+ if cost < 1:
196
+ color = "#d4edda" # light green
197
+ emoji = "🟒"
198
+ label = "Ultra Light"
199
+ elif cost < 2:
200
+ color = "#cce5ff" # light blue
201
+ emoji = "πŸ”΅"
202
+ label = "Lightweight"
203
+ elif cost < 3:
204
+ color = "#fff3cd" # light yellow
205
+ emoji = "🟑"
206
+ label = "Moderate"
207
+ elif cost < 4:
208
+ color = "#ffe5b4" # light orange
209
+ emoji = "🟠"
210
+ label = "Expensive"
211
+ else:
212
+ color = "#f8d7da" # light red
213
+ emoji = "πŸ”΄"
214
+ label = "Very Expensive"
215
+
216
+ html = f"""
217
+ <div style="
218
+ background: {color};
219
+ border-radius: 10px;
220
+ padding: 1em 1.5em;
221
+ margin: 0.5em 0;
222
+ display: flex;
223
+ align-items: center;
224
+ box-shadow: 0 2px 8px rgba(0,0,0,0.04);
225
+ font-size: 1.1em;
226
+ font-weight: 500;
227
+ ">
228
+ <span style="font-size:1.5em; margin-right: 0.7em;">{emoji}</span>
229
+ <span>
230
+ <span style="font-weight:600; color: #333;">Pipeline Cost:</span>
231
+ <span style="font-size:1.2em; color: #333;">${cost:.2f}</span>
232
+ <span style="font-size:0.85em; color: #555;">per 1k question prefixes</span>
233
+ <span style="margin-left:0.5em; color: #555;">({label})</span>
234
+ </span>
235
+ </div>
236
+ """
237
+ return gr.update(visible=True, value=html)
238
+
239
  # ------------------------------------- Agent Functions -----------------------------------------------------------
240
 
241
  def single_run(
 
400
  ],
401
  )
402
 
403
+ self.pipeline_state.change(
404
+ fn=self.refresh_model_cost,
405
+ inputs=[self.pipeline_state],
406
+ outputs=[self.model_cost_display],
407
+ )
408
+
409
  self.eval_btn.click(
410
  fn=self.evaluate,
411
  inputs=[self.pipeline_state],
src/components/quizbowl/tossup.py CHANGED
@@ -13,10 +13,11 @@ from components.model_pipeline.tossup_pipeline import TossupPipelineInterface, T
13
  from components.typed_dicts import TossupInterfaceDefaults, TossupPipelineStateDict
14
  from display.formatting import styled_error
15
  from shared.workflows import factory
16
- from shared.workflows.metrics import evaluate_prediction
17
  from shared.workflows.metrics.qb_metrics import prepare_tossup_results_df
18
  from shared.workflows.qb_agents import QuizBowlTossupAgent, TossupResult
19
  from shared.workflows.runners import run_and_eval_tossup_dataset, run_and_evaluate_tossup
 
20
  from submission import submit
21
 
22
  from . import populate
@@ -164,6 +165,7 @@ class TossupInterface:
164
  self.run_btn = gr.Button("Run on Tossup Question", variant="secondary")
165
  self.question_display = gr.HTML(label="Question", elem_id="tossup-question-display")
166
  self.error_display = gr.HTML(label="Error", elem_id="tossup-error-display", visible=False)
 
167
  with gr.Row():
168
  self.confidence_plot = gr.Plot(
169
  label="Buzz Confidence",
@@ -175,7 +177,6 @@ class TossupInterface:
175
  value=pd.DataFrame(columns=["Token Position", "Correct?", "Confidence", "Prediction"]),
176
  visible=False,
177
  )
178
-
179
  self.eval_btn, self.model_name_input, self.description_input, self.submit_btn, self.submit_status = (
180
  commons.get_model_submission_accordion(self.app)
181
  )
@@ -230,6 +231,55 @@ class TossupInterface:
230
  error_msg = styled_error(f"Error loading pipeline: {str(e)}")
231
  return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
232
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
  # ------------------------------------- Agent Functions -----------------------------------------------------------
234
 
235
  def single_run(
@@ -391,6 +441,12 @@ class TossupInterface:
391
  ],
392
  )
393
 
 
 
 
 
 
 
394
  self.eval_btn.click(
395
  fn=self.evaluate,
396
  inputs=[self.pipeline_state],
 
13
  from components.typed_dicts import TossupInterfaceDefaults, TossupPipelineStateDict
14
  from display.formatting import styled_error
15
  from shared.workflows import factory
16
+ from shared.workflows.metrics import evaluate_prediction, workflow_metrics
17
  from shared.workflows.metrics.qb_metrics import prepare_tossup_results_df
18
  from shared.workflows.qb_agents import QuizBowlTossupAgent, TossupResult
19
  from shared.workflows.runners import run_and_eval_tossup_dataset, run_and_evaluate_tossup
20
+ from shared.workflows.structs import TossupWorkflow
21
  from submission import submit
22
 
23
  from . import populate
 
165
  self.run_btn = gr.Button("Run on Tossup Question", variant="secondary")
166
  self.question_display = gr.HTML(label="Question", elem_id="tossup-question-display")
167
  self.error_display = gr.HTML(label="Error", elem_id="tossup-error-display", visible=False)
168
+ self.model_cost_display = gr.HTML(label="Pipeline Cost", value="", visible=False)
169
  with gr.Row():
170
  self.confidence_plot = gr.Plot(
171
  label="Buzz Confidence",
 
177
  value=pd.DataFrame(columns=["Token Position", "Correct?", "Confidence", "Prediction"]),
178
  visible=False,
179
  )
 
180
  self.eval_btn, self.model_name_input, self.description_input, self.submit_btn, self.submit_status = (
181
  commons.get_model_submission_accordion(self.app)
182
  )
 
231
  error_msg = styled_error(f"Error loading pipeline: {str(e)}")
232
  return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
233
 
234
+ def refresh_model_cost(self, pipeline_state: TossupPipelineStateDict):
235
+ """Refresh the model cost display with a styled HTML card and emoji."""
236
+ workflow = TossupWorkflow(**pipeline_state["workflow"])
237
+ cost = workflow_metrics.compute_workflow_cost(workflow)["cost"]
238
+ # Define color and emoji based on cost range
239
+ if cost < 1:
240
+ color = "#d4edda" # light green
241
+ emoji = "🟒"
242
+ label = "Ultra Light"
243
+ elif cost < 2:
244
+ color = "#cce5ff" # light blue
245
+ emoji = "πŸ”΅"
246
+ label = "Lightweight"
247
+ elif cost < 3:
248
+ color = "#fff3cd" # light yellow
249
+ emoji = "🟑"
250
+ label = "Moderate"
251
+ elif cost < 4:
252
+ color = "#ffe5b4" # light orange
253
+ emoji = "🟠"
254
+ label = "Expensive"
255
+ else:
256
+ color = "#f8d7da" # light red
257
+ emoji = "πŸ”΄"
258
+ label = "Very Expensive"
259
+
260
+ html = f"""
261
+ <div style="
262
+ background: {color};
263
+ border-radius: 10px;
264
+ padding: 1em 1.5em;
265
+ margin: 0.5em 0;
266
+ display: flex;
267
+ align-items: center;
268
+ box-shadow: 0 2px 8px rgba(0,0,0,0.04);
269
+ font-size: 1.1em;
270
+ font-weight: 500;
271
+ ">
272
+ <span style="font-size:1.5em; margin-right: 0.7em;">{emoji}</span>
273
+ <span>
274
+ <span style="font-weight:600; color: #333;">Pipeline Cost:</span>
275
+ <span style="font-size:1.2em; color: #333;">${cost:.2f}</span>
276
+ <span style="font-size:0.85em; color: #555;">per 1k question prefixes</span>
277
+ <span style="margin-left:0.5em; color: #555;">({label})</span>
278
+ </span>
279
+ </div>
280
+ """
281
+ return gr.update(visible=True, value=html)
282
+
283
  # ------------------------------------- Agent Functions -----------------------------------------------------------
284
 
285
  def single_run(
 
441
  ],
442
  )
443
 
444
+ self.pipeline_state.change(
445
+ fn=self.refresh_model_cost,
446
+ inputs=[self.pipeline_state],
447
+ outputs=[self.model_cost_display],
448
+ )
449
+
450
  self.eval_btn.click(
451
  fn=self.evaluate,
452
  inputs=[self.pipeline_state],