Enhances model selection and logging in pipeline components; adds logprobs support and improves UI feedback for disabled sliders.

src/app_configs.py

@@ -6,9 +6,11 @@ UNSELECTED_PIPELINE_NAME = "Select Pipeline to Import..."
 AVAILABLE_MODELS = {
     "OpenAI/gpt-4o": {
         "model": "gpt-4o-2024-11-20",
+        "logprobs": True,
     },
     "OpenAI/gpt-4o-mini": {
         "model": "gpt-4o-mini-2024-07-18",
+        "logprobs": True,
     },
     "OpenAI/gpt-3.5-turbo": {
         "model": "gpt-3.5-turbo-0125",

src/components/model_pipeline/model_pipeline.py

@@ -74,6 +74,7 @@ class PipelineInterface:
         self.ui_state = make_state(ui_state)
         self.pipeline_state = make_state(PipelineState(workflow=workflow, ui_state=ui_state))
         self.variables_state = make_state(workflow.get_available_variables())
+        self.model_selection_state = make_state({})
 
         self.sm = PipelineStateManager()
         self.input_variables = workflow.inputs
@@ -107,7 +108,7 @@ class PipelineInterface:
             step_interface.on_model_step_change(
                 self.sm.update_model_step_state,
                 inputs=[self.pipeline_state, step_interface.model_step_state, step_interface.ui_state],
-                outputs=[self.pipeline_state, self.ui_state, self.variables_state],
+                outputs=[self.pipeline_state, self.ui_state, self.variables_state, self.model_selection_state],
             )
 
             step_interface.on_ui_change(
@@ -239,7 +240,7 @@ class PipelineInterface:
         @gr.render(inputs=[self.pipeline_state, self.ui_state], concurrency_limit=1, concurrency_id="render_steps")
         def render_steps(state: PipelineState, ui_state: PipelineUIState):
             """Render all steps in the pipeline"""
-            logger.info(f"\nRerender triggered! Current UI State:{ui_state.model_dump()}")
+            logger.info(f"Rerender triggered! Current UI State:{ui_state.model_dump()}")
             workflow = state.workflow
             components = []
 
@@ -262,6 +263,7 @@ class PipelineInterface:
             concurrency_id="render_output_fields",
         )
         def render_output_fields(available_variables, pipeline_state):
+            logger.info(f"Rerendering output panel: {available_variables} {pipeline_state.workflow}")
             self._render_output_panel(available_variables, pipeline_state)
 
         export_btn = gr.Button("Export Pipeline", elem_classes="export-button")

src/components/model_pipeline/state_manager.py

@@ -176,7 +176,8 @@ class PipelineStateManager:
         state.ui_state.steps[model_step.id] = ui_state.model_copy()
         state.ui_state = state.ui_state.model_copy()
         state.update_output_variables_mapping()
-        return state, state.ui_state, state.available_variables
+        model_selections = {step_id: step.get_full_model_name() for step_id, step in state.workflow.steps.items()}
+        return state, state.ui_state, state.available_variables, model_selections
 
     def update_output_variables(self, state: PipelineState, target: str, produced_variable: str):
         if produced_variable == "Choose variable...":

src/components/model_pipeline/tossup_pipeline.py

@@ -1,8 +1,10 @@
 import gradio as gr
 import numpy as np
+from loguru import logger
 
 from app_configs import AVAILABLE_MODELS, UNSELECTED_VAR_NAME
 from components import commons
+from display.formatting import tiny_styled_warning
 from workflows.structs import Buzzer, TossupWorkflow
 
 from .model_pipeline import PipelineInterface, PipelineState, PipelineUIState
@@ -68,20 +70,36 @@ class TossupPipelineInterface(PipelineInterface):
     def update_prob_slider(self, state: TossupPipelineState, answer_var: str, tokens_prob: float | None):
         """Update the probability slider based on the answer variable."""
         if answer_var == UNSELECTED_VAR_NAME:
-            return gr.update(interactive=True)
+            return (
+                state,
+                gr.update(interactive=True),
+                gr.update(value="AND", interactive=True),
+                gr.update(visible=False),
+            )
         step_id = answer_var.split(".")[0]
-        model_name = state.workflow.steps[step_id].model
+        model_name = state.workflow.steps[step_id].get_full_model_name()
         model_config = AVAILABLE_MODELS[model_name]
         is_model_with_logprobs = model_config.get("logprobs", False)
         buzzer = state.workflow.buzzer
         tokens_prob_threshold = tokens_prob if is_model_with_logprobs else None
+        method = buzzer.method if is_model_with_logprobs else "AND"
         state = self.update_buzzer(
             state,
             confidence_threshold=buzzer.confidence_threshold,
-            method=buzzer.method,
+            method=method,
             tokens_prob=tokens_prob_threshold,
         )
-        return state, gr.update(interactive=not is_model_with_logprobs)
+        return (
+            state,
+            gr.update(interactive=is_model_with_logprobs),
+            gr.update(value=method, interactive=is_model_with_logprobs),
+            gr.update(
+                value=tiny_styled_warning(
+                    f"{model_name} does not support `logprobs`. The probability slider will be disabled."
+                ),
+                visible=not is_model_with_logprobs,
+            ),
+        )
 
     def _render_output_panel(self, available_variables: list[str], pipeline_state: TossupPipelineState):
         dropdowns = {}
@@ -136,6 +154,7 @@ class TossupPipelineInterface(PipelineInterface):
                     step=0.001,
                     elem_classes="slider-container",
                 )
+            self.buzzer_warning_display = gr.HTML(visible=False)
 
         def update_choices(available_variables):
             """Update the choices for the dropdowns"""
@@ -168,8 +187,14 @@ class TossupPipelineInterface(PipelineInterface):
         # TODO: Do Add model step change triggers as well. (Model name change triggers)
         answer_dropdown = dropdowns["answer"]
         if answer_dropdown is not None:
-            answer_dropdown.change(
-                self.update_prob_slider,
+            gr.on(
+                triggers=[answer_dropdown.input, self.model_selection_state.change],
+                fn=self.update_prob_slider,
                 inputs=[self.pipeline_state, answer_dropdown, self.prob_slider],
-                outputs=[self.pipeline_state, self.prob_slider],
+                outputs=[
+                    self.pipeline_state,
+                    self.prob_slider,
+                    self.buzzer_method_dropdown,
+                    self.buzzer_warning_display,
+                ],
             )

src/components/model_step/model_step.py

@@ -304,7 +304,7 @@ class ModelStepComponent(FormComponent):
         )
 
         # Model and system prompt
-        self.model_selection.change(
+        self.model_selection.input(
             fn=self.sm.update_model_and_provider,
             inputs=[self.model_step_state, self.model_selection],
             outputs=[self.model_step_state],
@@ -442,11 +442,11 @@ class ModelStepComponent(FormComponent):
 
     def on_model_step_change(self, fn, inputs, outputs):
         """Set up an event listener for the model change event."""
-        self.model_step_state.change(fn, inputs, outputs)
+        return self.model_step_state.change(fn, inputs, outputs)
 
     def on_ui_change(self, fn, inputs, outputs):
         """Set up an event listener for the UI change event."""
-        self.ui_state.change(fn, inputs, outputs)
+        return self.ui_state.change(fn, inputs, outputs)
 
     def _update_state_and_label(self, model_step: ModelStep, name: str):
         """Update both the state and the accordion label."""

src/display/custom_css.py

@@ -62,6 +62,10 @@ css_pipeline = """
     gap: var(--spacing-md) !important;
 }
 
+input[type=range][disabled] {
+    opacity: .3;
+}
+
 .json-node {
     /* On a light background (usually white), use darker and vivid colors */
     font-size: var(--text-sm) !important;

src/display/formatting.py

@@ -15,6 +15,10 @@ def styled_warning(warn):
     return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
 
 
+def tiny_styled_warning(warn):
+    return f"<p style='color: red; font-size: 12px; text-align: left;'>{warn}</p>"
+
+
 def styled_message(message):
     return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
 

src/workflows/structs.py

@@ -113,8 +113,8 @@ class ModelStep(BaseModel):
     def fields(self, field_type: FieldType) -> list[InputField | OutputField]:
         return self.input_fields if field_type == "input" else self.output_fields
 
-    def get_full_model_name(self):
-        return f"{self.provider} {self.model}"
+    def get_full_model_name(self) -> str:
+        return f"{self.provider}/{self.model}"
 
     def get_produced_variables(self) -> list[str]:
         return [f"{self.id}.{field.name}" for field in self.output_fields if field.name]