Restructure output panel rendering. This change fixes logprob issue and

src/app_configs.py

@@ -38,20 +38,33 @@ AVAILABLE_MODELS = {
     },
 }
 
+CONFIGS = {
+    "tossup": {
+        "required_input_vars": ["question_text"],
+        "required_output_vars": ["answer", "confidence"],
+    },
+    "bonus": {
+        "required_input_vars": ["leadin", "part"],
+        "required_output_vars": ["answer", "confidence", "explanation"],
+    },
+}
+
 DEFAULT_SELECTIONS = {
     "tossup": {
         "simple_workflow": False,
         "model": "OpenAI/gpt-4o-mini",
-        "temperature": 0.2,
+        "temperature": 0.7,
         "confidence_threshold": 0.85,
         "early_stop": True,
+        "max_temperature": 5.0,
     },
     "bonus": {
         "simple_workflow": False,
         "model": "OpenAI/gpt-4o-mini",
-        "temperature": 0.2,
+        "temperature": 0.7,
         "confidence_threshold": 0.85,
         "early_stop": True,
+        "max_temperature": 5.0,
     },
 }
 

src/components/model_pipeline/model_pipeline.py

@@ -18,6 +18,10 @@ from components.utils import make_state
 from workflows.structs import ModelStep, TossupWorkflow, Workflow
 from workflows.validators import WorkflowValidator
 
+from .state_manager import get_output_panel_state
+
+DEFAULT_MAX_TEMPERATURE = 5.0
+
 
 class PipelineInterface:
     """UI for the pipeline."""
@@ -28,17 +32,20 @@ class PipelineInterface:
         workflow: Workflow,
         ui_state: PipelineUIState | None = None,
         model_options: list[str] = None,
-        simple: bool = False,
+        config: dict = {},
     ):
         self.app = app
         self.model_options = model_options
-        self.simple = simple
+        self.config = config
+        self.simple = self.config.get("simple", False)
         ui_state = ui_state or PipelineUIState.from_workflow(workflow)
 
         # Gradio States
         self.workflow_state = make_state(workflow.model_dump())
         self.variables_state = make_state(workflow.get_available_variables())
-        self.model_selection_state = make_state(workflow.get_model_selections())
+        self.output_panel_state = make_state(get_output_panel_state(workflow))
+
+        # Maintains the toggle state change for pipeline changes through user input.
         self.pipeline_change = gr.State(False)
 
         if isinstance(workflow, TossupWorkflow):
@@ -51,24 +58,19 @@ class PipelineInterface:
 
         def get_aux_states(pipeline_state_dict: td.PipelineStateDict):
             """Get the auxiliary states for the pipeline."""
+            logger.debug("Pipeline changed! Getting aux states for pipeline state.")
             pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
             return (
                 pipeline_state.workflow.model_dump(),
                 pipeline_state.workflow.get_available_variables(),
-                pipeline_state.workflow.get_model_selections(),
+                get_output_panel_state(pipeline_state.workflow),
             )
 
         # Triggers for pipeline state changes
         self.pipeline_state.change(
             get_aux_states,
             inputs=[self.pipeline_state],
-            outputs=[self.workflow_state, self.variables_state, self.model_selection_state],
-        )
-
-        self.workflow_state.change(
-            lambda x: logger.debug(f"Workflow state changed: {x}"),
-            inputs=[self.workflow_state],
-            outputs=[],
+            outputs=[self.workflow_state, self.variables_state, self.output_panel_state],
         )
 
         # IO Variables
@@ -98,6 +100,7 @@ class PipelineInterface:
                 model_options=self.model_options,
                 input_variables=available_variables,
                 pipeline_state_manager=self.sm,
+                max_temperature=self.config.get("max_temperature", DEFAULT_MAX_TEMPERATURE),
             )
 
             step_interface.on_model_step_change(
@@ -259,14 +262,14 @@ class PipelineInterface:
             self._render_add_step_button(-1)
 
         @gr.render(
-            triggers=[self.workflow_state.change, self.app.load],
+            triggers=[self.output_panel_state.change, self.app.load],
             inputs=[self.pipeline_state],
             concurrency_limit=1,
             concurrency_id="render_output_fields",
         )
         def render_output_fields(pipeline_state_dict: td.PipelineStateDict):
             pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
-            logger.info(f"Rerendering output panel: {pipeline_state.workflow}")
+            logger.debug(f"Rerendering output panel: {get_output_panel_state(pipeline_state.workflow)}")
             self._render_output_panel(pipeline_state)
 
         export_btn = gr.Button("Export Pipeline", elem_classes="export-button")

src/components/model_pipeline/state_manager.py

@@ -8,7 +8,18 @@ from components import typed_dicts as td
 from components import utils
 from components.structs import ModelStepUIState, PipelineState, PipelineUIState, TossupPipelineState
 from workflows.factory import create_new_llm_step
-from workflows.structs import Buzzer, ModelStep, TossupWorkflow, Workflow
+from workflows.structs import Buzzer, BuzzerMethod, ModelStep, TossupWorkflow, Workflow
+
+
+def get_output_panel_state(workflow: Workflow) -> dict:
+    state = {
+        "variables": workflow.get_available_variables(),
+        "models": workflow.get_step_model_selections(),
+        "output_models": workflow.get_output_model_selections(),
+    }
+    if isinstance(workflow, TossupWorkflow):
+        state["buzzer"] = workflow.buzzer.model_dump(exclude_defaults=False)
+    return state
 
 
 class PipelineStateManager:
@@ -134,6 +145,26 @@ class TossupPipelineStateManager(PipelineStateManager):
         workflow = self.parse_yaml_workflow(yaml_str)
         return TossupPipelineState.from_workflow(workflow).model_dump(), not change_state
 
+    def update_model_step_state(
+        self, state_dict: td.TossupPipelineStateDict, model_step: ModelStep, ui_state: ModelStepUIState
+    ) -> td.TossupPipelineStateDict:
+        """Update a particular model step in the pipeline."""
+        state = self.make_pipeline_state(state_dict)
+        state = state.update_step(model_step, ui_state)
+        state.workflow = state.workflow.refresh_buzzer()
+        return state.model_dump()
+
+    def update_output_variables(
+        self, state_dict: td.TossupPipelineStateDict, target: str, produced_variable: str
+    ) -> td.TossupPipelineStateDict:
+        if produced_variable == UNSELECTED_VAR_NAME:
+            produced_variable = None
+        """Update the output variables for a step."""
+        state = self.make_pipeline_state(state_dict)
+        state.workflow.outputs[target] = produced_variable
+        state.workflow = state.workflow.refresh_buzzer()
+        return state.model_dump()
+
     def update_buzzer(
         self,
         state_dict: td.TossupPipelineStateDict,
@@ -144,6 +175,8 @@ class TossupPipelineStateManager(PipelineStateManager):
         """Update the buzzer."""
         state = self.make_pipeline_state(state_dict)
         prob_threshold = float(tokens_prob) if tokens_prob and tokens_prob > 0 else None
+        if method == BuzzerMethod.OR and prob_threshold is None:
+            prob_threshold = 0.0
         state.workflow.buzzer = Buzzer(
             method=method, confidence_threshold=confidence_threshold, prob_threshold=prob_threshold
         )

src/components/model_pipeline/tossup_pipeline.py

@@ -10,23 +10,6 @@ from display.formatting import tiny_styled_warning
 from workflows.structs import Buzzer, TossupWorkflow
 
 from .model_pipeline import PipelineInterface, PipelineState, PipelineUIState
-from .state_manager import PipelineStateManager, TossupPipelineStateManager
-
-
-def get_probs_model_name(workflow: TossupWorkflow, answer_var: str | None = None) -> str | None:
-    if answer_var is None:
-        answer_var = workflow.outputs["answer"]
-    if answer_var is None or answer_var == UNSELECTED_VAR_NAME:
-        return None
-    step_id = answer_var.split(".")[0]
-    return workflow.steps[step_id].get_full_model_name()
-
-
-def is_logprobs_supported(workflow: TossupWorkflow, answer_var: str | None = None) -> bool:
-    model_name = get_probs_model_name(workflow, answer_var)
-    if model_name is None:
-        return True
-    return AVAILABLE_MODELS[model_name].get("logprobs", False)
 
 
 def toggleable_slider(
@@ -56,17 +39,16 @@ class TossupPipelineInterface(PipelineInterface):
         workflow: TossupWorkflow,
         ui_state: PipelineUIState | None = None,
         model_options: list[str] = None,
-        simple: bool = False,
-        defaults: dict = {},
+        config: dict = {},
     ):
-        super().__init__(app, workflow, ui_state, model_options, simple)
-        self.defaults = defaults
+        super().__init__(app, workflow, ui_state, model_options, config)
+
+        self.buzzer_state = gr.State(workflow.buzzer.model_dump())
 
         self.pipeline_state.change(
-            lambda x: logger.debug(
-                f"Pipeline state changed. Type: {type(x)}. Has buzzer info: {x['workflow']['buzzer'] if isinstance(x, dict) else 'N/A'}"
-            ),
+            lambda x: TossupPipelineState(**x).workflow.buzzer.model_dump(),
             inputs=[self.pipeline_state],
+            outputs=[self.buzzer_state],
         )
 
     def update_prob_slider(
@@ -81,7 +63,7 @@ class TossupPipelineInterface(PipelineInterface):
                 gr.update(value="AND", interactive=True),
                 gr.update(visible=False),
             )
-        logprobs_supported = is_logprobs_supported(state.workflow, answer_var)
+        logprobs_supported = state.workflow.is_token_probs_supported(answer_var)
         buzzer = state.workflow.buzzer
         tokens_prob_threshold = tokens_prob if logprobs_supported else None
         method = buzzer.method if logprobs_supported else "AND"
@@ -90,10 +72,10 @@ class TossupPipelineInterface(PipelineInterface):
             confidence_threshold=buzzer.confidence_threshold,
             prob_threshold=tokens_prob_threshold,
         )
-        model_name = get_probs_model_name(state.workflow, answer_var)
+        model_name = state.workflow.get_answer_model(answer_var)
         return (
             state.model_dump(),
-            gr.update(interactive=logprobs_supported),
+            gr.update(interactive=logprobs_supported, value=tokens_prob if logprobs_supported else 0.0),
             gr.update(value=method, interactive=logprobs_supported),
             gr.update(
                 value=tiny_styled_warning(
@@ -172,20 +154,21 @@ class TossupPipelineInterface(PipelineInterface):
                 header="Buzzer settings:",
                 subheader="Set your thresholds for confidence and output tokens probability (computed using <code>logprobs</code>).",
             )
-            logprobs_supported = is_logprobs_supported(pipeline_state.workflow)
-            selected_model_name = get_probs_model_name(pipeline_state.workflow)
+            logprobs_supported = pipeline_state.workflow.is_token_probs_supported()
+            selected_model_name = pipeline_state.workflow.get_answer_model()
             self._render_buzzer_panel(pipeline_state.workflow.buzzer, logprobs_supported, selected_model_name)
 
-        # def update_choices(available_variables: list[str]):
-        #     """Update the choices for the dropdowns"""
-        #     return [gr.update(choices=available_variables, value=None, selected=None) for _ in dropdowns.values()]
+        def update_choices(available_variables: list[str]):
+            """Update the choices for the dropdowns"""
+            return [gr.update(choices=available_variables, value=None, selected=None) for _ in dropdowns.values()]
 
-        # self.variables_state.change(
-        #     update_choices,
-        #     inputs=[self.variables_state],
-        #     outputs=list(dropdowns.values()),
-        # )
+        self.variables_state.change(
+            update_choices,
+            inputs=[self.variables_state],
+            outputs=list(dropdowns.values()),
+        )
 
+        # Updating the pipeline buzzer on user input changes in Buzzer panel.
         gr.on(
             triggers=[
                 self.confidence_slider.release,
@@ -197,17 +180,17 @@ class TossupPipelineInterface(PipelineInterface):
             outputs=[self.pipeline_state],
         )
 
-        # TODO: Do Add model step change triggers as well. (Model name change triggers)
-        answer_dropdown = dropdowns["answer"]
-        if answer_dropdown is not None:
-            gr.on(
-                triggers=[answer_dropdown.input, self.model_selection_state.change],
-                fn=self.update_prob_slider,
-                inputs=[self.pipeline_state, answer_dropdown, self.prob_slider],
-                outputs=[
-                    self.pipeline_state,
-                    self.prob_slider,
-                    self.buzzer_method_dropdown,
-                    self.buzzer_warning_display,
-                ],
-            )
+        # THIS WASN't NEEDED SINCE WE RERENDER THE OUTPUT PANEL ENTIRELY ON CHANGES
+        # answer_dropdown = dropdowns["answer"]
+        # if answer_dropdown is not None:
+        #     gr.on(
+        #         triggers=[self.buzzer_state.change],
+        #         fn=self.update_prob_slider,
+        #         inputs=[self.buzzer_state, answer_dropdown, self.prob_slider],
+        #         outputs=[
+        #             self.pipeline_state,
+        #             self.prob_slider,
+        #             self.buzzer_method_dropdown,
+        #             self.buzzer_warning_display,
+        #         ],
+        #     )

src/components/model_step/model_step.py

@@ -44,6 +44,7 @@ class ModelStepComponent(FormComponent):
         input_variables: list[str] | None = None,
         max_input_fields=5,
         max_output_fields=5,
+        max_temperature=5.0,
         pipeline_state_manager: PipelineStateManager | None = None,
         **kwargs,
     ):
@@ -51,6 +52,7 @@ class ModelStepComponent(FormComponent):
             "input": max_input_fields,
             "output": max_output_fields,
         }
+        self.max_temperature = max_temperature
         self.model_options = model_options
         self.input_variables = [UNSELECTED_VAR_NAME] + input_variables
         self.sm = ModelStepStateManager(max_input_fields, max_output_fields)
@@ -238,7 +240,7 @@ class ModelStepComponent(FormComponent):
             self.temperature_slider = gr.Slider(
                 value=self.model_step.temperature,
                 minimum=0.0,
-                maximum=5,
+                maximum=self.max_temperature,
                 step=0.05,
                 info="Temperature",
                 show_label=False,

src/workflows/structs.py

@@ -257,7 +257,7 @@ class Workflow(BaseModel):
             variables.update(self.get_step_variables(step.id))
         return list(variables)
 
-    def get_model_selections(self) -> dict[str, str]:
+    def get_step_model_selections(self) -> dict[str, str]:
         """Get all model selections for all steps."""
         return {step_id: step.get_full_model_name() for step_id, step in self.steps.items()}