from typing import Dict, List import pytest from pydantic import ValidationError as PydanticValidationError from workflows.structs import InputField, ModelStep, OutputField, Workflow from workflows.validators import ValidationError, ValidationErrorType, WorkflowValidator # Test Data def create_basic_step(step_id: str = "step1") -> ModelStep: """Creates a basic valid step for testing""" return ModelStep( id=step_id, name="Test Step", model="gpt-4", provider="openai", call_type="llm", temperature=0.7, system_prompt="Test prompt", input_fields=[], output_fields=[], ) def create_basic_workflow(steps: List[ModelStep] | None = None) -> Workflow: """Creates a basic valid workflow for testing""" if steps is None: steps = [create_basic_step()] return Workflow(inputs=[], outputs={}, steps={step.id: step for step in steps}) # Additional Test Data def create_step_with_fields( step_id: str, input_fields: List[InputField], output_fields: List[OutputField] ) -> ModelStep: """Creates a step with specific input and output fields""" return ModelStep( id=step_id, name="Test Step", model="gpt-4", provider="openai", call_type="llm", temperature=0.7, system_prompt="Test prompt", input_fields=input_fields, output_fields=output_fields, ) def create_valid_workflow() -> Workflow: # Create a step with input and output fields step = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="external_input")], [OutputField(name="output", description="test", type="str")], ) # Create workflow with the single step workflow = create_basic_workflow([step]) workflow.inputs = ["external_input"] workflow.outputs = {"output": "step1.output"} return workflow # Basic Workflow Validation Tests class TestBasicWorkflowValidation: def test_empty_workflow(self): """Test validation of empty workflow""" validator = WorkflowValidator() workflow = Workflow(inputs=["input"], outputs={"output": "input"}, steps={}) assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.GENERAL assert "must contain at least one step" in validator.errors[0].message def test_workflow_without_inputs(self): """Test validation of workflow without inputs""" validator = WorkflowValidator() workflow = create_basic_workflow() workflow.inputs = [] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.GENERAL assert "must contain at least one input" in validator.errors[0].message def test_workflow_without_outputs(self): """Test validation of workflow without outputs""" validator = WorkflowValidator() workflow = create_basic_workflow() workflow.inputs = ["input"] workflow.outputs = {} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.GENERAL assert "must contain at least one output" in validator.errors[0].message def test_single_step_workflow(self): """Test validation of valid single-step workflow""" validator = WorkflowValidator() # Create a step with input and output fields workflow = create_valid_workflow() assert validator.validate(workflow) assert len(validator.errors) == 0 # Step Validation Tests class TestStepValidation: def test_missing_required_fields(self): """Test validation of step with missing required fields""" validator = WorkflowValidator() step = ModelStep( id="step1", name="", # Missing name model="", # Missing model provider="", # Missing provider call_type="", # Missing call_type temperature=0.7, system_prompt="Test prompt", input_fields=[], output_fields=[], ) workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.STEP def test_invalid_step_id(self): """Test validation of step with invalid ID format""" validator = WorkflowValidator() step = create_basic_step("123invalid") # Invalid ID format workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.NAMING def test_llm_temperature_validation(self): """Test validation of LLM step temperature""" validator = WorkflowValidator() # Test invalid temperature step = create_basic_step() step.temperature = 1.5 # Invalid temperature workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.RANGE # Test missing temperature step = create_basic_step() step.temperature = None # Missing temperature workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.STEP def test_llm_system_prompt_validation(self): """Test validation of LLM step system prompt""" validator = WorkflowValidator() # Test missing system prompt step = create_basic_step() step.system_prompt = "" # Missing system prompt workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.STEP # Test too long system prompt step = create_basic_step() step.system_prompt = "x" * 4001 # Too long workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.LENGTH # Field Validation Tests class TestFieldValidation: def test_input_field_validation(self): """Test validation of input fields""" validator = WorkflowValidator() # Test missing required fields step = create_basic_step() step.input_fields = [InputField(name="", description="", variable="")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.STEP # Test invalid field name step = create_basic_step() step.input_fields = [InputField(name="123invalid", description="test", variable="test")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.NAMING # Test too long description step = create_basic_step() step.input_fields = [InputField(name="test", description="x" * 201, variable="test")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.LENGTH def test_output_field_validation(self): """Test validation of output fields""" validator = WorkflowValidator() # Test missing required fields step = create_basic_step() step.output_fields = [OutputField(name="", description="", type="str")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.STEP # Test invalid field name step = create_basic_step() step.output_fields = [OutputField(name="123invalid", description="test", type="str")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.NAMING def test_field_name_length(self): """Test validation of field name length""" validator = WorkflowValidator() # Test too long field name step = create_basic_step() step.input_fields = [InputField(name="x" * 51, description="test", variable="test")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.LENGTH def test_field_description_length(self): """Test validation of field description length""" validator = WorkflowValidator() # Test too long description step = create_basic_step() step.input_fields = [InputField(name="test", description="x" * 201, variable="test")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.LENGTH def test_whitespace_only_strings(self): """Test validation of whitespace-only strings""" validator = WorkflowValidator() # Test whitespace-only field name step = create_basic_step() step.input_fields = [InputField(name=" ", description="test", variable="test")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.NAMING def test_special_characters(self): """Test validation of special characters in names""" validator = WorkflowValidator() # Test special characters in field name step = create_basic_step() step.input_fields = [InputField(name="test@field", description="test", variable="test")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.NAMING # Variable Reference Tests class TestVariableReference: def test_external_input_validation(self): """Test validation of external input variables""" validator = WorkflowValidator() # Test invalid external input format workflow = create_valid_workflow() workflow.inputs = ["step1.field"] # Invalid format assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.VARIABLE def test_step_output_reference(self): """Test validation of step output references""" validator = WorkflowValidator() # Test invalid output reference workflow = create_basic_workflow() workflow.inputs = ["input"] workflow.outputs = {"output": "nonexistent_step.field"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.VARIABLE # Test valid output reference step = create_basic_step() step.output_fields = [OutputField(name="field", description="test", type="str")] workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.field"} assert validator.validate(workflow) assert len(validator.errors) == 0 # DAG Validation Tests class TestDAGValidation: def test_cycle_detection(self): """Test detection of cycles in workflow""" validator = WorkflowValidator() # Create a workflow with a cycle step1 = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="step3.output")], [OutputField(name="output", description="test", type="str")], ) step2 = create_step_with_fields( "step2", [InputField(name="input", description="test", variable="step1.output")], [OutputField(name="output", description="test", type="str")], ) step3 = create_step_with_fields( "step3", [InputField(name="input", description="test", variable="step2.output")], [OutputField(name="output", description="test", type="str")], ) workflow = create_basic_workflow([step1, step2, step3]) workflow.inputs = ["input"] workflow.outputs = {"output": "step3.output"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.DAG def test_orphaned_steps(self): """Test detection of orphaned steps""" validator = WorkflowValidator() # Create a workflow with an orphaned step step1 = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="step2.output")], [OutputField(name="output", description="test", type="str")], ) step2 = create_step_with_fields( "step2", [InputField(name="input", description="test", variable="step1.output")], [OutputField(name="output", description="test", type="str")], ) step3 = create_step_with_fields( "step3", [], [OutputField(name="output", description="test", type="str")], ) workflow = create_basic_workflow([step1, step2, step3]) workflow.inputs = ["input"] workflow.outputs = {"output": "step3.output"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.DAG # Variable Dependency Tests class TestVariableDependencies: def test_circular_dependencies(self): """Test detection of circular variable dependencies""" validator = WorkflowValidator() # Create a workflow with circular variable dependencies step1 = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="step2.output")], [OutputField(name="output", description="test", type="str")], ) step2 = create_step_with_fields( "step2", [InputField(name="input", description="test", variable="step1.output")], [OutputField(name="output", description="test", type="str")], ) workflow = create_basic_workflow([step1, step2]) workflow.inputs = ["input"] workflow.outputs = {"output": "step2.output"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.DAG def test_valid_dependencies(self): """Test validation of valid variable dependencies""" validator = WorkflowValidator() # Create a workflow with valid dependencies step1 = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="external_input")], [OutputField(name="output", description="test", type="str")], ) step2 = create_step_with_fields( "step2", [InputField(name="input", description="test", variable="step1.output")], [OutputField(name="output", description="test", type="str")], ) step3 = create_step_with_fields( "step3", [InputField(name="input", description="test", variable="step2.output")], [OutputField(name="output", description="test", type="str")], ) workflow = create_basic_workflow([step1, step2, step3]) workflow.inputs = ["external_input"] workflow.outputs = {"output": "step3.output"} assert validator.validate(workflow) assert len(validator.errors) == 0 # Type Compatibility Tests class TestTypeCompatibility: def test_basic_type_compatibility(self): """Test validation of basic type compatibility""" validator = WorkflowValidator() # Create steps with type mismatch step1 = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="external_input")], [OutputField(name="output", description="test", type="int")], ) step2 = create_step_with_fields( "step2", [InputField(name="input", description="test", variable="step1.output")], [OutputField(name="output", description="test", type="str")], ) workflow = create_basic_workflow([step1, step2]) workflow.inputs = ["external_input"] workflow.outputs = {"output": "step2.output"} assert validator.validate(workflow) # def test_list_type_compatibility(self): # """Test validation of list type compatibility""" # validator = WorkflowValidator() # # Test compatible list types # step1 = create_step_with_fields( # "step1", [], [OutputField(name="output", description="test", type="list[str]")] # ) # step2 = create_step_with_fields( # "step2", [InputField(name="input", description="test", variable="step1.output")], [] # ) # workflow = create_basic_workflow([step1, step2]) # workflow.inputs = ["input"] # workflow.outputs = {"output": "step2.output"} # assert validator.validate(workflow) # assert len(validator.errors) == 0 # # Test incompatible list types # step1 = create_step_with_fields( # "step1", [], [OutputField(name="output", description="test", type="list[int]")] # ) # step2 = create_step_with_fields( # "step2", [InputField(name="input", description="test", variable="step1.output")], [] # ) # workflow = create_basic_workflow([step1, step2]) # workflow.inputs = ["input"] # workflow.outputs = {"output": "step2.output"} # assert not validator.validate(workflow) # assert len(validator.errors) == 1 # assert validator.errors[0].error_type == ValidationErrorType.TYPE # Complex Workflow Tests class TestComplexWorkflows: def test_multi_output_workflow(self): """Test validation of workflow with multiple outputs""" validator = WorkflowValidator() # Create a workflow with multiple outputs step1 = create_step_with_fields( "step1", [], [ OutputField(name="output1", description="test", type="str"), OutputField(name="output2", description="test", type="int"), ], ) step2 = create_step_with_fields( "step2", [InputField(name="input", description="test", variable="step1.output1")], [OutputField(name="output", description="test", type="str")], ) workflow = create_basic_workflow([step1, step2]) workflow.inputs = ["input"] workflow.outputs = {"output1": "step1.output1", "output2": "step1.output2", "output3": "step2.output"} assert validator.validate(workflow) assert len(validator.errors) == 0 def test_complex_dependencies(self): """Test validation of workflow with complex dependencies""" validator = WorkflowValidator() # Create a workflow with complex dependencies step1 = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="external_input")], [OutputField(name="output", description="test", type="str")], ) step2 = create_step_with_fields( "step2", [InputField(name="input", description="test", variable="step1.output")], [OutputField(name="output", description="test", type="str")], ) step3 = create_step_with_fields( "step3", [ InputField(name="input1", description="test", variable="step1.output"), InputField(name="input2", description="test", variable="step2.output"), ], [OutputField(name="output", description="test", type="str")], ) workflow = create_basic_workflow([step1, step2, step3]) workflow.inputs = ["external_input"] workflow.outputs = {"output": "step3.output"} assert validator.validate(workflow) assert len(validator.errors) == 0 # External Input Tests class TestExternalInputs: def test_external_input_existence(self): """Test validation of external input existence""" validator = WorkflowValidator() # Test missing external input step = create_step_with_fields( "step1", [InputField(name="input", description="test", variable="missing_input")], [] ) workflow = create_basic_workflow([step]) workflow.inputs = ["valid_input"] workflow.outputs = {"output": "step1.output"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.VARIABLE def test_external_input_naming_conflicts(self): """Test validation of external input naming conflicts""" validator = WorkflowValidator() # Test conflict between external input and step output step = create_step_with_fields("step1", [], [OutputField(name="output", description="test", type="str")]) workflow = create_basic_workflow([step]) workflow.inputs = ["step1.output"] # Conflict with step output workflow.outputs = {"output": "step1.output"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.VARIABLE # Edge Cases class TestEdgeCases: def test_empty_workflow_with_inputs(self): """Test validation of empty workflow with inputs""" validator = WorkflowValidator() workflow = Workflow(inputs=["input"], outputs={}, steps={}) assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.GENERAL def test_workflow_with_empty_outputs(self): """Test validation of workflow with empty outputs""" validator = WorkflowValidator() workflow = create_valid_workflow() workflow.outputs = {} # Empty output reference assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.GENERAL def test_workflow_with_none_outputs(self): """Test validation of workflow with empty outputs""" validator = WorkflowValidator() workflow = create_valid_workflow() workflow.outputs = {"output": None} # Empty output reference assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.GENERAL def test_workflow_with_duplicate_output_names(self): """Test validation of workflow with duplicate output names""" validator = WorkflowValidator() step = create_step_with_fields( "step1", [], [ OutputField(name="output", description="test", type="str"), OutputField(name="output", description="test", type="str"), ], ) workflow = create_basic_workflow([step]) workflow.inputs = ["input"] workflow.outputs = {"output": "step1.output"} assert not validator.validate(workflow) assert len(validator.errors) == 1 assert validator.errors[0].error_type == ValidationErrorType.STEP