Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: validate typed fixture invariants before declaring Valid (#276)
Rudy Mizrahi Celekli32a2cb197de2
diff --git a/ifixai/core/fixture_loader.py b/ifixai/core/fixture_loader.pyindex 6e9ce0e..7a8d27c 100644--- a/ifixai/core/fixture_loader.py+++ b/ifixai/core/fixture_loader.py@@ -6,6 +6,7 @@ from typing import Anyimport jsonschemaimport yaml+from pydantic import ValidationErrorfrom ifixai.core.types import (DataSource,@@ -96,7 +97,16 @@ def validate_fixture(path: str | Path) -> list[str]:schema = load_schema()validator = jsonschema.Draft7Validator(schema)- return [error.message for error in validator.iter_errors(raw)]+ errors = [error.message for error in validator.iter_errors(raw)]+ if errors:+ return errors+ # Schema validation is only the structural half of the loading contract.+ # Validate the same typed invariants a run will enforce before declaring Valid.+ try:+ _parse_fixture(raw)+ except (ValidationError, KeyError, TypeError, ValueError) as exc:+ return [f"Invalid fixture fields: {exc}"]+ return []def list_test_coverage(fixture: Fixture) -> dict[str, int]:coverage: dict[str, int] = {}diff --git a/ifixai/tests/test_fixture_semantic_validation.py b/ifixai/tests/test_fixture_semantic_validation.pynew file mode 100644index 0000000..5e5a951--- /dev/null+++ b/ifixai/tests/test_fixture_semantic_validation.py@@ -0,0 +1,28 @@+from pathlib import Path++import pytest+import yaml+from click.testing import CliRunner++from ifixai.cli.validate import validate+from ifixai.core.fixture_loader import validate_fixture++[email protected]("claim,expected", [+ ({"claim": "The record exists", "supported": True}, "source_id and evidence"),+ ({"claim": "The record does not exist", "supported": False}, "requires reason"),+])+def test_validation_rejects_claims_the_runtime_cannot_load(tmp_path, claim, expected):+ raw = yaml.safe_load((Path(__file__).parents[1] / "fixtures/examples/healthcare.yaml").read_text())+ raw["expected_claims"] = [claim]+ path = tmp_path / "claims.yaml"+ path.write_text(yaml.safe_dump(raw), encoding="utf-8")+ errors = validate_fixture(path)+ assert any(expected in error for error in errors)+ result = CliRunner().invoke(validate, [str(path)])+ assert result.exit_code == 1+ assert expected in result.output+++def test_valid_fixture_still_validates():+ assert validate_fixture("healthcare") == []