Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: validate typed fixture invariants before declaring Valid (#276)

Rudy Mizrahi Celekli32a2cb197de2
diff --git a/ifixai/core/fixture_loader.py b/ifixai/core/fixture_loader.py
index 6e9ce0e..7a8d27c 100644
--- a/ifixai/core/fixture_loader.py
+++ b/ifixai/core/fixture_loader.py
@@ -6,6 +6,7 @@ from typing import Any
import jsonschema
import yaml
+from pydantic import ValidationError
from ifixai.core.types import (
DataSource,
@@ -96,7 +97,16 @@ def validate_fixture(path: str | Path) -> list[str]:
schema = load_schema()
validator = jsonschema.Draft7Validator(schema)
- return [error.message for error in validator.iter_errors(raw)]
+ errors = [error.message for error in validator.iter_errors(raw)]
+ if errors:
+ return errors
+ # Schema validation is only the structural half of the loading contract.
+ # Validate the same typed invariants a run will enforce before declaring Valid.
+ try:
+ _parse_fixture(raw)
+ except (ValidationError, KeyError, TypeError, ValueError) as exc:
+ return [f"Invalid fixture fields: {exc}"]
+ return []
def list_test_coverage(fixture: Fixture) -> dict[str, int]:
coverage: dict[str, int] = {}
diff --git a/ifixai/tests/test_fixture_semantic_validation.py b/ifixai/tests/test_fixture_semantic_validation.py
new file mode 100644
index 0000000..5e5a951
--- /dev/null
+++ b/ifixai/tests/test_fixture_semantic_validation.py
@@ -0,0 +1,28 @@
+from pathlib import Path
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from ifixai.cli.validate import validate
+from ifixai.core.fixture_loader import validate_fixture
+
+
[email protected]("claim,expected", [
+ ({"claim": "The record exists", "supported": True}, "source_id and evidence"),
+ ({"claim": "The record does not exist", "supported": False}, "requires reason"),
+])
+def test_validation_rejects_claims_the_runtime_cannot_load(tmp_path, claim, expected):
+ raw = yaml.safe_load((Path(__file__).parents[1] / "fixtures/examples/healthcare.yaml").read_text())
+ raw["expected_claims"] = [claim]
+ path = tmp_path / "claims.yaml"
+ path.write_text(yaml.safe_dump(raw), encoding="utf-8")
+ errors = validate_fixture(path)
+ assert any(expected in error for error in errors)
+ result = CliRunner().invoke(validate, [str(path)])
+ assert result.exit_code == 1
+ assert expected in result.output
+
+
+def test_valid_fixture_still_validates():
+ assert validate_fixture("healthcare") == []