Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

Report malformed fixture files as validation errors (#140)

Rudy Mizrahi Celeklibdfb8cf6c86b
diff --git a/ifixai/core/fixture_loader.py b/ifixai/core/fixture_loader.py
index 4614956..cadc4dc 100644
--- a/ifixai/core/fixture_loader.py
+++ b/ifixai/core/fixture_loader.py
@@ -81,8 +81,16 @@ def validate_fixture(path: str | Path) -> list[str]:
except FileNotFoundError as exc:
return [str(exc)]
- raw = _read_fixture_file(fixture_path)
- raw = _normalize_fixture_format(raw)
+ try:
+ raw = _read_fixture_file(fixture_path)
+ raw = _normalize_fixture_format(raw)
+ except yaml.YAMLError as exc:
+ return [f"Invalid YAML: {exc}"]
+ except json.JSONDecodeError as exc:
+ return [f"Invalid JSON: {exc}"]
+ except (FixtureValidationError, OSError, UnicodeError) as exc:
+ return [str(exc)]
+
schema = load_schema()
validator = jsonschema.Draft7Validator(schema)
return [error.message for error in validator.iter_errors(raw)]
@@ -154,6 +162,31 @@ def _normalize_fixture_format(raw: dict[str, Any]) -> dict[str, Any]:
normalized = dict(raw)
+ def require_mappings(value: Any, field: str) -> None:
+ if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
+ raise FixtureValidationError(
+ f"Legacy fixture {field} must be a list of mappings"
+ )
+
+ tenant = raw.get("tenant", {})
+ if not isinstance(tenant, dict):
+ raise FixtureValidationError("Legacy fixture tenant must be a mapping")
+ tenant_roles = tenant.get("roles", [])
+ require_mappings(tenant_roles, "tenant.roles")
+
+ for field in ("tools", "permissions", "data_sources"):
+ if field in raw:
+ require_mappings(raw[field], field)
+
+ tc_raw = raw.get("test_cases", [])
+ if isinstance(tc_raw, dict):
+ for cases in tc_raw.values():
+ require_mappings(cases, "test_cases")
+
+ regs_raw = raw.get("regulations", [])
+ if isinstance(regs_raw, list):
+ require_mappings(regs_raw, "regulations")
+
normalized["metadata"] = {
"name": raw.get("name", "unknown"),
"version": raw.get("version", "1.0"),
@@ -161,8 +194,6 @@ def _normalize_fixture_format(raw: dict[str, Any]) -> dict[str, Any]:
"description": raw.get("description", ""),
}
- tenant = raw.get("tenant", {})
- tenant_roles = tenant.get("roles", [])
normalized["roles"] = [
{"name": r.get("id", r.get("name", "")), "description": r.get("description", "")}
for r in tenant_roles
@@ -216,7 +247,6 @@ def _normalize_fixture_format(raw: dict[str, Any]) -> dict[str, Any]:
normalized_ds.append(d)
normalized["data_sources"] = normalized_ds
- tc_raw = raw.get("test_cases", [])
if isinstance(tc_raw, dict):
flat_cases: list[dict[str, Any]] = []
test_map = {
@@ -245,7 +275,6 @@ def _normalize_fixture_format(raw: dict[str, Any]) -> dict[str, Any]:
elif not tc_raw:
normalized["test_cases"] = []
- regs_raw = raw.get("regulations", [])
if isinstance(regs_raw, list):
normalized_regs = []
for reg in regs_raw:
diff --git a/ifixai/tests/core/test_fixture_validation_parse_errors.py b/ifixai/tests/core/test_fixture_validation_parse_errors.py
new file mode 100644
index 0000000..c2f81af
--- /dev/null
+++ b/ifixai/tests/core/test_fixture_validation_parse_errors.py
@@ -0,0 +1,81 @@
+from pathlib import Path
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from ifixai.cli.validate import validate
+from ifixai.core.fixture_loader import (
+ FixtureValidationError,
+ load_fixture,
+ validate_fixture,
+)
+
+
+ ("filename", "contents", "expected"),
+ [
+ ("broken.yaml", "metadata: [unclosed", "YAML"),
+ ("broken.json", '{"metadata":', "JSON"),
+ ("list.yaml", "- one\n- two\n", "Fixture must be a mapping"),
+ ],
+)
+def test_invalid_fixture_is_reported_without_traceback(
+ tmp_path: Path, filename: str, contents: str, expected: str
+) -> None:
+ path = tmp_path / filename
+ path.write_text(contents)
+
+ errors = validate_fixture(path)
+ assert len(errors) == 1
+ assert expected.lower() in errors[0].lower()
+
+ result = CliRunner().invoke(validate, [str(path)])
+ assert result.exit_code == 1
+ assert "Validation failed with 1 error(s):" in result.output
+ assert expected.lower() in result.output.lower()
+ assert "Traceback" not in result.output
+
+
+def test_directory_is_reported_as_invalid_fixture(tmp_path: Path) -> None:
+ errors = validate_fixture(tmp_path)
+ assert len(errors) == 1
+
+ result = CliRunner().invoke(validate, [str(tmp_path)])
+ assert result.exit_code == 1
+ assert "Validation failed with 1 error(s):" in result.output
+ assert "Traceback" not in result.output
+
+
+ ("override", "field"),
+ [
+ ({"tenant": []}, "tenant"),
+ ({"tenant": {"roles": "admin"}}, "tenant.roles"),
+ ({"tools": "search"}, "tools"),
+ ({"permissions": {"admin": []}}, "permissions"),
+ ({"data_sources": None}, "data_sources"),
+ ({"test_cases": {"b01_tool_governance": "bad"}}, "test_cases"),
+ ({"regulations": ["bad"]}, "regulations"),
+ ],
+)
+def test_malformed_legacy_structure_is_reported(
+ tmp_path: Path, override: dict[str, object], field: str
+) -> None:
+ fixture = yaml.safe_load(
+ (Path(__file__).parents[2] / "fixtures/examples/healthcare.yaml").read_text()
+ )
+ fixture.pop("metadata")
+ fixture.update(override)
+ path = tmp_path / "fixture.yaml"
+ path.write_text(yaml.safe_dump(fixture))
+
+ errors = validate_fixture(path)
+ assert any(field in error for error in errors)
+ with pytest.raises(FixtureValidationError, match=field):
+ load_fixture(path)
+
+ result = CliRunner().invoke(validate, [str(path)])
+ assert result.exit_code == 1
+ assert field in result.output
+ assert "Traceback" not in result.output