Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: validate inspection definitions before parsing plans (#264)

Rudy Mizrahi Celekli12d1b0993e57
diff --git a/ifixai/harness/base.py b/ifixai/harness/base.py
index 63d3eaa..931a54c 100644
--- a/ifixai/harness/base.py
+++ b/ifixai/harness/base.py
@@ -253,22 +253,13 @@ class BaseTest(ABC):
)
def load_inspection_data(self) -> ConversationPlan | None:
- try:
- plan = load_inspection_definition(self.spec.test_id)
- except Exception as exc: # noqa: BLE001 — YAML load is best-effort; fall back to hardcoded logic
- _logger.warning(
- "%s: failed to load YAML inspection: %s — using hardcoded logic",
+ plan = load_inspection_definition(self.spec.test_id)
+ if plan is None:
+ _logger.debug(
+ "%s: no YAML inspection definition, using hardcoded logic",
self.spec.test_id,
- exc,
)
- return None
- else:
- if plan is None:
- _logger.debug(
- "%s: no YAML inspection definition, using hardcoded logic",
- self.spec.test_id,
- )
- return plan
+ return plan
def compute_score(self, evidence: list[EvidenceItem]) -> float:
if self.spec.count_extraction_errors_as_fail:
diff --git a/ifixai/rules/loader.py b/ifixai/rules/loader.py
index 41adf2d..79e86ea 100644
--- a/ifixai/rules/loader.py
+++ b/ifixai/rules/loader.py
@@ -109,13 +109,13 @@ def load_inspection_definition(test_id: str) -> ConversationPlan | None:
if _DEFAULT_LOADER is None:
_DEFAULT_LOADER = RuleLoader()
- plan = _DEFAULT_LOADER.load_rules(test_id)
- if not plan.steps:
- return None
-
yaml_path = _DEFAULT_LOADER._find_rule_file(test_id)
- if yaml_path is not None and _SCHEMA_PATH.exists():
- raw = _DEFAULT_LOADER._read_yaml(yaml_path)
+ if yaml_path is None:
+ return None
+ raw = _DEFAULT_LOADER._read_yaml(yaml_path)
+ if "steps" not in raw or raw["steps"] == []:
+ return None
+ if _SCHEMA_PATH.exists():
with open(_SCHEMA_PATH, encoding="utf-8") as fh:
schema = json.load(fh)
try:
@@ -125,4 +125,5 @@ def load_inspection_definition(test_id: str) -> ConversationPlan | None:
f"Schema validation failed for {test_id}: {exc.message}"
) from exc
- return plan
+ plan = _DEFAULT_LOADER._parse_plan(raw, test_id)
+ return plan if plan.steps else None
diff --git a/ifixai/tests/test_definition_validation_order.py b/ifixai/tests/test_definition_validation_order.py
new file mode 100644
index 0000000..1cd3b40
--- /dev/null
+++ b/ifixai/tests/test_definition_validation_order.py
@@ -0,0 +1,58 @@
+import pytest
+import yaml
+
+from ifixai.rules import loader as module
+
+
[email protected]("steps", [["not a step"], {"step_id": 1}])
+def test_existing_invalid_definition_reports_schema_error(tmp_path, monkeypatch, steps):
+ folder = tmp_path / "b05_custom"
+ folder.mkdir()
+ (folder / "definition.yaml").write_text(
+ yaml.safe_dump({"test_id": "B05", "fixture_requirements": [], "steps": steps})
+ )
+ monkeypatch.setattr(module, "_DEFAULT_LOADER", module.RuleLoader(tmp_path))
+ with pytest.raises(module.RuleLoadError, match="Schema validation failed"):
+ module.load_inspection_definition("B05")
+
+
+def test_missing_definition_remains_optional(tmp_path, monkeypatch):
+ monkeypatch.setattr(module, "_DEFAULT_LOADER", module.RuleLoader(tmp_path))
+ assert module.load_inspection_definition("B05") is None
+
+
+async def test_invalid_b26_definition_is_configuration_error_without_sut_calls(tmp_path, monkeypatch):
+ from ifixai.core.fixture_loader import load_fixture
+ from ifixai.core.runner import run_single
+ from ifixai.core.types import ProviderConfig, TestStatus
+ from ifixai.providers.base import ChatProvider
+
+ folder = tmp_path / "b26_rate_limiting"
+ folder.mkdir()
+ (folder / "definition.yaml").write_text(yaml.safe_dump({"test_id": "B26", "steps": ["not a step"]}))
+ monkeypatch.setattr(module, "_DEFAULT_LOADER", module.RuleLoader(tmp_path))
+ class OwnedProvider(ChatProvider):
+ calls = 0
+ async def send_message(self, messages, config):
+ self.calls += 1
+ return "owned reply"
+ provider = OwnedProvider()
+ result = await run_single("B26", provider, ProviderConfig(provider="owned"), load_fixture("customer_support"))
+ assert result.status == TestStatus.ERROR
+ assert "Schema validation failed" in result.error
+ assert provider.calls == 0
+
+
[email protected]("test_id", ["B28", "B29", "B31", "P01", "P08"])
+def test_shipped_custom_runner_definition_remains_optional(monkeypatch, test_id):
+ monkeypatch.setattr(module, "_DEFAULT_LOADER", module.RuleLoader())
+ assert module.load_inspection_definition(test_id) is None
+
+
[email protected]("raw", [{"test_id": "B05"}, {"test_id": "B05", "steps": []}])
+def test_definition_without_declarative_steps_remains_optional(tmp_path, monkeypatch, raw):
+ folder = tmp_path / "b05_custom"
+ folder.mkdir()
+ (folder / "definition.yaml").write_text(yaml.safe_dump(raw))
+ monkeypatch.setattr(module, "_DEFAULT_LOADER", module.RuleLoader(tmp_path))
+ assert module.load_inspection_definition("B05") is None