Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(inspections): preserve pipeline judge failure semantics (#161)

Rudy Mizrahi Celekli46d50f16aff7
diff --git a/ifixai/inspections/b15_long_horizon_drift/runner.py b/ifixai/inspections/b15_long_horizon_drift/runner.py
index 577b46c..0ab002f 100644
--- a/ifixai/inspections/b15_long_horizon_drift/runner.py
+++ b/ifixai/inspections/b15_long_horizon_drift/runner.py
@@ -293,6 +293,7 @@ class B15LongHorizonDrift(BaseTest):
if result.rubric_verdict is not None
else None
),
+ extraction_error=result.extraction_error,
details=details,
)
diff --git a/ifixai/inspections/b22_decision_reproducibility/runner.py b/ifixai/inspections/b22_decision_reproducibility/runner.py
index 95c08b8..0755b5c 100644
--- a/ifixai/inspections/b22_decision_reproducibility/runner.py
+++ b/ifixai/inspections/b22_decision_reproducibility/runner.py
@@ -23,6 +23,7 @@ from ifixai.evaluation.analytic_judge import (
load_analytic_rubric,
sanitize_response_payload,
)
+from ifixai.evaluation.errors import JudgeUnavailableError
from ifixai.harness.base import BaseTest, build_system_message, sample_capped
from ifixai.inspections.b22_decision_reproducibility.schemas import (
ArmDecisions,
@@ -235,6 +236,8 @@ class B22DecisionReproducibility(BaseTest):
)
results: list[EvidenceItem] = []
for (user, tool), item in zip(pairs, raw):
+ if isinstance(item, JudgeUnavailableError):
+ raise item
if isinstance(item, BaseException):
# A provider/transport error (e.g. a bad model id 404ing) is
# unscorable, not a reproducibility failure — stamp it so scoring
@@ -433,6 +436,7 @@ class B22DecisionReproducibility(BaseTest):
if result.rubric_verdict is not None
else None
),
+ extraction_error=result.extraction_error,
details=details,
)
diff --git a/ifixai/inspections/b28_rag_context_integrity/runner.py b/ifixai/inspections/b28_rag_context_integrity/runner.py
index 164394d..c8506f8 100644
--- a/ifixai/inspections/b28_rag_context_integrity/runner.py
+++ b/ifixai/inspections/b28_rag_context_integrity/runner.py
@@ -166,5 +166,6 @@ class B28RAGContextIntegrity(BaseTest):
if result.rubric_verdict is not None
else None
),
+ extraction_error=result.extraction_error,
details=details,
)
diff --git a/ifixai/tests/test_inspection_judge_error_propagation.py b/ifixai/tests/test_inspection_judge_error_propagation.py
new file mode 100644
index 0000000..7219682
--- /dev/null
+++ b/ifixai/tests/test_inspection_judge_error_propagation.py
@@ -0,0 +1,100 @@
+"""Drive real inspection/pipeline/judge/HTTP paths against owned offline replies."""
+
+import json
+
+import pytest
+from aiohttp import web
+
+from ifixai.core.fixture_loader import load_fixture
+from ifixai.core.types import (
+ EvaluationPipelineConfig,
+ ProviderConfig,
+ TestRunResult,
+ TestStatus,
+)
+from ifixai.evaluation.analytic_judge import AnalyticRubricJudge, load_analytic_rubric
+from ifixai.evaluation.errors import JudgeUnavailableError
+from ifixai.evaluation.pipeline import EvaluationPipeline
+from ifixai.harness.registry import create_inspection
+from ifixai.judge.config import JudgeConfig
+from ifixai.judge.evaluator import JudgeEvaluator
+from ifixai.providers.http import HttpProvider
+from ifixai.reporting.scorecard import generate_json_report
+
+
+async def _run_native_inspection(test_id, judge_reply):
+ dimensions = {}
+ for outcome in ["refuse", "comply"]:
+ rubric = await load_analytic_rubric(test_id, outcome)
+ dimensions.update({dimension.name: dimension for dimension in rubric.dimensions})
+ calls = {"sut": 0, "judge": 0}
+
+ async def complete(request):
+ payload = await request.json()
+ role = payload["model"]
+ calls[role] += 1
+ content = "Access denied; we continue the audit objective."
+ if role == "judge":
+ if judge_reply == "unavailable":
+ return web.Response(status=400, text="owned judge request rejected")
+ content = json.dumps({"dimensions": [
+ {"name": name, "passed": judge_reply == "pass", "reasoning": "offline control", "confidence": 1.0}
+ for name in dimensions
+ ]}) if judge_reply != "invalid" else "{}"
+ return web.json_response({"choices": [{"message": {"content": content}, "finish_reason": "stop"}]})
+
+ app = web.Application()
+ app.router.add_post("/chat/completions", complete)
+ runner = web.AppRunner(app)
+ await runner.setup()
+ site = web.TCPSite(runner, "127.0.0.1", 0)
+ await site.start()
+ endpoint = f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}"
+ sut = HttpProvider()
+ judge = JudgeEvaluator(JudgeConfig(provider="http", model="judge", endpoint=endpoint))
+ try:
+ inspection = create_inspection(test_id)
+ config = EvaluationPipelineConfig(judge_max_calls=0, b28_seed=28)
+ pipeline = EvaluationPipeline(config, AnalyticRubricJudge(judge))
+ result = await inspection.execute(
+ sut, ProviderConfig(provider="http", model="sut", endpoint=endpoint, seed=22),
+ load_fixture("software_engineering"), pipeline_config=config, pipeline=pipeline,
+ )
+ report = json.loads(generate_json_report(TestRunResult(test_results=[result])))
+ return result, report, calls
+ finally:
+ await sut.aclose()
+ await judge.aclose()
+ await runner.cleanup()
+
+
[email protected]("test_id", ["B15", "B22", "B28"])
[email protected]("judge_reply", ["invalid", "pass", "fail"])
+async def test_inspection_preserves_real_judge_error(test_id, judge_reply):
+ result, report, calls = await _run_native_inspection(test_id, judge_reply)
+ assert calls["sut"] > 0 and calls["judge"] > 0
+ expected = {"pass": TestStatus.PASS, "fail": TestStatus.FAIL, "invalid": TestStatus.INCONCLUSIVE}[judge_reply]
+ assert result.status == expected, result.model_dump(mode="json")
+ assert report["test_results"][0]["status"] == expected.value
+ if judge_reply != "invalid":
+ assert result.score == (1.0 if judge_reply == "pass" else 0.0)
+ assert all(item.extraction_error is None for item in result.evidence)
+ else:
+ assert result.insufficient_evidence
+ assert all(item.extraction_error is not None for item in result.evidence)
+ assert report["test_results"][0]["passing"] is False
+
+
+async def test_b22_propagates_real_fatal_judge_error(monkeypatch):
+ monkeypatch.setenv("IFIXAI_JUDGE_FAIL_FAST", "1")
+ with pytest.raises(JudgeUnavailableError, match="judge unreachable"):
+ await _run_native_inspection("B22", "unavailable")
+
+
+async def test_b22_preserves_explicit_fail_fast_opt_out(monkeypatch):
+ monkeypatch.setenv("IFIXAI_JUDGE_FAIL_FAST", "0")
+ result, report, calls = await _run_native_inspection("B22", "unavailable")
+ assert calls["sut"] > 0 and calls["judge"] > 0
+ assert result.status == TestStatus.INCONCLUSIVE
+ assert all(item.extraction_error is not None for item in result.evidence)
+ assert report["test_results"][0]["score"] is None