Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(judge): normalize verdict booleans before grading (#170)

Rudy Mizrahi Celekli03514ebf4e55
diff --git a/docs/testing-your-agent.md b/docs/testing-your-agent.md
index 0be833c..0bb6602 100644
--- a/docs/testing-your-agent.md
+++ b/docs/testing-your-agent.md
@@ -73,6 +73,11 @@ Atomic-claim verdicts accept JSON `true`/`false` and the strings `"true"`/`"fals
output. A string `"false"` always
scores as false; a nonempty string never becomes true merely because it exists.
+Analytic rubric dimensions accept JSON booleans, integer `0`/`1`, and recognized
+verdict words such as `"false"`/`"true"`, `"fail"`/`"pass"`, and `"no"`/`"yes"`.
+Other integers and floating-point values are invalid judge output and remain
+unscored. This normalization applies to every supported rubric envelope.
+
`--eval-mode self` is a smoke test, flagged `self-judge bias`. By default a second,
different provider grades the SUT, so nothing scores itself:
diff --git a/ifixai/evaluation/analytic_judge.py b/ifixai/evaluation/analytic_judge.py
index 2cbaa6e..6256b5a 100644
--- a/ifixai/evaluation/analytic_judge.py
+++ b/ifixai/evaluation/analytic_judge.py
@@ -501,6 +501,22 @@ def _normalize_dim_name(name: str) -> str:
return re.sub(r"[^a-z0-9]", "", cleaned.lower())
+def _parse_dimension_passed(value: object) -> bool:
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, int) and value in (0, 1):
+ return bool(value)
+ if isinstance(value, str):
+ token = value.strip().lower()
+ if token in _VERDICT_TRUE_TOKENS:
+ return True
+ if token in _VERDICT_FALSE_TOKENS:
+ return False
+ raise JudgeContractError(
+ f"Judge dimension 'passed' must be a boolean, integer 0/1, or a recognized verdict token; got {value!r}"
+ )
+
+
def _fuzzy_match_dim(name: str, rubric: AnalyticRubric) -> Optional[str]:
"""Return the canonical rubric dim name for a (possibly typo'd) judge name.
@@ -524,7 +540,7 @@ def _recover_dimensions_from_top_level(
Llama sometimes returns `{"dim_name_a": {...}, "dim_name_b": {...},
"overall_reasoning": "..."}` instead of the contract envelope. If any
- top-level key fuzzy-matches a rubric dim name and carries a bool / dict
+ top-level key fuzzy-matches a rubric dim name and carries a bool / integer 0/1 / dict
verdict, recover it.
"""
reserved = {"overall_reasoning", "dimensions"}
@@ -541,18 +557,12 @@ def _recover_dimensions_from_top_level(
recovered.append(
{
"name": canonical,
- "passed": bool(value.get("passed", False)),
+ "passed": _parse_dimension_passed(value.get("passed", False)),
"reasoning": str(value.get("reasoning", "")),
}
)
- elif isinstance(value, str):
- token = value.strip().lower()
- if token in _VERDICT_TRUE_TOKENS:
- recovered.append({"name": canonical, "passed": True, "reasoning": ""})
- elif token in _VERDICT_FALSE_TOKENS:
- recovered.append({"name": canonical, "passed": False, "reasoning": ""})
- # Unknown tokens fall through; caller raises JudgeContractError so
- # the existing retry loop still triggers.
+ elif isinstance(value, (str, int)):
+ recovered.append({"name": canonical, "passed": _parse_dimension_passed(value), "reasoning": ""})
return recovered or None
@@ -670,10 +680,13 @@ def build_judge_dim_map(
key = canonical.lower()
if key in conflicts:
continue
- normalized_entry = {**entry, "name": canonical}
+ normalized_entry = {
+ **entry, "name": canonical,
+ "passed": _parse_dimension_passed(entry.get("passed", False)),
+ }
if key in first_occurrence:
- existing = bool(first_occurrence[key].get("passed", False))
- incoming = bool(entry.get("passed", False))
+ existing = first_occurrence[key]["passed"]
+ incoming = normalized_entry["passed"]
if existing != incoming:
conflicts.add(key)
del first_occurrence[key]
@@ -796,7 +809,7 @@ def _dimension_score_from_judge(
if match is not None:
return DimensionScore(
dimension_name=dim.name,
- passed=bool(match.get("passed", False)),
+ passed=_parse_dimension_passed(match.get("passed", False)),
reasoning=str(match.get("reasoning", "")),
confidence=0.9,
is_mandatory=dim.mandatory,
diff --git a/ifixai/tests/test_analytic_judge_verdict_booleans.py b/ifixai/tests/test_analytic_judge_verdict_booleans.py
new file mode 100644
index 0000000..8767d45
--- /dev/null
+++ b/ifixai/tests/test_analytic_judge_verdict_booleans.py
@@ -0,0 +1,117 @@
+"""Exercise supported rubric verdict envelopes through real offline public runs."""
+
+import json
+
+import pytest
+from aiohttp import web
+
+from ifixai.api import run_selected
+from ifixai.core.types import EvaluationMode, EvaluationPipelineConfig, TestStatus
+from ifixai.evaluation.analytic_judge import (
+ JudgeExtractionError,
+ load_analytic_rubric,
+ parse_rubric_verdict,
+)
+from ifixai.judge.config import JudgeConfig
+from ifixai.reporting.scorecard import generate_json_report
+
+
+async def _run_verdict_envelope(envelope, verdict):
+ rubric = await load_analytic_rubric("B19", "comply")
+ dimensions = [{"name": dim.name, "passed": verdict, "reasoning": "offline control"} for dim in rubric.dimensions]
+ replies = {
+ "dimensions": {"dimensions": dimensions},
+ "alias": {"verdicts": dimensions},
+ "bare-list": dimensions,
+ "flat-dict": {dim.name: {"passed": verdict, "reasoning": "offline control"} for dim in rubric.dimensions},
+ "flat-scalar": {dim.name: verdict for dim in rubric.dimensions},
+ }
+ calls = {"sut": 0, "judge": 0}
+
+ async def complete(request):
+ payload = await request.json()
+ role = payload["model"]
+ calls[role] += 1
+ content = json.dumps(replies[envelope]) if role == "judge" else "The answer follows the supplied context."
+ return web.json_response({"choices": [{"message": {"content": content}, "finish_reason": "stop"}]})
+
+ app = web.Application()
+ app.router.add_post("/chat/completions", complete)
+ runner = web.AppRunner(app)
+ await runner.setup()
+ try:
+ site = web.TCPSite(runner, "127.0.0.1", 0)
+ await site.start()
+ endpoint = f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}"
+ run = await run_selected(
+ {"B19"}, provider="http", fixture="software_engineering", model="sut", endpoint=endpoint,
+ judge_config=JudgeConfig(provider="http", model="judge", endpoint=endpoint),
+ pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.FULL, judge_max_calls=0),
+ )
+ return run.test_results[0], calls, json.loads(generate_json_report(run))
+ finally:
+ await runner.cleanup()
+
+
[email protected]("envelope", ["dimensions", "alias", "bare-list", "flat-dict", "flat-scalar"])
[email protected]("verdict,passed", [(False, False), (True, True), (0, False), (1, True), ("false", False), (" FALSE ", False), ("true", True)])
+async def test_public_run_preserves_boolean_meaning_across_envelopes(envelope, verdict, passed):
+ result, calls, report = await _run_verdict_envelope(envelope, verdict)
+ assert calls["sut"] == 60 and calls["judge"] == 30
+ assert result.status == (TestStatus.PASS if passed else TestStatus.FAIL)
+ assert result.score == (1.0 if passed else 0.0)
+ assert all(item.passed == passed and item.extraction_error is None for item in result.evidence)
+ assert report["test_results"][0]["status"] == ("pass" if passed else "fail")
+ assert report["test_results"][0]["score"] == (1.0 if passed else 0.0)
+
+
[email protected]("first", [False, True])
+async def test_duplicate_dimension_compares_normalized_verdicts(first):
+ rubric = await load_analytic_rubric("B19", "comply")
+ dimensions = [entry for dim in rubric.dimensions for entry in [
+ {"name": dim.name, "passed": first}, {"name": dim.name, "passed": "false"},
+ ]]
+ raw = json.dumps({"dimensions": dimensions})
+ if first:
+ with pytest.raises(JudgeExtractionError, match="missing mandatory"):
+ parse_rubric_verdict(raw, rubric)
+ else:
+ verdict = parse_rubric_verdict(raw, rubric)
+ assert not verdict.passed
+ assert verdict.weighted_score == 0.0
+
+
[email protected]("envelope", ["dimensions", "alias", "bare-list", "flat-dict", "flat-scalar"])
+async def test_unknown_verdict_is_unscored_in_public_run(envelope):
+ result, calls, report = await _run_verdict_envelope(envelope, "not a verdict")
+ assert calls["sut"] == 60 and calls["judge"] == 30
+ assert result.status == TestStatus.INCONCLUSIVE
+ assert all(item.extraction_error is not None for item in result.evidence)
+ assert report["test_results"][0]["score"] is None
+
+
[email protected]("token,passed", [
+ ("pass", True), ("passed", True), ("true", True), ("yes", True), ("ok", True),
+ ("fail", False), ("failed", False), ("false", False), ("no", False),
+])
+async def test_existing_verdict_tokens_are_preserved(token, passed):
+ rubric = await load_analytic_rubric("B19", "comply")
+ raw = json.dumps({"dimensions": [{"name": dim.name, "passed": token} for dim in rubric.dimensions]})
+ assert parse_rubric_verdict(raw, rubric).passed == passed
+
+
+async def test_missing_passed_retains_default_false():
+ rubric = await load_analytic_rubric("B19", "comply")
+ raw = json.dumps({"dimensions": [{"name": dim.name} for dim in rubric.dimensions]})
+ verdict = parse_rubric_verdict(raw, rubric)
+ assert not verdict.passed
+ assert verdict.weighted_score == 0.0
+
+
[email protected]("value", [-1, 2, 0.0, 1.0, None, [], {}])
+async def test_other_numeric_or_structural_verdicts_remain_unscored(value):
+ result, calls, report = await _run_verdict_envelope("dimensions", value)
+ assert calls["sut"] == 60 and calls["judge"] == 30
+ assert result.status == TestStatus.INCONCLUSIVE
+ assert all(item.extraction_error is not None for item in result.evidence)
+ assert report["test_results"][0]["score"] is None