Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: keep errored judge samples from unscoring B15/P32 FAILs (#364)

n-papaioannouc8cb21e92c0b
diff --git a/ifixai/inspections/b15_long_horizon_drift/runner.py b/ifixai/inspections/b15_long_horizon_drift/runner.py
index 84ba27c..a8e8a35 100644
--- a/ifixai/inspections/b15_long_horizon_drift/runner.py
+++ b/ifixai/inspections/b15_long_horizon_drift/runner.py
@@ -20,6 +20,7 @@ from ifixai.core.types import (
from ifixai.evaluation.analytic_judge import load_analytic_rubric
from ifixai.evaluation.errors import JudgePipelineRequiredError
from ifixai.harness.base import BaseTest, build_system_message
+from ifixai.inspections.judge_consensus import choose_consensus_body
from ifixai.providers.base import ChatProvider, ProviderError, is_fatal_provider_error
from ifixai.rules.loader import RuleLoader
from ifixai.utils.template_renderer import render
@@ -406,7 +407,8 @@ async def _evaluate_with_consensus(
passes = sum(1 for r in results if r.passed)
majority_passed = passes > len(results) // 2
dispersion = JudgeDispersion(passes=passes, total=len(results))
- chosen = next((r for r in results if r.passed == majority_passed), results[0])
+ # An errored sample must not donate its extraction_error to a FAIL majority.
+ chosen = choose_consensus_body(results, majority_passed)
final = chosen.model_copy(update={"passed": majority_passed})
return ConsensusResult(result=final, dispersion=dispersion)
diff --git a/ifixai/inspections/p32_emergent_systemic_harm/runner.py b/ifixai/inspections/p32_emergent_systemic_harm/runner.py
index 362ed0a..dbe12ee 100644
--- a/ifixai/inspections/p32_emergent_systemic_harm/runner.py
+++ b/ifixai/inspections/p32_emergent_systemic_harm/runner.py
@@ -53,6 +53,7 @@ from ifixai.core.types import (
from ifixai.evaluation.analytic_judge import load_analytic_rubric
from ifixai.evaluation.errors import JudgePipelineRequiredError
from ifixai.harness.base import BaseTest, build_system_message
+from ifixai.inspections.judge_consensus import choose_consensus_body
from ifixai.providers.base import ChatProvider, ProviderEmptyContentError
from ifixai.rules.loader import RuleLoader, RuleLoadError
from ifixai.shared.evidence import build_unscorable_item
@@ -870,7 +871,8 @@ async def _evaluate_with_consensus(
passes = sum(1 for r in results if r.passed)
majority_passed = passes > len(results) // 2
dispersion = JudgeDispersion(passes=passes, total=len(results))
- chosen = next((r for r in results if r.passed == majority_passed), results[0])
+ # An errored sample must not donate its extraction_error to a FAIL majority.
+ chosen = choose_consensus_body(results, majority_passed)
final = chosen.model_copy(update={"passed": majority_passed})
return ConsensusResult(result=final, dispersion=dispersion)
diff --git a/ifixai/tests/test_private_consensus_clean_body.py b/ifixai/tests/test_private_consensus_clean_body.py
index 6f262a2..b974993 100644
--- a/ifixai/tests/test_private_consensus_clean_body.py
+++ b/ifixai/tests/test_private_consensus_clean_body.py
@@ -6,9 +6,11 @@ import inspect
import pytest
from ifixai.core.types import EvaluationMethod, JudgeErrorKind, PipelineResult
+from ifixai.inspections.b15_long_horizon_drift import runner as b15
from ifixai.inspections.p13_hidden_side_objective_persistence import runner as p13
from ifixai.inspections.p22_goal_edit_acceptance import runner as p22
from ifixai.inspections.p27_privilege_creep import runner as p27
+from ifixai.inspections.p32_emergent_systemic_harm import runner as p32
from ifixai.inspections.s02_configurer_stakeholder_conflict import runner as s02
_ERRORED = PipelineResult(
@@ -28,7 +30,7 @@ def _verdict(passed: bool) -> PipelineResult:
def _run(consensus, samples):
- # P13's private copy takes no context_vars.
+ # P13's and B15's private copies take no context_vars.
extra = ({},) if "context_vars" in inspect.signature(consensus).parameters else ()
return asyncio.run(consensus(_Judge(samples), "response", None, None, "context", *extra))["result"]
@@ -45,18 +47,29 @@ class _Judge:
_CONSENSUS = [s02._evaluate_with_consensus, p13._evaluate_with_consensus,
- p22._evaluate_with_consensus, p27._evaluate_with_consensus]
+ p22._evaluate_with_consensus, p27._evaluate_with_consensus,
+ b15._evaluate_with_consensus, p32._evaluate_with_consensus]
+_IDS = ["S02", "P13", "P22", "P27", "B15", "P32"]
[email protected]("consensus", _CONSENSUS, ids=["S02", "P13", "P22", "P27"])
[email protected]("consensus", _CONSENSUS, ids=_IDS)
def test_errored_first_sample_does_not_unscore_a_fail_majority(consensus):
result = _run(consensus, [_ERRORED, _verdict(False), _verdict(False)])
assert result.passed is False
assert result.extraction_error is None
[email protected]("consensus", _CONSENSUS, ids=["S02", "P13", "P22", "P27"])
[email protected]("consensus", _CONSENSUS, ids=_IDS)
def test_all_errored_samples_stay_unmeasurable(consensus):
result = _run(consensus, [_ERRORED, _ERRORED, _ERRORED])
assert result.passed is False
assert result.extraction_error is JudgeErrorKind.EXTRACTION
+
+
[email protected]("passed", [True, False])
[email protected]("consensus", _CONSENSUS, ids=_IDS)
+def test_clean_majority_keeps_first_agreeing_body(consensus, passed):
+ first = _verdict(passed).model_copy(update={"evaluation_result": "first"})
+ result = _run(consensus, [_verdict(not passed), first, _verdict(passed)])
+ assert result.passed is passed
+ assert result.evaluation_result == "first"