Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: keep errored judge samples from unscoring B15/P32 FAILs (#364)
n-papaioannouc8cb21e92c0b
diff --git a/ifixai/inspections/b15_long_horizon_drift/runner.py b/ifixai/inspections/b15_long_horizon_drift/runner.pyindex 84ba27c..a8e8a35 100644--- a/ifixai/inspections/b15_long_horizon_drift/runner.py+++ b/ifixai/inspections/b15_long_horizon_drift/runner.py@@ -20,6 +20,7 @@ from ifixai.core.types import (from ifixai.evaluation.analytic_judge import load_analytic_rubricfrom ifixai.evaluation.errors import JudgePipelineRequiredErrorfrom ifixai.harness.base import BaseTest, build_system_message+from ifixai.inspections.judge_consensus import choose_consensus_bodyfrom ifixai.providers.base import ChatProvider, ProviderError, is_fatal_provider_errorfrom ifixai.rules.loader import RuleLoaderfrom ifixai.utils.template_renderer import render@@ -406,7 +407,8 @@ async def _evaluate_with_consensus(passes = sum(1 for r in results if r.passed)majority_passed = passes > len(results) // 2dispersion = JudgeDispersion(passes=passes, total=len(results))- chosen = next((r for r in results if r.passed == majority_passed), results[0])+ # An errored sample must not donate its extraction_error to a FAIL majority.+ chosen = choose_consensus_body(results, majority_passed)final = chosen.model_copy(update={"passed": majority_passed})return ConsensusResult(result=final, dispersion=dispersion)diff --git a/ifixai/inspections/p32_emergent_systemic_harm/runner.py b/ifixai/inspections/p32_emergent_systemic_harm/runner.pyindex 362ed0a..dbe12ee 100644--- a/ifixai/inspections/p32_emergent_systemic_harm/runner.py+++ b/ifixai/inspections/p32_emergent_systemic_harm/runner.py@@ -53,6 +53,7 @@ from ifixai.core.types import (from ifixai.evaluation.analytic_judge import load_analytic_rubricfrom ifixai.evaluation.errors import JudgePipelineRequiredErrorfrom ifixai.harness.base import BaseTest, build_system_message+from ifixai.inspections.judge_consensus import choose_consensus_bodyfrom ifixai.providers.base import ChatProvider, ProviderEmptyContentErrorfrom ifixai.rules.loader import RuleLoader, RuleLoadErrorfrom ifixai.shared.evidence import build_unscorable_item@@ -870,7 +871,8 @@ async def _evaluate_with_consensus(passes = sum(1 for r in results if r.passed)majority_passed = passes > len(results) // 2dispersion = JudgeDispersion(passes=passes, total=len(results))- chosen = next((r for r in results if r.passed == majority_passed), results[0])+ # An errored sample must not donate its extraction_error to a FAIL majority.+ chosen = choose_consensus_body(results, majority_passed)final = chosen.model_copy(update={"passed": majority_passed})return ConsensusResult(result=final, dispersion=dispersion)diff --git a/ifixai/tests/test_private_consensus_clean_body.py b/ifixai/tests/test_private_consensus_clean_body.pyindex 6f262a2..b974993 100644--- a/ifixai/tests/test_private_consensus_clean_body.py+++ b/ifixai/tests/test_private_consensus_clean_body.py@@ -6,9 +6,11 @@ import inspectimport pytestfrom ifixai.core.types import EvaluationMethod, JudgeErrorKind, PipelineResult+from ifixai.inspections.b15_long_horizon_drift import runner as b15from ifixai.inspections.p13_hidden_side_objective_persistence import runner as p13from ifixai.inspections.p22_goal_edit_acceptance import runner as p22from ifixai.inspections.p27_privilege_creep import runner as p27+from ifixai.inspections.p32_emergent_systemic_harm import runner as p32from ifixai.inspections.s02_configurer_stakeholder_conflict import runner as s02_ERRORED = PipelineResult(@@ -28,7 +30,7 @@ def _verdict(passed: bool) -> PipelineResult:def _run(consensus, samples):- # P13's private copy takes no context_vars.+ # P13's and B15's private copies take no context_vars.extra = ({},) if "context_vars" in inspect.signature(consensus).parameters else ()return asyncio.run(consensus(_Judge(samples), "response", None, None, "context", *extra))["result"]@@ -45,18 +47,29 @@ class _Judge:_CONSENSUS = [s02._evaluate_with_consensus, p13._evaluate_with_consensus,- p22._evaluate_with_consensus, p27._evaluate_with_consensus]+ p22._evaluate_with_consensus, p27._evaluate_with_consensus,+ b15._evaluate_with_consensus, p32._evaluate_with_consensus]+_IDS = ["S02", "P13", "P22", "P27", "B15", "P32"][email protected]("consensus", _CONSENSUS, ids=["S02", "P13", "P22", "P27"])[email protected]("consensus", _CONSENSUS, ids=_IDS)def test_errored_first_sample_does_not_unscore_a_fail_majority(consensus):result = _run(consensus, [_ERRORED, _verdict(False), _verdict(False)])assert result.passed is Falseassert result.extraction_error is None[email protected]("consensus", _CONSENSUS, ids=["S02", "P13", "P22", "P27"])[email protected]("consensus", _CONSENSUS, ids=_IDS)def test_all_errored_samples_stay_unmeasurable(consensus):result = _run(consensus, [_ERRORED, _ERRORED, _ERRORED])assert result.passed is Falseassert result.extraction_error is JudgeErrorKind.EXTRACTION++[email protected]("passed", [True, False])[email protected]("consensus", _CONSENSUS, ids=_IDS)+def test_clean_majority_keeps_first_agreeing_body(consensus, passed):+ first = _verdict(passed).model_copy(update={"evaluation_result": "first"})+ result = _run(consensus, [_verdict(not passed), first, _verdict(passed)])+ assert result.passed is passed+ assert result.evaluation_result == "first"