Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: keep unselected mandatory minimums out of aborted scoped runs (#340)
Rudy Mizrahi Celekli37a2d2489a94
diff --git a/ifixai/core/runner.py b/ifixai/core/runner.pyindex 51c568e..32246e0 100644--- a/ifixai/core/runner.py+++ b/ifixai/core/runner.py@@ -491,6 +491,9 @@ def build_partial_result(run_mode=run_mode,sut_temperature=sut_temperature,sut_seed=sut_seed,+ # A scoped run reports unselected mandatory minimums as not run, the+ # same as when it completes; planned-but-missing ones still fail.+ selected_ids=None if run_mode == "full" else set(planned_ids),)result.partial = Trueresult.abort_reason = abort_reasondiff --git a/ifixai/tests/test_partial_selected_minimums.py b/ifixai/tests/test_partial_selected_minimums.pynew file mode 100644index 0000000..c72a6f1--- /dev/null+++ b/ifixai/tests/test_partial_selected_minimums.py@@ -0,0 +1,45 @@+from ifixai.core.runner import build_partial_result+from ifixai.core.types import InspectionCategory, TestResult, TestStatus+++def _passing(test_id: str) -> TestResult:+ return TestResult(+ test_id=test_id,+ category=InspectionCategory.DECEPTION,+ score=0.95,+ threshold=0.8,+ passed=True,+ passing=True,+ status=TestStatus.PASS,+ )+++def _partial(planned_ids, run_mode="selected"):+ return build_partial_result(+ completed=[_passing("B06")],+ planned_ids=planned_ids,+ abort_reason="owned cancellation",+ system_name="owned-run",+ system_version="1.0",+ fixture_name="default",+ provider_name="mock",+ run_mode=run_mode,+ )+++def test_aborted_selected_run_does_not_fail_unselected_minimums():+ result = _partial(["B06", "B12"])+ assert result.mandatory_minimum_violations == []+ assert {"B01", "B08", "P01"} <= set(result.mandatory_minimums_not_run)+ assert result.passed is False and result.partial+++def test_aborted_selected_run_still_fails_planned_missing_minimum():+ result = _partial(["B01", "B06"])+ assert "B01" in result.mandatory_minimum_violations+ assert "B01" not in result.mandatory_minimums_not_run+++def test_aborted_full_run_keeps_missing_minimums_as_violations():+ result = _partial(["B01", "B06", "B08", "P01"], run_mode="full")+ assert {"B01", "B08", "P01"} <= set(result.mandatory_minimum_violations)