Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: compare missing and unscored inspections without false gaps (#132)
Rudy Mizrahi Celekli4cebf8070765
diff --git a/ifixai/cli/compare.py b/ifixai/cli/compare.pyindex 6ee60c5..6f0a3e9 100644--- a/ifixai/cli/compare.py+++ b/ifixai/cli/compare.py@@ -9,6 +9,7 @@ from ifixai.core.types import (TestGrade,TestResult,TestRunResult,+ TestStatus,)from ifixai.reporting.comparison import compare_scorecards@@ -51,13 +52,22 @@ def compare(baseline: str, enhanced: str) -> None:click.echo()- header = f"{'ID':<12} {'Name':<30} {'Baseline':>8} {'Enhanced':>8} {'Delta':>8} {'Status':<12}"+ header = f"{'ID':<12} {'Name':<30} {'Baseline':>8} {'Enhanced':>8} {'Delta':>8} {'Status':<16}"click.echo(header)click.echo("-" * len(header))for delta in report.test_deltas:- delta_sign = "+" if delta.delta >= 0 else ""- delta_display = f"{delta_sign}{delta.delta:.0%}"+ delta_display = (+ f"{'+' if delta.delta >= 0 else ''}{delta.delta:.0%}"+ if delta.delta is not None+ else "n/a"+ )+ baseline_display = (+ f"{delta.baseline_score:.0%}" if delta.baseline_score is not None else "n/a"+ )+ enhanced_display = (+ f"{delta.enhanced_score:.0%}" if delta.enhanced_score is not None else "n/a"+ )status_color = _status_color(delta.status_change)status_styled = click.style(delta.status_change, fg=status_color)@@ -65,10 +75,10 @@ def compare(baseline: str, enhanced: str) -> None:click.echo(f"{delta.test_id:<12} "f"{delta.test_name:<30} "- f"{delta.baseline_score:>7.0%} "- f"{delta.enhanced_score:>8.0%} "+ f"{baseline_display:>8} "+ f"{enhanced_display:>8} "f"{delta_display:>8} "- f"{status_styled:<12}"+ f"{status_styled:<16}")click.echo()@@ -90,19 +100,7 @@ def load_result_from_json(path: str) -> TestRunResult:metadata = raw.get("metadata", {})overall = raw.get("overall", {})- test_results = [- TestResult(- test_id=br["test_id"],- name=br.get("name", ""),- category=_parse_category(br.get("category", "")),- score=br.get("score", 0.0),- threshold=br.get("threshold", 0.0),- passed=br.get("passing", False),- passing=br.get("passing", False),- error=br.get("error"),- )- for br in raw.get("test_results", [])- ]+ test_results = [_load_test_result(br) for br in raw.get("test_results", [])]category_scores = [CategoryScore(@@ -130,6 +128,29 @@ def load_result_from_json(path: str) -> TestRunResult:)+def _load_test_result(raw: dict) -> TestResult:+ score = raw.get("score", 0.0)+ passing = raw.get("passing", False)+ # Exported scorecards use JSON null for inconclusive/error inspections.+ # TestResult stores a numeric score, so preserve the unscored state via+ # status while using zero only as its internal placeholder.+ status = TestStatus(+ raw.get("status")+ or ("inconclusive" if score is None else "pass" if passing else "fail")+ )+ return TestResult(+ test_id=raw["test_id"],+ name=raw.get("name", ""),+ category=_parse_category(raw.get("category", "")),+ score=0.0 if score is None else score,+ threshold=raw.get("threshold", 0.0),+ passed=passing,+ passing=passing,+ status=status,+ error=raw.get("error"),+ )++def _parse_category(value: str) -> InspectionCategory:for cat in InspectionCategory:if cat.value == value:@@ -151,5 +172,10 @@ def _status_color(status: str) -> str:"unchanged": "white","regressed": "red","broken": "red",+ "missing_baseline": "yellow",+ "missing_enhanced": "yellow",+ "unscored_baseline": "yellow",+ "unscored_enhanced": "yellow",+ "unscored_both": "yellow",}return color_map.get(status, "white")diff --git a/ifixai/core/types.py b/ifixai/core/types.pyindex afc70ea..2a7c290 100644--- a/ifixai/core/types.py+++ b/ifixai/core/types.py@@ -2215,9 +2215,9 @@ class TestDelta(BaseModel):test_id: strtest_name: str = ""- baseline_score: float = 0.0- enhanced_score: float = 0.0- delta: float = 0.0+ baseline_score: Optional[float] = None+ enhanced_score: Optional[float] = None+ delta: Optional[float] = Nonestatus_change: str = "unchanged"gap_closed: bool = Falsediff --git a/ifixai/reporting/comparison.py b/ifixai/reporting/comparison.pyindex 8a9562e..744b871 100644--- a/ifixai/reporting/comparison.py+++ b/ifixai/reporting/comparison.py@@ -3,10 +3,18 @@from ifixai.core.types import (ComparisonReport,TestDelta,+ TestResult,TestRunResult,+ TestStatus,)+def _scored_value(result: TestResult | None) -> float | None:+ if result is None or result.status in {TestStatus.INCONCLUSIVE, TestStatus.ERROR}:+ return None+ return result.score++def compare_scorecards(baseline: TestRunResult,enhanced: TestRunResult,@@ -31,12 +39,47 @@ def compare_scorecards(base_result = baseline_scores.get(bid)enh_result = enhanced_scores.get(bid)- base_score = base_result.score if base_result else 0.0- enh_score = enh_result.score if enh_result else 0.0- base_passed = base_result.passing if base_result else False- enh_passed = enh_result.passing if enh_result else False- result = enh_result or base_result- name = result.name if result is not None else bid+ # Absence means that the inspection was not run. Treating it as a failed+ # zero-score result invents a regression or a newly closed gap.+ if base_result is None or enh_result is None:+ result = base_result or enh_result+ assert result is not None+ deltas.append(+ TestDelta(+ test_id=bid,+ test_name=result.name,+ baseline_score=_scored_value(base_result),+ enhanced_score=_scored_value(enh_result),+ status_change=(+ "missing_baseline" if base_result is None else "missing_enhanced"+ ),+ )+ )+ continue++ base_score = _scored_value(base_result)+ enh_score = _scored_value(enh_result)+ if base_score is None or enh_score is None:+ if base_score is None and enh_score is None:+ status = "unscored_both"+ elif base_score is None:+ status = "unscored_baseline"+ else:+ status = "unscored_enhanced"+ deltas.append(+ TestDelta(+ test_id=bid,+ test_name=enh_result.name or base_result.name or bid,+ baseline_score=base_score,+ enhanced_score=enh_score,+ status_change=status,+ )+ )+ continue++ base_passed = base_result.passing+ enh_passed = enh_result.passing+ name = enh_result.name or base_result.name or biddelta = enh_score - base_scorediff --git a/ifixai/tests/test_compare_unmatched_inspections.py b/ifixai/tests/test_compare_unmatched_inspections.pynew file mode 100644index 0000000..aa8a5ac--- /dev/null+++ b/ifixai/tests/test_compare_unmatched_inspections.py@@ -0,0 +1,112 @@+import json++from click.testing import CliRunner++from ifixai.cli.compare import compare+from ifixai.core.types import TestResult, TestRunResult, TestStatus+from ifixai.reporting.comparison import compare_scorecards+++def _result(test_id: str, score: float, passing: bool) -> TestResult:+ return TestResult(test_id=test_id, name=test_id, score=score, passing=passing)+++def test_unmatched_inspections_are_not_reported_as_fixed_or_broken():+ baseline = TestRunResult(test_results=[_result("B01", 0.9, True)])+ enhanced = TestRunResult(test_results=[_result("B02", 0.8, True)])++ report = compare_scorecards(baseline, enhanced)++ assert report.gaps_closed == []+ assert report.gaps_opened == []+ assert report.gaps_remaining == []+ assert [(delta.test_id, delta.status_change) for delta in report.test_deltas] == [+ ("B01", "missing_enhanced"),+ ("B02", "missing_baseline"),+ ]+ assert report.test_deltas[0].enhanced_score is None+ assert report.test_deltas[1].baseline_score is None+ assert all(delta.delta is None and not delta.gap_closed for delta in report.test_deltas)+++def test_shared_inspections_still_report_real_change():+ baseline = TestRunResult(test_results=[_result("B01", 0.2, False)])+ enhanced = TestRunResult(test_results=[_result("B01", 0.9, True)])++ report = compare_scorecards(baseline, enhanced)++ assert report.gaps_closed == ["B01"]+ assert report.test_deltas[0].status_change == "fixed"+ assert report.test_deltas[0].delta == 0.7+++def test_compare_cli_marks_unmatched_inspection_as_unscored(tmp_path):+ baseline = tmp_path / "baseline.json"+ enhanced = tmp_path / "enhanced.json"+ baseline.write_text(+ json.dumps({"test_results": [{"test_id": "B01", "score": 0.9, "passing": True}]}),+ encoding="utf-8",+ )+ enhanced.write_text(json.dumps({"test_results": []}), encoding="utf-8")++ result = CliRunner().invoke(compare, [str(baseline), str(enhanced)])++ assert result.exit_code == 0, result.exception+ assert "B01" in result.output+ assert "n/a" in result.output+ assert "missing_enhanced" in result.output+ assert "Gaps opened" not in result.output+++def test_inconclusive_inspection_is_not_counted_as_a_closed_gap():+ baseline = TestRunResult(+ test_results=[+ TestResult(test_id="B01", score=0.0, status=TestStatus.INCONCLUSIVE)+ ]+ )+ enhanced = TestRunResult(test_results=[_result("B01", 0.9, True)])++ report = compare_scorecards(baseline, enhanced)++ assert report.gaps_closed == []+ assert report.gaps_opened == []+ assert report.gaps_remaining == []+ assert report.test_deltas[0].status_change == "unscored_baseline"+ assert report.test_deltas[0].baseline_score is None+ assert report.test_deltas[0].delta is None+++def test_compare_cli_loads_exported_null_inspection_scores(tmp_path):+ baseline = tmp_path / "baseline.json"+ enhanced = tmp_path / "enhanced.json"+ baseline.write_text(+ json.dumps(+ {+ "test_results": [+ {+ "test_id": "B01",+ "score": None,+ "status": "inconclusive",+ "passing": False,+ }+ ]+ }+ ),+ encoding="utf-8",+ )+ enhanced.write_text(+ json.dumps(+ {+ "test_results": [+ {"test_id": "B01", "score": 0.9, "status": "pass", "passing": True}+ ]+ }+ ),+ encoding="utf-8",+ )++ result = CliRunner().invoke(compare, [str(baseline), str(enhanced)])++ assert result.exit_code == 0, result.exception+ assert "unscored_baseline" in result.output+ assert "Gaps closed" not in result.output