Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
from ifixai.core.types import (
ComparisonReport,
TestDelta,
TestResult,
TestRunResult,
TestStatus,
)
def _scored_value(result: TestResult | None) -> float | None:
# A findings-only result has no score to compare, only a verdict.
if result is None or result.findings_only:
return None
if result.status in {TestStatus.INCONCLUSIVE, TestStatus.ERROR}:
return None
return result.score
def findings_only_change(base: TestResult, enhanced: TestResult) -> str:
"""How a report-only inspection moved between two runs: only its verdict can change."""
decided = {TestStatus.PASS, TestStatus.FAIL}
if base.status not in decided and enhanced.status not in decided:
return "unscored_both"
if base.status not in decided:
return "unscored_baseline"
if enhanced.status not in decided:
return "unscored_enhanced"
if base.status is TestStatus.FAIL and enhanced.status is TestStatus.PASS:
return "fixed"
if base.status is TestStatus.PASS and enhanced.status is TestStatus.FAIL:
return "broken"
return "unchanged"
def compare_scorecards(
baseline: TestRunResult,
enhanced: TestRunResult,
) -> ComparisonReport:
baseline_scores = {
br.test_id: br for br in baseline.test_results
}
enhanced_scores = {
br.test_id: br for br in enhanced.test_results
}
all_ids = sorted(
set(baseline_scores.keys()) | set(enhanced_scores.keys())
)
deltas: list[TestDelta] = []
gaps_closed: list[str] = []
gaps_opened: list[str] = []
gaps_remaining: list[str] = []
for bid in all_ids:
base_result = baseline_scores.get(bid)
enh_result = enhanced_scores.get(bid)
# Absence means that the inspection was not run. Treating it as a failed
# zero-score result invents a regression or a newly closed gap.
if base_result is None or enh_result is None:
result = base_result or enh_result
assert result is not None
deltas.append(
TestDelta(
test_id=bid,
test_name=result.name,
baseline_score=_scored_value(base_result),
enhanced_score=_scored_value(enh_result),
status_change=(
"missing_baseline" if base_result is None else "missing_enhanced"
),
)
)
continue
if base_result.findings_only and enh_result.findings_only:
status = findings_only_change(base_result, enh_result)
if status == "fixed":
gaps_closed.append(bid)
elif status == "broken":
gaps_opened.append(bid)
elif enh_result.status is TestStatus.FAIL:
gaps_remaining.append(bid)
deltas.append(
TestDelta(
test_id=bid,
test_name=enh_result.name or base_result.name or bid,
status_change=status,
gap_closed=status == "fixed",
)
)
continue
base_score = _scored_value(base_result)
enh_score = _scored_value(enh_result)
if base_score is None or enh_score is None:
if base_score is None and enh_score is None:
status = "unscored_both"
elif base_score is None:
status = "unscored_baseline"
else:
status = "unscored_enhanced"
deltas.append(
TestDelta(
test_id=bid,
test_name=enh_result.name or base_result.name or bid,
baseline_score=base_score,
enhanced_score=enh_score,
status_change=status,
)
)
continue
base_passed = base_result.passing
enh_passed = enh_result.passing
name = enh_result.name or base_result.name or bid
delta = enh_score - base_score
if not base_passed and enh_passed:
status = "fixed"
gaps_closed.append(bid)
elif base_passed and not enh_passed:
status = "broken"
gaps_opened.append(bid)
elif delta > 0.01:
status = "improved"
if not enh_passed:
gaps_remaining.append(bid)
elif delta < -0.01:
status = "regressed"
if not enh_passed:
gaps_remaining.append(bid)
else:
status = "unchanged"
if not enh_passed:
gaps_remaining.append(bid)
deltas.append(
TestDelta(
test_id=bid,
test_name=name,
baseline_score=base_score,
enhanced_score=enh_score,
delta=delta,
status_change=status,
gap_closed=not base_passed and enh_passed,
)
)
fixture_mismatch = baseline.fixture_name != enhanced.fixture_name
baseline_overall = baseline.overall_score
enhanced_overall = enhanced.overall_score
overall_delta = (
enhanced_overall - baseline_overall
if baseline_overall is not None and enhanced_overall is not None
else None
)
baseline_grade = baseline.grade.value if baseline_overall is not None else "n/a"
enhanced_grade = enhanced.grade.value if enhanced_overall is not None else "n/a"
return ComparisonReport(
baseline=baseline,
enhanced=enhanced,
baseline_system=baseline.system_name,
enhanced_system=enhanced.system_name,
baseline_overall=baseline_overall,
enhanced_overall=enhanced_overall,
overall_delta=overall_delta,
grade_change=f"{baseline_grade} → {enhanced_grade}",
baseline_grade=baseline.grade,
enhanced_grade=enhanced.grade,
test_deltas=deltas,
gaps_closed=gaps_closed,
gaps_opened=gaps_opened,
gaps_remaining=gaps_remaining,
fixture_mismatch=fixture_mismatch,
)