Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: reject partial and unscored results in CI example (#136)
Rudy Mizrahi Celekliffda13cd6ecf
diff --git a/ifixai/examples/ci_check.py b/ifixai/examples/ci_check.pyindex 25cea79..43d95ad 100644--- a/ifixai/examples/ci_check.py+++ b/ifixai/examples/ci_check.py@@ -47,8 +47,10 @@ async def run_ci_check() -> int:f.write(generate_json_report(result))print(f"Report written to {config['report_path']}")- print(f"Grade: {result.grade.value}")- print(f"Overall Score: {result.overall_score:.0%}")+ grade = result.grade.value if result.overall_score is not None else "n/a"+ overall = f"{result.overall_score:.0%}" if result.overall_score is not None else "n/a"+ print(f"Grade: {grade}")+ print(f"Overall Score: {overall}")print(f"Strategic Score: {result.strategic_score:.0%}")passed_count = sum(1 for br in result.test_results if br.passed)@@ -60,7 +62,15 @@ async def run_ci_check() -> int:is_passing = True- if result.overall_score < min_score:+ if result.partial:+ reason = f": {result.abort_reason}" if result.abort_reason else ""+ print(f"\nFAIL: Partial run{reason}; finish the run before evaluating CI thresholds")+ is_passing = False++ if result.overall_score is None:+ print("\nFAIL: Overall score unavailable; the run is not gradeable")+ is_passing = False+ elif result.overall_score < min_score:print(f"\nFAIL: Overall score {result.overall_score:.0%} < {min_score:.0%}")is_passing = Falsediff --git a/ifixai/tests/test_ci_check_unscored.py b/ifixai/tests/test_ci_check_unscored.pynew file mode 100644index 0000000..0c7a361--- /dev/null+++ b/ifixai/tests/test_ci_check_unscored.py@@ -0,0 +1,62 @@+import asyncio++import pytest++from ifixai.core.types import TestGrade, TestRunResult+from ifixai.examples import ci_check+++def _config() -> dict[str, str]:+ return {+ "provider": "mock",+ "api_key": "test-key",+ "fixture": "test-fixture",+ "model": "",+ "system_name": "test-system",+ "system_version": "1.0",+ "min_score": "0.70",+ "min_strategic": "0.80",+ "report_path": "",+ }+++ ("overall_score", "partial", "expected_message"),+ [+ (None, False, "Overall Score: n/a"),+ (0.95, True, "partial run"),+ ],+)+def test_ci_check_fails_cleanly_when_run_is_not_gradeable(+ monkeypatch, capsys, overall_score, partial, expected_message+):+ async def fake_run_inspections(**kwargs):+ return TestRunResult(+ overall_score=overall_score,+ strategic_score=0.95,+ grade=TestGrade.A,+ mandatory_minimums_passed=True,+ partial=partial,+ abort_reason="judge quota exhausted" if partial else None,+ )++ monkeypatch.setattr(ci_check, "read_env_config", _config)+ monkeypatch.setattr(ci_check, "run_inspections", fake_run_inspections)++ assert asyncio.run(ci_check.run_ci_check()) == 1+ assert expected_message.lower() in capsys.readouterr().out.lower()+++def test_ci_check_still_passes_a_complete_scored_run(monkeypatch):+ async def fake_run_inspections(**kwargs):+ return TestRunResult(+ overall_score=0.95,+ strategic_score=0.95,+ grade=TestGrade.A,+ mandatory_minimums_passed=True,+ )++ monkeypatch.setattr(ci_check, "read_env_config", _config)+ monkeypatch.setattr(ci_check, "run_inspections", fake_run_inspections)++ assert asyncio.run(ci_check.run_ci_check()) == 0