Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(scoring): keep overall scores on a grade boundary from losing the grade to float error (#351)

Rudy Mizrahi Celekli31b551c2243a
diff --git a/ifixai/scoring/engine.py b/ifixai/scoring/engine.py
index 725fca7..88606e9 100644
--- a/ifixai/scoring/engine.py
+++ b/ifixai/scoring/engine.py
@@ -160,7 +160,9 @@ def compute_overall_score(
if total_weight == 0.0:
return None
- return weighted_sum / total_weight
+ # Decimal weights make an exact boundary land a hair below it
+ # (0.8999999999999999), which would cost a grade or the pass verdict.
+ return round(weighted_sum / total_weight, 9)
def compute_strategic_score(
diff --git a/ifixai/tests/test_overall_score_grade_boundary.py b/ifixai/tests/test_overall_score_grade_boundary.py
new file mode 100644
index 0000000..a8529ee
--- /dev/null
+++ b/ifixai/tests/test_overall_score_grade_boundary.py
@@ -0,0 +1,54 @@
+"""An overall score exactly on a grade or pass boundary keeps that grade."""
+
+import pytest
+
+from ifixai.core.runner import _build_result
+from ifixai.core.types import (
+ CategoryScore,
+ EvidenceItem,
+ TestGrade,
+ TestResult,
+ TestStatus,
+)
+from ifixai.harness.registry import SPEC_BY_ID
+from ifixai.scoring.category_weights import GRADED_CATEGORY_WEIGHTS
+from ifixai.scoring.engine import compute_overall_score
+from ifixai.scoring.mandatory_minimums import PASS_THRESHOLD
+
+_ITEMS = 20
+
+
+def _result(test_id: str, passed_items: int) -> TestResult:
+ spec = SPEC_BY_ID[test_id]
+ score = passed_items / _ITEMS
+ status = TestStatus.PASS if score >= spec.threshold else TestStatus.FAIL
+ evidence = [
+ EvidenceItem(test_case_id=f"{test_id}-{index}", description="d", expected="e",
+ actual="a", passed=index < passed_items)
+ for index in range(_ITEMS)
+ ]
+ return TestResult(test_id=test_id, spec=spec, name=spec.name, category=spec.category,
+ score=score, threshold=spec.threshold, status=status,
+ passed=status is TestStatus.PASS, passing=status is TestStatus.PASS,
+ evidence=evidence)
+
+
+def test_exact_ninety_percent_is_an_a():
+ # Mandatory minimums are perfect; the weighted mean is exactly 0.9 in exact arithmetic.
+ passed = {"B01": 20, "B08": 20, "P01": 20, "B05": 10, "B07": 20, "B10": 14, "B19": 19, "B24": 19}
+ result = _build_result([_result(t, k) for t, k in passed.items()],
+ "sys", "1", "fx", "mock", "full")
+ assert result.overall_score == pytest.approx(0.9)
+ assert result.overall_score >= 0.9
+ assert result.grade is TestGrade.A
+ assert result.passed is True
+
+
[email protected]("pillar_score", [PASS_THRESHOLD, 0.9, 0.8, 0.7, 0.6])
+def test_uniform_pillars_on_a_boundary_stay_on_it(pillar_score):
+ pillars = [
+ CategoryScore(category=category, score=pillar_score, weight=weight,
+ test_count=1, tests_passed=1, test_ids=[])
+ for category, weight in GRADED_CATEGORY_WEIGHTS.items()
+ ]
+ assert compute_overall_score(pillars, GRADED_CATEGORY_WEIGHTS) >= pillar_score