Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: show unscored artifact results without false grades or verdicts (#134)

Rudy Mizrahi Celeklidca29c35ab16
diff --git a/ifixai/reporting/artifact.py b/ifixai/reporting/artifact.py
index bb90b5c..4728913 100644
--- a/ifixai/reporting/artifact.py
+++ b/ifixai/reporting/artifact.py
@@ -111,7 +111,11 @@ def _check_payload(br) -> dict[str, Any]:
def _diff_payload(result: TestRunResult, previous: dict[str, Any]) -> dict[str, Any]:
"""A plain-data diff of this run against a previous run's results JSON."""
prev_overall = (previous.get("overall") or {}).get("score")
- prev_grade = (previous.get("overall") or {}).get("grade", "?")
+ prev_grade = (
+ (previous.get("overall") or {}).get("grade", "?")
+ if prev_overall is not None
+ else "n/a"
+ )
cur_overall = result.overall_score
overall_delta = (
round(cur_overall - prev_overall, 4)
@@ -138,11 +142,11 @@ def _diff_payload(result: TestRunResult, previous: dict[str, Any]) -> dict[str,
new_status = cur.status.value
prev_score = prev.get("score")
new_score = None if cur.status in {TestStatus.INCONCLUSIVE, TestStatus.ERROR} else round(cur.score, 4)
- was_pass = prev_status == "pass"
- now_pass = new_status == "pass"
- if not was_pass and now_pass:
+ # Only a measured FAIL can become "fixed", and only a measured PASS
+ # can become "broken". Inconclusive/error means no verdict was made.
+ if prev_status == "fail" and new_status == "pass":
kind = "fixed"
- elif was_pass and not now_pass:
+ elif prev_status == "pass" and new_status == "fail":
kind = "broken"
elif prev_score is not None and new_score is not None and new_score - prev_score > 0.01:
kind = "improved"
@@ -161,7 +165,7 @@ def _diff_payload(result: TestRunResult, previous: dict[str, Any]) -> dict[str,
})
return {
"overall_delta": overall_delta,
- "grade_change": f"{prev_grade} → {result.grade.value}",
+ "grade_change": f"{prev_grade} → {result.grade.value if cur_overall is not None else 'n/a'}",
"changes": changes,
}
@@ -212,8 +216,12 @@ def _build_payload(
"honesty_note": honesty_note,
},
"summary": {
- "grade": result.grade.value,
- "grade_class": _GRADE_CLASS.get(result.grade.value, "inconclusive"),
+ "grade": result.grade.value if overall is not None else "n/a",
+ "grade_class": (
+ _GRADE_CLASS.get(result.grade.value, "inconclusive")
+ if overall is not None
+ else "inconclusive"
+ ),
"overall_pct": "n/a (insufficient evidence)" if overall is None else f"{overall * 100:.1f}%",
"score_before_cap": None if result.overall_score_before_cap is None
else f"{result.overall_score_before_cap * 100:.1f}%",
diff --git a/ifixai/tests/test_artifact_diff_unscored.py b/ifixai/tests/test_artifact_diff_unscored.py
new file mode 100644
index 0000000..10ce66f
--- /dev/null
+++ b/ifixai/tests/test_artifact_diff_unscored.py
@@ -0,0 +1,79 @@
+import pytest
+
+from ifixai.core.types import TestResult, TestRunResult, TestStatus
+from ifixai.reporting.artifact import _build_payload, _diff_payload
+
+
[email protected]("previous_status", ["inconclusive", "error"])
+def test_unscored_to_pass_is_status_change_not_fixed(previous_status):
+ current = TestRunResult(
+ test_results=[
+ TestResult(test_id="B01", score=0.9, status=TestStatus.PASS, passing=True)
+ ]
+ )
+ previous = {
+ "test_results": [
+ {"test_id": "B01", "status": previous_status, "score": None}
+ ]
+ }
+
+ diff = _diff_payload(current, previous)
+
+ assert diff["changes"][0]["kind"] == "status"
+ assert diff["changes"][0]["prev_status"] == previous_status
+
+
[email protected]("current_status", [TestStatus.INCONCLUSIVE, TestStatus.ERROR])
+def test_pass_to_unscored_is_status_change_not_broken(current_status):
+ current = TestRunResult(
+ test_results=[TestResult(test_id="B01", status=current_status)]
+ )
+ previous = {
+ "test_results": [{"test_id": "B01", "status": "pass", "score": 0.9}]
+ }
+
+ diff = _diff_payload(current, previous)
+
+ assert diff["changes"][0]["kind"] == "status"
+ assert diff["changes"][0]["new_status"] == current_status.value
+
+
+ ("previous_status", "current_status", "expected_kind"),
+ [
+ ("fail", TestStatus.PASS, "fixed"),
+ ("pass", TestStatus.FAIL, "broken"),
+ ],
+)
+def test_scored_verdict_changes_remain_fixed_or_broken(
+ previous_status, current_status, expected_kind
+):
+ current = TestRunResult(
+ test_results=[TestResult(test_id="B01", status=current_status)]
+ )
+ previous = {
+ "test_results": [{"test_id": "B01", "status": previous_status, "score": 0.5}]
+ }
+
+ diff = _diff_payload(current, previous)
+
+ assert diff["changes"][0]["kind"] == expected_kind
+
+
+def test_unscored_artifact_does_not_display_a_failing_grade():
+ unscored = TestRunResult(overall_score=None)
+ previous = {"overall": {"score": None, "grade": "F"}, "test_results": []}
+
+ payload = _build_payload(
+ unscored,
+ live=False,
+ transport="offline",
+ sut_model=None,
+ judge_model=None,
+ honesty_note="",
+ previous=previous,
+ )
+
+ assert payload["summary"]["grade"] == "n/a"
+ assert payload["summary"]["grade_class"] == "inconclusive"
+ assert payload["diff"]["grade_change"] == "n/a → n/a"