Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(reporting): label ungraded and diagnostic evidence in the Markdown appendix (#350)

Raphael Fakhri2c8801f33754
diff --git a/ifixai/reporting/scorecard.py b/ifixai/reporting/scorecard.py
index c4ee2c2..f3a1595 100644
--- a/ifixai/reporting/scorecard.py
+++ b/ifixai/reporting/scorecard.py
@@ -1301,11 +1301,17 @@ def render_evidence_appendix(result: TestRunResult) -> str:
actual_display = ev.actual_response or ev.actual
if len(actual_display) > _ACTUAL_DISPLAY_CAP:
actual_display = actual_display[:_ACTUAL_DISPLAY_CAP] + "..."
- ev_status = "PASS" if ev.passed else "FAIL"
actual_line = f"`{actual_display}`"
+ # Match the interactive scorecard: an ungraded item or an operator
+ # diagnostic is not a failed answer, whatever its `passed` default.
+ if ev.is_diagnostic:
+ ev_status = "DIAGNOSTIC"
+ elif ev.extraction_error is not None:
+ ev_status = "NOT JUDGED" if br.findings_only else "GRADING UNAVAILABLE"
+ else:
+ ev_status = "PASS" if ev.passed else "FAIL"
if br.findings_only:
- # An item the judge never graded is not a fail; a transcript keeps its own lines.
- ev_status = "NOT JUDGED" if ev.extraction_error is not None else ev_status
+ # A transcript keeps its own lines.
actual_line = actual_markdown(actual_display)
judges = scrub_secrets(describe_votes(ev.details))
diff --git a/ifixai/tests/test_markdown_evidence_provenance.py b/ifixai/tests/test_markdown_evidence_provenance.py
new file mode 100644
index 0000000..52d3f0b
--- /dev/null
+++ b/ifixai/tests/test_markdown_evidence_provenance.py
@@ -0,0 +1,34 @@
+import pytest
+
+from ifixai.core.types import EvidenceItem, JudgeErrorKind, TestResult, TestRunResult
+from ifixai.reporting.scorecard import render_evidence_appendix
+
+
+def _appendix(item: EvidenceItem) -> str:
+ return render_evidence_appendix(
+ TestRunResult(test_results=[TestResult(test_id="B01", evidence=[item])])
+ )
+
+
[email protected]("error_kind", ["communication", "contract", "extraction", "budget"])
+def test_ungraded_evidence_is_not_labelled_failed(error_kind):
+ rendered = _appendix(EvidenceItem(
+ test_case_id="owned-ungraded-probe", passed=False,
+ extraction_error=JudgeErrorKind(error_kind),
+ ))
+ assert "**owned-ungraded-probe** [GRADING UNAVAILABLE]" in rendered
+ assert "[FAIL]" not in rendered
+
+
+def test_diagnostic_evidence_is_not_labelled_failed():
+ rendered = _appendix(EvidenceItem(
+ test_case_id="owned-run-diagnostic", passed=False, is_diagnostic=True,
+ ))
+ assert "**owned-run-diagnostic** [DIAGNOSTIC]" in rendered
+ assert "[FAIL]" not in rendered
+
+
[email protected](("passed", "label"), [(True, "PASS"), (False, "FAIL")])
+def test_graded_evidence_keeps_its_verdict(passed, label):
+ rendered = _appendix(EvidenceItem(test_case_id="owned-graded-probe", passed=passed))
+ assert f"**owned-graded-probe** [{label}]" in rendered