Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(reporting): preserve grading provenance in human exports (#201)
Rudy Mizrahi Celeklia27b238e9362
diff --git a/ifixai/reporting/artifact.py b/ifixai/reporting/artifact.pyindex d18e980..ebc1fa6 100644--- a/ifixai/reporting/artifact.py+++ b/ifixai/reporting/artifact.py@@ -78,6 +78,8 @@ def _evidence_payload(ev) -> dict[str, Any]:"expected": _scrub(ev.expected or ev.expected_behavior or ""),"actual": _scrub(ev.actual_response or ev.actual or ""),"evaluation_result": ev.evaluation_result or "",+ "extraction_error": ev.extraction_error.value if ev.extraction_error else None,+ "is_diagnostic": ev.is_diagnostic,"passed": ev.passed,"evaluation_method": ev.evaluation_method.value,"verdict": verdict_label,@@ -242,6 +244,8 @@ def _build_payload(},"categories": categories,"compliance": compliance,+ "warnings": [_scrub(warning) for warning in result.warnings],+ "validation_warnings": [_scrub(warning) for warning in result.validation_warnings],"checks": [_check_payload(br) for br in sorted(result.test_results, key=lambda b: b.test_id)],"diff": _diff_payload(result, previous) if previous else None,}@@ -354,6 +358,12 @@ function header(){</div>`;}+function warningBanners(){+ const render = (warnings, kind) => (warnings||[])+ .map(w => `<div class="banner ${kind}">${esc(w)}</div>`).join('');+ return render(D.validation_warnings, 'bad') + render(D.warnings, 'warn');+}+function categories(){if(!D.categories.length) return '';const rows = D.categories.map(c=>`<tr><td>${esc(c.category)}</td><td>${esc(c.score_pct)}</td><td>${esc((c.weight*100).toFixed(0))}%</td><td>${esc(c.coverage)}</td></tr>`).join('');@@ -379,12 +389,15 @@ function diffSection(){}function evidence(ev){+ const label = ev.is_diagnostic ? 'diagnostic' : ev.extraction_error ? 'grading unavailable' : ev.passed ? 'pass' : 'fail';+ const tagClass = ev.is_diagnostic || ev.extraction_error ? 'inconclusive' : label;const conf = ev.confidence==null?'':` · confidence ${(ev.confidence*100).toFixed(0)}%`;const dims = (ev.dimensions||[]).map(dm=>`<tr><td>${dm.passed?'✓':'✗'} ${esc(dm.name)}${dm.mandatory?' <span class="dim">(mandatory)</span>':''}</td><td>${dm.confidence==null?'':(dm.confidence*100).toFixed(0)+'%'}</td><td class="dim">${esc(dm.reasoning)}</td></tr>`).join('');const fld=(label,val)=> val? `<div class="kvp"><b>${label}</b><pre>${esc(val)}</pre></div>`:'';return `<div class="ev">- <div class="evhead"><span class="tag ${ev.passed?'pass':'fail'}">${ev.passed?'pass':'fail'}</span><span>${esc(ev.evaluation_method)}${conf}</span></div>+ <div class="evhead"><span class="tag ${tagClass}">${label}</span><span>${esc(ev.evaluation_method)}${conf}</span></div>${ev.description?`<div class="kvp dim">${esc(ev.description)}</div>`:''}+ ${ev.extraction_error?fld('Grading unavailable',ev.extraction_error):''}${fld('Prompt used',ev.prompt)}${fld('Expected',ev.expected)}${fld('Actual',ev.actual)}@@ -455,6 +468,7 @@ function checksSection(){const app = document.getElementById('app');app.insertAdjacentHTML('beforeend', header());+app.insertAdjacentHTML('beforeend', warningBanners());const diff = diffSection(); if(diff) app.insertAdjacentHTML('beforeend', diff);app.insertAdjacentHTML('beforeend', categories());app.insertAdjacentHTML('beforeend', compliance());diff --git a/ifixai/reporting/scorecard.py b/ifixai/reporting/scorecard.pyindex 945daaf..2a21e28 100644--- a/ifixai/reporting/scorecard.py+++ b/ifixai/reporting/scorecard.py@@ -425,6 +425,13 @@ def render_not_run_section(result: TestRunResult) -> str:return "\n".join(lines)+def render_run_warnings(result: TestRunResult) -> str:+ """Keep operator and grading caveats with exported results."""+ if not result.warnings:+ return ""+ return "## Run Warnings\n\n" + "\n".join(f"- {warning}" for warning in result.warnings)++def generate_markdown_report(result: TestRunResult) -> str:frameworks = load_all_mappings()@@ -436,6 +443,7 @@ def generate_markdown_report(result: TestRunResult) -> str:render_insights(result),render_category_table(result),render_mandatory_minimums(result),+ render_run_warnings(result),render_consistency_warnings(result),render_test_table(result),render_not_run_section(result),diff --git a/ifixai/tests/test_artifact_diff_unscored.py b/ifixai/tests/test_artifact_diff_unscored.pyindex 10ce66f..f31375f 100644--- a/ifixai/tests/test_artifact_diff_unscored.py+++ b/ifixai/tests/test_artifact_diff_unscored.py@@ -77,3 +77,69 @@ def test_unscored_artifact_does_not_display_a_failing_grade():assert payload["summary"]["grade"] == "n/a"assert payload["summary"]["grade_class"] == "inconclusive"assert payload["diff"]["grade_change"] == "n/a → n/a"+++def _warning_run():+ return TestRunResult(+ system_name="warning-provenance-control",+ warnings=["Pinned seeds: memorization resistance reduced", "Judge substituted after error"],+ validation_warnings=["run_invalid: ignore the grade", "judge_health: verdict contract failures"],+ )+++def test_markdown_preserves_operator_warnings():+ from ifixai.reporting.scorecard import generate_markdown_report++ result = _warning_run()+ rendered = generate_markdown_report(result)+ for warning in [*result.warnings, *result.validation_warnings]:+ assert warning in rendered+++def test_artifact_preserves_operator_and_measurement_warnings():+ result = _warning_run()+ payload = _build_payload(+ result, live=True, transport="http", sut_model=None,+ judge_model=None, honesty_note="", previous=None,+ )+ assert payload["warnings"] == result.warnings+ assert payload["validation_warnings"] == result.validation_warnings+++def test_artifact_warning_data_cannot_terminate_embedded_script():+ from ifixai.reporting.artifact import render_artifact++ warning = "Controlled warning </script><img src=x onerror=alert(1)>"+ rendered = render_artifact(+ TestRunResult(warnings=[warning]), live=True, transport="http",+ sut_model=None, judge_model=None, honesty_note="",+ )+ assert warning not in rendered+ assert "Controlled warning \\u003c/script>" in rendered++[email protected]("error_kind", ["communication", "contract", "extraction", "budget"])+def test_artifact_evidence_retains_unscorable_cause(error_kind):+ from ifixai.core.types import EvidenceItem, JudgeErrorKind+ from ifixai.reporting.artifact import _evidence_payload++ item = EvidenceItem(+ test_case_id="owned-ungraded-probe", passed=False,+ actual_response="A reply without a usable grading result",+ extraction_error=JudgeErrorKind(error_kind),+ )+ payload = _evidence_payload(item)+ assert payload["extraction_error"] == error_kind+ assert payload["is_diagnostic"] is False+++def test_artifact_evidence_retains_diagnostic_marker():+ from ifixai.core.types import EvidenceItem+ from ifixai.reporting.artifact import _evidence_payload++ payload = _evidence_payload(EvidenceItem(+ test_case_id="owned-run-diagnostic", passed=False, is_diagnostic=True,+ description="Transport diagnostics, excluded from scoring",+ ))+ assert payload["is_diagnostic"] is True+ assert payload["extraction_error"] is None