Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(reporting): preserve JSON evidence audit metadata (#178)
Rudy Mizrahi Celekli1e8e8755ce0a
diff --git a/ifixai/reporting/scorecard.py b/ifixai/reporting/scorecard.pyindex 003cc65..7f78099 100644--- a/ifixai/reporting/scorecard.py+++ b/ifixai/reporting/scorecard.py@@ -578,6 +578,12 @@ def build_test_results_section("inspection_method": ev.inspection_method.value,"evaluation_method": ev.evaluation_method.value,}+ # Keep the provenance used by runners to distinguish diagnostics+ # and unscorable probes from behavioral failures. Pydantic's JSON+ # mode also converts any enums nested in the structured details.+ ev_dict.update(ev.model_dump(+ mode="json", include={"details", "extraction_error", "is_diagnostic"}+ ))if ev.dimension_scores:ev_dict["dimension_scores"] = [{@@ -630,6 +636,11 @@ def build_test_results_section("sample_size": br.confidence_interval.sample_size,"warning": br.confidence_interval.warning,}+ if br.score_breakdown is not None:+ br_dict["score_breakdown"] = br.score_breakdown+ if br.variant_seed is not None:+ br_dict["variant_seed"] = br.variant_seed+ br_dict["variant_seed_pinned"] = br.variant_seed_pinnedif br.evaluation_mode:br_dict["evaluation_mode"] = br.evaluation_mode.valueif br.judge_calls_used:diff --git a/ifixai/tests/test_json_audit_metadata.py b/ifixai/tests/test_json_audit_metadata.pynew file mode 100644index 0000000..6a00615--- /dev/null+++ b/ifixai/tests/test_json_audit_metadata.py@@ -0,0 +1,81 @@+import json++import pytest++from ifixai.core.types import (+ EvaluationMethod,+ EvaluationPipelineConfig,+ EvidenceItem,+ Fixture,+ FixtureMetadata,+ JudgeErrorKind,+ PipelineResult,+ ProviderConfig,+ TestResult,+ TestRunResult,+ TestStatus,+ User,+)+from ifixai.inspections.b14_covert_side_task.runner import B14CovertSideTask+from ifixai.providers.base import ChatProvider, ProviderConnectionError+from ifixai.reporting.scorecard import generate_json_report, generate_markdown_report+from ifixai.shared.evidence import flag_diagnostics+++class SyntheticProvider(ChatProvider):+ def __init__(self):+ self.calls = 0++ async def send_message(self, messages, config):+ self.calls += 1+ if self.calls == 1:+ raise ProviderConnectionError(details="synthetic transport interruption")+ return "Synthetic refusal for export contract testing."+++class PassingPipeline:+ async def evaluate(self, **kwargs):+ return PipelineResult(passed=True, evaluation_result="synthetic pass",+ evaluation_method=EvaluationMethod.JUDGE)++[email protected]("projection", ["replay", "evidence"])+async def test_json_preserves_native_b14_audit_metadata(projection):+ fixture = Fixture(metadata=FixtureMetadata(name="synthetic audit metadata"),+ users=[User(user_id="u1", name="Operator", roles=["operator"])])+ result = await B14CovertSideTask().execute(+ SyntheticProvider(), ProviderConfig(provider="synthetic"), fixture,+ pipeline_config=EvaluationPipelineConfig(b14_seed=0, b14_seed_pinned=True),+ pipeline=PassingPipeline())+ assert result.status == TestStatus.PASS+ run = TestRunResult(test_results=[result])+ exported = json.loads(generate_json_report(run))["test_results"][0]+ if projection == "replay":+ assert "**B14** seed: 0 (pinned)" in generate_markdown_report(run)+ assert exported["variant_seed"] == result.variant_seed == 0+ assert exported["variant_seed_pinned"] is True+ assert exported["score_breakdown"] == result.score_breakdown+ else:+ failed = result.evidence[0]+ assert failed.extraction_error == JudgeErrorKind.COMMUNICATION+ assert exported["evidence"][0]["extraction_error"] == "communication"+ for original, item in zip(result.evidence, exported["evidence"], strict=True):+ assert item["details"] == original.model_dump(mode="json")["details"]+ assert item.get("is_diagnostic", False) == original.is_diagnostic+ assert item.get("extraction_error") == (+ original.extraction_error.value if original.extraction_error else None)+++def test_json_preserves_diagnostic_flag_and_structured_details():+ evidence = flag_diagnostics([+ EvidenceItem(test_case_id="coverage-summary", passed=False,+ details={"scored": 3, "bands": ["public", "private"]}),+ EvidenceItem(test_case_id="measurement", passed=True),+ ], ("coverage-",))+ run = TestRunResult(test_results=[TestResult(test_id="B14", evidence=evidence)])+ exported = json.loads(generate_json_report(run))["test_results"][0]["evidence"]+ assert exported[0]["is_diagnostic"] is True+ assert exported[0]["details"] == evidence[0].details+ assert exported[1].get("is_diagnostic", False) is False+ assert exported[1].get("extraction_error") is None