Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
from ifixai.core.types import (
GovernanceGap,
RegulatoryFramework,
TestResult,
TestStatus,
)
from ifixai.mappings.loader import get_mappings_for_test, load_all_mappings
TEST_CAPABILITY_MAP: dict[str, str] = {
"B01": "Tool Authorization Control",
"B02": "Non-LLM Governance Layer",
"B03": "Audit Trail Coverage",
"B04": "Deterministic Override Mechanism",
"B05": "Source Provenance Tracking",
"B06": "Uncertainty Signalling",
"B07": "Hallucination Prevention",
"B08": "Privilege Escalation Prevention",
"B09": "Policy Violation Detection",
"B10": "Evaluation Consistency",
"B11": "System Controllability",
"B12": "Prompt Injection Defence",
"B13": "Plan Propagation Traceability",
"B14": "Covert Side-Task Detection",
"B15": "Long-Horizon Policy Adherence",
"B16": "Silent Failure Reporting",
"B17": "Intra-System Response Consistency",
"B18": "Goal Stability Maintenance",
"B19": "Context Accuracy Retention",
"B20": "Instruction Adherence Enforcement",
"B21": "Cross-Turn Objective Persistence",
"B22": "Decision Reproducibility",
"B23": "Policy Version Traceability",
"B24": "Risk Scoring Capability",
"B25": "Regulatory Compliance Readiness",
"B26": "Rate Limiting Enforcement",
"B27": "Cross-Session Memory Integrity",
"B28": "RAG Context Integrity",
"B29": "Prompt Sensitivity Consistency",
}
def identify_gaps(
results: list[TestResult],
frameworks: dict[str, RegulatoryFramework] | None = None,
) -> list[GovernanceGap]:
if frameworks is None:
frameworks = load_all_mappings()
gaps: list[GovernanceGap] = []
for result in sorted(results, key=lambda r: r.test_id):
if result.status != TestStatus.FAIL:
continue
capability = TEST_CAPABILITY_MAP.get(
result.test_id,
result.name or f"Unknown Capability ({result.test_id})",
)
reg_refs = get_mappings_for_test(result.test_id, frameworks)
deficit = result.threshold - result.score
priority = "high" if deficit > 0.5 else ("medium" if deficit > 0.2 else "low")
gaps.append(
GovernanceGap(
test_id=result.test_id,
test_name=result.name,
category=result.category,
current_score=result.score,
required_score=result.threshold,
gap_description=describe_gap(result, capability),
capability_missing=capability,
priority=priority,
regulatory_references=reg_refs,
)
)
return gaps
def describe_gap(result: TestResult, capability: str) -> str:
"""The gap in words. A findings-only result names no percentage: its findings are the result."""
if result.findings_only:
return f"{capability} — failed on its findings (report only)"
return f"{capability} — scored {result.score:.0%} against {result.threshold:.0%} threshold"