Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(trajectory): report a defective delegation grant once, not once per use (#348)
Rudy Mizrahi Celekli079599dc22a3
diff --git a/ifixai/tests/trajectory/test_grant_violation_dedupe.py b/ifixai/tests/trajectory/test_grant_violation_dedupe.pynew file mode 100644index 0000000..cca37cc--- /dev/null+++ b/ifixai/tests/trajectory/test_grant_violation_dedupe.py@@ -0,0 +1,39 @@+"""A defective grant is one finding however many actions cite it."""++from collections import Counter++from ifixai.trajectory import audit_trajectory+from ifixai.trajectory.models import (+ ActorPolicy,+ AuditPolicy,+ DelegationPayload,+ PermissionRule,+)+from ifixai.trajectory.recorder import TraceRecorder+++def test_invalid_grant_is_reported_once_per_defect():+ recorder = TraceRecorder("tenant", "collector", clock_ns=lambda: 100)+ # The issuer has no direct permission, so the root grant itself is invalid.+ grant = recorder.emit(DelegationPayload(+ actor_id="manager", subject_id="worker", action="write",+ resource_scope="orders:*", expires_ns=1000,+ ))+ for index in range(3):+ recorder.invoke(+ lambda: None, actor_id="worker", action="write", resource=f"orders:{index}",+ arguments={}, delegation_id=grant.event_id, parents=(grant.event_id,),+ )+ policy = AuditPolicy(+ name="owned", tenant_id="tenant", trusted_collectors=("collector",),+ actors={actor: ActorPolicy(roles=("owner",) if actor == "owner" else ())+ for actor in ("owner", "manager", "worker")},+ permissions=(PermissionRule(role="owner", action="write", resource_scope="orders:*"),),+ require_citations=False,+ )+ report = audit_trajectory(recorder.seal(), policy)++ codes = Counter(violation.code for violation in report.violations)+ assert codes["DELEGATION_ISSUER_PERMISSION"] == 1+ # Each of the three actions is still its own permission finding.+ assert codes["PERMISSION_DENIED"] == 3diff --git a/ifixai/tests/trajectory/test_violation_dedupe_compat.py b/ifixai/tests/trajectory/test_violation_dedupe_compat.pynew file mode 100644index 0000000..aa02170--- /dev/null+++ b/ifixai/tests/trajectory/test_violation_dedupe_compat.py@@ -0,0 +1,92 @@+"""Only exact repeat findings collapse, and bundles saved with repeats still replay."""++import json++import pytest++from ifixai.trajectory import (+ audit_trajectory,+ canonical_digest,+ create_audit_bundle,+ load_audit_bundle,+)+from ifixai.trajectory.models import (+ ActorPolicy,+ AuditPolicy,+ BudgetRule,+ DelegationPayload,+ PermissionRule,+)+from ifixai.trajectory.recorder import TraceRecorder+++def test_two_budget_rules_broken_at_one_action_are_two_findings(native_example):+ capture, _, _ = native_example.record_scenario("budget-overrun")+ policy = native_example.demo_policy().model_copy(update={"budgets": (+ BudgetRule(scope="trace", limit_units=100),+ BudgetRule(scope="actor", actor_id="agent", limit_units=100),+ )})+ report = audit_trajectory(capture, policy)++ budget = sorted(v.message for v in report.violations if v.code == "BUDGET_EXCEEDED")+ assert budget == [+ "Configured actor:agent budget rule 1 exceeded its integer limit.",+ "Configured trace budget rule 0 exceeded its integer limit.",+ ]+++def _saved_with_repeats():+ """A bundle for one bad grant used twice, stored the way 4.2.0 wrote it: the+ grant's finding once at the grant and again before each action's own finding."""+ recorder = TraceRecorder("tenant", "collector", clock_ns=lambda: 100)+ grant = recorder.emit(DelegationPayload(+ actor_id="manager", subject_id="worker", action="write",+ resource_scope="orders:*", expires_ns=1000,+ ))+ for index in range(2):+ recorder.invoke(+ lambda: None, actor_id="worker", action="write", resource=f"orders:{index}",+ arguments={}, delegation_id=grant.event_id, parents=(grant.event_id,),+ )+ policy = AuditPolicy(+ name="owned", tenant_id="tenant", trusted_collectors=("collector",),+ actors={actor: ActorPolicy(roles=("owner",) if actor == "owner" else ())+ for actor in ("owner", "manager", "worker")},+ permissions=(PermissionRule(role="owner", action="write", resource_scope="orders:*"),),+ require_citations=False,+ )+ raw = create_audit_bundle(recorder.seal(), policy).model_dump(mode="json")+ issuer, *denied = raw["report"]["violations"]+ assert issuer["code"] == "DELEGATION_ISSUER_PERMISSION"+ assert [v["code"] for v in denied] == ["PERMISSION_DENIED", "PERMISSION_DENIED"]+ raw["report"]["violations"] = [issuer, issuer, denied[0], issuer, denied[1]]+ return raw+++def _seal(raw, path):+ raw["integrity_digest"] = canonical_digest({k: v for k, v in raw.items() if k != "integrity_digest"})+ path.write_text(json.dumps(raw), encoding="utf-8")+ return path+++def test_bundle_saved_with_repeated_findings_still_replays(tmp_path):+ bundle = load_audit_bundle(_seal(_saved_with_repeats(), tmp_path / "saved.json"))++ assert bundle.report.decision == "BLOCK"+ # The retained evidence is returned as saved, so its checksum still holds.+ assert len(bundle.report.violations) == 5++[email protected]("edit", ["reword", "rename", "drop"])+def test_changed_finding_in_saved_bundle_is_still_rejected(tmp_path, edit):+ raw = _saved_with_repeats()+ target = raw["report"]["violations"][2]+ if edit == "reword":+ target["message"] += " (edited)"+ elif edit == "rename":+ target["code"] = "EDITED_CODE"+ else:+ del raw["report"]["violations"][2]++ with pytest.raises(ValueError, match="audit report does not match"):+ load_audit_bundle(_seal(raw, tmp_path / "edited.json"))diff --git a/ifixai/trajectory/audit.py b/ifixai/trajectory/audit.pyindex c297b8c..45a7263 100644--- a/ifixai/trajectory/audit.py+++ b/ifixai/trajectory/audit.py@@ -57,6 +57,7 @@ class _Audit:self.policy = policyself.events = {event.event_id: event for event in capture.events}self.violations: list[Violation] = []+ self.violation_keys: set[tuple[str, str, str, str, tuple[str, ...], str | None]] = set()self.issues: list[CoverageIssue] = []self.edges = tuple((parent, event.event_id) for event in capture.events for parent in event.parents)self.paths: dict[str, tuple[str, ...]] = {}@@ -72,7 +73,14 @@ class _Audit:self.issues.append(CoverageIssue(code=code, event_id=event.event_id if event else None, message=message, witness_ids=witnesses))def violation(self, family: Literal["permission", "approval", "delegation", "budget", "citation"], code: str, event: TraceEvent, message: str, witnesses: tuple[str, ...] = (), actor: str | None = None) -> None:- self.violations.append(Violation(rule=family, code=code, event_id=event.event_id, message=message, witness_ids=witnesses or (event.event_id,), actor_id=actor))+ witness_ids = witnesses or (event.event_id,)+ # A grant is re-validated for every action or child that cites it; its own+ # defects are one finding, not one per citation. Only exact repeats are dropped.+ key = (family, code, event.event_id, message, witness_ids, actor)+ if key in self.violation_keys:+ return+ self.violation_keys.add(key)+ self.violations.append(Violation(rule=family, code=code, event_id=event.event_id, message=message, witness_ids=witness_ids, actor_id=actor))def ancestor(self, parent: str, child: str) -> bool:key = (parent, child)diff --git a/ifixai/trajectory/integrity.py b/ifixai/trajectory/integrity.pyindex 77cae1f..6a0991b 100644--- a/ifixai/trajectory/integrity.py+++ b/ifixai/trajectory/integrity.py@@ -59,6 +59,9 @@ def load_audit_bundle(path: str | Path) -> AuditBundle:raise ValueError("audit bundle must contain complete normalized evidence")if canonical_digest(bundle.model_dump(mode="json", exclude={"integrity_digest"})) != bundle.integrity_digest:raise ValueError("audit bundle integrity digest mismatch")- if audit_trajectory(bundle.capture, bundle.policy) != bundle.report:+ # Releases up to 4.2.0 stored a bad grant's finding once per use. An exact repeat adds+ # no fact, so drop repeats before comparing; any changed finding still mismatches.+ stored = bundle.report.model_copy(update={"violations": tuple(dict.fromkeys(bundle.report.violations))})+ if audit_trajectory(bundle.capture, bundle.policy) != stored:raise ValueError("audit report does not match the retained policy and capture")return bundle