Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(8.1): the audit retention window is enforced, and a truncated chain still verifies

Beenu Arorafd8382ae50f1
diff --git a/services/api/app/api/v1/endpoints/health.py b/services/api/app/api/v1/endpoints/health.py
index e7d97d800..83ff7a787 100644
--- a/services/api/app/api/v1/endpoints/health.py
+++ b/services/api/app/api/v1/endpoints/health.py
@@ -728,7 +728,53 @@ async def get_audit_chain_health(
.all()
)
replay = [dict(r) for r in reversed(rows)]
- breaks = verify_chain_breaks(replay)
+
+ # What the first replayed row should name as its predecessor.
+ #
+ # The replay can only verify the links *between the rows it fetched*,
+ # and the link from the first of them to whatever came before is
+ # unverifiable unless that predecessor is genesis. There are two ways
+ # it is not genesis, and both used to be reported as tampering:
+ #
+ # * **The window.** `window` is 500, so any tenant with more chained
+ # rows than that starts the replay mid-chain and the first row names
+ # a predecessor nobody fetched. Every such tenant reported
+ # `prev_hash mismatch` at index 0 — a healthy chain, read as broken,
+ # which is the one answer this surface must never give by accident.
+ # * **The retention window.** Depth 8.1 gave `audit_days` a purge
+ # (migration `111`), so the oldest retained row can legitimately
+ # name a predecessor that was lawfully removed. The checkpoint
+ # written in the same transaction records which hash that was.
+ #
+ # Anchoring on the checkpoint is a *verification*: the survivor must
+ # name the hash the truncation recorded. Anchoring on the row's own
+ # `prev_hash` is an *acknowledgement* that this window cannot check
+ # that link, and `replay_anchor` below says which of the two happened
+ # rather than leaving a reader to infer it.
+ truncation = (
+ (
+ await db.execute(
+ text(
+ "SELECT first_retained_prev_hash FROM aisoc_audit_checkpoints WHERE tenant_id = :tid ORDER BY created_at DESC LIMIT 1"
+ ),
+ {"tid": user.tenant_id},
+ )
+ )
+ .mappings()
+ .one_or_none()
+ )
+ chained_total = int(counts["chained"] or 0)
+ if replay and chained_total > len(replay):
+ anchor_prev_hash = replay[0].get("prev_hash")
+ replay_anchor = "window"
+ elif truncation is not None:
+ anchor_prev_hash = truncation["first_retained_prev_hash"]
+ replay_anchor = "retention_checkpoint"
+ else:
+ anchor_prev_hash = None
+ replay_anchor = "genesis"
+
+ breaks = verify_chain_breaks(replay, anchor_prev_hash=anchor_prev_hash)
tail_break = verify_chain_tail(replay, dict(head) if head else None)
intact = not breaks and tail_break is None
bad_index = int(breaks[0]["index"]) if breaks else None
@@ -753,6 +799,11 @@ async def get_audit_chain_health(
"oldest_unchained_at": counts["oldest_unchained"].isoformat() if counts["oldest_unchained"] else None,
"newest_unchained_at": counts["newest_unchained"].isoformat() if counts["newest_unchained"] else None,
"replay_window": len(replay),
+ # `genesis`, `retention_checkpoint`, or `window`. Only the first
+ # two mean the oldest link in this replay was checked; `window`
+ # means the chain is longer than the replay and the link below it
+ # was taken on trust.
+ "replay_anchor": replay_anchor,
# Deleting the newest rows leaves every remaining link valid; only the
# stored head shows the chain used to be longer.
"tail_intact": tail_break is None,
diff --git a/services/api/app/services/audit_hash.py b/services/api/app/services/audit_hash.py
index d7c50819f..f5efd303a 100644
--- a/services/api/app/services/audit_hash.py
+++ b/services/api/app/services/audit_hash.py
@@ -111,9 +111,27 @@ def compute_entry_hash(
return h.hexdigest()
-def verify_chain_breaks(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+def verify_chain_breaks(
+ rows: list[dict[str, Any]],
+ *,
+ anchor_prev_hash: str | None = None,
+) -> list[dict[str, Any]]:
"""Replay ``rows`` (oldest → newest, same tenant) and return **every** break.
+ ``anchor_prev_hash`` is the predecessor the first row should name, and
+ it exists for one reason: depth 8.1 gave the audit log an enforced
+ retention window, and a truncated chain no longer starts at the
+ genesis row. Without the anchor the oldest survivor names a
+ predecessor that has been lawfully removed, the replay reports
+ ``prev_hash mismatch`` at index 0, and a perfectly intact chain reads
+ as tampered with — which is the worst answer a control like this can
+ give, because it is indistinguishable from the real thing.
+
+ The anchor comes from ``aisoc_audit_checkpoints.first_retained_prev_hash``,
+ written by ``audit_retention_truncate`` in the same transaction as the
+ delete. It defaults to ``None``, which is the genesis case and what
+ every caller that has not been truncated still gets.
+
:func:`verify_chain` stops at the first violation, which is the right
answer to "is this chain intact?" and the wrong one for an auditor, who
needs to know whether they are looking at one historical fork or at
@@ -131,7 +149,7 @@ def verify_chain_breaks(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
broken too. One forked append otherwise reads as thousands of failures.
"""
breaks: list[dict[str, Any]] = []
- prev_hash: str | None = None
+ prev_hash: str | None = anchor_prev_hash
started = False
def _note(idx: int, row: dict[str, Any], reason: str) -> None:
@@ -240,7 +258,11 @@ def verify_chain_tail(rows: list[dict[str, Any]], head: dict[str, Any] | None) -
}
-def verify_chain(rows: list[dict[str, Any]]) -> tuple[bool, int | None, str | None]:
+def verify_chain(
+ rows: list[dict[str, Any]],
+ *,
+ anchor_prev_hash: str | None = None,
+) -> tuple[bool, int | None, str | None]:
"""Replay ``rows`` (oldest → newest, same tenant) and verify the chain.
Each row must expose the same field names as
@@ -255,7 +277,7 @@ def verify_chain(rows: list[dict[str, Any]]) -> tuple[bool, int | None, str | No
Use :func:`verify_chain_breaks` when the full picture matters; this is
the same replay reporting only its first finding.
"""
- breaks = verify_chain_breaks(rows)
+ breaks = verify_chain_breaks(rows, anchor_prev_hash=anchor_prev_hash)
if not breaks:
return True, None, None
first = breaks[0]
diff --git a/services/api/app/workers/retention_purge.py b/services/api/app/workers/retention_purge.py
index 4e8471866..63e35d476 100644
--- a/services/api/app/workers/retention_purge.py
+++ b/services/api/app/workers/retention_purge.py
@@ -24,13 +24,22 @@ same code path proves the predicate is right.
exist to pre-fill a form, and treating them as an instruction would delete data
nobody asked to delete.
-**Audit is excluded.** ``audit_days`` is storable but not purged here. The
-audit log is an append-only hash chain (migration 043) whose entire value is
-that no row was deleted, reordered, or rewritten; a retention purge would break
-the chain and make every subsequent verification fail. Enforcing an audit
-retention window needs chain-aware truncation with a re-anchored checkpoint,
-which is a different piece of work. Saying so in one place beats a purge that
-silently errors every tick.
+**Audit is included, through one function and nothing else.** ``audit_days``
+was storable and purged by nothing, because the audit log is an append-only
+hash chain (migration 043) whose entire value is that no row was deleted,
+reordered or rewritten — remove the oldest rows and a perfectly intact chain
+replays as `prev_hash mismatch` at index 0, which is the worst outcome for a
+control whose only job is to tell tampering from innocence.
+
+Depth 8.1 built the chain-aware truncation that was missing. Migration `111`
+adds `audit_retention_truncate`, a `SECURITY DEFINER` function that records a
+checkpoint — the cutoff, the count, the hash of the last row removed and the
+predecessor the oldest survivor names — and only then deletes, in the same
+transaction. Verification re-anchors on that checkpoint and the remainder
+replays clean, while the removal becomes a fact an auditor can read rather
+than an absence they have to infer. The application role has **no DELETE
+privilege on `audit_log`**, so that function is not merely the intended path,
+it is the only one.
"""
from __future__ import annotations
@@ -70,8 +79,10 @@ class TenantPurgeResult:
tenant_id: uuid.UUID
raw_events_days: int
alerts_days: int
+ audit_days: int = 0
lake_rows: int = 0
alert_rows: int = 0
+ audit_rows: int = 0
#: Rows that were past the window and were not deleted because a legal
#: hold covers this tenant. Counted rather than merely skipped: a hold
#: that silently stops a purge and a purge that silently had nothing to
@@ -79,6 +90,7 @@ class TenantPurgeResult:
#: work" can only be answered by a number.
held_lake_rows: int = 0
held_alert_rows: int = 0
+ held_audit_rows: int = 0
#: The matters the hold was placed under, so the operator reading the
#: log knows which one to go and release.
hold_refs: list[str] = field(default_factory=list)
@@ -103,6 +115,10 @@ class PurgeRun:
def alert_rows(self) -> int:
return sum(t.alert_rows for t in self.tenants)
+ @property
+ def audit_rows(self) -> int:
+ return sum(t.audit_rows for t in self.tenants)
+
@property
def held_lake_rows(self) -> int:
return sum(t.held_lake_rows for t in self.tenants)
@@ -111,6 +127,10 @@ class PurgeRun:
def held_alert_rows(self) -> int:
return sum(t.held_alert_rows for t in self.tenants)
+ @property
+ def held_audit_rows(self) -> int:
+ return sum(t.held_audit_rows for t in self.tenants)
+
@property
def tenants_held(self) -> int:
return sum(1 for t in self.tenants if t.held)
@@ -232,6 +252,46 @@ async def _purge_lake(tenant_id: uuid.UUID, days: int, *, dry_run: bool) -> int:
return doomed
+async def _purge_audit(db: AsyncSession, tenant_id: uuid.UUID, days: int, *, dry_run: bool) -> int:
+ """Truncate (or count) audit rows past the window for one tenant.
+
+ The delete goes through `audit_retention_truncate`, which is not a
+ style choice: the application role has no DELETE privilege on
+ `audit_log`, so a statement here would be refused by Postgres. That
+ function records the re-anchoring checkpoint and removes the rows in
+ one transaction, so there is no ordering in which the rows are gone
+ and the record of them is not.
+
+ A window of zero is "no audit window configured" and purges nothing.
+ `resolve_policy` fills the other two from `DEFAULT_RETENTION`; this one
+ is deliberately not defaulted, because a default that silently starts
+ deleting an audit trail is the kind of instruction nobody gave.
+ """
+ if days <= 0:
+ return 0
+
+ doomed = int(
+ (
+ await db.execute(
+ text("SELECT count(*) FROM audit_log WHERE tenant_id = :tenant_id AND created_at < now() - make_interval(days => :days)"),
+ {"tenant_id": str(tenant_id), "days": days},
+ )
+ ).scalar_one()
+ )
+ if doomed == 0 or dry_run:
+ return doomed
+
+ removed = (
+ await db.execute(
+ text("SELECT audit_retention_truncate(CAST(:tenant_id AS uuid), now() - make_interval(days => :days), :days)"),
+ {"tenant_id": str(tenant_id), "days": days},
+ )
+ ).scalar_one()
+ # The function's own count, not the one counted a moment ago. A row
+ # written between the two is retained rather than reported as removed.
+ return int(removed or 0)
+
+
async def _purge_alerts(db: AsyncSession, tenant_id: uuid.UUID, days: int, *, dry_run: bool) -> int:
"""Delete (or count) alerts past the window for one tenant.
@@ -277,6 +337,13 @@ async def run_once(
tenant_id=tenant_id,
raw_events_days=policy.raw_events_days,
alerts_days=policy.alerts_days,
+ # Read from the stored policy rather than from
+ # `resolve_policy`, which fills an unset window from
+ # `DEFAULT_RETENTION`. Those defaults exist to pre-fill a
+ # form; treating one as an instruction to start truncating
+ # an audit trail would delete evidence nobody asked to
+ # delete.
+ audit_days=int(config.get("audit_days") or 0),
)
try:
@@ -324,13 +391,32 @@ async def run_once(
type(exc).__name__,
)
+ try:
+ purged = await _purge_audit(db, tenant_id, result.audit_days, dry_run=dry_run or held)
+ if held:
+ result.held_audit_rows = purged
+ else:
+ result.audit_rows = purged
+ except Exception as exc:
+ # Same reasoning as the two above: one failing store must
+ # not read as "nothing needed deleting". It matters more
+ # here, because the thing that did not happen is an
+ # erasure somebody may have been told was performed.
+ result.errors.append(f"audit: {type(exc).__name__}")
+ logger.warning(
+ "retention_purge.audit_failed tenant=%s err=%s",
+ tenant_id,
+ type(exc).__name__,
+ )
+
if held:
logger.info(
- "retention_purge.held tenant=%s matters=%s lake_rows=%d alert_rows=%d",
+ "retention_purge.held tenant=%s matters=%s lake_rows=%d alert_rows=%d audit_rows=%d",
tenant_id,
",".join(result.hold_refs),
result.held_lake_rows,
result.held_alert_rows,
+ result.held_audit_rows,
)
run.tenants.append(result)