Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(8.1): migration 111 — chain-aware audit truncation with a re-anchored checkpoint

Beenu Aroraf2839ade7ed6
diff --git a/services/api/migrations/111_audit_retention.sql b/services/api/migrations/111_audit_retention.sql
new file mode 100644
index 000000000..15bf4568c
--- /dev/null
+++ b/services/api/migrations/111_audit_retention.sql
@@ -0,0 +1,222 @@
+-- An enforced retention window for the audit log, without breaking the chain.
+--
+-- Depth 8.1 (parity 4.4's remainder).
+--
+-- `retention_policies.audit_days` has been storable since the table existed
+-- and purged by nothing. `app/workers/retention_purge.py` says so in its own
+-- module docstring, and says why:
+--
+-- > The audit log is an append-only hash chain (migration 043) whose entire
+-- > value is that no row was deleted, reordered, or rewritten; a retention
+-- > purge would break the chain and make every subsequent verification
+-- > fail. Enforcing an audit retention window needs chain-aware truncation
+-- > with a re-anchored checkpoint, which is a different piece of work.
+--
+-- This is that piece of work. Saying it in one place beat a purge that
+-- silently errored every tick, but it left an operator who configured
+-- `audit_days` with a setting the product ignored — and a regulator's
+-- erasure request with no mechanism behind it.
+--
+-- ── Why a checkpoint rather than a plain DELETE ─────────────────────────────
+--
+-- `verify_chain_breaks` replays a tenant's rows oldest-first starting from
+-- `prev_hash = NULL`. Remove the oldest rows and the new first row names a
+-- predecessor that is gone, so the replay reports `prev_hash mismatch` at
+-- index 0 — a chain that is perfectly intact reads as tampered with, which
+-- is the worst possible outcome for a control whose only job is to tell
+-- those two apart.
+--
+-- A checkpoint re-anchors it. It records the cutoff that was applied, how
+-- many rows went, the hash of the last one removed, and the hash the oldest
+-- survivor names as its predecessor. Verification then starts from that
+-- anchor and the surviving chain replays clean — and the removal itself is
+-- now a fact an auditor can read, which a plain DELETE would not be.
+
+CREATE TABLE IF NOT EXISTS aisoc_audit_checkpoints (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
+
+ -- The cutoff applied, and the policy that produced it. Both, because
+ -- "everything before 1 March went" and "a 90-day window was in force"
+ -- answer different questions and an auditor asks the second one.
+ truncated_before TIMESTAMPTZ NOT NULL,
+ window_days INTEGER NOT NULL,
+
+ rows_removed BIGINT NOT NULL,
+
+ -- The last link of what was removed, and the first link of what stayed.
+ -- `first_retained_prev_hash` is what makes the remainder verifiable: it
+ -- is the anchor the replay starts from, and it must equal
+ -- `last_removed_hash` on an untampered chain. Storing both means the
+ -- checkpoint can be checked against itself.
+ last_removed_hash TEXT,
+ first_retained_prev_hash TEXT,
+ first_retained_hash TEXT,
+
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS aisoc_audit_checkpoints_tenant_idx
+ ON aisoc_audit_checkpoints (tenant_id, created_at DESC);
+
+-- ── The checkpoint is immutable too ─────────────────────────────────────────
+--
+-- A record of what was deleted, which can itself be edited or deleted, is
+-- not a record. This is the same trigger shape migration `004` uses on
+-- `audit_log` and `101` repaired: DELETE is refused first and
+-- unconditionally, because on DELETE there is no NEW and `RETURN NEW`
+-- returns NULL, which in a BEFORE trigger silently *cancels* the statement
+-- and reports "DELETE 0" as success.
+CREATE OR REPLACE FUNCTION public.audit_checkpoint_immutable()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $fn$
+BEGIN
+ RAISE EXCEPTION 'aisoc_audit_checkpoints rows are immutable (attempted %)', TG_OP;
+END;
+$fn$;
+
+DROP TRIGGER IF EXISTS audit_checkpoint_immutable ON aisoc_audit_checkpoints;
+CREATE TRIGGER audit_checkpoint_immutable
+ BEFORE UPDATE OR DELETE ON aisoc_audit_checkpoints
+ FOR EACH ROW EXECUTE FUNCTION public.audit_checkpoint_immutable();
+
+ALTER TABLE aisoc_audit_checkpoints ENABLE ROW LEVEL SECURITY;
+ALTER TABLE aisoc_audit_checkpoints FORCE ROW LEVEL SECURITY;
+DROP POLICY IF EXISTS tenant_isolation ON aisoc_audit_checkpoints;
+CREATE POLICY tenant_isolation ON aisoc_audit_checkpoints
+ USING (tenant_id = current_tenant_id() OR current_tenant_id() IS NULL)
+ WITH CHECK (tenant_id = current_tenant_id() OR current_tenant_id() IS NULL);
+
+-- ── The one sanctioned way to remove an audit row ───────────────────────────
+--
+-- `SECURITY DEFINER`, so it runs as the table owner, and the application
+-- role has **no DELETE privilege on `audit_log` at all** after this
+-- migration. That is the gate that cannot be talked around: the
+-- transaction-local setting below is a second one, but a custom GUC can be
+-- set by anybody, so on its own it would be a comment rather than a
+-- control.
+--
+-- `SET search_path` is pinned on the function. Without it a caller who
+-- controls `search_path` can put their own `audit_log` in front of the real
+-- one and have a definer-rights function operate on it.
+CREATE OR REPLACE FUNCTION public.audit_retention_truncate(
+ p_tenant_id UUID,
+ p_cutoff TIMESTAMPTZ,
+ p_window_days INTEGER
+)
+RETURNS BIGINT
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public, pg_temp
+AS $fn$
+DECLARE
+ v_rows BIGINT;
+ v_last_removed_hash TEXT;
+ v_first_prev_hash TEXT;
+ v_first_hash TEXT;
+BEGIN
+ IF p_tenant_id IS NULL OR p_cutoff IS NULL THEN
+ RAISE EXCEPTION 'audit_retention_truncate needs a tenant and a cutoff';
+ END IF;
+
+ SELECT count(*) INTO v_rows
+ FROM audit_log
+ WHERE tenant_id = p_tenant_id AND created_at < p_cutoff;
+
+ IF v_rows = 0 THEN
+ -- No checkpoint for a no-op. A table of "removed 0 rows" entries
+ -- every tick buries the ones that mean something.
+ RETURN 0;
+ END IF;
+
+ SELECT entry_hash INTO v_last_removed_hash
+ FROM audit_log
+ WHERE tenant_id = p_tenant_id AND created_at < p_cutoff
+ ORDER BY created_at DESC, chain_index DESC
+ LIMIT 1;
+
+ SELECT prev_hash, entry_hash INTO v_first_prev_hash, v_first_hash
+ FROM audit_log
+ WHERE tenant_id = p_tenant_id AND created_at >= p_cutoff
+ ORDER BY created_at ASC, chain_index ASC
+ LIMIT 1;
+
+ -- Recorded before the delete and in the same transaction, so there is
+ -- no ordering in which rows are gone and the record of them is not.
+ INSERT INTO aisoc_audit_checkpoints (
+ tenant_id, truncated_before, window_days, rows_removed,
+ last_removed_hash, first_retained_prev_hash, first_retained_hash
+ )
+ VALUES (
+ p_tenant_id, p_cutoff, p_window_days, v_rows,
+ v_last_removed_hash, v_first_prev_hash, v_first_hash
+ );
+
+ -- The trigger on `audit_log` refuses every DELETE unless this is set,
+ -- and it is `SET LOCAL`, so it lapses at the end of this transaction
+ -- rather than leaving the door open for the rest of the session.
+ PERFORM set_config('app.audit_retention_purge', 'on', TRUE);
+ DELETE FROM audit_log WHERE tenant_id = p_tenant_id AND created_at < p_cutoff;
+ PERFORM set_config('app.audit_retention_purge', 'off', TRUE);
+
+ RETURN v_rows;
+END;
+$fn$;
+
+-- ── Teach the immutability trigger about the one exception ──────────────────
+--
+-- Byte-identical to `101_audit_immutable_nullify.sql` apart from the DELETE
+-- arm. Restated in full rather than patched, because a trigger function
+-- assembled from two migrations is one nobody can read in one place.
+CREATE OR REPLACE FUNCTION public.audit_log_immutable()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $fn$
+DECLARE
+ key text;
+ old_row jsonb := to_jsonb(OLD);
+ new_row jsonb := to_jsonb(NEW);
+BEGIN
+ IF TG_OP = 'DELETE' THEN
+ -- The retention window, and nothing else. `audit_retention_truncate`
+ -- is the only thing that sets this, it is `SET LOCAL` so it cannot
+ -- outlive the statement that needed it, and the application role has
+ -- no DELETE privilege here anyway — so a caller who sets the setting
+ -- by hand is refused by Postgres before this trigger is reached.
+ IF coalesce(current_setting('app.audit_retention_purge', TRUE), 'off') = 'on' THEN
+ RETURN OLD;
+ END IF;
+ RAISE EXCEPTION 'audit_log rows are immutable (attempted %)', TG_OP;
+ END IF;
+
+ FOR key IN SELECT jsonb_object_keys(new_row) LOOP
+ IF old_row -> key IS DISTINCT FROM new_row -> key THEN
+ -- The ONE legal mutation, pinned to its column: only actor_id
+ -- carries ON DELETE SET NULL.
+ IF key = 'actor_id'
+ AND jsonb_typeof(old_row -> key) <> 'null'
+ AND jsonb_typeof(new_row -> key) = 'null'
+ THEN
+ CONTINUE;
+ END IF;
+ RAISE EXCEPTION 'audit_log rows are immutable (attempted UPDATE of %)', key;
+ END IF;
+ END LOOP;
+ RETURN NEW;
+END;
+$fn$;
+
+DO $$
+BEGIN
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'aisoc_app') THEN
+ GRANT SELECT, INSERT ON aisoc_audit_checkpoints TO aisoc_app;
+ GRANT EXECUTE ON FUNCTION public.audit_retention_truncate(UUID, TIMESTAMPTZ, INTEGER) TO aisoc_app;
+
+ -- The privilege half of the control. The trigger has always
+ -- refused a direct DELETE; this makes the refusal happen one layer
+ -- lower, where no setting a caller can reach has any say.
+ REVOKE DELETE ON audit_log FROM aisoc_app;
+ END IF;
+END
+$$;