Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(approvals): expire approvals nobody answered (9b) (#1038)
Beenu Arorad2b8ee3492b3
diff --git a/scripts/check_tenant_query_predicates.py b/scripts/check_tenant_query_predicates.pyindex bfef2186c..fc678f2c1 100644--- a/scripts/check_tenant_query_predicates.py+++ b/scripts/check_tenant_query_predicates.py@@ -271,6 +271,11 @@ RATCHET: dict[str, str] = {"writes back to the connector row the sweep above already selected, by primary key"),"services/api/app/workers/oauth_refresh.py::_record_failure::Connector": "records the failure of that same swept row, by primary key",+ "services/api/app/workers/approval_expiry.py::run_once::agent_approvals": (+ "counts pending approvals that carry no expires_at, across every tenant, for one operator log line. "+ "The sweep it sits beside is tenant-wide by nature and calls assert_cross_tenant_session first, so a "+ "tenant predicate here would report one tenant's gap as the whole estate's"+ ),# -- the row is addressed by an opaque capability, and yields the tenant --"services/honeytokens/app/api/routes.py::webhook_trigger::Honeytoken": ("canary callback: the token id is the capability and the row supplies the tenant it belongs to"@@ -325,7 +330,15 @@ RATCHET: dict[str, str] = {#: the identical query as raw SQL, which would have moved it out of this#: gate's view without changing what it reads, and that is worse than an entry#: with a reason beside it.-MAX_RATCHET = 34+#:+#: 34 -> 35 (deferral 9b): `approval_expiry.run_once`'s no-deadline count. The+#: worker expires stale approvals across every tenant and asserts an unbound+#: session before it does, so the aggregate beside it is tenant-wide for the+#: same reason. The count feeds one log line and returns no tenant's rows. It+#: was moved out of module scope to earn this entry: keyed on `<module>` the+#: waiver would have matched any later unscoped query in the same file, which+#: is an allowlist wearing a ratchet's name.+MAX_RATCHET = 35# ---------------------------------------------------------------------------diff --git a/services/api/app/core/config.py b/services/api/app/core/config.pyindex ad3b80000..6e1f29395 100644--- a/services/api/app/core/config.py+++ b/services/api/app/core/config.py@@ -352,6 +352,17 @@ class Settings(BaseSettings):)RETENTION_WORKER_INTERVAL_SECONDS: int = 21600 # 6h+ # Approval-SLA expiry (deferral 9b). On by default, unlike the retention+ # purge: that one deletes data, this one marks an already-abandoned+ # request as abandoned and dispatches nothing. Leaving it off would keep+ # the state it closes, where a pending containment waits forever and+ # nobody can tell it from one still under consideration.+ APPROVAL_EXPIRY_ENABLED: bool = Field(+ default=True,+ validation_alias=AliasChoices("APPROVAL_EXPIRY_ENABLED", "AISOC_APPROVAL_EXPIRY_ENABLED"),+ )+ APPROVAL_EXPIRY_INTERVAL_SECONDS: int = 300 # 5m+# Shadow-reconciliation sweep (gap-closure Phase 2.1, D15). Polls each# measuring tenant's own SIEM for the closures their analysts made there,# so a tenant whose queue lives in Splunk ES still accumulates the trackdiff --git a/services/api/app/main.py b/services/api/app/main.pyindex c00f13bfd..53a28559f 100644--- a/services/api/app/main.py+++ b/services/api/app/main.py@@ -36,6 +36,7 @@ from app.models import Basefrom app.services.plugin_manager import get_plugin_managerfrom app.services.scim.resources import SCIM_CONTENT_TYPEfrom app.services.scim.resources import error_response as scim_error_response+from app.workers.approval_expiry import run_forever as run_approval_expiryfrom app.workers.hunt_scheduler import run_forever as run_hunt_schedulerfrom app.workers.oauth_refresh import run_forever as run_oauth_refreshfrom app.workers.retention_purge import run_forever as run_retention_purge@@ -62,6 +63,10 @@ _HUNT_SCHEDULER_LOCK_TTL_SECONDS = 300# ClickHouse mutation with mutations_sync=1 blocks until it lands. 30m keeps# a second replica from starting a concurrent sweep mid-delete._RETENTION_PURGE_LOCK_TTL_SECONDS = 1800+# Shorter than the others: the sweep is a single bounded UPDATE, so a lock+# held for half an hour after a crashed replica would leave approvals+# un-expired for far longer than the work takes.+_APPROVAL_EXPIRY_LOCK_TTL_SECONDS = 300# A shadow-reconciliation pass makes up to SHADOW_RECONCILE_MAX_CONNECTORS_PER_TICK# vendor searches, each bounded at 120s. 30m covers a slow pass without letting# a second replica start one on top of it, which would double a customer's@@ -475,6 +480,36 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:"reconciled. Set SHADOW_RECONCILE_ENABLED=true to poll them")+ # Approval-SLA expiry (9b). `agent_approvals` has carried an `expires_at`+ # and an `expired` status since migration 009; nothing wrote the status+ # and nothing swept the column, so a console-raised approval nobody+ # answered waited forever instead of timing out to its declared safe+ # default. Expiring dispatches nothing -- `/decide` still accepts an+ # expired row -- so this removes the pretence that the request is live+ # without spending the decision on the operator's behalf.+ approval_expiry_task: asyncio.Task | None = None+ if settings.APPROVAL_EXPIRY_ENABLED:+ try:+ approval_expiry_task = asyncio.create_task(+ _run_guarded_scheduler_worker(+ job_name="approval_expiry",+ ttl_seconds=_APPROVAL_EXPIRY_LOCK_TTL_SECONDS,+ worker=run_approval_expiry,+ ),+ name="approval_expiry_worker",+ )+ logger.info("approval_expiry worker started")+ except Exception as exc:+ logger.warning("approval_expiry worker failed to start", error=str(exc))+ else:+ # Said out loud: with this off, a pending approval never times out,+ # and an operator reading the queue cannot tell an abandoned request+ # from one still being considered.+ logger.info(+ "approval_expiry worker disabled; approvals with an expires_at will never move to 'expired'. "+ "Set APPROVAL_EXPIRY_ENABLED=true to sweep them"+ )+# Gap-closure Phase 8.1. The consumer for the `NEW_IOC` events# `services/threatintel` has always emitted and nothing has ever read.#@@ -534,6 +569,15 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:except Exception as exc:logger.warning("hunt_scheduler worker shutdown error", error=type(exc).__name__)+ if approval_expiry_task is not None and not approval_expiry_task.done():+ approval_expiry_task.cancel()+ try:+ await approval_expiry_task+ except asyncio.CancelledError:+ logger.debug("approval_expiry worker cancelled during shutdown")+ except Exception as exc:+ logger.warning("approval_expiry worker shutdown error", error=type(exc).__name__)+if retention_purge_task is not None and not retention_purge_task.done():retention_purge_task.cancel()try:diff --git a/services/api/app/workers/approval_expiry.py b/services/api/app/workers/approval_expiry.pynew file mode 100644index 000000000..7b3574a0c--- /dev/null+++ b/services/api/app/workers/approval_expiry.py@@ -0,0 +1,174 @@+"""Expire approvals nobody answered (deferral 9b).++Phase 9's live-router half landed: a decision on `/approvals/{id}/decide`+carries through to `services/actions` and the row records whether it+executed. The durable timer table landed too, in `062_approval_timers.sql`.++What was left is every approval the bot never sees. The expiry lives in+`services/slack-bot`, a `chatops`-profile service, and is armed per ChatOps+approval. The API's own `agent_approvals` row carries an `expires_at` and its+status vocabulary includes `expired` — and **nothing wrote that status and no+worker swept the column**. An approval raised in the console and never+answered waited forever rather than timing out to its declared safe default.++Forever is the wrong default for a pending containment. It is also the+*invisible* wrong default: the row stays `pending`, the queue keeps showing+it, and nobody can tell a request still being considered from one abandoned+three weeks ago.++The safe default, and where it comes from+-----------------------------------------+`rejected`. Not invented here — it is what `services/slack-bot` has always+used (`SafeDefault = Literal["rejected", "approved"]` defaulting to+`"rejected"`, and `safe_default TEXT NOT NULL DEFAULT 'rejected'` in+migration 062), stated there as "so a forgotten approval can never+accidentally execute". This worker expires to the same default the other half+of the system already declares, because two halves timing out two different+ways would be worse than either.++Expiring is not deciding+------------------------+An expired approval is marked `expired`, and **nothing is dispatched**. The+`/decide` endpoint deliberately still accepts an `expired` row — its guard is+`status not in {"pending", "expired"}` — so a human who returns to a timed-out+request can still act on it. Timing out removes the pretence that the request+is live; it does not spend the decision on the operator's behalf.++Only rows that opted in+-----------------------+`expires_at` is nullable and only `POST /approvals` ever writes it, from a+caller-supplied value. Rows without one are left alone rather than given a+default window here: inventing a deadline for an approval whose creator did+not set one would start expiring containments on a schedule nobody chose.+That gap is reported in the run summary rather than silently closed, so the+number of never-expiring approvals is visible instead of assumed to be zero.+"""++from __future__ import annotations++import asyncio+import logging+from dataclasses import dataclass, field+from datetime import UTC, datetime++from sqlalchemy import text+from sqlalchemy.ext.asyncio import AsyncSession++from app.core.config import settings+from app.db.cross_tenant import assert_cross_tenant_session+from app.db.database import AsyncSessionLocal+from app.workers._tick_failures import TickFailures++logger = logging.getLogger("aisoc.approval_expiry")++#: The safe default, mirrored from `services/slack-bot` and migration 062.+#: A forgotten approval must never resolve to something that executes.+SAFE_DEFAULT = "rejected"++#: Expiring writes `expired`, which is in the status vocabulary the migration+#: comments and `approvals.py::_VALID_STATUSES` both declare, and which+#: nothing wrote before this worker.+EXPIRED_STATUS = "expired"++_SWEEP_SQL = text(+ """+ UPDATE agent_approvals+ SET status = :expired, decision_comment = COALESCE(decision_comment, :note)+ WHERE status = 'pending'+ AND expires_at IS NOT NULL+ AND expires_at < now()+ RETURNING id, tenant_id, risk_level+ """+)++_NOTE = (+ "Expired automatically: the approval window elapsed with no decision. "+ f"Safe default is '{SAFE_DEFAULT}' — nothing was dispatched, and this request can still be decided."+)+++@dataclass+class ExpirySweep:+ """What one sweep found."""++ started_at: datetime+ expired: int = 0+ #: Pending approvals that carry no deadline and therefore cannot expire.+ #: Reported so the gap is visible rather than read as zero.+ pending_without_deadline: int = 0+ by_risk: dict[str, int] = field(default_factory=dict)+++async def run_once(*, db: AsyncSession | None = None) -> ExpirySweep:+ """Expire every pending approval whose window has elapsed, across all tenants."""+ own_session = db is None+ if db is None:+ db = AsyncSessionLocal()+ sweep = ExpirySweep(started_at=datetime.now(UTC))++ try:+ # Cross-tenant by nature. The precondition that makes the policy's+ # `OR current_tenant_id() IS NULL` arm apply is asserted rather than+ # assumed — a session bound to one tenant would expire that tenant's+ # approvals and report success for all of them.+ await assert_cross_tenant_session(db, "approval expiry sweep")++ rows = (await db.execute(_SWEEP_SQL, {"expired": EXPIRED_STATUS, "note": _NOTE})).mappings().all()+ sweep.expired = len(rows)+ for row in rows:+ level = str(row["risk_level"] or "unknown")+ sweep.by_risk[level] = sweep.by_risk.get(level, 0) + 1++ # Counted, not swept. An approval with no deadline is one whose creator+ # did not set one, and a worker that invented deadlines would start+ # expiring containments on a schedule nobody chose. Deliberately+ # tenant-wide, like the sweep above, and on the ratchet as such: the+ # statement lives here rather than at module scope so the waiver names+ # this function and cannot be inherited by some later query.+ no_deadline_sql = text("SELECT count(*) FROM agent_approvals WHERE status = 'pending' AND expires_at IS NULL")+ sweep.pending_without_deadline = int((await db.execute(no_deadline_sql)).scalar_one())+ await db.commit()+ finally:+ if own_session:+ await db.close()++ if sweep.expired:+ # At warning, not info. An approval timing out means a human was+ # asked and did not answer, which is a fact about the rota.+ logger.warning(+ "approval_expiry expired=%d by_risk=%s safe_default=%s",+ sweep.expired,+ sweep.by_risk,+ SAFE_DEFAULT,+ )+ if sweep.pending_without_deadline:+ logger.info(+ "approval_expiry %d pending approval(s) carry no expires_at and can never time out; "+ "set one at POST /approvals to bring them under this sweep",+ sweep.pending_without_deadline,+ )+ return sweep+++async def run_forever() -> None:+ """Tick until cancelled. Owned by the API ``lifespan``, like the other workers."""+ interval = max(int(getattr(settings, "APPROVAL_EXPIRY_INTERVAL_SECONDS", 300)), 30)+ logger.info("approval_expiry started interval=%ds safe_default=%s", interval, SAFE_DEFAULT)+ failures = TickFailures("approval_expiry", logger)+ try:+ while True:+ try:+ await run_once()+ except asyncio.CancelledError:+ raise+ except Exception as exc: # pragma: no cover - defensive+ failures.record_failure(exc)+ else:+ failures.record_success()+ await asyncio.sleep(interval)+ except asyncio.CancelledError:+ logger.info("approval_expiry stopped")+ raise+++__all__ = ["EXPIRED_STATUS", "SAFE_DEFAULT", "ExpirySweep", "run_forever", "run_once"]diff --git a/services/api/tests/test_approval_expiry.py b/services/api/tests/test_approval_expiry.pynew file mode 100644index 000000000..567b0130b--- /dev/null+++ b/services/api/tests/test_approval_expiry.py@@ -0,0 +1,197 @@+"""Approvals nobody answered stop waiting forever (deferral 9b).++Four things must hold, and each closes a specific way this could go wrong:+only elapsed rows are touched, expiring dispatches nothing, the safe default+matches the one the other half of the system already declares, and an+approval with no deadline is *counted* rather than given one.+"""++from __future__ import annotations++import re+import uuid+from datetime import UTC, datetime+from pathlib import Path++import pytest+from app.workers.approval_expiry import EXPIRED_STATUS, SAFE_DEFAULT, run_once++REPO_ROOT = Path(__file__).resolve().parents[3]+++class _Result:+ def __init__(self, rows: list[dict] | None = None, scalar: int = 0) -> None:+ self._rows = rows or []+ self._scalar = scalar++ def mappings(self) -> _Result:+ return self++ def all(self) -> list[dict]:+ return self._rows++ def scalar_one(self) -> int:+ return self._scalar+++class _Session:+ """Answers the sweep's two statements and records what it was asked."""++ def __init__(self, expired_rows: list[dict] | None = None, no_deadline: int = 0) -> None:+ self.expired_rows = expired_rows or []+ self.no_deadline = no_deadline+ self.sql: list[str] = []+ self.params: list[dict] = []+ self.commits = 0++ async def execute(self, statement: object, params: dict | None = None) -> _Result:+ sql = str(statement)+ self.sql.append(sql)+ self.params.append(params or {})+ if "current_setting" in sql: # the cross-tenant precondition probe+ return _Result(scalar=0)+ if sql.strip().upper().startswith("UPDATE"):+ return _Result(rows=self.expired_rows)+ return _Result(scalar=self.no_deadline)++ async def scalar(self, _statement: object) -> str | None:+ # `assert_cross_tenant_session` reads the bound tenant; None means+ # nothing is bound, which is the precondition the sweep requires.+ return None++ async def commit(self) -> None:+ self.commits += 1++ async def close(self) -> None:+ return None+++def _row(risk: str = "high") -> dict:+ return {"id": uuid.uuid4(), "tenant_id": uuid.uuid4(), "risk_level": risk}+++class TestOnlyElapsedApprovalsAreExpired:+ @pytest.mark.asyncio+ async def test_the_predicate_names_pending_a_deadline_and_the_past(self) -> None:+ session = _Session(expired_rows=[_row()])++ await run_once(db=session)++ update = next(s for s in session.sql if s.strip().upper().startswith("UPDATE"))+ normalised = " ".join(update.split())+ assert "status = 'pending'" in normalised+ assert "expires_at IS NOT NULL" in normalised+ assert "expires_at < now()" in normalised++ @pytest.mark.asyncio+ async def test_an_empty_sweep_reports_zero_and_commits(self) -> None:+ session = _Session(expired_rows=[])++ sweep = await run_once(db=session)++ assert sweep.expired == 0+ assert session.commits == 1++ @pytest.mark.asyncio+ async def test_it_refuses_a_session_bound_to_one_tenant(self) -> None:+ """A bound session would expire one tenant's approvals and report+ success for every tenant — the cross-tenant defect the precondition+ helper exists to catch."""+ from app.db.cross_tenant import CrossTenantPreconditionError++ class _Bound(_Session):+ async def scalar(self, _statement: object) -> str | None:+ return str(uuid.uuid4())++ with pytest.raises(CrossTenantPreconditionError):+ await run_once(db=_Bound())+++class TestExpiringIsNotDeciding:+ @pytest.mark.asyncio+ async def test_the_sweep_writes_only_a_status_and_a_note(self) -> None:+ """Nothing is dispatched. An expired approval is a request that+ stopped pretending to be live, not a decision taken for the+ operator — and `/decide` still accepts an expired row."""+ session = _Session(expired_rows=[_row()])++ await run_once(db=session)++ update = " ".join(next(s for s in session.sql if s.strip().upper().startswith("UPDATE")).split())+ assert "SET status = :expired" in update+ for forbidden in ("decided_by_id", "decided_at", "action ="):+ assert forbidden not in update, f"the sweep must not write {forbidden}"++ def test_the_decide_endpoint_still_accepts_an_expired_row(self) -> None:+ source = (REPO_ROOT / "services" / "api" / "app" / "api" / "v1" / "endpoints" / "approvals.py").read_text(encoding="utf-8")+ assert 'row.status not in {"pending", "expired"}' in source, (+ "the guard changed; if `expired` no longer decides, this worker silently becomes a way to close approvals"+ )++ @pytest.mark.asyncio+ async def test_the_note_says_nothing_was_dispatched(self) -> None:+ session = _Session(expired_rows=[_row()])++ await run_once(db=session)++ note = next(p["note"] for p in session.params if "note" in p)+ assert "nothing was dispatched" in note+ assert "can still be decided" in note+++class TestTheSafeDefaultIsTheOneAlreadyDeclared:+ def test_it_matches_the_slack_bot_and_migration_062(self) -> None:+ """Two halves timing out two different ways is worse than either."""+ migration = (REPO_ROOT / "services" / "api" / "migrations" / "062_approval_timers.sql").read_text(encoding="utf-8")+ scheduler = (REPO_ROOT / "services" / "slack-bot" / "app" / "services" / "approval_timeout.py").read_text(encoding="utf-8")++ assert f"DEFAULT '{SAFE_DEFAULT}'" in migration+ assert re.search(rf'safe_default: SafeDefault = "{SAFE_DEFAULT}"', scheduler)++ def test_the_status_written_is_in_the_declared_vocabulary(self) -> None:+ approvals = (REPO_ROOT / "services" / "api" / "app" / "api" / "v1" / "endpoints" / "approvals.py").read_text(encoding="utf-8")+ assert f'"{EXPIRED_STATUS}"' in approvals, "the worker writes a status the API does not recognise"+++class TestAnApprovalWithNoDeadlineIsCountedNotInvented:+ @pytest.mark.asyncio+ async def test_they_are_reported_rather_than_swept(self) -> None:+ """Inventing a window for an approval whose creator did not set one+ would start expiring containments on a schedule nobody chose. The+ number is surfaced so the gap is visible instead of read as zero."""+ session = _Session(expired_rows=[], no_deadline=17)++ sweep = await run_once(db=session)++ assert sweep.pending_without_deadline == 17+ assert sweep.expired == 0++ @pytest.mark.asyncio+ async def test_the_breakdown_by_risk_travels_with_the_count(self) -> None:+ """'Three expired' is a number; 'two of them critical' is a finding."""+ session = _Session(expired_rows=[_row("critical"), _row("critical"), _row("low")])++ sweep = await run_once(db=session)++ assert sweep.expired == 3+ assert sweep.by_risk == {"critical": 2, "low": 1}+++class TestItIsWiredIntoTheApiLifespan:+ def test_the_worker_is_started_and_cancelled(self) -> None:+ """A worker nobody starts is the defect this deferral is about."""+ main = (REPO_ROOT / "services" / "api" / "app" / "main.py").read_text(encoding="utf-8")++ assert "run_approval_expiry" in main+ assert "approval_expiry_task.cancel()" in main+ assert "APPROVAL_EXPIRY_ENABLED" in main++ def test_a_disabled_worker_says_so_out_loud(self) -> None:+ main = (REPO_ROOT / "services" / "api" / "app" / "main.py").read_text(encoding="utf-8")+ assert "approval_expiry worker disabled" in main++ def test_the_sweep_started_at_is_timezone_aware(self) -> None:+ from app.workers.approval_expiry import ExpirySweep++ sweep = ExpirySweep(started_at=datetime.now(UTC))+ assert sweep.started_at.tzinfo is not None