Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(approvals): expire approvals nobody answered (9b) (#1038)

Beenu Arorad2b8ee3492b3
diff --git a/scripts/check_tenant_query_predicates.py b/scripts/check_tenant_query_predicates.py
index bfef2186c..fc678f2c1 100644
--- a/scripts/check_tenant_query_predicates.py
+++ b/scripts/check_tenant_query_predicates.py
@@ -271,6 +271,11 @@ RATCHET: dict[str, str] = {
"writes back to the connector row the sweep above already selected, by primary key"
),
"services/api/app/workers/oauth_refresh.py::_record_failure::Connector": "records the failure of that same swept row, by primary key",
+ "services/api/app/workers/approval_expiry.py::run_once::agent_approvals": (
+ "counts pending approvals that carry no expires_at, across every tenant, for one operator log line. "
+ "The sweep it sits beside is tenant-wide by nature and calls assert_cross_tenant_session first, so a "
+ "tenant predicate here would report one tenant's gap as the whole estate's"
+ ),
# -- the row is addressed by an opaque capability, and yields the tenant --
"services/honeytokens/app/api/routes.py::webhook_trigger::Honeytoken": (
"canary callback: the token id is the capability and the row supplies the tenant it belongs to"
@@ -325,7 +330,15 @@ RATCHET: dict[str, str] = {
#: the identical query as raw SQL, which would have moved it out of this
#: gate's view without changing what it reads, and that is worse than an entry
#: with a reason beside it.
-MAX_RATCHET = 34
+#:
+#: 34 -> 35 (deferral 9b): `approval_expiry.run_once`'s no-deadline count. The
+#: worker expires stale approvals across every tenant and asserts an unbound
+#: session before it does, so the aggregate beside it is tenant-wide for the
+#: same reason. The count feeds one log line and returns no tenant's rows. It
+#: was moved out of module scope to earn this entry: keyed on `<module>` the
+#: waiver would have matched any later unscoped query in the same file, which
+#: is an allowlist wearing a ratchet's name.
+MAX_RATCHET = 35
# ---------------------------------------------------------------------------
diff --git a/services/api/app/core/config.py b/services/api/app/core/config.py
index ad3b80000..6e1f29395 100644
--- a/services/api/app/core/config.py
+++ b/services/api/app/core/config.py
@@ -352,6 +352,17 @@ class Settings(BaseSettings):
)
RETENTION_WORKER_INTERVAL_SECONDS: int = 21600 # 6h
+ # Approval-SLA expiry (deferral 9b). On by default, unlike the retention
+ # purge: that one deletes data, this one marks an already-abandoned
+ # request as abandoned and dispatches nothing. Leaving it off would keep
+ # the state it closes, where a pending containment waits forever and
+ # nobody can tell it from one still under consideration.
+ APPROVAL_EXPIRY_ENABLED: bool = Field(
+ default=True,
+ validation_alias=AliasChoices("APPROVAL_EXPIRY_ENABLED", "AISOC_APPROVAL_EXPIRY_ENABLED"),
+ )
+ APPROVAL_EXPIRY_INTERVAL_SECONDS: int = 300 # 5m
+
# Shadow-reconciliation sweep (gap-closure Phase 2.1, D15). Polls each
# measuring tenant's own SIEM for the closures their analysts made there,
# so a tenant whose queue lives in Splunk ES still accumulates the track
diff --git a/services/api/app/main.py b/services/api/app/main.py
index c00f13bfd..53a28559f 100644
--- a/services/api/app/main.py
+++ b/services/api/app/main.py
@@ -36,6 +36,7 @@ from app.models import Base
from app.services.plugin_manager import get_plugin_manager
from app.services.scim.resources import SCIM_CONTENT_TYPE
from app.services.scim.resources import error_response as scim_error_response
+from app.workers.approval_expiry import run_forever as run_approval_expiry
from app.workers.hunt_scheduler import run_forever as run_hunt_scheduler
from app.workers.oauth_refresh import run_forever as run_oauth_refresh
from app.workers.retention_purge import run_forever as run_retention_purge
@@ -62,6 +63,10 @@ _HUNT_SCHEDULER_LOCK_TTL_SECONDS = 300
# ClickHouse mutation with mutations_sync=1 blocks until it lands. 30m keeps
# a second replica from starting a concurrent sweep mid-delete.
_RETENTION_PURGE_LOCK_TTL_SECONDS = 1800
+# Shorter than the others: the sweep is a single bounded UPDATE, so a lock
+# held for half an hour after a crashed replica would leave approvals
+# un-expired for far longer than the work takes.
+_APPROVAL_EXPIRY_LOCK_TTL_SECONDS = 300
# A shadow-reconciliation pass makes up to SHADOW_RECONCILE_MAX_CONNECTORS_PER_TICK
# vendor searches, each bounded at 120s. 30m covers a slow pass without letting
# a second replica start one on top of it, which would double a customer's
@@ -475,6 +480,36 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"reconciled. Set SHADOW_RECONCILE_ENABLED=true to poll them"
)
+ # Approval-SLA expiry (9b). `agent_approvals` has carried an `expires_at`
+ # and an `expired` status since migration 009; nothing wrote the status
+ # and nothing swept the column, so a console-raised approval nobody
+ # answered waited forever instead of timing out to its declared safe
+ # default. Expiring dispatches nothing -- `/decide` still accepts an
+ # expired row -- so this removes the pretence that the request is live
+ # without spending the decision on the operator's behalf.
+ approval_expiry_task: asyncio.Task | None = None
+ if settings.APPROVAL_EXPIRY_ENABLED:
+ try:
+ approval_expiry_task = asyncio.create_task(
+ _run_guarded_scheduler_worker(
+ job_name="approval_expiry",
+ ttl_seconds=_APPROVAL_EXPIRY_LOCK_TTL_SECONDS,
+ worker=run_approval_expiry,
+ ),
+ name="approval_expiry_worker",
+ )
+ logger.info("approval_expiry worker started")
+ except Exception as exc:
+ logger.warning("approval_expiry worker failed to start", error=str(exc))
+ else:
+ # Said out loud: with this off, a pending approval never times out,
+ # and an operator reading the queue cannot tell an abandoned request
+ # from one still being considered.
+ logger.info(
+ "approval_expiry worker disabled; approvals with an expires_at will never move to 'expired'. "
+ "Set APPROVAL_EXPIRY_ENABLED=true to sweep them"
+ )
+
# Gap-closure Phase 8.1. The consumer for the `NEW_IOC` events
# `services/threatintel` has always emitted and nothing has ever read.
#
@@ -534,6 +569,15 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception as exc:
logger.warning("hunt_scheduler worker shutdown error", error=type(exc).__name__)
+ if approval_expiry_task is not None and not approval_expiry_task.done():
+ approval_expiry_task.cancel()
+ try:
+ await approval_expiry_task
+ except asyncio.CancelledError:
+ logger.debug("approval_expiry worker cancelled during shutdown")
+ except Exception as exc:
+ logger.warning("approval_expiry worker shutdown error", error=type(exc).__name__)
+
if retention_purge_task is not None and not retention_purge_task.done():
retention_purge_task.cancel()
try:
diff --git a/services/api/app/workers/approval_expiry.py b/services/api/app/workers/approval_expiry.py
new file mode 100644
index 000000000..7b3574a0c
--- /dev/null
+++ b/services/api/app/workers/approval_expiry.py
@@ -0,0 +1,174 @@
+"""Expire approvals nobody answered (deferral 9b).
+
+Phase 9's live-router half landed: a decision on `/approvals/{id}/decide`
+carries through to `services/actions` and the row records whether it
+executed. The durable timer table landed too, in `062_approval_timers.sql`.
+
+What was left is every approval the bot never sees. The expiry lives in
+`services/slack-bot`, a `chatops`-profile service, and is armed per ChatOps
+approval. The API's own `agent_approvals` row carries an `expires_at` and its
+status vocabulary includes `expired` — and **nothing wrote that status and no
+worker swept the column**. An approval raised in the console and never
+answered waited forever rather than timing out to its declared safe default.
+
+Forever is the wrong default for a pending containment. It is also the
+*invisible* wrong default: the row stays `pending`, the queue keeps showing
+it, and nobody can tell a request still being considered from one abandoned
+three weeks ago.
+
+The safe default, and where it comes from
+-----------------------------------------
+`rejected`. Not invented here — it is what `services/slack-bot` has always
+used (`SafeDefault = Literal["rejected", "approved"]` defaulting to
+`"rejected"`, and `safe_default TEXT NOT NULL DEFAULT 'rejected'` in
+migration 062), stated there as "so a forgotten approval can never
+accidentally execute". This worker expires to the same default the other half
+of the system already declares, because two halves timing out two different
+ways would be worse than either.
+
+Expiring is not deciding
+------------------------
+An expired approval is marked `expired`, and **nothing is dispatched**. The
+`/decide` endpoint deliberately still accepts an `expired` row — its guard is
+`status not in {"pending", "expired"}` — so a human who returns to a timed-out
+request can still act on it. Timing out removes the pretence that the request
+is live; it does not spend the decision on the operator's behalf.
+
+Only rows that opted in
+-----------------------
+`expires_at` is nullable and only `POST /approvals` ever writes it, from a
+caller-supplied value. Rows without one are left alone rather than given a
+default window here: inventing a deadline for an approval whose creator did
+not set one would start expiring containments on a schedule nobody chose.
+That gap is reported in the run summary rather than silently closed, so the
+number of never-expiring approvals is visible instead of assumed to be zero.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import settings
+from app.db.cross_tenant import assert_cross_tenant_session
+from app.db.database import AsyncSessionLocal
+from app.workers._tick_failures import TickFailures
+
+logger = logging.getLogger("aisoc.approval_expiry")
+
+#: The safe default, mirrored from `services/slack-bot` and migration 062.
+#: A forgotten approval must never resolve to something that executes.
+SAFE_DEFAULT = "rejected"
+
+#: Expiring writes `expired`, which is in the status vocabulary the migration
+#: comments and `approvals.py::_VALID_STATUSES` both declare, and which
+#: nothing wrote before this worker.
+EXPIRED_STATUS = "expired"
+
+_SWEEP_SQL = text(
+ """
+ UPDATE agent_approvals
+ SET status = :expired, decision_comment = COALESCE(decision_comment, :note)
+ WHERE status = 'pending'
+ AND expires_at IS NOT NULL
+ AND expires_at < now()
+ RETURNING id, tenant_id, risk_level
+ """
+)
+
+_NOTE = (
+ "Expired automatically: the approval window elapsed with no decision. "
+ f"Safe default is '{SAFE_DEFAULT}' — nothing was dispatched, and this request can still be decided."
+)
+
+
+@dataclass
+class ExpirySweep:
+ """What one sweep found."""
+
+ started_at: datetime
+ expired: int = 0
+ #: Pending approvals that carry no deadline and therefore cannot expire.
+ #: Reported so the gap is visible rather than read as zero.
+ pending_without_deadline: int = 0
+ by_risk: dict[str, int] = field(default_factory=dict)
+
+
+async def run_once(*, db: AsyncSession | None = None) -> ExpirySweep:
+ """Expire every pending approval whose window has elapsed, across all tenants."""
+ own_session = db is None
+ if db is None:
+ db = AsyncSessionLocal()
+ sweep = ExpirySweep(started_at=datetime.now(UTC))
+
+ try:
+ # Cross-tenant by nature. The precondition that makes the policy's
+ # `OR current_tenant_id() IS NULL` arm apply is asserted rather than
+ # assumed — a session bound to one tenant would expire that tenant's
+ # approvals and report success for all of them.
+ await assert_cross_tenant_session(db, "approval expiry sweep")
+
+ rows = (await db.execute(_SWEEP_SQL, {"expired": EXPIRED_STATUS, "note": _NOTE})).mappings().all()
+ sweep.expired = len(rows)
+ for row in rows:
+ level = str(row["risk_level"] or "unknown")
+ sweep.by_risk[level] = sweep.by_risk.get(level, 0) + 1
+
+ # Counted, not swept. An approval with no deadline is one whose creator
+ # did not set one, and a worker that invented deadlines would start
+ # expiring containments on a schedule nobody chose. Deliberately
+ # tenant-wide, like the sweep above, and on the ratchet as such: the
+ # statement lives here rather than at module scope so the waiver names
+ # this function and cannot be inherited by some later query.
+ no_deadline_sql = text("SELECT count(*) FROM agent_approvals WHERE status = 'pending' AND expires_at IS NULL")
+ sweep.pending_without_deadline = int((await db.execute(no_deadline_sql)).scalar_one())
+ await db.commit()
+ finally:
+ if own_session:
+ await db.close()
+
+ if sweep.expired:
+ # At warning, not info. An approval timing out means a human was
+ # asked and did not answer, which is a fact about the rota.
+ logger.warning(
+ "approval_expiry expired=%d by_risk=%s safe_default=%s",
+ sweep.expired,
+ sweep.by_risk,
+ SAFE_DEFAULT,
+ )
+ if sweep.pending_without_deadline:
+ logger.info(
+ "approval_expiry %d pending approval(s) carry no expires_at and can never time out; "
+ "set one at POST /approvals to bring them under this sweep",
+ sweep.pending_without_deadline,
+ )
+ return sweep
+
+
+async def run_forever() -> None:
+ """Tick until cancelled. Owned by the API ``lifespan``, like the other workers."""
+ interval = max(int(getattr(settings, "APPROVAL_EXPIRY_INTERVAL_SECONDS", 300)), 30)
+ logger.info("approval_expiry started interval=%ds safe_default=%s", interval, SAFE_DEFAULT)
+ failures = TickFailures("approval_expiry", logger)
+ try:
+ while True:
+ try:
+ await run_once()
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc: # pragma: no cover - defensive
+ failures.record_failure(exc)
+ else:
+ failures.record_success()
+ await asyncio.sleep(interval)
+ except asyncio.CancelledError:
+ logger.info("approval_expiry stopped")
+ raise
+
+
+__all__ = ["EXPIRED_STATUS", "SAFE_DEFAULT", "ExpirySweep", "run_forever", "run_once"]
diff --git a/services/api/tests/test_approval_expiry.py b/services/api/tests/test_approval_expiry.py
new file mode 100644
index 000000000..567b0130b
--- /dev/null
+++ b/services/api/tests/test_approval_expiry.py
@@ -0,0 +1,197 @@
+"""Approvals nobody answered stop waiting forever (deferral 9b).
+
+Four things must hold, and each closes a specific way this could go wrong:
+only elapsed rows are touched, expiring dispatches nothing, the safe default
+matches the one the other half of the system already declares, and an
+approval with no deadline is *counted* rather than given one.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+from app.workers.approval_expiry import EXPIRED_STATUS, SAFE_DEFAULT, run_once
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+
+
+class _Result:
+ def __init__(self, rows: list[dict] | None = None, scalar: int = 0) -> None:
+ self._rows = rows or []
+ self._scalar = scalar
+
+ def mappings(self) -> _Result:
+ return self
+
+ def all(self) -> list[dict]:
+ return self._rows
+
+ def scalar_one(self) -> int:
+ return self._scalar
+
+
+class _Session:
+ """Answers the sweep's two statements and records what it was asked."""
+
+ def __init__(self, expired_rows: list[dict] | None = None, no_deadline: int = 0) -> None:
+ self.expired_rows = expired_rows or []
+ self.no_deadline = no_deadline
+ self.sql: list[str] = []
+ self.params: list[dict] = []
+ self.commits = 0
+
+ async def execute(self, statement: object, params: dict | None = None) -> _Result:
+ sql = str(statement)
+ self.sql.append(sql)
+ self.params.append(params or {})
+ if "current_setting" in sql: # the cross-tenant precondition probe
+ return _Result(scalar=0)
+ if sql.strip().upper().startswith("UPDATE"):
+ return _Result(rows=self.expired_rows)
+ return _Result(scalar=self.no_deadline)
+
+ async def scalar(self, _statement: object) -> str | None:
+ # `assert_cross_tenant_session` reads the bound tenant; None means
+ # nothing is bound, which is the precondition the sweep requires.
+ return None
+
+ async def commit(self) -> None:
+ self.commits += 1
+
+ async def close(self) -> None:
+ return None
+
+
+def _row(risk: str = "high") -> dict:
+ return {"id": uuid.uuid4(), "tenant_id": uuid.uuid4(), "risk_level": risk}
+
+
+class TestOnlyElapsedApprovalsAreExpired:
+ @pytest.mark.asyncio
+ async def test_the_predicate_names_pending_a_deadline_and_the_past(self) -> None:
+ session = _Session(expired_rows=[_row()])
+
+ await run_once(db=session)
+
+ update = next(s for s in session.sql if s.strip().upper().startswith("UPDATE"))
+ normalised = " ".join(update.split())
+ assert "status = 'pending'" in normalised
+ assert "expires_at IS NOT NULL" in normalised
+ assert "expires_at < now()" in normalised
+
+ @pytest.mark.asyncio
+ async def test_an_empty_sweep_reports_zero_and_commits(self) -> None:
+ session = _Session(expired_rows=[])
+
+ sweep = await run_once(db=session)
+
+ assert sweep.expired == 0
+ assert session.commits == 1
+
+ @pytest.mark.asyncio
+ async def test_it_refuses_a_session_bound_to_one_tenant(self) -> None:
+ """A bound session would expire one tenant's approvals and report
+ success for every tenant — the cross-tenant defect the precondition
+ helper exists to catch."""
+ from app.db.cross_tenant import CrossTenantPreconditionError
+
+ class _Bound(_Session):
+ async def scalar(self, _statement: object) -> str | None:
+ return str(uuid.uuid4())
+
+ with pytest.raises(CrossTenantPreconditionError):
+ await run_once(db=_Bound())
+
+
+class TestExpiringIsNotDeciding:
+ @pytest.mark.asyncio
+ async def test_the_sweep_writes_only_a_status_and_a_note(self) -> None:
+ """Nothing is dispatched. An expired approval is a request that
+ stopped pretending to be live, not a decision taken for the
+ operator — and `/decide` still accepts an expired row."""
+ session = _Session(expired_rows=[_row()])
+
+ await run_once(db=session)
+
+ update = " ".join(next(s for s in session.sql if s.strip().upper().startswith("UPDATE")).split())
+ assert "SET status = :expired" in update
+ for forbidden in ("decided_by_id", "decided_at", "action ="):
+ assert forbidden not in update, f"the sweep must not write {forbidden}"
+
+ def test_the_decide_endpoint_still_accepts_an_expired_row(self) -> None:
+ source = (REPO_ROOT / "services" / "api" / "app" / "api" / "v1" / "endpoints" / "approvals.py").read_text(encoding="utf-8")
+ assert 'row.status not in {"pending", "expired"}' in source, (
+ "the guard changed; if `expired` no longer decides, this worker silently becomes a way to close approvals"
+ )
+
+ @pytest.mark.asyncio
+ async def test_the_note_says_nothing_was_dispatched(self) -> None:
+ session = _Session(expired_rows=[_row()])
+
+ await run_once(db=session)
+
+ note = next(p["note"] for p in session.params if "note" in p)
+ assert "nothing was dispatched" in note
+ assert "can still be decided" in note
+
+
+class TestTheSafeDefaultIsTheOneAlreadyDeclared:
+ def test_it_matches_the_slack_bot_and_migration_062(self) -> None:
+ """Two halves timing out two different ways is worse than either."""
+ migration = (REPO_ROOT / "services" / "api" / "migrations" / "062_approval_timers.sql").read_text(encoding="utf-8")
+ scheduler = (REPO_ROOT / "services" / "slack-bot" / "app" / "services" / "approval_timeout.py").read_text(encoding="utf-8")
+
+ assert f"DEFAULT '{SAFE_DEFAULT}'" in migration
+ assert re.search(rf'safe_default: SafeDefault = "{SAFE_DEFAULT}"', scheduler)
+
+ def test_the_status_written_is_in_the_declared_vocabulary(self) -> None:
+ approvals = (REPO_ROOT / "services" / "api" / "app" / "api" / "v1" / "endpoints" / "approvals.py").read_text(encoding="utf-8")
+ assert f'"{EXPIRED_STATUS}"' in approvals, "the worker writes a status the API does not recognise"
+
+
+class TestAnApprovalWithNoDeadlineIsCountedNotInvented:
+ @pytest.mark.asyncio
+ async def test_they_are_reported_rather_than_swept(self) -> None:
+ """Inventing a window for an approval whose creator did not set one
+ would start expiring containments on a schedule nobody chose. The
+ number is surfaced so the gap is visible instead of read as zero."""
+ session = _Session(expired_rows=[], no_deadline=17)
+
+ sweep = await run_once(db=session)
+
+ assert sweep.pending_without_deadline == 17
+ assert sweep.expired == 0
+
+ @pytest.mark.asyncio
+ async def test_the_breakdown_by_risk_travels_with_the_count(self) -> None:
+ """'Three expired' is a number; 'two of them critical' is a finding."""
+ session = _Session(expired_rows=[_row("critical"), _row("critical"), _row("low")])
+
+ sweep = await run_once(db=session)
+
+ assert sweep.expired == 3
+ assert sweep.by_risk == {"critical": 2, "low": 1}
+
+
+class TestItIsWiredIntoTheApiLifespan:
+ def test_the_worker_is_started_and_cancelled(self) -> None:
+ """A worker nobody starts is the defect this deferral is about."""
+ main = (REPO_ROOT / "services" / "api" / "app" / "main.py").read_text(encoding="utf-8")
+
+ assert "run_approval_expiry" in main
+ assert "approval_expiry_task.cancel()" in main
+ assert "APPROVAL_EXPIRY_ENABLED" in main
+
+ def test_a_disabled_worker_says_so_out_loud(self) -> None:
+ main = (REPO_ROOT / "services" / "api" / "app" / "main.py").read_text(encoding="utf-8")
+ assert "approval_expiry worker disabled" in main
+
+ def test_the_sweep_started_at_is_timezone_aware(self) -> None:
+ from app.workers.approval_expiry import ExpirySweep
+
+ sweep = ExpirySweep(started_at=datetime.now(UTC))
+ assert sweep.started_at.tzinfo is not None