Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
test(agents,api): verification pause against live Postgres, plus its workflow
Beenu Arora403207061231
diff --git a/.github/workflows/verification-live.yml b/.github/workflows/verification-live.ymlnew file mode 100644index 000000000..a8f148e40--- /dev/null+++ b/.github/workflows/verification-live.yml@@ -0,0 +1,104 @@+name: Stateful user verification (live)++# Depth 5.4. The verification workflow is covered offline against a double+# that honours the predicates the real statements carry, and a double cannot+# answer the two questions that matter here: whether migration 102 applies,+# and whether the conditional UPDATEs really are atomic claims. Postgres+# answers both.+#+# The offline suite is the one that runs on every pull request; this one+# grades the storage layer the offline suite stands in for.++on:+ pull_request:+ paths:+ - "services/api/migrations/102_playbook_verification.sql"+ - "services/agents/app/playbook/pause.py"+ - "services/agents/app/playbook/verification.py"+ - "tests/isolation/test_verification_pause_live.py"+ - ".github/workflows/verification-live.yml"+ push:+ branches: [main]+ workflow_dispatch:++# Per commit on a push and per pull request on a PR. Keying on `github.ref`+# makes a push to `main` cancel the run before it, which across this+# repository left 0 of 22 required checks graded on all of the last 30+# commits.+concurrency:+ group: verification-live-${{ github.event_name == 'push' && github.sha || github.event.pull_request.number }}+ cancel-in-progress: false++jobs:+ verification-live:+ name: A verification pause survives, escalates once and answers once+ runs-on: ubuntu-latest+ timeout-minutes: 15++ services:+ postgres:+ image: public.ecr.aws/docker/library/postgres:16-alpine+ env:+ POSTGRES_USER: aisoc+ POSTGRES_PASSWORD: aisoc_ci_only+ POSTGRES_DB: aisoc+ ports: ["5432:5432"]+ options: >-+ --health-cmd "pg_isready -U aisoc"+ --health-interval 5s+ --health-timeout 5s+ --health-retries 20++ env:+ LIVE_DSN: postgresql://aisoc:aisoc_ci_only@localhost:5432/aisoc+ PGPASSWORD: aisoc_ci_only+ OTEL_SDK_DISABLED: "true"++ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7++ - uses: actions/setup-python@e9d6f990972a57673cdb72ec29e19d42ba28880f # v6+ with:+ python-version: "3.11"++ - name: Install agents from its lockfile+ working-directory: services/agents+ run: |+ set -euo pipefail+ python -m pip install --quiet --upgrade pip "poetry==2.4.1"+ poetry config virtualenvs.create false+ poetry install --no-root --no-interaction+ python -m pip install --quiet pytest pytest-asyncio++ - name: Apply every migration, in order+ run: |+ set -euo pipefail+ for f in services/api/migrations/*.sql; do+ psql -h localhost -U aisoc -d aisoc -v ON_ERROR_STOP=1 -q -f "$f"+ done++ # The migration applying is itself the first assertion: `102` alters a+ # CHECK constraint, and a constraint name that drifted would fail here+ # rather than on somebody's deployment.+ - name: The pause table carries the verification columns+ run: |+ set -euo pipefail+ psql -h localhost -U aisoc -d aisoc -v ON_ERROR_STOP=1 -q -c \+ "SELECT verification_stage, verification_outcome FROM aisoc_playbook_pauses WHERE false;"++ - name: The verification pause, against the real schema+ working-directory: services/agents+ env:+ ISOLATION_VERIFICATION_DSN: ${{ env.LIVE_DSN }}+ run: python -m pytest ../../tests/isolation/test_verification_pause_live.py -v --tb=short -rs++ # A suite that skipped reports the same word as one that passed, so the+ # job asserts the DSN was set rather than trusting the run.+ - name: No suite may report success having skipped+ if: always()+ run: |+ set -euo pipefail+ if [ -z "${LIVE_DSN}" ]; then+ echo "::error::LIVE_DSN is empty, so the live suite skipped and this job proved nothing."+ exit 1+ fidiff --git a/playbooks/packs/v1/suspicious-signin/user-verification-containment.playbook.json b/playbooks/packs/v1/suspicious-signin/user-verification-containment.playbook.jsonnew file mode 100644index 000000000..2f9317017--- /dev/null+++ b/playbooks/packs/v1/suspicious-signin/user-verification-containment.playbook.json@@ -0,0 +1,111 @@+{+ "id": "user-verification-containment-v1",+ "name": "Suspicious Sign-In: Ask the User, Then Their Manager",+ "description": "Asks the person whose account signed in whether they recognise it, escalates to their manager from the directory when nobody answers, and branches on the signed answer. 'No, that wasn't me' and 'nobody answered' both reach containment behind an analyst approval; 'yes, that was me' closes the case. Never asked on a confirmed true positive, and auto-sent only for alert classes the tenant opted into.",+ "version": "1.0.0",+ "author": "AiSOC",+ "tags": [+ "suspicious-signin",+ "identity",+ "verification",+ "mitre.t1078",+ "mitre.t1556"+ ],+ "trigger": {+ "on": "alert",+ "severity": ["medium", "high"],+ "tags": ["suspicious-signin", "impossible-travel", "new-device"]+ },+ "enabled": true,+ "dry_run_support": true,+ "created_at": "2026-10-11T00:00:00+00:00",+ "updated_at": "2026-10-11T00:00:00+00:00",+ "inputs": {+ "user": {+ "description": "The account the suspicious sign-in was for",+ "field": "alert.user",+ "required": true+ },+ "source_ip": {+ "description": "Where the sign-in came from, quoted back to the person being asked",+ "field": "alert.source_ip",+ "required": false+ }+ },+ "steps": [+ {+ "id": "ask",+ "name": "Ask the account holder",+ "type": "verify_user",+ "params": {+ "user": "{{alert.user}}",+ "channel": "slack",+ "message_template": "AiSOC saw a sign-in to your account from {{alert.source_ip}}. Was this you?",+ "expires_in_hours": 24+ },+ "on_failure": "abort",+ "retry_max": 0,+ "timeout_seconds": 30+ },+ {+ "id": "approve_contain",+ "name": "Approve containment",+ "type": "approval",+ "params": {+ "prompt": "{{alert.user}} did not confirm this sign-in. Disable the account and revoke its sessions?",+ "expires_in_hours": 4+ },+ "condition": "verification_outcome != 'acknowledged'",+ "on_failure": "abort",+ "retry_max": 0,+ "timeout_seconds": 30+ },+ {+ "id": "disable",+ "name": "Disable the account",+ "type": "disable_user",+ "params": {+ "user": "{{alert.user}}"+ },+ "condition": "verification_outcome != 'acknowledged'",+ "on_failure": "abort",+ "retry_max": 0,+ "timeout_seconds": 30+ },+ {+ "id": "revoke",+ "name": "Revoke the account's sessions",+ "type": "revoke_session",+ "params": {+ "user": "{{alert.user}}"+ },+ "condition": "verification_outcome != 'acknowledged'",+ "on_failure": "abort",+ "retry_max": 0,+ "timeout_seconds": 30+ },+ {+ "id": "notify_soc",+ "name": "Tell the SOC what the answer was",+ "type": "notify",+ "params": {+ "channel": "slack",+ "message_template": "Verification for {{alert.user}} came back '{{verification_outcome}}' (asked: {{verification.stage}}). Containment {{verification_outcome}} != acknowledged.",+ "webhook_env": "SLACK_SOC_WEBHOOK"+ },+ "on_failure": "continue",+ "retry_max": 0,+ "timeout_seconds": 10+ },+ {+ "id": "close",+ "name": "Close the case when the user confirmed it was them",+ "type": "close_case",+ "params": {},+ "condition": "verification_outcome == 'acknowledged'",+ "on_failure": "continue",+ "retry_max": 0,+ "timeout_seconds": 20+ }+ ]+}diff --git a/tests/isolation/test_verification_pause_live.py b/tests/isolation/test_verification_pause_live.pynew file mode 100644index 000000000..6519ea71c--- /dev/null+++ b/tests/isolation/test_verification_pause_live.py@@ -0,0 +1,336 @@+"""The verification pause, against real Postgres and the real SQL.++Depth plan 5.4.++Why this suite exists at all+---------------------------+``services/agents/tests/test_playbook_verification.py`` drives the whole+workflow against a double, and that double is written to honour the+predicates the real statements carry. It is still a double, and this+repository's most expensive recurring defect is a double that is more+capable than the thing it stands in for. Specifically, nothing offline can+tell you:++* whether migration 102's DDL applies at all, or whether its ``CHECK``+ constraints accept the five outcome values the code writes — a rejected+ value would surface as "the answer was not recorded" on a live+ deployment and nowhere else;+* whether the conditional ``UPDATE``s really are atomic claims. The double+ asserts the predicates by reading them out of the SQL text; Postgres+ asserts them by refusing the second writer.++So the same four functions run here against the real schema.++What is deliberately not covered+--------------------------------+The engine. This suite is about the storage layer and says so: the+branching, the escalation and the refusals are covered offline, where they+can run on every pull request without a container.+"""++from __future__ import annotations++import json+import os+import uuid+from collections.abc import AsyncIterator+from datetime import UTC, datetime, timedelta+from pathlib import Path++import pytest+import pytest_asyncio++pytestmark = [+ pytest.mark.asyncio(loop_scope="module"),+ pytest.mark.skipif(+ not os.environ.get("ISOLATION_VERIFICATION_DSN", "").strip(),+ reason="ISOLATION_VERIFICATION_DSN is not set; this suite needs a live Postgres",+ ),+]++REPO_ROOT = Path(__file__).resolve().parents[2]++#: The three migrations that own this table, applied in order. Not the whole+#: chain: `001_init` brings up the entire product schema and this suite is+#: about three columns and one constraint. The two stub tables below stand+#: in for the only foreign keys that reach outside those files.+_MIGRATIONS = (+ "081_playbook_pauses.sql",+ "096_playbook_wait_pauses.sql",+ "102_playbook_verification.sql",+)++#: Enough of `tenants`, `users` and `aisoc_scim_users` for the foreign keys+#: and the manager lookup. Written here rather than imported from+#: `001_init` so this file states exactly what it depends on, and so a+#: change to an unrelated column in the real schema cannot break it.+_STUBS = """+CREATE EXTENSION IF NOT EXISTS pgcrypto;+CREATE TABLE IF NOT EXISTS tenants (id UUID PRIMARY KEY, name TEXT);+CREATE TABLE IF NOT EXISTS users (+ id UUID PRIMARY KEY,+ tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,+ email TEXT NOT NULL+);+CREATE TABLE IF NOT EXISTS aisoc_scim_users (+ user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,+ tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,+ external_id TEXT+);+CREATE OR REPLACE FUNCTION current_tenant_id() RETURNS UUID AS $$+ SELECT NULLIF(current_setting('app.current_tenant_id', true), '')::uuid;+$$ LANGUAGE sql STABLE;+"""+++def _dsn() -> str:+ value = os.environ.get("ISOLATION_VERIFICATION_DSN", "").strip()+ if not value:+ pytest.skip("ISOLATION_VERIFICATION_DSN is not set")+ return value+++@pytest_asyncio.fixture(loop_scope="module", scope="module")+async def pool() -> AsyncIterator[object]:+ import asyncpg++ created = await asyncpg.create_pool(_dsn(), min_size=1, max_size=4)+ assert created is not None+ async with created.acquire() as conn:+ # In CI the whole chain is already applied by the job, and+ # re-running the stubs there would replace `current_tenant_id()` —+ # the function every RLS policy in the product calls — with the+ # simplified one below. So the schema is detected rather than+ # assumed, and the standalone path is the only one that writes.+ migrated = await conn.fetchval(+ "SELECT 1 FROM information_schema.columns WHERE table_name = 'aisoc_playbook_pauses' AND column_name = 'verification_stage'"+ )+ if not migrated:+ await conn.execute(_STUBS)+ for name in _MIGRATIONS:+ await conn.execute((REPO_ROOT / "services" / "api" / "migrations" / name).read_text(encoding="utf-8"))+ yield created+ await created.close()+++@pytest_asyncio.fixture(loop_scope="module")+async def wired(pool: object, monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[object, str]]:+ """Point `app.playbook.pause` at the live pool and seed one tenant."""+ import sys++ sys.path.insert(0, str(REPO_ROOT / "services" / "agents"))+ from app.playbook import pause as playbook_pause++ async def _pool() -> object:+ return pool++ monkeypatch.setattr(playbook_pause, "_pool", _pool)++ tenant_id = str(uuid.uuid4())+ async with pool.acquire() as conn: # type: ignore[attr-defined]+ await conn.execute("INSERT INTO tenants (id, name) VALUES ($1::uuid, 'live')", tenant_id)+ yield playbook_pause, tenant_id+++async def _suspend(playbook_pause: object, tenant_id: str, token: str) -> object:+ return await playbook_pause.suspend( # type: ignore[attr-defined]+ kind="verification",+ resume_at=datetime.now(UTC) + timedelta(minutes=30),+ resume_token=token,+ tenant_id=tenant_id,+ run_id=f"run-{uuid.uuid4().hex[:12]}",+ playbook_id="pb-live",+ playbook_name="live verification",+ step_index=0,+ step_id="ask",+ run_context={"tenant_id": tenant_id},+ step_results=[],+ verification_stage="user",+ verification_subject="[email protected]",+ )+++async def test_the_migration_accepts_a_verification_pause(wired: tuple[object, str]) -> None:+ """`kind = 'verification'` and a stage both pass the real CHECKs."""+ playbook_pause, tenant_id = wired+ pause = await _suspend(playbook_pause, tenant_id, uuid.uuid4().hex)++ assert pause is not None+ assert pause.kind == "verification"+ assert pause.verification_stage == "user"+ assert pause.verification_subject == "[email protected]"++[email protected]("outcome", ["acknowledged", "denied", "unsure", "timed_out", "no_manager"])+async def test_every_outcome_the_code_writes_passes_the_check(wired: tuple[object, str], outcome: str) -> None:+ """The constraint and the vocabulary must agree.++ They are written in two files — the migration's ``CHECK`` and+ ``verification.OUTCOMES`` — and a value in one and not the other is a+ row Postgres refuses at the moment somebody answers.+ """+ playbook_pause, tenant_id = wired+ pause = await _suspend(playbook_pause, tenant_id, uuid.uuid4().hex)+ assert pause is not None++ assert await playbook_pause.record_verification_outcome( # type: ignore[attr-defined]+ pause_id=pause.id, tenant_id=tenant_id, outcome=outcome+ )+++async def test_an_outcome_is_written_once(wired: tuple[object, str]) -> None:+ """Postgres refuses the second answer, not the double.++ ``verification_outcome IS NULL`` in the predicate is what makes a+ double click resolve once, and this is the only place that claim is+ tested against a real transaction.+ """+ playbook_pause, tenant_id = wired+ pause = await _suspend(playbook_pause, tenant_id, uuid.uuid4().hex)+ assert pause is not None++ assert await playbook_pause.record_verification_outcome(pause_id=pause.id, tenant_id=tenant_id, outcome="denied") # type: ignore[attr-defined]+ assert not await playbook_pause.record_verification_outcome( # type: ignore[attr-defined]+ pause_id=pause.id, tenant_id=tenant_id, outcome="acknowledged"+ )+++async def test_a_pause_escalates_once_and_keeps_its_token(wired: tuple[object, str]) -> None:+ """Two sweeper replicas racing the same row escalate it once."""+ playbook_pause, tenant_id = wired+ token = uuid.uuid4().hex+ pause = await _suspend(playbook_pause, tenant_id, token)+ assert pause is not None+ deadline = datetime.now(UTC) + timedelta(hours=1)++ assert await playbook_pause.escalate_to_manager( # type: ignore[attr-defined]+ pause_id=pause.id, tenant_id=tenant_id, manager="[email protected]", resume_at=deadline+ )+ assert not await playbook_pause.escalate_to_manager( # type: ignore[attr-defined]+ pause_id=pause.id, tenant_id=tenant_id, manager="[email protected]", resume_at=deadline+ )++ found = await playbook_pause.find_verification_by_token(resume_token=token) # type: ignore[attr-defined]+ assert found is not None+ assert found.verification_stage == "manager"+ assert found.verification_manager == "[email protected]"+++async def test_a_wait_token_does_not_resolve_as_a_verification(wired: tuple[object, str]) -> None:+ """``kind`` is in the predicate, so the two callbacks cannot cross."""+ playbook_pause, tenant_id = wired+ token = uuid.uuid4().hex+ await playbook_pause.suspend( # type: ignore[attr-defined]+ kind="wait",+ resume_at=None,+ resume_token=token,+ tenant_id=tenant_id,+ run_id=f"run-wait-{uuid.uuid4().hex[:12]}",+ playbook_id="pb-live",+ playbook_name="live wait",+ step_index=0,+ step_id="hold",+ run_context={},+ step_results=[],+ )++ assert await playbook_pause.find_verification_by_token(resume_token=token) is None # type: ignore[attr-defined]+ assert await playbook_pause.find_wait_by_token(resume_token=token) is not None # type: ignore[attr-defined]+++async def test_due_verifications_sees_only_waiting_verification_rows(wired: tuple[object, str]) -> None:+ """The sweep's predicate, against the real partial index."""+ playbook_pause, tenant_id = wired+ pause = await _suspend(playbook_pause, tenant_id, uuid.uuid4().hex)+ assert pause is not None++ later = datetime.now(UTC) + timedelta(hours=2)+ due_ids = {p.id for p in await playbook_pause.due_verifications(now=later, limit=100)} # type: ignore[attr-defined]+ assert pause.id in due_ids++ await playbook_pause.resolve(pause_id=pause.id, tenant_id=tenant_id, status="resumed", resolution="answered") # type: ignore[attr-defined]+ due_ids = {p.id for p in await playbook_pause.due_verifications(now=later, limit=100)} # type: ignore[attr-defined]+ assert pause.id not in due_ids+++async def test_the_policy_table_is_tenant_scoped_and_default_deny(pool: object) -> None:+ """A tenant with no row opts into nothing, and cannot read another's.++ The RLS policy carries an unbound arm so the agents service — which+ reads this from a consumer that binds no tenant — is not locked out.+ That arm is exactly the thing worth checking: with a tenant bound, it+ must not leak.+ """+ import asyncpg++ tenant_a, tenant_b = str(uuid.uuid4()), str(uuid.uuid4())+ async with pool.acquire() as conn: # type: ignore[attr-defined]+ await conn.execute("INSERT INTO tenants (id, name) VALUES ($1::uuid, 'a'), ($2::uuid, 'b')", tenant_a, tenant_b)+ await conn.execute(+ "INSERT INTO aisoc_verification_policies (tenant_id, alert_class) VALUES ($1::uuid, 'impossible_travel')",+ tenant_a,+ )++ # A non-superuser, because RLS is bypassed by the owner and by a+ # superuser — the exact mistake that made 92 policies decorative in+ # this repository once already.+ async with pool.acquire() as conn: # type: ignore[attr-defined]+ # Created if absent rather than dropped and recreated: the role+ # holds grants, so `DROP ROLE` fails on the second run of this+ # suite against the same database and the failure reads as a+ # tenancy problem rather than as test housekeeping.+ await conn.execute(+ "DO $$ BEGIN "+ "IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'lanec_reader') THEN "+ "CREATE ROLE lanec_reader LOGIN PASSWORD 'lanec_reader_pw'; END IF; END $$"+ )+ await conn.execute("GRANT SELECT ON aisoc_verification_policies TO lanec_reader")+ await conn.execute("GRANT USAGE ON SCHEMA public TO lanec_reader")++ dsn = _dsn()+ reader_dsn = dsn.replace("postgres:lanec@", "lanec_reader:lanec_reader_pw@").replace("postgres@", "lanec_reader@")+ reader = await asyncpg.connect(reader_dsn)+ try:+ await reader.execute("SELECT set_config('app.current_tenant_id', $1, false)", tenant_b)+ rows = await reader.fetch("SELECT alert_class FROM aisoc_verification_policies")+ assert rows == [], "tenant B read tenant A's verification policy"++ await reader.execute("SELECT set_config('app.current_tenant_id', $1, false)", tenant_a)+ rows = await reader.fetch("SELECT alert_class FROM aisoc_verification_policies")+ assert [r["alert_class"] for r in rows] == ["impossible_travel"]+ finally:+ await reader.close()+++async def test_the_stored_context_survives_a_round_trip(wired: tuple[object, str]) -> None:+ """The escalation reads the step's params back off the pause.++ They go in as JSON and come back as a string from asyncpg, which+ ``_row_to_pause`` decodes. A change there would leave the escalation+ asking on the default channel with the default message.+ """+ playbook_pause, tenant_id = wired+ token = uuid.uuid4().hex+ await playbook_pause.suspend( # type: ignore[attr-defined]+ kind="verification",+ resume_at=datetime.now(UTC) + timedelta(minutes=30),+ resume_token=token,+ tenant_id=tenant_id,+ run_id=f"run-ctx-{uuid.uuid4().hex[:12]}",+ playbook_id="pb-live",+ playbook_name="live verification",+ step_index=0,+ step_id="ask",+ run_context={"tenant_id": tenant_id, "_verification_step_params": {"channel": "teams"}},+ step_results=[{"step_id": "earlier", "status": "success"}],+ verification_stage="user",+ verification_subject="[email protected]",+ )++ found = await playbook_pause.find_verification_by_token(resume_token=token) # type: ignore[attr-defined]+ assert found is not None+ assert found.run_context["_verification_step_params"] == {"channel": "teams"}+ assert found.step_results == [{"step_id": "earlier", "status": "success"}]+ # Proves the decode rather than a passthrough: a raw row would be a str.+ assert not isinstance(found.run_context, str)+ json.dumps(found.run_context)