Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""Every automatic closure of a production alert passes one authorization check.
Phase 1 of the closure remediation. See
`docs/remediation/phase-1-closure-authorization.md`.
What was wrong
--------------
`alerts.status` reaches `'resolved'` from exactly one statement — the `UPDATE
alerts` at the end of `app.investigator.ledger.persist_auto_triage`, whose
`CASE WHEN $7 AND NOT $10 THEN 'resolved'` turns on the caller-supplied
`auto_closed` flag. Two callers set that flag and only one of them asked
permission:
* `run_auto_triage` consulted `closure_policy.decide(...)` — the kill switch,
the tenant's policy row, the earned `auto_close` grant;
* `FusedAlertTriageWorker._maybe_suppress_from_memory` set
`state.status = COMPLETED` straight from a stored outcome prior, and
`_record` forwarded `auto_closed=state.status is AgentStatus.COMPLETED`.
The strings `closure_policy` and `kill_switch` appear in that module only
inside comments.
So a tenant with `enabled = false` still got automatic closures, the kill
switch did not stop them, `require_grant` was not consulted, and the
confidence written against the decision was read off the prior rather than
measured. A remembered verdict was being treated as remembered permission.
Why this file is a live suite
-----------------------------
The defect is a disagreement between two modules about who asked permission,
resolved by SQL. A fake connection answers both sides happily: the offline
suite had a `_Pool` whose `fetchrow` returns whatever it was constructed with,
which cannot tell a policy that refused from a policy that was never read, and
cannot run the `CASE` expression that is the actual closure. Everything here
drives the real `persist_auto_triage`, the real `app.closure` modules and real
Postgres with every migration applied, and asserts on the row afterwards.
Against the pre-fix tree the first, third, fourth, fifth, sixth, seventh,
eighth and eleventh tests fail with the alert sitting at `status='resolved'`.
"""
from __future__ import annotations
import os
import uuid
from datetime import UTC, datetime
from typing import Any
import pytest
import pytest_asyncio
pytestmark = [
pytest.mark.asyncio(loop_scope="module"),
pytest.mark.skipif(
not os.environ.get("ISOLATION_CLOSURE_AUTHZ_DSN", "").strip(),
reason="ISOLATION_CLOSURE_AUTHZ_DSN is not set; this suite needs a live Postgres",
),
]
#: The verdict every test here uses. Auto-closeable, so nothing in these
#: results turns on verdict eligibility — the question under test is
#: authorization, and a non-closeable verdict would pass for the wrong reason.
BENIGN = "benign"
def _dsn() -> str:
value = os.environ.get("ISOLATION_CLOSURE_AUTHZ_DSN", "").strip()
if not value:
pytest.skip("ISOLATION_CLOSURE_AUTHZ_DSN is not set")
return value
def _admin_dsn() -> str:
"""The schema owner, used only for fixtures and assertions.
The code under test runs as the *runtime* role, where RLS is forced and
a superuser's exemption cannot make a cross-tenant assertion pass for the
wrong reason. Test setup is a different job: it writes rows for several
tenants at once and reads them back, which under those same policies
needs the owner. Keeping the two roles apart is what lets the production
path be judged under the policies it really runs under.
"""
return os.environ.get("ISOLATION_CLOSURE_AUTHZ_ADMIN_DSN", "").strip() or _dsn()
@pytest_asyncio.fixture(scope="module", loop_scope="module")
async def pool():
"""A real asyncpg pool, plus the `DATABASE_URL` production reads.
Both the ledger and the closure policy resolve their own pool from the
environment. Setting it here rather than handing each a pool is the
point: a test that injected pools would not notice a module that
resolves the wrong one.
"""
asyncpg = pytest.importorskip("asyncpg")
os.environ["DATABASE_URL"] = _dsn()
from app.closure import policy as closure_policy
from app.investigator import ledger as ledger_module
created = await asyncpg.create_pool(
_admin_dsn().replace("postgresql+asyncpg://", "postgresql://"),
min_size=1,
max_size=6,
)
yield created
await created.close()
await ledger_module.close_pool()
# The closure package resolves the institutional pool. Closed through the
# accessor it exports rather than by reaching into a private global, so
# this teardown does not depend on the shape of a module it is testing.
shared = await closure_policy.shared_pool()
if shared is not None:
await shared.close()
closure_policy.reset_cache()
@pytest.fixture(autouse=True)
def _clear_policy_cache():
"""The policy caches its answer for ten seconds.
Every test here changes a policy row and then asks again, which is
exactly the window the cache covers, so leaving it would make tests pass
or fail on their ordering.
"""
from app.closure import policy as closure_policy
closure_policy.reset_cache()
yield
closure_policy.reset_cache()
async def _tenant(pool) -> str:
tenant_id = str(uuid.uuid4())
async with pool.acquire() as conn:
await conn.execute(
"INSERT INTO tenants (id, name, slug) VALUES ($1::uuid, $2, $3)",
tenant_id,
"closure-authz",
f"closure-authz-{tenant_id[:8]}",
)
return tenant_id
async def _alert(pool, tenant_id: str, *, category: str = "identity") -> str:
alert_id = str(uuid.uuid4())
async with pool.acquire() as conn:
await conn.execute(
"""
INSERT INTO alerts (id, tenant_id, title, severity, status, category)
VALUES ($1::uuid, $2::uuid, 'closure authorization probe', 'medium', 'new', $3)
""",
alert_id,
tenant_id,
category,
)
return alert_id
async def _policy(
pool,
tenant_id: str,
*,
enabled: bool,
threshold: float | None = None,
require_grant: bool = False,
alert_class: str | None = None,
) -> None:
async with pool.acquire() as conn:
await conn.execute(
"""
INSERT INTO aisoc_closure_policies
(tenant_id, alert_class, enabled, threshold, require_grant)
VALUES ($1::uuid, $2, $3, $4, $5)
""",
tenant_id,
alert_class,
enabled,
threshold,
require_grant,
)
async def _grant(pool, tenant_id: str, *, alert_class: str, state: str = "granted") -> None:
async with pool.acquire() as conn:
await conn.execute(
"""
INSERT INTO aisoc_autonomy_grants
(tenant_id, scope_kind, scope_key, capability, state, source, evidence)
VALUES ($1::uuid, 'alert_class', $2, 'auto_close', $3, 'earned', '{}'::jsonb)
""",
tenant_id,
alert_class,
state,
)
async def _engage_kill_switch(pool, tenant_id: str | None, *, reason: str) -> None:
async with pool.acquire() as conn:
await conn.execute(
"""
INSERT INTO aisoc_kill_switch (tenant_id, engaged, reason, engaged_by)
VALUES ($1::uuid, TRUE, $2, 'isolation-suite')
""",
tenant_id,
reason,
)
async def _release_kill_switches(pool) -> None:
async with pool.acquire() as conn:
await conn.execute("DELETE FROM aisoc_kill_switch")
async def _row(pool, alert_id: str) -> dict[str, Any]:
async with pool.acquire() as conn:
found = await conn.fetchrow(
"SELECT status, disposition, resolved_at, ai_score FROM alerts WHERE id = $1::uuid",
alert_id,
)
return dict(found)
async def _persist(
*,
tenant_id: str,
alert_id: str,
auto_closed: bool = True,
confidence: float = 0.99,
tier: str = "llm",
verdict: str = BENIGN,
shadow: bool = False,
category: str = "identity",
) -> bool:
"""The real durable write, called exactly as the worker calls it."""
from app.investigator import ledger as ledger_module
return await ledger_module.persist_auto_triage(
run_id=uuid.uuid4(),
alert_id=alert_id,
tenant_ref=tenant_id,
alert_summary="closure authorization probe",
raw_alert={"id": alert_id, "category": category},
tier=tier,
verdict=verdict,
confidence=confidence,
rationale="probe",
findings=[],
proposed_actions=[],
auto_closed=auto_closed,
shadow=shadow,
)
class TestClosureNeedsExplicitAuthorization:
"""A tenant that has authorized nothing gets no automatic closures."""
async def test_an_eligible_verdict_does_not_close_without_an_explicit_policy(self, pool) -> None:
"""Finding 14. The absence of a policy row was read as permission.
`resolve_closure_policy` returned `confidence >= default_threshold`
for a tenant with no row, so every tenant auto-closed from the moment
the agent produced a confident benign verdict — including one that had
never opened the screen.
"""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _persist(tenant_id=tenant_id, alert_id=alert_id)
row = await _row(pool, alert_id)
assert row["status"] != "resolved", "an alert auto-resolved for a tenant that has authorized no automatic closure at all"
assert row["resolved_at"] is None
async def test_an_explicitly_authorized_tenant_still_closes(self, pool) -> None:
"""The positive control, and the reason it is not optional.
A gate that refuses everything satisfies every other assertion in
this file. Without this test, wiring the chokepoint to `return False`
would look like a complete fix.
"""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=True, threshold=0.80, require_grant=False)
await _persist(tenant_id=tenant_id, alert_id=alert_id)
row = await _row(pool, alert_id)
assert row["status"] == "resolved", "an explicitly authorized tenant stopped auto-closing"
assert row["resolved_at"] is not None
assert row["disposition"] == BENIGN
async def test_a_disabled_policy_blocks_closure(self, pool) -> None:
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=False)
await _persist(tenant_id=tenant_id, alert_id=alert_id)
assert (await _row(pool, alert_id))["status"] != "resolved"
async def test_an_unearned_grant_blocks_closure(self, pool) -> None:
"""`require_grant` is the tenant saying "only once shadow mode earns it"."""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=True, threshold=0.5, require_grant=True)
await _persist(tenant_id=tenant_id, alert_id=alert_id)
assert (await _row(pool, alert_id))["status"] != "resolved"
from app.closure import policy as closure_policy
earned = await _alert(pool, tenant_id)
await _grant(pool, tenant_id, alert_class="identity")
closure_policy.reset_cache()
await _persist(tenant_id=tenant_id, alert_id=earned)
assert (await _row(pool, earned))["status"] == "resolved", "an earned grant did not unblock closure"
class TestEveryClosurePathIsGoverned:
"""The LLM path asked permission. The other two did not."""
async def test_the_memory_suppression_path_is_governed(self, pool) -> None:
"""Finding 1, at the durable boundary.
`_maybe_suppress_from_memory` reaches `persist_auto_triage` with
`auto_closed=True` and `tier='memory'` having consulted no policy at
all. A trusted prior is evidence about the alert; it is not a grant.
"""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=False)
await _persist(tenant_id=tenant_id, alert_id=alert_id, tier="memory", confidence=0.9)
assert (await _row(pool, alert_id))["status"] != "resolved", (
"outcome-memory suppression closed an alert for a tenant whose policy disables closure"
)
async def test_the_cached_verdict_path_is_governed(self, pool) -> None:
"""A cached verdict is a remembered answer, not remembered permission.
The cost governor returns `Decision.DEDUPLICATED` with a verdict
produced for an earlier alert, possibly under a policy that has since
changed.
"""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=False)
await _persist(tenant_id=tenant_id, alert_id=alert_id, tier="cached")
assert (await _row(pool, alert_id))["status"] != "resolved"
async def test_the_real_suppression_path_does_not_close_an_unauthorized_alert(self, pool) -> None:
"""The whole of Finding 1, driven through the worker rather than the ledger.
Everything here is production: the real `FusedAlertTriageWorker`
method, the real `LiveTriageWriter`, the real ledger and the real
policy. Only the outcome prior is supplied, because that is the
worker's *input* — a reader is the seam replay already uses, and
faking it fakes the alert's history, not the authorization.
"""
from app.agents.dispositions import BENIGN as BENIGN_DISPOSITION
from app.memory.outcomes import HUMAN
from app.models.state import InvestigationState
from app.workers.fused_alert_consumer import FusedAlertTriageWorker
from app.workers.triage_persistence import LiveTriageWriter
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=False)
class _PriorReader:
"""Returns one trusted prior. Every other read is empty."""
async def lookup_prior(self, tenant: str, signature: str) -> dict[str, Any]: # noqa: ARG002
return {
"disposition": BENIGN_DISPOSITION,
"confidence": 0.97,
"count": 5,
"author": HUMAN,
"last_seen": datetime.now(UTC).isoformat(),
}
worker = FusedAlertTriageWorker.__new__(FusedAlertTriageWorker)
worker._reader = _PriorReader() # type: ignore[assignment]
worker._writer = LiveTriageWriter()
state = InvestigationState(
incident_id=uuid.uuid4(),
tenant_id=uuid.UUID(tenant_id),
alert_summary="repeat alert",
raw_alert={"id": alert_id, "category": "identity"},
)
suppressed = await worker._maybe_suppress_from_memory(
state,
"sig-authz-probe",
[],
writer=LiveTriageWriter(),
)
assert suppressed is not None, "the prior should still suppress re-triage; only the closure is withheld"
assert (await _row(pool, alert_id))["status"] != "resolved", (
"the production memory-suppression path closed an alert the tenant's policy forbids"
)
class TestAuthorizationIsEvaluatedAtTheWrite:
async def test_a_changed_policy_invalidates_an_earlier_authorization(self, pool) -> None:
"""Recovery must re-evaluate, never replay a stale permissive answer.
A retried or resumed alert is persisted long after the decision that
produced its verdict. If the authorization travelled with the verdict,
a tenant disabling closure would not stop work already in flight.
"""
from app.closure import policy as closure_policy
tenant_id = await _tenant(pool)
await _policy(pool, tenant_id, enabled=True, threshold=0.5, require_grant=False)
first = await _alert(pool, tenant_id)
await _persist(tenant_id=tenant_id, alert_id=first)
assert (await _row(pool, first))["status"] == "resolved"
async with pool.acquire() as conn:
await conn.execute("UPDATE aisoc_closure_policies SET enabled = FALSE WHERE tenant_id = $1::uuid", tenant_id)
closure_policy.reset_cache()
second = await _alert(pool, tenant_id)
await _persist(tenant_id=tenant_id, alert_id=second)
assert (await _row(pool, second))["status"] != "resolved", (
"closure continued after the tenant disabled it, so the authorization was retained rather than re-evaluated"
)
async def test_one_tenants_policy_cannot_authorize_anothers_closure(self, pool) -> None:
authorized = await _tenant(pool)
await _policy(pool, authorized, enabled=True, threshold=0.5, require_grant=False)
other = await _tenant(pool)
alert_id = await _alert(pool, other)
await _persist(tenant_id=other, alert_id=alert_id)
assert (await _row(pool, alert_id))["status"] != "resolved"
class TestStopControls:
async def test_a_global_kill_switch_stops_closure(self, pool) -> None:
from app.closure import policy as closure_policy
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=True, threshold=0.5, require_grant=False)
await _engage_kill_switch(pool, None, reason="incident 4471")
closure_policy.reset_cache()
try:
await _persist(tenant_id=tenant_id, alert_id=alert_id)
assert (await _row(pool, alert_id))["status"] != "resolved"
finally:
await _release_kill_switches(pool)
closure_policy.reset_cache()
async def test_a_tenant_kill_switch_stops_only_that_tenant(self, pool) -> None:
from app.closure import policy as closure_policy
stopped = await _tenant(pool)
running = await _tenant(pool)
for tenant_id in (stopped, running):
await _policy(pool, tenant_id, enabled=True, threshold=0.5, require_grant=False)
await _engage_kill_switch(pool, stopped, reason="tuning")
closure_policy.reset_cache()
try:
stopped_alert = await _alert(pool, stopped)
await _persist(tenant_id=stopped, alert_id=stopped_alert)
assert (await _row(pool, stopped_alert))["status"] != "resolved"
closure_policy.reset_cache()
running_alert = await _alert(pool, running)
await _persist(tenant_id=running, alert_id=running_alert)
assert (await _row(pool, running_alert))["status"] == "resolved", "one tenant's stop control froze another tenant's closures"
finally:
await _release_kill_switches(pool)
closure_policy.reset_cache()
async def test_the_memory_path_obeys_the_kill_switch(self, pool) -> None:
"""The switch somebody reaches for at 3am did not reach this path at all."""
from app.closure import policy as closure_policy
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=True, threshold=0.5, require_grant=False)
await _engage_kill_switch(pool, None, reason="incident 4471")
closure_policy.reset_cache()
try:
await _persist(tenant_id=tenant_id, alert_id=alert_id, tier="memory", confidence=0.9)
assert (await _row(pool, alert_id))["status"] != "resolved", "the kill switch did not stop an outcome-memory closure"
finally:
await _release_kill_switches(pool)
closure_policy.reset_cache()
class TestShadowAndReplay:
async def test_a_shadow_run_leaves_production_status_untouched(self, pool) -> None:
"""Already fixed before this phase; pinned here so the chokepoint cannot regress it."""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
await _policy(pool, tenant_id, enabled=True, threshold=0.5, require_grant=False)
await _persist(tenant_id=tenant_id, alert_id=alert_id, shadow=True)
row = await _row(pool, alert_id)
assert row["status"] != "resolved"
assert row["disposition"] is None, "a shadow verdict landed in the column the analyst fills in"
class TestManualClosureIsUnaffected:
async def test_an_analyst_can_still_close_an_alert(self, pool) -> None:
"""The gate governs *automatic* closure only.
A human resolving an alert through the API writes this row directly
and must keep working for a tenant that has authorized no automation.
"""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
async with pool.acquire() as conn:
await conn.execute(
"UPDATE alerts SET status = 'closed', disposition = $2, resolved_at = now() WHERE id = $1::uuid",
alert_id,
"false_positive",
)
row = await _row(pool, alert_id)
assert row["status"] == "closed"
assert row["disposition"] == "false_positive"
async def test_an_unauthorized_verdict_is_still_recorded(self, pool) -> None:
"""Withholding the closure must not withhold the triage.
The alert stays open with the agent's verdict visible on it, which is
the difference between "held for a human" and "lost".
"""
tenant_id = await _tenant(pool)
alert_id = await _alert(pool, tenant_id)
written = await _persist(tenant_id=tenant_id, alert_id=alert_id)
assert written is True
row = await _row(pool, alert_id)
assert row["status"] == "triaged", "an alert held back from closure should still show as triaged"
assert row["ai_score"] == pytest.approx(0.99)