Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""End-to-end event-spine test against real containers (Phase 3.1).
Proves the product's central claim with zero mocks:
raw vendor event
→ POST /v1/ingest/batch (services/ingest, Go)
→ OCSF normalize (services/ingest)
→ Kafka `aisoc.raw_events`
→ promotion + dedup + correlate (services/fusion)
→ Kafka `aisoc.alerts.fused`
→ WebSocket `alert.fused` frame (services/realtime)
→ Postgres `alerts` row (services/fusion AlertSink)
→ GET /api/v1/alerts (services/api)
Also asserts duplicate suppression: re-submitting the same event must NOT
create a second alert row.
Run against a booted docker-compose stack (see .github/workflows/integration.yml):
python3 scripts/integration/spine_test.py # full spine test
python3 scripts/integration/spine_test.py --post-event --title "chaos-1"
python3 scripts/integration/spine_test.py --await-alert --title "chaos-1"
The --post-event / --await-alert modes are used by the chaos step (kill the
fusion consumer, produce, restart, assert recovery).
Dependencies: httpx, websockets (installed by the workflow).
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import sys
import time
import uuid
import httpx
import websockets
API = "http://localhost:8000"
INGEST = "http://localhost:8081"
FUSION = "http://localhost:8003"
REALTIME_HTTP = "http://localhost:8086"
REALTIME_WS = "ws://localhost:8086"
TENANT = "00000000-0000-0000-0000-000000000001"
#: /v1/ingest is authenticated and the tenant is derived from the
#: credential, so this test has to hold one like any other pusher. The
#: workflow mints it with `python -m app.scripts.mint_ingest_token` and
#: exports it here. Required rather than optional: an unauthenticated
#: fallback would mean the spine test stopped exercising the path a real
#: push takes.
INGEST_TOKEN = os.environ.get("AISOC_INGEST_TOKEN", "").strip()
HEALTH_BUDGET_S = 300
SPINE_BUDGET_S = 180
REPOST_INTERVAL_S = 20
def log(msg: str) -> None:
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
#: The bearer the API reads and the realtime ticket need. These calls used
#: to carry no credential at all and succeeded, because an uncredentialed
#: request resolved to a demo administrator and every documented path
#: produced one. Minted by `app.scripts.mint_api_token`; the workflow sets it.
API_TOKEN = os.environ.get("AISOC_API_TOKEN", "").strip()
def api_headers() -> dict[str, str]:
"""Headers for a call to the API. Empty when no token was supplied, so a
401 names the missing variable rather than looking like an outage."""
return {"Authorization": f"Bearer {API_TOKEN}"} if API_TOKEN else {}
def ingest_headers() -> dict[str, str]:
"""Credential plus the tenant the credential is expected to cover.
The tenant header is not authority — the ingest service intersects it
with the token's own scope — but sending it keeps this test honest
about which tenant it believes it is writing for.
"""
if not INGEST_TOKEN:
raise SystemExit(
"AISOC_INGEST_TOKEN is unset. /v1/ingest requires a credential; mint one with\n"
" docker compose run --rm -T api python -m app.scripts.mint_ingest_token --quiet"
)
return {
"X-Tenant-ID": TENANT,
"Authorization": f"Bearer {INGEST_TOKEN}",
}
async def wait_healthy(client: httpx.AsyncClient) -> None:
"""Block until every spine service answers its health probe."""
probes = {
"api": f"{API}/readyz",
"ingest": f"{INGEST}/health",
"fusion": f"{FUSION}/readyz",
"realtime": f"{REALTIME_HTTP}/health",
}
deadline = time.monotonic() + HEALTH_BUDGET_S
pending = dict(probes)
while pending and time.monotonic() < deadline:
for name, url in list(pending.items()):
try:
r = await client.get(url, timeout=5)
if r.status_code == 200:
log(f"healthy: {name}")
del pending[name]
except httpx.HTTPError:
# Service not up yet — keep it pending and retry on the next loop.
pass
if pending:
await asyncio.sleep(3)
if pending:
raise SystemExit(f"services never became healthy: {sorted(pending)}")
def event_payload(title: str) -> dict:
"""A CrowdStrike-shaped critical finding. `event_simpleName` maps to
OCSF `activity_name`, which the fusion promoter uses as the alert title."""
return {
"connector_id": "spine-test",
"connector_type": "crowdstrike_falcon",
"source_format": "json",
"events": [
{
"event_simpleName": title,
"ComputerName": "SPINE-TEST-HOST",
"UserName": "spine-test-user",
"SHA256HashData": "f" * 64,
"severity": "Critical",
"technique_id": "T1003",
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
],
}
async def post_event(client: httpx.AsyncClient, title: str) -> None:
r = await client.post(
f"{INGEST}/v1/ingest/batch",
json=event_payload(title),
headers=ingest_headers(),
timeout=10,
)
r.raise_for_status()
body = r.json()
if body.get("accepted") != 1:
raise SystemExit(f"ingest rejected the event: {body}")
log(f"ingest accepted event title={title!r}")
# Phase A2 — an event whose recovered raw fields match the native
# `aws-root-account-login` detection rule. The rule (product `aws`) fires in
# the fusion detection engine and produces an alert titled below, proving the
# executable corpus runs against the live stream (not just CI fixtures).
DETECTION_ALERT_TITLE = "AWS Root Account Console Login"
async def post_detection_event(client: httpx.AsyncClient) -> None:
payload = {
"connector_id": "spine-detection-test",
"connector_type": "aws_cloudtrail",
"source_format": "json",
"events": [
{
"event_name": "ConsoleLogin",
"user_type": "Root",
"error_code": None,
"src_ip": "203.0.113.7",
"severity": "critical",
}
],
}
r = await client.post(
f"{INGEST}/v1/ingest/batch",
json=payload,
headers=ingest_headers(),
timeout=10,
)
r.raise_for_status()
body = r.json()
if body.get("accepted") != 1:
raise SystemExit(f"ingest rejected the detection event: {body}")
log("ingest accepted detection-triggering event (aws root login)")
async def count_alerts_titled(client: httpx.AsyncClient, title: str) -> int:
r = await client.get(f"{API}/api/v1/alerts", params={"page_size": 200}, headers=api_headers(), timeout=10)
r.raise_for_status()
items = r.json().get("items", [])
return sum(1 for a in items if a.get("title") == title)
async def get_alert_titled(client: httpx.AsyncClient, title: str) -> dict | None:
r = await client.get(f"{API}/api/v1/alerts", params={"page_size": 200}, headers=api_headers(), timeout=10)
r.raise_for_status()
for a in r.json().get("items", []):
if a.get("title") == title:
return a
return None
async def mint_ws_ticket(client: httpx.AsyncClient) -> str:
r = await client.post(f"{API}/api/v1/realtime/ticket", headers=api_headers(), timeout=10)
r.raise_for_status()
return r.json()["token"]
async def ws_listener(ticket: str, title: str, found: asyncio.Event) -> None:
"""Listen on the alerts channel until a fused frame carrying `title` arrives."""
url = f"{REALTIME_WS}/ws/alerts?token={ticket}"
async with websockets.connect(url, open_timeout=15) as ws:
log("websocket connected")
while not found.is_set():
raw = await ws.recv()
try:
frame = json.loads(raw)
except (TypeError, ValueError):
continue
if frame.get("type") != "alert.fused":
continue
payload = frame.get("payload") or {}
alert = payload.get("alert") or {}
if alert.get("title") == title:
log("websocket frame received for spine event")
found.set()
async def await_alert_row(client: httpx.AsyncClient, title: str, timeout_s: int, repost: bool) -> None:
"""Poll the API until the alert row exists. Optionally re-POST the event to
ride out consumer-group assignment races (dedup keeps this idempotent).
Transient HTTP errors are swallowed and retried rather than fatal: during
the Postgres-outage chaos step the API and fusion are mid-reconnect, so a
5xx blip must extend the poll window, not fail the whole test."""
deadline = time.monotonic() + timeout_s
last_post = time.monotonic()
while time.monotonic() < deadline:
try:
if await count_alerts_titled(client, title) >= 1:
log("alert row visible via GET /api/v1/alerts")
return
if repost and time.monotonic() - last_post > REPOST_INTERVAL_S:
await post_event(client, title)
last_post = time.monotonic()
except httpx.HTTPError as exc:
log(f"transient error while polling (will retry): {exc}")
await asyncio.sleep(3)
raise SystemExit(f"alert row for {title!r} never appeared within {timeout_s}s")
async def full_spine_test() -> None:
title = f"SpineTest-{uuid.uuid4().hex[:12]}"
async with httpx.AsyncClient() as client:
await wait_healthy(client)
# WS listener must be up before the event flows.
ticket = await mint_ws_ticket(client)
ws_found = asyncio.Event()
ws_task = asyncio.create_task(ws_listener(ticket, title, ws_found))
# Give the fusion + realtime consumer groups a moment to be assigned
# partitions after boot; the repost loop below covers the rest.
await asyncio.sleep(5)
started = time.monotonic()
await post_event(client, title)
await await_alert_row(client, title, SPINE_BUDGET_S, repost=True)
elapsed = time.monotonic() - started
log(f"raw event → alert row in {elapsed:.1f}s")
# WS frame: every fused alert (including dedup republish) is broadcast,
# and the repost loop keeps producing until the row exists, so the
# frame must arrive promptly if the realtime consumer is alive.
try:
await asyncio.wait_for(ws_found.wait(), timeout=60)
except TimeoutError:
raise SystemExit("no alert.fused WebSocket frame received within 60s") from None
finally:
ws_task.cancel()
# Alert content assertions — the row must carry the fused pipeline's
# fingerprint of the original telemetry, not a bare title.
alert = await get_alert_titled(client, title)
assert alert is not None
if alert.get("severity") != "critical":
raise SystemExit(f"alert.severity={alert.get('severity')!r}, expected 'critical'")
# 'new' is the state the row is born in; 'triaged' means auto-triage
# already reached it, which depth 4.4's per-entity concurrency now
# routinely manages inside this test's own window. Both prove the
# spine delivered the alert; anything else is a real wrong state.
if alert.get("status") not in ("new", "triaged"):
raise SystemExit(f"alert.status={alert.get('status')!r}, expected 'new' or 'triaged'")
techniques = alert.get("mitre_techniques") or []
if "T1003" not in techniques:
raise SystemExit(f"ATT&CK enrichment lost in transit: {techniques}")
# Duplicate suppression: same event again must NOT create a second row.
await post_event(client, title)
await asyncio.sleep(15)
n = await count_alerts_titled(client, title)
if n != 1:
raise SystemExit(f"duplicate suppression failed: {n} rows for {title!r}")
log("duplicate suppression verified (1 row after re-submit)")
log("SPINE TEST PASSED")
async def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--post-event", action="store_true", help="only POST an event")
parser.add_argument("--await-alert", action="store_true", help="only await the alert row")
parser.add_argument("--title", help="event title for --post-event / --await-alert")
parser.add_argument("--timeout", type=int, default=120)
parser.add_argument(
"--detection",
action="store_true",
help="Phase A2: post a detection-rule-triggering event and await the resulting alert",
)
args = parser.parse_args()
if args.detection:
async with httpx.AsyncClient() as client:
await wait_healthy(client)
await asyncio.sleep(5)
# Re-post periodically to ride out consumer-group assignment races,
# exactly like the spine test (the detection alert is dedup-guarded
# so repeats never create duplicate rows).
timeout_s = args.timeout or 180
deadline = time.monotonic() + timeout_s
last_post = 0.0
while time.monotonic() < deadline:
try:
if time.monotonic() - last_post > REPOST_INTERVAL_S:
await post_detection_event(client)
last_post = time.monotonic()
if await count_alerts_titled(client, DETECTION_ALERT_TITLE) >= 1:
log("DETECTION ENGINE FIRED (live-stream rule match produced an alert)")
return
except httpx.HTTPError as exc:
log(f"transient error while polling (will retry): {exc}")
await asyncio.sleep(3)
raise SystemExit(f"detection alert {DETECTION_ALERT_TITLE!r} never appeared within {timeout_s}s")
if args.post_event or args.await_alert:
if not args.title:
parser.error("--title is required with --post-event / --await-alert")
async with httpx.AsyncClient() as client:
if args.post_event:
await post_event(client, args.title)
if args.await_alert:
await await_alert_row(client, args.title, args.timeout, repost=False)
return
await full_spine_test()
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
sys.exit(130)