Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
import asyncio
import os
from contextlib import asynccontextmanager
from typing import Any
import redis.asyncio as aioredis
from fastapi import FastAPI
from app._health import install_health_routes, register_dependency, register_subscription
from app.api.router import router, set_worker
from app.core.config import settings
from app.core.logging import configure_logging, logger
from app.memory.provider import MemoryPriorProvider
from app.services import ioc_match
from app.services.ai_asset_recorder import AiAssetRecorder
from app.services.alert_enricher import AlertEnricher
from app.services.alert_outbox import AlertNotificationOutbox
from app.services.alert_sink import AlertSink
from app.services.asset_context import AssetContextResolver
from app.services.attack_chain_grouper import AttackChainGrouper
from app.services.baselines import BaselineStore
from app.services.confidence import ConfidenceScorer
from app.services.correlator import Correlator
from app.services.deduplicator import Deduplicator
from app.services.detection_engine import DetectionEngine
from app.services.dlq_sink import PostgresDLQ
from app.services.entity_risk import EntityRiskEngine
from app.services.first_seen import FirstSeenStore
from app.services.fusion_engine import FusionEngine
from app.services.identity_privilege_context import IdentityPrivilegeResolver
from app.services.ioc_match import TenantIocMatcher
from app.services.lake_writer import LakeWriter
from app.services.sessions import SessionStore
from app.services.tenant_overlay import OverlayCache
from app.services.ueba_signal import UebaSignalCache
from app.services.windowed_detection import WindowedDetectionEngine
from app.workers.consumer import FusionWorker
def _log_worker_exit(task: asyncio.Task) -> None:
"""Say why the consume loop stopped, whatever the reason.
A worker that ends cleanly is as much of a problem as one that raises:
either way nothing is consuming ``raw_events`` any more, and both used to
be silent.
"""
if task.cancelled():
logger.info("fusion.worker_cancelled")
return
exc = task.exception()
if exc is None:
logger.error("fusion.worker_exited", detail="consume loop returned; the subscription is gone")
return
logger.error("fusion.worker_died", error=str(exc), error_type=type(exc).__name__, exc_info=exc)
@asynccontextmanager
async def lifespan(app: FastAPI):
configure_logging()
logger.info("Starting AiSOC Alert Fusion Service", port=settings.http_port)
# Short socket timeouts are part of the Redis fallback: a black-holed
# connection must fail open into Postgres-backed dedup, not stall the
# alert path waiting for a TCP read that never arrives.
redis_client = aioredis.from_url(
settings.redis_url,
decode_responses=False,
socket_connect_timeout=settings.redis_connect_timeout_seconds,
socket_timeout=settings.redis_socket_timeout_seconds,
health_check_interval=30,
)
dedup = Deduplicator(redis_client)
correlator = Correlator(redis_client)
entity_risk = EntityRiskEngine(redis_client)
confidence_scorer = ConfidenceScorer(enabled=settings.confidence_enabled)
# Phase A4 — behavioral-model fusion: one cache shared by the engine
# (fuse-time lookup) and the worker (records the ueba.anomalies stream).
ueba_cache = UebaSignalCache(redis_client, ttl_seconds=settings.ueba_signal_ttl_seconds) if settings.ueba_fusion_enabled else None
# Phase C4 — fuse-time attack-chain grouping (shares the fusion Redis).
chain_grouper = (
AttackChainGrouper(redis_client, window_seconds=settings.attack_chain_window_seconds)
if settings.attack_chain_grouping_enabled
else None
)
# Wave 1 — fuse-time TI/vuln enrichment via the enrichment service.
# Parity 3.1. The matcher reads `threat_intel_iocs` from the Postgres
# this service already connects to, so it works on CORE where the
# enrichment service (a `full` profile component) does not run. Without
# it, `enrich()` caught a connection error, logged at debug and
# returned `{}`, and the investigation agent received "could not check"
# for every indicator on every alert.
ioc_matcher = TenantIocMatcher(settings.database_url) if ioc_match.enabled() else None
enricher = (
AlertEnricher(
base_url=settings.enrichment_service_url,
timeout_seconds=settings.fuse_enrichment_timeout_seconds,
malicious_risk_floor=settings.fuse_enrichment_risk_floor,
ioc_matcher=ioc_matcher,
)
if settings.fuse_enrichment_enabled
else None
)
# Wave 1 — live institutional-memory nudge: distil disposition history into
# per-tenant per-signature priors and feed them to the confidence scorer.
memory_provider = (
MemoryPriorProvider() if os.getenv("AISOC_FUSION_MEMORY_NUDGE", "1").strip().lower() not in {"0", "false", "no", "off"} else None
)
# Business-context plan PR-3 — fuse-time asset-context stamp. Same
# Postgres this service already connects to (the assets table the console
# import writes); unset DATABASE_URL leaves the scorer's factor honestly
# reading "unknown (no inventory match)".
asset_resolver = AssetContextResolver(settings.database_url) if settings.database_url else None
# Depth 2 wave 2, 1.1 — the actor's privilege as of the event, read from
# `identity_privilege_facts` (migration 124) in the same Postgres. Unset
# DATABASE_URL leaves every alert stamped "unknown", which is what a
# deployment that has collected no posture should see.
privilege_resolver = IdentityPrivilegeResolver(settings.database_url) if settings.database_url else None
engine = FusionEngine(
dedup,
correlator,
entity_risk=entity_risk,
confidence_scorer=confidence_scorer,
ueba_cache=ueba_cache,
chain_grouper=chain_grouper,
enricher=enricher,
asset_resolver=asset_resolver,
memory_provider=memory_provider,
privilege_resolver=privilege_resolver,
)
# Phase 3.1 — fused alerts land in the Postgres alert store so the spine
# is continuous (raw event → alert row). Fail-soft: a missing/unreachable
# DB never blocks the Kafka pipeline.
sink = AlertSink(settings.database_url) if settings.alert_sink_enabled else None
# Phase A1 — populate the ClickHouse event lake from the raw-events stream.
lake = (
LakeWriter(
host=settings.clickhouse_host,
port=settings.clickhouse_port,
database=settings.clickhouse_database,
user=settings.clickhouse_user,
password=settings.clickhouse_password,
batch_size=settings.lake_batch_size,
batch_max_age_seconds=settings.lake_batch_max_age_seconds,
)
if settings.lake_writer_enabled
else None
)
# Phase A2 — evaluate the executable detection corpus against the stream.
detector = DetectionEngine() if settings.detection_engine_enabled else None
# Wave 2 — windowed detections share the fusion Redis for sliding-window state.
windowed_detector = WindowedDetectionEngine(redis_client) if settings.windowed_detection_enabled else None
# Depth plan 3.3 — per-tenant first-seen memory, sharing the fusion Redis
# the windowed engine already uses for sliding-window state.
first_seen = FirstSeenStore(redis_client)
# Depth plan 3.4 — per-principal behavioural baselines in CORE, on the
# same Redis. ADR-0011 records why this is a fusion module rather than
# the `full`-profile UEBA service.
baselines = BaselineStore(redis_client)
# Dead letters go to Postgres so they can be read back. Until now the
# worker defaulted to LoggingDLQ, so a dropped event produced a log
# line and nothing else — and an invisible drop is indistinguishable
# from an event that never arrived.
#
# The pool is passed as a callable because `sink.start()` opens it
# during `worker.start()`, after this point; capturing it eagerly would
# capture None and drop every dead letter.
# `ensure_pool`/`invalidate_pool` matter here: the DLQ shares the sink's
# pool, and while the consumer is holding a message neither the alert path
# nor anything else reconnects. Without them a Postgres outage that
# invalidated the pool would turn the hold into a permanent stall.
dlq = (
PostgresDLQ(
lambda: sink._pool,
ensure_pool=sink.ensure_pool,
invalidate_pool=sink.invalidate_pool,
)
if sink is not None
else None
)
# Depth 6.1 — the observed AI estate. Shares the alert sink's pool for
# the same reason the dead-letter queue does: one Postgres connection
# pool per service, opened by `sink.start()` during `worker.start()`.
# Passed as a callable because that happens after this point, and
# capturing `sink._pool` eagerly would capture None and silently drop
# every sighting for the life of the process.
ai_assets = AiAssetRecorder(lambda: sink._pool) if sink is not None else None
# Remediation phase 4 — the recovery record for a fused alert that is
# durable in Postgres but whose Kafka notification never landed. Shares
# the sink's pool, and is passed as a callable for the same reason the
# two above are: `sink.start()` opens it during `worker.start()`, after
# this point. Without a sink there is no pool and no alert row either,
# so there is nothing for the journal to be about.
outbox = (
AlertNotificationOutbox(lambda: sink._pool, ensure_pool=sink.ensure_pool)
if sink is not None and settings.alert_outbox_enabled
else None
)
if sink is None:
# Explicit, not inferred: this mode writes no alert rows and has no
# durable DLQ. The consumer still retries and holds, but its committed
# offsets are not evidence that an alert survived. Development-only.
logger.warning(
"fusion.alert_persistence_disabled",
setting="AISOC_ALERT_SINK_ENABLED=false",
detail=(
"alerts are not persisted to Postgres and dead letters are log-only; "
"Kafka offsets commit after processing, not after durable capture"
),
)
worker = FusionWorker(
engine,
sink=sink,
dlq=dlq,
lake=lake,
detector=detector,
windowed_detector=windowed_detector,
first_seen=first_seen,
baselines=baselines,
# Parity 5.4. Per-tenant detection tuning as a versioned overlay
# with hot reload. Without it the engine evaluated the shared
# corpus and nothing else, so a tenant who disabled a noisy rule
# in the console kept receiving its alerts.
overlays=OverlayCache(_fusion_pool()),
# Depth plan 2.4. Sessions per principal: the stateful half of the
# projection `session.go` derives per event. Disabled by the same
# switch as the alert sink, because a session row nothing can join
# an alert to is a row nobody reads.
sessions=SessionStore(_fusion_pool()) if settings.alert_sink_enabled else None,
ueba_cache=ueba_cache,
ai_assets=ai_assets,
outbox=outbox,
)
set_worker(worker)
# Start Kafka worker as a background task.
#
# The done-callback is what makes a dead worker findable. The task is held
# on app.state, so it is never garbage-collected and asyncio never emits
# its "Task exception was never retrieved" warning — without this, a
# consume loop that died took the reason with it. The matching readiness
# probe below is what stops /readyz answering 200 afterwards.
worker_task = asyncio.create_task(worker.start())
worker_task.add_done_callback(_log_worker_exit)
register_subscription(app, "alerts+raw_events", lambda: worker.attached)
# Two different questions, which is why they are two probes and not one
# flag read twice. The subscription above is "am I consuming"; the
# dependency below is "can I reach the broker", and only the second can
# describe an outage -- `attached` stays true across one, because
# aiokafka's retry lives inside `__anext__` and the `async for` never
# exits. See `FusionWorker.broker_reachable`.
register_dependency(app, "kafka", worker.broker_reachable)
app.state.worker_task = worker_task
app.state.redis = redis_client
# Wave 1 — periodically distil disposition history into memory priors so
# the confidence nudge stays current (default every 6h). Best-effort.
async def _refresh_memory_priors() -> None:
interval = max(int(os.getenv("AISOC_FUSION_MEMORY_REFRESH_SECONDS", "21600")), 300)
while True:
try:
if memory_provider is not None and sink is not None:
pool = await sink._ensure_pool() # noqa: SLF001 — reuse the sink's pool
if pool is not None:
await memory_provider.refresh(pool)
except Exception as exc: # noqa: BLE001 — never crash the service on a refresh
logger.warning("memory_prior_refresh_failed", error=str(exc))
await asyncio.sleep(interval)
memory_task = asyncio.create_task(_refresh_memory_priors()) if memory_provider is not None else None
app.state.memory_task = memory_task
# Phase 2.6 — flip /readyz to 200 now that Redis is open + the
# Kafka consumer is running.
app.state.mark_ready()
logger.info("Alert Fusion Service ready")
yield
# Phase 2.6 — flip /readyz to 503 at the start of shutdown so the
# orchestrator stops sending traffic before we tear Kafka down.
app.state.mark_not_ready()
# Shutdown
logger.info("Shutting down Alert Fusion Service")
await worker.stop()
worker_task.cancel()
if memory_task is not None:
memory_task.cancel()
await redis_client.aclose()
logger.info("Alert Fusion Service stopped")
app = FastAPI(
title="AiSOC Alert Fusion Service",
description="Real-time alert deduplication and correlation engine",
version="0.1.0",
lifespan=lifespan,
)
# Phase 2.6 — k8s liveness + readiness probes (see app/_health.py).
_mark_ready, _mark_not_ready = install_health_routes(app, service_name="aisoc-fusion")
app.state.mark_ready = _mark_ready
app.state.mark_not_ready = _mark_not_ready
async def _postgres_ready() -> bool:
import asyncpg
dsn = settings.database_url
for prefix in ("postgresql+asyncpg://", "postgres+asyncpg://"):
if dsn.startswith(prefix):
dsn = "postgresql://" + dsn[len(prefix) :]
break
conn = await asyncpg.connect(dsn, timeout=1.4)
await conn.close()
return True
def _clickhouse_ready() -> bool:
if not settings.lake_writer_enabled:
return False
from clickhouse_driver import Client
client = Client(
host=settings.clickhouse_host,
port=settings.clickhouse_port,
database=settings.clickhouse_database,
user=settings.clickhouse_user,
password=settings.clickhouse_password or "",
connect_timeout=0.7,
send_receive_timeout=0.7,
)
try:
return client.execute("SELECT 1", settings={"max_execution_time": 1}) == [(1,)]
finally:
client.disconnect()
register_dependency(app, "database", _postgres_ready)
register_dependency(app, "lake_writer", _clickhouse_ready, required=False)
app.include_router(router)
_OVERLAY_POOL: Any = None
def _fusion_pool() -> Any:
"""One lazy Postgres pool, shared by the consumer's side readers.
Built on first use rather than at import, because `main` runs before
an event loop exists and `asyncpg.create_pool` needs one. The overlay
cache tolerates `None` and returns the empty overlay, so a deployment
with no database reachable detects exactly as it did before 5.4, and
the session store logs and returns None for the same reason.
Shared rather than one pool each: both sit on the consumer's hot path
against the same database, and two pools is two connection budgets to
size instead of one. Renamed from `_overlay_pool` when the session
store (depth plan 2.4) became the second reader, because a name that
says "overlay" while two things use it is how the next contributor
opens a third.
"""
global _OVERLAY_POOL # noqa: PLW0603
if _OVERLAY_POOL is None:
_OVERLAY_POOL = _LazyPool(settings.database_url)
return _OVERLAY_POOL
class _LazyPool:
"""Opens an asyncpg pool on first acquire."""
def __init__(self, dsn: str) -> None:
self._dsn = dsn
self._pool: Any = None
def acquire(self): # noqa: ANN201
return _LazyAcquire(self)
async def _ensure(self) -> Any:
if self._pool is not None:
return self._pool
import asyncpg
dsn = self._dsn
for prefix in ("postgresql+asyncpg://", "postgres+asyncpg://"):
if dsn.startswith(prefix):
dsn = "postgresql://" + dsn[len(prefix) :]
break
self._pool = await asyncpg.create_pool(dsn, min_size=1, max_size=2)
return self._pool
class _LazyAcquire:
def __init__(self, parent: _LazyPool) -> None:
self._parent = parent
self._ctx: Any = None
async def __aenter__(self) -> Any:
pool = await self._parent._ensure()
self._ctx = pool.acquire()
return await self._ctx.__aenter__()
async def __aexit__(self, *exc: Any) -> None:
if self._ctx is not None:
await self._ctx.__aexit__(*exc)