Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(fusion): stateful/windowed detection engine + ClickHouse skip-indexes (Wave 2) (#535)
Beenu Arora055f528564b5
diff --git a/services/api/clickhouse/001_init.sql b/services/api/clickhouse/001_init.sqlindex 33c5d1aec..590b2799f 100644--- a/services/api/clickhouse/001_init.sql+++ b/services/api/clickhouse/001_init.sql@@ -32,7 +32,15 @@ CREATE TABLE IF NOT EXISTS aisoc.raw_events (ocsf_json String CODEC(ZSTD(3)),mitre_techniques Array(String),mitre_tactics Array(String),- iocs Array(String)+ iocs Array(String),+ -- Data-skipping (bloom-filter) indexes so hunts over high-cardinality+ -- needles (file hash, user, host, IOCs) skip granules instead of scanning+ -- the whole ZSTD blob. GRANULARITY 4 = one index block per 4 * 8192 rows.+ INDEX idx_hash hash_sha256 TYPE bloom_filter(0.01) GRANULARITY 4,+ INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4,+ INDEX idx_src_host src_hostname TYPE bloom_filter(0.01) GRANULARITY 4,+ INDEX idx_iocs iocs TYPE bloom_filter(0.01) GRANULARITY 4,+ INDEX idx_techniques mitre_techniques TYPE bloom_filter(0.01) GRANULARITY 4)-- ReplacingMergeTree collapses rows sharing the ORDER BY key, keeping the row-- with the greatest ingest_time. Ingest now stamps a replay-stable event_iddiff --git a/services/fusion/app/core/config.py b/services/fusion/app/core/config.pyindex 864fced03..4d3c436d0 100644--- a/services/fusion/app/core/config.py+++ b/services/fusion/app/core/config.py@@ -46,6 +46,9 @@ class Settings(BaseSettings):# live event stream (app/services/detection_engine.py). Each firing rule# becomes a RawAlert routed through fusion.detection_engine_enabled: bool = Field(default=True, alias="AISOC_DETECTION_ENGINE_ENABLED")+ # Stateful / windowed detections (brute force, spray, scans) — Redis-backed+ # sliding-window thresholds evaluated alongside the stateless corpus.+ windowed_detection_enabled: bool = Field(default=True, alias="AISOC_WINDOWED_DETECTION_ENABLED")# Phase A4 — consume the UEBA behavioral-anomaly stream and fold each# entity's latest anomaly into alert confidence + anomaly score at fusediff --git a/services/fusion/app/main.py b/services/fusion/app/main.pyindex 09e25e041..5231893ac 100644--- a/services/fusion/app/main.py+++ b/services/fusion/app/main.py@@ -19,6 +19,7 @@ from app.services.entity_risk import EntityRiskEnginefrom app.services.fusion_engine import FusionEnginefrom app.services.lake_writer import LakeWriterfrom app.services.ueba_signal import UebaSignalCache+from app.services.windowed_detection import WindowedDetectionEnginefrom app.workers.consumer import FusionWorker@@ -81,7 +82,16 @@ async def lifespan(app: FastAPI):)# Phase A2 — evaluate the executable detection corpus against the stream.detector = DetectionEngine() if settings.detection_engine_enabled else None- worker = FusionWorker(engine, sink=sink, lake=lake, detector=detector, ueba_cache=ueba_cache)+ # Wave 2 — windowed detections share the fusion Redis for sliding-window state.+ windowed_detector = WindowedDetectionEngine(redis_client) if settings.windowed_detection_enabled else None+ worker = FusionWorker(+ engine,+ sink=sink,+ lake=lake,+ detector=detector,+ windowed_detector=windowed_detector,+ ueba_cache=ueba_cache,+ )set_worker(worker)# Start Kafka worker as a background taskdiff --git a/services/fusion/app/services/windowed_detection.py b/services/fusion/app/services/windowed_detection.pynew file mode 100644index 000000000..b393b1ef5--- /dev/null+++ b/services/fusion/app/services/windowed_detection.py@@ -0,0 +1,194 @@+"""Stateful / windowed detection engine (Wave 2).++The live :class:`app.services.detection_engine.DetectionEngine` is stateless: it+matches one event at a time. Whole classes of real attacks are only visible+across MULTIPLE events in a time window — brute force, password spray, port+scans, data-staging bursts. This engine adds sliding-window threshold detection:+count events matching a rule, grouped by an entity, and fire once when the count+crosses a threshold inside the window.++State lives in Redis sorted sets (member = event id, score = epoch seconds) so+the window is a cheap ``ZREMRANGEBYSCORE`` + ``ZCARD``, and a short-lived+"fired" marker suppresses duplicate alerts for the same window. Everything is+fail-soft: a Redis outage degrades to "windowed detections are skipped for the+outage", never crashing the fusion pipeline.++Only security-relevant bursts fire — a benign event that doesn't match a rule's+``match_when`` never even touches Redis.+"""++from __future__ import annotations++import json+import time+import uuid+from dataclasses import dataclass+from typing import Any++import structlog++from app.models.alert import AlertSeverity, RawAlert+from app.services.detection_engine import DetectionHit+from app.services.detection_matcher import matches++logger = structlog.get_logger()++_SEVERITY_MAP = {+ "critical": AlertSeverity.CRITICAL,+ "high": AlertSeverity.HIGH,+ "medium": AlertSeverity.MEDIUM,+ "low": AlertSeverity.LOW,+ "info": AlertSeverity.INFO,+}+++@dataclass(frozen=True)+class WindowRule:+ id: str+ name: str+ severity: str+ category: str+ mitre: list[str]+ # Selects the events this rule counts (evaluated against recovered flat fields).+ match_when: dict[str, Any]+ # Flat field whose value is the entity the count is grouped by.+ group_by: str+ threshold: int+ window_seconds: int+++# Built-in windowed rules. Intentionally small + high-signal; the corpus can grow+# via the same JSON export path as the stateless engine later.+_BUILTIN_RULES: tuple[WindowRule, ...] = (+ WindowRule(+ id="wd-bruteforce-auth",+ name="Brute-force: repeated authentication failures",+ severity="high",+ category="identity",+ mitre=["T1110"],+ # Un-suffixed fields are plain-equality clauses in the matcher DSL.+ match_when={"event_type": "authentication", "outcome": "failure"},+ group_by="user",+ threshold=5,+ window_seconds=600,+ ),+ WindowRule(+ id="wd-password-spray",+ name="Password spray: auth failures across many accounts from one source",+ severity="high",+ category="identity",+ mitre=["T1110.003"],+ match_when={"event_type": "authentication", "outcome": "failure"},+ group_by="src_ip",+ threshold=10,+ window_seconds=600,+ ),+ WindowRule(+ id="wd-port-scan",+ name="Port scan: many distinct connections from one source",+ severity="medium",+ category="network",+ mitre=["T1046"],+ match_when={"event_type": "network"},+ group_by="src_ip",+ threshold=50,+ window_seconds=120,+ ),+)+++class WindowedDetectionEngine:+ """Redis-backed sliding-window threshold detections."""++ def __init__(self, redis: Any, rules: tuple[WindowRule, ...] = _BUILTIN_RULES, *, key_prefix: str = "aisoc:wd") -> None:+ self._redis = redis+ self._rules = rules+ self._prefix = key_prefix++ @property+ def rule_count(self) -> int:+ return len(self._rules)++ @staticmethod+ def _fields(message: dict[str, Any]) -> dict[str, Any]:+ ocsf = message.get("ocsf_event")+ if not isinstance(ocsf, dict):+ return {}+ raw = ocsf.get("raw_data")+ if isinstance(raw, str) and raw.strip():+ try:+ parsed = json.loads(raw)+ if isinstance(parsed, dict):+ return parsed+ except (ValueError, TypeError):+ pass+ return ocsf++ async def evaluate(self, message: dict[str, Any]) -> list[DetectionHit]:+ """Count this event into any matching window; return threshold-crossing hits."""+ ocsf = message.get("ocsf_event")+ if not isinstance(ocsf, dict):+ return []+ tenant = str(message.get("tenant_id") or ocsf.get("tenant_uid") or "")+ if not tenant:+ return []+ fields = self._fields(message)+ now = time.time()+ hits: list[DetectionHit] = []+ for rule in self._rules:+ try:+ if not matches(rule.match_when, fields):+ continue+ entity = fields.get(rule.group_by)+ if not entity:+ continue+ if await self._observe_and_check(rule, tenant, str(entity), now):+ hits.append(+ DetectionHit(+ rule_id=rule.id,+ name=rule.name,+ severity=rule.severity,+ category=rule.category,+ mitre=list(rule.mitre),+ )+ )+ except Exception as exc: # noqa: BLE001 — one rule/Redis error must not wedge detection+ logger.debug("windowed_detection.rule_error", rule=rule.id, error=str(exc))+ return hits++ async def _observe_and_check(self, rule: WindowRule, tenant: str, entity: str, now: float) -> bool:+ key = f"{self._prefix}:{tenant}:{rule.id}:{entity}"+ member = uuid.uuid4().hex+ await self._redis.zadd(key, {member: now})+ await self._redis.zremrangebyscore(key, 0, now - rule.window_seconds)+ # Expire the key a window after the last event so idle entities are reaped.+ await self._redis.expire(key, rule.window_seconds + 60)+ count = await self._redis.zcard(key)+ if count < rule.threshold:+ return False+ # Fire once per window: a short-lived marker suppresses re-firing on every+ # subsequent event until the window rolls.+ fired_key = f"{key}:fired"+ already = await self._redis.set(fired_key, "1", nx=True, ex=rule.window_seconds)+ return bool(already)++ def build_alert(self, message: dict[str, Any], hit: DetectionHit) -> RawAlert | None:+ ocsf = message.get("ocsf_event") or {}+ tenant_raw = message.get("tenant_id") or ocsf.get("tenant_uid")+ try:+ tenant_id = uuid.UUID(str(tenant_raw))+ except (ValueError, TypeError):+ return None+ fields = self._fields(message)+ return RawAlert(+ tenant_id=tenant_id,+ source=f"detection:{hit.rule_id}",+ title=hit.name,+ description=f"Windowed detection {hit.rule_id} ({hit.category}) crossed its threshold.",+ severity=_SEVERITY_MAP.get(hit.severity, AlertSeverity.MEDIUM),+ src_ip=fields.get("src_ip"),+ hostname=fields.get("hostname") or fields.get("host"),+ username=fields.get("user"),+ mitre_techniques=hit.mitre,+ raw_event=ocsf,+ )diff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.pyindex 4e293910b..813f0ec4e 100644--- a/services/fusion/app/workers/consumer.py+++ b/services/fusion/app/workers/consumer.py@@ -21,6 +21,7 @@ from app.services.fusion_engine import FusionEnginefrom app.services.lake_writer import LakeWriterfrom app.services.promoter import promote_normalized_eventfrom app.services.ueba_signal import UebaSignalCache+from app.services.windowed_detection import WindowedDetectionEnginelogger = structlog.get_logger()@@ -49,12 +50,14 @@ class FusionWorker:dlq: DeadLetterQueue | None = None,lake: LakeWriter | None = None,detector: DetectionEngine | None = None,+ windowed_detector: WindowedDetectionEngine | None = None,ueba_cache: UebaSignalCache | None = None,) -> None:self._engine = engineself._sink = sinkself._lake = lakeself._detector = detector+ self._windowed = windowed_detectorself._ueba_cache = ueba_cache# A poison message must never vanish silently; default to a structured# logging DLQ so persistence-free deployments still get the signal.@@ -72,7 +75,7 @@ class FusionWorker:topics = [settings.kafka_topic_alerts_raw]# Subscribe to raw_events when EITHER promotion or lake archival needs# it — the lake must fill even if promotion is turned off.- if settings.event_promotion_enabled or self._lake is not None or self._detector is not None:+ if settings.event_promotion_enabled or self._lake is not None or self._detector is not None or self._windowed is not None:topics.append(settings.kafka_topic_raw_events)# Phase A4 — also consume the UEBA behavioral-anomaly stream so the# per-entity signal cache stays warm for fuse-time boosting.@@ -234,6 +237,15 @@ class FusionWorker:_METRICS["detected"] += 1await self._fuse_and_persist(det_alert, source_event_id=validation.source_event_id)+ # Wave 2 — stateful/windowed detections (brute force, spray, scans)+ # count this event into its sliding window and fire once on threshold.+ if self._windowed is not None:+ for hit in await self._windowed.evaluate(payload):+ win_alert = self._windowed.build_alert(payload, hit)+ if win_alert is not None:+ _METRICS["detected"] += 1+ await self._fuse_and_persist(win_alert, source_event_id=validation.source_event_id)+# Ingest-normalized OCSF event — run the deterministic promotion# policy (see app/services/promoter.py). Non-promoted events are# dropped here by design; the detect stage (above) owns rule eval.diff --git a/services/fusion/tests/test_windowed_detection.py b/services/fusion/tests/test_windowed_detection.pynew file mode 100644index 000000000..9e4fa4e31--- /dev/null+++ b/services/fusion/tests/test_windowed_detection.py@@ -0,0 +1,104 @@+"""Wave 2 — stateful/windowed detection fires when a threshold is crossed in a+sliding window, and only once per window."""++from __future__ import annotations++import json+from uuid import uuid4++import pytest+from app.services.windowed_detection import WindowedDetectionEngine+++class _FakeRedis:+ """Minimal in-memory Redis for the sorted-set window + fired marker."""++ def __init__(self) -> None:+ self.zsets: dict[str, dict[str, float]] = {}+ self.strings: dict[str, str] = {}++ async def zadd(self, key: str, mapping: dict[str, float]) -> None:+ self.zsets.setdefault(key, {}).update(mapping)++ async def zremrangebyscore(self, key: str, min_s: float, max_s: float) -> None:+ z = self.zsets.get(key, {})+ for m in [m for m, s in list(z.items()) if min_s <= s <= max_s]:+ del z[m]++ async def expire(self, key: str, ttl: int) -> None:+ return None++ async def zcard(self, key: str) -> int:+ return len(self.zsets.get(key, {}))++ async def set(self, key: str, value: str, nx: bool = False, ex: int | None = None):+ if nx and key in self.strings:+ return None+ self.strings[key] = value+ return True+++def _auth_fail_event(tenant: str, user: str = "alice", src: str = "1.2.3.4") -> dict:+ return {+ "tenant_id": tenant,+ "ocsf_event": {+ "tenant_uid": tenant,+ "raw_data": json.dumps({"event_type": "authentication", "outcome": "failure", "user": user, "src_ip": src}),+ },+ }+++async def test_bruteforce_fires_at_threshold_once():+ eng = WindowedDetectionEngine(_FakeRedis())+ tenant = str(uuid4())++ # First 4 failures: below the threshold of 5 -> no brute-force hit.+ for i in range(4):+ hits = await eng.evaluate(_auth_fail_event(tenant))+ assert not any(h.rule_id == "wd-bruteforce-auth" for h in hits), f"fired early on attempt {i + 1}"++ # 5th crosses the threshold.+ hits = await eng.evaluate(_auth_fail_event(tenant))+ assert any(h.rule_id == "wd-bruteforce-auth" for h in hits)++ # 6th within the same window must NOT re-fire (fired-marker suppresses).+ hits6 = await eng.evaluate(_auth_fail_event(tenant))+ assert not any(h.rule_id == "wd-bruteforce-auth" for h in hits6)+++async def test_group_by_isolates_entities():+ eng = WindowedDetectionEngine(_FakeRedis())+ tenant = str(uuid4())+ # 4 failures for alice + 4 for bob: neither user reaches the per-user threshold.+ for _ in range(4):+ await eng.evaluate(_auth_fail_event(tenant, user="alice"))+ for _ in range(4):+ hits = await eng.evaluate(_auth_fail_event(tenant, user="bob"))+ assert not any(h.rule_id == "wd-bruteforce-auth" for h in hits)+++async def test_non_matching_event_never_fires():+ eng = WindowedDetectionEngine(_FakeRedis())+ tenant = str(uuid4())+ benign = {"tenant_id": tenant, "ocsf_event": {"raw_data": json.dumps({"event_type": "process", "user": "alice"})}}+ for _ in range(20):+ hits = await eng.evaluate(benign)+ assert hits == []+++async def test_build_alert_from_hit():+ eng = WindowedDetectionEngine(_FakeRedis())+ tenant = str(uuid4())+ hit = None+ for _ in range(5):+ for h in await eng.evaluate(_auth_fail_event(tenant)):+ hit = h+ assert hit is not None+ alert = eng.build_alert(_auth_fail_event(tenant), hit)+ assert alert is not None+ assert alert.source == "detection:wd-bruteforce-auth"+ assert alert.username == "alice"