Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(fusion): stateful/windowed detection engine + ClickHouse skip-indexes (Wave 2) (#535)

Beenu Arora055f528564b5
diff --git a/services/api/clickhouse/001_init.sql b/services/api/clickhouse/001_init.sql
index 33c5d1aec..590b2799f 100644
--- a/services/api/clickhouse/001_init.sql
+++ b/services/api/clickhouse/001_init.sql
@@ -32,7 +32,15 @@ CREATE TABLE IF NOT EXISTS aisoc.raw_events (
ocsf_json String CODEC(ZSTD(3)),
mitre_techniques Array(String),
mitre_tactics Array(String),
- iocs Array(String)
+ iocs Array(String),
+ -- Data-skipping (bloom-filter) indexes so hunts over high-cardinality
+ -- needles (file hash, user, host, IOCs) skip granules instead of scanning
+ -- the whole ZSTD blob. GRANULARITY 4 = one index block per 4 * 8192 rows.
+ INDEX idx_hash hash_sha256 TYPE bloom_filter(0.01) GRANULARITY 4,
+ INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4,
+ INDEX idx_src_host src_hostname TYPE bloom_filter(0.01) GRANULARITY 4,
+ INDEX idx_iocs iocs TYPE bloom_filter(0.01) GRANULARITY 4,
+ INDEX idx_techniques mitre_techniques TYPE bloom_filter(0.01) GRANULARITY 4
)
-- ReplacingMergeTree collapses rows sharing the ORDER BY key, keeping the row
-- with the greatest ingest_time. Ingest now stamps a replay-stable event_id
diff --git a/services/fusion/app/core/config.py b/services/fusion/app/core/config.py
index 864fced03..4d3c436d0 100644
--- a/services/fusion/app/core/config.py
+++ b/services/fusion/app/core/config.py
@@ -46,6 +46,9 @@ class Settings(BaseSettings):
# live event stream (app/services/detection_engine.py). Each firing rule
# becomes a RawAlert routed through fusion.
detection_engine_enabled: bool = Field(default=True, alias="AISOC_DETECTION_ENGINE_ENABLED")
+ # Stateful / windowed detections (brute force, spray, scans) — Redis-backed
+ # sliding-window thresholds evaluated alongside the stateless corpus.
+ windowed_detection_enabled: bool = Field(default=True, alias="AISOC_WINDOWED_DETECTION_ENABLED")
# Phase A4 — consume the UEBA behavioral-anomaly stream and fold each
# entity's latest anomaly into alert confidence + anomaly score at fuse
diff --git a/services/fusion/app/main.py b/services/fusion/app/main.py
index 09e25e041..5231893ac 100644
--- a/services/fusion/app/main.py
+++ b/services/fusion/app/main.py
@@ -19,6 +19,7 @@ from app.services.entity_risk import EntityRiskEngine
from app.services.fusion_engine import FusionEngine
from app.services.lake_writer import LakeWriter
from app.services.ueba_signal import UebaSignalCache
+from app.services.windowed_detection import WindowedDetectionEngine
from app.workers.consumer import FusionWorker
@@ -81,7 +82,16 @@ async def lifespan(app: FastAPI):
)
# Phase A2 — evaluate the executable detection corpus against the stream.
detector = DetectionEngine() if settings.detection_engine_enabled else None
- worker = FusionWorker(engine, sink=sink, lake=lake, detector=detector, ueba_cache=ueba_cache)
+ # Wave 2 — windowed detections share the fusion Redis for sliding-window state.
+ windowed_detector = WindowedDetectionEngine(redis_client) if settings.windowed_detection_enabled else None
+ worker = FusionWorker(
+ engine,
+ sink=sink,
+ lake=lake,
+ detector=detector,
+ windowed_detector=windowed_detector,
+ ueba_cache=ueba_cache,
+ )
set_worker(worker)
# Start Kafka worker as a background task
diff --git a/services/fusion/app/services/windowed_detection.py b/services/fusion/app/services/windowed_detection.py
new file mode 100644
index 000000000..b393b1ef5
--- /dev/null
+++ b/services/fusion/app/services/windowed_detection.py
@@ -0,0 +1,194 @@
+"""Stateful / windowed detection engine (Wave 2).
+
+The live :class:`app.services.detection_engine.DetectionEngine` is stateless: it
+matches one event at a time. Whole classes of real attacks are only visible
+across MULTIPLE events in a time window — brute force, password spray, port
+scans, data-staging bursts. This engine adds sliding-window threshold detection:
+count events matching a rule, grouped by an entity, and fire once when the count
+crosses a threshold inside the window.
+
+State lives in Redis sorted sets (member = event id, score = epoch seconds) so
+the window is a cheap ``ZREMRANGEBYSCORE`` + ``ZCARD``, and a short-lived
+"fired" marker suppresses duplicate alerts for the same window. Everything is
+fail-soft: a Redis outage degrades to "windowed detections are skipped for the
+outage", never crashing the fusion pipeline.
+
+Only security-relevant bursts fire — a benign event that doesn't match a rule's
+``match_when`` never even touches Redis.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import uuid
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+
+from app.models.alert import AlertSeverity, RawAlert
+from app.services.detection_engine import DetectionHit
+from app.services.detection_matcher import matches
+
+logger = structlog.get_logger()
+
+_SEVERITY_MAP = {
+ "critical": AlertSeverity.CRITICAL,
+ "high": AlertSeverity.HIGH,
+ "medium": AlertSeverity.MEDIUM,
+ "low": AlertSeverity.LOW,
+ "info": AlertSeverity.INFO,
+}
+
+
+@dataclass(frozen=True)
+class WindowRule:
+ id: str
+ name: str
+ severity: str
+ category: str
+ mitre: list[str]
+ # Selects the events this rule counts (evaluated against recovered flat fields).
+ match_when: dict[str, Any]
+ # Flat field whose value is the entity the count is grouped by.
+ group_by: str
+ threshold: int
+ window_seconds: int
+
+
+# Built-in windowed rules. Intentionally small + high-signal; the corpus can grow
+# via the same JSON export path as the stateless engine later.
+_BUILTIN_RULES: tuple[WindowRule, ...] = (
+ WindowRule(
+ id="wd-bruteforce-auth",
+ name="Brute-force: repeated authentication failures",
+ severity="high",
+ category="identity",
+ mitre=["T1110"],
+ # Un-suffixed fields are plain-equality clauses in the matcher DSL.
+ match_when={"event_type": "authentication", "outcome": "failure"},
+ group_by="user",
+ threshold=5,
+ window_seconds=600,
+ ),
+ WindowRule(
+ id="wd-password-spray",
+ name="Password spray: auth failures across many accounts from one source",
+ severity="high",
+ category="identity",
+ mitre=["T1110.003"],
+ match_when={"event_type": "authentication", "outcome": "failure"},
+ group_by="src_ip",
+ threshold=10,
+ window_seconds=600,
+ ),
+ WindowRule(
+ id="wd-port-scan",
+ name="Port scan: many distinct connections from one source",
+ severity="medium",
+ category="network",
+ mitre=["T1046"],
+ match_when={"event_type": "network"},
+ group_by="src_ip",
+ threshold=50,
+ window_seconds=120,
+ ),
+)
+
+
+class WindowedDetectionEngine:
+ """Redis-backed sliding-window threshold detections."""
+
+ def __init__(self, redis: Any, rules: tuple[WindowRule, ...] = _BUILTIN_RULES, *, key_prefix: str = "aisoc:wd") -> None:
+ self._redis = redis
+ self._rules = rules
+ self._prefix = key_prefix
+
+ @property
+ def rule_count(self) -> int:
+ return len(self._rules)
+
+ @staticmethod
+ def _fields(message: dict[str, Any]) -> dict[str, Any]:
+ ocsf = message.get("ocsf_event")
+ if not isinstance(ocsf, dict):
+ return {}
+ raw = ocsf.get("raw_data")
+ if isinstance(raw, str) and raw.strip():
+ try:
+ parsed = json.loads(raw)
+ if isinstance(parsed, dict):
+ return parsed
+ except (ValueError, TypeError):
+ pass
+ return ocsf
+
+ async def evaluate(self, message: dict[str, Any]) -> list[DetectionHit]:
+ """Count this event into any matching window; return threshold-crossing hits."""
+ ocsf = message.get("ocsf_event")
+ if not isinstance(ocsf, dict):
+ return []
+ tenant = str(message.get("tenant_id") or ocsf.get("tenant_uid") or "")
+ if not tenant:
+ return []
+ fields = self._fields(message)
+ now = time.time()
+ hits: list[DetectionHit] = []
+ for rule in self._rules:
+ try:
+ if not matches(rule.match_when, fields):
+ continue
+ entity = fields.get(rule.group_by)
+ if not entity:
+ continue
+ if await self._observe_and_check(rule, tenant, str(entity), now):
+ hits.append(
+ DetectionHit(
+ rule_id=rule.id,
+ name=rule.name,
+ severity=rule.severity,
+ category=rule.category,
+ mitre=list(rule.mitre),
+ )
+ )
+ except Exception as exc: # noqa: BLE001 — one rule/Redis error must not wedge detection
+ logger.debug("windowed_detection.rule_error", rule=rule.id, error=str(exc))
+ return hits
+
+ async def _observe_and_check(self, rule: WindowRule, tenant: str, entity: str, now: float) -> bool:
+ key = f"{self._prefix}:{tenant}:{rule.id}:{entity}"
+ member = uuid.uuid4().hex
+ await self._redis.zadd(key, {member: now})
+ await self._redis.zremrangebyscore(key, 0, now - rule.window_seconds)
+ # Expire the key a window after the last event so idle entities are reaped.
+ await self._redis.expire(key, rule.window_seconds + 60)
+ count = await self._redis.zcard(key)
+ if count < rule.threshold:
+ return False
+ # Fire once per window: a short-lived marker suppresses re-firing on every
+ # subsequent event until the window rolls.
+ fired_key = f"{key}:fired"
+ already = await self._redis.set(fired_key, "1", nx=True, ex=rule.window_seconds)
+ return bool(already)
+
+ def build_alert(self, message: dict[str, Any], hit: DetectionHit) -> RawAlert | None:
+ ocsf = message.get("ocsf_event") or {}
+ tenant_raw = message.get("tenant_id") or ocsf.get("tenant_uid")
+ try:
+ tenant_id = uuid.UUID(str(tenant_raw))
+ except (ValueError, TypeError):
+ return None
+ fields = self._fields(message)
+ return RawAlert(
+ tenant_id=tenant_id,
+ source=f"detection:{hit.rule_id}",
+ title=hit.name,
+ description=f"Windowed detection {hit.rule_id} ({hit.category}) crossed its threshold.",
+ severity=_SEVERITY_MAP.get(hit.severity, AlertSeverity.MEDIUM),
+ src_ip=fields.get("src_ip"),
+ hostname=fields.get("hostname") or fields.get("host"),
+ username=fields.get("user"),
+ mitre_techniques=hit.mitre,
+ raw_event=ocsf,
+ )
diff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.py
index 4e293910b..813f0ec4e 100644
--- a/services/fusion/app/workers/consumer.py
+++ b/services/fusion/app/workers/consumer.py
@@ -21,6 +21,7 @@ from app.services.fusion_engine import FusionEngine
from app.services.lake_writer import LakeWriter
from app.services.promoter import promote_normalized_event
from app.services.ueba_signal import UebaSignalCache
+from app.services.windowed_detection import WindowedDetectionEngine
logger = structlog.get_logger()
@@ -49,12 +50,14 @@ class FusionWorker:
dlq: DeadLetterQueue | None = None,
lake: LakeWriter | None = None,
detector: DetectionEngine | None = None,
+ windowed_detector: WindowedDetectionEngine | None = None,
ueba_cache: UebaSignalCache | None = None,
) -> None:
self._engine = engine
self._sink = sink
self._lake = lake
self._detector = detector
+ self._windowed = windowed_detector
self._ueba_cache = ueba_cache
# A poison message must never vanish silently; default to a structured
# logging DLQ so persistence-free deployments still get the signal.
@@ -72,7 +75,7 @@ class FusionWorker:
topics = [settings.kafka_topic_alerts_raw]
# Subscribe to raw_events when EITHER promotion or lake archival needs
# it — the lake must fill even if promotion is turned off.
- if settings.event_promotion_enabled or self._lake is not None or self._detector is not None:
+ if settings.event_promotion_enabled or self._lake is not None or self._detector is not None or self._windowed is not None:
topics.append(settings.kafka_topic_raw_events)
# Phase A4 — also consume the UEBA behavioral-anomaly stream so the
# per-entity signal cache stays warm for fuse-time boosting.
@@ -234,6 +237,15 @@ class FusionWorker:
_METRICS["detected"] += 1
await self._fuse_and_persist(det_alert, source_event_id=validation.source_event_id)
+ # Wave 2 — stateful/windowed detections (brute force, spray, scans)
+ # count this event into its sliding window and fire once on threshold.
+ if self._windowed is not None:
+ for hit in await self._windowed.evaluate(payload):
+ win_alert = self._windowed.build_alert(payload, hit)
+ if win_alert is not None:
+ _METRICS["detected"] += 1
+ await self._fuse_and_persist(win_alert, source_event_id=validation.source_event_id)
+
# Ingest-normalized OCSF event — run the deterministic promotion
# policy (see app/services/promoter.py). Non-promoted events are
# dropped here by design; the detect stage (above) owns rule eval.
diff --git a/services/fusion/tests/test_windowed_detection.py b/services/fusion/tests/test_windowed_detection.py
new file mode 100644
index 000000000..9e4fa4e31
--- /dev/null
+++ b/services/fusion/tests/test_windowed_detection.py
@@ -0,0 +1,104 @@
+"""Wave 2 — stateful/windowed detection fires when a threshold is crossed in a
+sliding window, and only once per window."""
+
+from __future__ import annotations
+
+import json
+from uuid import uuid4
+
+import pytest
+from app.services.windowed_detection import WindowedDetectionEngine
+
+
+class _FakeRedis:
+ """Minimal in-memory Redis for the sorted-set window + fired marker."""
+
+ def __init__(self) -> None:
+ self.zsets: dict[str, dict[str, float]] = {}
+ self.strings: dict[str, str] = {}
+
+ async def zadd(self, key: str, mapping: dict[str, float]) -> None:
+ self.zsets.setdefault(key, {}).update(mapping)
+
+ async def zremrangebyscore(self, key: str, min_s: float, max_s: float) -> None:
+ z = self.zsets.get(key, {})
+ for m in [m for m, s in list(z.items()) if min_s <= s <= max_s]:
+ del z[m]
+
+ async def expire(self, key: str, ttl: int) -> None:
+ return None
+
+ async def zcard(self, key: str) -> int:
+ return len(self.zsets.get(key, {}))
+
+ async def set(self, key: str, value: str, nx: bool = False, ex: int | None = None):
+ if nx and key in self.strings:
+ return None
+ self.strings[key] = value
+ return True
+
+
+def _auth_fail_event(tenant: str, user: str = "alice", src: str = "1.2.3.4") -> dict:
+ return {
+ "tenant_id": tenant,
+ "ocsf_event": {
+ "tenant_uid": tenant,
+ "raw_data": json.dumps({"event_type": "authentication", "outcome": "failure", "user": user, "src_ip": src}),
+ },
+ }
+
+
+async def test_bruteforce_fires_at_threshold_once():
+ eng = WindowedDetectionEngine(_FakeRedis())
+ tenant = str(uuid4())
+
+ # First 4 failures: below the threshold of 5 -> no brute-force hit.
+ for i in range(4):
+ hits = await eng.evaluate(_auth_fail_event(tenant))
+ assert not any(h.rule_id == "wd-bruteforce-auth" for h in hits), f"fired early on attempt {i + 1}"
+
+ # 5th crosses the threshold.
+ hits = await eng.evaluate(_auth_fail_event(tenant))
+ assert any(h.rule_id == "wd-bruteforce-auth" for h in hits)
+
+ # 6th within the same window must NOT re-fire (fired-marker suppresses).
+ hits6 = await eng.evaluate(_auth_fail_event(tenant))
+ assert not any(h.rule_id == "wd-bruteforce-auth" for h in hits6)
+
+
+async def test_group_by_isolates_entities():
+ eng = WindowedDetectionEngine(_FakeRedis())
+ tenant = str(uuid4())
+ # 4 failures for alice + 4 for bob: neither user reaches the per-user threshold.
+ for _ in range(4):
+ await eng.evaluate(_auth_fail_event(tenant, user="alice"))
+ for _ in range(4):
+ hits = await eng.evaluate(_auth_fail_event(tenant, user="bob"))
+ assert not any(h.rule_id == "wd-bruteforce-auth" for h in hits)
+
+
+async def test_non_matching_event_never_fires():
+ eng = WindowedDetectionEngine(_FakeRedis())
+ tenant = str(uuid4())
+ benign = {"tenant_id": tenant, "ocsf_event": {"raw_data": json.dumps({"event_type": "process", "user": "alice"})}}
+ for _ in range(20):
+ hits = await eng.evaluate(benign)
+ assert hits == []
+
+
+async def test_build_alert_from_hit():
+ eng = WindowedDetectionEngine(_FakeRedis())
+ tenant = str(uuid4())
+ hit = None
+ for _ in range(5):
+ for h in await eng.evaluate(_auth_fail_event(tenant)):
+ hit = h
+ assert hit is not None
+ alert = eng.build_alert(_auth_fail_event(tenant), hit)
+ assert alert is not None
+ assert alert.source == "detection:wd-bruteforce-auth"
+ assert alert.username == "alice"