Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""Compile Sigma correlation rules into the windowed engine's form.
Specification: https://github.com/SigmaHQ/sigma-specification — "Sigma
Correlations", the ``correlation:`` document type. Read before writing this;
every field name and correlation type below is taken from it rather than
guessed.
What the engine can and cannot express
--------------------------------------
The windowed engine counts events or distinct values for one entity over a
sliding window, and stages ordered or unordered sets of events for one entity
over a sliding window. That covers four of Sigma's correlation types:
``event_count`` → a counting rule.
``value_count`` → a counting rule with ``distinct_by``.
``temporal`` → an unordered staged rule.
``temporal_ordered`` → an ordered staged rule.
It does not cover the rest, and the rule here is **refuse rather than
approximate**. A correlation that is translated with one of its constraints
dropped is a different detection shipping under the original's name and
severity, which is the specific failure the windowed translation work in
depth plan 3.1 existed to stop. Every refusal carries a reason, and the
refusals are counted and published the same way the Sigma compiler's are.
The two refusals worth knowing about in advance, because they look like
oversights:
* **A ``lt`` / ``lte`` condition is a rarity signal.** "Fewer than five in an
hour" fires when nothing much happened, so it needs a store of what is
normal rather than a counter. The engine fires when a count is high.
* **A multi-field ``group-by`` has no single entity.** The engine accumulates
against one field. Picking the first would group by something the rule
author did not ask for and silently widen or narrow every count.
"""
from __future__ import annotations
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
__all__ = [
"CORRELATION_TYPES",
"CorrelationRefusal",
"compile_correlation",
"parse_timespan",
]
#: Correlation types this compiler accepts, mapped to the shape they become.
CORRELATION_TYPES: dict[str, str] = {
"event_count": "counting",
"value_count": "counting",
"temporal": "staged",
"temporal_ordered": "staged",
}
R_UNKNOWN_TYPE = "correlation type is not one this engine can express"
R_CONDITION_SHAPE = "the condition is not a single comparison the engine can threshold"
R_RARITY = "a `lt`/`lte` condition fires when a count is low, which needs a baseline rather than a counter"
R_EQ = "an `eq` condition fires on an exact count, and a sliding window cannot hold a count still"
R_MULTI_GROUP = "`group-by` names more than one field and the engine accumulates against one entity"
R_NO_GROUP = "`group-by` is absent, so the count has no entity and would be global"
R_TIMESPAN = "`timespan` is missing or not a Sigma duration"
R_NO_RULES = "`rules` names no referenced rule"
R_UNRESOLVED = "a referenced rule is not in the corpus or did not compile"
R_ONE_STAGE = "a temporal correlation over a single rule is a plain count, not a sequence"
R_VALUE_FIELD = "`value_count` needs a `field` naming what to count distinctly"
R_ALIASES = "`aliases` map different field names across referenced rules and the engine has no aliasing"
R_GENERATE = "`generate: true` also emits the referenced rules, which this path does not model"
R_CHAINED = "the correlation references another correlation, which the engine cannot stage"
#: Sigma duration suffixes, from the specification.
_UNITS: dict[str, int] = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
class CorrelationRefusal(Exception):
"""This correlation cannot be expressed without changing what it detects."""
def __init__(self, reason: str, detail: str = "") -> None:
super().__init__(f"{reason}{f' ({detail})' if detail else ''}")
self.reason = reason
self.detail = detail
def parse_timespan(raw: Any) -> int:
"""Sigma duration (``5m``, ``1h``, ``30s``) to seconds."""
text = str(raw or "").strip()
if len(text) < 2 or text[-1] not in _UNITS or not text[:-1].isdigit():
raise CorrelationRefusal(R_TIMESPAN, text or "absent")
seconds = int(text[:-1]) * _UNITS[text[-1]]
if seconds < 1:
raise CorrelationRefusal(R_TIMESPAN, text)
return seconds
@dataclass(frozen=True)
class _Condition:
operator: str
value: int
def _parse_condition(block: Any, correlation_type: str) -> _Condition:
"""The ``condition:`` mapping, which Sigma writes as one comparison."""
if not isinstance(block, dict) or len(block) != 1:
raise CorrelationRefusal(R_CONDITION_SHAPE, repr(block))
((operator, value),) = block.items()
operator = str(operator).lower()
if operator in {"lt", "lte"}:
raise CorrelationRefusal(R_RARITY, f"{operator}: {value}")
if operator == "eq":
raise CorrelationRefusal(R_EQ, f"eq: {value}")
if operator not in {"gt", "gte"}:
raise CorrelationRefusal(R_CONDITION_SHAPE, operator)
try:
bound = int(value)
except (TypeError, ValueError) as exc:
raise CorrelationRefusal(R_CONDITION_SHAPE, f"{operator}: {value!r}") from exc
if bound < 0:
raise CorrelationRefusal(R_CONDITION_SHAPE, f"{operator}: {bound}")
# `correlation_type` is carried so a later type with different threshold
# semantics cannot silently reuse this.
del correlation_type
return _Condition(operator=operator, value=bound)
def _group_by(block: Any) -> str:
fields = block if isinstance(block, list) else [block] if block else []
names = [str(f) for f in fields if str(f or "").strip()]
if not names:
raise CorrelationRefusal(R_NO_GROUP)
if len(names) > 1:
raise CorrelationRefusal(R_MULTI_GROUP, ", ".join(names))
return names[0]
def _referenced(block: dict[str, Any]) -> list[str]:
"""`rules:` lives inside the correlation block, per the specification."""
raw = block.get("rules")
names = [str(r) for r in (raw if isinstance(raw, list) else [raw] if raw else []) if str(r or "").strip()]
if not names:
raise CorrelationRefusal(R_NO_RULES)
return names
def compile_correlation(
doc: dict[str, Any],
*,
selectors: dict[str, dict[str, Any]],
correlations: frozenset[str] = frozenset(),
) -> dict[str, Any]:
"""One Sigma correlation document to one windowed rule.
``selectors`` maps a referenced rule's name (or id) to the ``match_when``
it compiled to. A correlation whose referenced rule is missing is refused
rather than compiled against an empty selector — an empty selector
matches every event, so that refusal is the difference between a rule and
an outage.
"""
block = doc.get("correlation")
if not isinstance(block, dict):
raise CorrelationRefusal(R_UNKNOWN_TYPE, "no correlation block")
correlation_type = str(block.get("type") or "").strip().lower()
if correlation_type not in CORRELATION_TYPES:
raise CorrelationRefusal(R_UNKNOWN_TYPE, correlation_type or "absent")
if block.get("aliases"):
raise CorrelationRefusal(R_ALIASES)
if block.get("generate"):
raise CorrelationRefusal(R_GENERATE)
window_seconds = parse_timespan(block.get("timespan"))
group_by = _group_by(block.get("group-by"))
names = _referenced(block)
if any(name in correlations for name in names):
raise CorrelationRefusal(R_CHAINED, ", ".join(n for n in names if n in correlations))
missing = [name for name in names if name not in selectors]
if missing:
raise CorrelationRefusal(R_UNRESOLVED, ", ".join(missing))
common = {
"id": str(doc.get("id") or "").strip(),
"name": str(doc.get("title") or doc.get("name") or "").strip(),
"severity": str(doc.get("level") or doc.get("severity") or "medium").strip().lower(),
"category": str(doc.get("category") or "identity").strip().lower(),
"mitre": sorted({str(t).upper().removeprefix("ATTACK.") for t in doc.get("tags") or [] if str(t).lower().startswith("attack.t")}),
"group_by": group_by,
"window_seconds": window_seconds,
"correlation_type": correlation_type,
"correlates": names,
}
if not common["id"] or not common["name"]:
raise CorrelationRefusal(R_CONDITION_SHAPE, "the document has no id or title")
if CORRELATION_TYPES[correlation_type] == "counting":
if len(names) != 1:
# Sigma allows several, meaning "events matching any of them".
# The engine's `match_when` is one clause set, and an `any_of`
# over compiled selectors is a faithful translation only when the
# selectors are disjoint, which nothing here can establish.
raise CorrelationRefusal(R_CONDITION_SHAPE, f"{correlation_type} over {len(names)} rules")
condition = _parse_condition(block.get("condition"), correlation_type)
rule: dict[str, Any] = {
**common,
"match_when": dict(selectors[names[0]]),
# `gt: N` crosses at N+1; `gte: N` crosses at N.
"threshold": condition.value + 1 if condition.operator == "gt" else condition.value,
}
if correlation_type == "value_count":
field = str(block.get("field") or "").strip()
if not field:
raise CorrelationRefusal(R_VALUE_FIELD)
rule["distinct_by"] = field
if rule["threshold"] < 1:
raise CorrelationRefusal(R_CONDITION_SHAPE, "threshold below one fires on the first event")
return rule
if len(names) < 2:
raise CorrelationRefusal(R_ONE_STAGE, names[0])
return {
**common,
"sequence": [dict(selectors[name]) for name in names],
"ordered": correlation_type == "temporal_ordered",
}