Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""Curate the AiSOC v1.0 buyer-promised detection set.
The repo contains thousands of detection files across native (``detections/<category>/``)
and four upstream import tiers (``sigma-imports/``, ``car-imports/``,
``splunk-imports/``, ``chronicle-imports/``). Most imported rules are
``_quarantine``-d because their query language isn't directly executable
yet. The buyer-value plan (WS-B2) requires us to *curate* a focused set
of **≥ 300 MITRE-mapped, executable detections** that explicitly cover
eight buyer-prioritised threat families:
ransomware, credential-access, lateral, exfil, cloud, identity,
supply-chain, k8s
This script does the curation in three passes:
1. **Discovery** — walk every ``*.yaml`` / ``*.yml`` under ``detections/``
excluding ``_quarantine`` and ``fixtures``. Pull MITRE technique IDs,
severity, category, and provenance from each rule.
2. **Classification** — for each rule decide which of the eight families
it covers. Mapping is deterministic and uses three signals:
- explicit MITRE technique IDs (matched against curated family
technique sets, see ``FAMILIES`` below),
- native rule category (``cloud``, ``identity``, ``data-exfil``,
etc.) — used to bucket buyer-meaningful categories regardless of
whether the technique tag is present (e.g. cloud rules without
MITRE tags still belong to *cloud*),
- log source product name (``kubernetes`` → k8s, ``aws.ec2`` →
cloud, etc.).
3. **Quality scoring + selection** — score each rule by
``has_mitre × has_severity × has_executable_body`` and select the
top-scoring rules per family until each family has at least its
target floor (configurable, default 25 per family). Ties are broken
alphabetically for determinism.
The output is written to:
- ``marketplace/curated.json`` — machine-readable curation manifest
consumed by the marketplace UI's "Curated v1.0" tab.
- ``apps/docs/docs/detections/coverage.md`` — public coverage
report with per-family counts, MITRE technique density, and
a flat list of curated rule IDs (so reviewers can audit the
promised set without reading JSON).
Usage::
python3 scripts/curate_detections.py # build & write
python3 scripts/curate_detections.py --check # CI: no-op verify
python3 scripts/curate_detections.py --print # stdout JSON
python3 scripts/curate_detections.py --min 400 # raise minimum
Exit codes:
0 — curation succeeded and on-disk artefacts match (or were updated)
1 — quality gate failed (couldn't reach minimum, family uncovered)
2 — drift: ``--check`` and on-disk artefacts disagree
The script is idempotent and side-effect-free outside the two output
files. It does **not** delete or move any detection rules — pruning a
rule means it doesn't appear in the curated manifest, not that the
file is removed.
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import re
import sys
from collections.abc import Iterable
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
from typing import Any
import yaml
# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from gate_toolkit import repo_root, self_test_if_requested
self_test_if_requested(__file__)
REPO_ROOT = repo_root()
DETECTIONS_DIR = REPO_ROOT / "detections"
OUT_MANIFEST = REPO_ROOT / "marketplace" / "curated.json"
OUT_REPORT = REPO_ROOT / "apps" / "docs" / "docs" / "detections" / "coverage.md"
# ─── MITRE family definitions ────────────────────────────────────────────────
#
# Each entry maps a buyer-meaningful threat family to the MITRE ATT&CK
# techniques that *most directly* indicate that family. We keep this
# tight on purpose — claiming a rule covers "ransomware" because it
# tags T1059 (Command and Scripting Interpreter) is dishonest. We only
# match a family if the rule carries one of the family-specific
# techniques *or* (for category-based families like cloud/identity) the
# rule lives in the corresponding native category.
#
# Sources:
# - MITRE ATT&CK Enterprise v15 matrix (ransomware techniques per the
# ATT&CK Group g0046 / S0372 ransomware playbook references).
# - SigmaHQ tag conventions for k8s / supply-chain coverage.
#
# Tactic IDs are included in ``tactics`` so rules tagged at the tactic
# level (``attack.exfiltration``) without a specific technique still
# get bucketed correctly — at the cost of a slightly looser match.
# This trade-off is deliberate: imported Sigma rules sometimes only
# carry the tactic tag.
FAMILIES: dict[str, dict[str, Any]] = {
"ransomware": {
"label": "Ransomware",
"techniques": {
"T1486", # Data Encrypted for Impact
"T1490", # Inhibit System Recovery
"T1485", # Data Destruction
"T1489", # Service Stop
"T1491", # Defacement
"T1561", # Disk Wipe
"T1490.001",
"T1486.001",
},
"tactics": set(),
"categories": set(),
"log_sources": set(),
},
"credential-access": {
"label": "Credential Access",
"techniques": {
"T1003", # OS Credential Dumping (+ subtechniques)
"T1110", # Brute Force
"T1110.001",
"T1110.002",
"T1110.003",
"T1110.004",
"T1555", # Credentials from Password Stores
"T1212", # Exploitation for Credential Access
"T1187", # Forced Authentication
"T1556", # Modify Authentication Process
"T1606", # Forge Web Credentials
"T1528", # Steal Application Access Token
"T1558", # Steal or Forge Kerberos Tickets
"T1539", # Steal Web Session Cookie
"T1552", # Unsecured Credentials
"T1040", # Network Sniffing (creds in transit)
},
"tactics": {"TA0006"},
"categories": set(),
"log_sources": set(),
},
"lateral": {
"label": "Lateral Movement",
"techniques": {
"T1021", # Remote Services (+ subtechniques)
"T1021.001",
"T1021.002",
"T1021.003",
"T1021.004",
"T1021.005",
"T1021.006",
"T1570", # Lateral Tool Transfer
"T1550", # Use Alternate Authentication Material
"T1080", # Taint Shared Content
"T1210", # Exploitation of Remote Services
"T1563", # Remote Service Session Hijacking
},
"tactics": {"TA0008"},
"categories": set(),
"log_sources": set(),
},
"exfil": {
"label": "Data Exfiltration",
"techniques": {
"T1041", # Exfiltration Over C2 Channel
"T1048", # Exfiltration Over Alternative Protocol
"T1567", # Exfiltration Over Web Service
"T1029", # Scheduled Transfer
"T1011", # Exfiltration Over Other Network Medium
"T1052", # Exfiltration Over Physical Medium
"T1020", # Automated Exfiltration
"T1537", # Transfer Data to Cloud Account
},
"tactics": {"TA0010"},
"categories": {"data-exfil"},
"log_sources": set(),
},
"cloud": {
"label": "Cloud",
"techniques": {
"T1078.004", # Valid Accounts: Cloud Accounts
"T1538", # Cloud Service Dashboard
"T1526", # Cloud Service Discovery
"T1580", # Cloud Infrastructure Discovery
"T1530", # Data from Cloud Storage
"T1578", # Modify Cloud Compute Infrastructure
"T1535", # Unused/Unsupported Cloud Regions
"T1098.001", # Account Manipulation: Additional Cloud Credentials
"T1098.003", # Account Manipulation: Additional Cloud Roles
"T1556.007", # Hybrid Identity
"T1496", # Resource Hijacking (cryptomining in cloud)
"T1537", # Transfer Data to Cloud Account (overlaps exfil)
},
"tactics": set(),
"categories": {"cloud"},
"log_sources": {"aws", "azure", "gcp", "okta", "office365", "google_workspace"},
},
"identity": {
"label": "Identity",
"techniques": {
"T1078", # Valid Accounts (incl. all sub)
"T1078.001",
"T1078.002",
"T1078.003",
"T1078.004",
"T1098", # Account Manipulation
"T1098.001",
"T1098.002",
"T1098.003",
"T1098.004",
"T1098.005",
"T1136", # Create Account
"T1136.001",
"T1136.002",
"T1136.003",
"T1556", # Modify Authentication Process (+ subtechniques)
"T1621", # MFA Request Generation
"T1606.002", # Forge Web Credentials: SAML
},
"tactics": set(),
"categories": {"identity"},
"log_sources": {"okta", "azuread", "duo", "auth0", "ping", "google_workspace"},
},
"supply-chain": {
"label": "Supply Chain",
"techniques": {
"T1195", # Supply Chain Compromise (+ subtechniques)
"T1195.001",
"T1195.002",
"T1195.003",
"T1199", # Trusted Relationship
"T1554", # Compromise Client Software Binary
"T1505.003", # Server Software Component: Web Shell (proxied via supply-chain)
"T1059.006", # Python (most malicious package PoCs are python)
},
"tactics": set(),
"categories": set(),
"log_sources": {"github", "gitlab", "npm", "pypi", "bitbucket"},
},
"k8s": {
"label": "Kubernetes / Containers",
"techniques": {
"T1610", # Deploy Container
"T1611", # Escape to Host
"T1613", # Container and Resource Discovery
"T1612", # Build Image on Host
"T1525", # Implant Internal Image
"T1609", # Container Administration Command
"T1496", # Resource Hijacking (container cryptomining)
},
"tactics": set(),
"categories": set(),
"log_sources": {"kubernetes", "docker", "containerd", "k8s"},
},
}
# Categories that are considered native AiSOC tiers (executable-by-default).
NATIVE_CATEGORIES = {
"cloud",
"identity",
"endpoint",
"network",
"application",
"data-exfil",
}
# Imported tier directories. A rule that does not execute is never curated —
# but "does not execute" is decided by asking the engine, not by reading the
# path. ``_quarantine/`` stopped meaning "cannot run" when the Sigma compiler
# began translating rules where they sat: 1,724 files under that directory are
# now loaded and fire. Curating by directory name skipped every one of them,
# so the coverage page was selecting from a corpus smaller than the one that
# runs. ``build_marketplace.py`` and ``detection_truth_table.py`` both moved to
# the engine as the authority for the same reason.
IMPORTED_TIERS: dict[str, str] = {
"sigma-imports": "sigmahq",
"car-imports": "mitre-car",
"splunk-imports": "splunk-security-content",
"chronicle-imports": "chronicle-detection-rules",
}
# Per-family minimum count required to claim coverage. Total floor
# (default 300) is enforced across the union of these.
DEFAULT_MIN_PER_FAMILY = 25
DEFAULT_MIN_TOTAL = 300
# ─── Regex patterns ─────────────────────────────────────────────────────────
MITRE_DOTTED_RE = re.compile(r"mitre\.attack\.(t\d{4}(?:\.\d{3})?)", re.IGNORECASE)
MITRE_LOOSE_RE = re.compile(r"mitre\.(t\d{4}(?:\.\d{3})?)", re.IGNORECASE)
MITRE_BARE_RE = re.compile(r"^t\d{4}(?:\.\d{3})?$", re.IGNORECASE)
MITRE_ATTACK_PREFIX_RE = re.compile(r"^attack\.(t\d{4}(?:\.\d{3})?)$", re.IGNORECASE)
MITRE_TACTIC_RE = re.compile(r"^attack\.([a-z_-]+)$", re.IGNORECASE)
TACTIC_NAME_TO_ID = {
"reconnaissance": "TA0043",
"resource_development": "TA0042",
"initial_access": "TA0001",
"execution": "TA0002",
"persistence": "TA0003",
"privilege_escalation": "TA0004",
"defense_evasion": "TA0005",
"credential_access": "TA0006",
"discovery": "TA0007",
"lateral_movement": "TA0008",
"collection": "TA0009",
"exfiltration": "TA0010",
"command_and_control": "TA0011",
"impact": "TA0040",
}
# ─── Data model ──────────────────────────────────────────────────────────────
@dataclass
class Rule:
"""A single detection rule, parsed and classified."""
rule_id: str
name: str
path: Path
tier: str # native | imported | community
source: str # core | sigmahq | mitre-car | splunk-security-content | chronicle-detection-rules | community
category: str | None
severity: str | None
techniques: list[str] = field(default_factory=list)
tactics: list[str] = field(default_factory=list)
log_source_product: str | None = None
has_executable_body: bool = False
families: list[str] = field(default_factory=list)
quality_score: float = 0.0
quality_notes: list[str] = field(default_factory=list)
# ─── Parsing helpers ─────────────────────────────────────────────────────────
def _normalise_techniques(raw_tags: Any) -> tuple[list[str], list[str]]:
"""Pull (technique_ids, tactic_ids) from a tag block.
Accepts:
* native list form: ``['mitre.attack.t1234', 'tlp.white']``,
* importer dict form: ``{'mitre': ['T1234'], 'categories': ['cloud']}``,
* sigma-style: ``['attack.t1234', 'attack.credential_access']``.
"""
techniques: list[str] = []
tactics: list[str] = []
def _add_t(t: str) -> None:
u = t.upper()
if u not in techniques:
techniques.append(u)
def _add_ta(t: str) -> None:
if t not in tactics:
tactics.append(t)
if isinstance(raw_tags, dict):
for v in raw_tags.get("mitre") or []:
if isinstance(v, str) and MITRE_BARE_RE.match(v):
_add_t(v)
iterable: Iterable[Any]
if isinstance(raw_tags, dict):
# Already consumed mitre key above; nothing else useful here.
iterable = []
elif isinstance(raw_tags, list):
iterable = raw_tags
else:
iterable = []
for tag in iterable:
if not isinstance(tag, str):
continue
normalised = tag.strip().lower()
m = MITRE_DOTTED_RE.search(tag) or MITRE_LOOSE_RE.search(tag)
if m:
_add_t(m.group(1))
continue
m2 = MITRE_ATTACK_PREFIX_RE.match(normalised)
if m2:
_add_t(m2.group(1))
continue
if MITRE_BARE_RE.match(normalised):
_add_t(normalised)
continue
m3 = MITRE_TACTIC_RE.match(normalised)
if m3:
tactic_key = m3.group(1).replace("-", "_")
ta = TACTIC_NAME_TO_ID.get(tactic_key)
if ta:
_add_ta(ta)
return techniques, tactics
def _detect_executable_body(data: dict[str, Any]) -> bool:
"""Heuristic: does the rule have a body the engine can execute?
Native rules ship a ``detection`` block with a ``condition``. Imported
Sigma rules ship a ``detection`` block with selections + condition.
Quarantined / SPL / Chronicle rules typically only have a raw
``query`` field, which the engine doesn't execute today.
"""
detection = data.get("detection")
if not isinstance(detection, dict):
return False
if not detection.get("condition"):
return False
# Also need at least one selection or a condition that doesn't ref
# missing identifiers.
has_selection = any(k for k in detection.keys() if k != "condition" and k != "timeframe")
return has_selection or bool(detection.get("condition"))
def _parse_rule(path: Path, *, tier: str, source: str, quarantined: bool) -> Rule | None:
"""Load a YAML rule file and return a :class:`Rule`.
Returns ``None`` for rules that fail to parse, are quarantined,
or are missing required fields entirely.
"""
if quarantined:
return None
try:
data = yaml.safe_load(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001 — corrupt YAML, just skip
return None
if not isinstance(data, dict):
return None
rule_id = data.get("id") or path.stem
name = data.get("name") or rule_id
# Severity normalisation. Sigma uses ``low | medium | high | critical``;
# native uses the same. Coerce missing to None so the quality scorer
# can penalise it.
severity = data.get("severity")
if isinstance(severity, str):
severity = severity.strip().lower() or None
else:
severity = None
# Category is taken explicitly when set, otherwise from the parent
# directory name (which is how native rules are organised).
category = data.get("category")
if isinstance(category, str):
category = category.strip().lower() or None
if not category:
# A rule the engine loads has a category in the compiled ruleset,
# which is the one the engine evaluates it under. Prefer that over
# anything inferred from where the file sits: the Sigma corpus is
# written flat under `_quarantine/` with no `category:` key, so the
# parent-directory fallback bucketed 135 curated rules under a
# literal category named `_quarantine` and published it.
category = compiled_categories().get(str(rule_id))
if not category:
# For imported tiers, the directory above the file is the
# category bucket the importer picked.
category = path.parent.name.strip().lower() or None
# Some imports are direct children of e.g. sigma-imports/ with no
# category subdir — those have category == 'sigma-imports' from
# the parent name; null those out so they don't bucket-leak. The
# same applies to `_quarantine`, which is a lifecycle location and
# was never a category.
if category in IMPORTED_TIERS or category == "_quarantine":
category = None
techniques, tactics = _normalise_techniques(data.get("tags"))
log_source = data.get("log_source") or data.get("logsource") or {}
if not isinstance(log_source, dict):
log_source = {}
log_source_product = log_source.get("product")
if isinstance(log_source_product, str):
log_source_product = log_source_product.strip().lower() or None
else:
log_source_product = None
rule = Rule(
rule_id=str(rule_id),
name=str(name),
path=path,
tier=tier,
source=source,
category=category,
severity=severity,
techniques=techniques,
tactics=tactics,
log_source_product=log_source_product,
has_executable_body=_detect_executable_body(data),
)
rule.families = _classify(rule)
rule.quality_score, rule.quality_notes = _score(rule)
return rule
# ─── Classification ─────────────────────────────────────────────────────────
def _classify(rule: Rule) -> list[str]:
"""Decide which buyer-required families a rule covers.
A rule may cover multiple families (e.g. T1537 belongs to both
*cloud* and *exfil*). Multi-family membership is fine — it lets us
count one good rule against multiple coverage targets, and the
quality scorer prevents low-quality rules from cheating their way
in via a single tag overlap.
"""
matched: list[str] = []
tech_set = {t.upper() for t in rule.techniques}
tactic_set = {t.upper() for t in rule.tactics}
log_source = (rule.log_source_product or "").lower()
category = (rule.category or "").lower()
for family_id, spec in FAMILIES.items():
# Technique match
if tech_set & spec["techniques"]:
matched.append(family_id)
continue
# Tactic match (only for families that opt into tactic-only matching)
if spec["tactics"] and tactic_set & spec["tactics"]:
matched.append(family_id)
continue
# Category match
if category and category in spec["categories"]:
matched.append(family_id)
continue
# Log source match
if log_source and any(s in log_source for s in spec["log_sources"]):
matched.append(family_id)
continue
return matched
# ─── Quality scoring ────────────────────────────────────────────────────────
_SEVERITY_WEIGHT = {
"critical": 1.0,
"high": 0.85,
"medium": 0.7,
"low": 0.55,
}
def _score(rule: Rule) -> tuple[float, list[str]]:
"""Score a rule from 0 to 1. Native rules start at a higher base."""
score = 0.0
notes: list[str] = []
# Base by tier — native rules are fixture-tested, imported are not.
if rule.tier == "native":
score += 0.5
notes.append("tier:native(+0.5)")
elif rule.tier == "imported":
score += 0.3
notes.append("tier:imported(+0.3)")
else:
score += 0.2
notes.append("tier:other(+0.2)")
# MITRE — tagged with at least one technique.
if rule.techniques:
score += 0.2
notes.append(f"mitre_techniques:{len(rule.techniques)}(+0.2)")
elif rule.tactics:
score += 0.05
notes.append(f"mitre_tactics_only:{len(rule.tactics)}(+0.05)")
else:
notes.append("no_mitre(0)")
# Severity — present and high-confidence.
if rule.severity in _SEVERITY_WEIGHT:
weight = _SEVERITY_WEIGHT[rule.severity] * 0.15
score += weight
notes.append(f"severity:{rule.severity}(+{weight:.3f})")
else:
notes.append("no_severity(0)")
# Executable body.
if rule.has_executable_body:
score += 0.15
notes.append("executable(+0.15)")
else:
notes.append("no_executable_body(0)")
return min(score, 1.0), notes
# ─── Discovery ──────────────────────────────────────────────────────────────
@lru_cache(maxsize=1)
def _compiled_rules() -> dict[str, dict[str, Any]]:
"""Every rule the detection engine loads, keyed by id.
Empty when neither artefact is present, in which case callers fall back
to the directory layout rather than curating from nothing.
"""
out: dict[str, dict[str, Any]] = {}
for name in ("detection_ruleset.json", "detection_ruleset_imported.json"):
path = REPO_ROOT / "services" / "fusion" / "app" / "data" / name
if not path.exists():
continue
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
for rule in data.get("rules") or []:
if rule.get("id"):
out[str(rule["id"])] = rule
return out
def engine_rule_ids() -> frozenset[str]:
"""Ids the detection engine loads, across both compiled rulesets."""
return frozenset(_compiled_rules())
@lru_cache(maxsize=1)
def compiled_categories() -> dict[str, str]:
"""Rule id -> the category the engine evaluates that rule under."""
return {
rule_id: str(rule["category"]).strip().lower()
for rule_id, rule in _compiled_rules().items()
if isinstance(rule.get("category"), str) and rule["category"].strip()
}
def discover_rules() -> list[Rule]:
"""Parsed, classified rules that are candidates for curation.
A candidate is a rule the detection engine loads. Curating anything else
puts a rule that cannot fire into a page that promises coverage, which is
the failure this repository guards against everywhere else. The engine is
also the only authority that agrees with the truth table: the previous
filter read ``_quarantine/`` off the path, and that stopped meaning
"cannot run" once the Sigma compiler began translating rules in place.
With no compiled ruleset on disk the filter is skipped rather than
curating from nothing — a fresh checkout that has not exported the ruleset
still produces a report, and the count says how many it considered.
"""
rules: list[Rule] = []
loaded = engine_rule_ids()
def _keep(rule: Rule | None) -> None:
if rule is None:
return
if loaded and rule.rule_id not in loaded:
return
rules.append(rule)
# Native tiers
for cat in sorted(NATIVE_CATEGORIES):
cat_dir = DETECTIONS_DIR / cat
if not cat_dir.exists():
continue
for ext in ("*.yaml", "*.yml"):
for path in sorted(cat_dir.rglob(ext)):
_keep(_parse_rule(path, tier="native", source="core", quarantined=False))
# Imported tiers
for tier_dir, source_name in IMPORTED_TIERS.items():
root = DETECTIONS_DIR / tier_dir
if not root.exists():
continue
for ext in ("*.yaml", "*.yml"):
for path in sorted(root.rglob(ext)):
_keep(_parse_rule(path, tier="imported", source=source_name, quarantined=False))
# Community tier — opt-in, treated as supplementary not core
community_dir = DETECTIONS_DIR / "community"
if community_dir.exists():
for ext in ("*.yaml", "*.yml"):
for path in sorted(community_dir.rglob(ext)):
_keep(_parse_rule(path, tier="community", source="community", quarantined=False))
return rules
# ─── Selection ──────────────────────────────────────────────────────────────
def select_curated(
rules: list[Rule],
*,
min_per_family: int = DEFAULT_MIN_PER_FAMILY,
min_total: int = DEFAULT_MIN_TOTAL,
quality_floor: float = 0.55,
) -> tuple[list[Rule], dict[str, list[str]], dict[str, Any]]:
"""Pick the curated v1.0 set.
Strategy:
1. Drop rules below the quality floor immediately — these are the
"low-quality" candidates the plan asks us to prune.
2. For each family, sort eligible rules by quality (desc) and
pick the top N until the family floor is reached.
3. Union all picks. If union is below ``min_total``, top-up with
the highest-scoring remaining rules regardless of family.
Returns:
(selected_rules, per_family_ids, stats)
"""
eligible = [r for r in rules if r.quality_score >= quality_floor]
# Per-family ranking — highest quality first, deterministic on ties.
per_family: dict[str, list[Rule]] = {fid: [] for fid in FAMILIES}
for r in eligible:
for fid in r.families:
per_family[fid].append(r)
for lst in per_family.values():
lst.sort(key=lambda r: (-r.quality_score, r.rule_id))
selected: dict[str, Rule] = {}
family_picks: dict[str, list[str]] = {fid: [] for fid in FAMILIES}
for fid, ranked in per_family.items():
for r in ranked:
if len(family_picks[fid]) >= min_per_family * 4:
# Cap per-family contribution so a single deep family
# (cloud has 225+ rules) doesn't crowd out the others
# in the manifest. Floor is min_per_family × 4 = 100,
# which is generous and keeps families balanced.
break
selected.setdefault(r.rule_id, r)
family_picks[fid].append(r.rule_id)
# Top-up if union is below min_total.
if len(selected) < min_total:
remaining = [r for r in sorted(eligible, key=lambda r: (-r.quality_score, r.rule_id)) if r.rule_id not in selected]
for r in remaining:
if len(selected) >= min_total:
break
selected[r.rule_id] = r
# Final sort for deterministic output.
final_rules = sorted(selected.values(), key=lambda r: r.rule_id)
stats = {
"considered": len(rules),
"eligible": len(eligible),
"selected": len(final_rules),
"min_per_family": min_per_family,
"min_total": min_total,
"quality_floor": quality_floor,
}
return final_rules, family_picks, stats
# ─── Output ──────────────────────────────────────────────────────────────────
def _utc_now() -> str:
return dt.datetime.now(dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def build_manifest(
selected: list[Rule],
family_picks: dict[str, list[str]],
stats: dict[str, Any],
) -> dict[str, Any]:
"""Build the curated.json document."""
items = []
for r in selected:
items.append(
{
"id": r.rule_id,
"name": r.name,
"path": str(r.path.relative_to(REPO_ROOT)),
"tier": r.tier,
"source": r.source,
"category": r.category,
"severity": r.severity,
"mitre_techniques": r.techniques,
"mitre_tactics": r.tactics,
"families": r.families,
"quality_score": round(r.quality_score, 3),
"quality_notes": r.quality_notes,
"log_source_product": r.log_source_product,
"has_executable_body": r.has_executable_body,
}
)
families = {}
for fid, spec in FAMILIES.items():
ids = family_picks.get(fid, [])
families[fid] = {
"label": spec["label"],
"count": len(ids),
"rule_ids": sorted(ids),
"min_target": stats["min_per_family"],
"covered": len(ids) >= stats["min_per_family"],
}
# MITRE technique density
technique_counts: dict[str, int] = {}
for r in selected:
for t in r.techniques:
technique_counts[t] = technique_counts.get(t, 0) + 1
return {
"$schema": "https://aisoc.example.com/schemas/curated/v1.json",
"version": "1.0.0",
"generated": _utc_now(),
"stats": {
**stats,
"unique_techniques": len(technique_counts),
"selected_by_tier": _count_by(selected, "tier"),
"selected_by_severity": _count_by(selected, "severity"),
"selected_by_category": _count_by(selected, "category"),
},
"families": families,
"mitre_techniques": dict(sorted(technique_counts.items())),
"items": items,
}
def _count_by(rules: list[Rule], attr: str) -> dict[str, int]:
counts: dict[str, int] = {}
for r in rules:
v = getattr(r, attr) or "unknown"
counts[v] = counts.get(v, 0) + 1
return dict(sorted(counts.items()))
def build_report(manifest: dict[str, Any]) -> str:
"""Render the buyer-facing coverage report."""
s = manifest["stats"]
lines: list[str] = []
lines.append("---")
lines.append("title: Detection Coverage")
lines.append("description: |")
lines.append(" A curated selection of MITRE ATT&CK-mapped detections")
lines.append(" covering eight buyer-prioritised threat families, chosen")
lines.append(" from the rules the detection engine actually loads and")
lines.append(" generated by ``scripts/curate_detections.py``.")
lines.append("sidebar_position: 2")
lines.append("---")
lines.append("")
lines.append("# Detection Coverage")
lines.append("")
lines.append(f"Generated: `{manifest['generated']}`")
lines.append("")
lines.append("This page is a **curated selection**, not the size of the corpus.")
lines.append("It picks the highest-scoring rules per threat family from the set")
lines.append("the engine loads, so its numbers are smaller than the executable")
lines.append("total by design. For the corpus figures — executable, on disk, and")
lines.append("why they differ — see")
lines.append("[the detection truth table](https://github.com/beenuar/AiSOC/blob/main/docs/detections/truth-table.md).")
lines.append("")
lines.append("Candidates are the rules whose ids appear in the compiled ruleset,")
lines.append("which is the same authority the truth table uses. It used to be the")
lines.append("directory layout, and that stopped being the same thing once the")
lines.append("Sigma compiler began translating rules where they sat: rules under")
lines.append("`_quarantine/` that the engine loads were skipped for their path.")
lines.append("")
lines.append("## Headline numbers")
lines.append("")
lines.append(f"- **Curated detections**: `{s['selected']}` (target: ≥ {s['min_total']})")
lines.append(f"- **Executable rules considered**: `{s['considered']}` (quality floor: {s['quality_floor']})")
lines.append(f"- **Unique MITRE techniques covered**: `{s['unique_techniques']}`")
lines.append("")
lines.append("## Coverage by buyer family")
lines.append("")
lines.append("| Family | Count | Target | Covered |")
lines.append("|---|---|---|---|")
for info in manifest["families"].values():
check = "✅" if info["covered"] else "❌"
lines.append(f"| **{info['label']}** | {info['count']} | ≥ {info['min_target']} | {check} |")
lines.append("")
lines.append("## Distribution")
lines.append("")
lines.append("### By tier")
lines.append("")
for tier, count in s["selected_by_tier"].items():
lines.append(f"- `{tier}`: {count}")
lines.append("")
lines.append("### By severity")
lines.append("")
for sev, count in s["selected_by_severity"].items():
lines.append(f"- `{sev}`: {count}")
lines.append("")
lines.append("### By category")
lines.append("")
for cat, count in s["selected_by_category"].items():
lines.append(f"- `{cat}`: {count}")
lines.append("")
lines.append("## How to audit")
lines.append("")
lines.append(
"The curated rule IDs are listed in "
"[`marketplace/curated.json`](https://github.com/beenuar/AiSOC/blob/main/marketplace/curated.json) "
"under each family. Every entry has a `path` field pointing at the "
"on-disk YAML. Run `pnpm marketplace:curate --check` in CI to enforce "
"drift; run `python3 scripts/curate_detections.py` locally to regenerate."
)
lines.append("")
return "\n".join(lines) + "\n"
# ─── CLI ──────────────────────────────────────────────────────────────────────
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--check",
action="store_true",
help="CI mode: verify on-disk artefacts match the build, no writes.",
)
parser.add_argument(
"--print",
dest="print_only",
action="store_true",
help="Print built manifest to stdout instead of writing files.",
)
parser.add_argument(
"--min",
type=int,
default=DEFAULT_MIN_TOTAL,
help=f"Minimum total curated rules (default {DEFAULT_MIN_TOTAL}).",
)
parser.add_argument(
"--per-family",
type=int,
default=DEFAULT_MIN_PER_FAMILY,
help=f"Minimum per-family rules (default {DEFAULT_MIN_PER_FAMILY}).",
)
parser.add_argument(
"--quality-floor",
type=float,
default=0.55,
help="Drop rules with quality score below this (default 0.55).",
)
args = parser.parse_args()
rules = discover_rules()
selected, family_picks, stats = select_curated(
rules,
min_per_family=args.per_family,
min_total=args.min,
quality_floor=args.quality_floor,
)
manifest = build_manifest(selected, family_picks, stats)
serialised = json.dumps(manifest, indent=2, sort_keys=False) + "\n"
if args.print_only:
sys.stdout.write(serialised)
return 0
# Quality gate: every family must be covered, total must hit min.
ok = stats["selected"] >= args.min and all(info["covered"] for info in manifest["families"].values())
if args.check:
existing = OUT_MANIFEST.read_text(encoding="utf-8") if OUT_MANIFEST.exists() else ""
report = build_report(manifest)
existing_report = OUT_REPORT.read_text(encoding="utf-8") if OUT_REPORT.exists() else ""
def _strip_generated(s: str) -> str:
try:
obj = json.loads(s)
except Exception:
return s
obj.pop("generated", None)
return json.dumps(obj, indent=2, sort_keys=False) + "\n"
def _strip_md_generated(s: str) -> str:
return re.sub(r"Generated: `[^`]+`", "Generated: `<ts>`", s)
if _strip_generated(existing) != _strip_generated(serialised) or _strip_md_generated(existing_report) != _strip_md_generated(
report
):
print(
"marketplace/curated.json or coverage.md is stale. Run: pnpm marketplace:curate",
file=sys.stderr,
)
return 2
print(f"curation up to date ({stats['selected']} curated, {manifest['stats']['unique_techniques']} techniques).")
return 0 if ok else 1
OUT_MANIFEST.parent.mkdir(parents=True, exist_ok=True)
OUT_MANIFEST.write_text(serialised, encoding="utf-8")
OUT_REPORT.parent.mkdir(parents=True, exist_ok=True)
OUT_REPORT.write_text(build_report(manifest), encoding="utf-8")
uncovered = [info["label"] for info in manifest["families"].values() if not info["covered"]]
print(
f"Wrote curated manifest: selected={stats['selected']} "
f"techniques={manifest['stats']['unique_techniques']} "
f"families_covered={sum(1 for f in manifest['families'].values() if f['covered'])}"
f"/{len(manifest['families'])}"
)
if uncovered:
print(f" ❌ uncovered families: {', '.join(uncovered)}", file=sys.stderr)
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(main())