Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""
AiSOC Detection Rule Validator
================================
Validates all YAML detection rules under ``detections/`` for:
- Valid YAML syntax
- Required fields (``id``, ``name``, ``severity``, ``detection``)
- Severity is one of: ``low | medium | high | critical``
- Category is one of:
``network | endpoint | cloud | identity | application | data-exfil | threat-anomaly``
- No duplicate ``id`` values across all tiers
- ``id`` prefix matches the rule's tier (``det-`` for native,
``<source>-...`` for imported tiers)
- For native rules: matching positive + negative fixtures under
``detections/fixtures/`` and successful fixture replay against the
canonical ``match_when`` spec
- For imported rules: a populated ``provenance`` block with the required
fields documented in ``tools/detection_import/README.md``
Tiering
-------
* **native** — ``detections/{network,endpoint,cloud,identity,application,
data-exfil,threat-anomaly}/*.yaml``. Strict: fixtures + provenance optional.
* **imported** — ``detections/{sigma,car,splunk,chronicle}-imports/<category>
/*.yaml`` (and ``.../_quarantine/<category>/*.yaml`` for rules that
parsed but cannot run as-is). Provenance required, fixtures optional.
* **community** — ``detections/community/<category>/*.yaml``. Permissive;
provenance encouraged but not required.
Exit codes
----------
* ``0`` — all rules valid
* ``1`` — one or more validation errors
Run ``python3 scripts/validate_detections.py --strict-fixtures`` to promote
fixture warnings into hard failures (used by CI for native rules).
"""
from __future__ import annotations
import json
import sys
from functools import lru_cache
from pathlib import Path
from typing import Any
# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from gate_toolkit import repo_root, self_test_if_requested
self_test_if_requested(__file__)
try:
import yaml
except ImportError:
print("ERROR: PyYAML not installed. Run: pip install pyyaml")
sys.exit(1)
ROOT = repo_root()
DETECTIONS_DIR = ROOT / "detections"
FIXTURES_DIR = DETECTIONS_DIR / "fixtures"
SCRIPTS_DIR = ROOT / "scripts"
RULESET_DIR = ROOT / "services" / "fusion" / "app" / "data"
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
# Imported as modules, not as bound attributes (py/import-of-mutable-attribute).
import detection_specs_index # noqa: E402
import generate_detections # noqa: E402
all_specs = detection_specs_index.all_specs
enrich = generate_detections.enrich
matches = generate_detections.matches
requested_derived_fields = generate_detections.requested_derived_fields
VALID_SEVERITIES = {"low", "medium", "high", "critical"}
VALID_CATEGORIES = {
"network",
"endpoint",
"cloud",
"identity",
"application",
"data-exfil",
# Depth 3.4: rules whose clauses read the behavioural-baseline fields the
# fusion enrichment computes (rarity, peer deviation, volume). Built
# through the same generator as every other native rule, with the same
# det- ids and paired fixtures, so the native contract applies unchanged.
"threat-anomaly",
}
REQUIRED_FIELDS = ["id", "name", "severity", "detection"]
@lru_cache(maxsize=1)
def engine_rule_ids() -> frozenset[str]:
"""Ids the detection engine loads, across both compiled rulesets.
Empty when neither artefact is on disk, which the summary reports as
unknown rather than as zero — a fresh checkout that has not exported the
ruleset should not be told none of its rules run.
"""
ids: set[str] = set()
for name in ("detection_ruleset.json", "detection_ruleset_imported.json"):
path = RULESET_DIR / name
if not path.exists():
continue
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
ids |= {str(r["id"]) for r in data.get("rules") or [] if r.get("id")}
return frozenset(ids)
# Provenance fields required on every imported rule. License and
# license_url should be present so we can prove the redistribution chain;
# imported_by is the importer module so future audits can grep for the
# tool that produced the file.
REQUIRED_PROVENANCE_FIELDS = (
"source",
"source_id",
"source_commit",
"license",
"imported_at",
"imported_by",
"upstream_path",
)
# Each imported tier directory maps to (display name, expected id prefix).
# The id prefix matches what ``tools.detection_import.common.stable_id``
# produces for that tier. Bumping importer source names? Update these too.
IMPORTED_TIERS: dict[str, dict[str, str]] = {
"sigma-imports": {
"name": "sigmahq",
"id_prefix": "sigmahq-sigma-",
},
"car-imports": {
"name": "mitre-car",
"id_prefix": "mitre-car-",
},
"splunk-imports": {
"name": "splunk-security-content",
"id_prefix": "splunk-security-content-",
},
"chronicle-imports": {
"name": "chronicle-detection-rules",
"id_prefix": "chronicle-detection-rules-",
},
}
# =============================================================================
# Spec lookup — for native fixture replay only.
# =============================================================================
_SPEC_BY_KEY: dict[tuple[str, str], dict[str, Any]] = {(cat, spec["slug"]): spec for cat, spec in all_specs()}
# =============================================================================
# Tier classification
# =============================================================================
def classify(path: Path) -> dict[str, Any]:
"""Classify a rule file by tier from its on-disk path.
Returns a dict with keys:
* ``tier`` — ``"native" | "imported" | "community" | "unknown"``
* ``source`` — short source name for imported rules, else ``None``
* ``id_prefix`` — required id prefix for the tier, else ``None``
* ``category`` — the AiSOC category this rule belongs to (e.g.
``cloud``), inferred from the directory layout
* ``is_quarantined`` — true if the rule lives under ``_quarantine/``
"""
try:
rel_parts = path.relative_to(DETECTIONS_DIR).parts
except ValueError:
return {
"tier": "unknown",
"source": None,
"id_prefix": None,
"category": None,
"is_quarantined": False,
}
if not rel_parts:
return {
"tier": "unknown",
"source": None,
"id_prefix": None,
"category": None,
"is_quarantined": False,
}
head = rel_parts[0]
# Native: detections/<category>/<rule>.yaml
if head in VALID_CATEGORIES:
return {
"tier": "native",
"source": None,
"id_prefix": "det-",
"category": head,
"is_quarantined": False,
}
# Imported: detections/<source>-imports/[<_quarantine>/]<category>/<rule>.yaml
if head in IMPORTED_TIERS:
tier_meta = IMPORTED_TIERS[head]
is_quarantined = len(rel_parts) >= 2 and rel_parts[1] == "_quarantine"
if is_quarantined and len(rel_parts) >= 3:
category = rel_parts[2]
elif len(rel_parts) >= 2:
category = rel_parts[1]
else:
category = None
return {
"tier": "imported",
"source": tier_meta["name"],
"id_prefix": tier_meta["id_prefix"],
"category": category,
"is_quarantined": is_quarantined,
}
if head == "community":
category = rel_parts[1] if len(rel_parts) >= 2 else None
return {
"tier": "community",
"source": "community",
"id_prefix": None,
"category": category,
"is_quarantined": False,
}
return {
"tier": "unknown",
"source": None,
"id_prefix": None,
"category": None,
"is_quarantined": False,
}
# =============================================================================
# Validation
# =============================================================================
def validate_rule(
path: Path,
seen_ids: dict[str, Path],
classification: dict[str, Any],
) -> tuple[list[str], dict[str, Any] | None]:
"""Validate a single detection rule file.
Returns ``(errors, parsed_rule)``. ``parsed_rule`` is ``None`` if YAML
parsing failed.
"""
errors: list[str] = []
try:
with open(path) as f:
rule = yaml.safe_load(f)
except yaml.YAMLError as exc:
return [f"YAML parse error: {exc}"], None
if not isinstance(rule, dict):
return ["Rule is not a YAML mapping (expected key: value pairs at top level)"], None
for field in REQUIRED_FIELDS:
if field not in rule:
errors.append(f"Missing required field: '{field}'")
if errors:
return errors, rule
if rule["severity"] not in VALID_SEVERITIES:
errors.append(f"Invalid severity '{rule['severity']}'; must be one of: {', '.join(sorted(VALID_SEVERITIES))}")
rule_category = rule.get("category")
if rule_category and rule_category not in VALID_CATEGORIES:
errors.append(f"Invalid category '{rule_category}'; must be one of: {', '.join(sorted(VALID_CATEGORIES))}")
expected_category = classification["category"]
if rule_category and expected_category and rule_category != expected_category:
errors.append(f"Rule category '{rule_category}' does not match directory '{expected_category}'")
rule_id = str(rule["id"])
expected_prefix = classification["id_prefix"]
if expected_prefix and not rule_id.startswith(expected_prefix):
errors.append(f"Rule id '{rule_id}' must start with '{expected_prefix}' (tier: {classification['tier']})")
if rule_id in seen_ids:
errors.append(f"Duplicate id '{rule_id}' — already defined in {seen_ids[rule_id]}")
else:
seen_ids[rule_id] = path
detection = rule.get("detection")
if not isinstance(detection, dict):
errors.append(
"'detection' field must be a YAML mapping with 'condition', "
"'keywords', or a tier-specific block (e.g. 'splunk_spl', "
"'chronicle_yaral')"
)
if classification["tier"] == "imported":
errors.extend(_validate_provenance(rule))
return errors, rule
def _validate_provenance(rule: dict[str, Any]) -> list[str]:
"""Validate the ``provenance`` block on an imported rule."""
errors: list[str] = []
provenance = rule.get("provenance")
if not isinstance(provenance, dict):
return ["Imported rule is missing required 'provenance' block (see tools/detection_import/README.md)"]
for field in REQUIRED_PROVENANCE_FIELDS:
value = provenance.get(field)
if value in (None, ""):
errors.append(f"provenance.{field} is required and must be non-empty")
return errors
def replay_fixture(rule_path: Path, rule: dict[str, Any], strict: bool) -> list[str]:
"""Replay positive + negative fixtures against the canonical spec.
Native tier only. Looks up the spec for this rule by ``(category,
slug)``, then evaluates the structured ``match_when`` dict against the
on-disk fixtures using the same ``matches()`` function the runtime
engine would use.
Returns a list of error messages. If ``strict`` is ``False``, missing
fixtures or missing specs degrade to soft warnings (``WARN: ...``).
"""
errors: list[str] = []
slug = rule_path.stem
category = rule_path.parent.name
pos_path = FIXTURES_DIR / "positive" / f"{slug}.json"
neg_path = FIXTURES_DIR / "negative" / f"{slug}.json"
pos_missing = not pos_path.exists()
neg_missing = not neg_path.exists()
if pos_missing or neg_missing:
msg_parts = []
if pos_missing:
msg_parts.append(f"missing positive fixture {pos_path.relative_to(ROOT)}")
if neg_missing:
msg_parts.append(f"missing negative fixture {neg_path.relative_to(ROOT)}")
msg = "; ".join(msg_parts)
if strict:
errors.append(msg)
else:
errors.append(f"WARN: {msg}")
return errors
spec = _SPEC_BY_KEY.get((category, slug))
if spec is None:
msg = f"no canonical spec found for ({category}, {slug}); hand-authored rule — fixture replay skipped"
errors.append(f"WARN: {msg}")
return errors
match_when = spec.get("match_when")
if not match_when:
return errors
try:
with open(pos_path) as f:
pos_event = json.load(f)
with open(neg_path) as f:
neg_event = json.load(f)
except (OSError, json.JSONDecodeError) as exc:
errors.append(f"fixture load error: {exc}")
return errors
# Enriched the way the engine enriches. Replaying against the bare
# matcher tests a pipeline production does not run — a rule matching a
# derived field would fail here while working live, which trains people
# to weaken the gate.
wanted = requested_derived_fields([{"match_when": match_when}])
pos_event = enrich(pos_event, wanted)
neg_event = enrich(neg_event, wanted)
if not matches(match_when, pos_event):
errors.append("positive fixture did NOT match match_when (expected match)")
if matches(match_when, neg_event):
errors.append("negative fixture DID match match_when (expected no match)")
return errors
def main() -> int:
strict = "--strict-fixtures" in sys.argv
if not DETECTIONS_DIR.exists():
print(f"ERROR: detections/ directory not found at {DETECTIONS_DIR}")
return 1
yaml_files = sorted(DETECTIONS_DIR.rglob("*.yaml"))
if not yaml_files:
# This was a WARNING and an exit 0 — the validator for the corpus the
# product's headline number describes, certifying an empty corpus as
# valid. "No rules are broken" and "no rules were opened" are the same
# sentence here, and only one of them is good news.
print(
f"ERROR: no .yaml files found under {DETECTIONS_DIR}. Zero rules validated is "
f"not zero rules broken — either the corpus is gone or this is not the tree "
f"it was meant to read.",
file=sys.stderr,
)
return 1
seen_ids: dict[str, Path] = {}
total = 0
failed = 0
fixture_warnings = 0
tier_counts: dict[str, int] = {
"native": 0,
"imported": 0,
"community": 0,
"unknown": 0,
}
loaded_ids = engine_rule_ids()
executable_count = 0
for path in yaml_files:
rel = path.relative_to(ROOT)
# Skip any meta/index file at the top of detections/ that is not a
# rule file. A rule lives under a category directory; anything
# directly under detections/ with no category dir is not one.
try:
rel_under = path.relative_to(DETECTIONS_DIR)
except ValueError:
continue
if len(rel_under.parts) < 2:
continue
if rel_under.parts[0] in ("fixtures", "playbooks"):
continue
classification = classify(path)
tier = classification["tier"]
tier_counts[tier] = tier_counts.get(tier, 0) + 1
total += 1
errors, rule = validate_rule(path, seen_ids, classification)
rule_failed = bool(errors)
# Whether a rule runs is reported by asking the engine, not by reading
# the path. `_quarantine/` stopped meaning "cannot run" when the Sigma
# compiler began translating rules where they sat — 1,724 files under
# that directory are loaded and fire — so the old path-derived figure
# said 5,937 while `detection_truth_table.py` said 4,213 about the same
# tree. Two counts of one thing that disagree is how a stale number
# survives; this one derives from the same artefact as the truth table.
rule_loaded = str((rule or {}).get("id") or "") in loaded_ids
if rule_loaded:
executable_count += 1
replay_errors: list[str] = []
if rule and not rule_failed and tier == "native" and path.parent.name in VALID_CATEGORIES:
replay_errors = replay_fixture(path, rule, strict=strict)
warnings = [e for e in replay_errors if e.startswith("WARN:")]
hard = [e for e in replay_errors if not e.startswith("WARN:")]
fixture_warnings += len(warnings)
if hard:
rule_failed = True
errors.extend(hard)
if rule_failed:
failed += 1
print(f"\nFAIL [{tier}] {rel}")
for e in errors:
print(f" - {e}")
else:
warn_suffix = f" ({len(warnings)} warn)" if warnings else ""
state_suffix = " [executable]" if rule_loaded else " [not loaded]"
print(f"PASS [{tier}] {rel}{warn_suffix}{state_suffix}")
for w in warnings:
print(f" {w}")
print(f"\n{'─' * 60}")
print(f"Validated {total} rules — {total - failed} passed, {failed} failed, {fixture_warnings} fixture warnings")
print(" Tiers: " + ", ".join(f"{tier}={count}" for tier, count in sorted(tier_counts.items()) if count > 0))
if loaded_ids:
print(f" Executable (loaded by the engine): {executable_count}; not loaded: {total - executable_count}")
else:
print(" Executable: unknown — no compiled ruleset on disk; run scripts/export_detection_ruleset.py")
return 1 if failed > 0 else 0
if __name__ == "__main__":
sys.exit(main())