Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

17.6 KB
Raw Blame
#!/usr/bin/env python3
"""
AiSOC Detection Rule Validator
================================
Validates all YAML detection rules under ``detections/`` for:

  - Valid YAML syntax
  - Required fields (``id``, ``name``, ``severity``, ``detection``)
  - Severity is one of: ``low | medium | high | critical``
  - Category is one of:
    ``network | endpoint | cloud | identity | application | data-exfil | threat-anomaly``
  - No duplicate ``id`` values across all tiers
  - ``id`` prefix matches the rule's tier (``det-`` for native,
    ``<source>-...`` for imported tiers)
  - For native rules: matching positive + negative fixtures under
    ``detections/fixtures/`` and successful fixture replay against the
    canonical ``match_when`` spec
  - For imported rules: a populated ``provenance`` block with the required
    fields documented in ``tools/detection_import/README.md``

Tiering
-------
* **native** — ``detections/{network,endpoint,cloud,identity,application,
  data-exfil,threat-anomaly}/*.yaml``. Strict: fixtures + provenance optional.
* **imported** — ``detections/{sigma,car,splunk,chronicle}-imports/<category>
  /*.yaml`` (and ``.../_quarantine/<category>/*.yaml`` for rules that
  parsed but cannot run as-is). Provenance required, fixtures optional.
* **community** — ``detections/community/<category>/*.yaml``. Permissive;
  provenance encouraged but not required.

Exit codes
----------
* ``0`` — all rules valid
* ``1`` — one or more validation errors

Run ``python3 scripts/validate_detections.py --strict-fixtures`` to promote
fixture warnings into hard failures (used by CI for native rules).
"""

from __future__ import annotations

import json
import sys
from functools import lru_cache
from pathlib import Path
from typing import Any

# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))

from gate_toolkit import repo_root, self_test_if_requested

self_test_if_requested(__file__)

try:
    import yaml
except ImportError:
    print("ERROR: PyYAML not installed. Run: pip install pyyaml")
    sys.exit(1)

ROOT = repo_root()
DETECTIONS_DIR = ROOT / "detections"
FIXTURES_DIR = DETECTIONS_DIR / "fixtures"
SCRIPTS_DIR = ROOT / "scripts"
RULESET_DIR = ROOT / "services" / "fusion" / "app" / "data"

if str(SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(SCRIPTS_DIR))

# Imported as modules, not as bound attributes (py/import-of-mutable-attribute).
import detection_specs_index  # noqa: E402
import generate_detections  # noqa: E402

all_specs = detection_specs_index.all_specs
enrich = generate_detections.enrich
matches = generate_detections.matches
requested_derived_fields = generate_detections.requested_derived_fields

VALID_SEVERITIES = {"low", "medium", "high", "critical"}
VALID_CATEGORIES = {
    "network",
    "endpoint",
    "cloud",
    "identity",
    "application",
    "data-exfil",
    # Depth 3.4: rules whose clauses read the behavioural-baseline fields the
    # fusion enrichment computes (rarity, peer deviation, volume). Built
    # through the same generator as every other native rule, with the same
    # det- ids and paired fixtures, so the native contract applies unchanged.
    "threat-anomaly",
}

REQUIRED_FIELDS = ["id", "name", "severity", "detection"]


@lru_cache(maxsize=1)
def engine_rule_ids() -> frozenset[str]:
    """Ids the detection engine loads, across both compiled rulesets.

    Empty when neither artefact is on disk, which the summary reports as
    unknown rather than as zero — a fresh checkout that has not exported the
    ruleset should not be told none of its rules run.
    """
    ids: set[str] = set()
    for name in ("detection_ruleset.json", "detection_ruleset_imported.json"):
        path = RULESET_DIR / name
        if not path.exists():
            continue
        try:
            data = json.loads(path.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError):
            continue
        ids |= {str(r["id"]) for r in data.get("rules") or [] if r.get("id")}
    return frozenset(ids)


# Provenance fields required on every imported rule. License and
# license_url should be present so we can prove the redistribution chain;
# imported_by is the importer module so future audits can grep for the
# tool that produced the file.
REQUIRED_PROVENANCE_FIELDS = (
    "source",
    "source_id",
    "source_commit",
    "license",
    "imported_at",
    "imported_by",
    "upstream_path",
)

# Each imported tier directory maps to (display name, expected id prefix).
# The id prefix matches what ``tools.detection_import.common.stable_id``
# produces for that tier. Bumping importer source names? Update these too.
IMPORTED_TIERS: dict[str, dict[str, str]] = {
    "sigma-imports": {
        "name": "sigmahq",
        "id_prefix": "sigmahq-sigma-",
    },
    "car-imports": {
        "name": "mitre-car",
        "id_prefix": "mitre-car-",
    },
    "splunk-imports": {
        "name": "splunk-security-content",
        "id_prefix": "splunk-security-content-",
    },
    "chronicle-imports": {
        "name": "chronicle-detection-rules",
        "id_prefix": "chronicle-detection-rules-",
    },
}


# =============================================================================
# Spec lookup — for native fixture replay only.
# =============================================================================

_SPEC_BY_KEY: dict[tuple[str, str], dict[str, Any]] = {(cat, spec["slug"]): spec for cat, spec in all_specs()}


# =============================================================================
# Tier classification
# =============================================================================


def classify(path: Path) -> dict[str, Any]:
    """Classify a rule file by tier from its on-disk path.

    Returns a dict with keys:

    * ``tier`` — ``"native" | "imported" | "community" | "unknown"``
    * ``source`` — short source name for imported rules, else ``None``
    * ``id_prefix`` — required id prefix for the tier, else ``None``
    * ``category`` — the AiSOC category this rule belongs to (e.g.
      ``cloud``), inferred from the directory layout
    * ``is_quarantined`` — true if the rule lives under ``_quarantine/``
    """
    try:
        rel_parts = path.relative_to(DETECTIONS_DIR).parts
    except ValueError:
        return {
            "tier": "unknown",
            "source": None,
            "id_prefix": None,
            "category": None,
            "is_quarantined": False,
        }

    if not rel_parts:
        return {
            "tier": "unknown",
            "source": None,
            "id_prefix": None,
            "category": None,
            "is_quarantined": False,
        }

    head = rel_parts[0]

    # Native: detections/<category>/<rule>.yaml
    if head in VALID_CATEGORIES:
        return {
            "tier": "native",
            "source": None,
            "id_prefix": "det-",
            "category": head,
            "is_quarantined": False,
        }

    # Imported: detections/<source>-imports/[<_quarantine>/]<category>/<rule>.yaml
    if head in IMPORTED_TIERS:
        tier_meta = IMPORTED_TIERS[head]
        is_quarantined = len(rel_parts) >= 2 and rel_parts[1] == "_quarantine"
        if is_quarantined and len(rel_parts) >= 3:
            category = rel_parts[2]
        elif len(rel_parts) >= 2:
            category = rel_parts[1]
        else:
            category = None
        return {
            "tier": "imported",
            "source": tier_meta["name"],
            "id_prefix": tier_meta["id_prefix"],
            "category": category,
            "is_quarantined": is_quarantined,
        }

    if head == "community":
        category = rel_parts[1] if len(rel_parts) >= 2 else None
        return {
            "tier": "community",
            "source": "community",
            "id_prefix": None,
            "category": category,
            "is_quarantined": False,
        }

    return {
        "tier": "unknown",
        "source": None,
        "id_prefix": None,
        "category": None,
        "is_quarantined": False,
    }


# =============================================================================
# Validation
# =============================================================================


def validate_rule(
    path: Path,
    seen_ids: dict[str, Path],
    classification: dict[str, Any],
) -> tuple[list[str], dict[str, Any] | None]:
    """Validate a single detection rule file.

    Returns ``(errors, parsed_rule)``. ``parsed_rule`` is ``None`` if YAML
    parsing failed.
    """
    errors: list[str] = []

    try:
        with open(path) as f:
            rule = yaml.safe_load(f)
    except yaml.YAMLError as exc:
        return [f"YAML parse error: {exc}"], None

    if not isinstance(rule, dict):
        return ["Rule is not a YAML mapping (expected key: value pairs at top level)"], None

    for field in REQUIRED_FIELDS:
        if field not in rule:
            errors.append(f"Missing required field: '{field}'")

    if errors:
        return errors, rule

    if rule["severity"] not in VALID_SEVERITIES:
        errors.append(f"Invalid severity '{rule['severity']}'; must be one of: {', '.join(sorted(VALID_SEVERITIES))}")

    rule_category = rule.get("category")
    if rule_category and rule_category not in VALID_CATEGORIES:
        errors.append(f"Invalid category '{rule_category}'; must be one of: {', '.join(sorted(VALID_CATEGORIES))}")

    expected_category = classification["category"]
    if rule_category and expected_category and rule_category != expected_category:
        errors.append(f"Rule category '{rule_category}' does not match directory '{expected_category}'")

    rule_id = str(rule["id"])
    expected_prefix = classification["id_prefix"]
    if expected_prefix and not rule_id.startswith(expected_prefix):
        errors.append(f"Rule id '{rule_id}' must start with '{expected_prefix}' (tier: {classification['tier']})")

    if rule_id in seen_ids:
        errors.append(f"Duplicate id '{rule_id}' — already defined in {seen_ids[rule_id]}")
    else:
        seen_ids[rule_id] = path

    detection = rule.get("detection")
    if not isinstance(detection, dict):
        errors.append(
            "'detection' field must be a YAML mapping with 'condition', "
            "'keywords', or a tier-specific block (e.g. 'splunk_spl', "
            "'chronicle_yaral')"
        )

    if classification["tier"] == "imported":
        errors.extend(_validate_provenance(rule))

    return errors, rule


def _validate_provenance(rule: dict[str, Any]) -> list[str]:
    """Validate the ``provenance`` block on an imported rule."""
    errors: list[str] = []
    provenance = rule.get("provenance")
    if not isinstance(provenance, dict):
        return ["Imported rule is missing required 'provenance' block (see tools/detection_import/README.md)"]
    for field in REQUIRED_PROVENANCE_FIELDS:
        value = provenance.get(field)
        if value in (None, ""):
            errors.append(f"provenance.{field} is required and must be non-empty")
    return errors


def replay_fixture(rule_path: Path, rule: dict[str, Any], strict: bool) -> list[str]:
    """Replay positive + negative fixtures against the canonical spec.

    Native tier only. Looks up the spec for this rule by ``(category,
    slug)``, then evaluates the structured ``match_when`` dict against the
    on-disk fixtures using the same ``matches()`` function the runtime
    engine would use.

    Returns a list of error messages. If ``strict`` is ``False``, missing
    fixtures or missing specs degrade to soft warnings (``WARN: ...``).
    """
    errors: list[str] = []
    slug = rule_path.stem
    category = rule_path.parent.name
    pos_path = FIXTURES_DIR / "positive" / f"{slug}.json"
    neg_path = FIXTURES_DIR / "negative" / f"{slug}.json"

    pos_missing = not pos_path.exists()
    neg_missing = not neg_path.exists()

    if pos_missing or neg_missing:
        msg_parts = []
        if pos_missing:
            msg_parts.append(f"missing positive fixture {pos_path.relative_to(ROOT)}")
        if neg_missing:
            msg_parts.append(f"missing negative fixture {neg_path.relative_to(ROOT)}")
        msg = "; ".join(msg_parts)
        if strict:
            errors.append(msg)
        else:
            errors.append(f"WARN: {msg}")
        return errors

    spec = _SPEC_BY_KEY.get((category, slug))
    if spec is None:
        msg = f"no canonical spec found for ({category}, {slug}); hand-authored rule — fixture replay skipped"
        errors.append(f"WARN: {msg}")
        return errors

    match_when = spec.get("match_when")
    if not match_when:
        return errors

    try:
        with open(pos_path) as f:
            pos_event = json.load(f)
        with open(neg_path) as f:
            neg_event = json.load(f)
    except (OSError, json.JSONDecodeError) as exc:
        errors.append(f"fixture load error: {exc}")
        return errors

    # Enriched the way the engine enriches. Replaying against the bare
    # matcher tests a pipeline production does not run — a rule matching a
    # derived field would fail here while working live, which trains people
    # to weaken the gate.
    wanted = requested_derived_fields([{"match_when": match_when}])
    pos_event = enrich(pos_event, wanted)
    neg_event = enrich(neg_event, wanted)

    if not matches(match_when, pos_event):
        errors.append("positive fixture did NOT match match_when (expected match)")
    if matches(match_when, neg_event):
        errors.append("negative fixture DID match match_when (expected no match)")
    return errors


def main() -> int:
    strict = "--strict-fixtures" in sys.argv
    if not DETECTIONS_DIR.exists():
        print(f"ERROR: detections/ directory not found at {DETECTIONS_DIR}")
        return 1

    yaml_files = sorted(DETECTIONS_DIR.rglob("*.yaml"))
    if not yaml_files:
        # This was a WARNING and an exit 0 — the validator for the corpus the
        # product's headline number describes, certifying an empty corpus as
        # valid. "No rules are broken" and "no rules were opened" are the same
        # sentence here, and only one of them is good news.
        print(
            f"ERROR: no .yaml files found under {DETECTIONS_DIR}. Zero rules validated is "
            f"not zero rules broken — either the corpus is gone or this is not the tree "
            f"it was meant to read.",
            file=sys.stderr,
        )
        return 1

    seen_ids: dict[str, Path] = {}
    total = 0
    failed = 0
    fixture_warnings = 0
    tier_counts: dict[str, int] = {
        "native": 0,
        "imported": 0,
        "community": 0,
        "unknown": 0,
    }
    loaded_ids = engine_rule_ids()
    executable_count = 0

    for path in yaml_files:
        rel = path.relative_to(ROOT)

        # Skip any meta/index file at the top of detections/ that is not a
        # rule file. A rule lives under a category directory; anything
        # directly under detections/ with no category dir is not one.
        try:
            rel_under = path.relative_to(DETECTIONS_DIR)
        except ValueError:
            continue
        if len(rel_under.parts) < 2:
            continue
        if rel_under.parts[0] in ("fixtures", "playbooks"):
            continue

        classification = classify(path)
        tier = classification["tier"]
        tier_counts[tier] = tier_counts.get(tier, 0) + 1

        total += 1

        errors, rule = validate_rule(path, seen_ids, classification)
        rule_failed = bool(errors)

        # Whether a rule runs is reported by asking the engine, not by reading
        # the path. `_quarantine/` stopped meaning "cannot run" when the Sigma
        # compiler began translating rules where they sat — 1,724 files under
        # that directory are loaded and fire — so the old path-derived figure
        # said 5,937 while `detection_truth_table.py` said 4,213 about the same
        # tree. Two counts of one thing that disagree is how a stale number
        # survives; this one derives from the same artefact as the truth table.
        rule_loaded = str((rule or {}).get("id") or "") in loaded_ids
        if rule_loaded:
            executable_count += 1

        replay_errors: list[str] = []
        if rule and not rule_failed and tier == "native" and path.parent.name in VALID_CATEGORIES:
            replay_errors = replay_fixture(path, rule, strict=strict)

        warnings = [e for e in replay_errors if e.startswith("WARN:")]
        hard = [e for e in replay_errors if not e.startswith("WARN:")]
        fixture_warnings += len(warnings)
        if hard:
            rule_failed = True
            errors.extend(hard)

        if rule_failed:
            failed += 1
            print(f"\nFAIL  [{tier}] {rel}")
            for e in errors:
                print(f"    - {e}")
        else:
            warn_suffix = f" ({len(warnings)} warn)" if warnings else ""
            state_suffix = "  [executable]" if rule_loaded else "  [not loaded]"
            print(f"PASS  [{tier}] {rel}{warn_suffix}{state_suffix}")
            for w in warnings:
                print(f"    {w}")

    print(f"\n{'─' * 60}")
    print(f"Validated {total} rules — {total - failed} passed, {failed} failed, {fixture_warnings} fixture warnings")
    print("  Tiers: " + ", ".join(f"{tier}={count}" for tier, count in sorted(tier_counts.items()) if count > 0))
    if loaded_ids:
        print(f"  Executable (loaded by the engine): {executable_count}; not loaded: {total - executable_count}")
    else:
        print("  Executable: unknown — no compiled ruleset on disk; run scripts/export_detection_ruleset.py")

    return 1 if failed > 0 else 0


if __name__ == "__main__":
    sys.exit(main())