Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

23.8 KB
Raw Blame
#!/usr/bin/env python3
"""Export, validate, and regenerate the AiSOC security knowledge graph schema.

This script is the drift gate that keeps three representations of the schema
in lockstep:

1. ``schemas/graph-schema.yaml`` — the source of truth checked into the repo.
2. ``services/ingest/internal/graph/schema.go`` — the Go enums consumed by
   the ingest writer (may not exist yet — T1.1 is concurrent).
3. The live Neo4j database materialised by ingest (when reachable).

Modes
-----

Default (no flags)
    Connect to Neo4j (env: ``AISOC_NEO4J_URI`` / ``AISOC_NEO4J_USER`` /
    ``AISOC_NEO4J_PASSWORD``, falling back to ``NEO4J_URI`` etc.) and write
    the materialised schema to ``schemas/graph-schema-current.yaml``. When
    Neo4j is unreachable, fall back to parsing the Go enums and dumping
    those.

``--check``
    Parse the on-disk YAML, parse the Go enums (when present), and the live
    database (when reachable), and fail non-zero on any drift. CI calls this.

``--from-go``
    Regenerate ``schemas/graph-schema.yaml`` from the Go enums. One-shot
    helper for the rare case where Go is the authoritative change. Refuses
    to run when the Go source file is missing.

Exit codes
----------
0
    All checked artefacts agree.
1
    Drift detected (in ``--check`` mode).
2
    Operational failure: missing input file, unreadable YAML, missing Go
    source in ``--from-go`` mode.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
from collections.abc import Iterable
from dataclasses import dataclass, field
from pathlib import Path

# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))

from gate_toolkit import repo_root, self_test_if_requested

self_test_if_requested(__file__)

REPO_ROOT = repo_root()
DEFAULT_YAML = REPO_ROOT / "schemas" / "graph-schema.yaml"
DEFAULT_CURRENT_YAML = REPO_ROOT / "schemas" / "graph-schema-current.yaml"
DEFAULT_GO_SOURCE = REPO_ROOT / "services" / "ingest" / "internal" / "graph" / "schema.go"


def _go_schema_version(path: Path = DEFAULT_GO_SOURCE) -> str:
    """Read SchemaVersion out of schema.go.

    Derived rather than duplicated. The version previously lived here as a
    literal as well as in the Go source and the YAML, so a bump meant editing
    three files and the gate's failure message was a reminder to edit the
    third. Two representations that must agree is a gate; three, one of which
    is the gate itself, is a chore.
    """
    if path.exists():
        match = re.search(r'SchemaVersion\s*=\s*"([^"]+)"', path.read_text(encoding="utf-8"))
        if match:
            return match.group(1)
    return "v1.0"


SCHEMA_VERSION = _go_schema_version()

# Required event-edge property set. Every relationship marked
# ``event_edge: true`` must declare all three.
EVENT_EDGE_REQUIRED = {"ts", "source_event_id", "snapshot_id"}


# ---------------------------------------------------------------------------
# YAML parsing
# ---------------------------------------------------------------------------


def _require_yaml():
    """Import PyYAML lazily so the module can be imported in environments
    without PyYAML (e.g. for ``--help``)."""
    try:
        import yaml  # type: ignore[import-untyped]
    except ImportError as exc:  # pragma: no cover - import guard
        sys.stderr.write(f"PyYAML is required: pip install pyyaml\n  (original error: {exc})\n")
        sys.exit(2)
    return yaml


@dataclass
class Schema:
    """In-memory representation of the schema."""

    version: str
    node_labels: list[str] = field(default_factory=list)
    relationships: list[str] = field(default_factory=list)
    label_properties: dict[str, set[str]] = field(default_factory=dict)
    relationship_properties: dict[str, set[str]] = field(default_factory=dict)
    event_edges: set[str] = field(default_factory=set)

    def summary(self) -> str:
        return (
            f"version={self.version} "
            f"labels={len(self.node_labels)} "
            f"relationships={len(self.relationships)} "
            f"event_edges={len(self.event_edges)}"
        )


def load_yaml_schema(path: Path) -> Schema:
    yaml = _require_yaml()
    if not path.exists():
        sys.stderr.write(f"schema yaml not found at {path}\n")
        sys.exit(2)
    with path.open("r", encoding="utf-8") as fh:
        data = yaml.safe_load(fh) or {}

    version = str(data.get("version", "")).strip()
    if not version:
        sys.stderr.write(f"{path}: missing top-level `version`\n")
        sys.exit(2)

    schema = Schema(version=version)
    for label_entry in data.get("node_labels", []) or []:
        if not isinstance(label_entry, dict):
            continue
        name = label_entry.get("label")
        if not name:
            continue
        schema.node_labels.append(name)
        props = {p["name"] for p in (label_entry.get("properties") or []) if isinstance(p, dict) and p.get("name")}
        schema.label_properties[name] = props

    for rel_entry in data.get("relationships", []) or []:
        if not isinstance(rel_entry, dict):
            continue
        name = rel_entry.get("name")
        if not name:
            continue
        schema.relationships.append(name)
        props = {p["name"] for p in (rel_entry.get("properties") or []) if isinstance(p, dict) and p.get("name")}
        schema.relationship_properties[name] = props
        if rel_entry.get("event_edge"):
            schema.event_edges.add(name)

    return schema


# ---------------------------------------------------------------------------
# Go source parsing
# ---------------------------------------------------------------------------


# Heuristic regex: match string literals on the right-hand side of a Go
# constant or variable assignment. We deliberately do not try to fully parse
# Go — labels and relationships are classified by their declared type
# (``NodeLabel`` vs ``RelType``), not by the shape of the string. The
# previous approach bucketed CamelCase as a label and UPPER_SNAKE as a
# relationship, which misfiled ``IOC`` — an all-caps node label.


def parse_go_source(path: Path) -> tuple[set[str], set[str]] | None:
    """Extract node labels and relationship names from the Go enums.

    Classifies by the declared Go type (``NodeLabel`` / ``RelType``) rather
    than by the shape of the string. The previous heuristic — "contains a
    lowercase letter, so it is a CamelCase label, otherwise ALL_CAPS, so it
    is a relationship" — put the ``IOC`` node label in the relationship set,
    and would do the same to any acronym label added later. The type is
    already in the source and is not a guess.

    Go permits omitting the type on subsequent entries in a ``const`` block,
    so the last explicit type carries forward, as the compiler does.

    Returns ``None`` when the file does not exist (T1.1 not yet landed).
    """
    if not path.exists():
        return None

    labels: set[str] = set()
    rels: set[str] = set()

    typed = re.compile(r'^\s*\w+\s+(NodeLabel|RelType)\s*=\s*"([^"]+)"')
    untyped = re.compile(r'^\s*\w+\s*=\s*"([^"]+)"')

    current: str | None = None
    in_const = False
    for line in path.read_text(encoding="utf-8").splitlines():
        stripped = line.strip()
        if stripped.startswith("const ("):
            in_const, current = True, None
            continue
        if in_const and stripped == ")":
            in_const, current = False, None
            continue

        match = typed.match(line)
        if match:
            current = match.group(1)
            (labels if current == "NodeLabel" else rels).add(match.group(2))
            continue

        if in_const and current:
            match = untyped.match(line)
            if match:
                (labels if current == "NodeLabel" else rels).add(match.group(1))

    return labels, rels


# ---------------------------------------------------------------------------
# Neo4j parsing
# ---------------------------------------------------------------------------


def _neo4j_credentials() -> tuple[str, str, str] | None:
    uri = os.environ.get("AISOC_NEO4J_URI") or os.environ.get("NEO4J_URI")
    user = os.environ.get("AISOC_NEO4J_USER") or os.environ.get("NEO4J_USER")
    password = os.environ.get("AISOC_NEO4J_PASSWORD") or os.environ.get("NEO4J_PASSWORD")
    if not uri or not user or password is None:
        return None
    return uri, user, password


def parse_live_neo4j() -> tuple[set[str], set[str]] | None:
    """Connect to Neo4j and dump labels + relationship types.

    Returns ``None`` when the ``neo4j`` driver is missing, when no credentials
    are configured, or when the connection fails — every one of these is a
    soft fall-through, not an error. Callers should treat ``None`` as "skip
    the live check".
    """
    creds = _neo4j_credentials()
    if creds is None:
        return None
    try:
        from neo4j import GraphDatabase  # type: ignore[import-untyped]
    except ImportError:
        return None

    uri, user, password = creds
    try:
        driver = GraphDatabase.driver(uri, auth=(user, password))
    except Exception:
        return None

    try:
        with driver.session() as session:
            labels = {record["label"] for record in session.run("CALL db.labels() YIELD label")}
            rels = {record["relationshipType"] for record in session.run("CALL db.relationshipTypes() YIELD relationshipType")}
    except Exception:
        return None
    finally:
        try:
            driver.close()
        except Exception:
            # Closing the driver is strictly best-effort cleanup —
            # we're already on the error path. If the underlying
            # connection is already torn down (or never opened),
            # surfacing that here would just shadow the real failure
            # from the calling block above.
            pass

    return labels, rels


# ---------------------------------------------------------------------------
# Validators
# ---------------------------------------------------------------------------


def _format_set(items: Iterable[str]) -> str:
    return ", ".join(sorted(items)) if items else "<none>"


def validate_yaml_internal(schema: Schema) -> list[str]:
    """Internal-consistency checks on the YAML itself."""
    errors: list[str] = []

    if schema.version != SCHEMA_VERSION:
        errors.append(
            f"schema version mismatch: YAML declares {schema.version!r}, "
            f"Go source declares {SCHEMA_VERSION!r}. Update the `version:` "
            f"field in schemas/graph-schema.yaml to match schema.go."
        )

    duplicates = [name for name in schema.node_labels if schema.node_labels.count(name) > 1]
    if duplicates:
        errors.append(f"duplicate node labels: {_format_set(set(duplicates))}")

    rel_duplicates = [name for name in schema.relationships if schema.relationships.count(name) > 1]
    if rel_duplicates:
        errors.append(f"duplicate relationships: {_format_set(set(rel_duplicates))}")

    for rel_name in schema.event_edges:
        props = schema.relationship_properties.get(rel_name, set())
        missing = EVENT_EDGE_REQUIRED - props
        if missing:
            errors.append(f"event-edge `{rel_name}` is missing required properties: {_format_set(missing)}")

    return errors


def compare_against_go(schema: Schema, go_parsed: tuple[set[str], set[str]]) -> list[str]:
    """Compare the YAML against the parsed Go enums, in both directions.

    This used to check one direction only — a YAML label missing from Go —
    and called it "intentionally lenient". The leniency had no upside: if Go
    declares a label the YAML already covers, the set difference is empty
    anyway, so the only thing the missing direction permitted was the failure
    case. And it is the direction the code actually moves in: someone adds a
    label to schema.go, ships it, and the YAML and docs silently fall behind
    while the gate reports the schema consistent. That happened — the Go
    source carried 28 labels against the YAML's 17 and this printed OK.

    Both directions now fail.
    """
    errors: list[str] = []
    go_labels, go_rels = go_parsed

    yaml_labels = set(schema.node_labels)
    yaml_rels = set(schema.relationships)

    missing_in_go_labels = yaml_labels - go_labels
    if missing_in_go_labels:
        errors.append(f"node labels declared in YAML but missing from Go source: {_format_set(missing_in_go_labels)}")

    missing_in_yaml_labels = go_labels - yaml_labels
    if missing_in_yaml_labels:
        errors.append(
            f"node labels declared in Go source but missing from YAML: "
            f"{_format_set(missing_in_yaml_labels)} — run "
            f"`python scripts/export_graph_schema.py --from-go` and update "
            f"apps/docs/docs/architecture/graph-schema.md"
        )

    missing_in_go_rels = yaml_rels - go_rels
    if missing_in_go_rels:
        errors.append(f"relationships declared in YAML but missing from Go source: {_format_set(missing_in_go_rels)}")

    missing_in_yaml_rels = go_rels - yaml_rels
    if missing_in_yaml_rels:
        errors.append(
            f"relationships declared in Go source but missing from YAML: "
            f"{_format_set(missing_in_yaml_rels)} — run "
            f"`python scripts/export_graph_schema.py --from-go` and update "
            f"apps/docs/docs/architecture/graph-schema.md"
        )

    return errors


def compare_against_live(schema: Schema, live_parsed: tuple[set[str], set[str]]) -> list[str]:
    """Compare the YAML against the live database.

    The live DB may contain labels not yet documented (drift in the other
    direction — Neo4j running ahead of the schema). Both directions are
    reported.
    """
    errors: list[str] = []
    live_labels, live_rels = live_parsed

    yaml_labels = set(schema.node_labels)
    yaml_rels = set(schema.relationships)

    in_db_not_yaml_labels = live_labels - yaml_labels
    if in_db_not_yaml_labels:
        errors.append(f"node labels materialised in Neo4j but not in YAML: {_format_set(in_db_not_yaml_labels)}")

    in_yaml_not_db_labels = yaml_labels - live_labels
    if in_yaml_not_db_labels:
        errors.append(
            "node labels declared in YAML but not yet materialised in Neo4j: "
            f"{_format_set(in_yaml_not_db_labels)} "
            "(non-fatal on a fresh DB — the writer materialises lazily)"
        )

    in_db_not_yaml_rels = live_rels - yaml_rels
    if in_db_not_yaml_rels:
        errors.append(f"relationships materialised in Neo4j but not in YAML: {_format_set(in_db_not_yaml_rels)}")

    return errors


# ---------------------------------------------------------------------------
# YAML writers
# ---------------------------------------------------------------------------


def dump_runtime_yaml(
    out_path: Path,
    *,
    source: str,
    labels: set[str],
    relationships: set[str],
    schema_version: str = SCHEMA_VERSION,
) -> None:
    """Dump the currently-materialised schema in a stable shape.

    The output is intentionally minimal: it captures what is *running*, not
    a re-spec of the source-of-truth YAML. The ``--check`` mode compares
    label/relationship *sets*, so a thin dump is enough.
    """
    yaml = _require_yaml()
    payload = {
        "version": schema_version,
        "generated_from": source,
        "node_labels": sorted(labels),
        "relationships": sorted(relationships),
    }
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with out_path.open("w", encoding="utf-8") as fh:
        fh.write("# Auto-generated by scripts/export_graph_schema.py — do not edit.\n")
        yaml.safe_dump(payload, fh, sort_keys=False)


def regenerate_yaml_from_go(go_parsed: tuple[set[str], set[str]], out_path: Path) -> None:
    """Regenerate the source-of-truth YAML from Go enums.

    Property declarations come from the existing on-disk YAML when the label
    or relationship is already known; brand-new entries get a stub
    ``properties: []`` that the author is expected to fill in. The point of
    ``--from-go`` is to get a fresh skeleton, not to replace human curation.
    """
    yaml = _require_yaml()
    labels, rels = go_parsed
    existing = load_yaml_schema(out_path) if out_path.exists() else Schema(version=SCHEMA_VERSION)

    existing_label_props: dict[str, list[dict]] = {}
    existing_rel_props: dict[str, list[dict]] = {}
    if out_path.exists():
        raw = yaml.safe_load(out_path.read_text(encoding="utf-8")) or {}
        for entry in raw.get("node_labels", []) or []:
            if isinstance(entry, dict) and entry.get("label"):
                existing_label_props[entry["label"]] = entry.get("properties", []) or []
        for entry in raw.get("relationships", []) or []:
            if isinstance(entry, dict) and entry.get("name"):
                existing_rel_props[entry["name"]] = entry.get("properties", []) or []

    out = {
        "version": existing.version or SCHEMA_VERSION,
        "node_labels": [
            {
                "label": label,
                "description": f"TODO: describe {label}",
                "properties": existing_label_props.get(label, []),
            }
            for label in sorted(labels)
        ],
        "relationships": [
            {
                "name": rel,
                "source_label": "TODO",
                "target_label": "TODO",
                "event_edge": False,
                "description": f"TODO: describe {rel}",
                "properties": existing_rel_props.get(rel, []),
            }
            for rel in sorted(rels)
        ],
    }
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with out_path.open("w", encoding="utf-8") as fh:
        fh.write(
            "# Regenerated from services/ingest/internal/graph/schema.go.\n"
            "# Fill in TODOs by hand — the script can extract names, not intent.\n"
        )
        yaml.safe_dump(out, fh, sort_keys=False)


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def _parse_args(argv: list[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description=(
            "Export, validate, and regenerate the AiSOC graph schema. "
            "Default mode: dump the currently materialised schema to "
            "schemas/graph-schema-current.yaml."
        ),
    )
    parser.add_argument(
        "--check",
        action="store_true",
        help="Compare YAML, Go source, and (if reachable) live Neo4j. Exit non-zero on drift.",
    )
    parser.add_argument(
        "--from-go",
        action="store_true",
        help="Regenerate schemas/graph-schema.yaml from the Go enums in services/ingest/internal/graph/schema.go.",
    )
    parser.add_argument(
        "--yaml-path",
        default=str(DEFAULT_YAML),
        help="Path to schemas/graph-schema.yaml (default: %(default)s).",
    )
    parser.add_argument(
        "--go-path",
        default=str(DEFAULT_GO_SOURCE),
        help="Path to the Go schema source (default: %(default)s).",
    )
    parser.add_argument(
        "--out-path",
        default=str(DEFAULT_CURRENT_YAML),
        help="Where to write the materialised-schema dump (default: %(default)s).",
    )
    parser.add_argument(
        "--json",
        action="store_true",
        help="Emit machine-readable JSON summary on stdout.",
    )
    return parser.parse_args(argv)


def _emit_human(args: argparse.Namespace, lines: list[str]) -> None:
    if args.json:
        return
    for line in lines:
        print(line)


def main(argv: list[str] | None = None) -> int:
    args = _parse_args(argv if argv is not None else sys.argv[1:])

    yaml_path = Path(args.yaml_path)
    go_path = Path(args.go_path)
    out_path = Path(args.out_path)

    if args.check and args.from_go:
        sys.stderr.write("--check and --from-go are mutually exclusive\n")
        return 2

    # --from-go mode ---------------------------------------------------------
    if args.from_go:
        go_parsed = parse_go_source(go_path)
        if go_parsed is None:
            sys.stderr.write(f"--from-go requires {go_path} to exist (T1.1 not landed yet?)\n")
            return 2
        regenerate_yaml_from_go(go_parsed, yaml_path)
        _emit_human(
            args,
            [
                f"regenerated {yaml_path} from {go_path}",
                f"  labels: {len(go_parsed[0])}",
                f"  relationships: {len(go_parsed[1])}",
            ],
        )
        if args.json:
            json.dump(
                {
                    "mode": "from-go",
                    "yaml_path": str(yaml_path),
                    "labels": sorted(go_parsed[0]),
                    "relationships": sorted(go_parsed[1]),
                },
                sys.stdout,
                indent=2,
            )
            print()
        return 0

    # Both --check and default mode want the YAML loaded.
    schema = load_yaml_schema(yaml_path)
    internal_errors = validate_yaml_internal(schema)

    go_parsed = parse_go_source(go_path)
    live_parsed = parse_live_neo4j()

    # --check mode -----------------------------------------------------------
    if args.check:
        errors = list(internal_errors)
        if go_parsed is not None:
            errors.extend(compare_against_go(schema, go_parsed))
        if live_parsed is not None:
            errors.extend(compare_against_live(schema, live_parsed))

        result = {
            "mode": "check",
            "schema_summary": schema.summary(),
            "go_source_present": go_parsed is not None,
            "neo4j_reachable": live_parsed is not None,
            "errors": errors,
        }
        if args.json:
            json.dump(result, sys.stdout, indent=2)
            print()
        else:
            print(f"schema: {schema.summary()}")
            print(
                "go source: "
                + (
                    f"present ({len(go_parsed[0])} labels, {len(go_parsed[1])} rels)"
                    if go_parsed is not None
                    else f"missing — skipping (expected {go_path})"
                )
            )
            print(
                "live db: "
                + (
                    f"reachable ({len(live_parsed[0])} labels, {len(live_parsed[1])} rels)"
                    if live_parsed is not None
                    else "unreachable — skipping"
                )
            )
            if errors:
                print()
                print("DRIFT DETECTED:")
                for err in errors:
                    print(f"  - {err}")
            else:
                print()
                print("OK: schema is consistent across all available representations.")

        return 1 if errors else 0

    # Default mode -----------------------------------------------------------
    if live_parsed is not None:
        labels, rels = live_parsed
        source = "neo4j"
    elif go_parsed is not None:
        labels, rels = go_parsed
        source = "go"
    else:
        # Last-resort fallback: dump what's in the YAML so we always produce
        # an output file. This is the bootstrap case where neither T1.1 nor
        # the database is up yet.
        labels = set(schema.node_labels)
        rels = set(schema.relationships)
        source = "yaml"

    dump_runtime_yaml(
        out_path,
        source=source,
        labels=labels,
        relationships=rels,
    )

    summary = {
        "mode": "dump",
        "source": source,
        "out_path": str(out_path),
        "labels": sorted(labels),
        "relationships": sorted(rels),
    }
    if args.json:
        json.dump(summary, sys.stdout, indent=2)
        print()
    else:
        print(f"wrote {out_path} (source={source})")
        print(f"  labels: {len(labels)}")
        print(f"  relationships: {len(rels)}")

    return 0


if __name__ == "__main__":  # pragma: no cover
    raise SystemExit(main())