Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

8.5 KB
Raw Blame
#!/usr/bin/env python3
"""Gate: count the response arms the actions service actually registers.

Depth 5.5 (carried to wave-2 Phase 6.1) sets a floor on executor arms and
names the counting rule, because the number had already been quoted three
ways. A loose grep over ``services/actions/app/live_actions/`` also matches
``reverse_capability=`` arguments and reports 111; the declared rule is
class-level ``capability = "..."`` declarations, which reports 89. Quoting
the larger figure would publish 22 arms that do not exist.

So this counts twice and makes the two agree:

``declared``
    An ``ast`` walk for class-level ``capability = "<literal>"`` under
    ``services/actions/app/live_actions/``. This is the plan's rule, kept
    so the published figure stays comparable with the baseline it is
    measured against. It never imports, so it cannot be fooled by an
    import-time side effect.

``registered``
    The adapters in ``builtins._BUILTIN_ADAPTERS``, which is what
    ``register_builtin_executors()`` puts in the registry and therefore what
    a dispatch can actually reach.

A class that declares a capability and is never registered answers
``executor_not_found`` at dispatch while counting towards the headline — the
exact shape of padding this gate exists to refuse — so the two sets must be
identical, not merely the same size.

The artefact is generated, never hand-edited: ``--write`` regenerates
``docs/audit/executor-arms.json`` and ``--check`` fails when the committed
file disagrees with the tree.

Run:  python3 scripts/check_executor_arm_count.py --check
      python3 scripts/check_executor_arm_count.py --write
"""

from __future__ import annotations

import argparse
import ast
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

from gate_toolkit import repo_root, self_test_if_requested

self_test_if_requested(__file__)

REPO_ROOT = repo_root()
LIVE_ACTIONS = REPO_ROOT / "services" / "actions" / "app" / "live_actions"
ARTEFACT = REPO_ROOT / "docs" / "audit" / "executor-arms.json"

#: The floor wave-2 Phase 6.1 sets, which is depth 5.5's floor. A floor is
#: only a gate while it is above the tree; this one was 89 when it was
#: written, which is the measurement that made the phase necessary.
REQUIRED_ARMS = 150


def declared_arms() -> list[tuple[str, str, str]]:
    """``(module, class, capability)`` for every class-level declaration.

    Pure source analysis, matching the plan's counting rule exactly. A
    ``capability`` assigned anything but a non-empty string literal is not a
    declaration — ``capability: str = ""`` on the base class is the shape
    that would otherwise inflate the count by one per abstract class.
    """
    found: list[tuple[str, str, str]] = []
    for path in sorted(LIVE_ACTIONS.glob("*.py")):
        tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
        for node in ast.walk(tree):
            if not isinstance(node, ast.ClassDef):
                continue
            for stmt in node.body:
                if not isinstance(stmt, ast.Assign) or len(stmt.targets) != 1:
                    continue
                target = stmt.targets[0]
                if not isinstance(target, ast.Name) or target.id != "capability":
                    continue
                value = stmt.value
                if isinstance(value, ast.Constant) and isinstance(value.value, str) and value.value:
                    found.append((path.stem, node.name, value.value))
    return sorted(found)


def registered_arms() -> list[dict[str, str]]:
    """``vendor_id``/``capability`` for every adapter the registry gets.

    Imports the actions service the way ``check_action_contract.py`` does.
    Eight adapters take their ``vendor_id`` from a module constant rather
    than a literal, so this is the only honest way to read the vendor —
    which is also why the artefact is written from here and the ``ast`` walk
    only supplies the count.
    """
    sys.path.insert(0, str(REPO_ROOT / "services" / "actions"))
    from app.live_actions.builtins import _BUILTIN_ADAPTERS  # noqa: PLC0415 — needs the sys.path edit above

    return sorted(
        (
            {
                "module": cls.__module__.rsplit(".", 1)[-1],
                "class": cls.__name__,
                "vendor_id": str(cls.vendor_id),
                "capability": str(cls.capability),
            }
            for cls in _BUILTIN_ADAPTERS
        ),
        key=lambda row: (row["capability"], row["vendor_id"], row["class"]),
    )


def build() -> dict:
    arms = registered_arms()
    return {
        "_comment": ("Generated by scripts/check_executor_arm_count.py --write. Do not hand-edit; regenerate."),
        "counting_rule": (
            'Class-level `capability = "..."` declarations under '
            "services/actions/app/live_actions/, cross-checked against "
            "builtins._BUILTIN_ADAPTERS."
        ),
        "arms": len(arms),
        "verbs": len({row["capability"] for row in arms}),
        "vendors": len({row["vendor_id"] for row in arms}),
        "required_minimum": REQUIRED_ARMS,
        "by_capability": {
            capability: sorted(row["vendor_id"] for row in arms if row["capability"] == capability)
            for capability in sorted({row["capability"] for row in arms})
        },
        "registry": arms,
    }


def problems() -> list[str]:
    errors: list[str] = []
    declared = declared_arms()
    registered = registered_arms()

    if not declared:
        return [
            f"no capability declarations found under {LIVE_ACTIONS}. A gate that "
            f"scans nothing reports the same word as a gate that found nothing, "
            f"so this refuses rather than passing."
        ]

    declared_classes = {(module, cls) for module, cls, _ in declared}
    registered_classes = {(row["module"], row["class"]) for row in registered}

    for module, cls in sorted(declared_classes - registered_classes):
        errors.append(
            f"{module}.{cls} declares a capability and is not in "
            f"_BUILTIN_ADAPTERS, so it counts towards the arm total and "
            f"answers executor_not_found at dispatch. Register it or stop "
            f"declaring it."
        )
    for module, cls in sorted(registered_classes - declared_classes):
        errors.append(
            f"{module}.{cls} is registered but declares no class-level "
            f"capability literal, so the declared-rule count cannot see it "
            f"and the published figure is lower than the surface."
        )

    if len(declared) < REQUIRED_ARMS:
        errors.append(
            f"{len(declared)} executor arms over {len({c for _, _, c in declared})} "
            f"verbs; the plan's floor is {REQUIRED_ARMS}. Counted by the declared "
            f'rule (class-level `capability = "..."`), not the looser grep that '
            f"also matches reverse_capability arguments."
        )

    if ARTEFACT.is_file():
        committed = json.loads(ARTEFACT.read_text(encoding="utf-8"))
        if committed != build():
            errors.append(
                f"{ARTEFACT.relative_to(REPO_ROOT)} disagrees with the tree. "
                f"Regenerate it with --write; never hand-edit a generated figure."
            )
    else:
        errors.append(f"{ARTEFACT.relative_to(REPO_ROOT)} is missing. Generate it with --write.")

    return errors


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--check", action="store_true", help="fail when the tree disagrees with the artefact or the floor")
    parser.add_argument("--write", action="store_true", help="regenerate the artefact from the tree")
    args = parser.parse_args()

    if args.write:
        ARTEFACT.parent.mkdir(parents=True, exist_ok=True)
        ARTEFACT.write_text(json.dumps(build(), indent=2) + "\n", encoding="utf-8")
        payload = build()
        print(
            f"wrote {ARTEFACT.relative_to(REPO_ROOT)}: {payload['arms']} arms over {payload['verbs']} verbs, {payload['vendors']} vendors"
        )
        return 0

    errors = problems()
    if errors:
        print("executor-arm-count: FAIL\n", file=sys.stderr)
        for error in errors:
            print(f"  {error}", file=sys.stderr)
        print("", file=sys.stderr)
        return 1

    payload = build()
    print(
        f"executor-arm-count: OK — {payload['arms']} arms over "
        f"{payload['verbs']} verbs across {payload['vendors']} vendors "
        f"(floor {REQUIRED_ARMS})"
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())