Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""Generate the single source of truth for AiSOC's first-party connector count.
Why this exists
---------------
Before this script, five different surfaces in the repo each carried their own
hard-coded "N connectors" claim:
* ``README.md`` (52)
* ``apps/docs/docs/connectors/index.md`` (50)
* ``apps/web/src/components/landing/sections/Hero.tsx`` (69)
* ``apps/web/src/components/landing/sections/FeatureGrid.tsx`` (69)
* ``apps/web/src/components/landing/sections/ConnectorsMarquee.tsx`` (69)
The registry of record is ``services/connectors/app/connectors/__init__.py`` —
``_CONNECTOR_CLASSES`` is the tuple that backs ``CONNECTOR_REGISTRY``. This
script reads that tuple via the Python AST (so the script has no runtime
dependency on the ``connectors`` package itself) and writes:
* ``apps/web/src/data/connector-count.json`` — machine-readable count.
* ``apps/web/src/data/connectorCount.ts`` — TypeScript constants
(``CONNECTOR_COUNT`` + ``CONNECTOR_CATEGORIES``) imported by every
landing / pricing / FAQ surface.
Usage
-----
python3 scripts/generate_connector_count.py # write outputs
python3 scripts/generate_connector_count.py --check # fail if drift
The ``--check`` mode is wired into ``.github/workflows/ci.yml`` so any time
someone adds a connector class without regenerating the count, CI catches it.
"""
from __future__ import annotations
import argparse
import ast
import json
import re
import sys
from collections import Counter
from pathlib import Path
from typing import Any
# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from gate_toolkit import repo_root, self_test_if_requested
self_test_if_requested(__file__)
REPO_ROOT = repo_root()
REGISTRY_FILE = REPO_ROOT / "services" / "connectors" / "app" / "connectors" / "__init__.py"
CONNECTORS_DIR = REPO_ROOT / "services" / "connectors" / "app" / "connectors"
JSON_OUT = REPO_ROOT / "apps" / "web" / "src" / "data" / "connector-count.json"
TS_OUT = REPO_ROOT / "apps" / "web" / "src" / "data" / "connectorCount.ts"
# Files that quote a connector count in prose. We rewrite the number in place,
# and CI verifies no other surface diverges. Each entry is a regex that MUST
# match exactly one number per file, in the form ``\b<count>\b`` plus a
# disambiguating prefix/suffix so we never rewrite an unrelated integer.
COUNT_BEARING_FILES: tuple[tuple[Path, tuple[re.Pattern[str], ...]], ...] = (
(
# README went on a Phase-2 diet: the "declares **N first-party
# connectors**" and "a N-connector click-and-connect catalog"
# phrasings were absorbed into RELEASES.md / ROADMAP.md (already
# covered below). The lean README now anchors the count in a single
# bullet near the top of "What's in the box".
REPO_ROOT / "README.md",
(re.compile(r"(?P<pre>\*\*)(?P<n>\d+)(?P<post> click-and-connect data connectors\*\*)"),),
),
(
# The "corrected **N-connector count**" prose used to live in README.md
# but was moved to RELEASES.md when the "What's new" section was
# collapsed under <details>. The anchor still needs to be reconciled
# against the registry, so we track it where it actually lives now.
REPO_ROOT / "RELEASES.md",
(
re.compile(r"(?P<pre>corrected \*\*)(?P<n>\d+)(?P<post>-connector count\*\*)"),
re.compile(r"(?P<pre>declares \*\*)(?P<n>\d+)(?P<post> first-party connectors)"),
),
),
(
REPO_ROOT / "apps" / "docs" / "docs" / "connectors" / "index.md",
(re.compile(r"(?P<pre>with \*\*)(?P<n>\d+) connectors(?P<post>\*\*)"),),
),
(
REPO_ROOT / "apps" / "docs" / "docs" / "connectors" / "endpoint-decision-matrix.md",
(re.compile(r"(?P<pre>the full )(?P<n>\d+)(?P<post>-connector catalog)"),),
),
(
REPO_ROOT / "apps" / "docs" / "docs" / "connectors" / "api-coverage.md",
(re.compile(r"(?P<pre>## Coverage table — )(?P<n>\d+)(?P<post> connectors)"),),
),
(
REPO_ROOT / "apps" / "docs" / "docs" / "architecture.md",
(re.compile(r"(?P<pre>now )(?P<n>\d+) connectors(?P<post>\*\*)"),),
),
(
REPO_ROOT / "apps" / "docs" / "docs" / "intro.md",
(re.compile(r"(?P<pre>a \*\*)(?P<n>\d+)(?P<post>-connector\*\* catalog)"),),
),
(
REPO_ROOT / "ROADMAP.md",
(re.compile(r"(?P<pre>declares \*\*)(?P<n>\d+)(?P<post> first-party connectors)"),),
),
(
REPO_ROOT / "docs" / "architecture" / "SYSTEM_DESIGN.md",
(
re.compile(r"(?P<pre>│ )(?P<n>\d+)(?P<post> connector classes│)"),
re.compile(r"(?P<pre>\| Python 3\.11 \| )(?P<n>\d+)(?P<post> connector classes,)"),
re.compile(r"(?P<pre>with \*\*)(?P<n>\d+)(?P<post> registered connector classes\*\*)"),
),
),
)
def parse_registry_count() -> int:
"""Parse ``_CONNECTOR_CLASSES`` length via AST (no runtime imports)."""
source = REGISTRY_FILE.read_text(encoding="utf-8")
tree = ast.parse(source)
for node in tree.body:
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
if node.target.id == "_CONNECTOR_CLASSES" and isinstance(node.value, ast.Tuple):
return len(node.value.elts)
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id == "_CONNECTOR_CLASSES":
if isinstance(node.value, ast.Tuple):
return len(node.value.elts)
raise RuntimeError(f"could not find _CONNECTOR_CLASSES tuple in {REGISTRY_FILE.relative_to(REPO_ROOT)}")
_CATEGORY_PATTERN = re.compile(r'^\s*connector_category\s*=\s*"([a-z][a-z0-9_-]*)"', re.MULTILINE)
def parse_category_counts() -> dict[str, int]:
"""Count ``connector_category = "..."`` declarations across registered files."""
counts: Counter[str] = Counter()
for py in sorted(CONNECTORS_DIR.glob("*.py")):
if py.name in {"__init__.py", "base.py"}:
continue
text = py.read_text(encoding="utf-8")
match = _CATEGORY_PATTERN.search(text)
if match:
counts[match.group(1)] += 1
return dict(sorted(counts.items()))
def build_payload() -> dict[str, Any]:
"""Produce the on-disk payload.
`Any` rather than `object`: this is a JSON document with heterogeneous
values, and declaring `object` forced the two read sites below to carry
`# type: ignore[arg-type]` comments for an error code mypy does not emit
here — a suppression that suppresses nothing while telling the next
reader a checker was consulted and agreed.
"""
count = parse_registry_count()
categories = parse_category_counts()
return {
"count": count,
"categories": categories,
"generatedFrom": "services/connectors/app/connectors/__init__.py",
"regenerateWith": "python3 scripts/generate_connector_count.py",
}
def render_typescript(payload: dict[str, object]) -> str:
"""Render the TS constants module imported by the landing/pricing UI."""
return (
"// AUTO-GENERATED by scripts/generate_connector_count.py.\n"
"// Do not edit by hand. Run the script (or `make connector-count`) instead.\n"
"//\n"
"// This module is the single source of truth for every prose claim of the\n"
'// form "N connectors" or "All N connectors" across the marketing site,\n'
"// docs portal, and pricing surfaces. The count itself comes from the\n"
"// _CONNECTOR_CLASSES tuple in services/connectors/app/connectors/__init__.py.\n"
"\n"
"import data from './connector-count.json';\n"
"\n"
"export const CONNECTOR_COUNT: number = data.count;\n"
"export const CONNECTOR_CATEGORIES: Readonly<Record<string, number>> ="
" data.categories;\n"
"\n"
"// Human-readable label used in marquees / chips.\n"
"export const CONNECTOR_COUNT_LABEL = `${CONNECTOR_COUNT} connectors`;\n"
)
def write_outputs(payload: dict[str, object]) -> None:
JSON_OUT.parent.mkdir(parents=True, exist_ok=True)
JSON_OUT.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
TS_OUT.write_text(render_typescript(payload), encoding="utf-8")
def reconcile_prose(count: int, *, check_only: bool) -> list[str]:
"""Rewrite or verify every COUNT_BEARING_FILES entry.
Returns a list of human-readable drift messages. Empty list means clean.
"""
drift: list[str] = []
for path, patterns in COUNT_BEARING_FILES:
if not path.exists():
drift.append(f"missing file {path.relative_to(REPO_ROOT)}")
continue
before = path.read_text(encoding="utf-8")
after = before
for pattern in patterns:
matches = list(pattern.finditer(after))
if not matches:
drift.append(f"{path.relative_to(REPO_ROOT)}: pattern {pattern.pattern!r} did not match")
continue
for match in matches:
if match.group("n") != str(count):
after = after[: match.start("n")] + str(count) + after[match.end("n") :]
if after != before:
if check_only:
drift.append(f"{path.relative_to(REPO_ROOT)}: connector count drift (expected {count})")
else:
path.write_text(after, encoding="utf-8")
return drift
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--check",
action="store_true",
help="Fail (exit 1) on any drift instead of rewriting files.",
)
args = parser.parse_args(argv)
payload = build_payload()
count = int(payload["count"])
drift: list[str] = []
if args.check:
json_existing = json.loads(JSON_OUT.read_text(encoding="utf-8")) if JSON_OUT.exists() else None
if json_existing != payload:
drift.append(f"{JSON_OUT.relative_to(REPO_ROOT)}: regenerate with `python3 scripts/generate_connector_count.py`")
ts_existing = TS_OUT.read_text(encoding="utf-8") if TS_OUT.exists() else ""
if ts_existing != render_typescript(payload):
drift.append(f"{TS_OUT.relative_to(REPO_ROOT)}: regenerate with `python3 scripts/generate_connector_count.py`")
else:
write_outputs(payload)
drift.extend(reconcile_prose(count, check_only=args.check))
if drift:
print(f"connector-count drift detected (registry={count}):", file=sys.stderr)
for line in drift:
print(f" - {line}", file=sys.stderr)
return 1
print(f"connector-count OK — {count} registered, {len(payload['categories'])} categories")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))