Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""Generate a setup page per connector from its schema, and gate the coverage.
35 of the 84 connectors had no documentation page. Writing 35 by hand is the
obvious response and the wrong one: a hand-written page states the fields a
connector needed on the day it was written, and says nothing when a field is
added, renamed or made secret. The catalogue would be documented once and
wrong thereafter.
The schema is already the source of truth — the setup wizard renders from it,
the marketplace manifest mirrors it, and the API validates against it. So the
page is generated from the same place, and `--check` fails the build when a
connector has no page or a page has drifted from its schema.
What is generated is the mechanical half: fields, which are secret, required
versus optional, capabilities, category, the vendor docs link. What a
generator cannot know — which console page issues the credential, which
scopes to tick, what the vendor calls its rate limit — goes in a hand-written
block the generator preserves across regeneration. Overwriting that on every
run would make the human half unwritable, which is how generated docs end up
worse than none.
Pages written before the generator are richer than it can produce and are
left alone. They are not exempt from checking, though: the gate asserts every
**required** and every **secret** field in the schema is mentioned somewhere
in the page. That catches the drift that matters — a new required field
nobody documented, or a field that became secret without the page saying so —
without dictating prose to a page a human wrote well.
Run: python3 scripts/generate_connector_docs.py [--check]
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from gate_toolkit import repo_root, self_test_if_requested
self_test_if_requested(__file__)
REPO_ROOT = repo_root()
CONNECTORS_SRC = REPO_ROOT / "services" / "connectors"
DOCS_DIR = REPO_ROOT / "apps" / "docs" / "docs" / "connectors"
SIDEBAR = REPO_ROOT / "apps" / "docs" / "sidebars.ts"
sys.path.insert(0, str(CONNECTORS_SRC))
#: Everything between these markers survives regeneration. A generator that
#: clobbers the human half makes the human half not worth writing.
HUMAN_START = "<!-- BEGIN HUMAN NOTES -->"
HUMAN_END = "<!-- END HUMAN NOTES -->"
GENERATED_WARNING = (
"{/* Generated by scripts/generate_connector_docs.py from the connector's\n"
" schema(). Edit the schema, not this block. Prose between the HUMAN\n"
" NOTES markers below is preserved across regeneration. */}"
)
#: Pages that predate the generator and are richer than it can produce.
#: Listed rather than silently skipped, so the exemption is reviewable.
HAND_WRITTEN: frozenset[str] = frozenset()
#: Ownership marker. A fully-generated page carries the HUMAN NOTES block;
#: a hand-written page does not. Deliberately not the "Generated by" string,
#: which also appears in the field reference injected into hand-written pages
#: — keying on that made every augmented page look generator-owned on the
#: next run, and the generator then offered to overwrite the prose it had
#: just been careful to preserve.
GENERATOR_MARKER = HUMAN_START
#: A schema-generated field table appended to a hand-written page. Bounded by
#: markers so it can be regenerated without touching the prose around it.
#:
#: Eighteen hand-written pages omitted a required or secret field — several
#: omitted the credential itself. Hand-editing them fixes today and drifts
#: again at the next schema change, which is how they got there. Injecting a
#: generated table keeps the human page and makes the field list a derived
#: artifact.
REF_START = "<!-- BEGIN GENERATED FIELD REFERENCE -->"
REF_END = "<!-- END GENERATED FIELD REFERENCE -->"
def undescribed_fields(schema, page: str) -> list[str]:
"""Required or secret schema fields that the page never mentions.
Deliberately a substring test rather than a structural one. The point is
to catch a field nobody documented, not to dictate how a human writes a
page that already reads well — and a gate that demands a table would push
every author towards the generated shape, which would make keeping the
hand-written pages pointless.
"""
lowered = page.lower()
gaps = []
for field in schema.fields:
if not (field.required or _is_secret(field)):
continue
if field.name.lower() in lowered:
continue
# Some pages name the environment variable rather than the field.
if field.name.upper() in page:
continue
gaps.append(field.name)
return gaps
def unescaped_jsx_tags(page: str) -> list[str]:
"""Angle-bracket tags MDX would read as JSX, outside code.
Docusaurus parses `.md` under MDX, so `<account>` in a vendor's help
text is an unclosed JSX element and the **entire docs build** fails —
on a page the generator wrote, for a string the vendor chose. That is
a build break nobody can trace to its cause without reading the MDX
error, so it is checked here rather than discovered on deploy.
"""
found: list[str] = []
in_fence = False
for number, line in enumerate(page.splitlines(), 1):
if line.lstrip().startswith("```"):
in_fence = not in_fence
continue
if in_fence:
continue
outside = "".join(part for part in re.split(r"(`{1,3}[^`]*`{1,3})", line) if not part.startswith("`"))
for match in re.finditer(r"<([A-Za-z][\w.-]*)>", outside):
# Real HTML that MDX accepts.
if match.group(1).lower() in {"br", "hr", "img"}:
continue
found.append(f"line {number}: {match.group(0)}")
return found
def _mdx_safe(text: str) -> str:
"""Escape angle brackets so MDX does not read them as JSX.
Vendor help text carries placeholders like `<account>` and
`<your-tenant>`. Docusaurus parses `.md` under MDX, so an unclosed
`<account>` is a JSX tag with no closing partner and the whole docs
build fails — on a page the generator wrote, for a string the vendor
chose.
Wrapped in backticks rather than HTML-escaped: `<account>` is
correct and unreadable, and a placeholder is code-shaped anyway.
Segments already inside a code span are left alone — MDX does not
parse JSX there, and adding backticks inside backticks produces a
nested span that renders as literal punctuation.
"""
# Split on code spans (longest fences first) and escape only outside.
parts = re.split(r"(`{1,3}[^`]*`{1,3})", text)
return "".join(part if part.startswith("`") else re.sub(r"<([A-Za-z][\w.-]*)>", r"`<\1>`", part) for part in parts)
def _is_secret(field) -> bool:
"""Whether a field holds a credential.
Checked two ways because the schema expresses it two ways: most
connectors set ``type="secret"``, a few carry a ``secret=True``
attribute. Reading only one missed every ``type="secret"`` field, which
is most of them — so the generated pages said nothing about credential
handling for exactly the fields that needed it.
"""
return getattr(field, "secret", False) or str(getattr(field, "type", "")) == "secret"
def duplicate_slugs() -> list[tuple[str, list[str]]]:
"""Pages that differ only by separator.
Six connectors had a hand-written page using underscores while the
generator assumed hyphens, so it saw no page and wrote a second,
thinner one beside the original — and listed only its own in the
sidebar, dropping the better half of each pair out of navigation.
"""
groups: dict[str, list[str]] = {}
for path in sorted(DOCS_DIR.glob("*.md")):
groups.setdefault(path.stem.replace("_", "-"), []).append(path.stem)
return [(key, names) for key, names in groups.items() if len(names) > 1]
def _slug(connector_id: str) -> str:
"""Doc filename for a connector id.
Hyphens are the convention, but six pages predate it and use
underscores. Assuming the convention meant the coverage check saw no
page for those connectors and generated a second, thinner one beside
the hand-written original — six duplicate pages, with the better half
of each pair dropped from the sidebar.
So an existing page wins over the convention, whichever separator it
used. Only a connector with no page at all gets the hyphenated form.
"""
hyphenated = connector_id.replace("_", "-")
if (DOCS_DIR / f"{hyphenated}.md").exists():
return hyphenated
underscored = connector_id.replace("-", "_")
if (DOCS_DIR / f"{underscored}.md").exists():
return underscored
return hyphenated
def _human_block(path: Path) -> str:
"""Preserve the hand-written section from an existing page."""
if not path.exists():
return ""
text = path.read_text(encoding="utf-8")
match = re.search(re.escape(HUMAN_START) + r"(.*?)" + re.escape(HUMAN_END), text, re.S)
return match.group(1).strip() if match else ""
def _field_rows(schema) -> list[str]:
rows = []
for field in schema.fields:
secret = _is_secret(field)
required = "yes" if field.required else "no"
default = f"`{field.default}`" if field.default not in (None, "") else "—"
notes = []
if secret:
# The single most important thing a reader needs to know about a
# field, so it is not buried in a help string.
notes.append("**encrypted at rest**")
if getattr(field, "help_text", None):
notes.append(_mdx_safe(str(field.help_text)))
options = getattr(field, "options", None)
if options:
# Options are either plain strings or {value,label} dicts; render
# the value either way. Dumping the dict put a JSON blob in a
# table cell, which is how a generated doc becomes less readable
# than no doc.
values = [str(o.get("value", o)) if isinstance(o, dict) else str(o) for o in options]
shown = ", ".join(f"`{v}`" for v in values[:8])
if len(values) > 8:
shown += f", … ({len(values)} total)"
notes.append("one of: " + shown)
rows.append(
f"| `{field.name}` | {_mdx_safe(str(field.label or field.name))} "
f"| {field.type} | "
f"{required} | {default} | {' · '.join(notes) or '—'} |"
)
return rows
def render(schema, human: str) -> str:
title = schema.connector_name or schema.connector_id
secret_fields = [f for f in schema.fields if _is_secret(f)]
capabilities = sorted(str(c.value if hasattr(c, "value") else c) for c in (schema.capabilities or []))
lines = [
"---",
f"title: {title}",
f"description: Connect {title} to AiSOC.",
"---",
"",
f"# {title}",
"",
GENERATED_WARNING,
"",
_mdx_safe(schema.description or f"Ingest security telemetry from {title}."),
"",
f"- **Connector id:** `{schema.connector_id}`",
f"- **Category:** {schema.category}",
]
# Several connectors set docs_url to this page's own path rather than the
# vendor's documentation. Rendering that produces a link to itself, which
# is worse than no link because it looks like it goes somewhere.
docs_url = str(schema.docs_url or "")
if docs_url and not docs_url.startswith(("/docs/", "docs/")):
lines.append(f"- **Vendor documentation:** {docs_url}")
if getattr(schema, "oauth", None):
lines.append("- **Authentication:** OAuth")
lines += ["", "## Configuration", ""]
if schema.fields:
lines += [
"| Field | Label | Type | Required | Default | Notes |",
"|-------|-------|------|----------|---------|-------|",
*_field_rows(schema),
"",
]
else:
lines += ["This connector takes no configuration.", ""]
if secret_fields:
names = ", ".join(f"`{f.name}`" for f in secret_fields)
lines += [
":::note Credential handling",
(
f"{names} " + ("is" if len(secret_fields) == 1 else "are") + " encrypted at rest by the credential vault before the row is "
"written, and decrypted only at poll time. The value is never "
"logged and never returned by the API — a read of the saved "
"connector shows the field as set, not its contents. See "
"[credential vault](../operations/credentials.md)."
),
":::",
"",
]
if capabilities:
lines += [
"## Capabilities",
"",
"\n".join(f"- `{c}`" for c in capabilities),
"",
]
# Built as named strings rather than inline. Adjacent string literals
# inside a list are implicitly concatenated, which is indistinguishable
# from a missing comma — the reader cannot tell three list items from
# one, and neither can a reviewer.
step_pick = f"1. In the console, go to **Connectors → Add connector** and pick **{title}**."
step_test = "2. Fill in the fields above and run **Test connection**. The test is read-only and does not save anything."
step_save = (
"3. Save. Polling starts on a jittered schedule within about a minute; "
"the connector row shows `last_sync` once the first poll completes."
)
empty_sync_tip = (
"The first poll is the honest signal that the credential works. A "
"connector that saves cleanly and never syncs usually means the "
"credential lacks a scope the vendor requires for reads — check the "
"connectors service log for the vendor's own error, which is passed "
"through verbatim rather than summarised."
)
lines += [
"## Setup",
"",
step_pick,
step_test,
step_save,
"",
":::tip If `last_sync` stays empty",
empty_sync_tip,
":::",
"",
HUMAN_START,
"",
human
or (
"_Vendor-specific setup notes go here: which console page issues the "
"credential, which scopes to grant, rate limits, and anything the "
"schema cannot express. This section is preserved when the page is "
"regenerated._"
),
"",
HUMAN_END,
"",
]
return "\n".join(lines)
def render_field_reference(schema) -> str:
"""The schema-derived field table, for appending to a hand-written page."""
secret_fields = [f for f in schema.fields if _is_secret(f)]
lines = [
REF_START,
"",
"## Configuration reference",
"",
"{/* Generated by scripts/generate_connector_docs.py from the connector's",
" schema(). Edit the schema, not this table. */}",
"",
]
if schema.fields:
lines += [
"| Field | Label | Type | Required | Default | Notes |",
"|-------|-------|------|----------|---------|-------|",
*_field_rows(schema),
"",
]
else:
lines += ["This connector takes no configuration.", ""]
if secret_fields:
names = ", ".join(f"`{f.name}`" for f in secret_fields)
lines += [
":::note Credential handling",
(
f"{names} " + ("is" if len(secret_fields) == 1 else "are") + " encrypted at rest by the credential vault before the row is "
"written, and decrypted only at poll time. The value is never "
"logged and never returned by the API. See "
"[credential vault](../operations/credentials.md)."
),
":::",
"",
]
lines.append(REF_END)
return "\n".join(lines)
def augment(page: str, schema) -> str:
"""Insert or refresh the generated field reference in a hand-written page."""
block = render_field_reference(schema)
if REF_START in page and REF_END in page:
head, _, rest = page.partition(REF_START)
_, _, tail = rest.partition(REF_END)
return head + block + tail
return page.rstrip("\n") + "\n\n" + block + "\n"
#: Sidebar entries that are not per-connector pages and keep their position
#: at the top of the category.
SIDEBAR_PREAMBLE = (
"connectors/index",
"connectors/hello-connector",
"connectors/endpoint-decision-matrix",
"connectors/api-coverage",
"connectors/universal-capture",
# A transport rather than a connector: the syslog listener in
# services/ingest serves every appliance that speaks RFC 5424, RFC 3164,
# CEF or LEEF, so there is no schema() for the generated list below to
# pick up and the page has to be named here or it drops out of
# navigation on the next regeneration.
"connectors/syslog-listener",
# The same reason, for the two receivers depth 4.2 added: the OTLP log
# endpoint and the Kafka input are transports in services/ingest with no
# schema() either, so they have to be named here or the generator drops
# their hand-written pages from navigation.
"connectors/otlp-logs",
"connectors/kafka-input",
# Not a connector, so the generated list below does not cover it.
"connectors/osquery-extensions",
# A push-side inbox template rather than a connector: Agentmetry's own
# webhook sink POSTs to /v1/inbox/{token}, so there is no schema() for
# the generated list below to pick up and the page has to be named here
# or it drops out of navigation on the next regeneration.
"connectors/agentmetry",
)
SIDEBAR_START = "// BEGIN GENERATED CONNECTOR LIST"
SIDEBAR_END = "// END GENERATED CONNECTOR LIST"
def render_sidebar_entries(schemas) -> str:
"""The connector category's item list, one entry per page.
The sidebar carried 34 entries against 96 pages, so most connector
documentation existed and could not be navigated to. A page nobody can
reach is barely a page, and the gap is invisible unless something counts
both sides — which is what makes this generated rather than curated.
"""
entries = [f' "{e}",' for e in SIDEBAR_PREAMBLE]
for schema in schemas:
entries.append(f' "connectors/{_slug(schema.connector_id)}",')
return "\n".join([f" {SIDEBAR_START}", *entries, f" {SIDEBAR_END}"])
def sync_sidebar(schemas, *, check: bool) -> bool:
"""Rewrite the connector category's items. Returns True when in sync."""
if not SIDEBAR.exists():
return True
source = SIDEBAR.read_text(encoding="utf-8")
block = render_sidebar_entries(schemas)
if SIDEBAR_START in source and SIDEBAR_END in source:
head, _, rest = source.partition(f" {SIDEBAR_START}")
_, _, tail = rest.partition(f" {SIDEBAR_END}")
updated = head + block + tail
else:
marker = 'label: "Connectors",'
if marker not in source:
return True
start = source.index(marker)
items_at = source.index("items: [", start)
close_at = source.index("],", items_at)
updated = source[: items_at + len("items: [")] + "\n" + block + "\n " + source[close_at:]
if updated == source:
return True
if check:
return False
SIDEBAR.write_text(updated, encoding="utf-8")
return True
def load_schemas() -> list:
from app.connectors import _CONNECTOR_CLASSES
schemas = []
for cls in _CONNECTOR_CLASSES:
try:
schemas.append(cls.schema())
except Exception as exc: # noqa: BLE001 - reported, never fatal
print(f" warning: {cls.__name__}.schema() raised {type(exc).__name__}", file=sys.stderr)
return sorted(schemas, key=lambda s: s.connector_id)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--check",
action="store_true",
help="Fail if any connector has no page, or a page has drifted from its schema.",
)
args = parser.parse_args(argv)
try:
schemas = load_schemas()
except ImportError as exc:
print(f"connector-docs: cannot import the connectors package: {exc}", file=sys.stderr)
return 2
DOCS_DIR.mkdir(parents=True, exist_ok=True)
missing: list[str] = []
drifted: list[str] = []
undocumented_fields: list[tuple[str, list[str]]] = []
mdx_breaks: list[tuple[str, list[str]]] = []
written = 0
generated_pages = 0
hand_written_seen = 0
for schema in schemas:
if schema.connector_id in HAND_WRITTEN:
continue
path = DOCS_DIR / f"{_slug(schema.connector_id)}.md"
existing = path.read_text(encoding="utf-8") if path.exists() else ""
generator_owned = GENERATOR_MARKER in existing
if not path.exists():
if args.check:
missing.append(schema.connector_id)
else:
path.write_text(render(schema, ""), encoding="utf-8")
written += 1
generated_pages += 1
continue
jsx = unescaped_jsx_tags(existing)
if jsx and args.check:
mdx_breaks.append((schema.connector_id, jsx))
if not generator_owned:
# Written by a person, and richer than this can produce. The prose
# is left alone; the field list is injected as a generated block so
# it cannot drift away from the schema again.
hand_written_seen += 1
augmented = augment(existing, schema)
if args.check:
if existing != augmented:
gaps = undescribed_fields(schema, existing)
undocumented_fields.append((schema.connector_id, gaps or ["field reference out of date"]))
elif existing != augmented:
path.write_text(augmented, encoding="utf-8")
written += 1
continue
generated_pages += 1
rendered = render(schema, _human_block(path))
if args.check:
if existing != rendered:
drifted.append(schema.connector_id)
elif existing != rendered:
path.write_text(rendered, encoding="utf-8")
written += 1
dupes = duplicate_slugs()
sidebar_ok = sync_sidebar(schemas, check=args.check)
if args.check:
if not sidebar_ok:
print(
"CONNECTOR DOC GATE FAILED: the docs sidebar does not list every "
"connector page.\n"
" A page that exists and cannot be navigated to is barely a page.\n"
" Run: python3 scripts/generate_connector_docs.py",
file=sys.stderr,
)
if missing:
print(
f"CONNECTOR DOC GATE FAILED: {len(missing)} connector(s) have no page:\n"
f" {', '.join(missing)}\n"
f" Run: python3 scripts/generate_connector_docs.py",
file=sys.stderr,
)
if drifted:
print(
f"CONNECTOR DOC GATE FAILED: {len(drifted)} generated page(s) no longer "
f"match their schema:\n {', '.join(drifted)}\n"
f" Run: python3 scripts/generate_connector_docs.py",
file=sys.stderr,
)
if undocumented_fields:
print(
f"CONNECTOR DOC GATE FAILED: {len(undocumented_fields)} hand-written page(s) omit a required or secret field:",
file=sys.stderr,
)
for connector_id, fields in undocumented_fields:
print(f" {connector_id}: {', '.join(fields)}", file=sys.stderr)
print(
" A field was added, renamed or made secret and the page's generated "
"field reference is stale or absent.\n"
" Run: python3 scripts/generate_connector_docs.py — the prose is left "
"alone, only the reference table is rewritten.",
file=sys.stderr,
)
if dupes:
print(
f"CONNECTOR DOC GATE FAILED: {len(dupes)} connector(s) have two "
f"pages differing only by separator. One will be orphaned from "
f"the sidebar:",
file=sys.stderr,
)
for _key, names in dupes:
print(f" {' and '.join(n + '.md' for n in names)}", file=sys.stderr)
return 1
if mdx_breaks:
print(
f"CONNECTOR DOC GATE FAILED: {len(mdx_breaks)} page(s) contain "
f"angle-bracket tags MDX reads as unclosed JSX. The whole docs "
f"build fails on these:",
file=sys.stderr,
)
for connector_id, tags in mdx_breaks:
print(f" {connector_id}: {', '.join(tags[:4])}", file=sys.stderr)
print(
" Wrap them in backticks. Generated pages do this automatically; a hand-written page has to.",
file=sys.stderr,
)
if missing or drifted or undocumented_fields or mdx_breaks or not sidebar_ok:
return 1
print(
f"connector-docs: OK — {len(schemas)} connectors documented "
f"({generated_pages} generated, {len(schemas) - generated_pages} hand-written); "
f"every required and secret field is described"
)
return 0
print(f"connector-docs: {len(schemas)} connectors, {written} page(s) written, {hand_written_seen} hand-written page(s) left alone")
return 0
if __name__ == "__main__":
raise SystemExit(main())