Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""Render AiSOC white-paper markdown files to PDF.
Phase 4.3 update: the script now discovers every markdown file under
``apps/web/content/papers/`` and renders each one to
``apps/web/public/papers/<slug>.pdf``. This is what the CI papers
workflow drives.
Backwards-compatible CLI: passing ``--input`` / ``--output`` still
renders a single paper at an explicit path so the local dev workflow
documented in ``apps/web/public/papers/README.md`` keeps working.
It uses the WeasyPrint stack already required by the executive-digest
PDF in ``services/api/app/services/digest_pdf.py`` plus a ``markdown``
package for HTML conversion. Both are pure-Python installs on top of
native libs (Pango / Cairo / GLib) that are pinned in
``services/api/Dockerfile`` and in the ``papers`` CI workflow.
If WeasyPrint is not available (typical for a CI runner without the
native stack) the script exits with code 2 and a clear message so
callers can fall back to shipping the markdown unrendered.
This script is deliberately dependency-light. It does not introduce a
new heavy dependency (e.g. puppeteer / headless Chromium); WeasyPrint
is already part of the AiSOC build profile for the API service.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import sys
import tempfile
from pathlib import Path
# `scripts/` is on sys.path when this file is run as a program, but not when a
# sibling imports it.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from gate_toolkit import repo_root, self_test_main # noqa: E402
REPO_ROOT = repo_root()
DEFAULT_SRC_DIR = REPO_ROOT / "apps" / "web" / "content" / "papers"
DEFAULT_OUT_DIR = REPO_ROOT / "apps" / "web" / "public" / "papers"
# Records the digest of each markdown source at the moment its PDF was
# rendered, so `--check` can tell a stale PDF from a current one.
#
# The digest is taken over the *source closure* — the markdown plus every
# local figure it embeds — and never over the rendered PDF.
# WeasyPrint stamps a creation timestamp into its output and glyph metrics
# depend on which fonts the host happens to have installed, so two correct
# renders of one source do not produce equal bytes. A PDF byte comparison
# would therefore fail for reasons that have nothing to do with staleness.
MANIFEST_PATH = DEFAULT_OUT_DIR / "render-manifest.json"
PRINT_CSS = """
@page {
size: A4;
margin: 22mm 18mm 22mm 18mm;
@bottom-center {
content: counter(page) " / " counter(pages);
font-family: "Inter", "Helvetica Neue", Arial, sans-serif;
font-size: 9pt;
color: #6b7280;
}
@bottom-left {
content: "Funded and supported by Cyble — cyble.com";
font-family: "Inter", "Helvetica Neue", Arial, sans-serif;
font-size: 8pt;
color: #6b7280;
}
@top-right {
content: string(paper-title);
font-family: "Inter", "Helvetica Neue", Arial, sans-serif;
font-size: 8pt;
color: #9ca3af;
}
}
html, body {
font-family: "Inter", "Helvetica Neue", Arial, sans-serif;
font-size: 10.5pt;
line-height: 1.55;
color: #111827;
}
h1 {
font-size: 22pt;
margin: 0 0 0.4em 0;
color: #0f172a;
page-break-before: auto;
string-set: paper-title content();
}
h2 {
font-size: 15pt;
margin: 1.6em 0 0.4em 0;
color: #0f172a;
page-break-after: avoid;
}
figure {
margin: 1.4em 0;
page-break-inside: avoid;
}
figure img, p > img {
width: 100%;
max-width: 100%;
border: 1px solid #d1d5db;
border-radius: 3px;
}
figcaption, .figure-caption {
font-size: 8.5pt;
color: #4b5563;
margin-top: 0.4em;
font-style: italic;
}
h3 {
font-size: 12pt;
margin: 1.2em 0 0.3em 0;
color: #1f2937;
page-break-after: avoid;
}
p { margin: 0 0 0.7em 0; }
code, pre {
font-family: "JetBrains Mono", "Menlo", "Consolas", monospace;
font-size: 9pt;
background: #f1f5f9;
color: #0f172a;
}
code { padding: 0 2px; }
pre {
padding: 10px 12px;
border-radius: 4px;
overflow-x: auto;
page-break-inside: avoid;
}
blockquote {
border-left: 3px solid #94a3b8;
margin: 0.6em 0;
padding: 0.2em 0 0.2em 0.8em;
color: #334155;
font-style: italic;
}
table {
border-collapse: collapse;
width: 100%;
margin: 0.6em 0;
font-size: 9.5pt;
page-break-inside: avoid;
}
th, td {
border: 1px solid #cbd5e1;
padding: 5px 7px;
text-align: left;
vertical-align: top;
}
th { background: #e2e8f0; }
a { color: #1d4ed8; text-decoration: none; }
a:hover { text-decoration: underline; }
hr {
border: none;
border-top: 1px solid #cbd5e1;
margin: 1em 0;
}
ul, ol { margin: 0.2em 0 0.7em 1.4em; padding: 0; }
li { margin: 0.2em 0; }
"""
COVER_HTML = """
<div style="page-break-after: always;">
<div style="margin-top: 60mm; text-align: center;">
<div style="font-size: 32pt; font-weight: 700; color: #0f172a;">
{title}
</div>
<div style="margin-top: 12mm; font-size: 13pt; color: #475569;">
{subtitle}
</div>
<div style="margin-top: 18mm; font-size: 10pt; color: #64748b;">
AiSOC project · {version} · Released {date}
</div>
<div style="margin-top: 4mm; font-size: 10pt; color: #94a3b8;">
MIT licensed · github.com/beenuar/AiSOC
</div>
</div>
</div>
"""
_IMG_RE = re.compile(r"!\[[^\]]*\]\(([^)\s]+)")
# Papers reference figures from the repository root, which is also the
# `base_url` WeasyPrint renders them against. One name for that root, read by
# both the missing-figure refusal and the staleness digest, so the two cannot
# disagree about which file a caption points at.
IMAGE_ROOT = REPO_ROOT
def _resolve_image(ref: str) -> Path:
return IMAGE_ROOT / ref.lstrip("/")
def _assert_images_resolved(markdown_path: Path, output_path: Path) -> None:
"""Refuse a PDF whose figures point at files that are not there.
WeasyPrint renders a missing image as nothing at all, so the
caption survives and describes a blank space. A reader takes that
as a layout bug; it is actually a claim about a screenshot the
document does not have.
Checked against the source markdown rather than the rendered PDF
because the PDF is where the evidence has already been lost.
"""
source = markdown_path.read_text(encoding="utf-8")
missing = [
ref for ref in _IMG_RE.findall(source) if not ref.startswith(("http://", "https://", "data:")) and not _resolve_image(ref).is_file()
]
if missing:
output_path.unlink(missing_ok=True)
raise SystemExit(
f"[render_white_paper] {markdown_path.name} references "
f"{len(missing)} image(s) that do not exist: {', '.join(missing[:5])}. "
"A caption describing an absent screenshot is worse than no screenshot, "
"so the PDF was not written."
)
def _strip_frontmatter(source: str) -> tuple[dict[str, str], str]:
"""Return (metadata, body) after stripping YAML frontmatter."""
if not source.startswith("---"):
return {}, source
parts = source.split("---", 2)
if len(parts) < 3:
return {}, source
raw_meta, body = parts[1], parts[2]
meta: dict[str, str] = {}
for line in raw_meta.strip().splitlines():
if ":" in line and not line.startswith(" "):
key, _, value = line.partition(":")
meta[key.strip()] = value.strip().strip('"').strip("'")
return meta, body.lstrip("\n")
def _render_one(markdown_path: Path, output_path: Path) -> None:
try:
import markdown as md # type: ignore[import-untyped]
except ImportError as exc:
print(
"[render_white_paper] ERROR: the `markdown` package is required. "
"Install with `pip install markdown weasyprint` or run inside the "
"services/api Dockerfile build profile.",
file=sys.stderr,
)
raise SystemExit(2) from exc
try:
from weasyprint import CSS, HTML # type: ignore[import-untyped]
except (ImportError, OSError) as exc:
print(
"[render_white_paper] ERROR: WeasyPrint (or its native libs) is "
"missing. See services/api/Dockerfile for the required apt "
"packages (libpango, libcairo, libgdk-pixbuf, libffi, libssl). "
"Install with `pip install weasyprint` once the libs are present.",
file=sys.stderr,
)
raise SystemExit(2) from exc
source = markdown_path.read_text(encoding="utf-8")
meta, body = _strip_frontmatter(source)
rendered_body = md.markdown(
body,
extensions=["extra", "tables", "fenced_code", "toc", "sane_lists"],
)
cover = COVER_HTML.format(
title=meta.get("title", markdown_path.stem.replace("-", " ").title()),
subtitle=meta.get("subtitle", ""),
version=meta.get("version", "v1.0"),
date=meta.get("date", ""),
)
html = (
"<!doctype html><html><head><meta charset='utf-8'/>"
f"<title>{meta.get('title', 'AiSOC White Paper')}</title>"
"</head><body>"
f"{cover}"
f"{rendered_body}"
"</body></html>"
)
# Strip the redundant top-level H1 added by the markdown body so the
# cover page is the only title surface. We only do this for the
# first occurrence to avoid wrecking section headings.
html = re.sub(r"<h1[^>]*>.*?</h1>", "", html, count=1)
output_path.parent.mkdir(parents=True, exist_ok=True)
# `base_url` is the repository root so a paper can reference the
# real console captures under `apps/web/public/screenshots/` by
# repo-relative path. Without it WeasyPrint resolves relative `src`
# against the current directory and silently renders nothing where
# the image should be — a figure caption describing an absent
# screenshot, which is worse than no screenshot.
HTML(string=html, base_url=str(repo_root())).write_pdf(
str(output_path),
stylesheets=[CSS(string=PRINT_CSS)],
)
_assert_images_resolved(markdown_path, output_path)
size = output_path.stat().st_size
print(
f"[render_white_paper] wrote {output_path} ({size:,} bytes)",
file=sys.stderr,
)
def render(markdown_path: Path, output_path: Path) -> None:
"""Public single-file entrypoint kept for backwards compatibility."""
_render_one(markdown_path, output_path)
def _discover() -> list[tuple[Path, Path]]:
"""Pair every ``content/papers/*.md`` with its target PDF path."""
pairs: list[tuple[Path, Path]] = []
for src in sorted(DEFAULT_SRC_DIR.glob("*.md")):
if src.name.startswith("_"):
continue
pdf = DEFAULT_OUT_DIR / f"{src.stem}.pdf"
pairs.append((src, pdf))
return pairs
def _source_digest(path: Path) -> str:
"""Digest a paper's markdown *and every local figure it embeds*.
A PDF is only as current as its screenshots. Digesting the markdown alone
meant refreshing a capture left the recorded digest unchanged, so
``--check`` reported the PDF as current while it still showed the previous
console. That is not hypothetical: the v20.1.0 release replaced fifteen
console captures without re-rendering the technical guide, and this check
passed throughout — the workflow even runs on `screenshots/**` for exactly
this reason, so the trigger was right and the comparison was blind to it.
"""
digest = hashlib.sha256()
digest.update(path.read_bytes())
for ref in sorted(set(_IMG_RE.findall(path.read_text(encoding="utf-8", errors="replace")))):
if ref.startswith(("http://", "https://", "data:")):
continue
resolved = _resolve_image(ref)
if not resolved.is_file():
# A missing figure is `_assert_images_resolved`'s refusal to make,
# at render time, where it can say which caption is empty.
continue
# The path is part of the digest, so repointing a caption at a
# different file counts as a change even if the bytes are identical.
digest.update(b"\x00" + ref.encode("utf-8") + b"\x00")
digest.update(resolved.read_bytes())
return digest.hexdigest()
def _display(path: Path) -> str:
"""Repository-relative where possible, absolute where not.
The gate-contract harness and ``--self-test`` both point this script at a
tree outside ``REPO_ROOT``, where ``relative_to`` raises.
"""
try:
return str(path.relative_to(REPO_ROOT))
except ValueError:
return str(path)
def _read_manifest() -> dict[str, str]:
if not MANIFEST_PATH.exists():
return {}
try:
loaded = json.loads(MANIFEST_PATH.read_text())
except json.JSONDecodeError:
return {}
sources = loaded.get("sources")
return sources if isinstance(sources, dict) else {}
def _record_rendered(sources: dict[str, str]) -> None:
"""Merge digests for the papers just rendered into the manifest."""
merged = {**_read_manifest(), **sources}
MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True)
MANIFEST_PATH.write_text(json.dumps({"sources": dict(sorted(merged.items()))}, indent=2) + "\n")
def _render_all() -> None:
pairs = _discover()
if not pairs:
print(
f"[render_white_paper] no markdown sources found under {DEFAULT_SRC_DIR}",
file=sys.stderr,
)
return
for src, dst in pairs:
_render_one(src, dst)
_record_rendered({src.name: _source_digest(src) for src, _ in pairs})
def _check() -> int:
"""Report papers whose PDF is missing or older than its source.
Needs no rendering stack, so it runs anywhere python does.
"""
pairs = _discover()
if not pairs:
print(
f"[render_white_paper] refusing to pass: no markdown sources under {DEFAULT_SRC_DIR}",
file=sys.stderr,
)
return 1
manifest = _read_manifest()
problems: list[str] = []
for src, pdf in pairs:
if not pdf.exists():
problems.append(f"{_display(pdf)} does not exist")
continue
recorded = manifest.get(src.name)
if recorded is None:
problems.append(f"{src.name} has a PDF but no recorded render")
elif recorded != _source_digest(src):
problems.append(f"{src.name} changed since its PDF was rendered")
known = {src.name for src, _ in pairs}
for orphan in sorted(set(manifest) - known):
problems.append(f"{orphan} is recorded but its source is gone")
if problems:
print("[render_white_paper] rendered PDFs are out of date:", file=sys.stderr)
for problem in problems:
print(f" - {problem}", file=sys.stderr)
print(
"\nRun `make papers` and commit the result. CI cannot refresh them for\n"
"you: main is branch-protected, so a workflow cannot push to it.",
file=sys.stderr,
)
return 1
print(f"[render_white_paper] {len(pairs)} paper(s) up to date with their sources")
return 0
def _injected_defect_cases() -> list[tuple[str, bool]]:
"""Plant each defect ``--check`` claims to catch and require a refusal.
The control case matters as much as the rest: a gate that refused
everything would "detect" all four defects and be useless, so a
well-formed paper is required to pass.
"""
global DEFAULT_SRC_DIR, DEFAULT_OUT_DIR, MANIFEST_PATH, IMAGE_ROOT
def plant(root: Path, *, pdf: bool, recorded: str | None, orphan: bool, figure: str | None = None) -> None:
# Without this, the assignments below bind locals to `plant` and every
# case silently grades the real tree instead of the planted one.
global DEFAULT_SRC_DIR, DEFAULT_OUT_DIR, MANIFEST_PATH, IMAGE_ROOT
src_dir, out_dir = root / "content", root / "public"
src_dir.mkdir(parents=True)
out_dir.mkdir(parents=True)
IMAGE_ROOT = root
source = src_dir / "paper.md"
source.write_text("# a paper\n" if figure is None else "# a paper\n\n\n")
if figure is not None:
# Record the digest against the *first* capture, then replace the
# file the way a re-capture does. The markdown never changes, which
# is precisely how the real staleness went unnoticed.
(root / "shot.png").write_bytes(b"first capture")
if pdf:
(out_dir / "paper.pdf").write_bytes(b"%PDF-1.7 not a real render\n")
entries: dict[str, str] = {}
if recorded == "match":
entries["paper.md"] = _source_digest(source)
elif recorded == "stale":
entries["paper.md"] = "0" * 64
if orphan:
entries["deleted.md"] = "0" * 64
if figure == "refreshed":
(root / "shot.png").write_bytes(b"re-captured against a later release")
DEFAULT_SRC_DIR, DEFAULT_OUT_DIR = src_dir, out_dir
MANIFEST_PATH = out_dir / "render-manifest.json"
MANIFEST_PATH.write_text(json.dumps({"sources": entries}) + "\n")
cases: list[tuple[str, dict[str, object], bool]] = [
("a paper whose PDF matches its recorded source passes", {"pdf": True, "recorded": "match", "orphan": False}, True),
("a source edited since its PDF was rendered is refused", {"pdf": True, "recorded": "stale", "orphan": False}, False),
("a source with no PDF at all is refused", {"pdf": False, "recorded": "match", "orphan": False}, False),
("a PDF with no recorded render is refused", {"pdf": True, "recorded": None, "orphan": False}, False),
("a recorded paper whose source is gone is refused", {"pdf": True, "recorded": "match", "orphan": True}, False),
# The control for the pair below: an embedded figure that has not moved
# must still pass, or the two cases together prove nothing.
("a paper whose embedded figure is unchanged passes", {"pdf": True, "recorded": "match", "orphan": False, "figure": "kept"}, True),
(
"a figure re-captured since the PDF was rendered is refused",
{"pdf": True, "recorded": "match", "orphan": False, "figure": "refreshed"},
False,
),
]
saved = (DEFAULT_SRC_DIR, DEFAULT_OUT_DIR, MANIFEST_PATH, IMAGE_ROOT)
results: list[tuple[str, bool]] = []
try:
for description, shape, should_pass in cases:
with tempfile.TemporaryDirectory() as tmp:
plant(Path(tmp), **shape) # type: ignore[arg-type]
results.append((description, (_check() == 0) is should_pass))
finally:
DEFAULT_SRC_DIR, DEFAULT_OUT_DIR, MANIFEST_PATH, IMAGE_ROOT = saved
return results
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--input",
help=("Render a single paper from this markdown path. If omitted, render every paper under apps/web/content/papers/."),
)
parser.add_argument(
"--output",
help=("Write the rendered PDF to this path. Required when --input is supplied; ignored otherwise."),
)
parser.add_argument(
"--all",
action="store_true",
help=("Render every paper under apps/web/content/papers/. Default when neither --input nor --output is supplied."),
)
parser.add_argument(
"--check",
action="store_true",
help=("Report papers whose PDF is missing or older than its source. Renders nothing."),
)
parser.add_argument(
"--self-test",
action="store_true",
help=("Prove --check still detects each defect it claims to, then exit. Renders nothing."),
)
args = parser.parse_args(argv)
if args.self_test:
return self_test_main(Path(__file__).name, ["--check"], extra=_injected_defect_cases())
if args.check and (args.input or args.output or args.all):
parser.error("--check renders nothing, so it takes no other arguments")
if args.input and not args.output:
parser.error("--output is required when --input is supplied")
if args.output and not args.input:
parser.error("--input is required when --output is supplied")
if args.check:
return _check()
if args.input and args.output:
source = Path(args.input)
_render_one(source, Path(args.output))
# Keep the manifest honest when a single paper is rendered in place,
# so the next --check does not report a paper that was just refreshed.
if source.resolve().parent == DEFAULT_SRC_DIR:
_record_rendered({source.name: _source_digest(source)})
else:
_render_all()
return 0
if __name__ == "__main__":
raise SystemExit(main())