Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

64.1 KB
Raw Blame
#!/usr/bin/env python3
"""Build the Markdown article sections of xerj.org.

The generator has no runtime dependencies beyond Python's standard library.
Sources live under ``content/answers`` and ``content/compare``.  The output is
deliberately small, deterministic HTML that uses the site's existing nav,
docs-shell, CSS classes, head emitter and footer.
"""

from __future__ import annotations

import argparse
import collections
import datetime as dt
import html
import html as html_lib
import json
import pathlib
import posixpath
import re
import subprocess
import sys
import tempfile
import urllib.parse
from typing import Any, Callable

SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))

import article_data  # noqa: E402
import fix_links  # noqa: E402
import set_nav  # noqa: E402
import pagedata  # noqa: E402
import seo_head  # noqa: E402
import urlmap  # noqa: E402


GENERATOR_PREFIX = "<!-- generated by build_articles.py — edit content/"
GENERATOR_RE = re.compile(
    r"<!-- generated by build_articles\.py — edit content/(answers|compare)/"
    r"(?:[a-z0-9-]+|\*)\.md instead -->"
)
LLMS_BEGIN = "<!-- BEGIN GENERATED ARTICLE TREE: build_articles.py -->"
LLMS_END = "<!-- END GENERATED ARTICLE TREE: build_articles.py -->"
HTML_TAG_RE = re.compile(r"<!--|<![A-Za-z]|</?[A-Za-z][^>]*>")
FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
HEADING_RE = re.compile(r"^ {0,3}(#{1,6})\s+(.+?)\s*$")
UL_RE = re.compile(r"^ {0,3}([-+*])\s+(.+?)\s*$")
OL_RE = re.compile(r"^ {0,3}(\d+)[.)]\s+(.+?)\s*$")
LANG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9+_.-]*$")
TLDR_RE = re.compile(r"^\s*\*\*TL[;:]?DR\*\*\s*[—–:.-]?\s*", re.I)


class BuildError(ValueError):
    """A user-actionable build error."""


class LinkTarget:
    __slots__ = ("rel", "href", "label")

    def __init__(self, rel: str, href: str, label: str):
        self.rel = rel
        self.href = href
        self.label = label


def esc_text(value: str) -> str:
    return html.escape(value, quote=False)


def esc_attr(value: str) -> str:
    return html.escape(value, quote=True)


def line_error(article: article_data.Article, line: int, message: str) -> BuildError:
    return BuildError(f"{article.path}:{line}: {message}")


class BuildContext:
    def __init__(self, repo: pathlib.Path, landing: pathlib.Path,
                 articles: list[article_data.Article], dates: urlmap.DateSource):
        self.repo = repo
        self.landing = landing
        self.dates = dates
        self.articles = articles
        self.by_rel = {a.output_rel: a for a in articles}
        self.by_slug: dict[str, list[article_data.Article]] = collections.defaultdict(list)
        for article in articles:
            self.by_slug[article.slug].append(article)
        self.existing = set(urlmap.iter_pages(landing))
        self.known = self.existing | set(self.by_rel) | {
            "answers/index.html", "compare/index.html",
        }

    def _normalise_path(self, raw: str, category: str) -> tuple[str, str, str]:
        value = raw.strip()
        if not value:
            raise BuildError("empty internal link target")
        parsed = urllib.parse.urlsplit(value)
        if parsed.scheme or value.startswith("//"):
            if parsed.scheme not in ("http", "https") or parsed.netloc not in (
                    "xerj.org", "www.xerj.org"):
                raise BuildError(f"internal article link must stay on xerj.org: {raw!r}")
            path = parsed.path or "/"
        else:
            path = parsed.path
            if not path.startswith("/"):
                if path.startswith(("answers/", "compare/", "docs/", "use-cases/",
                                    "case-studies/", "industries/")):
                    path = "/" + path
                else:
                    # A bare value is an article slug in the current section.
                    path = f"/{category}/{path}"
        if "\\" in path:
            raise BuildError(f"backslashes are not valid in internal link target {raw!r}")
        path = posixpath.normpath(path)
        if not path.startswith("/"):
            path = "/" + path
        if path == "/.":
            path = "/"
        return path, parsed.query, parsed.fragment

    def resolve(self, raw: str, category: str, article: article_data.Article | None = None,
                line: int | None = None) -> LinkTarget:
        try:
            path, query, fragment = self._normalise_path(raw, category)
        except BuildError as exc:
            if article is not None and line is not None:
                raise line_error(article, line, str(exc)) from exc
            raise
        rel = urlmap.rel_for_path(path)
        if rel is None or rel not in self.known or not (
                (self.landing / rel).is_file() or rel in self.by_rel):
            message = f"dangling internal target {raw!r} (resolved as {path})"
            if article is not None and line is not None:
                raise line_error(article, line, message)
            raise BuildError(message)
        target = self.by_rel.get(rel)
        if target is not None:
            label = target.h1
        else:
            try:
                label = str(pagedata.entry(rel).get("label", rel))
            except KeyError:
                label = rel
        href = urlmap.deployed_path(rel)
        if query:
            href += "?" + query
        if fragment:
            href += "#" + fragment
        return LinkTarget(rel, href, label)

    def resolve_article_target(self, raw: str, article: article_data.Article,
                               line: int | None = None) -> LinkTarget:
        """Resolve a links_out item, including a bare slug."""
        value = raw.strip()
        parsed = urllib.parse.urlsplit(value)
        bare = parsed.path.strip("/")
        if not parsed.scheme and not value.startswith("/") and "/" not in bare:
            matches = self.by_slug.get(bare, [])
            if len(matches) == 1:
                return self.resolve(matches[0].output_rel, article.category, article, line)
            if len(matches) > 1:
                raise line_error(article, line or 1,
                                 f"ambiguous article slug {raw!r}; use /answers/... or /compare/...")
        return self.resolve(value, article.category, article, line)

    def markdown_href(self, raw: str, article: article_data.Article,
                      line: int) -> str:
        parsed = urllib.parse.urlsplit(raw.strip())
        if parsed.scheme in ("http", "https") and parsed.netloc:
            return raw.strip()
        if parsed.scheme or raw.strip().startswith("//"):
            raise line_error(article, line, f"unsupported Markdown link URL {raw!r}")
        return self.resolve(raw, article.category, article, line).href


# A fenced block whose language is `visual` holds one line, the name of a figure in
# content/visuals/<name>.html. The figure is hand-written HTML and inline SVG, inlined
# into the page as-is, so it is held to an allowlist here: one <figure class="vz">
# root with a <figcaption>, no script, no style element, no event handler, no
# javascript: URL and no external media. Behaviour lives in landing/visuals.js and
# styling in landing/visuals.css; both load only on pages that carry a visual. The
# Markdown twin gets the caption, and factcheck.py reads the figure's visible text.
VISUAL_DIR = ("content", "visuals")
VISUAL_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
VISUAL_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})\s*visual\s*$")
VISUAL_FORBIDDEN = (
    (re.compile(r"<\s*/?\s*(?:script|style|iframe|object|embed|link|meta|base|form|img|video|audio|source|foreignobject)\b", re.I),
     "element is not allowed in a visual"),
    (re.compile(r"\son[a-z]+\s*=", re.I), "event handler attributes are not allowed in a visual"),
    (re.compile(r"javascript:", re.I), "javascript: URLs are not allowed in a visual"),
    (re.compile(r"\s(?:src|srcset|xlink:href)\s*=", re.I), "external media is not allowed in a visual"),
    (re.compile(r"\shref\s*=\s*[\"'](?!/|#|https://)", re.I), "a visual link must be /path, #anchor or https://"),
    (re.compile("[\u2013\u2014]"), "en and em dashes are not allowed in a visual"),
)
_VISUAL_CACHE: dict[str, tuple[str, str]] = {}


def _indent_visual(html: str) -> str:
    """Indent a figure by two spaces, except lines inside <textarea> or <pre>,
    whose leading whitespace is content."""
    out: list[str] = []
    raw = False
    for line in html.splitlines():
        out.append(line if raw or not line.strip() else "  " + line)
        low = line.lower()
        opens = low.count("<textarea") + low.count("<pre")
        closes = low.count("</textarea>") + low.count("</pre>")
        if opens > closes:
            raw = True
        elif closes > opens:
            raw = False
    return "\n".join(out)


def load_visual(article: article_data.Article, name: str, line: int,
                context: "BuildContext") -> tuple[str, str]:
    """Return (html, caption text) for content/visuals/<name>.html."""
    if not VISUAL_NAME_RE.fullmatch(name):
        raise line_error(article, line, f"visual name {name!r} must be lowercase-hyphenated")
    path = context.repo.joinpath(*VISUAL_DIR, f"{name}.html")
    key = str(path)
    if key in _VISUAL_CACHE:
        return _VISUAL_CACHE[key]
    if not path.is_file():
        raise line_error(article, line, f"visual {name!r} not found at {path.relative_to(context.repo)}")
    html = path.read_text(encoding="utf-8").strip()
    if not (html.startswith('<figure class="vz') and html.endswith("</figure>")):
        raise line_error(article, line, f"visual {name!r} must be one <figure class=\"vz ...\"> element")
    for pattern, message in VISUAL_FORBIDDEN:
        hit = pattern.search(html)
        if hit:
            raise line_error(article, line, f"visual {name!r}: {message} ({hit.group(0).strip()!r})")
    caption = re.search(r"<figcaption[^>]*>(.*?)</figcaption>", html, re.S)
    if not caption:
        raise line_error(article, line, f"visual {name!r} needs a <figcaption>")
    text = " ".join(html_lib.unescape(re.sub(r"<[^>]+>", " ", caption.group(1))).split())
    _VISUAL_CACHE[key] = (html, text)
    return _VISUAL_CACHE[key]


def has_visuals(article: article_data.Article) -> bool:
    return any(VISUAL_FENCE_RE.match(line) for line in article.body.splitlines())


def _twin_visuals(body: str, article: article_data.Article, context: "BuildContext") -> str:
    lines = body.splitlines()
    out: list[str] = []
    i = 0
    while i < len(lines):
        if VISUAL_FENCE_RE.match(lines[i]) and i + 2 < len(lines) + 1:
            name = lines[i + 1].strip() if i + 1 < len(lines) else ""
            _, caption = load_visual(article, name, article.body_start_line, context)
            out.append(f"*Figure: {caption}*")
            i += 3
            continue
        out.append(lines[i])
        i += 1
    return "\n".join(out)


def _check_raw_html(lines: list[str], article: article_data.Article) -> None:
    in_fence = False
    marker = ""
    for offset, raw in enumerate(lines):
        line_no = article.body_start_line + offset
        fence = FENCE_RE.match(raw)
        if fence:
            token = fence.group(1)
            if not in_fence:
                in_fence = True
                marker = token[0]
            elif token[0] == marker:
                in_fence = False
            continue
        if in_fence:
            continue
        masked = re.sub(r"`[^`]*`", "", raw)
        if HTML_TAG_RE.search(masked):
            raise line_error(article, line_no, "raw HTML is not allowed in Markdown")
    if in_fence:
        raise line_error(article, article.body_start_line + len(lines),
                         "fenced code block has no closing fence")


def _split_table_row(raw: str) -> list[str]:
    text = raw.strip()
    if text.startswith("|"):
        text = text[1:]
    if text.endswith("|") and not text.endswith("\\|"):
        text = text[:-1]
    cells: list[str] = []
    current: list[str] = []
    escaped = False
    for ch in text:
        if escaped:
            current.append(ch)
            escaped = False
        elif ch == "\\":
            escaped = True
        elif ch == "|":
            cells.append("".join(current).strip())
            current = []
        else:
            current.append(ch)
    if escaped:
        current.append("\\")
    cells.append("".join(current).strip())
    return cells


def _is_table_separator(raw: str) -> bool:
    if "|" not in raw:
        return False
    cells = _split_table_row(raw)
    return bool(cells) and all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)


def _is_word_char(ch: str) -> bool:
    """Return True for characters that make ``_`` intraword, not emphasis."""
    return ch.isalnum() or ch == "_"


def _inline(text: str, article: article_data.Article, line: int,
            href_resolver: Callable[[str, article_data.Article, int], str]) -> str:
    """Render the safe inline subset with escaping at the text boundary."""
    out: list[str] = []
    i = 0
    n = len(text)
    while i < n:
        if text.startswith("![", i):
            raise line_error(article, line, "images are not supported in article Markdown")
        if text[i] == "`":
            end = text.find("`", i + 1)
            if end < 0:
                raise line_error(article, line, "inline code span has no closing backtick")
            out.append(f'<code class="inline">{esc_text(text[i + 1:end])}</code>')
            i = end + 1
            continue
        marker = None
        width = 0
        if text.startswith("**", i) or text.startswith("__", i):
            marker, width = text[i:i + 2], 2
        elif text[i] in "*_":
            marker, width = text[i], 1
        if marker is not None and marker[0] == "_" and i > 0 and _is_word_char(text[i - 1]):
            # CommonMark forbids intraword ``_`` emphasis so that bare
            # identifiers such as ``ignored_files`` stay literal text.
            marker = None
        if marker is not None:
            end = text.find(marker, i + width)
            if marker[0] == "_":
                # A closing ``_`` may not be followed by a word character
                # either, for the same reason.
                while end >= 0 and end + width < n and _is_word_char(text[end + width]):
                    end = text.find(marker, end + width)
            if end > i + width and not text[i + width].isspace():
                inner = _inline(text[i + width:end], article, line, href_resolver)
                tag = "strong" if width == 2 else "em"
                out.append(f"<{tag}>{inner}</{tag}>")
                i = end + width
                continue
        if text[i] == "[":
            close = text.find("]", i + 1)
            if close > i + 1 and close + 1 < n and text[close + 1] == "(":
                end = text.find(")", close + 2)
                if end < 0:
                    raise line_error(article, line, "Markdown link has no closing parenthesis")
                label = text[i + 1:close]
                destination = text[close + 2:end].strip()
                if not destination or any(ch.isspace() for ch in destination):
                    raise line_error(article, line, "Markdown links need a single URL without spaces")
                href = href_resolver(destination, article, line)
                out.append(f'<a href="{esc_attr(href)}">'
                           f"{_inline(label, article, line, href_resolver)}</a>")
                i = end + 1
                continue
        start = i
        while i < n and text[i] not in "`[*_":
            i += 1
        if start == i:
            out.append(esc_text(text[i]))
            i += 1
        else:
            out.append(esc_text(text[start:i]))
    return "".join(out)


def _block_start(lines: list[str], i: int) -> bool:
    raw = lines[i]
    return bool(FENCE_RE.match(raw) or HEADING_RE.match(raw) or UL_RE.match(raw)
                or OL_RE.match(raw) or raw.lstrip().startswith(">"))


def render_markdown_blocks(article: article_data.Article, context: BuildContext) -> list[str]:
    lines = article.body.splitlines()
    _check_raw_html(lines, article)
    first = next((line for line in lines if line.strip()), "")
    if not TLDR_RE.match(first):
        raise line_error(article, article.body_start_line,
                         "the first body block must be the TL;DR paragraph")

    rendered: list[str] = []
    i = 0
    previous_heading = 1
    while i < len(lines):
        raw = lines[i]
        line_no = article.body_start_line + i
        if not raw.strip():
            i += 1
            continue

        fence = FENCE_RE.match(raw)
        if fence:
            marker = fence.group(1)[0]
            info = fence.group(2).strip()
            if not info or not LANG_RE.fullmatch(info):
                raise line_error(article, line_no,
                                 "fenced code blocks need a simple language name")
            i += 1
            code: list[str] = []
            closed = False
            while i < len(lines):
                close = FENCE_RE.match(lines[i])
                if close and close.group(1)[0] == marker and not close.group(2).strip():
                    closed = True
                    i += 1
                    break
                code.append(lines[i])
                i += 1
            if not closed:
                raise line_error(article, line_no, "fenced code block has no closing fence")
            if info == "visual":
                names = [c.strip() for c in code if c.strip()]
                if len(names) != 1:
                    raise line_error(article, line_no, "a visual block names exactly one figure")
                html, _ = load_visual(article, names[0], line_no, context)
                rendered.append(_indent_visual(html))
                continue
            body = esc_text("\n".join(code))
            if code:
                body += "\n"
            rendered.append(f'  <pre class="code"><code class="language-{info}">'
                            f"{body}</code></pre>")
            continue

        heading = HEADING_RE.match(raw)
        if heading:
            level = len(heading.group(1))
            text = heading.group(2).rstrip("#").rstrip()
            if level == 1:
                raise line_error(article, line_no,
                                 "Markdown body must not contain an H1; use frontmatter h1")
            if level not in (2, 3):
                raise line_error(article, line_no,
                                 "only ATX h2 and h3 headings are supported")
            if level > previous_heading + 1:
                raise line_error(article, line_no,
                                 f"heading level jumps from h{previous_heading} to h{level}")
            if level == 2 and text.strip().lower() == "faq":
                raise line_error(article, line_no,
                                 "FAQ is generated from frontmatter; remove the body FAQ heading")
            rendered.append(f"  <h{level}>"
                            f"{_inline(text, article, line_no, context.markdown_href)}"
                            f"</h{level}>")
            previous_heading = level
            i += 1
            continue

        if (i + 1 < len(lines) and "|" in raw
                and _is_table_separator(lines[i + 1])):
            header = _split_table_row(raw)
            separator = _split_table_row(lines[i + 1])
            if len(header) != len(separator):
                raise line_error(article, line_no, "table header and separator widths differ")
            i += 2
            rows: list[list[str]] = []
            while i < len(lines) and lines[i].strip() and "|" in lines[i]:
                row = _split_table_row(lines[i])
                if len(row) != len(header):
                    raise line_error(article, article.body_start_line + i,
                                     "table row width differs from the header")
                rows.append(row)
                i += 1
            rendered.append("  <table>")
            rendered.append("    <thead><tr>" + "".join(
                f"<th>{_inline(cell, article, line_no, context.markdown_href)}</th>"
                for cell in header) + "</tr></thead>")
            if rows:
                rendered.append("    <tbody>")
                for row_index, row in enumerate(rows):
                    row_line = line_no + 2 + row_index
                    rendered.append("      <tr>" + "".join(
                        f"<td>{_inline(cell, article, row_line, context.markdown_href)}</td>"
                        for cell in row) + "</tr>")
                rendered.append("    </tbody>")
            rendered.append("  </table>")
            continue

        unordered = UL_RE.match(raw)
        ordered = OL_RE.match(raw)
        if unordered or ordered:
            ordered_mode = ordered is not None
            items: list[tuple[int, str]] = []
            while i < len(lines):
                match = OL_RE.match(lines[i]) if ordered_mode else UL_RE.match(lines[i])
                if not match:
                    break
                items.append((article.body_start_line + i, match.group(2)))
                i += 1
            tag = "ol" if ordered_mode else "ul"
            rendered.append(f"  <{tag}>")
            for item_line, item in items:
                rendered.append(f"    <li>{_inline(item, article, item_line, context.markdown_href)}</li>")
            rendered.append(f"  </{tag}>")
            continue

        if raw.lstrip().startswith(">"):
            quote_lines: list[tuple[int, str]] = []
            while i < len(lines) and lines[i].lstrip().startswith(">"):
                quote_lines.append((article.body_start_line + i,
                                    re.sub(r"^ {0,3}>\s?", "", lines[i])))
                i += 1
            text = " ".join(value for _, value in quote_lines)
            rendered.append("  <blockquote><p>"
                            + _inline(text, article, quote_lines[0][0], context.markdown_href)
                            + "</p></blockquote>")
            continue

        paragraph: list[tuple[int, str]] = []
        while i < len(lines) and lines[i].strip() and not _block_start(lines, i):
            paragraph.append((article.body_start_line + i, lines[i].strip()))
            i += 1
        if not paragraph:
            raise line_error(article, line_no, "unsupported Markdown syntax")
        text = " ".join(value for _, value in paragraph)
        rendered.append("  <p>" + _inline(text, article, paragraph[0][0],
                                               context.markdown_href) + "</p>")
    return rendered


def render_markdown(article: article_data.Article, context: BuildContext) -> str:
    return "\n".join(render_markdown_blocks(article, context))


EVIDENCE_LINE_RE = re.compile(r"^(.*):(\d+)$")


def _evidence_repo_path(source: str) -> str:
    """Return the filesystem portion of a repo evidence source."""
    match = EVIDENCE_LINE_RE.fullmatch(source)
    return match.group(1) if match else source


def _metadata_link(source: str, article: article_data.Article,
                   context: BuildContext) -> str | None:
    """Resolve an evidence source, or return ``None`` for repo provenance.

    Evidence sources are not all site links.  A repo-relative source is
    validated against the repository and deliberately has no href so callers
    render it as provenance text rather than an ordinary document link.
    """
    value = source.strip()
    repo_path = _evidence_repo_path(value)
    parsed = urllib.parse.urlsplit(value)
    if parsed.scheme in ("http", "https") and parsed.netloc:
        return value
    if parsed.scheme or value.startswith("//"):
        # urlsplit treats ``ROADMAP.md:80`` as a URI with scheme
        # ``ROADMAP.md``.  Once the trailing line number is removed, a path
        # with no scheme is still a valid repo-relative evidence source.
        path_parsed = urllib.parse.urlsplit(repo_path)
        if not (repo_path != value and not path_parsed.scheme):
            raise line_error(article, article.body_start_line,
                             f"unsupported evidence URL {source!r}")
    if value.startswith("//"):
        raise line_error(article, article.body_start_line,
                         f"unsupported evidence URL {source!r}")
    if value.startswith("/"):
        return context.resolve(value, article.category, article,
                               article.body_start_line).href
    if repo_path and (context.repo / repo_path).exists():
        return None

    # Keep the existing dangling-target wording for missing relative sources,
    # but do not let a relative value resolve as a site article by accident.
    try:
        path, _query, _fragment = context._normalise_path(repo_path, article.category)
    except BuildError as exc:
        raise line_error(article, article.body_start_line, str(exc)) from exc
    raise line_error(article, article.body_start_line,
                     f"dangling internal target {source!r} (resolved as {path})")


GITHUB_BLOB = "https://github.com/xerj-org/xerj/blob/main/"


CAPTURE_LABEL = "the captured run documented on this page"


def key_fact_href(article: article_data.Article, context: BuildContext) -> str | None:
    """Where a reader checks the key number: the URL, the site page, or the
    file on GitHub (a repo path gets a real link here, unlike evidence rows,
    because an answer engine needs a URL it can cite). None for "capture":
    the number comes from this page's own run, so there is nothing to link."""
    source = (article.key_fact_source or "").strip()
    if source == article_data.CAPTURE_SOURCE:
        return None
    if source.startswith("https://"):
        return source
    if source.startswith("/"):
        return context.resolve(source, article.category, article,
                               article.body_start_line).href
    return GITHUB_BLOB + _evidence_repo_path(source)


def key_fact_label(article: article_data.Article) -> str:
    source = (article.key_fact_source or "").strip()
    if source.startswith("https://"):
        parsed = urllib.parse.urlsplit(source)
        return parsed.netloc + (parsed.path.rstrip("/") or "")
    return source


_INTENT_TOKEN_RE = re.compile(r"[a-z0-9]+")
_INTENT_STOP = frozenset("a an and are can do does for from how i in is it my of on or "
                         "the to what when which with without you your xerj".split())


def _intent_tokens(article: article_data.Article) -> set[str]:
    text = " ".join([article.title, article.question, *article.queries]).lower()
    return {t for t in _INTENT_TOKEN_RE.findall(text) if t not in _INTENT_STOP and len(t) > 2}


def intent_siblings(article: article_data.Article, context: BuildContext,
                    limit: int = 3) -> list[article_data.Article]:
    """Up to ``limit`` other articles in the same intent group, not already in
    links_out, most-overlapping first. Deterministic: ties break on slug."""
    if not article.intent_group:
        return []
    linked: set[str] = set()
    for raw in article.links_out:
        target = context.resolve_article_target(raw, article, article.body_start_line)
        linked.add(target.rel)
    mine = _intent_tokens(article)
    pool = [a for a in context.articles
            if a is not article and a.intent_group == article.intent_group
            and a.output_rel not in linked and not a.noindex]
    pool.sort(key=lambda a: (-len(mine & _intent_tokens(a)), a.category, a.slug))
    return pool[:limit]


def render_answer_card(article: article_data.Article, context: BuildContext,
                       modified: str) -> str:
    """The answer-first block under the H1: a 40-60 word answer that restates
    the question, then one dated, sourced number."""
    if not article.answer and not article.key_fact and not article.last_checked:
        return ""
    out = ['  <div class="prompt-card answer-first" id="short-answer">',
           f'    <span class="prompt-label">SHORT ANSWER · UPDATED {esc_text(modified)}</span>']
    if article.answer:
        out.append("    <p>" + _inline(article.answer, article, article.body_start_line,
                                       context.markdown_href) + "</p>")
    if article.key_fact:
        href = key_fact_href(article, context)
        out.append('    <p class="key-fact"><strong>Key number:</strong> '
                   + _inline(article.key_fact, article, article.body_start_line,
                             context.markdown_href)
                   + f' <span class="mute">(as of {esc_text(article.key_fact_date or "")} · source: '
                   + (CAPTURE_LABEL if href is None else
                      f'<a href="{esc_attr(href)}">{esc_text(key_fact_label(article))}</a>')
                   + ')</span></p>')
    if article.last_checked:
        out.append(f'    <p class="mute">Last checked: {esc_text(article.last_checked)}. Every '
                   "third-party fact on this page was re-read from the tool's own docs or "
                   "repository on that date.</p>")
    out.append("  </div>")
    return "\n".join(out)


def markdown_rel(article: article_data.Article) -> str:
    return f"{article.category}/{article.slug}.md"


def markdown_path(article: article_data.Article) -> str:
    return "/" + markdown_rel(article)


def markdown_url(article: article_data.Article) -> str:
    return urlmap.SITE_ORIGIN + markdown_path(article)


def article_modified(article: article_data.Article, context: BuildContext) -> str:
    return article.updated or context.dates.last_modified(article.path)


def render_agent_prompt(article: article_data.Article) -> str:
    if not article.agent_prompt:
        return ""
    prompt = article.agent_prompt
    return ("  <div class=\"prompt-card\">\n"
            "    <span class=\"prompt-label\">PASTE THIS TO YOUR AI AGENT — IT DOES THE REST</span>\n"
            f"    <code>{esc_text(prompt)}</code>\n"
            f"    <button type=\"button\" class=\"prompt-copy\" data-copy=\"{esc_attr(prompt)}\">"
            "COPY THE PROMPT</button>\n"
            "  </div>")


def render_commands(article: article_data.Article) -> str:
    if not article.commands:
        return ""
    out: list[str] = []
    for index, item in enumerate(article.commands, start=1):
        command = item["cmd"]
        note = item["note"]
        step_id = f' id="step-{index}"' if "howto" in article.structured_data else ""
        out.append(f"  <div class=\"prompt-card\"{step_id}>")
        out.append("    <span class=\"prompt-label\">RUN THIS XERJ COMMAND</span>")
        out.append(f"    <code>{esc_text(command)}</code>")
        out.append(f"    <button type=\"button\" class=\"prompt-copy\" data-copy=\"{esc_attr(command)}\">"
                   "COPY COMMAND</button>")
        out.append(f"    <span class=\"hero-proof\">{esc_text(note)}</span>")
        out.append("  </div>")
    return "\n".join(out)


def _split_tldr(body: str) -> tuple[str, str]:
    lines = body.splitlines()
    start = next((i for i, line in enumerate(lines) if line.strip()), 0)
    end = start
    while end < len(lines) and lines[end].strip():
        end += 1
    return "\n".join(lines[start:end]).strip(), "\n".join(lines[end:]).strip()


def _md_header_value(value: str) -> str:
    return json.dumps(value, ensure_ascii=False)


def render_markdown_twin(article: article_data.Article, context: BuildContext) -> str:
    """Emit the agent-readable Markdown twin without HTML presentation chrome."""
    modified = article_modified(article, context)
    tldr, body = _split_tldr(article.body)
    source = article.path.relative_to(context.repo).as_posix()
    out = [
        f"{GENERATOR_PREFIX}{article.category}/{article.slug}.md instead -->",
        "---",
        f"title: {_md_header_value(article.title)}",
        f"canonical: {_md_header_value(urlmap.deployed_url(article.output_rel))}",
        f"updated: {_md_header_value(modified)}",
        f"source: {_md_header_value(source)}",
        "---",
        "",
        f"# {article.h1}",
        "",
    ]
    if article.answer:
        out += [f"**Short answer** (updated {modified}): {article.answer}", ""]
    if article.key_fact:
        href = key_fact_href(article, context)
        where = CAPTURE_LABEL if href is None else f"[{key_fact_label(article)}]({href})"
        out += [f"**Key number:** {article.key_fact} (as of {article.key_fact_date}; "
                f"source: {where})", ""]
    if article.last_checked:
        out += [f"Last checked: {article.last_checked}. Every third-party fact on this page "
                "was re-read from the tool's own docs or repository on that date.", ""]
    out += [tldr]
    if article.agent_prompt:
        out += ["", "## Agent prompt", "", "```text", article.agent_prompt, "```"]
    if article.commands:
        out += ["", "## Commands", ""]
        for index, item in enumerate(article.commands, start=1):
            out += [f"### Command {index}", "", f"Note: {item['note']}", "",
                    "```sh", item["cmd"], "```", ""]
        while out and out[-1] == "":
            out.pop()
    if body:
        out += ["", _twin_visuals(body, article, context)]

    out += ["", "## FAQ", ""]
    for item in article.faq:
        out += [f"### {item['q'].strip()}", "", item["a"].strip(), ""]
    if out[-1] == "":
        out.pop()

    # No evidence block means no provenance section at all.  An "## Evidence"
    # heading with a placeholder under it is worse than silence: it tells a
    # reader the article was sourced and then shows nothing.  Sources that ARE
    # present still go through _metadata_link and still have to resolve.
    if article.evidence:
        out += ["", "## Evidence", ""]
        for item in article.evidence:
            href = _metadata_link(item["source"], article, context)
            source = item["source"].strip()
            rendered_source = (f"[{source}]({href})" if href is not None
                               else f"`{source}`")
            out.append(f"- {item['claim'].strip()} — {rendered_source}")

    out += ["", "## Related", ""]
    siblings = intent_siblings(article, context)
    if article.links_out or siblings:
        for raw in article.links_out:
            target = context.resolve_article_target(raw, article, article.body_start_line)
            out.append(f"- [{target.label}]({target.href})")
        for sibling in siblings:
            out.append(f"- [{sibling.h1}]({urlmap.deployed_path(sibling.output_rel)}) "
                       f"(same intent: {article.intent_group})")
    else:
        out.append("No related pages are listed.")
    return "\n".join(out).rstrip() + "\n"


def render_article_sections(article: article_data.Article, context: BuildContext) -> str:
    body_blocks = render_markdown_blocks(article, context)
    out: list[str] = []
    if body_blocks:
        out.append(body_blocks[0])
    rest = body_blocks[1:]
    # A visual placed directly under the TL;DR is the page's lead figure: it goes
    # above the prompt and command cards so the answer, then the picture, come first.
    if rest and rest[0].startswith('  <figure class="vz'):
        out.append(rest.pop(0))
    prompt = render_agent_prompt(article)
    if prompt:
        out.append(prompt)
    commands = render_commands(article)
    if commands:
        out.append(commands)
    out.extend(rest)
    out.append("  <h2>FAQ</h2>")
    for item in article.faq:
        # No grammar rule on the question string. A real user types "The
        # indexer died overnight." and matching that phrasing verbatim is the
        # whole mechanism this content depends on; a question-word validator
        # rejected exactly the strings worth matching. Structure (4-8 entries)
        # is still enforced in article_data.py.
        question = item["q"].strip()
        out.append(f"  <h3>{_inline(question, article, article.body_start_line, context.markdown_href)}</h3>")
        out.append("  <p>" + _inline(item["a"].strip(), article, article.body_start_line,
                                      context.markdown_href) + "</p>")

    # Same rule as the Markdown twin: the whole section is omitted when there
    # is no evidence block, so no page ships an empty "Evidence" heading.
    if article.evidence:
        out.append("  <h2>Evidence</h2>")
        out.append("  <ul>")
        for item in article.evidence:
            href = _metadata_link(item["source"], article, context)
            source = item["source"].strip()
            if href is None:
                source_html = f"<code>{esc_text(source)}</code>"
            else:
                source_html = f'<a href="{esc_attr(href)}">{esc_text(source)}</a>'
            out.append("    <li>" + _inline(item["claim"].strip(), article,
                                              article.body_start_line,
                                              context.markdown_href)
                       + " — " + source_html + "</li>")
        out.append("  </ul>")

    out.append("  <h2>Related</h2>")
    siblings = intent_siblings(article, context)
    if article.links_out or siblings:
        out.append("  <ul>")
        for raw in article.links_out:
            target = context.resolve_article_target(raw, article, article.body_start_line)
            out.append(f'    <li><a href="{esc_attr(target.href)}">'
                       f"{esc_text(target.label)}</a></li>")
        for sibling in siblings:
            out.append(f'    <li><a href="{urlmap.deployed_path(sibling.output_rel)}">'
                       f"{esc_text(sibling.h1)}</a> <span class=\"mute\">· same intent: "
                       f"{esc_text(article.intent_group or '')}</span></li>")
        out.append("  </ul>")
    else:
        out.append("  <p>No related pages are listed.</p>")
    return "\n".join(out)


def primary_nav(rel: str) -> str:
    """The site-wide primary nav; scripts/seo/set_nav.py owns its definition."""
    return set_nav.render(rel)


def article_nav(context: BuildContext, active: article_data.Article | None) -> str:
    groups = [("ANSWERS", "answers"), ("COMPARISONS", "compare")]
    out = ['<nav class="docs-nav" aria-label="Article sections">']
    for label, category in groups:
        out.append('  <div class="group">')
        out.append(f'    <span class="label">{label}</span>')
        out.append(f'    <a href="/{category}/"><span class="eyebrow">00</span>All {label.title()}</a>')
        members = sorted((a for a in context.articles if a.category == category),
                         key=lambda a: (a.cluster.lower(), a.h1.lower()))
        for index, member in enumerate(members, start=1):
            active_class = " active" if active is member else ""
            out.append(f'    <a href="{urlmap.deployed_path(member.output_rel)}"'
                       f' class="{active_class.strip()}"><span class="eyebrow">{index:02}</span>'
                       f"{esc_text(member.h1)}</a>")
        out.append("  </div>")
    out.append("</nav>")
    return "\n".join(out)


def breadcrumb(rel: str, meta: dict[str, Any]) -> str:
    parts = [f'<a href="/">HOME</a>']
    directories = rel.split("/")[:-1]
    if rel.endswith("/index.html"):
        directories = directories[:-1]
    for i in range(len(directories)):
        directory = "/".join(directories[:i + 1])
        hub = pagedata.SECTION_HUBS.get(directory)
        if hub:
            label, path = hub
            parts.append(f'<a href="{esc_attr(path)}">{esc_text(label.upper())}</a>')
    parts.append(esc_text(str(meta["label"]).upper()))
    return '<div class="breadcrumb">' + '<span class="sep">·</span>'.join(parts) + "</div>"


# These pages carry NO release version, anywhere — not in the prose, not in
# the footer brand line. That is deliberate.
#
# A version stamp is a claim that goes stale on its own. This footer was
# hardcoded to `V1.0.0-RC.15`, copied from a page written when that was
# current, and it sat there for eight releases until
# `.github/scripts/landing-constants-guard.sh` failed the build. Deriving it
# from the newest tag fixes the drift but keeps a claim these pages do not
# need to make: an answer page tells a reader to install the latest XERJ, so
# naming a release only invites the question of whether the page still applies
# to it.
#
# The guard is satisfied by absence, not by agreement. It matches
# `XERJ.AI[^<]*V1.0.0-RC.?[0-9]+` and flags only stamps that DISAGREE with the
# newest tag, so a footer without one is never examined. Do not "fix" this by
# adding a version back — that re-creates the drift the guard exists to catch,
# and it would be the only version string on pages whose whole point is that
# they do not depend on one.
def footer(category: str) -> str:
    return fix_links.render_footer() + f'''\n<footer class="footer">
  <span>XERJ.AI · {esc_text(category.upper())}</span>
  <span>READ THE SOURCE</span>
  <span><a href="/">HOME</a> · <a href="/product">PRODUCT</a> · <a href="/answers/">ANSWERS</a> · <a href="/playground">DASHBOARDS</a> &middot; <a href="/llms.txt">LLMS.TXT</a></span>
</footer>'''


def _head(title: str, description: str, rel: str, block: str,
          alternate_markdown: str | None = None, visuals: bool = False) -> str:
    alternate = (f'<link rel="alternate" type="text/markdown" href="{esc_attr(alternate_markdown)}">\n'
                 if alternate_markdown else "")
    return f'''<!DOCTYPE html>
<html lang="en" data-theme="day">
<head>
{seo_head.ga4_snippet()}
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{seo_head.text_esc(title)}</title>
<meta name="description" content="{seo_head.esc(description)}">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Big+Shoulders+Display:wght@400;700;800;900&family=Inter:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;700&display=swap" rel="stylesheet">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' fill='%230b0b0d'/><text x='50' y='68' font-size='60' font-family='Big Shoulders Display,sans-serif' font-weight='900' fill='%23ffc400' text-anchor='middle'>X</text></svg>">
<script src="/theme.js"></script>
<link rel="stylesheet" href="/style.css?v=as2">
{'<link rel="stylesheet" href="/visuals.css?v=2">' + chr(10) if visuals else ''}{alternate}{block}
</head>'''


# Questions longer than this get a smaller H1 on phones (style.css, .h1-long)
# so a verbatim user question still fits in about five lines at 390px.
# Desktop sizing does not change.
H1_LONG_CHARS = 80


def render_article(article: article_data.Article, context: BuildContext) -> str:
    rel = article.output_rel
    meta = article.head_meta()
    modified = article_modified(article, context)
    block = seo_head.build_block(rel, article.title, article.description,
                                 article.published, modified, meta)
    body = render_article_sections(article, context)
    visuals = has_visuals(article)
    answer_card = render_answer_card(article, context, modified)
    if answer_card:
        answer_card += "\n"
    marker = f"{GENERATOR_PREFIX}{article.category}/{article.slug}.md instead -->"
    source_note = (f"AUTHOR · {article.author} · REVIEWED BY · {article.reviewer} · "
                   f"PUBLISHED · {article.published} · UPDATED · {modified}")
    return f'''{marker}
{_head(article.title, article.description, rel, block, markdown_path(article), visuals)}
<body>
{primary_nav(rel)}
<div class="docs-shell article-shell">
<main class="docs-content">
  {breadcrumb(rel, meta)}
  <div class="kicker"><span>{esc_text(article.category.upper())} · {esc_text(article.cluster.upper())}</span></div>
  <h1{' class="h1-long"' if len(article.h1) > H1_LONG_CHARS else ''}>{_inline(article.h1, article, article.body_start_line, context.markdown_href)}</h1>
  <div class="source-note">{esc_text(source_note)}</div>
{answer_card}{body}
</main>
{article_nav(context, article)}
</div>
{footer(article.category)}
{'<script src="/visuals.js?v=2"></script>' + chr(10) if visuals else ''}<script src="/interact.js"></script>
</body>
</html>
'''


def render_hub(category: str, articles: list[article_data.Article],
               context: BuildContext) -> str:
    rel = f"{category}/index.html"
    meta = dict(pagedata.entry(rel))
    # A hub's dates come from the articles it lists, never from the hub file
    # itself: reading the generated file's own git date makes the output depend
    # on the commit that wrote it, so committing a regenerated hub changes the
    # value it should have had. Every merge that touches the file on a later
    # date then re-stales it and `--check` fails on main (that is #942's red
    # gate, and #972's re-red one commit later). Article pages never had the
    # problem because they date from their content source.
    listed = [a for a in articles if a.category == category]
    if listed:
        modified = max(article_modified(a, context) for a in listed)
        published = min(a.published for a in listed if a.published) or modified
    else:
        modified = published = dt.datetime.now(dt.timezone.utc).date().isoformat()
    block = seo_head.build_block(rel, meta["title"], meta["description"],
                                 published, modified, meta)
    groups: dict[str, list[article_data.Article]] = collections.defaultdict(list)
    for article in articles:
        if article.category == category:
            groups[article.cluster].append(article)
    content = [
        f'  <p>{esc_text(meta["description"])}</p>',
    ]
    if groups:
        for cluster in sorted(groups, key=str.lower):
            content.append(f"  <h2>{esc_text(cluster)}</h2>")
            content.append("  <ul>")
            for article in sorted(groups[cluster], key=lambda a: a.h1.lower()):
                content.append(f'    <li><a href="{urlmap.deployed_path(article.output_rel)}">'
                               f"{esc_text(article.h1)}</a> — {esc_text(article.description)}</li>")
            content.append("  </ul>")
    else:
        content.append("  <h2>Articles</h2>")
        content.append("  <p>No articles are published in this section yet.</p>")
    marker = f"{GENERATOR_PREFIX}{category}/*.md instead -->"
    return f'''{marker}
{_head(meta["title"], meta["description"], rel, block)}
<body>
{primary_nav(rel)}
<div class="docs-shell article-shell">
<main class="docs-content">
  {breadcrumb(rel, meta)}
  <div class="kicker"><span>{esc_text(category.upper())}</span></div>
  <h1>{esc_text(meta["label"])}</h1>
{chr(10).join(content)}
</main>
{article_nav(context, None)}
</div>
{footer(category)}
<script src="/interact.js"></script>
</body>
</html>
'''


def discover(repo: pathlib.Path) -> list[article_data.Article]:
    articles: list[article_data.Article] = []
    for category in ("answers", "compare"):
        directory = repo / "content" / category
        if not directory.exists():
            continue
        for path in sorted(directory.glob("*.md")):
            articles.append(article_data.load(path, category))
    slugs: dict[str, pathlib.Path] = {}
    for article in articles:
        previous = slugs.get(article.slug)
        if previous is not None:
            raise BuildError(f"{article.path}: slug {article.slug!r} duplicates {previous}")
        slugs[article.slug] = article.path
    return articles


def article_index_entry(article: article_data.Article, context: BuildContext) -> dict[str, str]:
    return {
        "title": article.title,
        "question": article.question,
        "canonical_url": urlmap.deployed_url(article.output_rel),
        "markdown_url": markdown_url(article),
        "cluster": article.cluster,
        "updated": article_modified(article, context),
        "summary": " ".join(article.description.split()),
        **({"intent_group": article.intent_group} if article.intent_group else {}),
        **({"answer": article.answer} if article.answer else {}),
        **({"queries": list(article.queries)} if article.queries else {}),
    }


def render_index_json(category: str, articles: list[article_data.Article],
                      context: BuildContext) -> str:
    entries = [article_index_entry(article, context)
               for article in sorted(articles, key=lambda a: (a.cluster.lower(), a.title.lower()))
               if article.category == category]
    payload = {
        "generated_by": "scripts/seo/build_articles.py",
        "section": category,
        "articles": entries,
    }
    text = json.dumps(payload, ensure_ascii=False, indent=2) + "\n"
    try:
        json.loads(text)
    except json.JSONDecodeError as exc:  # pragma: no cover - json.dumps is deterministic
        raise BuildError(f"generated {category}/index.json is not valid JSON: {exc}") from exc
    return text


def render_llms_section(articles: list[article_data.Article], context: BuildContext) -> str:
    """Render the generated article index in the direct-to-agent llms.txt voice.

    One line per article, in the shape the llms.txt spec asks for —
    ``[name](url): notes`` — with the **Markdown twin as the href**, because the
    spec says the links "should point to LLM-friendly content".  The previous
    form spent three lines per article (title + question + summary, then a bare
    ``Markdown twin: <url>`` line that was not even a link) and cost ~33 KB for
    76 pages, roughly 440 bytes per link against a 155-byte corpus median.

    Nothing became unreachable: the canonical HTML page is the same URL without
    ``.md`` (stated once above the list), and the per-article summary, cluster
    and last-modified date stay machine-readable in ``answers/index.json`` and
    ``compare/index.json``, both linked from the block.
    """
    lines = [
        LLMS_BEGIN,
        "## Answers and comparisons, grouped by what the agent is trying to do",
        "",
        "Direct answers for you, the AI agent, grouped by intent: the clusters come from a "
        "1,000-query study of what agents actually type into search, so find the group that "
        "matches your task first. Each line is the link, a tag (`answer` or `compare`), then the "
        "question that page answers. "
        "**Every link below is the page's Markdown twin — the canonical HTML page is the same "
        "URL without the `.md`.** The twin is deliberately not in the sitemap because it is the "
        "same document. Titles, questions, short answers, target queries, intents and "
        "last-modified dates for every page, as JSON: "
        "[answers/index.json](https://xerj.org/answers/index.json) · "
        "[compare/index.json](https://xerj.org/compare/index.json). Human hubs: "
        "[/answers](https://xerj.org/answers) · [/compare](https://xerj.org/compare).",
        "",
    ]
    if not articles:
        lines.append("- No pages are published in this section yet.")
        lines.append("")
    groups: dict[str, list[article_data.Article]] = collections.defaultdict(list)
    for article in articles:
        groups[article.intent_group or article.cluster].append(article)
    # Biggest intent first (that is where most agents land), then by name.
    for group in sorted(groups, key=lambda g: (-len(groups[g]), g.lower())):
        lines.append(f"### {group}")
        lines.append("")
        members = sorted(groups[group],
                         key=lambda a: (a.category != "compare", a.title.lower()))
        for article in members:
            entry = article_index_entry(article, context)
            # The "canonical = twin minus .md" rule is stated once above the list,
            # so assert the relationship the prose promises still holds.
            if entry["markdown_url"] != entry["canonical_url"] + ".md":
                raise BuildError(
                    f"{article.path}: Markdown twin {entry['markdown_url']} is not "
                    f"{entry['canonical_url']}.md, but llms.txt states that rule once")
            tag = "compare" if article.category == "compare" else "answer"
            lines.append(f"- [{entry['title']}]({entry['markdown_url']}) `{tag}`: "
                         f"{entry['question']}")
        lines.append("")
    lines.append(LLMS_END)
    return "\n".join(lines).rstrip() + "\n"


def splice_llms(text: str, section: str) -> str:
    """Replace the generated block, or insert it before machine artifacts."""
    begin = text.find(LLMS_BEGIN)
    end = text.find(LLMS_END)
    if (begin < 0) != (end < 0):
        raise BuildError("landing/llms.txt has only one article-tree generated marker")
    if begin >= 0:
        if end < begin:
            raise BuildError("landing/llms.txt article-tree markers are out of order")
        end += len(LLMS_END)
        prefix = text[:begin].rstrip("\n")
        suffix = text[end:].lstrip("\n")
        return prefix + "\n\n" + section.rstrip("\n") + ("\n\n" + suffix if suffix else "\n")

    # Fallback only — the markers above are the normal path.  Listed in the
    # order the article index should appear before.
    for anchor in ("\n## Use cases and recipes", "\n## Machine-readable artifacts",
                   "\n## Optional"):
        position = text.find(anchor)
        if position >= 0:
            break
    else:
        raise BuildError("landing/llms.txt has no article-index insertion point")
    prefix = text[:position].rstrip("\n")
    suffix = text[position:].lstrip("\n")
    return prefix + "\n\n" + section.rstrip("\n") + "\n\n" + suffix


def run_ste(articles: list[article_data.Article], allow_errors: bool) -> dict[str, Any]:
    result: dict[str, Any] = {"files": len(articles), "errors": 0, "warnings": 0, "info": 0}
    script = SCRIPT_DIR / "ste_check.py"
    for article in articles:
        proc = subprocess.run(
            [sys.executable, str(script), "--json", str(article.path)],
            capture_output=True, text=True,
        )
        if proc.returncode == 2:
            raise BuildError(proc.stderr.strip() or f"ste_check.py failed for {article.path}")
        try:
            report = json.loads(proc.stdout)
        except json.JSONDecodeError as exc:
            raise BuildError(f"ste_check.py returned invalid JSON for {article.path}: {exc}") from exc
        totals = report.get("totals", {})
        count_keys = {"ERROR": "errors", "WARN": "warnings", "INFO": "info"}
        for key, result_key in count_keys.items():
            result[result_key] += int(totals.get(key, 0))
        findings = [f for f in report.get("findings", []) if f.get("severity") == "ERROR"]
        if findings:
            result["errors"] += len(findings)
            for finding in findings:
                message = (f"{finding.get('path', article.path)}:{finding.get('line', 1)}:"
                           f"{finding.get('col', 1)}: ERROR {finding.get('rule', '')} "
                           f"{finding.get('message', '')}")
                print("STE " + message, file=sys.stderr)
            # Reported, never raised.  ASD-STE100 is a house style for technical
            # writing, and its rules fight the one thing these pages exist to
            # do: match the sentence a person actually types.  An 84-prompt
            # evaluation returned 0 brand mentions across two engines, and the
            # queries people fired were statements — "The indexer died
            # overnight", "My codebase indexer says indexed but I don't see my
            # code".  Blocking a build over the grammar of a quoted user
            # question optimises the wrong thing.
            #
            # `ste_check.py` stays in the repo and stays useful; run it while
            # drafting. It is advisory here and advisory in CI, which is what
            # README §7 already says — until 2026-08-22 this line quietly
            # contradicted it, so STE was a hard gate through the back door.
    return result


def assert_reachability(context: BuildContext, categories: set[str]) -> None:
    footer_paths = {href for _heading, links in fix_links.FOOTER_MAP for href, _ in links}
    for category in categories:
        hub_path = f"/{category}/"
        if hub_path not in footer_paths:
            raise BuildError(f"{category}/index.html is not linked from the site footer")
        hub_rel = f"{category}/index.html"
        category_articles = [a for a in context.articles if a.category == category]
        if hub_rel not in context.known:
            raise BuildError(f"missing generated hub target {hub_rel}")
        for article in category_articles:
            if article.output_rel not in context.known:
                raise BuildError(f"missing generated article target {article.output_rel}")
    for article in context.articles:
        for raw in article.links_out:
            context.resolve_article_target(raw, article, article.body_start_line)


def select_articles(articles: list[article_data.Article], only: str | None) -> list[article_data.Article]:
    if not only:
        return articles
    value = only.strip().strip("/")
    if value.endswith(".md"):
        value = value[:-3]
    if value.endswith(".html"):
        value = value[:-5]
    if value.startswith("answers/") or value.startswith("compare/"):
        value = value.split("/", 1)[1]
    selected = [article for article in articles if article.slug == value]
    if not selected:
        raise BuildError(f"--only {only!r} did not match an article slug")
    if len(selected) > 1:
        raise BuildError(f"--only {only!r} matches more than one article; include its section")
    return selected


def build_expected(repo: pathlib.Path, landing: pathlib.Path, all_articles: list[article_data.Article],
                   selected: list[article_data.Article], only: str | None) -> tuple[dict[str, str], dict[str, Any]]:
    dates = urlmap.DateSource(repo)
    context = BuildContext(repo, landing, all_articles, dates)
    categories = {a.category for a in selected} if only else {"answers", "compare"}
    assert_reachability(context, categories)
    expected: dict[str, str] = {}
    for article in selected:
        expected[article.output_rel] = render_article(article, context)
        expected[markdown_rel(article)] = render_markdown_twin(article, context)
    for category in sorted(categories):
        expected[f"{category}/index.html"] = render_hub(category, all_articles, context)
    if not only:
        for category in ("answers", "compare"):
            expected[f"{category}/index.json"] = render_index_json(category, all_articles, context)
        llms = landing / "llms.txt"
        current = llms.read_text(encoding="utf-8") if llms.exists() else ""
        expected["llms.txt"] = splice_llms(current, render_llms_section(all_articles, context))
    assert_generated_contract(expected, landing, selected, only)
    return expected, {"dates": dates, "context": context}


def assert_generated_contract(expected: dict[str, str], landing: pathlib.Path,
                              selected: list[article_data.Article], only: str | None) -> None:
    for article in selected:
        html_output = expected.get(article.output_rel, "")
        alternate = (f'<link rel="alternate" type="text/markdown" href="{markdown_path(article)}">')
        if alternate not in html_output:
            raise BuildError(f"{article.output_rel}: missing text/markdown alternate link")
        if re.search(r"<a\b[^>]*href=[\"'](?![A-Za-z][A-Za-z0-9+.-]*:|//)[^\"']+\.md(?:[?#][^\"']*)?[\"']",
                     html_output, re.I):
            raise BuildError(f"{article.output_rel}: Markdown twin must not be an ordinary <a> link")
        md_path = landing / markdown_rel(article)
        if not md_path.is_file() and markdown_rel(article) not in expected:
            raise BuildError(f"{article.output_rel}: Markdown twin {md_path} is missing")
    if not only:
        for rel in ("answers/index.json", "compare/index.json"):
            try:
                json.loads(expected[rel])
            except (KeyError, json.JSONDecodeError) as exc:
                raise BuildError(f"{rel}: generated JSON does not parse: {exc}") from exc


def _write_or_compare(landing: pathlib.Path, expected: dict[str, str], write: bool,
                      only: str | None) -> list[str]:
    changed: list[str] = []
    if write:
        for rel, output in sorted(expected.items()):
            path = landing / rel
            if path.exists():
                current = path.read_text(encoding="utf-8")
                if rel.endswith((".html", ".md")) and not GENERATOR_RE.search(current):
                    raise BuildError(f"{path}: exists but is not generated by build_articles.py")
            if not path.exists() or path.read_text(encoding="utf-8") != output:
                changed.append(rel)
                path.parent.mkdir(parents=True, exist_ok=True)
                path.write_text(output, encoding="utf-8")
        return changed

    with tempfile.TemporaryDirectory(prefix="xerj-articles-") as temp_name:
        temp = pathlib.Path(temp_name)
        for rel, output in expected.items():
            path = temp / rel
            path.parent.mkdir(parents=True, exist_ok=True)
            path.write_text(output, encoding="utf-8")
            current_path = landing / rel
            current = current_path.read_text(encoding="utf-8") if current_path.exists() else ""
            if current != output:
                changed.append(rel)
    if not only:
        for category in ("answers", "compare"):
            directory = landing / category
            if not directory.exists():
                continue
            expected_names = set(expected)
            for path in sorted(directory.glob("*.html")):
                rel = path.relative_to(landing).as_posix()
                if rel not in expected_names and GENERATOR_RE.search(path.read_text(encoding="utf-8")):
                    raise BuildError(f"{path}: generated output has no matching Markdown source")
            for path in sorted(directory.glob("*.md")):
                rel = path.relative_to(landing).as_posix()
                if rel not in expected_names and GENERATOR_RE.search(path.read_text(encoding="utf-8")):
                    raise BuildError(f"{path}: generated Markdown twin has no matching source")
    return changed


def run(argv: list[str] | None = None) -> tuple[int, dict[str, Any]]:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    mode = ap.add_mutually_exclusive_group(required=True)
    mode.add_argument("--write", action="store_true", help="write generated HTML")
    mode.add_argument("--check", action="store_true",
                      help="regenerate in a temporary directory and compare committed HTML")
    ap.add_argument("--only", metavar="SLUG", help="build one article slug and its section hub")
    ap.add_argument("--allow-ste-errors", action="store_true",
                    help="report, but do not block on STE ERROR-tier findings")
    ap.add_argument("--json", action="store_true", help="emit a machine-readable report")
    ap.add_argument("--root", default=None, help="repository root")
    args = ap.parse_args(argv)

    repo = pathlib.Path(args.root).resolve() if args.root else urlmap.repo_root()
    landing = repo / "landing"
    try:
        all_articles = discover(repo)
        selected = select_articles(all_articles, args.only)
        ste = run_ste(selected, args.allow_ste_errors)
        expected, state = build_expected(repo, landing, all_articles, selected, args.only)
        changed = _write_or_compare(landing, expected, args.write, args.only)
        report = {
            "status": "ok" if not changed or args.write else "stale",
            "mode": "write" if args.write else "check",
            "only": args.only,
            "articles": len(selected),
            "all_sources": len(all_articles),
            "outputs": sorted(expected),
            "changed": sorted(changed),
            "ste": ste,
            "git_dates": state["dates"].git_ok,
        }
        if args.json:
            print(json.dumps(report, indent=2, ensure_ascii=False))
        elif args.check:
            if changed:
                print(f"FAIL {len(changed)} generated file(s) differ:")
                for rel in changed:
                    print(f"  {rel}")
            else:
                print(f"ok   build_articles.py output is current ({len(expected)} files; "
                      f"{len(selected)} selected article(s))")
        else:
            print(f"wrote {len(changed)} changed generated file(s) ({len(expected)} outputs; "
                  f"{len(selected)} selected article(s))")
        return (1 if changed and args.check else 0), report
    except (BuildError, article_data.FrontmatterError, OSError) as exc:
        report = {"status": "error", "error": str(exc)}
        if args.json:
            print(json.dumps(report, indent=2, ensure_ascii=False))
        else:
            print(f"build_articles: {exc}", file=sys.stderr)
        return 1, report


if __name__ == "__main__":
    raise SystemExit(run()[0])