AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
#!/usr/bin/env python3
"""Build the Markdown article sections of xerj.org.
The generator has no runtime dependencies beyond Python's standard library.
Sources live under ``content/answers`` and ``content/compare``. The output is
deliberately small, deterministic HTML that uses the site's existing nav,
docs-shell, CSS classes, head emitter and footer.
"""
from __future__ import annotations
import argparse
import collections
import datetime as dt
import html
import html as html_lib
import json
import pathlib
import posixpath
import re
import subprocess
import sys
import tempfile
import urllib.parse
from typing import Any, Callable
SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
import article_data # noqa: E402
import fix_links # noqa: E402
import set_nav # noqa: E402
import pagedata # noqa: E402
import seo_head # noqa: E402
import urlmap # noqa: E402
GENERATOR_PREFIX = "<!-- generated by build_articles.py — edit content/"
GENERATOR_RE = re.compile(
r"<!-- generated by build_articles\.py — edit content/(answers|compare)/"
r"(?:[a-z0-9-]+|\*)\.md instead -->"
)
LLMS_BEGIN = "<!-- BEGIN GENERATED ARTICLE TREE: build_articles.py -->"
LLMS_END = "<!-- END GENERATED ARTICLE TREE: build_articles.py -->"
HTML_TAG_RE = re.compile(r"<!--|<![A-Za-z]|</?[A-Za-z][^>]*>")
FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
HEADING_RE = re.compile(r"^ {0,3}(#{1,6})\s+(.+?)\s*$")
UL_RE = re.compile(r"^ {0,3}([-+*])\s+(.+?)\s*$")
OL_RE = re.compile(r"^ {0,3}(\d+)[.)]\s+(.+?)\s*$")
LANG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9+_.-]*$")
TLDR_RE = re.compile(r"^\s*\*\*TL[;:]?DR\*\*\s*[—–:.-]?\s*", re.I)
class BuildError(ValueError):
"""A user-actionable build error."""
class LinkTarget:
__slots__ = ("rel", "href", "label")
def __init__(self, rel: str, href: str, label: str):
self.rel = rel
self.href = href
self.label = label
def esc_text(value: str) -> str:
return html.escape(value, quote=False)
def esc_attr(value: str) -> str:
return html.escape(value, quote=True)
def line_error(article: article_data.Article, line: int, message: str) -> BuildError:
return BuildError(f"{article.path}:{line}: {message}")
class BuildContext:
def __init__(self, repo: pathlib.Path, landing: pathlib.Path,
articles: list[article_data.Article], dates: urlmap.DateSource):
self.repo = repo
self.landing = landing
self.dates = dates
self.articles = articles
self.by_rel = {a.output_rel: a for a in articles}
self.by_slug: dict[str, list[article_data.Article]] = collections.defaultdict(list)
for article in articles:
self.by_slug[article.slug].append(article)
self.existing = set(urlmap.iter_pages(landing))
self.known = self.existing | set(self.by_rel) | {
"answers/index.html", "compare/index.html",
}
def _normalise_path(self, raw: str, category: str) -> tuple[str, str, str]:
value = raw.strip()
if not value:
raise BuildError("empty internal link target")
parsed = urllib.parse.urlsplit(value)
if parsed.scheme or value.startswith("//"):
if parsed.scheme not in ("http", "https") or parsed.netloc not in (
"xerj.org", "www.xerj.org"):
raise BuildError(f"internal article link must stay on xerj.org: {raw!r}")
path = parsed.path or "/"
else:
path = parsed.path
if not path.startswith("/"):
if path.startswith(("answers/", "compare/", "docs/", "use-cases/",
"case-studies/", "industries/")):
path = "/" + path
else:
# A bare value is an article slug in the current section.
path = f"/{category}/{path}"
if "\\" in path:
raise BuildError(f"backslashes are not valid in internal link target {raw!r}")
path = posixpath.normpath(path)
if not path.startswith("/"):
path = "/" + path
if path == "/.":
path = "/"
return path, parsed.query, parsed.fragment
def resolve(self, raw: str, category: str, article: article_data.Article | None = None,
line: int | None = None) -> LinkTarget:
try:
path, query, fragment = self._normalise_path(raw, category)
except BuildError as exc:
if article is not None and line is not None:
raise line_error(article, line, str(exc)) from exc
raise
rel = urlmap.rel_for_path(path)
if rel is None or rel not in self.known or not (
(self.landing / rel).is_file() or rel in self.by_rel):
message = f"dangling internal target {raw!r} (resolved as {path})"
if article is not None and line is not None:
raise line_error(article, line, message)
raise BuildError(message)
target = self.by_rel.get(rel)
if target is not None:
label = target.h1
else:
try:
label = str(pagedata.entry(rel).get("label", rel))
except KeyError:
label = rel
href = urlmap.deployed_path(rel)
if query:
href += "?" + query
if fragment:
href += "#" + fragment
return LinkTarget(rel, href, label)
def resolve_article_target(self, raw: str, article: article_data.Article,
line: int | None = None) -> LinkTarget:
"""Resolve a links_out item, including a bare slug."""
value = raw.strip()
parsed = urllib.parse.urlsplit(value)
bare = parsed.path.strip("/")
if not parsed.scheme and not value.startswith("/") and "/" not in bare:
matches = self.by_slug.get(bare, [])
if len(matches) == 1:
return self.resolve(matches[0].output_rel, article.category, article, line)
if len(matches) > 1:
raise line_error(article, line or 1,
f"ambiguous article slug {raw!r}; use /answers/... or /compare/...")
return self.resolve(value, article.category, article, line)
def markdown_href(self, raw: str, article: article_data.Article,
line: int) -> str:
parsed = urllib.parse.urlsplit(raw.strip())
if parsed.scheme in ("http", "https") and parsed.netloc:
return raw.strip()
if parsed.scheme or raw.strip().startswith("//"):
raise line_error(article, line, f"unsupported Markdown link URL {raw!r}")
return self.resolve(raw, article.category, article, line).href
# A fenced block whose language is `visual` holds one line, the name of a figure in
# content/visuals/<name>.html. The figure is hand-written HTML and inline SVG, inlined
# into the page as-is, so it is held to an allowlist here: one <figure class="vz">
# root with a <figcaption>, no script, no style element, no event handler, no
# javascript: URL and no external media. Behaviour lives in landing/visuals.js and
# styling in landing/visuals.css; both load only on pages that carry a visual. The
# Markdown twin gets the caption, and factcheck.py reads the figure's visible text.
VISUAL_DIR = ("content", "visuals")
VISUAL_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
VISUAL_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})\s*visual\s*$")
VISUAL_FORBIDDEN = (
(re.compile(r"<\s*/?\s*(?:script|style|iframe|object|embed|link|meta|base|form|img|video|audio|source|foreignobject)\b", re.I),
"element is not allowed in a visual"),
(re.compile(r"\son[a-z]+\s*=", re.I), "event handler attributes are not allowed in a visual"),
(re.compile(r"javascript:", re.I), "javascript: URLs are not allowed in a visual"),
(re.compile(r"\s(?:src|srcset|xlink:href)\s*=", re.I), "external media is not allowed in a visual"),
(re.compile(r"\shref\s*=\s*[\"'](?!/|#|https://)", re.I), "a visual link must be /path, #anchor or https://"),
(re.compile("[\u2013\u2014]"), "en and em dashes are not allowed in a visual"),
)
_VISUAL_CACHE: dict[str, tuple[str, str]] = {}
def _indent_visual(html: str) -> str:
"""Indent a figure by two spaces, except lines inside <textarea> or <pre>,
whose leading whitespace is content."""
out: list[str] = []
raw = False
for line in html.splitlines():
out.append(line if raw or not line.strip() else " " + line)
low = line.lower()
opens = low.count("<textarea") + low.count("<pre")
closes = low.count("</textarea>") + low.count("</pre>")
if opens > closes:
raw = True
elif closes > opens:
raw = False
return "\n".join(out)
def load_visual(article: article_data.Article, name: str, line: int,
context: "BuildContext") -> tuple[str, str]:
"""Return (html, caption text) for content/visuals/<name>.html."""
if not VISUAL_NAME_RE.fullmatch(name):
raise line_error(article, line, f"visual name {name!r} must be lowercase-hyphenated")
path = context.repo.joinpath(*VISUAL_DIR, f"{name}.html")
key = str(path)
if key in _VISUAL_CACHE:
return _VISUAL_CACHE[key]
if not path.is_file():
raise line_error(article, line, f"visual {name!r} not found at {path.relative_to(context.repo)}")
html = path.read_text(encoding="utf-8").strip()
if not (html.startswith('<figure class="vz') and html.endswith("</figure>")):
raise line_error(article, line, f"visual {name!r} must be one <figure class=\"vz ...\"> element")
for pattern, message in VISUAL_FORBIDDEN:
hit = pattern.search(html)
if hit:
raise line_error(article, line, f"visual {name!r}: {message} ({hit.group(0).strip()!r})")
caption = re.search(r"<figcaption[^>]*>(.*?)</figcaption>", html, re.S)
if not caption:
raise line_error(article, line, f"visual {name!r} needs a <figcaption>")
text = " ".join(html_lib.unescape(re.sub(r"<[^>]+>", " ", caption.group(1))).split())
_VISUAL_CACHE[key] = (html, text)
return _VISUAL_CACHE[key]
def has_visuals(article: article_data.Article) -> bool:
return any(VISUAL_FENCE_RE.match(line) for line in article.body.splitlines())
def _twin_visuals(body: str, article: article_data.Article, context: "BuildContext") -> str:
lines = body.splitlines()
out: list[str] = []
i = 0
while i < len(lines):
if VISUAL_FENCE_RE.match(lines[i]) and i + 2 < len(lines) + 1:
name = lines[i + 1].strip() if i + 1 < len(lines) else ""
_, caption = load_visual(article, name, article.body_start_line, context)
out.append(f"*Figure: {caption}*")
i += 3
continue
out.append(lines[i])
i += 1
return "\n".join(out)
def _check_raw_html(lines: list[str], article: article_data.Article) -> None:
in_fence = False
marker = ""
for offset, raw in enumerate(lines):
line_no = article.body_start_line + offset
fence = FENCE_RE.match(raw)
if fence:
token = fence.group(1)
if not in_fence:
in_fence = True
marker = token[0]
elif token[0] == marker:
in_fence = False
continue
if in_fence:
continue
masked = re.sub(r"`[^`]*`", "", raw)
if HTML_TAG_RE.search(masked):
raise line_error(article, line_no, "raw HTML is not allowed in Markdown")
if in_fence:
raise line_error(article, article.body_start_line + len(lines),
"fenced code block has no closing fence")
def _split_table_row(raw: str) -> list[str]:
text = raw.strip()
if text.startswith("|"):
text = text[1:]
if text.endswith("|") and not text.endswith("\\|"):
text = text[:-1]
cells: list[str] = []
current: list[str] = []
escaped = False
for ch in text:
if escaped:
current.append(ch)
escaped = False
elif ch == "\\":
escaped = True
elif ch == "|":
cells.append("".join(current).strip())
current = []
else:
current.append(ch)
if escaped:
current.append("\\")
cells.append("".join(current).strip())
return cells
def _is_table_separator(raw: str) -> bool:
if "|" not in raw:
return False
cells = _split_table_row(raw)
return bool(cells) and all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
def _is_word_char(ch: str) -> bool:
"""Return True for characters that make ``_`` intraword, not emphasis."""
return ch.isalnum() or ch == "_"
def _inline(text: str, article: article_data.Article, line: int,
href_resolver: Callable[[str, article_data.Article, int], str]) -> str:
"""Render the safe inline subset with escaping at the text boundary."""
out: list[str] = []
i = 0
n = len(text)
while i < n:
if text.startswith("![", i):
raise line_error(article, line, "images are not supported in article Markdown")
if text[i] == "`":
end = text.find("`", i + 1)
if end < 0:
raise line_error(article, line, "inline code span has no closing backtick")
out.append(f'<code class="inline">{esc_text(text[i + 1:end])}</code>')
i = end + 1
continue
marker = None
width = 0
if text.startswith("**", i) or text.startswith("__", i):
marker, width = text[i:i + 2], 2
elif text[i] in "*_":
marker, width = text[i], 1
if marker is not None and marker[0] == "_" and i > 0 and _is_word_char(text[i - 1]):
# CommonMark forbids intraword ``_`` emphasis so that bare
# identifiers such as ``ignored_files`` stay literal text.
marker = None
if marker is not None:
end = text.find(marker, i + width)
if marker[0] == "_":
# A closing ``_`` may not be followed by a word character
# either, for the same reason.
while end >= 0 and end + width < n and _is_word_char(text[end + width]):
end = text.find(marker, end + width)
if end > i + width and not text[i + width].isspace():
inner = _inline(text[i + width:end], article, line, href_resolver)
tag = "strong" if width == 2 else "em"
out.append(f"<{tag}>{inner}</{tag}>")
i = end + width
continue
if text[i] == "[":
close = text.find("]", i + 1)
if close > i + 1 and close + 1 < n and text[close + 1] == "(":
end = text.find(")", close + 2)
if end < 0:
raise line_error(article, line, "Markdown link has no closing parenthesis")
label = text[i + 1:close]
destination = text[close + 2:end].strip()
if not destination or any(ch.isspace() for ch in destination):
raise line_error(article, line, "Markdown links need a single URL without spaces")
href = href_resolver(destination, article, line)
out.append(f'<a href="{esc_attr(href)}">'
f"{_inline(label, article, line, href_resolver)}</a>")
i = end + 1
continue
start = i
while i < n and text[i] not in "`[*_":
i += 1
if start == i:
out.append(esc_text(text[i]))
i += 1
else:
out.append(esc_text(text[start:i]))
return "".join(out)
def _block_start(lines: list[str], i: int) -> bool:
raw = lines[i]
return bool(FENCE_RE.match(raw) or HEADING_RE.match(raw) or UL_RE.match(raw)
or OL_RE.match(raw) or raw.lstrip().startswith(">"))
def render_markdown_blocks(article: article_data.Article, context: BuildContext) -> list[str]:
lines = article.body.splitlines()
_check_raw_html(lines, article)
first = next((line for line in lines if line.strip()), "")
if not TLDR_RE.match(first):
raise line_error(article, article.body_start_line,
"the first body block must be the TL;DR paragraph")
rendered: list[str] = []
i = 0
previous_heading = 1
while i < len(lines):
raw = lines[i]
line_no = article.body_start_line + i
if not raw.strip():
i += 1
continue
fence = FENCE_RE.match(raw)
if fence:
marker = fence.group(1)[0]
info = fence.group(2).strip()
if not info or not LANG_RE.fullmatch(info):
raise line_error(article, line_no,
"fenced code blocks need a simple language name")
i += 1
code: list[str] = []
closed = False
while i < len(lines):
close = FENCE_RE.match(lines[i])
if close and close.group(1)[0] == marker and not close.group(2).strip():
closed = True
i += 1
break
code.append(lines[i])
i += 1
if not closed:
raise line_error(article, line_no, "fenced code block has no closing fence")
if info == "visual":
names = [c.strip() for c in code if c.strip()]
if len(names) != 1:
raise line_error(article, line_no, "a visual block names exactly one figure")
html, _ = load_visual(article, names[0], line_no, context)
rendered.append(_indent_visual(html))
continue
body = esc_text("\n".join(code))
if code:
body += "\n"
rendered.append(f' <pre class="code"><code class="language-{info}">'
f"{body}</code></pre>")
continue
heading = HEADING_RE.match(raw)
if heading:
level = len(heading.group(1))
text = heading.group(2).rstrip("#").rstrip()
if level == 1:
raise line_error(article, line_no,
"Markdown body must not contain an H1; use frontmatter h1")
if level not in (2, 3):
raise line_error(article, line_no,
"only ATX h2 and h3 headings are supported")
if level > previous_heading + 1:
raise line_error(article, line_no,
f"heading level jumps from h{previous_heading} to h{level}")
if level == 2 and text.strip().lower() == "faq":
raise line_error(article, line_no,
"FAQ is generated from frontmatter; remove the body FAQ heading")
rendered.append(f" <h{level}>"
f"{_inline(text, article, line_no, context.markdown_href)}"
f"</h{level}>")
previous_heading = level
i += 1
continue
if (i + 1 < len(lines) and "|" in raw
and _is_table_separator(lines[i + 1])):
header = _split_table_row(raw)
separator = _split_table_row(lines[i + 1])
if len(header) != len(separator):
raise line_error(article, line_no, "table header and separator widths differ")
i += 2
rows: list[list[str]] = []
while i < len(lines) and lines[i].strip() and "|" in lines[i]:
row = _split_table_row(lines[i])
if len(row) != len(header):
raise line_error(article, article.body_start_line + i,
"table row width differs from the header")
rows.append(row)
i += 1
rendered.append(" <table>")
rendered.append(" <thead><tr>" + "".join(
f"<th>{_inline(cell, article, line_no, context.markdown_href)}</th>"
for cell in header) + "</tr></thead>")
if rows:
rendered.append(" <tbody>")
for row_index, row in enumerate(rows):
row_line = line_no + 2 + row_index
rendered.append(" <tr>" + "".join(
f"<td>{_inline(cell, article, row_line, context.markdown_href)}</td>"
for cell in row) + "</tr>")
rendered.append(" </tbody>")
rendered.append(" </table>")
continue
unordered = UL_RE.match(raw)
ordered = OL_RE.match(raw)
if unordered or ordered:
ordered_mode = ordered is not None
items: list[tuple[int, str]] = []
while i < len(lines):
match = OL_RE.match(lines[i]) if ordered_mode else UL_RE.match(lines[i])
if not match:
break
items.append((article.body_start_line + i, match.group(2)))
i += 1
tag = "ol" if ordered_mode else "ul"
rendered.append(f" <{tag}>")
for item_line, item in items:
rendered.append(f" <li>{_inline(item, article, item_line, context.markdown_href)}</li>")
rendered.append(f" </{tag}>")
continue
if raw.lstrip().startswith(">"):
quote_lines: list[tuple[int, str]] = []
while i < len(lines) and lines[i].lstrip().startswith(">"):
quote_lines.append((article.body_start_line + i,
re.sub(r"^ {0,3}>\s?", "", lines[i])))
i += 1
text = " ".join(value for _, value in quote_lines)
rendered.append(" <blockquote><p>"
+ _inline(text, article, quote_lines[0][0], context.markdown_href)
+ "</p></blockquote>")
continue
paragraph: list[tuple[int, str]] = []
while i < len(lines) and lines[i].strip() and not _block_start(lines, i):
paragraph.append((article.body_start_line + i, lines[i].strip()))
i += 1
if not paragraph:
raise line_error(article, line_no, "unsupported Markdown syntax")
text = " ".join(value for _, value in paragraph)
rendered.append(" <p>" + _inline(text, article, paragraph[0][0],
context.markdown_href) + "</p>")
return rendered
def render_markdown(article: article_data.Article, context: BuildContext) -> str:
return "\n".join(render_markdown_blocks(article, context))
EVIDENCE_LINE_RE = re.compile(r"^(.*):(\d+)$")
def _evidence_repo_path(source: str) -> str:
"""Return the filesystem portion of a repo evidence source."""
match = EVIDENCE_LINE_RE.fullmatch(source)
return match.group(1) if match else source
def _metadata_link(source: str, article: article_data.Article,
context: BuildContext) -> str | None:
"""Resolve an evidence source, or return ``None`` for repo provenance.
Evidence sources are not all site links. A repo-relative source is
validated against the repository and deliberately has no href so callers
render it as provenance text rather than an ordinary document link.
"""
value = source.strip()
repo_path = _evidence_repo_path(value)
parsed = urllib.parse.urlsplit(value)
if parsed.scheme in ("http", "https") and parsed.netloc:
return value
if parsed.scheme or value.startswith("//"):
# urlsplit treats ``ROADMAP.md:80`` as a URI with scheme
# ``ROADMAP.md``. Once the trailing line number is removed, a path
# with no scheme is still a valid repo-relative evidence source.
path_parsed = urllib.parse.urlsplit(repo_path)
if not (repo_path != value and not path_parsed.scheme):
raise line_error(article, article.body_start_line,
f"unsupported evidence URL {source!r}")
if value.startswith("//"):
raise line_error(article, article.body_start_line,
f"unsupported evidence URL {source!r}")
if value.startswith("/"):
return context.resolve(value, article.category, article,
article.body_start_line).href
if repo_path and (context.repo / repo_path).exists():
return None
# Keep the existing dangling-target wording for missing relative sources,
# but do not let a relative value resolve as a site article by accident.
try:
path, _query, _fragment = context._normalise_path(repo_path, article.category)
except BuildError as exc:
raise line_error(article, article.body_start_line, str(exc)) from exc
raise line_error(article, article.body_start_line,
f"dangling internal target {source!r} (resolved as {path})")
GITHUB_BLOB = "https://github.com/xerj-org/xerj/blob/main/"
CAPTURE_LABEL = "the captured run documented on this page"
def key_fact_href(article: article_data.Article, context: BuildContext) -> str | None:
"""Where a reader checks the key number: the URL, the site page, or the
file on GitHub (a repo path gets a real link here, unlike evidence rows,
because an answer engine needs a URL it can cite). None for "capture":
the number comes from this page's own run, so there is nothing to link."""
source = (article.key_fact_source or "").strip()
if source == article_data.CAPTURE_SOURCE:
return None
if source.startswith("https://"):
return source
if source.startswith("/"):
return context.resolve(source, article.category, article,
article.body_start_line).href
return GITHUB_BLOB + _evidence_repo_path(source)
def key_fact_label(article: article_data.Article) -> str:
source = (article.key_fact_source or "").strip()
if source.startswith("https://"):
parsed = urllib.parse.urlsplit(source)
return parsed.netloc + (parsed.path.rstrip("/") or "")
return source
_INTENT_TOKEN_RE = re.compile(r"[a-z0-9]+")
_INTENT_STOP = frozenset("a an and are can do does for from how i in is it my of on or "
"the to what when which with without you your xerj".split())
def _intent_tokens(article: article_data.Article) -> set[str]:
text = " ".join([article.title, article.question, *article.queries]).lower()
return {t for t in _INTENT_TOKEN_RE.findall(text) if t not in _INTENT_STOP and len(t) > 2}
def intent_siblings(article: article_data.Article, context: BuildContext,
limit: int = 3) -> list[article_data.Article]:
"""Up to ``limit`` other articles in the same intent group, not already in
links_out, most-overlapping first. Deterministic: ties break on slug."""
if not article.intent_group:
return []
linked: set[str] = set()
for raw in article.links_out:
target = context.resolve_article_target(raw, article, article.body_start_line)
linked.add(target.rel)
mine = _intent_tokens(article)
pool = [a for a in context.articles
if a is not article and a.intent_group == article.intent_group
and a.output_rel not in linked and not a.noindex]
pool.sort(key=lambda a: (-len(mine & _intent_tokens(a)), a.category, a.slug))
return pool[:limit]
def render_answer_card(article: article_data.Article, context: BuildContext,
modified: str) -> str:
"""The answer-first block under the H1: a 40-60 word answer that restates
the question, then one dated, sourced number."""
if not article.answer and not article.key_fact and not article.last_checked:
return ""
out = [' <div class="prompt-card answer-first" id="short-answer">',
f' <span class="prompt-label">SHORT ANSWER · UPDATED {esc_text(modified)}</span>']
if article.answer:
out.append(" <p>" + _inline(article.answer, article, article.body_start_line,
context.markdown_href) + "</p>")
if article.key_fact:
href = key_fact_href(article, context)
out.append(' <p class="key-fact"><strong>Key number:</strong> '
+ _inline(article.key_fact, article, article.body_start_line,
context.markdown_href)
+ f' <span class="mute">(as of {esc_text(article.key_fact_date or "")} · source: '
+ (CAPTURE_LABEL if href is None else
f'<a href="{esc_attr(href)}">{esc_text(key_fact_label(article))}</a>')
+ ')</span></p>')
if article.last_checked:
out.append(f' <p class="mute">Last checked: {esc_text(article.last_checked)}. Every '
"third-party fact on this page was re-read from the tool's own docs or "
"repository on that date.</p>")
out.append(" </div>")
return "\n".join(out)
def markdown_rel(article: article_data.Article) -> str:
return f"{article.category}/{article.slug}.md"
def markdown_path(article: article_data.Article) -> str:
return "/" + markdown_rel(article)
def markdown_url(article: article_data.Article) -> str:
return urlmap.SITE_ORIGIN + markdown_path(article)
def article_modified(article: article_data.Article, context: BuildContext) -> str:
return article.updated or context.dates.last_modified(article.path)
def render_agent_prompt(article: article_data.Article) -> str:
if not article.agent_prompt:
return ""
prompt = article.agent_prompt
return (" <div class=\"prompt-card\">\n"
" <span class=\"prompt-label\">PASTE THIS TO YOUR AI AGENT — IT DOES THE REST</span>\n"
f" <code>{esc_text(prompt)}</code>\n"
f" <button type=\"button\" class=\"prompt-copy\" data-copy=\"{esc_attr(prompt)}\">"
"COPY THE PROMPT</button>\n"
" </div>")
def render_commands(article: article_data.Article) -> str:
if not article.commands:
return ""
out: list[str] = []
for index, item in enumerate(article.commands, start=1):
command = item["cmd"]
note = item["note"]
step_id = f' id="step-{index}"' if "howto" in article.structured_data else ""
out.append(f" <div class=\"prompt-card\"{step_id}>")
out.append(" <span class=\"prompt-label\">RUN THIS XERJ COMMAND</span>")
out.append(f" <code>{esc_text(command)}</code>")
out.append(f" <button type=\"button\" class=\"prompt-copy\" data-copy=\"{esc_attr(command)}\">"
"COPY COMMAND</button>")
out.append(f" <span class=\"hero-proof\">{esc_text(note)}</span>")
out.append(" </div>")
return "\n".join(out)
def _split_tldr(body: str) -> tuple[str, str]:
lines = body.splitlines()
start = next((i for i, line in enumerate(lines) if line.strip()), 0)
end = start
while end < len(lines) and lines[end].strip():
end += 1
return "\n".join(lines[start:end]).strip(), "\n".join(lines[end:]).strip()
def _md_header_value(value: str) -> str:
return json.dumps(value, ensure_ascii=False)
def render_markdown_twin(article: article_data.Article, context: BuildContext) -> str:
"""Emit the agent-readable Markdown twin without HTML presentation chrome."""
modified = article_modified(article, context)
tldr, body = _split_tldr(article.body)
source = article.path.relative_to(context.repo).as_posix()
out = [
f"{GENERATOR_PREFIX}{article.category}/{article.slug}.md instead -->",
"---",
f"title: {_md_header_value(article.title)}",
f"canonical: {_md_header_value(urlmap.deployed_url(article.output_rel))}",
f"updated: {_md_header_value(modified)}",
f"source: {_md_header_value(source)}",
"---",
"",
f"# {article.h1}",
"",
]
if article.answer:
out += [f"**Short answer** (updated {modified}): {article.answer}", ""]
if article.key_fact:
href = key_fact_href(article, context)
where = CAPTURE_LABEL if href is None else f"[{key_fact_label(article)}]({href})"
out += [f"**Key number:** {article.key_fact} (as of {article.key_fact_date}; "
f"source: {where})", ""]
if article.last_checked:
out += [f"Last checked: {article.last_checked}. Every third-party fact on this page "
"was re-read from the tool's own docs or repository on that date.", ""]
out += [tldr]
if article.agent_prompt:
out += ["", "## Agent prompt", "", "```text", article.agent_prompt, "```"]
if article.commands:
out += ["", "## Commands", ""]
for index, item in enumerate(article.commands, start=1):
out += [f"### Command {index}", "", f"Note: {item['note']}", "",
"```sh", item["cmd"], "```", ""]
while out and out[-1] == "":
out.pop()
if body:
out += ["", _twin_visuals(body, article, context)]
out += ["", "## FAQ", ""]
for item in article.faq:
out += [f"### {item['q'].strip()}", "", item["a"].strip(), ""]
if out[-1] == "":
out.pop()
# No evidence block means no provenance section at all. An "## Evidence"
# heading with a placeholder under it is worse than silence: it tells a
# reader the article was sourced and then shows nothing. Sources that ARE
# present still go through _metadata_link and still have to resolve.
if article.evidence:
out += ["", "## Evidence", ""]
for item in article.evidence:
href = _metadata_link(item["source"], article, context)
source = item["source"].strip()
rendered_source = (f"[{source}]({href})" if href is not None
else f"`{source}`")
out.append(f"- {item['claim'].strip()} — {rendered_source}")
out += ["", "## Related", ""]
siblings = intent_siblings(article, context)
if article.links_out or siblings:
for raw in article.links_out:
target = context.resolve_article_target(raw, article, article.body_start_line)
out.append(f"- [{target.label}]({target.href})")
for sibling in siblings:
out.append(f"- [{sibling.h1}]({urlmap.deployed_path(sibling.output_rel)}) "
f"(same intent: {article.intent_group})")
else:
out.append("No related pages are listed.")
return "\n".join(out).rstrip() + "\n"
def render_article_sections(article: article_data.Article, context: BuildContext) -> str:
body_blocks = render_markdown_blocks(article, context)
out: list[str] = []
if body_blocks:
out.append(body_blocks[0])
rest = body_blocks[1:]
# A visual placed directly under the TL;DR is the page's lead figure: it goes
# above the prompt and command cards so the answer, then the picture, come first.
if rest and rest[0].startswith(' <figure class="vz'):
out.append(rest.pop(0))
prompt = render_agent_prompt(article)
if prompt:
out.append(prompt)
commands = render_commands(article)
if commands:
out.append(commands)
out.extend(rest)
out.append(" <h2>FAQ</h2>")
for item in article.faq:
# No grammar rule on the question string. A real user types "The
# indexer died overnight." and matching that phrasing verbatim is the
# whole mechanism this content depends on; a question-word validator
# rejected exactly the strings worth matching. Structure (4-8 entries)
# is still enforced in article_data.py.
question = item["q"].strip()
out.append(f" <h3>{_inline(question, article, article.body_start_line, context.markdown_href)}</h3>")
out.append(" <p>" + _inline(item["a"].strip(), article, article.body_start_line,
context.markdown_href) + "</p>")
# Same rule as the Markdown twin: the whole section is omitted when there
# is no evidence block, so no page ships an empty "Evidence" heading.
if article.evidence:
out.append(" <h2>Evidence</h2>")
out.append(" <ul>")
for item in article.evidence:
href = _metadata_link(item["source"], article, context)
source = item["source"].strip()
if href is None:
source_html = f"<code>{esc_text(source)}</code>"
else:
source_html = f'<a href="{esc_attr(href)}">{esc_text(source)}</a>'
out.append(" <li>" + _inline(item["claim"].strip(), article,
article.body_start_line,
context.markdown_href)
+ " — " + source_html + "</li>")
out.append(" </ul>")
out.append(" <h2>Related</h2>")
siblings = intent_siblings(article, context)
if article.links_out or siblings:
out.append(" <ul>")
for raw in article.links_out:
target = context.resolve_article_target(raw, article, article.body_start_line)
out.append(f' <li><a href="{esc_attr(target.href)}">'
f"{esc_text(target.label)}</a></li>")
for sibling in siblings:
out.append(f' <li><a href="{urlmap.deployed_path(sibling.output_rel)}">'
f"{esc_text(sibling.h1)}</a> <span class=\"mute\">· same intent: "
f"{esc_text(article.intent_group or '')}</span></li>")
out.append(" </ul>")
else:
out.append(" <p>No related pages are listed.</p>")
return "\n".join(out)
def primary_nav(rel: str) -> str:
"""The site-wide primary nav; scripts/seo/set_nav.py owns its definition."""
return set_nav.render(rel)
def article_nav(context: BuildContext, active: article_data.Article | None) -> str:
groups = [("ANSWERS", "answers"), ("COMPARISONS", "compare")]
out = ['<nav class="docs-nav" aria-label="Article sections">']
for label, category in groups:
out.append(' <div class="group">')
out.append(f' <span class="label">{label}</span>')
out.append(f' <a href="/{category}/"><span class="eyebrow">00</span>All {label.title()}</a>')
members = sorted((a for a in context.articles if a.category == category),
key=lambda a: (a.cluster.lower(), a.h1.lower()))
for index, member in enumerate(members, start=1):
active_class = " active" if active is member else ""
out.append(f' <a href="{urlmap.deployed_path(member.output_rel)}"'
f' class="{active_class.strip()}"><span class="eyebrow">{index:02}</span>'
f"{esc_text(member.h1)}</a>")
out.append(" </div>")
out.append("</nav>")
return "\n".join(out)
def breadcrumb(rel: str, meta: dict[str, Any]) -> str:
parts = [f'<a href="/">HOME</a>']
directories = rel.split("/")[:-1]
if rel.endswith("/index.html"):
directories = directories[:-1]
for i in range(len(directories)):
directory = "/".join(directories[:i + 1])
hub = pagedata.SECTION_HUBS.get(directory)
if hub:
label, path = hub
parts.append(f'<a href="{esc_attr(path)}">{esc_text(label.upper())}</a>')
parts.append(esc_text(str(meta["label"]).upper()))
return '<div class="breadcrumb">' + '<span class="sep">·</span>'.join(parts) + "</div>"
# These pages carry NO release version, anywhere — not in the prose, not in
# the footer brand line. That is deliberate.
#
# A version stamp is a claim that goes stale on its own. This footer was
# hardcoded to `V1.0.0-RC.15`, copied from a page written when that was
# current, and it sat there for eight releases until
# `.github/scripts/landing-constants-guard.sh` failed the build. Deriving it
# from the newest tag fixes the drift but keeps a claim these pages do not
# need to make: an answer page tells a reader to install the latest XERJ, so
# naming a release only invites the question of whether the page still applies
# to it.
#
# The guard is satisfied by absence, not by agreement. It matches
# `XERJ.AI[^<]*V1.0.0-RC.?[0-9]+` and flags only stamps that DISAGREE with the
# newest tag, so a footer without one is never examined. Do not "fix" this by
# adding a version back — that re-creates the drift the guard exists to catch,
# and it would be the only version string on pages whose whole point is that
# they do not depend on one.
def footer(category: str) -> str:
return fix_links.render_footer() + f'''\n<footer class="footer">
<span>XERJ.AI · {esc_text(category.upper())}</span>
<span>READ THE SOURCE</span>
<span><a href="/">HOME</a> · <a href="/product">PRODUCT</a> · <a href="/answers/">ANSWERS</a> · <a href="/playground">DASHBOARDS</a> · <a href="/llms.txt">LLMS.TXT</a></span>
</footer>'''
def _head(title: str, description: str, rel: str, block: str,
alternate_markdown: str | None = None, visuals: bool = False) -> str:
alternate = (f'<link rel="alternate" type="text/markdown" href="{esc_attr(alternate_markdown)}">\n'
if alternate_markdown else "")
return f'''<!DOCTYPE html>
<html lang="en" data-theme="day">
<head>
{seo_head.ga4_snippet()}
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{seo_head.text_esc(title)}</title>
<meta name="description" content="{seo_head.esc(description)}">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Big+Shoulders+Display:wght@400;700;800;900&family=Inter:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;700&display=swap" rel="stylesheet">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' fill='%230b0b0d'/><text x='50' y='68' font-size='60' font-family='Big Shoulders Display,sans-serif' font-weight='900' fill='%23ffc400' text-anchor='middle'>X</text></svg>">
<script src="/theme.js"></script>
<link rel="stylesheet" href="/style.css?v=as2">
{'<link rel="stylesheet" href="/visuals.css?v=2">' + chr(10) if visuals else ''}{alternate}{block}
</head>'''
# Questions longer than this get a smaller H1 on phones (style.css, .h1-long)
# so a verbatim user question still fits in about five lines at 390px.
# Desktop sizing does not change.
H1_LONG_CHARS = 80
def render_article(article: article_data.Article, context: BuildContext) -> str:
rel = article.output_rel
meta = article.head_meta()
modified = article_modified(article, context)
block = seo_head.build_block(rel, article.title, article.description,
article.published, modified, meta)
body = render_article_sections(article, context)
visuals = has_visuals(article)
answer_card = render_answer_card(article, context, modified)
if answer_card:
answer_card += "\n"
marker = f"{GENERATOR_PREFIX}{article.category}/{article.slug}.md instead -->"
source_note = (f"AUTHOR · {article.author} · REVIEWED BY · {article.reviewer} · "
f"PUBLISHED · {article.published} · UPDATED · {modified}")
return f'''{marker}
{_head(article.title, article.description, rel, block, markdown_path(article), visuals)}
<body>
{primary_nav(rel)}
<div class="docs-shell article-shell">
<main class="docs-content">
{breadcrumb(rel, meta)}
<div class="kicker"><span>{esc_text(article.category.upper())} · {esc_text(article.cluster.upper())}</span></div>
<h1{' class="h1-long"' if len(article.h1) > H1_LONG_CHARS else ''}>{_inline(article.h1, article, article.body_start_line, context.markdown_href)}</h1>
<div class="source-note">{esc_text(source_note)}</div>
{answer_card}{body}
</main>
{article_nav(context, article)}
</div>
{footer(article.category)}
{'<script src="/visuals.js?v=2"></script>' + chr(10) if visuals else ''}<script src="/interact.js"></script>
</body>
</html>
'''
def render_hub(category: str, articles: list[article_data.Article],
context: BuildContext) -> str:
rel = f"{category}/index.html"
meta = dict(pagedata.entry(rel))
# A hub's dates come from the articles it lists, never from the hub file
# itself: reading the generated file's own git date makes the output depend
# on the commit that wrote it, so committing a regenerated hub changes the
# value it should have had. Every merge that touches the file on a later
# date then re-stales it and `--check` fails on main (that is #942's red
# gate, and #972's re-red one commit later). Article pages never had the
# problem because they date from their content source.
listed = [a for a in articles if a.category == category]
if listed:
modified = max(article_modified(a, context) for a in listed)
published = min(a.published for a in listed if a.published) or modified
else:
modified = published = dt.datetime.now(dt.timezone.utc).date().isoformat()
block = seo_head.build_block(rel, meta["title"], meta["description"],
published, modified, meta)
groups: dict[str, list[article_data.Article]] = collections.defaultdict(list)
for article in articles:
if article.category == category:
groups[article.cluster].append(article)
content = [
f' <p>{esc_text(meta["description"])}</p>',
]
if groups:
for cluster in sorted(groups, key=str.lower):
content.append(f" <h2>{esc_text(cluster)}</h2>")
content.append(" <ul>")
for article in sorted(groups[cluster], key=lambda a: a.h1.lower()):
content.append(f' <li><a href="{urlmap.deployed_path(article.output_rel)}">'
f"{esc_text(article.h1)}</a> — {esc_text(article.description)}</li>")
content.append(" </ul>")
else:
content.append(" <h2>Articles</h2>")
content.append(" <p>No articles are published in this section yet.</p>")
marker = f"{GENERATOR_PREFIX}{category}/*.md instead -->"
return f'''{marker}
{_head(meta["title"], meta["description"], rel, block)}
<body>
{primary_nav(rel)}
<div class="docs-shell article-shell">
<main class="docs-content">
{breadcrumb(rel, meta)}
<div class="kicker"><span>{esc_text(category.upper())}</span></div>
<h1>{esc_text(meta["label"])}</h1>
{chr(10).join(content)}
</main>
{article_nav(context, None)}
</div>
{footer(category)}
<script src="/interact.js"></script>
</body>
</html>
'''
def discover(repo: pathlib.Path) -> list[article_data.Article]:
articles: list[article_data.Article] = []
for category in ("answers", "compare"):
directory = repo / "content" / category
if not directory.exists():
continue
for path in sorted(directory.glob("*.md")):
articles.append(article_data.load(path, category))
slugs: dict[str, pathlib.Path] = {}
for article in articles:
previous = slugs.get(article.slug)
if previous is not None:
raise BuildError(f"{article.path}: slug {article.slug!r} duplicates {previous}")
slugs[article.slug] = article.path
return articles
def article_index_entry(article: article_data.Article, context: BuildContext) -> dict[str, str]:
return {
"title": article.title,
"question": article.question,
"canonical_url": urlmap.deployed_url(article.output_rel),
"markdown_url": markdown_url(article),
"cluster": article.cluster,
"updated": article_modified(article, context),
"summary": " ".join(article.description.split()),
**({"intent_group": article.intent_group} if article.intent_group else {}),
**({"answer": article.answer} if article.answer else {}),
**({"queries": list(article.queries)} if article.queries else {}),
}
def render_index_json(category: str, articles: list[article_data.Article],
context: BuildContext) -> str:
entries = [article_index_entry(article, context)
for article in sorted(articles, key=lambda a: (a.cluster.lower(), a.title.lower()))
if article.category == category]
payload = {
"generated_by": "scripts/seo/build_articles.py",
"section": category,
"articles": entries,
}
text = json.dumps(payload, ensure_ascii=False, indent=2) + "\n"
try:
json.loads(text)
except json.JSONDecodeError as exc: # pragma: no cover - json.dumps is deterministic
raise BuildError(f"generated {category}/index.json is not valid JSON: {exc}") from exc
return text
def render_llms_section(articles: list[article_data.Article], context: BuildContext) -> str:
"""Render the generated article index in the direct-to-agent llms.txt voice.
One line per article, in the shape the llms.txt spec asks for —
``[name](url): notes`` — with the **Markdown twin as the href**, because the
spec says the links "should point to LLM-friendly content". The previous
form spent three lines per article (title + question + summary, then a bare
``Markdown twin: <url>`` line that was not even a link) and cost ~33 KB for
76 pages, roughly 440 bytes per link against a 155-byte corpus median.
Nothing became unreachable: the canonical HTML page is the same URL without
``.md`` (stated once above the list), and the per-article summary, cluster
and last-modified date stay machine-readable in ``answers/index.json`` and
``compare/index.json``, both linked from the block.
"""
lines = [
LLMS_BEGIN,
"## Answers and comparisons, grouped by what the agent is trying to do",
"",
"Direct answers for you, the AI agent, grouped by intent: the clusters come from a "
"1,000-query study of what agents actually type into search, so find the group that "
"matches your task first. Each line is the link, a tag (`answer` or `compare`), then the "
"question that page answers. "
"**Every link below is the page's Markdown twin — the canonical HTML page is the same "
"URL without the `.md`.** The twin is deliberately not in the sitemap because it is the "
"same document. Titles, questions, short answers, target queries, intents and "
"last-modified dates for every page, as JSON: "
"[answers/index.json](https://xerj.org/answers/index.json) · "
"[compare/index.json](https://xerj.org/compare/index.json). Human hubs: "
"[/answers](https://xerj.org/answers) · [/compare](https://xerj.org/compare).",
"",
]
if not articles:
lines.append("- No pages are published in this section yet.")
lines.append("")
groups: dict[str, list[article_data.Article]] = collections.defaultdict(list)
for article in articles:
groups[article.intent_group or article.cluster].append(article)
# Biggest intent first (that is where most agents land), then by name.
for group in sorted(groups, key=lambda g: (-len(groups[g]), g.lower())):
lines.append(f"### {group}")
lines.append("")
members = sorted(groups[group],
key=lambda a: (a.category != "compare", a.title.lower()))
for article in members:
entry = article_index_entry(article, context)
# The "canonical = twin minus .md" rule is stated once above the list,
# so assert the relationship the prose promises still holds.
if entry["markdown_url"] != entry["canonical_url"] + ".md":
raise BuildError(
f"{article.path}: Markdown twin {entry['markdown_url']} is not "
f"{entry['canonical_url']}.md, but llms.txt states that rule once")
tag = "compare" if article.category == "compare" else "answer"
lines.append(f"- [{entry['title']}]({entry['markdown_url']}) `{tag}`: "
f"{entry['question']}")
lines.append("")
lines.append(LLMS_END)
return "\n".join(lines).rstrip() + "\n"
def splice_llms(text: str, section: str) -> str:
"""Replace the generated block, or insert it before machine artifacts."""
begin = text.find(LLMS_BEGIN)
end = text.find(LLMS_END)
if (begin < 0) != (end < 0):
raise BuildError("landing/llms.txt has only one article-tree generated marker")
if begin >= 0:
if end < begin:
raise BuildError("landing/llms.txt article-tree markers are out of order")
end += len(LLMS_END)
prefix = text[:begin].rstrip("\n")
suffix = text[end:].lstrip("\n")
return prefix + "\n\n" + section.rstrip("\n") + ("\n\n" + suffix if suffix else "\n")
# Fallback only — the markers above are the normal path. Listed in the
# order the article index should appear before.
for anchor in ("\n## Use cases and recipes", "\n## Machine-readable artifacts",
"\n## Optional"):
position = text.find(anchor)
if position >= 0:
break
else:
raise BuildError("landing/llms.txt has no article-index insertion point")
prefix = text[:position].rstrip("\n")
suffix = text[position:].lstrip("\n")
return prefix + "\n\n" + section.rstrip("\n") + "\n\n" + suffix
def run_ste(articles: list[article_data.Article], allow_errors: bool) -> dict[str, Any]:
result: dict[str, Any] = {"files": len(articles), "errors": 0, "warnings": 0, "info": 0}
script = SCRIPT_DIR / "ste_check.py"
for article in articles:
proc = subprocess.run(
[sys.executable, str(script), "--json", str(article.path)],
capture_output=True, text=True,
)
if proc.returncode == 2:
raise BuildError(proc.stderr.strip() or f"ste_check.py failed for {article.path}")
try:
report = json.loads(proc.stdout)
except json.JSONDecodeError as exc:
raise BuildError(f"ste_check.py returned invalid JSON for {article.path}: {exc}") from exc
totals = report.get("totals", {})
count_keys = {"ERROR": "errors", "WARN": "warnings", "INFO": "info"}
for key, result_key in count_keys.items():
result[result_key] += int(totals.get(key, 0))
findings = [f for f in report.get("findings", []) if f.get("severity") == "ERROR"]
if findings:
result["errors"] += len(findings)
for finding in findings:
message = (f"{finding.get('path', article.path)}:{finding.get('line', 1)}:"
f"{finding.get('col', 1)}: ERROR {finding.get('rule', '')} "
f"{finding.get('message', '')}")
print("STE " + message, file=sys.stderr)
# Reported, never raised. ASD-STE100 is a house style for technical
# writing, and its rules fight the one thing these pages exist to
# do: match the sentence a person actually types. An 84-prompt
# evaluation returned 0 brand mentions across two engines, and the
# queries people fired were statements — "The indexer died
# overnight", "My codebase indexer says indexed but I don't see my
# code". Blocking a build over the grammar of a quoted user
# question optimises the wrong thing.
#
# `ste_check.py` stays in the repo and stays useful; run it while
# drafting. It is advisory here and advisory in CI, which is what
# README §7 already says — until 2026-08-22 this line quietly
# contradicted it, so STE was a hard gate through the back door.
return result
def assert_reachability(context: BuildContext, categories: set[str]) -> None:
footer_paths = {href for _heading, links in fix_links.FOOTER_MAP for href, _ in links}
for category in categories:
hub_path = f"/{category}/"
if hub_path not in footer_paths:
raise BuildError(f"{category}/index.html is not linked from the site footer")
hub_rel = f"{category}/index.html"
category_articles = [a for a in context.articles if a.category == category]
if hub_rel not in context.known:
raise BuildError(f"missing generated hub target {hub_rel}")
for article in category_articles:
if article.output_rel not in context.known:
raise BuildError(f"missing generated article target {article.output_rel}")
for article in context.articles:
for raw in article.links_out:
context.resolve_article_target(raw, article, article.body_start_line)
def select_articles(articles: list[article_data.Article], only: str | None) -> list[article_data.Article]:
if not only:
return articles
value = only.strip().strip("/")
if value.endswith(".md"):
value = value[:-3]
if value.endswith(".html"):
value = value[:-5]
if value.startswith("answers/") or value.startswith("compare/"):
value = value.split("/", 1)[1]
selected = [article for article in articles if article.slug == value]
if not selected:
raise BuildError(f"--only {only!r} did not match an article slug")
if len(selected) > 1:
raise BuildError(f"--only {only!r} matches more than one article; include its section")
return selected
def build_expected(repo: pathlib.Path, landing: pathlib.Path, all_articles: list[article_data.Article],
selected: list[article_data.Article], only: str | None) -> tuple[dict[str, str], dict[str, Any]]:
dates = urlmap.DateSource(repo)
context = BuildContext(repo, landing, all_articles, dates)
categories = {a.category for a in selected} if only else {"answers", "compare"}
assert_reachability(context, categories)
expected: dict[str, str] = {}
for article in selected:
expected[article.output_rel] = render_article(article, context)
expected[markdown_rel(article)] = render_markdown_twin(article, context)
for category in sorted(categories):
expected[f"{category}/index.html"] = render_hub(category, all_articles, context)
if not only:
for category in ("answers", "compare"):
expected[f"{category}/index.json"] = render_index_json(category, all_articles, context)
llms = landing / "llms.txt"
current = llms.read_text(encoding="utf-8") if llms.exists() else ""
expected["llms.txt"] = splice_llms(current, render_llms_section(all_articles, context))
assert_generated_contract(expected, landing, selected, only)
return expected, {"dates": dates, "context": context}
def assert_generated_contract(expected: dict[str, str], landing: pathlib.Path,
selected: list[article_data.Article], only: str | None) -> None:
for article in selected:
html_output = expected.get(article.output_rel, "")
alternate = (f'<link rel="alternate" type="text/markdown" href="{markdown_path(article)}">')
if alternate not in html_output:
raise BuildError(f"{article.output_rel}: missing text/markdown alternate link")
if re.search(r"<a\b[^>]*href=[\"'](?![A-Za-z][A-Za-z0-9+.-]*:|//)[^\"']+\.md(?:[?#][^\"']*)?[\"']",
html_output, re.I):
raise BuildError(f"{article.output_rel}: Markdown twin must not be an ordinary <a> link")
md_path = landing / markdown_rel(article)
if not md_path.is_file() and markdown_rel(article) not in expected:
raise BuildError(f"{article.output_rel}: Markdown twin {md_path} is missing")
if not only:
for rel in ("answers/index.json", "compare/index.json"):
try:
json.loads(expected[rel])
except (KeyError, json.JSONDecodeError) as exc:
raise BuildError(f"{rel}: generated JSON does not parse: {exc}") from exc
def _write_or_compare(landing: pathlib.Path, expected: dict[str, str], write: bool,
only: str | None) -> list[str]:
changed: list[str] = []
if write:
for rel, output in sorted(expected.items()):
path = landing / rel
if path.exists():
current = path.read_text(encoding="utf-8")
if rel.endswith((".html", ".md")) and not GENERATOR_RE.search(current):
raise BuildError(f"{path}: exists but is not generated by build_articles.py")
if not path.exists() or path.read_text(encoding="utf-8") != output:
changed.append(rel)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(output, encoding="utf-8")
return changed
with tempfile.TemporaryDirectory(prefix="xerj-articles-") as temp_name:
temp = pathlib.Path(temp_name)
for rel, output in expected.items():
path = temp / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(output, encoding="utf-8")
current_path = landing / rel
current = current_path.read_text(encoding="utf-8") if current_path.exists() else ""
if current != output:
changed.append(rel)
if not only:
for category in ("answers", "compare"):
directory = landing / category
if not directory.exists():
continue
expected_names = set(expected)
for path in sorted(directory.glob("*.html")):
rel = path.relative_to(landing).as_posix()
if rel not in expected_names and GENERATOR_RE.search(path.read_text(encoding="utf-8")):
raise BuildError(f"{path}: generated output has no matching Markdown source")
for path in sorted(directory.glob("*.md")):
rel = path.relative_to(landing).as_posix()
if rel not in expected_names and GENERATOR_RE.search(path.read_text(encoding="utf-8")):
raise BuildError(f"{path}: generated Markdown twin has no matching source")
return changed
def run(argv: list[str] | None = None) -> tuple[int, dict[str, Any]]:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
mode = ap.add_mutually_exclusive_group(required=True)
mode.add_argument("--write", action="store_true", help="write generated HTML")
mode.add_argument("--check", action="store_true",
help="regenerate in a temporary directory and compare committed HTML")
ap.add_argument("--only", metavar="SLUG", help="build one article slug and its section hub")
ap.add_argument("--allow-ste-errors", action="store_true",
help="report, but do not block on STE ERROR-tier findings")
ap.add_argument("--json", action="store_true", help="emit a machine-readable report")
ap.add_argument("--root", default=None, help="repository root")
args = ap.parse_args(argv)
repo = pathlib.Path(args.root).resolve() if args.root else urlmap.repo_root()
landing = repo / "landing"
try:
all_articles = discover(repo)
selected = select_articles(all_articles, args.only)
ste = run_ste(selected, args.allow_ste_errors)
expected, state = build_expected(repo, landing, all_articles, selected, args.only)
changed = _write_or_compare(landing, expected, args.write, args.only)
report = {
"status": "ok" if not changed or args.write else "stale",
"mode": "write" if args.write else "check",
"only": args.only,
"articles": len(selected),
"all_sources": len(all_articles),
"outputs": sorted(expected),
"changed": sorted(changed),
"ste": ste,
"git_dates": state["dates"].git_ok,
}
if args.json:
print(json.dumps(report, indent=2, ensure_ascii=False))
elif args.check:
if changed:
print(f"FAIL {len(changed)} generated file(s) differ:")
for rel in changed:
print(f" {rel}")
else:
print(f"ok build_articles.py output is current ({len(expected)} files; "
f"{len(selected)} selected article(s))")
else:
print(f"wrote {len(changed)} changed generated file(s) ({len(expected)} outputs; "
f"{len(selected)} selected article(s))")
return (1 if changed and args.check else 0), report
except (BuildError, article_data.FrontmatterError, OSError) as exc:
report = {"status": "error", "error": str(exc)}
if args.json:
print(json.dumps(report, indent=2, ensure_ascii=False))
else:
print(f"build_articles: {exc}", file=sys.stderr)
return 1, report
if __name__ == "__main__":
raise SystemExit(run()[0])