A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
docs: build llms.txt locally as a compact index instead of crawling the site (#10383)
vincent d warmerdam7482361ea332
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.ymlindex 8e42209c46..ad3dd12a35 100644--- a/.github/workflows/docs.yml+++ b/.github/workflows/docs.yml@@ -71,12 +71,15 @@ jobs:uv run --group docs python docs/scripts/patch_vercel_config.py .vercel/output/config.json- name: 📚 llms.txt- # This is technically off by 1 commit, but generating the llms.txt file is easiest from a- # deployed site.- # We also ignore txt and xml files- run: npx sitefetch https://docs.marimo.io -o ${{ env.OUTPUT_FILE }} --match '!**/*.{txt,xml}'- env:- OUTPUT_FILE: .vercel/output/static/llms.txt+ # Generated locally from the build artifact (the .md files produced by+ # the step above) and the mkdocs nav — no live-site crawl, so it's+ # always in sync with this commit. It's a compact index of links to the+ # per-page .md files, not a full-content dump.+ run: |+ uv run --group docs python docs/scripts/generate_llms_txt.py \+ --input-dir .vercel/output/static \+ --base-url https://docs.marimo.io \+ --output-index .vercel/output/static/llms.txt- name: 📚 CLAUDE.mdrun: cp docs/_static/CLAUDE.md .vercel/output/static/CLAUDE.mddiff --git a/Makefile b/Makefileindex 1d4bf637e7..e68111f040 100644--- a/Makefile+++ b/Makefile@@ -190,6 +190,15 @@ docs:docs-serve:uv run --group docs mkdocs serve --clean+.PHONY: docs-llms+# 📚 Build docs + generate per-page .md files and the llms.txt index into site/+docs-llms: docs+ uv run --group docs python docs/scripts/html_to_markdown.py \+ --input-dir site --base-url https://docs.marimo.io+ uv run --group docs python docs/scripts/generate_llms_txt.py \+ --input-dir site --base-url https://docs.marimo.io \+ --output-index site/llms.txt+.PHONY: storybook# 🧩 Start Storybook for UI developmentstorybook:diff --git a/docs/scripts/generate_llms_txt.py b/docs/scripts/generate_llms_txt.pynew file mode 100644index 0000000000..c1702ac0ba--- /dev/null+++ b/docs/scripts/generate_llms_txt.py@@ -0,0 +1,283 @@+"""Generate an `llms.txt` index from the built docs.++Reads structure and page titles from the mkdocs `nav`, and page content from+the per-page markdown emitted by `html_to_markdown.py` (an `index.md` written+beside every `index.html`), then writes a compact [llmstxt.org](https://llmstxt.org)+index: a title, a blurb, and one section per top-level nav group, each a bullet+list linking to the `.md` version of every page.++Run this after `html_to_markdown.py` so the `.md` files exist:++ python docs/scripts/generate_llms_txt.py \\+ --input-dir .vercel/output/static \\+ --base-url https://docs.marimo.io \\+ --output-index .vercel/output/static/llms.txt+"""++# /// script+# requires-python = ">=3.10"+# dependencies = ["mkdocs>=1.6.1"]+# ///++import argparse+import re+from pathlib import Path++# mkdocs' loader understands the custom tags (!ENV, !relative) in mkdocs.yml;+# a plain yaml.safe_load would choke on them. mkdocs is always installed here+# (this script only runs under `uv run --group docs`).+from mkdocs.utils import yaml_load+++def load_yaml(path: Path) -> dict:+ with open(path, encoding="utf-8") as f:+ return yaml_load(f)+++# Cap each page's one-line description so the index stays a compact, scannable+# map; anything longer is truncated with an ellipsis (see extract_description).+MAX_DESCRIPTION_LEN = 100+++class Leaf:+ """A single documentation page reachable from the nav."""++ def __init__(self, title: str | None, path: str):+ self.title = title+ self.path = path++ @property+ def is_external(self) -> bool:+ return self.path.startswith(("http://", "https://"))+++def walk(entry: object) -> list[Leaf]:+ """Flatten a nav subtree into an ordered list of leaves."""+ leaves: list[Leaf] = []+ if isinstance(entry, str):+ leaves.append(Leaf(None, entry))+ elif isinstance(entry, dict):+ for title, value in entry.items():+ if isinstance(value, str):+ leaves.append(Leaf(title, value))+ elif isinstance(value, list):+ for sub in value:+ leaves.extend(walk(sub))+ return leaves+++def parse_nav(nav: list) -> tuple[list[Leaf], list[tuple[str, list[Leaf]]]]:+ """Split the top-level nav into a prelude and titled sections.++ Returns `(prelude, sections)` where `prelude` is the top-level pages that+ aren't grouped under a section, and `sections` is `(title, leaves)` pairs.+ """+ prelude: list[Leaf] = []+ sections: list[tuple[str, list[Leaf]]] = []+ for entry in nav:+ if isinstance(entry, str):+ prelude.append(Leaf(None, entry))+ elif isinstance(entry, dict):+ for title, value in entry.items():+ if isinstance(value, str):+ prelude.append(Leaf(title, value))+ elif isinstance(value, list):+ leaves = [leaf for sub in value for leaf in walk(sub)]+ sections.append((title, leaves))+ return prelude, sections+++def resolve(path: str, input_dir: Path, base_url: str) -> tuple[Path, str]:+ """Map a nav source path to its built `.md` file and public `.md` URL.++ mkdocs (with `use_directory_urls`) builds `foo/bar.md` to `foo/bar/index.html`+ and `foo/index.md` to `foo/index.html`; `html_to_markdown.py` writes an+ `index.md` beside each. We link to the pretty `.md` URL, e.g.+ `guides/reactivity.md` -> `<base>/guides/reactivity.md`.+ """+ rel = path.removesuffix(".md")+ if rel == "index" or rel.endswith("/index"):+ disk = input_dir / f"{rel}.md"+ route = "" if rel == "index" else rel[: -len("/index")]+ else:+ disk = input_dir / rel / "index.md"+ route = rel+ url = f"{base_url}/{route}.md" if route else f"{base_url}/index.md"+ return disk, url+++def strip_markdown(text: str) -> str:+ """Reduce inline markdown to plain-ish text for a one-line description."""+ text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", text) # images+ text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text) # links -> text+ text = re.sub(r"[`*_]", "", text) # emphasis / code marks+ text = text.replace("¶", "") # stray pilcrows+ return re.sub(r"\s+", " ", text).strip()+++def extract_h1(md: str) -> str | None:+ for line in md.splitlines():+ s = line.strip()+ if s.startswith("# "):+ return strip_markdown(s[2:])+ return None+++def extract_description(md: str) -> str:+ """First prose paragraph after the H1, collapsed to a single line.++ Skips fenced code blocks (e.g. mkdocstrings signatures), headings, and+ other block markers so the description is real summary prose.+ """+ para: list[str] = []+ in_fence = False+ for line in md.splitlines():+ s = line.strip()+ if s.startswith("```"):+ in_fence = not in_fence+ if para:+ break+ continue+ if in_fence:+ continue+ if not s:+ if para:+ break+ continue+ # Skip source comment, headings, block markers, and the mkdocstrings+ # "Bases: ..." line, until real summary prose starts.+ if s.startswith(("<!--", "#", "|", ">", "-", "*", "!", "[", "Bases:")):+ if para:+ break+ continue+ para.append(s)+ text = strip_markdown(" ".join(para))+ if len(text) > MAX_DESCRIPTION_LEN:+ cut = text[:MAX_DESCRIPTION_LEN]+ if " " in cut:+ cut = cut.rsplit(" ", 1)[0]+ text = f"{cut.rstrip()}…"+ return text+++def bullet(leaf: Leaf, input_dir: Path, base_url: str) -> str | None:+ """Render one index bullet, or None if the page has no built markdown."""+ if leaf.is_external:+ title = leaf.title or leaf.path+ return f"- [{title}]({leaf.path})"++ disk, url = resolve(leaf.path, input_dir, base_url)+ if not disk.exists():+ print(+ f" warning: no built markdown for nav entry {leaf.path!r}, skipping"+ )+ return None++ md = disk.read_text(encoding="utf-8")+ title = (+ leaf.title or extract_h1(md) or Path(leaf.path).stem.replace("_", " ")+ )+ description = extract_description(md)+ return (+ f"- [{title}]({url}): {description}"+ if description+ else f"- [{title}]({url})"+ )+++def render_index(+ prelude: list[Leaf],+ sections: list[tuple[str, list[Leaf]]],+ input_dir: Path,+ base_url: str,+ site_name: str,+ site_description: str,+) -> str:+ lines = [f"# {site_name}"]+ if site_description:+ lines.append(f"> {site_description}")+ lines.append("")++ for leaf in prelude:+ line = bullet(leaf, input_dir, base_url)+ if line:+ lines.append(line)+ if prelude:+ lines.append("")++ for title, leaves in sections:+ rendered = [+ b+ for b in (bullet(leaf, input_dir, base_url) for leaf in leaves)+ if b+ ]+ if not rendered:+ continue+ lines.append(f"## {title}")+ lines.extend(rendered)+ lines.append("")++ return "\n".join(lines).rstrip() + "\n"+++def main() -> None:+ parser = argparse.ArgumentParser(+ description="Generate an llms.txt index from the built docs"+ )+ parser.add_argument(+ "--input-dir",+ type=Path,+ required=True,+ help="Path to the built site directory (containing the generated .md files)",+ )+ parser.add_argument(+ "--base-url",+ type=str,+ default="https://docs.marimo.io",+ help="Base URL for page links",+ )+ parser.add_argument(+ "--config",+ type=Path,+ default=Path("mkdocs.yml"),+ help="Path to mkdocs.yml (for nav, site_name, site_description)",+ )+ parser.add_argument(+ "--output-index",+ type=Path,+ required=True,+ help="Where to write llms.txt",+ )+ args = parser.parse_args()++ input_dir: Path = args.input_dir+ base_url: str = args.base_url.rstrip("/")++ if not input_dir.exists():+ raise SystemExit(f"Error: input directory {input_dir} does not exist")+ if not args.config.exists():+ raise SystemExit(f"Error: config {args.config} does not exist")++ config = load_yaml(args.config)+ nav = config.get("nav")+ if not nav:+ raise SystemExit(f"Error: no `nav` found in {args.config}")++ prelude, sections = parse_nav(nav)++ index = render_index(+ prelude,+ sections,+ input_dir,+ base_url,+ site_name=config.get("site_name", "marimo"),+ site_description=config.get("site_description", ""),+ )+ args.output_index.parent.mkdir(parents=True, exist_ok=True)+ args.output_index.write_text(index, encoding="utf-8")+ n_index = index.count("\n- ")+ print(f"Wrote {args.output_index} ({n_index} links, {len(index)} bytes)")+++if __name__ == "__main__":+ main()