A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
docs: Add markdown support for documentation with edge middleware and conversion scripts (#8796)
Myles Scolnick2635b5e7cf01
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.ymlindex 216c3059a4..f05c72bdc3 100644--- a/.github/workflows/docs.yml+++ b/.github/workflows/docs.yml@@ -42,6 +42,19 @@ jobs:VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}+ - name: 📝 Generate markdown versions of docs+ run: |+ uv run --group docs python docs/scripts/html_to_markdown.py \+ --input-dir .vercel/output/static \+ --base-url https://docs.marimo.io++ - name: 🔀 Add edge middleware for markdown content negotiation+ run: |+ mkdir -p .vercel/output/functions/_middleware.func+ cp docs/scripts/middleware.js .vercel/output/functions/_middleware.func/index.js+ echo '{"runtime":"edge","entrypoint":"index.js"}' > .vercel/output/functions/_middleware.func/.vc-config.json+ uv run --group docs python docs/scripts/patch_vercel_config.py .vercel/output/config.json+- name: 📚 llms.txt# This is technically off by 1 commit, but generating the llms.txt file is easiest from a# deployed site.diff --git a/docs/scripts/html_to_markdown.py b/docs/scripts/html_to_markdown.pynew file mode 100644index 0000000000..6bb8e5e8f1--- /dev/null+++ b/docs/scripts/html_to_markdown.py@@ -0,0 +1,438 @@+"""Convert mkdocs-built HTML pages to clean markdown for AI agent consumption.++Walks the build output directory, extracts article content from each page,+and writes a markdown version alongside each index.html.+"""++from __future__ import annotations++import argparse+import os+import re+from pathlib import Path+from urllib.parse import urljoin++from bs4 import BeautifulSoup, Tag+from markdownify import MarkdownConverter+++class DocsMarkdownConverter(MarkdownConverter):+ """Custom markdownify converter for mkdocs-material HTML."""++ def __init__(self, base_url: str, page_path: str, **kwargs):+ self.base_url = base_url.rstrip("/")+ self.page_path = page_path+ super().__init__(**kwargs)++ def convert_pre(self, el: Tag, text: str, **kwargs) -> str:+ """Convert <pre> code blocks by stripping Pygments spans."""+ code_el = el.find("code")+ if code_el:+ # Extract raw text, stripping all Pygments <span> tags+ raw_code = code_el.get_text()+ # Try to detect language from class+ lang = ""+ classes = list(el.get("class", [])) + list(+ code_el.get("class", [])+ )+ for cls in classes:+ if isinstance(cls, str) and cls.startswith("language-"):+ lang = cls.replace("language-", "")+ break+ # Also check parent div for language class+ parent = el.parent+ if not lang and parent and parent.name == "div":+ for cls in parent.get("class", []):+ if isinstance(cls, str) and cls.startswith("language-"):+ lang = cls.replace("language-", "")+ break+ return f"\n```{lang}\n{raw_code.strip()}\n```\n\n"+ return f"\n```\n{text.strip()}\n```\n\n"++ def convert_details(self, el: Tag, text: str, **kwargs) -> str:+ """Convert <details> elements to blockquote with summary."""+ summary_el = el.find("summary")+ summary = summary_el.get_text(strip=True) if summary_el else "Details"++ # Get the content after the summary+ content_parts = []+ for child in el.children:+ if isinstance(child, Tag) and child.name == "summary":+ continue+ converted = (+ self.convert(str(child)) if isinstance(child, Tag) else str(child)+ )+ if converted.strip():+ content_parts.append(converted.strip())++ content = "\n".join(content_parts)+ # Indent content as blockquote+ indented = "\n".join(+ f"> {line}" if line.strip() else ">" for line in content.split("\n")+ )+ return f"\n> **{summary}**\n>\n{indented}\n\n"++ def convert_div(self, el: Tag, text: str, **kwargs) -> str:+ """Handle special div types: admonitions, embeds, doc objects."""+ classes = el.get("class", [])++ # Admonitions (tip, note, warning, etc.)+ if "admonition" in classes:+ return self._convert_admonition(el, classes)++ # marimo embed containers+ if "marimo-embed-container" in classes:+ iframe = el.find("iframe")+ if iframe and iframe.get("src"):+ return f"\n[Interactive marimo example]({iframe['src']})\n\n"+ return ""++ # Tabbed content sets+ if "tabbed-set" in classes:+ return self._convert_tabbed(el)++ # mkdocstrings doc signature blocks+ if "doc-signature" in classes or (+ "highlight" in classes and "doc-signature" in " ".join(classes)+ ):+ code_el = el.find("code")+ if code_el:+ raw = code_el.get_text()+ return f"\n```python\n{raw.strip()}\n```\n\n"++ # Doc contents - just pass through+ if "doc-contents" in classes:+ return f"\n{text}\n"++ # Default: return inner text+ return text++ def _convert_admonition(self, el: Tag, classes: list) -> str:+ """Convert admonition divs to blockquotes."""+ # Determine type+ admonition_type = "Note"+ for cls in classes:+ if cls != "admonition" and cls not in (+ "md-typeset",+ "inline",+ "end",+ ):+ admonition_type = cls.capitalize()+ break++ title_el = el.find("p", class_="admonition-title")+ title = title_el.get_text(strip=True) if title_el else admonition_type++ # Get content after title+ content_parts = []+ for child in el.children:+ if isinstance(child, Tag) and "admonition-title" in (+ child.get("class", [])+ ):+ continue+ converted = (+ self.convert(str(child)) if isinstance(child, Tag) else str(child)+ )+ if converted.strip():+ content_parts.append(converted.strip())++ content = "\n".join(content_parts)+ indented = "\n".join(+ f"> {line}" if line.strip() else ">" for line in content.split("\n")+ )+ return f"\n> **{title}**\n>\n{indented}\n\n"++ def _convert_tabbed(self, el: Tag) -> str:+ """Convert tabbed content to sections."""+ result = []+ contents = el.find_all("div", class_="tabbed-block")++ # Get label text from <label> elements inside .tabbed-labels+ label_texts = []+ labels_div = el.find("div", class_="tabbed-labels")+ if labels_div:+ for label in labels_div.find_all("label"):+ text = label.get_text(strip=True)+ if text:+ label_texts.append(text)++ # Fallback: try tabbed-set__label class+ if not label_texts:+ for label in el.find_all("label", class_="tabbed-set__label"):+ text = label.get_text(strip=True)+ if text:+ label_texts.append(text)++ for i, content in enumerate(contents):+ tab_label = label_texts[i] if i < len(label_texts) else f"Tab {i + 1}"+ converted = self.convert(str(content))+ result.append(f"**{tab_label}:**\n\n{converted.strip()}")++ return "\n\n".join(result) + "\n\n"++ def convert_a(self, el: Tag, text: str, **kwargs) -> str:+ """Convert links, resolving relative paths to absolute URLs."""+ href = el.get("href", "")++ # Skip headerlinks (permalink anchors)+ if "headerlink" in (el.get("class") or []):+ return ""++ if not text.strip():+ return ""++ # Resolve relative URLs to absolute+ if href and not href.startswith(("http://", "https://", "#", "mailto:")):+ # Build the absolute URL+ page_url = f"{self.base_url}/{self.page_path}"+ href = urljoin(page_url, href)++ if href:+ return f"[{text.strip()}]({href})"+ return text++ def convert_img(self, el: Tag, text: str, **kwargs) -> str:+ """Convert images with resolved URLs."""+ src = el.get("src", "")+ alt = el.get("alt", "")++ if not src:+ return ""++ # Skip tiny decorative images / icons+ width = el.get("width", "")+ if width and str(width).isdigit() and int(width) < 20:+ return ""++ # Resolve relative URLs+ if not src.startswith(("http://", "https://", "data:")):+ page_url = f"{self.base_url}/{self.page_path}"+ src = urljoin(page_url, src)++ return f""++ def _resolve_url(self, url: str) -> str:+ """Resolve a relative URL to an absolute URL."""+ if url and not url.startswith(("http://", "https://", "data:")):+ page_url = f"{self.base_url}/{self.page_path}"+ return urljoin(page_url, url)+ return url++ def convert_video(self, el: Tag, text: str, **kwargs) -> str:+ """Convert <video> elements to a markdown link."""+ src = el.get("src", "")+ # Also check for <source> child+ if not src:+ source = el.find("source")+ if source:+ src = source.get("src", "")+ if not src:+ return ""+ src = self._resolve_url(src)+ return f"\n[Video: {src}]({src})\n"++ def convert_figure(self, el: Tag, text: str, **kwargs) -> str:+ """Convert <figure> elements, preserving caption."""+ parts = []+ for child in el.children:+ if isinstance(child, Tag):+ if child.name == "figcaption":+ caption = child.get_text(strip=True)+ if caption:+ parts.append(f"*{caption}*")+ else:+ converted = self.convert(str(child)).strip()+ if converted:+ parts.append(converted)+ return "\n" + "\n\n".join(parts) + "\n\n"++ def convert_figcaption(self, el: Tag, text: str, **kwargs) -> str:+ """Convert <figcaption> to italicized text (standalone, outside figure)."""+ caption = el.get_text(strip=True)+ if caption:+ return f"\n*{caption}*\n"+ return ""++ def convert_table(self, el: Tag, text: str, **kwargs) -> str:+ """Convert tables to markdown tables."""+ rows = el.find_all("tr")+ if not rows:+ return text++ table_data: list[list[str]] = []+ for row in rows:+ cells = row.find_all(["th", "td"])+ row_data = []+ for cell in cells:+ # Convert cell content+ cell_text = self.convert(str(cell))+ # Clean up: collapse whitespace, remove newlines+ cell_text = re.sub(r"\s+", " ", cell_text).strip()+ # Escape pipe characters so they don't break the table+ cell_text = cell_text.replace("|", "\\|")+ row_data.append(cell_text)+ table_data.append(row_data)++ if not table_data:+ return text++ # Build markdown table+ result = []+ # Header+ result.append("| " + " | ".join(table_data[0]) + " |")+ result.append("| " + " | ".join("---" for _ in table_data[0]) + " |")+ # Body+ for row in table_data[1:]:+ # Pad row to match header length+ while len(row) < len(table_data[0]):+ row.append("")+ result.append("| " + " | ".join(row) + " |")++ return "\n" + "\n".join(result) + "\n\n"++ def convert_td(self, el: Tag, text: str, **kwargs) -> str:+ """Pass through td content without wrapping."""+ return text.strip()++ def convert_th(self, el: Tag, text: str, **kwargs) -> str:+ """Pass through th content without wrapping."""+ return text.strip()+++def extract_title(soup: BeautifulSoup) -> str:+ """Extract the page title from the HTML."""+ title_el = soup.find("title")+ if title_el:+ title = title_el.get_text(strip=True)+ # Remove " - marimo" suffix+ if title.endswith(" - marimo"):+ title = title[: -len(" - marimo")]+ return title+ return ""+++def clean_markdown(md: str) -> str:+ """Post-process markdown to clean up artifacts."""+ # Remove excessive blank lines (more than 2 consecutive)+ md = re.sub(r"\n{4,}", "\n\n\n", md)+ # Remove trailing whitespace on lines+ md = "\n".join(line.rstrip() for line in md.split("\n"))+ # Remove the pilcrow sign that appears in headerlinks+ md = md.replace("\u00b6", "")+ # Unescape Jinja2-escaped braces (\{\{ -> {{, \}\} -> }})+ md = md.replace("\\{\\{", "{{").replace("\\}\\}", "}}")+ # Strip leading/trailing whitespace+ md = md.strip()+ return md+++def convert_page(html_path: Path, input_dir: Path, base_url: str) -> str | None:+ """Convert a single HTML page to markdown."""+ with open(html_path, encoding="utf-8") as f:+ html = f.read()++ soup = BeautifulSoup(html, "html.parser")++ # Extract the article content+ article = soup.find("article", class_="md-content__inner")+ if not article:+ return None++ # Remove elements we don't want+ # Remove copy buttons from code blocks+ for btn in article.find_all("button", class_="md-clipboard"):+ btn.decompose()+ # Remove annotation markers+ for marker in article.find_all("span", class_="md-annotation"):+ marker.decompose()+ # Remove "Last updated" footer+ for aside in article.find_all("aside", class_="md-source-file"):+ aside.decompose()+ # Remove SVG icons+ for svg in article.find_all("svg"):+ svg.decompose()+ # Remove inline style tags+ for style in article.find_all("style"):+ style.decompose()++ title = extract_title(soup)++ # Calculate the page path relative to the input dir+ rel_path = html_path.parent.relative_to(input_dir)+ page_path = str(rel_path)+ if page_path == ".":+ page_path = ""+ page_path = page_path + "/"++ # Get canonical URL+ canonical = f"{base_url}/{page_path}".replace("//", "/").replace(":/", "://")++ # Convert to markdown+ converter = DocsMarkdownConverter(+ base_url=base_url,+ page_path=page_path,+ heading_style="ATX",+ bullets="-",+ strong_em_symbol="*",+ wrap=False,+ wrap_width=0,+ strip=["nav", "footer", "script", "style", "noscript"],+ )+ md = converter.convert(str(article))+ md = clean_markdown(md)++ # Add source URL as a comment at the top (the article already has an H1)+ header = f"<!-- Source: {canonical} -->\n\n"+ return header + md+++def main():+ parser = argparse.ArgumentParser(description="Convert mkdocs HTML to markdown")+ parser.add_argument(+ "--input-dir",+ type=Path,+ required=True,+ help="Path to the built site directory",+ )+ parser.add_argument(+ "--base-url",+ type=str,+ default="https://docs.marimo.io",+ help="Base URL for resolving links",+ )+ args = parser.parse_args()++ input_dir: Path = args.input_dir+ base_url: str = args.base_url++ if not input_dir.exists():+ print(f"Error: input directory {input_dir} does not exist")+ raise SystemExit(1)++ converted = 0+ skipped = 0++ for root, _dirs, files in os.walk(input_dir):+ for filename in files:+ if filename != "index.html":+ continue++ html_path = Path(root) / filename+ md = convert_page(html_path, input_dir, base_url)++ if md is None:+ skipped += 1+ continue++ # Write markdown alongside the HTML+ md_path = html_path.with_name("index.md")+ with open(md_path, "w", encoding="utf-8") as f:+ f.write(md)++ converted += 1++ print(f"Converted {converted} pages, skipped {skipped}")+++if __name__ == "__main__":+ main()diff --git a/docs/scripts/middleware.js b/docs/scripts/middleware.jsnew file mode 100644index 0000000000..c54589ccdb--- /dev/null+++ b/docs/scripts/middleware.js@@ -0,0 +1,91 @@+/**+ * Vercel Edge Middleware for serving markdown versions of docs pages.+ *+ * Handles two cases:+ * 1. Accept: text/markdown header -> rewrite to serve .md file+ * 2. URL ending in .md -> serve the corresponding .md file+ *+ * Testing (deployed):+ *+ * # Via Accept header (content negotiation)+ * curl -H "Accept: text/markdown" https://docs.marimo.io/guides/reactivity/+ *+ * # Via .md URL extension+ * curl https://docs.marimo.io/guides/reactivity.md+ *+ * Testing (local build):+ *+ * # 1. Build docs and generate markdown+ * make docs+ * python docs/scripts/html_to_markdown.py --input-dir site --base-url https://docs.marimo.io+ *+ * # 2. Verify .md files were created+ * cat site/guides/reactivity/index.md+ *+ * Note: The middleware only runs on Vercel (edge runtime), so Accept header+ * negotiation can't be tested locally. The .md files can be served directly+ * by any static file server, e.g.:+ * cd site && python -m http.server 8000+ * curl http://localhost:8000/guides/reactivity/index.md+ */+export default async function middleware(request) {+ const url = new URL(request.url);+ const pathname = url.pathname;++ // Already a direct .md static file request (e.g. /guides/reactivity/index.md)+ // — let Vercel serve it as-is to avoid a rewrite loop.+ if (pathname.endsWith("/index.md")) {+ return;+ }++ const acceptHeader = request.headers.get("accept") || "";+ const wantsMarkdown = acceptHeader.includes("text/markdown");+ const isMdUrl = pathname.endsWith(".md");++ if (!wantsMarkdown && !isMdUrl) {+ return;+ }++ // Determine the .md file path+ let mdPath;+ if (isMdUrl) {+ // /guides/reactivity.md -> /guides/reactivity/index.md+ mdPath = pathname.replace(/\.md$/, "/index.md");+ } else {+ // /guides/reactivity/ -> /guides/reactivity/index.md+ mdPath = pathname.endsWith("/")+ ? pathname + "index.md"+ : pathname + "/index.md";+ }++ // Rewrite to the static .md file. In Vercel's edge runtime, fetch() to the+ // same origin goes directly to the static file server without re-triggering+ // middleware. The /index.md early-return above is an extra safety guard.+ const rewriteUrl = new URL(mdPath, request.url);+ const response = await fetch(rewriteUrl);++ if (!response.ok) {+ // .md file doesn't exist — fall through to normal static serving+ return;+ }++ const headers = new Headers(response.headers);+ headers.set("content-type", "text/markdown; charset=utf-8");+ if (!headers.has("cache-control")) {+ headers.set("cache-control", "public, max-age=3600");+ }+ headers.set("access-control-allow-origin", "*");+ headers.set("vary", "Accept");++ return new Response(response.body, {+ status: 200,+ headers,+ });+}++export const config = {+ matcher: [+ // Match all paths except static assets and direct .md file requests+ "/((?!_static|assets|stylesheets|search/search_index\\.json|sitemap|robots\\.txt|favicon|llms\\.txt|CLAUDE\\.md).*)",+ ],+};diff --git a/docs/scripts/patch_vercel_config.py b/docs/scripts/patch_vercel_config.pynew file mode 100644index 0000000000..261059d6a2--- /dev/null+++ b/docs/scripts/patch_vercel_config.py@@ -0,0 +1,49 @@+"""Patch .vercel/output/config.json to add edge middleware routing."""++from __future__ import annotations++import json+import sys+from pathlib import Path+++def main():+ if len(sys.argv) < 2:+ print("Usage: patch_vercel_config.py <config.json>")+ raise SystemExit(1)++ config_path = Path(sys.argv[1])++ if config_path.exists():+ with open(config_path) as f:+ config = json.load(f)+ else:+ config = {"version": 3}++ # Ensure routes list exists+ if "routes" not in config:+ config["routes"] = []++ # Add middleware route at the beginning if not already present+ middleware_route = {+ "src": "/(?!_static/|assets/|stylesheets/|favicon\\.ico)(.*)",+ "middlewarePath": "_middleware",+ "continue": True,+ }++ # Check if middleware route already exists+ has_middleware = any(+ r.get("middlewarePath") == "_middleware" for r in config["routes"]+ )++ if not has_middleware:+ config["routes"].insert(0, middleware_route)++ with open(config_path, "w") as f:+ json.dump(config, f, indent=2)++ print(f"Patched {config_path} with middleware route")+++if __name__ == "__main__":+ main()diff --git a/docs/vercel.json b/docs/vercel.jsonindex 5e857419ec..c25c15c9d2 100644--- a/docs/vercel.json+++ b/docs/vercel.json@@ -1,4 +1,19 @@{+ "headers": [+ {+ "source": "/(.*)\\.md",+ "headers": [+ {+ "key": "Content-Type",+ "value": "text/markdown; charset=utf-8"+ },+ {+ "key": "Access-Control-Allow-Origin",+ "value": "*"+ }+ ]+ }+ ],"redirects": [{"source": "/guides/package_reproducibility/",diff --git a/pyproject.toml b/pyproject.tomlindex 8c5d94020d..6dd7efc297 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -269,6 +269,8 @@ docs = ["mdx-include>=1.4.2","pymdown-extensions>=10.7","lzstring>=1.0.4",+ "beautifulsoup4>=4.12.0",+ "markdownify>=0.13.0",]