Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

docs: Add markdown support for documentation with edge middleware and conversion scripts (#8796)

Myles Scolnick2635b5e7cf01
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 216c3059a4..f05c72bdc3 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -42,6 +42,19 @@ jobs:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
+ - name: 📝 Generate markdown versions of docs
+ run: |
+ uv run --group docs python docs/scripts/html_to_markdown.py \
+ --input-dir .vercel/output/static \
+ --base-url https://docs.marimo.io
+
+ - name: 🔀 Add edge middleware for markdown content negotiation
+ run: |
+ mkdir -p .vercel/output/functions/_middleware.func
+ cp docs/scripts/middleware.js .vercel/output/functions/_middleware.func/index.js
+ echo '{"runtime":"edge","entrypoint":"index.js"}' > .vercel/output/functions/_middleware.func/.vc-config.json
+ uv run --group docs python docs/scripts/patch_vercel_config.py .vercel/output/config.json
+
- name: 📚 llms.txt
# This is technically off by 1 commit, but generating the llms.txt file is easiest from a
# deployed site.
diff --git a/docs/scripts/html_to_markdown.py b/docs/scripts/html_to_markdown.py
new file mode 100644
index 0000000000..6bb8e5e8f1
--- /dev/null
+++ b/docs/scripts/html_to_markdown.py
@@ -0,0 +1,438 @@
+"""Convert mkdocs-built HTML pages to clean markdown for AI agent consumption.
+
+Walks the build output directory, extracts article content from each page,
+and writes a markdown version alongside each index.html.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+from pathlib import Path
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup, Tag
+from markdownify import MarkdownConverter
+
+
+class DocsMarkdownConverter(MarkdownConverter):
+ """Custom markdownify converter for mkdocs-material HTML."""
+
+ def __init__(self, base_url: str, page_path: str, **kwargs):
+ self.base_url = base_url.rstrip("/")
+ self.page_path = page_path
+ super().__init__(**kwargs)
+
+ def convert_pre(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert <pre> code blocks by stripping Pygments spans."""
+ code_el = el.find("code")
+ if code_el:
+ # Extract raw text, stripping all Pygments <span> tags
+ raw_code = code_el.get_text()
+ # Try to detect language from class
+ lang = ""
+ classes = list(el.get("class", [])) + list(
+ code_el.get("class", [])
+ )
+ for cls in classes:
+ if isinstance(cls, str) and cls.startswith("language-"):
+ lang = cls.replace("language-", "")
+ break
+ # Also check parent div for language class
+ parent = el.parent
+ if not lang and parent and parent.name == "div":
+ for cls in parent.get("class", []):
+ if isinstance(cls, str) and cls.startswith("language-"):
+ lang = cls.replace("language-", "")
+ break
+ return f"\n```{lang}\n{raw_code.strip()}\n```\n\n"
+ return f"\n```\n{text.strip()}\n```\n\n"
+
+ def convert_details(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert <details> elements to blockquote with summary."""
+ summary_el = el.find("summary")
+ summary = summary_el.get_text(strip=True) if summary_el else "Details"
+
+ # Get the content after the summary
+ content_parts = []
+ for child in el.children:
+ if isinstance(child, Tag) and child.name == "summary":
+ continue
+ converted = (
+ self.convert(str(child)) if isinstance(child, Tag) else str(child)
+ )
+ if converted.strip():
+ content_parts.append(converted.strip())
+
+ content = "\n".join(content_parts)
+ # Indent content as blockquote
+ indented = "\n".join(
+ f"> {line}" if line.strip() else ">" for line in content.split("\n")
+ )
+ return f"\n> **{summary}**\n>\n{indented}\n\n"
+
+ def convert_div(self, el: Tag, text: str, **kwargs) -> str:
+ """Handle special div types: admonitions, embeds, doc objects."""
+ classes = el.get("class", [])
+
+ # Admonitions (tip, note, warning, etc.)
+ if "admonition" in classes:
+ return self._convert_admonition(el, classes)
+
+ # marimo embed containers
+ if "marimo-embed-container" in classes:
+ iframe = el.find("iframe")
+ if iframe and iframe.get("src"):
+ return f"\n[Interactive marimo example]({iframe['src']})\n\n"
+ return ""
+
+ # Tabbed content sets
+ if "tabbed-set" in classes:
+ return self._convert_tabbed(el)
+
+ # mkdocstrings doc signature blocks
+ if "doc-signature" in classes or (
+ "highlight" in classes and "doc-signature" in " ".join(classes)
+ ):
+ code_el = el.find("code")
+ if code_el:
+ raw = code_el.get_text()
+ return f"\n```python\n{raw.strip()}\n```\n\n"
+
+ # Doc contents - just pass through
+ if "doc-contents" in classes:
+ return f"\n{text}\n"
+
+ # Default: return inner text
+ return text
+
+ def _convert_admonition(self, el: Tag, classes: list) -> str:
+ """Convert admonition divs to blockquotes."""
+ # Determine type
+ admonition_type = "Note"
+ for cls in classes:
+ if cls != "admonition" and cls not in (
+ "md-typeset",
+ "inline",
+ "end",
+ ):
+ admonition_type = cls.capitalize()
+ break
+
+ title_el = el.find("p", class_="admonition-title")
+ title = title_el.get_text(strip=True) if title_el else admonition_type
+
+ # Get content after title
+ content_parts = []
+ for child in el.children:
+ if isinstance(child, Tag) and "admonition-title" in (
+ child.get("class", [])
+ ):
+ continue
+ converted = (
+ self.convert(str(child)) if isinstance(child, Tag) else str(child)
+ )
+ if converted.strip():
+ content_parts.append(converted.strip())
+
+ content = "\n".join(content_parts)
+ indented = "\n".join(
+ f"> {line}" if line.strip() else ">" for line in content.split("\n")
+ )
+ return f"\n> **{title}**\n>\n{indented}\n\n"
+
+ def _convert_tabbed(self, el: Tag) -> str:
+ """Convert tabbed content to sections."""
+ result = []
+ contents = el.find_all("div", class_="tabbed-block")
+
+ # Get label text from <label> elements inside .tabbed-labels
+ label_texts = []
+ labels_div = el.find("div", class_="tabbed-labels")
+ if labels_div:
+ for label in labels_div.find_all("label"):
+ text = label.get_text(strip=True)
+ if text:
+ label_texts.append(text)
+
+ # Fallback: try tabbed-set__label class
+ if not label_texts:
+ for label in el.find_all("label", class_="tabbed-set__label"):
+ text = label.get_text(strip=True)
+ if text:
+ label_texts.append(text)
+
+ for i, content in enumerate(contents):
+ tab_label = label_texts[i] if i < len(label_texts) else f"Tab {i + 1}"
+ converted = self.convert(str(content))
+ result.append(f"**{tab_label}:**\n\n{converted.strip()}")
+
+ return "\n\n".join(result) + "\n\n"
+
+ def convert_a(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert links, resolving relative paths to absolute URLs."""
+ href = el.get("href", "")
+
+ # Skip headerlinks (permalink anchors)
+ if "headerlink" in (el.get("class") or []):
+ return ""
+
+ if not text.strip():
+ return ""
+
+ # Resolve relative URLs to absolute
+ if href and not href.startswith(("http://", "https://", "#", "mailto:")):
+ # Build the absolute URL
+ page_url = f"{self.base_url}/{self.page_path}"
+ href = urljoin(page_url, href)
+
+ if href:
+ return f"[{text.strip()}]({href})"
+ return text
+
+ def convert_img(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert images with resolved URLs."""
+ src = el.get("src", "")
+ alt = el.get("alt", "")
+
+ if not src:
+ return ""
+
+ # Skip tiny decorative images / icons
+ width = el.get("width", "")
+ if width and str(width).isdigit() and int(width) < 20:
+ return ""
+
+ # Resolve relative URLs
+ if not src.startswith(("http://", "https://", "data:")):
+ page_url = f"{self.base_url}/{self.page_path}"
+ src = urljoin(page_url, src)
+
+ return f"![{alt}]({src})"
+
+ def _resolve_url(self, url: str) -> str:
+ """Resolve a relative URL to an absolute URL."""
+ if url and not url.startswith(("http://", "https://", "data:")):
+ page_url = f"{self.base_url}/{self.page_path}"
+ return urljoin(page_url, url)
+ return url
+
+ def convert_video(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert <video> elements to a markdown link."""
+ src = el.get("src", "")
+ # Also check for <source> child
+ if not src:
+ source = el.find("source")
+ if source:
+ src = source.get("src", "")
+ if not src:
+ return ""
+ src = self._resolve_url(src)
+ return f"\n[Video: {src}]({src})\n"
+
+ def convert_figure(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert <figure> elements, preserving caption."""
+ parts = []
+ for child in el.children:
+ if isinstance(child, Tag):
+ if child.name == "figcaption":
+ caption = child.get_text(strip=True)
+ if caption:
+ parts.append(f"*{caption}*")
+ else:
+ converted = self.convert(str(child)).strip()
+ if converted:
+ parts.append(converted)
+ return "\n" + "\n\n".join(parts) + "\n\n"
+
+ def convert_figcaption(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert <figcaption> to italicized text (standalone, outside figure)."""
+ caption = el.get_text(strip=True)
+ if caption:
+ return f"\n*{caption}*\n"
+ return ""
+
+ def convert_table(self, el: Tag, text: str, **kwargs) -> str:
+ """Convert tables to markdown tables."""
+ rows = el.find_all("tr")
+ if not rows:
+ return text
+
+ table_data: list[list[str]] = []
+ for row in rows:
+ cells = row.find_all(["th", "td"])
+ row_data = []
+ for cell in cells:
+ # Convert cell content
+ cell_text = self.convert(str(cell))
+ # Clean up: collapse whitespace, remove newlines
+ cell_text = re.sub(r"\s+", " ", cell_text).strip()
+ # Escape pipe characters so they don't break the table
+ cell_text = cell_text.replace("|", "\\|")
+ row_data.append(cell_text)
+ table_data.append(row_data)
+
+ if not table_data:
+ return text
+
+ # Build markdown table
+ result = []
+ # Header
+ result.append("| " + " | ".join(table_data[0]) + " |")
+ result.append("| " + " | ".join("---" for _ in table_data[0]) + " |")
+ # Body
+ for row in table_data[1:]:
+ # Pad row to match header length
+ while len(row) < len(table_data[0]):
+ row.append("")
+ result.append("| " + " | ".join(row) + " |")
+
+ return "\n" + "\n".join(result) + "\n\n"
+
+ def convert_td(self, el: Tag, text: str, **kwargs) -> str:
+ """Pass through td content without wrapping."""
+ return text.strip()
+
+ def convert_th(self, el: Tag, text: str, **kwargs) -> str:
+ """Pass through th content without wrapping."""
+ return text.strip()
+
+
+def extract_title(soup: BeautifulSoup) -> str:
+ """Extract the page title from the HTML."""
+ title_el = soup.find("title")
+ if title_el:
+ title = title_el.get_text(strip=True)
+ # Remove " - marimo" suffix
+ if title.endswith(" - marimo"):
+ title = title[: -len(" - marimo")]
+ return title
+ return ""
+
+
+def clean_markdown(md: str) -> str:
+ """Post-process markdown to clean up artifacts."""
+ # Remove excessive blank lines (more than 2 consecutive)
+ md = re.sub(r"\n{4,}", "\n\n\n", md)
+ # Remove trailing whitespace on lines
+ md = "\n".join(line.rstrip() for line in md.split("\n"))
+ # Remove the pilcrow sign that appears in headerlinks
+ md = md.replace("\u00b6", "")
+ # Unescape Jinja2-escaped braces (\{\{ -> {{, \}\} -> }})
+ md = md.replace("\\{\\{", "{{").replace("\\}\\}", "}}")
+ # Strip leading/trailing whitespace
+ md = md.strip()
+ return md
+
+
+def convert_page(html_path: Path, input_dir: Path, base_url: str) -> str | None:
+ """Convert a single HTML page to markdown."""
+ with open(html_path, encoding="utf-8") as f:
+ html = f.read()
+
+ soup = BeautifulSoup(html, "html.parser")
+
+ # Extract the article content
+ article = soup.find("article", class_="md-content__inner")
+ if not article:
+ return None
+
+ # Remove elements we don't want
+ # Remove copy buttons from code blocks
+ for btn in article.find_all("button", class_="md-clipboard"):
+ btn.decompose()
+ # Remove annotation markers
+ for marker in article.find_all("span", class_="md-annotation"):
+ marker.decompose()
+ # Remove "Last updated" footer
+ for aside in article.find_all("aside", class_="md-source-file"):
+ aside.decompose()
+ # Remove SVG icons
+ for svg in article.find_all("svg"):
+ svg.decompose()
+ # Remove inline style tags
+ for style in article.find_all("style"):
+ style.decompose()
+
+ title = extract_title(soup)
+
+ # Calculate the page path relative to the input dir
+ rel_path = html_path.parent.relative_to(input_dir)
+ page_path = str(rel_path)
+ if page_path == ".":
+ page_path = ""
+ page_path = page_path + "/"
+
+ # Get canonical URL
+ canonical = f"{base_url}/{page_path}".replace("//", "/").replace(":/", "://")
+
+ # Convert to markdown
+ converter = DocsMarkdownConverter(
+ base_url=base_url,
+ page_path=page_path,
+ heading_style="ATX",
+ bullets="-",
+ strong_em_symbol="*",
+ wrap=False,
+ wrap_width=0,
+ strip=["nav", "footer", "script", "style", "noscript"],
+ )
+ md = converter.convert(str(article))
+ md = clean_markdown(md)
+
+ # Add source URL as a comment at the top (the article already has an H1)
+ header = f"<!-- Source: {canonical} -->\n\n"
+ return header + md
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Convert mkdocs HTML to markdown")
+ parser.add_argument(
+ "--input-dir",
+ type=Path,
+ required=True,
+ help="Path to the built site directory",
+ )
+ parser.add_argument(
+ "--base-url",
+ type=str,
+ default="https://docs.marimo.io",
+ help="Base URL for resolving links",
+ )
+ args = parser.parse_args()
+
+ input_dir: Path = args.input_dir
+ base_url: str = args.base_url
+
+ if not input_dir.exists():
+ print(f"Error: input directory {input_dir} does not exist")
+ raise SystemExit(1)
+
+ converted = 0
+ skipped = 0
+
+ for root, _dirs, files in os.walk(input_dir):
+ for filename in files:
+ if filename != "index.html":
+ continue
+
+ html_path = Path(root) / filename
+ md = convert_page(html_path, input_dir, base_url)
+
+ if md is None:
+ skipped += 1
+ continue
+
+ # Write markdown alongside the HTML
+ md_path = html_path.with_name("index.md")
+ with open(md_path, "w", encoding="utf-8") as f:
+ f.write(md)
+
+ converted += 1
+
+ print(f"Converted {converted} pages, skipped {skipped}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/scripts/middleware.js b/docs/scripts/middleware.js
new file mode 100644
index 0000000000..c54589ccdb
--- /dev/null
+++ b/docs/scripts/middleware.js
@@ -0,0 +1,91 @@
+/**
+ * Vercel Edge Middleware for serving markdown versions of docs pages.
+ *
+ * Handles two cases:
+ * 1. Accept: text/markdown header -> rewrite to serve .md file
+ * 2. URL ending in .md -> serve the corresponding .md file
+ *
+ * Testing (deployed):
+ *
+ * # Via Accept header (content negotiation)
+ * curl -H "Accept: text/markdown" https://docs.marimo.io/guides/reactivity/
+ *
+ * # Via .md URL extension
+ * curl https://docs.marimo.io/guides/reactivity.md
+ *
+ * Testing (local build):
+ *
+ * # 1. Build docs and generate markdown
+ * make docs
+ * python docs/scripts/html_to_markdown.py --input-dir site --base-url https://docs.marimo.io
+ *
+ * # 2. Verify .md files were created
+ * cat site/guides/reactivity/index.md
+ *
+ * Note: The middleware only runs on Vercel (edge runtime), so Accept header
+ * negotiation can't be tested locally. The .md files can be served directly
+ * by any static file server, e.g.:
+ * cd site && python -m http.server 8000
+ * curl http://localhost:8000/guides/reactivity/index.md
+ */
+export default async function middleware(request) {
+ const url = new URL(request.url);
+ const pathname = url.pathname;
+
+ // Already a direct .md static file request (e.g. /guides/reactivity/index.md)
+ // — let Vercel serve it as-is to avoid a rewrite loop.
+ if (pathname.endsWith("/index.md")) {
+ return;
+ }
+
+ const acceptHeader = request.headers.get("accept") || "";
+ const wantsMarkdown = acceptHeader.includes("text/markdown");
+ const isMdUrl = pathname.endsWith(".md");
+
+ if (!wantsMarkdown && !isMdUrl) {
+ return;
+ }
+
+ // Determine the .md file path
+ let mdPath;
+ if (isMdUrl) {
+ // /guides/reactivity.md -> /guides/reactivity/index.md
+ mdPath = pathname.replace(/\.md$/, "/index.md");
+ } else {
+ // /guides/reactivity/ -> /guides/reactivity/index.md
+ mdPath = pathname.endsWith("/")
+ ? pathname + "index.md"
+ : pathname + "/index.md";
+ }
+
+ // Rewrite to the static .md file. In Vercel's edge runtime, fetch() to the
+ // same origin goes directly to the static file server without re-triggering
+ // middleware. The /index.md early-return above is an extra safety guard.
+ const rewriteUrl = new URL(mdPath, request.url);
+ const response = await fetch(rewriteUrl);
+
+ if (!response.ok) {
+ // .md file doesn't exist — fall through to normal static serving
+ return;
+ }
+
+ const headers = new Headers(response.headers);
+ headers.set("content-type", "text/markdown; charset=utf-8");
+ if (!headers.has("cache-control")) {
+ headers.set("cache-control", "public, max-age=3600");
+ }
+ headers.set("access-control-allow-origin", "*");
+ headers.set("vary", "Accept");
+
+ return new Response(response.body, {
+ status: 200,
+ headers,
+ });
+}
+
+export const config = {
+ matcher: [
+ // Match all paths except static assets and direct .md file requests
+ "/((?!_static|assets|stylesheets|search/search_index\\.json|sitemap|robots\\.txt|favicon|llms\\.txt|CLAUDE\\.md).*)",
+ ],
+};
diff --git a/docs/scripts/patch_vercel_config.py b/docs/scripts/patch_vercel_config.py
new file mode 100644
index 0000000000..261059d6a2
--- /dev/null
+++ b/docs/scripts/patch_vercel_config.py
@@ -0,0 +1,49 @@
+"""Patch .vercel/output/config.json to add edge middleware routing."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+
+def main():
+ if len(sys.argv) < 2:
+ print("Usage: patch_vercel_config.py <config.json>")
+ raise SystemExit(1)
+
+ config_path = Path(sys.argv[1])
+
+ if config_path.exists():
+ with open(config_path) as f:
+ config = json.load(f)
+ else:
+ config = {"version": 3}
+
+ # Ensure routes list exists
+ if "routes" not in config:
+ config["routes"] = []
+
+ # Add middleware route at the beginning if not already present
+ middleware_route = {
+ "src": "/(?!_static/|assets/|stylesheets/|favicon\\.ico)(.*)",
+ "middlewarePath": "_middleware",
+ "continue": True,
+ }
+
+ # Check if middleware route already exists
+ has_middleware = any(
+ r.get("middlewarePath") == "_middleware" for r in config["routes"]
+ )
+
+ if not has_middleware:
+ config["routes"].insert(0, middleware_route)
+
+ with open(config_path, "w") as f:
+ json.dump(config, f, indent=2)
+
+ print(f"Patched {config_path} with middleware route")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/vercel.json b/docs/vercel.json
index 5e857419ec..c25c15c9d2 100644
--- a/docs/vercel.json
+++ b/docs/vercel.json
@@ -1,4 +1,19 @@
{
+ "headers": [
+ {
+ "source": "/(.*)\\.md",
+ "headers": [
+ {
+ "key": "Content-Type",
+ "value": "text/markdown; charset=utf-8"
+ },
+ {
+ "key": "Access-Control-Allow-Origin",
+ "value": "*"
+ }
+ ]
+ }
+ ],
"redirects": [
{
"source": "/guides/package_reproducibility/",
diff --git a/pyproject.toml b/pyproject.toml
index 8c5d94020d..6dd7efc297 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -269,6 +269,8 @@ docs = [
"mdx-include>=1.4.2",
"pymdown-extensions>=10.7",
"lzstring>=1.0.4",
+ "beautifulsoup4>=4.12.0",
+ "markdownify>=0.13.0",
]