DSPy: The framework for programming—not prompting—language models
Python
docs(dspy) generate missing .md-files shown in llms.txt (#10576)
Thomas Hjelde Thoresen0d0646bd9bf8
diff --git a/docs/README.md b/docs/README.mdindex 48c9ed782..c82747952 100644--- a/docs/README.md+++ b/docs/README.md@@ -76,5 +76,5 @@ This guide is for contributors looking to make changes to the documentation in t## LLMs.txt-The build process generates an `/llms.txt` file for LLM consumption using [mkdocs-llmstxt](https://github.com/pawamoy/mkdocs-llmstxt). Configure sections in `mkdocs.yml` under the `llmstxt` plugin.+The build process generates an `/llms.txt` file for LLM consumption, plus a Markdown copy of every rendered page at `<page>/index.md` for its links to point to. Configure sections in `build.yml` under `llms`; the generator lives in `scripts/zensical_build.py`.diff --git a/docs/requirements.txt b/docs/requirements.txtindex 724605388..da3337a3c 100644--- a/docs/requirements.txt+++ b/docs/requirements.txt@@ -1,3 +1,7 @@+beautifulsoup4==4.15.0+markdownify==1.2.3+mdformat==0.7.22+mdformat-tables==1.0.0mkdocstrings==1.0.6mkdocstrings-python==2.0.9nbconvert==7.17.1diff --git a/docs/scripts/zensical_build.py b/docs/scripts/zensical_build.pyindex c54106882..51dd70786 100644--- a/docs/scripts/zensical_build.py+++ b/docs/scripts/zensical_build.py@@ -17,7 +17,7 @@ import textwrapfrom html import escapefrom html.parser import HTMLParserfrom pathlib import Path-from urllib.parse import urljoin+from urllib.parse import urljoin, urlparseimport yaml@@ -210,6 +210,69 @@ def generate_llms(docs: Path, output: Path, site_url: str, settings: dict[str, ooutput.write_text("\n".join(parts))+MARKDOWN_EXCLUDED_CLASSES = {"headerlink", "twemoji", "tabbed-labels", "doc-labels"}+++def page_markdown(html: str, page_url: str, site_url: str, markdown_urls: dict[str, str]) -> str | None:+ """Convert a rendered page's article to Markdown, as mkdocs-llmstxt did for Material builds.++ `markdown_urls` maps page URLs, including static redirects, to the Markdown copy they resolve to.+ """+ import mdformat+ from bs4 import BeautifulSoup+ from markdownify import ATX, MarkdownConverter++ article = BeautifulSoup(html, "html.parser").find("article")+ if article is None:+ return None+ for element in article.find_all(["img", "svg"]):+ element.decompose()+ for element in article.find_all(class_=lambda value: value in MARKDOWN_EXCLUDED_CLASSES):+ element.decompose()+ for element in article.find_all("div", class_="doc-md-description"):+ element.replace_with(element.get_text().strip())+ for element in article.find_all("table", class_="highlighttable"):+ code = element.find("code")+ element.replace_with(BeautifulSoup(f"<pre>{escape(code.get_text() if code else '')}</pre>", "html.parser"))+ for link in article.find_all("a", href=True):+ href = link["href"]+ if href.startswith(("#", "//")) or urlparse(href).scheme:+ continue+ # Root-relative links stay within the version, as build_docs.py scopes them in HTML.+ target = urljoin(site_url, href.lstrip("/")) if href.startswith("/") else urljoin(page_url, href)+ path, _, fragment = target.partition("#")+ markdown_url = markdown_urls.get(path) or markdown_urls.get(f"{path}/")+ link["href"] = f"{markdown_url}{'#' + fragment if fragment else ''}" if markdown_url else target++ def language(tag) -> str:+ classes = [*(tag.get("class") or ()), *((tag.parent.get("class") or ()) if tag.parent else ())]+ return next((name.removeprefix("language-") for name in classes if name.startswith("language-")), "")++ converter = MarkdownConverter(+ bullets="-", code_language_callback=language, escape_underscores=False, heading_style=ATX+ )+ return mdformat.text(converter.convert_soup(article), options={"wrap": "no"}, extensions=("tables",))+++def write_page_markdown(site: Path, site_url: str, redirects: dict[str, str]) -> None:+ """Write `<route>/index.md` beside every rendered page so llms.txt links resolve."""+ pages = {+ page: urljoin(site_url, "" if route == "." else f"{route}/")+ for page in site.rglob("index.html")+ if "<article" in page.read_text()+ for route in [page.parent.relative_to(site).as_posix()]+ }+ markdown_urls = {url: f"{url}index.md" for url in pages.values()}+ for source, target in redirects.items():+ target_url = urljoin(site_url, route_for_source(target.replace(".ipynb", ".md")).lstrip("/"))+ if target_url in markdown_urls:+ markdown_urls[urljoin(site_url, route_for_source(source).lstrip("/"))] = markdown_urls[target_url]+ for page, url in pages.items():+ markdown = page_markdown(page.read_text(), url, site_url, markdown_urls)+ if markdown is not None:+ page.with_suffix(".md").write_text(markdown)++class HeadParser(HTMLParser):def __init__(self) -> None:super().__init__()@@ -320,7 +383,10 @@ def social_cards(site: Path, site_url: str, titles: dict[str, str], logo: Path)HREF = re.compile(r"""href=["']?([^"'\s>]+)""")-def validate_output(site: Path, notebook_routes: set[str], redirects: dict[str, str]) -> None:+LLMS_LINK = re.compile(r"^- \[.*\]\((?P<url>[^)\s]+)\)(?::.*)?$", re.MULTILINE)+++def validate_output(site: Path, notebook_routes: set[str], redirects: dict[str, str], site_url: str) -> None:"""Fail the build when an existing documentation feature has no output."""required = ("index.html", "api/index.html", "search.json", "llms.txt", "sitemap.xml", "sitemap.xml.gz")missing = [path for path in required if not (site / path).is_file()]@@ -330,6 +396,11 @@ def validate_output(site: Path, notebook_routes: set[str], redirects: dict[str,for source in redirectsif not (site / (route_for_source(source).lstrip("/") + "index.html")).is_file())+ missing.extend(+ url+ for url in LLMS_LINK.findall((site / "llms.txt").read_text() if (site / "llms.txt").is_file() else "")+ if not url.startswith(site_url) or not (site / url.removeprefix(site_url)).is_file()+ )if missing:raise RuntimeError(f"Zensical output is missing required files: {', '.join(sorted(missing))}")@@ -397,13 +468,14 @@ def build_zensical_site(str(source_config["site_url"]),llms,)+ write_page_markdown(output, str(source_config["site_url"]), redirect_maps(prepared))social_cards(output, str(source_config["site_url"]), titles, project / "docs" / "static" / "img" / "logo.png")sitemap = output / "sitemap.xml"with gzip.open(output / "sitemap.xml.gz", "wb") as compressed:compressed.write(sitemap.read_bytes())home = output / "index.html"home.write_text(re.sub(r"<title>Index\s+-\s+DSPy</title>", "<title>DSPy</title>", home.read_text(), count=1))- validate_output(output, notebook_routes, redirect_maps(prepared))+ validate_output(output, notebook_routes, redirect_maps(prepared), str(source_config["site_url"]))return notebook_routesfinally:if temporary:diff --git a/tests/docs/test_zensical_build.py b/tests/docs/test_zensical_build.pyindex a01173e7a..5037ff96c 100644--- a/tests/docs/test_zensical_build.py+++ b/tests/docs/test_zensical_build.py@@ -17,6 +17,7 @@ from docs.scripts.zensical_build import (remove_redirect_sources,social_cards,validate_output,+ write_page_markdown,write_redirects,)@@ -113,6 +114,43 @@ def test_llms_generation_uses_the_configured_inventory(tmp_path):assert "Excluded" not in result+def test_page_markdown_is_written_beside_rendered_pages(tmp_path):+ for module in ("bs4", "markdownify", "mdformat"):+ pytest.importorskip(module)+ guide = tmp_path / "guides" / "first" / "index.html"+ guide.parent.mkdir(parents=True)+ guide.write_text(+ "<html><body><nav>Navigation</nav><article>"+ '<h1 id="first">First<a class="headerlink" href="#first">¶</a></h1>'+ '<p>See <a href="../second/#setup">Second</a>, <a href="../../learn/">Learn</a>,'+ ' <a href="/learn#steps">Steps</a>, <a href="../../gone/">Gone</a>'+ ' and <a href="https://example.com/">elsewhere</a>.</p>'+ "<div class=\"highlight\"><pre><code>print('hi')</code></pre></div>"+ "</article></body></html>"+ )+ second = tmp_path / "guides" / "second" / "index.html"+ second.parent.mkdir()+ second.write_text("<article><h1>Second</h1></article>")+ redirect = tmp_path / "learn" / "index.html"+ redirect.parent.mkdir()+ redirect.write_text('<meta http-equiv="refresh" content="0; url=../guides/first/">')++ write_page_markdown(tmp_path, "https://dspy.ai/current/", {"learn/index.md": "guides/second.md"})++ markdown = (tmp_path / "guides" / "first" / "index.md").read_text()+ assert markdown.startswith("# First\n")+ assert "Navigation" not in markdown+ assert "¶" not in markdown+ assert "[Second](https://dspy.ai/current/guides/second/index.md#setup)" in markdown+ assert "[Learn](https://dspy.ai/current/guides/second/index.md)" in markdown+ assert "[Steps](https://dspy.ai/current/guides/second/index.md#steps)" in markdown+ assert "[Gone](https://dspy.ai/current/gone/)" in markdown+ assert "[elsewhere](https://example.com/)" in markdown+ assert "```\nprint('hi')\n```" in markdown+ assert (tmp_path / "guides" / "second" / "index.md").read_text() == "# Second\n"+ assert not (tmp_path / "learn" / "index.md").exists()++def test_public_stats_api_does_not_leak_cache_metadata(tmp_path, monkeypatch):cache = tmp_path / "stats.json"fetched = {"stars": "10k"}@@ -207,8 +245,17 @@ def test_output_validation_covers_native_and_compatibility_features(tmp_path):cards.mkdir(parents=True)(cards / "home.png").write_bytes(b"card")- validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"})+ (tmp_path / "llms.txt").write_text("- [Home](https://dspy.ai/current/index.md): Overview\n")+ (tmp_path / "index.md").write_text("# Home\n")+ site_url = "https://dspy.ai/current/"++ validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"}, site_url)++ (tmp_path / "index.md").unlink()+ with pytest.raises(RuntimeError, match=r"missing required files: https://dspy.ai/current/index.md"):+ validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"}, site_url)+ (tmp_path / "index.md").write_text("# Home\n")(tmp_path / "search.json").write_text(json.dumps({"items": []}))with pytest.raises(RuntimeError, match="search index is empty"):- validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"})+ validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"}, site_url)