Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

docs(dspy) generate missing .md-files shown in llms.txt (#10576)

Thomas Hjelde Thoresen0d0646bd9bf8
diff --git a/docs/README.md b/docs/README.md
index 48c9ed782..c82747952 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -76,5 +76,5 @@ This guide is for contributors looking to make changes to the documentation in t
## LLMs.txt
-The build process generates an `/llms.txt` file for LLM consumption using [mkdocs-llmstxt](https://github.com/pawamoy/mkdocs-llmstxt). Configure sections in `mkdocs.yml` under the `llmstxt` plugin.
+The build process generates an `/llms.txt` file for LLM consumption, plus a Markdown copy of every rendered page at `<page>/index.md` for its links to point to. Configure sections in `build.yml` under `llms`; the generator lives in `scripts/zensical_build.py`.
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 724605388..da3337a3c 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,3 +1,7 @@
+beautifulsoup4==4.15.0
+markdownify==1.2.3
+mdformat==0.7.22
+mdformat-tables==1.0.0
mkdocstrings==1.0.6
mkdocstrings-python==2.0.9
nbconvert==7.17.1
diff --git a/docs/scripts/zensical_build.py b/docs/scripts/zensical_build.py
index c54106882..51dd70786 100644
--- a/docs/scripts/zensical_build.py
+++ b/docs/scripts/zensical_build.py
@@ -17,7 +17,7 @@ import textwrap
from html import escape
from html.parser import HTMLParser
from pathlib import Path
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
import yaml
@@ -210,6 +210,69 @@ def generate_llms(docs: Path, output: Path, site_url: str, settings: dict[str, o
output.write_text("\n".join(parts))
+MARKDOWN_EXCLUDED_CLASSES = {"headerlink", "twemoji", "tabbed-labels", "doc-labels"}
+
+
+def page_markdown(html: str, page_url: str, site_url: str, markdown_urls: dict[str, str]) -> str | None:
+ """Convert a rendered page's article to Markdown, as mkdocs-llmstxt did for Material builds.
+
+ `markdown_urls` maps page URLs, including static redirects, to the Markdown copy they resolve to.
+ """
+ import mdformat
+ from bs4 import BeautifulSoup
+ from markdownify import ATX, MarkdownConverter
+
+ article = BeautifulSoup(html, "html.parser").find("article")
+ if article is None:
+ return None
+ for element in article.find_all(["img", "svg"]):
+ element.decompose()
+ for element in article.find_all(class_=lambda value: value in MARKDOWN_EXCLUDED_CLASSES):
+ element.decompose()
+ for element in article.find_all("div", class_="doc-md-description"):
+ element.replace_with(element.get_text().strip())
+ for element in article.find_all("table", class_="highlighttable"):
+ code = element.find("code")
+ element.replace_with(BeautifulSoup(f"<pre>{escape(code.get_text() if code else '')}</pre>", "html.parser"))
+ for link in article.find_all("a", href=True):
+ href = link["href"]
+ if href.startswith(("#", "//")) or urlparse(href).scheme:
+ continue
+ # Root-relative links stay within the version, as build_docs.py scopes them in HTML.
+ target = urljoin(site_url, href.lstrip("/")) if href.startswith("/") else urljoin(page_url, href)
+ path, _, fragment = target.partition("#")
+ markdown_url = markdown_urls.get(path) or markdown_urls.get(f"{path}/")
+ link["href"] = f"{markdown_url}{'#' + fragment if fragment else ''}" if markdown_url else target
+
+ def language(tag) -> str:
+ classes = [*(tag.get("class") or ()), *((tag.parent.get("class") or ()) if tag.parent else ())]
+ return next((name.removeprefix("language-") for name in classes if name.startswith("language-")), "")
+
+ converter = MarkdownConverter(
+ bullets="-", code_language_callback=language, escape_underscores=False, heading_style=ATX
+ )
+ return mdformat.text(converter.convert_soup(article), options={"wrap": "no"}, extensions=("tables",))
+
+
+def write_page_markdown(site: Path, site_url: str, redirects: dict[str, str]) -> None:
+ """Write `<route>/index.md` beside every rendered page so llms.txt links resolve."""
+ pages = {
+ page: urljoin(site_url, "" if route == "." else f"{route}/")
+ for page in site.rglob("index.html")
+ if "<article" in page.read_text()
+ for route in [page.parent.relative_to(site).as_posix()]
+ }
+ markdown_urls = {url: f"{url}index.md" for url in pages.values()}
+ for source, target in redirects.items():
+ target_url = urljoin(site_url, route_for_source(target.replace(".ipynb", ".md")).lstrip("/"))
+ if target_url in markdown_urls:
+ markdown_urls[urljoin(site_url, route_for_source(source).lstrip("/"))] = markdown_urls[target_url]
+ for page, url in pages.items():
+ markdown = page_markdown(page.read_text(), url, site_url, markdown_urls)
+ if markdown is not None:
+ page.with_suffix(".md").write_text(markdown)
+
+
class HeadParser(HTMLParser):
def __init__(self) -> None:
super().__init__()
@@ -320,7 +383,10 @@ def social_cards(site: Path, site_url: str, titles: dict[str, str], logo: Path)
HREF = re.compile(r"""href=["']?([^"'\s>]+)""")
-def validate_output(site: Path, notebook_routes: set[str], redirects: dict[str, str]) -> None:
+LLMS_LINK = re.compile(r"^- \[.*\]\((?P<url>[^)\s]+)\)(?::.*)?$", re.MULTILINE)
+
+
+def validate_output(site: Path, notebook_routes: set[str], redirects: dict[str, str], site_url: str) -> None:
"""Fail the build when an existing documentation feature has no output."""
required = ("index.html", "api/index.html", "search.json", "llms.txt", "sitemap.xml", "sitemap.xml.gz")
missing = [path for path in required if not (site / path).is_file()]
@@ -330,6 +396,11 @@ def validate_output(site: Path, notebook_routes: set[str], redirects: dict[str,
for source in redirects
if not (site / (route_for_source(source).lstrip("/") + "index.html")).is_file()
)
+ missing.extend(
+ url
+ for url in LLMS_LINK.findall((site / "llms.txt").read_text() if (site / "llms.txt").is_file() else "")
+ if not url.startswith(site_url) or not (site / url.removeprefix(site_url)).is_file()
+ )
if missing:
raise RuntimeError(f"Zensical output is missing required files: {', '.join(sorted(missing))}")
@@ -397,13 +468,14 @@ def build_zensical_site(
str(source_config["site_url"]),
llms,
)
+ write_page_markdown(output, str(source_config["site_url"]), redirect_maps(prepared))
social_cards(output, str(source_config["site_url"]), titles, project / "docs" / "static" / "img" / "logo.png")
sitemap = output / "sitemap.xml"
with gzip.open(output / "sitemap.xml.gz", "wb") as compressed:
compressed.write(sitemap.read_bytes())
home = output / "index.html"
home.write_text(re.sub(r"<title>Index\s+-\s+DSPy</title>", "<title>DSPy</title>", home.read_text(), count=1))
- validate_output(output, notebook_routes, redirect_maps(prepared))
+ validate_output(output, notebook_routes, redirect_maps(prepared), str(source_config["site_url"]))
return notebook_routes
finally:
if temporary:
diff --git a/tests/docs/test_zensical_build.py b/tests/docs/test_zensical_build.py
index a01173e7a..5037ff96c 100644
--- a/tests/docs/test_zensical_build.py
+++ b/tests/docs/test_zensical_build.py
@@ -17,6 +17,7 @@ from docs.scripts.zensical_build import (
remove_redirect_sources,
social_cards,
validate_output,
+ write_page_markdown,
write_redirects,
)
@@ -113,6 +114,43 @@ def test_llms_generation_uses_the_configured_inventory(tmp_path):
assert "Excluded" not in result
+def test_page_markdown_is_written_beside_rendered_pages(tmp_path):
+ for module in ("bs4", "markdownify", "mdformat"):
+ pytest.importorskip(module)
+ guide = tmp_path / "guides" / "first" / "index.html"
+ guide.parent.mkdir(parents=True)
+ guide.write_text(
+ "<html><body><nav>Navigation</nav><article>"
+ '<h1 id="first">First<a class="headerlink" href="#first">¶</a></h1>'
+ '<p>See <a href="../second/#setup">Second</a>, <a href="../../learn/">Learn</a>,'
+ ' <a href="/learn#steps">Steps</a>, <a href="../../gone/">Gone</a>'
+ ' and <a href="https://example.com/">elsewhere</a>.</p>'
+ "<div class=\"highlight\"><pre><code>print('hi')</code></pre></div>"
+ "</article></body></html>"
+ )
+ second = tmp_path / "guides" / "second" / "index.html"
+ second.parent.mkdir()
+ second.write_text("<article><h1>Second</h1></article>")
+ redirect = tmp_path / "learn" / "index.html"
+ redirect.parent.mkdir()
+ redirect.write_text('<meta http-equiv="refresh" content="0; url=../guides/first/">')
+
+ write_page_markdown(tmp_path, "https://dspy.ai/current/", {"learn/index.md": "guides/second.md"})
+
+ markdown = (tmp_path / "guides" / "first" / "index.md").read_text()
+ assert markdown.startswith("# First\n")
+ assert "Navigation" not in markdown
+ assert "¶" not in markdown
+ assert "[Second](https://dspy.ai/current/guides/second/index.md#setup)" in markdown
+ assert "[Learn](https://dspy.ai/current/guides/second/index.md)" in markdown
+ assert "[Steps](https://dspy.ai/current/guides/second/index.md#steps)" in markdown
+ assert "[Gone](https://dspy.ai/current/gone/)" in markdown
+ assert "[elsewhere](https://example.com/)" in markdown
+ assert "```\nprint('hi')\n```" in markdown
+ assert (tmp_path / "guides" / "second" / "index.md").read_text() == "# Second\n"
+ assert not (tmp_path / "learn" / "index.md").exists()
+
+
def test_public_stats_api_does_not_leak_cache_metadata(tmp_path, monkeypatch):
cache = tmp_path / "stats.json"
fetched = {"stars": "10k"}
@@ -207,8 +245,17 @@ def test_output_validation_covers_native_and_compatibility_features(tmp_path):
cards.mkdir(parents=True)
(cards / "home.png").write_bytes(b"card")
- validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"})
+ (tmp_path / "llms.txt").write_text("- [Home](https://dspy.ai/current/index.md): Overview\n")
+ (tmp_path / "index.md").write_text("# Home\n")
+ site_url = "https://dspy.ai/current/"
+
+ validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"}, site_url)
+
+ (tmp_path / "index.md").unlink()
+ with pytest.raises(RuntimeError, match=r"missing required files: https://dspy.ai/current/index.md"):
+ validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"}, site_url)
+ (tmp_path / "index.md").write_text("# Home\n")
(tmp_path / "search.json").write_text(json.dumps({"items": []}))
with pytest.raises(RuntimeError, match="search index is empty"):
- validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"})
+ validate_output(tmp_path, {"tutorial/index.html"}, {"old.md": "index.md"}, site_url)