Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
20.3 KB
Raw Blame
#!/usr/bin/env python3
"""Build DSPy documentation with Zensical from a disposable prepared project."""

from __future__ import annotations

import gzip
import hashlib
import importlib.util
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import textwrap
from html import escape
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urljoin, urlparse

import yaml


def load_config(path: Path) -> dict[str, object]:
    return dict(yaml.safe_load(path.read_text()))


def write_config(path: Path, config: dict[str, object]) -> None:
    path.write_text(yaml.safe_dump(config, sort_keys=False, allow_unicode=True))


def nav_entries(value: object) -> dict[str, str]:
    entries: dict[str, str] = {}
    if isinstance(value, list):
        for item in value:
            entries.update(nav_entries(item))
    elif isinstance(value, dict):
        for title, item in value.items():
            if isinstance(item, str) and item.endswith((".md", ".ipynb")):
                entries[item] = str(title)
            else:
                entries.update(nav_entries(item))
    return entries


def route_for_source(path: str) -> str:
    source = Path(path)
    route = (
        source.parent.as_posix().strip("/")
        if source.name == "index.md"
        else source.with_suffix("").as_posix().strip("/")
    )
    return f"/{route}/" if route and route != "." else "/"


def fetch_stats(project: Path) -> dict[str, object]:
    """Run the existing MkDocs stats hook for Zensical's template context."""
    hook = project / "hooks" / "fetch_stats.py"
    spec = importlib.util.spec_from_file_location("dspy_docs_fetch_stats", hook)
    if spec is None or spec.loader is None:
        raise RuntimeError(f"cannot load {hook}")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return dict(module.fetch_stats())


def prepare_config(config: dict[str, object], stats: dict[str, object]) -> dict[str, object]:
    prepared = dict(config)
    extra = dict(prepared.get("extra", {}))
    extra["stats"] = stats
    prepared["extra"] = extra
    return prepared


def redirect_maps(config: dict[str, object]) -> dict[str, str]:
    for plugin in config.get("plugins", []):
        if isinstance(plugin, dict) and "redirects" in plugin:
            return dict(plugin["redirects"].get("redirect_maps", {}))
    return {}


def remove_redirect_sources(docs: Path, config: dict[str, object]) -> None:
    """Reserve legacy routes for the static redirect adapter."""
    for source in redirect_maps(config):
        (docs / source).unlink(missing_ok=True)


def redirect_document(target: str) -> str:
    encoded = json.dumps(target)
    escaped = escape(target, quote=True)
    return (
        '<!doctype html><html lang="en"><head><meta charset="utf-8">'
        f'<link rel="canonical" href="{escaped}">'
        f"<script>location.replace({encoded}+location.search+location.hash)</script>"
        f'<noscript><meta http-equiv="refresh" content="0; url={escaped}"></noscript>'
        f'</head><body><a href="{escaped}">Continue to the documentation</a></body></html>'
    )


def write_redirects(site: Path, redirects: dict[str, str]) -> None:
    """Material's redirects plugin is not implemented by Zensical."""
    for source, target in redirects.items():
        route = route_for_source(source)
        destination = site / route.lstrip("/") / "index.html"
        destination.parent.mkdir(parents=True, exist_ok=True)
        target_route = route_for_source(target.replace(".ipynb", ".md"))
        relative = os.path.relpath(target_route.lstrip("/") or ".", route.lstrip("/") or ".")
        if relative == ".":
            relative = "../" if route != "/" else "./"
        elif not relative.endswith("/"):
            relative += "/"
        destination.write_text(redirect_document(relative))


NOTEBOOK_LINK = re.compile(r"\]\((?P<target>[^)\s#]+)\.ipynb(?P<suffix>#[^)\s]*)?\)")


def rewrite_notebook_links(docs: Path, notebooks: set[Path]) -> None:
    """Point Markdown links at converted notebooks to their generated pages, as mkdocs-jupyter did."""

    for page in docs.rglob("*.md"):
        text = page.read_text()

        def replace(match: re.Match[str], page: Path = page) -> str:
            target = match["target"]
            if (page.parent / f"{target}.ipynb").resolve() not in notebooks:
                return match[0]
            return f"]({target}.md{match['suffix'] or ''})"

        rewritten = NOTEBOOK_LINK.sub(replace, text)
        if rewritten != text:
            page.write_text(rewritten)


def convert_notebooks(docs: Path) -> set[str]:
    from nbconvert import MarkdownExporter

    exporter = MarkdownExporter()
    routes = set()
    notebooks = {notebook.resolve() for notebook in docs.rglob("*.ipynb")}
    for notebook in notebooks:
        relative = notebook.relative_to(docs.resolve()).with_suffix(".md").as_posix()
        routes.add(route_for_source(relative).lstrip("/") + "index.html")
        body, resources = exporter.from_filename(
            str(notebook), resources={"output_files_dir": f"{notebook.stem}_files"}
        )
        notebook.with_suffix(".md").write_text(body.replace("<details>", '<details markdown="1">'))
        for name, content in resources.get("outputs", {}).items():
            output = notebook.parent / name
            output.parent.mkdir(parents=True, exist_ok=True)
            output.write_bytes(content)
        notebook.unlink()
    rewrite_notebook_links(docs, notebooks)
    for source in docs.rglob("*.py"):
        relative = source.relative_to(docs).with_suffix(".md").as_posix()
        routes.add(route_for_source(relative).lstrip("/") + "index.html")
        source.with_suffix(".md").write_text(
            f"# {source.stem.replace('_', ' ').title()}\n\n```python\n{source.read_text()}\n```\n"
        )
    return routes


def add_missing_titles(docs: Path, entries: dict[str, str]) -> None:
    for source, title in entries.items():
        path = docs / source.replace(".ipynb", ".md")
        if path.name == "index.md" and path == docs / "index.md":
            continue
        if path.exists() and not re.search(r"(?m)^#\s+", path.read_text()):
            path.write_text(f"# {title}\n\n{path.read_text()}")


def source_title(source: Path) -> str:
    if source.suffix == ".ipynb":
        notebook = json.loads(source.read_text())
        text = "\n".join("".join(cell["source"]) for cell in notebook["cells"] if cell["cell_type"] == "markdown")
    else:
        text = source.read_text(errors="replace")
    match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
    return re.sub(r"\s*\{[^}]+\}\s*$", "", match.group(1)) if match else source.stem.replace("-", " ").title()


def matching_sources(docs: Path, pattern: str) -> list[Path]:
    if "**." in pattern:
        prefix, suffix = pattern.split("**.", 1)
        return sorted((docs / prefix).rglob(f"*.{suffix}"))
    path = docs / pattern
    return [path] if path.exists() else []


def generate_llms(docs: Path, output: Path, site_url: str, settings: dict[str, object]) -> None:
    parts = [
        "# DSPy",
        "",
        "> DSPy is the framework for programming—rather than prompting—language models.",
        "",
        str(settings["markdown_description"]).strip(),
        "",
    ]
    for section, entries in settings["sections"].items():
        parts.extend((f"## {section}", ""))
        for entry in entries:
            pattern, description = next(iter(entry.items())) if isinstance(entry, dict) else (entry, None)
            for source in matching_sources(docs, pattern):
                route = route_for_source(source.relative_to(docs).as_posix().replace(".ipynb", ".md"))
                target = "index.md" if route == "/" else f"{route.lstrip('/')}index.md"
                line = f"- [{source_title(source)}]({urljoin(site_url, target)})"
                parts.append(f"{line}: {description}" if description else line)
        parts.append("")
    output.write_text("\n".join(parts))


MARKDOWN_EXCLUDED_CLASSES = {"headerlink", "twemoji", "tabbed-labels", "doc-labels"}


def page_markdown(html: str, page_url: str, site_url: str, markdown_urls: dict[str, str]) -> str | None:
    """Convert a rendered page's article to Markdown, as mkdocs-llmstxt did for Material builds.

    `markdown_urls` maps page URLs, including static redirects, to the Markdown copy they resolve to.
    """
    import mdformat
    from bs4 import BeautifulSoup
    from markdownify import ATX, MarkdownConverter

    article = BeautifulSoup(html, "html.parser").find("article")
    if article is None:
        return None
    for element in article.find_all(["img", "svg"]):
        element.decompose()
    for element in article.find_all(class_=lambda value: value in MARKDOWN_EXCLUDED_CLASSES):
        element.decompose()
    for element in article.find_all("div", class_="doc-md-description"):
        element.replace_with(element.get_text().strip())
    for element in article.find_all("table", class_="highlighttable"):
        code = element.find("code")
        element.replace_with(BeautifulSoup(f"<pre>{escape(code.get_text() if code else '')}</pre>", "html.parser"))
    for link in article.find_all("a", href=True):
        href = link["href"]
        if href.startswith(("#", "//")) or urlparse(href).scheme:
            continue
        # Root-relative links stay within the version, as build_docs.py scopes them in HTML.
        target = urljoin(site_url, href.lstrip("/")) if href.startswith("/") else urljoin(page_url, href)
        path, _, fragment = target.partition("#")
        markdown_url = markdown_urls.get(path) or markdown_urls.get(f"{path}/")
        link["href"] = f"{markdown_url}{'#' + fragment if fragment else ''}" if markdown_url else target

    def language(tag) -> str:
        classes = [*(tag.get("class") or ()), *((tag.parent.get("class") or ()) if tag.parent else ())]
        return next((name.removeprefix("language-") for name in classes if name.startswith("language-")), "")

    converter = MarkdownConverter(
        bullets="-", code_language_callback=language, escape_underscores=False, heading_style=ATX
    )
    return mdformat.text(converter.convert_soup(article), options={"wrap": "no"}, extensions=("tables",))


def write_page_markdown(site: Path, site_url: str, redirects: dict[str, str]) -> None:
    """Write `<route>/index.md` beside every rendered page so llms.txt links resolve."""
    pages = {
        page: urljoin(site_url, "" if route == "." else f"{route}/")
        for page in site.rglob("index.html")
        if "<article" in page.read_text()
        for route in [page.parent.relative_to(site).as_posix()]
    }
    markdown_urls = {url: f"{url}index.md" for url in pages.values()}
    for source, target in redirects.items():
        target_url = urljoin(site_url, route_for_source(target.replace(".ipynb", ".md")).lstrip("/"))
        if target_url in markdown_urls:
            markdown_urls[urljoin(site_url, route_for_source(source).lstrip("/"))] = markdown_urls[target_url]
    for page, url in pages.items():
        markdown = page_markdown(page.read_text(), url, site_url, markdown_urls)
        if markdown is not None:
            page.with_suffix(".md").write_text(markdown)


class HeadParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__()
        self.metadata: dict[str, str] = {}

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        values = dict(attrs)
        if tag == "link" and values.get("rel") == "canonical" and values.get("href"):
            self.metadata["canonical"] = values["href"]
        elif tag == "meta":
            key = values.get("property") or values.get("name")
            if key and values.get("content"):
                self.metadata[key] = values["content"]


def navigation_titles(config: dict[str, object]) -> dict[str, str]:
    titles = {"/": str(config.get("site_name", "DSPy"))}
    for source, title in nav_entries(config.get("nav", [])).items():
        route = route_for_source(source.replace(".ipynb", ".md"))
        if route != "/":
            titles[route] = title
    return titles


def page_titles(config: dict[str, object], docs: Path) -> dict[str, str]:
    titles = navigation_titles(config)
    sources = [*docs.rglob("*.md"), *docs.rglob("*.ipynb"), *docs.rglob("*.py")]
    for source in sources:
        relative = source.relative_to(docs).as_posix()
        route = route_for_source(relative.replace(".ipynb", ".md").replace(".py", ".md"))
        titles.setdefault(route, source_title(source))
    return titles


def social_cards(site: Path, site_url: str, titles: dict[str, str], logo: Path) -> None:
    from PIL import Image, ImageDraw, ImageFont

    card_dir = site / "assets" / "images" / "social-zensical"
    card_dir.mkdir(parents=True, exist_ok=True)

    def load_font(size: int):
        try:
            return ImageFont.truetype("DejaVuSans.ttf", size)
        except OSError:
            # DejaVu is only guaranteed on Linux; fall back to Pillow's bundled font elsewhere.
            return ImageFont.load_default(size)

    font = load_font(32)
    title_font = load_font(84)
    description_font = load_font(25)
    logo_image = Image.open(logo).convert("RGBA").resize((140, 140))
    for page in site.rglob("*.html"):
        html = page.read_text()
        if "<article" not in html:
            continue
        relative = page.relative_to(site)
        route = "/" if relative == Path("index.html") else f"/{relative.parent.as_posix()}/"
        parser = HeadParser()
        parser.feed(html)
        title = titles.get(route, Path(route.strip("/")).name.replace("-", " ").title() or "DSPy")
        description = parser.metadata.get(
            "description", "The framework for programming—rather than prompting—language models."
        )
        card = card_dir / f"{hashlib.sha256(route.encode()).hexdigest()[:16]}.png"
        image = Image.new("RGB", (1200, 630), "white")
        draw = ImageDraw.Draw(image)
        draw.text((64, 65), "DSPy", fill="black", font=font)
        lines = textwrap.wrap(title, width=23)
        draw.multiline_text((64, 270 if len(lines) > 1 else 170), "\n".join(lines), fill="black", font=title_font)
        image.paste(logo_image, (995, 65), logo_image)
        draw.multiline_text(
            (64, 515), "\n".join(textwrap.wrap(description, width=48)), fill="black", font=description_font, spacing=8
        )
        image.save(card)

        card_url = urljoin(site_url, f"assets/images/social-zensical/{card.name}")
        page_url = urljoin(site_url, route.lstrip("/"))
        metadata = {
            "og:type": "website",
            "og:title": title if route == "/" else f"{title} - DSPy",
            "og:description": description,
            "og:url": page_url,
            "og:image": card_url,
            "og:image:type": "image/png",
            "og:image:width": "1200",
            "og:image:height": "630",
            "twitter:card": "summary_large_image",
            "twitter:title": title if route == "/" else f"{title} - DSPy",
            "twitter:description": description,
            "twitter:image": card_url,
        }
        managed_images = {"og:image", "twitter:image"}
        html = re.sub(
            r'<meta\b(?=[^>]*(?:property|name)=["\'](?:og:image|twitter:image)["\'])[^>]*>\s*',
            "",
            html,
            flags=re.IGNORECASE,
        )
        tags = "".join(
            f'<meta {"name" if key.startswith("twitter:") else "property"}="{escape(key)}" '
            f'content="{escape(value, quote=True)}">\n'
            for key, value in metadata.items()
            if key in managed_images or key not in parser.metadata
        )
        page.write_text(html.replace("</head>", f"{tags}</head>", 1))


HREF = re.compile(r"""href=["']?([^"'\s>]+)""")


LLMS_LINK = re.compile(r"^- \[.*\]\((?P<url>[^)\s]+)\)(?::.*)?$", re.MULTILINE)


def validate_output(site: Path, notebook_routes: set[str], redirects: dict[str, str], site_url: str) -> None:
    """Fail the build when an existing documentation feature has no output."""
    required = ("index.html", "api/index.html", "search.json", "llms.txt", "sitemap.xml", "sitemap.xml.gz")
    missing = [path for path in required if not (site / path).is_file()]
    missing.extend(route for route in notebook_routes if not (site / route).is_file())
    missing.extend(
        route_for_source(source).lstrip("/") + "index.html"
        for source in redirects
        if not (site / (route_for_source(source).lstrip("/") + "index.html")).is_file()
    )
    missing.extend(
        url
        for url in LLMS_LINK.findall((site / "llms.txt").read_text() if (site / "llms.txt").is_file() else "")
        if not url.startswith(site_url) or not (site / url.removeprefix(site_url)).is_file()
    )
    if missing:
        raise RuntimeError(f"Zensical output is missing required files: {', '.join(sorted(missing))}")

    search = json.loads((site / "search.json").read_text())
    if not search.get("items"):
        raise RuntimeError("Zensical search index is empty")
    if gzip.decompress((site / "sitemap.xml.gz").read_bytes()) != (site / "sitemap.xml").read_bytes():
        raise RuntimeError("compressed sitemap does not match sitemap.xml")
    home = (site / "index.html").read_text()
    cards = site / "assets" / "images" / "social-zensical"
    if not any(cards.glob("*.png")) or 'property="og:image"' not in home:
        raise RuntimeError("social cards or Open Graph metadata were not generated")

    notebook_links = [
        f"{page.relative_to(site).as_posix()} -> {href}"
        for page in site.rglob("*.html")
        for href in HREF.findall(page.read_text())
        if ":" not in href and not href.startswith("//") and re.split(r"[?#]", href)[0].endswith(".ipynb")
    ]
    if notebook_links:
        raise RuntimeError(f"pages link to notebook sources instead of rendered pages: {', '.join(notebook_links)}")


def build_zensical_site(
    *,
    config: Path,
    output: Path,
    python: Path = Path(sys.executable),
    keep_project: Path | None = None,
) -> set[str]:
    source_project = config.parent
    source_config = load_config(config)
    build_config = load_config(source_project / "build.yml")
    llms = dict(build_config["llms"])
    source_docs = source_project / str(source_config.get("docs_dir", "docs"))
    titles = page_titles(source_config, source_docs)

    temporary = None if keep_project else tempfile.TemporaryDirectory(prefix="dspy-zensical-build-")
    project = keep_project.resolve() if keep_project else Path(temporary.name) / "project"
    shutil.rmtree(project, ignore_errors=True)
    shutil.copytree(
        source_project, project, ignore=shutil.ignore_patterns("site", ".cache", "__pycache__", ".zensical-*")
    )
    try:
        prepared = prepare_config(source_config, fetch_stats(source_project))
        prepared_config = project / "zensical.generated.yml"
        write_config(prepared_config, prepared)
        prepared_docs = project / str(prepared.get("docs_dir", "docs"))
        remove_redirect_sources(prepared_docs, prepared)
        notebook_routes = convert_notebooks(prepared_docs)
        add_missing_titles(prepared_docs, nav_entries(prepared.get("nav", [])))

        subprocess.run(
            [str(python), "-m", "zensical", "build", "--clean", "--config-file", prepared_config],
            cwd=project,
            check=True,
        )
        built = project / str(prepared["site_dir"])
        shutil.rmtree(output, ignore_errors=True)
        shutil.copytree(built, output)
        write_redirects(output, redirect_maps(prepared))
        generate_llms(
            source_project / str(source_config.get("docs_dir", "docs")),
            output / "llms.txt",
            str(source_config["site_url"]),
            llms,
        )
        write_page_markdown(output, str(source_config["site_url"]), redirect_maps(prepared))
        social_cards(output, str(source_config["site_url"]), titles, project / "docs" / "static" / "img" / "logo.png")
        sitemap = output / "sitemap.xml"
        with gzip.open(output / "sitemap.xml.gz", "wb") as compressed:
            compressed.write(sitemap.read_bytes())
        home = output / "index.html"
        home.write_text(re.sub(r"<title>Index\s+-\s+DSPy</title>", "<title>DSPy</title>", home.read_text(), count=1))
        validate_output(output, notebook_routes, redirect_maps(prepared), str(source_config["site_url"]))
        return notebook_routes
    finally:
        if temporary:
            temporary.cleanup()