Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
11.1 KB
Raw Blame
#!/usr/bin/env python3
"""Build Current or one immutable documentation release snapshot."""

from __future__ import annotations

import argparse
import hashlib
import importlib.metadata
import json
import os
import re
import subprocess
import sys
import tempfile
from pathlib import Path

if __package__:
    from .zensical_build import HeadParser, build_zensical_site
else:  # Direct script execution from the deployed docs repository.
    from zensical_build import HeadParser, build_zensical_site


RELEASE_VERSION = re.compile(
    r"^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:(?:a|b|rc)\d+)?$"
)
SOURCE_MAP_COMMENT = re.compile(rb"(?:\n?//# sourceMappingURL=[^\r\n]*|/\*# sourceMappingURL=.*?\*/)", re.DOTALL)
ROOT_URL_ATTRIBUTE = re.compile(
    r'(?P<prefix>\b(?:href|src|action)=["\x27])(?P<url>/(?!/)[^"\x27]*)(?P<suffix>["\x27])',
    re.IGNORECASE,
)
VERSIONED_PATH = re.compile(r"^/(?:current|\d+\.\d+(?:\.\d+(?:(?:a|b|rc)\d+)?)?)(?:/|$)")
RELEASE_BADGE_VERSION = re.compile(
    r'(?s)(<div class=(?:"hp-hero-badge"|hp-hero-badge)>.*?\bDSPy )(\S+)(\s+&mdash;)'
)
SHARED_HEADER_ASSETS = Path(__file__).parent.parent / "versioning"


def release_version(value: str) -> str:
    match = RELEASE_VERSION.fullmatch(value)
    if not match or int(match.group("major")) < 3:
        raise argparse.ArgumentTypeError("expected an X.Y.Z release version at or after 3.0.0")
    return value


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def git_value(repository: Path, *arguments: str) -> str:
    return subprocess.check_output(["git", "-C", str(repository), *arguments], text=True).strip()


def patched_config(config: Path, identifier: str, *, edit_ref: str | None = None) -> Path:
    text = config.read_text()
    text, site_count = re.subn(r"(?m)^site_url:\s*.*$", f"site_url: https://dspy.ai/{identifier}/", text, count=1)
    if site_count != 1:
        raise RuntimeError(f"could not patch site_url in {config}")
    if edit_ref:
        text, edit_count = re.subn(r"(?m)^edit_uri:\s*.*$", f"edit_uri: blob/{edit_ref}/docs/docs/", text, count=1)
        if edit_count != 1:
            raise RuntimeError(f"could not patch edit_uri in {config}")
        # Release API docs must import the installed wheel, not this checkout.
        text = re.sub(r'(?m)^[ \t]+paths:\s*\[\s*["\x27]?\.\.["\x27]?\s*\][ \t]*\n', "", text, count=1)
    if not re.search(r"(?m)^\s+provider:\s*mike\s*$", text):
        version_config = "    version:\n        provider: mike\n        alias: true\n"
        text, extra_count = re.subn(r"(?m)^extra:\s*$", f"extra:\n{version_config}", text, count=1)
        if extra_count == 0:
            text = f"{text.rstrip()}\n\nextra:\n{version_config}"

    handle, name = tempfile.mkstemp(prefix=f".mkdocs-{identifier}-", suffix=".yml", dir=config.parent)
    os.close(handle)
    output = Path(name)
    output.write_text(text)
    return output


def remove_source_maps(site: Path) -> dict[str, int]:
    """Remove browser source maps after Zensical performs native minification."""
    before = sum(path.stat().st_size for path in site.rglob("*") if path.is_file())
    source_maps = list(site.rglob("*.map"))
    for path in source_maps:
        path.unlink()
    for pattern in ("*.js", "*.css"):
        for path in site.rglob(pattern):
            source = path.read_bytes()
            optimized = SOURCE_MAP_COMMENT.sub(b"", source)
            if optimized != source:
                path.write_bytes(optimized)
    after = sum(path.stat().st_size for path in site.rglob("*") if path.is_file())
    return {"before": before, "after": after, "source_maps": len(source_maps)}


def install_shared_header(site: Path, source: Path = SHARED_HEADER_ASSETS) -> None:
    """Give every renderer and historical snapshot the same header controls."""
    destination = site / "_static"
    destination.mkdir(parents=True, exist_ok=True)
    styles = destination / "dspy-header.css"
    script = destination / "dspy-header.js"
    styles.write_bytes((source / "header.css").read_bytes())
    script.write_bytes((source / "header.js").read_bytes())
    for page in site.rglob("*.html"):
        relative_styles = Path(os.path.relpath(styles, page.parent)).as_posix()
        relative_script = Path(os.path.relpath(script, page.parent)).as_posix()
        style_tag = f'<link rel="stylesheet" href="{relative_styles}">'
        script_tag = f'<script src="{relative_script}" defer></script>'
        html = page.read_text()
        if style_tag not in html:
            html = html.replace("</head>", f"{style_tag}</head>", 1)
        if script_tag not in html:
            html = html.replace("</head>", f"{script_tag}</head>", 1)
        page.write_text(html)


def scope_root_relative_urls(site: Path, identifier: str) -> None:
    """Keep hand-authored root-relative links inside the selected version."""

    def replace(match: re.Match[str]) -> str:
        url = match.group("url")
        if VERSIONED_PATH.match(url):
            return match.group(0)
        scoped = f"/{identifier}{url}"
        return f"{match.group('prefix')}{scoped}{match.group('suffix')}"

    for page in site.rglob("*.html"):
        html = page.read_text()
        scoped = ROOT_URL_ATTRIBUTE.sub(replace, html)
        if scoped != html:
            page.write_text(scoped)


def set_release_badge_version(site: Path, version: str) -> None:
    """Keep a historical home page from advertising the latest live release."""
    home = site / "index.html"
    html = home.read_text()
    updated, count = RELEASE_BADGE_VERSION.subn(
        lambda match: f"{match.group(1)}{version}{match.group(3)}",
        html,
        count=1,
    )
    if count:
        home.write_text(updated)


def installed_packages() -> dict[str, str]:
    return dict(
        sorted(
            (distribution.metadata["Name"], distribution.version)
            for distribution in importlib.metadata.distributions()
            if distribution.metadata["Name"]
        )
    )


def renderer_version() -> str:
    return importlib.metadata.version("zensical")


def validate_release_site(site: Path, config: Path, version: str) -> None:
    expected = (site / "index.html", site / "api" / "index.html", site / "search.json", site / "llms.txt")
    missing = [str(path.relative_to(site)) for path in expected if not path.exists()]
    if missing:
        raise RuntimeError(f"release site is missing required output: {', '.join(missing)}")

    home = (site / "index.html").read_text()
    metadata = HeadParser()
    metadata.feed(home)
    canonical = f"https://dspy.ai/{version}/"
    if metadata.metadata.get("canonical") != canonical:
        raise RuntimeError(f"release home page is missing canonical URL {canonical}")

    docs_dir = config.parent / "docs"
    missing_notebooks = []
    for notebook in docs_dir.rglob("*.ipynb"):
        relative = notebook.relative_to(docs_dir)
        outputs = (
            site / relative.with_suffix(".html"),
            site / relative.with_suffix("") / "index.html",
        )
        if not any(output.exists() for output in outputs):
            missing_notebooks.append(str(notebook.relative_to(docs_dir)))
    if missing_notebooks:
        raise RuntimeError(f"notebooks were not rendered: {', '.join(missing_notebooks)}")

    cards = site / "assets" / "images" / "social-zensical"
    if not cards.exists() or not any(cards.rglob("*.png")) or "og:image" not in metadata.metadata:
        raise RuntimeError("social cards or Open Graph metadata were not generated")

    try:
        dspy_version = importlib.metadata.version("dspy")
    except importlib.metadata.PackageNotFoundError as error:
        raise RuntimeError("release build environment does not contain DSPy") from error
    if dspy_version != version:
        raise RuntimeError(f"release build imported dspy=={dspy_version}, expected {version}")


def build(
    *,
    config: Path,
    output: Path,
    version: str | None = None,
    artifact: Path | None = None,
    package_source: str | None = None,
) -> None:
    output = output.resolve()
    output.parent.mkdir(parents=True, exist_ok=True)
    identifier = version or "current"
    effective_config = patched_config(config, identifier, edit_ref=version)
    try:
        build_zensical_site(config=effective_config, output=output, python=Path(sys.executable))
    finally:
        effective_config.unlink(missing_ok=True)

    if version:
        if artifact is None or package_source is None:
            raise ValueError("release builds require an artifact and package source")
        set_release_badge_version(output, version)
    scope_root_relative_urls(output, identifier)
    if version:
        validate_release_site(output, config, version)
    install_shared_header(output)
    optimization = remove_source_maps(output)
    if version:
        repository = config.parent.parent
        source_commit = git_value(repository, "rev-list", "-n", "1", version)
        metadata = {
            "version": version,
            "source_tag": version,
            "source_commit": source_commit,
            "source_commit_time": git_value(repository, "show", "-s", "--format=%cI", source_commit),
            "renderer": "zensical",
            "renderer_version": renderer_version(),
            "package_source": package_source,
            "package_artifact": artifact.name,
            "package_sha256": sha256(artifact),
            "optimization": optimization,
            "python": sys.version.split()[0],
            "packages": installed_packages(),
            "intentional_differences": [
                "The renderer's Mike version selector is enabled in a transient build configuration.",
                "Build provenance metadata added under _meta/build.json.",
                "Generated HTML is minified and source maps are omitted from production snapshots.",
            ],
        }
        metadata_dir = output / "_meta"
        metadata_dir.mkdir(exist_ok=True)
        (metadata_dir / "build.json").write_text(json.dumps(metadata, indent=2) + "\n")


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument("mode", choices=("current", "release"))
    parser.add_argument("--config", type=Path, default=Path("mkdocs.yml"))
    parser.add_argument("--output", type=Path, default=Path("site"))
    parser.add_argument("--version", type=release_version)
    parser.add_argument("--artifact", type=Path)
    parser.add_argument("--package-source", choices=("pypi-wheel", "workflow-wheel", "tag-built-wheel"))
    args = parser.parse_args()
    if args.mode == "release" and not (args.version and args.artifact and args.package_source):
        parser.error("release requires --version, --artifact, and --package-source")
    return args


def main() -> None:
    args = parse_args()
    build(
        config=args.config.resolve(),
        output=args.output,
        version=args.version if args.mode == "release" else None,
        artifact=args.artifact.resolve() if args.artifact else None,
        package_source=args.package_source,
    )


if __name__ == "__main__":
    main()