DSPy: The framework for programming—not prompting—language models
Python
#!/usr/bin/env python3
"""Build Current or one immutable documentation release snapshot."""
from __future__ import annotations
import argparse
import hashlib
import importlib.metadata
import json
import os
import re
import subprocess
import sys
import tempfile
from pathlib import Path
if __package__:
from .zensical_build import HeadParser, build_zensical_site
else: # Direct script execution from the deployed docs repository.
from zensical_build import HeadParser, build_zensical_site
RELEASE_VERSION = re.compile(
r"^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:(?:a|b|rc)\d+)?$"
)
SOURCE_MAP_COMMENT = re.compile(rb"(?:\n?//# sourceMappingURL=[^\r\n]*|/\*# sourceMappingURL=.*?\*/)", re.DOTALL)
ROOT_URL_ATTRIBUTE = re.compile(
r'(?P<prefix>\b(?:href|src|action)=["\x27])(?P<url>/(?!/)[^"\x27]*)(?P<suffix>["\x27])',
re.IGNORECASE,
)
VERSIONED_PATH = re.compile(r"^/(?:current|\d+\.\d+(?:\.\d+(?:(?:a|b|rc)\d+)?)?)(?:/|$)")
RELEASE_BADGE_VERSION = re.compile(
r'(?s)(<div class=(?:"hp-hero-badge"|hp-hero-badge)>.*?\bDSPy )(\S+)(\s+—)'
)
SHARED_HEADER_ASSETS = Path(__file__).parent.parent / "versioning"
def release_version(value: str) -> str:
match = RELEASE_VERSION.fullmatch(value)
if not match or int(match.group("major")) < 3:
raise argparse.ArgumentTypeError("expected an X.Y.Z release version at or after 3.0.0")
return value
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def git_value(repository: Path, *arguments: str) -> str:
return subprocess.check_output(["git", "-C", str(repository), *arguments], text=True).strip()
def patched_config(config: Path, identifier: str, *, edit_ref: str | None = None) -> Path:
text = config.read_text()
text, site_count = re.subn(r"(?m)^site_url:\s*.*$", f"site_url: https://dspy.ai/{identifier}/", text, count=1)
if site_count != 1:
raise RuntimeError(f"could not patch site_url in {config}")
if edit_ref:
text, edit_count = re.subn(r"(?m)^edit_uri:\s*.*$", f"edit_uri: blob/{edit_ref}/docs/docs/", text, count=1)
if edit_count != 1:
raise RuntimeError(f"could not patch edit_uri in {config}")
# Release API docs must import the installed wheel, not this checkout.
text = re.sub(r'(?m)^[ \t]+paths:\s*\[\s*["\x27]?\.\.["\x27]?\s*\][ \t]*\n', "", text, count=1)
if not re.search(r"(?m)^\s+provider:\s*mike\s*$", text):
version_config = " version:\n provider: mike\n alias: true\n"
text, extra_count = re.subn(r"(?m)^extra:\s*$", f"extra:\n{version_config}", text, count=1)
if extra_count == 0:
text = f"{text.rstrip()}\n\nextra:\n{version_config}"
handle, name = tempfile.mkstemp(prefix=f".mkdocs-{identifier}-", suffix=".yml", dir=config.parent)
os.close(handle)
output = Path(name)
output.write_text(text)
return output
def remove_source_maps(site: Path) -> dict[str, int]:
"""Remove browser source maps after Zensical performs native minification."""
before = sum(path.stat().st_size for path in site.rglob("*") if path.is_file())
source_maps = list(site.rglob("*.map"))
for path in source_maps:
path.unlink()
for pattern in ("*.js", "*.css"):
for path in site.rglob(pattern):
source = path.read_bytes()
optimized = SOURCE_MAP_COMMENT.sub(b"", source)
if optimized != source:
path.write_bytes(optimized)
after = sum(path.stat().st_size for path in site.rglob("*") if path.is_file())
return {"before": before, "after": after, "source_maps": len(source_maps)}
def install_shared_header(site: Path, source: Path = SHARED_HEADER_ASSETS) -> None:
"""Give every renderer and historical snapshot the same header controls."""
destination = site / "_static"
destination.mkdir(parents=True, exist_ok=True)
styles = destination / "dspy-header.css"
script = destination / "dspy-header.js"
styles.write_bytes((source / "header.css").read_bytes())
script.write_bytes((source / "header.js").read_bytes())
for page in site.rglob("*.html"):
relative_styles = Path(os.path.relpath(styles, page.parent)).as_posix()
relative_script = Path(os.path.relpath(script, page.parent)).as_posix()
style_tag = f'<link rel="stylesheet" href="{relative_styles}">'
script_tag = f'<script src="{relative_script}" defer></script>'
html = page.read_text()
if style_tag not in html:
html = html.replace("</head>", f"{style_tag}</head>", 1)
if script_tag not in html:
html = html.replace("</head>", f"{script_tag}</head>", 1)
page.write_text(html)
def scope_root_relative_urls(site: Path, identifier: str) -> None:
"""Keep hand-authored root-relative links inside the selected version."""
def replace(match: re.Match[str]) -> str:
url = match.group("url")
if VERSIONED_PATH.match(url):
return match.group(0)
scoped = f"/{identifier}{url}"
return f"{match.group('prefix')}{scoped}{match.group('suffix')}"
for page in site.rglob("*.html"):
html = page.read_text()
scoped = ROOT_URL_ATTRIBUTE.sub(replace, html)
if scoped != html:
page.write_text(scoped)
def set_release_badge_version(site: Path, version: str) -> None:
"""Keep a historical home page from advertising the latest live release."""
home = site / "index.html"
html = home.read_text()
updated, count = RELEASE_BADGE_VERSION.subn(
lambda match: f"{match.group(1)}{version}{match.group(3)}",
html,
count=1,
)
if count:
home.write_text(updated)
def installed_packages() -> dict[str, str]:
return dict(
sorted(
(distribution.metadata["Name"], distribution.version)
for distribution in importlib.metadata.distributions()
if distribution.metadata["Name"]
)
)
def renderer_version() -> str:
return importlib.metadata.version("zensical")
def validate_release_site(site: Path, config: Path, version: str) -> None:
expected = (site / "index.html", site / "api" / "index.html", site / "search.json", site / "llms.txt")
missing = [str(path.relative_to(site)) for path in expected if not path.exists()]
if missing:
raise RuntimeError(f"release site is missing required output: {', '.join(missing)}")
home = (site / "index.html").read_text()
metadata = HeadParser()
metadata.feed(home)
canonical = f"https://dspy.ai/{version}/"
if metadata.metadata.get("canonical") != canonical:
raise RuntimeError(f"release home page is missing canonical URL {canonical}")
docs_dir = config.parent / "docs"
missing_notebooks = []
for notebook in docs_dir.rglob("*.ipynb"):
relative = notebook.relative_to(docs_dir)
outputs = (
site / relative.with_suffix(".html"),
site / relative.with_suffix("") / "index.html",
)
if not any(output.exists() for output in outputs):
missing_notebooks.append(str(notebook.relative_to(docs_dir)))
if missing_notebooks:
raise RuntimeError(f"notebooks were not rendered: {', '.join(missing_notebooks)}")
cards = site / "assets" / "images" / "social-zensical"
if not cards.exists() or not any(cards.rglob("*.png")) or "og:image" not in metadata.metadata:
raise RuntimeError("social cards or Open Graph metadata were not generated")
try:
dspy_version = importlib.metadata.version("dspy")
except importlib.metadata.PackageNotFoundError as error:
raise RuntimeError("release build environment does not contain DSPy") from error
if dspy_version != version:
raise RuntimeError(f"release build imported dspy=={dspy_version}, expected {version}")
def build(
*,
config: Path,
output: Path,
version: str | None = None,
artifact: Path | None = None,
package_source: str | None = None,
) -> None:
output = output.resolve()
output.parent.mkdir(parents=True, exist_ok=True)
identifier = version or "current"
effective_config = patched_config(config, identifier, edit_ref=version)
try:
build_zensical_site(config=effective_config, output=output, python=Path(sys.executable))
finally:
effective_config.unlink(missing_ok=True)
if version:
if artifact is None or package_source is None:
raise ValueError("release builds require an artifact and package source")
set_release_badge_version(output, version)
scope_root_relative_urls(output, identifier)
if version:
validate_release_site(output, config, version)
install_shared_header(output)
optimization = remove_source_maps(output)
if version:
repository = config.parent.parent
source_commit = git_value(repository, "rev-list", "-n", "1", version)
metadata = {
"version": version,
"source_tag": version,
"source_commit": source_commit,
"source_commit_time": git_value(repository, "show", "-s", "--format=%cI", source_commit),
"renderer": "zensical",
"renderer_version": renderer_version(),
"package_source": package_source,
"package_artifact": artifact.name,
"package_sha256": sha256(artifact),
"optimization": optimization,
"python": sys.version.split()[0],
"packages": installed_packages(),
"intentional_differences": [
"The renderer's Mike version selector is enabled in a transient build configuration.",
"Build provenance metadata added under _meta/build.json.",
"Generated HTML is minified and source maps are omitted from production snapshots.",
],
}
metadata_dir = output / "_meta"
metadata_dir.mkdir(exist_ok=True)
(metadata_dir / "build.json").write_text(json.dumps(metadata, indent=2) + "\n")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=("current", "release"))
parser.add_argument("--config", type=Path, default=Path("mkdocs.yml"))
parser.add_argument("--output", type=Path, default=Path("site"))
parser.add_argument("--version", type=release_version)
parser.add_argument("--artifact", type=Path)
parser.add_argument("--package-source", choices=("pypi-wheel", "workflow-wheel", "tag-built-wheel"))
args = parser.parse_args()
if args.mode == "release" and not (args.version and args.artifact and args.package_source):
parser.error("release requires --version, --artifact, and --package-source")
return args
def main() -> None:
args = parse_args()
build(
config=args.config.resolve(),
output=args.output,
version=args.version if args.mode == "release" else None,
artifact=args.artifact.resolve() if args.artifact else None,
package_source=args.package_source,
)
if __name__ == "__main__":
main()