DSPy: The framework for programming—not prompting—language models
Python
#!/usr/bin/env python3
"""Publish a prebuilt documentation site through Zensical's Mike version store."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import subprocess
from contextlib import contextmanager
from html import escape
from pathlib import Path
RELEASE_VERSION = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?$")
STABLE_VERSION = re.compile(r"^\d+\.\d+\.\d+$")
DEFAULT_BRANCH = "versioned-docs"
LEGACY_REDIRECTS_MANIFEST = ".dspy-legacy-redirects.json"
HOST_CONFIG = (
json.dumps(
{
"framework": None,
"buildCommand": "true",
"outputDirectory": ".",
"redirects": [
{
"source": "/:version(\\d+\\.\\d+(?:\\.\\d+(?:(?:a|b|rc)\\d+)?)?)",
"destination": "/:version/",
"permanent": True,
},
{
"source": "/:path((?:.*/)?[^./]+)",
"destination": "/:path/",
"permanent": True,
},
],
"headers": [
{
"source": "/(.*).md",
"headers": [{"key": "Content-Type", "value": "text/markdown; charset=utf-8"}],
}
],
},
indent=2,
)
+ "\n"
)
def version_tuple(version: str) -> tuple[int, int, int, int, int]:
match = RELEASE_VERSION.fullmatch(version)
if not match or int(match.group(1)) < 3:
raise argparse.ArgumentTypeError("expected an X.Y.Z release version at or after 3.0.0")
major, minor, patch, prerelease, prerelease_number = match.groups()
phase = {"a": 0, "b": 1, "rc": 2, None: 3}[prerelease]
return int(major), int(minor), int(patch), phase, int(prerelease_number or 0)
@contextmanager
def working_directory(path: Path):
original = Path.cwd()
os.chdir(path)
try:
yield
finally:
os.chdir(original)
def tree_digest(root: Path) -> str:
digest = hashlib.sha256()
paths = sorted(
(item for item in root.rglob("*") if item.is_file()),
key=lambda item: item.relative_to(root).as_posix(),
)
for path in paths:
digest.update(path.relative_to(root).as_posix().encode())
digest.update(path.read_bytes())
return digest.hexdigest()
def deployed_tree_digest(repository: Path, branch: str, identifier: str) -> str | None:
result = subprocess.run(
["git", "-C", str(repository), "ls-tree", "-r", "-z", branch, "--", identifier],
check=False,
capture_output=True,
)
if result.returncode != 0 or not result.stdout:
return None
digest = hashlib.sha256()
for record in result.stdout.rstrip(b"\0").split(b"\0"):
metadata, path = record.split(b"\t", 1)
object_id = metadata.split()[2]
relative = path.decode().removeprefix(f"{identifier}/")
digest.update(relative.encode())
digest.update(subprocess.check_output(["git", "-C", str(repository), "cat-file", "blob", object_id.decode()]))
return digest.hexdigest()
def branch_file(repository: Path, branch: str, path: str) -> str | None:
result = subprocess.run(
["git", "-C", str(repository), "show", f"{branch}:{path}"],
check=False,
capture_output=True,
text=True,
)
return result.stdout if result.returncode == 0 else None
def require_current_renderer(repository: Path, branch: str, renderer: str) -> None:
inventory_text = branch_file(repository, branch, "versions.json")
inventory = json.loads(inventory_text) if inventory_text else []
current = next((entry for entry in inventory if entry.get("version") == "current"), None)
actual = current.get("properties", {}).get("renderer") if current else None
if actual != renderer:
raise RuntimeError(f"production Current renderer is {actual!r}, expected {renderer!r}")
def monotonic_aliases(repository: Path, branch: str, identifier: str, aliases: list[str]) -> list[str]:
"""Do not let a delayed older patch move a minor alias backward."""
inventory_text = branch_file(repository, branch, "versions.json")
if not inventory_text:
return aliases
requested = version_tuple(identifier)
inventory = json.loads(inventory_text)
holders = {
alias: version_tuple(entry["version"])
for entry in inventory
if STABLE_VERSION.fullmatch(entry["version"])
for alias in entry.get("aliases", [])
}
return [alias for alias in aliases if alias not in holders or holders[alias] <= requested]
def redirect_document(target: str) -> str:
encoded = json.dumps(target)
escaped = escape(target, quote=True)
return (
'<!doctype html><html lang="en"><head><meta charset="utf-8">'
f'<link rel="canonical" href="{escaped}">'
f"<script>location.replace({encoded}+location.search+location.hash)</script>"
f'<noscript><meta http-equiv="refresh" content="0; url={escaped}"></noscript>'
f'</head><body><a href="{escaped}">Continue to the documentation</a></body></html>'
)
def legacy_redirects(site: Path, identifier: str) -> dict[str, str]:
redirects = {}
for page in sorted(site.rglob("*.html")):
relative = page.relative_to(site)
if relative == Path("index.html"):
continue
if relative.name == "index.html":
route = relative.parent.as_posix().strip("/")
if not route:
continue
destination = f"/{identifier}/{route}/"
else:
route = relative.as_posix()
destination = f"/{identifier}/{route}"
top_level = route.split("/", 1)[0]
if top_level == "current" or RELEASE_VERSION.fullmatch(top_level) or re.fullmatch(r"\d+\.\d+", top_level):
continue
redirects[relative.as_posix()] = redirect_document(destination)
return redirects
def write_legacy_redirects(repository: Path, branch: str, site: Path, identifier: str) -> None:
from mike import git_utils
redirects = legacy_redirects(site, identifier)
previous_text = branch_file(repository, branch, LEGACY_REDIRECTS_MANIFEST)
previous = set(json.loads(previous_text)) if previous_text else set()
manifest = json.dumps(sorted(redirects), indent=2) + "\n"
if previous_text == manifest:
return
with git_utils.Commit(branch, "Update unversioned documentation redirects") as commit:
commit.delete_files(sorted(previous - redirects.keys()))
for path, document in redirects.items():
commit.add_file(git_utils.FileInfo(path, document))
commit.add_file(git_utils.FileInfo(LEGACY_REDIRECTS_MANIFEST, manifest))
def publish_site(
*,
repository: Path,
site: Path,
identifier: str,
aliases: list[str],
package_source: str,
branch: str = DEFAULT_BRANCH,
required_current_renderer: str | None = None,
) -> bool:
from mike import commands, git_utils
if required_current_renderer:
require_current_renderer(repository, branch, required_current_renderer)
current = identifier == "current"
if not current:
# Automated publication is append-only. Intentional corrections to an
# existing snapshot go through review in the deployment repository.
version_tuple(identifier)
aliases = monotonic_aliases(repository, branch, identifier, aliases)
deployed = deployed_tree_digest(repository, branch, identifier)
if deployed:
if deployed != tree_digest(site):
raise RuntimeError(f"immutable Mike snapshot {identifier} already exists with different content")
return False
config = {"site_dir": str(site), "use_directory_urls": True}
message = f"Publish DSPy documentation {identifier}"
with working_directory(repository):
with commands.deploy(
config,
identifier,
"Current" if current else identifier,
aliases,
update_aliases=True,
alias_type=commands.AliasType.redirect,
branch=branch,
message=message,
set_props=[("renderer", "zensical"), ("package_source", package_source)],
):
pass
root = branch_file(repository, branch, "index.html")
if current and (root is None or f"url={identifier}/" not in root):
commands.set_default(
identifier,
branch=branch,
message=f"Set default documentation to {identifier}",
)
if current:
write_legacy_redirects(repository, branch, site, identifier)
if branch_file(repository, branch, "vercel.json") != HOST_CONFIG:
with git_utils.Commit(branch, "Configure static documentation hosting") as commit:
commit.add_file(git_utils.FileInfo("vercel.json", HOST_CONFIG))
return True
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--repository", type=Path, required=True)
parser.add_argument("--site", type=Path, required=True)
parser.add_argument("--identifier", required=True)
parser.add_argument("--alias", action="append", default=[])
parser.add_argument(
"--package-source",
choices=("pypi-wheel", "workflow-wheel", "tag-built-wheel", "working-tree"),
required=True,
)
parser.add_argument("--branch", default=DEFAULT_BRANCH)
parser.add_argument("--require-current-renderer")
args = parser.parse_args()
changed = publish_site(
repository=args.repository.resolve(),
site=args.site.resolve(),
identifier=args.identifier,
aliases=args.alias,
package_source=args.package_source,
branch=args.branch,
required_current_renderer=args.require_current_renderer,
)
print("published" if changed else "already published")
if __name__ == "__main__":
main()