Apache DataFusion SQL Query Engine
docs: publish older version docs from versions.json with a workflow and build script (#26828)
Andrew Lamb10fb0700b24a
diff --git a/.github/actions/setup-docs-builder/action.yaml b/.github/actions/setup-docs-builder/action.yamlnew file mode 100644index 0000000000..380109b722--- /dev/null+++ b/.github/actions/setup-docs-builder/action.yaml@@ -0,0 +1,39 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++name: Prepare Docs Builder+description: 'Install the tools needed to build the documentation in docs'+runs:+ using: "composite"+ steps:+ - name: Setup Rust toolchain+ uses: ./.github/actions/setup-rustup+ - name: Setup uv+ uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2+ - name: Install Python dependencies+ shell: bash+ run: uv sync --package datafusion-docs+ - name: Install Graphviz+ shell: bash+ run: |+ set -x+ sudo apt-get update+ sudo apt-get install -y graphviz+ - name: Install cargo-depgraph+ uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31+ with:+ tool: [email protected]diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yamlindex 292a710508..783ec8f26a 100644--- a/.github/workflows/docs.yaml+++ b/.github/workflows/docs.yaml@@ -26,12 +26,17 @@ on:name: Deploy DataFusion site+# run the publishing workflows one at a time.+concurrency:+ group: publish-asf-site+ cancel-in-progress: false+jobs:build-docs:permissions:contents: writename: Build docs- runs-on: ubuntu-latest+ runs-on: ubuntu-slimsteps:- name: Checkout docs sourcesuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -42,20 +47,8 @@ jobs:ref: asf-sitepath: asf-site- - name: Setup uv- uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2-- - name: Install dependencies- run: uv sync --package datafusion-docs- - name: Install Graphviz- run: |- set -x- sudo apt-get update- sudo apt-get install -y graphviz- - name: Install cargo-depgraph- uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31- with:- tool: [email protected]+ - name: Setup docs builder+ uses: ./.github/actions/setup-docs-builder- name: Build docsrun: |diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yamlindex 571995a108..f62e6c2768 100644--- a/.github/workflows/docs_pr.yaml+++ b/.github/workflows/docs_pr.yaml@@ -51,20 +51,7 @@ jobs:with:submodules: truefetch-depth: 1- - name: Setup Rust toolchain- uses: ./.github/actions/setup-rustup- - name: Setup uv- uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2- - name: Install doc dependencies- run: uv sync --package datafusion-docs- - name: Install Graphviz- run: |- set -x- sudo apt-get update- sudo apt-get install -y graphviz- - name: Install cargo-depgraph- uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31- with:- tool: [email protected]+ - name: Setup docs builder+ uses: ./.github/actions/setup-docs-builder- name: Build docs html and check for warningsrun: ./ci/scripts/check_docs_html.sh # fails on errorsdiff --git a/.github/workflows/docs_versions.yaml b/.github/workflows/docs_versions.yamlnew file mode 100644index 0000000000..a22727847e--- /dev/null+++ b/.github/workflows/docs_versions.yaml@@ -0,0 +1,84 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Publishes the documentation for released versions to+# https://datafusion.apache.org/versions/<version>/+#+# The list of versions is in docs/source/_static/versions.json+#+# The current development docs are published from `main` by docs.yaml.++on:+ push:+ branches:+ - main+ paths:+ - .github/workflows/docs_versions.yaml+ - docs/scripts/build_versions.sh+ - docs/scripts/release/conf.py+ - docs/source/_static/versions.json+ - docs/pyproject.toml+ - uv.lock++name: Deploy DataFusion versioned docs++# run the publishing workflows one at a time.+concurrency:+ group: publish-asf-site+ cancel-in-progress: false++jobs:+ build-versions:+ permissions:+ contents: write+ name: Build versioned docs+ runs-on: ubuntu-slim+ steps:+ - name: Checkout docs sources+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1++ - name: Checkout asf-site branch+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: asf-site+ path: asf-site++ - name: Setup docs builder+ uses: ./.github/actions/setup-docs-builder++ - name: Build versioned docs+ run: |+ set -x+ docs/scripts/build_versions.sh --output docs/build/versions++ - name: Copy & push the generated HTML+ run: |+ set -x+ cd asf-site/+ # versions/ mirrors the releases in versions.json; the current docs+ # at the site root are published by docs.yaml+ mkdir -p versions+ # remove any old versions and replace with newly built ones+ rsync -a --delete ../docs/build/versions/ versions/+ git status --porcelain+ if [ "$(git status --porcelain)" != "" ]; then+ git config user.name "github-actions[bot]"+ git config user.email "github-actions[bot]@users.noreply.github.com"+ git add --all+ git commit -m 'Publish versioned docs triggered by ${{ github.sha }}'+ git push+ fidiff --git a/dev/release/README.md b/dev/release/README.mdindex 71714ae1d9..cc2501a14c 100644--- a/dev/release/README.md+++ b/dev/release/README.md@@ -311,71 +311,27 @@ git push apache 55.2.0#### Publish the versioned documentation-**1. Build the documentation from the release tag** using the configuration-overlay (`docs/scripts/release/conf.py`) from the main checkout. For example,-for `55.0.0`, from the repository root:+Open a PR against `main` that adds the release to+`docs/source/_static/versions.json` immediately after the `Development (main)`+entry mark it with `"preferred": true`.-```shell-export DATAFUSION_DOCS_OVERLAY="$PWD/docs/scripts/release"-git fetch apache tag 55.0.0-git worktree add --detach /tmp/datafusion-55.0.0 55.0.0-cd /tmp/datafusion-55.0.0/docs-DATAFUSION_DOCS_SOURCE="$PWD/source" DATAFUSION_DOCS_VERSION=55.0.0 \- SPHINXOPTS="-W -c $DATAFUSION_DOCS_OVERLAY" \- uv run --package datafusion-docs --with sphinx-sitemap ./build.sh-cd --```--Notes:--1. You need to use an absolute path to the overlay in your current checkout.-2. The `--with sphinx-sitemap` option supplies the sitemap extension for older tags.--To preview the result, build the docs from your main checkout-(`uv run --package datafusion-docs ./build.sh` in `docs/`), copy the release-build in under `versions/<version>/`, and serve the tree over HTTP (the version-picker does not work when the pages are opened directly as files):--```shell-mkdir -p docs/build/html/versions/55.0.0-rsync -a /tmp/datafusion-55.0.0/docs/build/html/ docs/build/html/versions/55.0.0/-python3 -m http.server --directory docs/build/html 8000-```--Then open http://localhost:8000/ and http://localhost:8000/versions/55.0.0/-and check the version picker in the navigation bar.+Once the PR is merged, the+[Deploy DataFusion versioned docs](../../.github/workflows/docs_versions.yaml)+workflow builds the documentation for each release and+pushes the result to the `asf-site` branch.-**2. Add the built site to the `asf-site` branch.** Open a PR against-`asf-site` that adds the release under `versions/<version>/` (for example-[#25881](https://github.com/apache/datafusion/pull/25881)):+To preview a release locally before opening the PR, build the current site+and then the release with `docs/scripts/build_versions.sh <version>` and serve+`docs/build/html` over HTTP:```shell-git fetch apache asf-site-git worktree add -b docs/publish-55.0.0 /tmp/datafusion-asf-site apache/asf-site-rsync -a --exclude '/.buildinfo' \- /tmp/datafusion-55.0.0/docs/build/html/ \- /tmp/datafusion-asf-site/versions/55.0.0/-cd /tmp/datafusion-asf-site-git add versions/55.0.0-git commit -m "Publish documentation for 55.0.0"-git push origin docs/publish-55.0.0+cd docs+uv run --package datafusion-docs ./build.sh+scripts/build_versions.sh 55.2.0+python3 -m http.server --directory build/html 8000```-The release is live at `https://datafusion.apache.org/versions/55.0.0/` once-the PR is merged.--**3. Add the release to the version picker.** Open a PR against `main` that-adds the release to `docs/source/_static/versions.json`. Put the new release-directly after the `Development (main)` entry and move `"preferred": true` to it, so that-it is treated as the current stable release. Only add releases that are already-published, since the picker links to them directly.--Finally, remove the temporary worktrees:--```shell-git worktree remove /tmp/datafusion-55.0.0-git worktree remove /tmp/datafusion-asf-site-```+Then open http://localhost:8000/ and check the version picker in the navigation bar.### 10. Publish on Crates.iodiff --git a/docs/README.md b/docs/README.mdindex b55d770cdc..a94d2bc109 100644--- a/docs/README.md+++ b/docs/README.md@@ -88,3 +88,8 @@ The Apache Software Foundation provides https://datafusion.apache.org/,which serves content based on the configuration in[.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml),which specifies the target as https://datafusion.apache.org/.++The documentation for previously released versions is served under+`/versions/<version>/`. See the+[release instructions](../dev/release/README.md#publish-the-versioned-documentation)+for how it is published.diff --git a/docs/scripts/build_versions.sh b/docs/scripts/build_versions.shnew file mode 100755index 0000000000..04636471a4--- /dev/null+++ b/docs/scripts/build_versions.sh@@ -0,0 +1,201 @@+#!/usr/bin/env bash+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Build the documentation for each release listed in docs/source/_static/versions.json+#+# The release is checked out into a temporary worktree, writing the result+# to <output>/<version>/.++set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+DOCS_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"+REPO_DIR="$(cd "${DOCS_DIR}/.." && pwd)"+OVERLAY_DIR="${SCRIPT_DIR}/release"+MANIFEST="${DOCS_DIR}/source/_static/versions.json"++# Where to fetch release tags from (forks usually do not contain release tags)+UPSTREAM_URL="${DATAFUSION_UPSTREAM_URL:-https://github.com/apache/datafusion.git}"++OUTPUT_DIR="${DOCS_DIR}/build/html/versions"+WORK_DIR=""+VERSIONS=()++usage() {+ cat <<USAGE+Build the documentation for released DataFusion versions.++Usage: $(basename "$0") [options] [VERSION ...]++Builds the documentation for each VERSION from its git tag. With no VERSION,+builds every release listed in ${MANIFEST#"${REPO_DIR}/"}.++Options:+ -o, --output DIR Write the documentation to DIR/<version>/+ (default: docs/build/html/versions, so that the versions+ appear under a site built with docs/build.sh)+ -w, --work DIR Check the tags out under DIR/<version> instead of a+ temporary directory+ -h, --help Show this help message++Environment:+ DATAFUSION_UPSTREAM_URL Repository to fetch missing release tags from+ (default: ${UPSTREAM_URL})++Example: preview the site with the 55.0.0 docs under versions/55.0.0/+ cd docs+ uv run --package datafusion-docs ./build.sh+ scripts/build_versions.sh 55.0.0+ python3 -m http.server --directory build/html 8000+USAGE+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ -o|--output)+ OUTPUT_DIR="$2"+ shift 2+ ;;+ -w|--work)+ WORK_DIR="$2"+ shift 2+ ;;+ -h|--help)+ usage+ exit 0+ ;;+ -*)+ echo "Unknown option: $1" >&2+ usage >&2+ exit 1+ ;;+ *)+ VERSIONS+=("$1")+ shift+ ;;+ esac+done++for tool in git uv python3; do+ if ! command -v "${tool}" >/dev/null 2>&1; then+ echo "${tool} is required." >&2+ exit 1+ fi+done++# Releases listed in the version picker manifest, newest first.+list_versions() {+ python3 - "${MANIFEST}" <<'PY'+import json+import sys++with open(sys.argv[1]) as f:+ for entry in json.load(f):+ if entry["version"] != "main":+ print(entry["version"])+PY+}++if [[ ${#VERSIONS[@]} -eq 0 ]]; then+ while IFS= read -r version; do+ VERSIONS+=("${version}")+ done < <(list_versions)+fi++if [[ ${#VERSIONS[@]} -eq 0 ]]; then+ echo "No versions to build." >&2+ exit 1+fi++TEMP_WORK_DIR=""+if [[ -z "${WORK_DIR}" ]]; then+ TEMP_WORK_DIR="$(mktemp -d -t datafusion-docs-versions.XXXXXX)"+ WORK_DIR="${TEMP_WORK_DIR}"+fi+mkdir -p "${WORK_DIR}" "${OUTPUT_DIR}"+WORK_DIR="$(cd "${WORK_DIR}" && pwd)"+OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)"++cleanup() {+ if [[ -n "${TEMP_WORK_DIR}" ]]; then+ for worktree in "${TEMP_WORK_DIR}"/*/; do+ [[ -d "${worktree}" ]] || continue+ git -C "${REPO_DIR}" worktree remove --force "${worktree}" >/dev/null 2>&1 || true+ done+ rm -rf "${TEMP_WORK_DIR}"+ git -C "${REPO_DIR}" worktree prune+ fi+}+trap cleanup EXIT++# Make sure this checkout has the release tag, fetching it from upstream+# otherwise (a shallow CI checkout stays shallow).+fetch_tag() {+ local version="$1"+ if git -C "${REPO_DIR}" rev-parse -q --verify "refs/tags/${version}^{commit}" >/dev/null; then+ return+ fi+ echo "Fetching tag ${version} from ${UPSTREAM_URL}"+ local depth=()+ if [[ "$(git -C "${REPO_DIR}" rev-parse --is-shallow-repository)" == "true" ]]; then+ depth=(--depth 1)+ fi+ git -C "${REPO_DIR}" fetch "${depth[@]}" "${UPSTREAM_URL}" "refs/tags/${version}:refs/tags/${version}"+}++build_version() {+ local version="$1"+ local worktree="${WORK_DIR}/${version}"+ local output="${OUTPUT_DIR}/${version}"++ echo "==> Building documentation for ${version}"+ fetch_tag "${version}"++ if [[ -e "${worktree}" ]]; then+ git -C "${REPO_DIR}" worktree remove --force "${worktree}" >/dev/null 2>&1 || rm -rf "${worktree}"+ fi+ git -C "${REPO_DIR}" worktree add --detach "${worktree}" "refs/tags/${version}"++ # The workspace dependency graph is generated at build time and embedded in+ # the contributor guide (docs/build.sh does the same for the current docs).+ if [[ -x "${worktree}/docs/scripts/generate_dependency_graph.sh" ]]; then+ "${worktree}/docs/scripts/generate_dependency_graph.sh"+ fi++ # Sphinx's incremental build state (doctrees, .buildinfo) is not part of the+ # published site: keep the doctrees in the worktree, which is removed below.+ rm -rf "${output}"+ (+ cd "${DOCS_DIR}"+ DATAFUSION_DOCS_SOURCE="${worktree}/docs/source" \+ DATAFUSION_DOCS_VERSION="${version}" \+ uv run --package datafusion-docs \+ sphinx-build -b html -W -c "${OVERLAY_DIR}" \+ -d "${worktree}/docs/build/doctrees" \+ "${worktree}/docs/source" "${output}"+ )+ rm -f "${output}/.buildinfo"++ git -C "${REPO_DIR}" worktree remove --force "${worktree}"+ echo "==> Wrote ${output}"+}++for version in "${VERSIONS[@]}"; do+ build_version "${version}"+donediff --git a/docs/scripts/release/conf.py b/docs/scripts/release/conf.pyindex a4281e1726..8b706b08cc 100644--- a/docs/scripts/release/conf.py+++ b/docs/scripts/release/conf.py@@ -21,6 +21,8 @@ the release process builds docs with the following command from a maincheckout:sphinx-build -c docs/scripts/release++See docs/scripts/build_versions.sh for building past releases"""import os