Skip to content

Apache DataFusion SQL Query Engine

docs: publish older version docs from versions.json with a workflow and build script (#26828)

Andrew Lamb10fb0700b24a
diff --git a/.github/actions/setup-docs-builder/action.yaml b/.github/actions/setup-docs-builder/action.yaml
new file mode 100644
index 0000000000..380109b722
--- /dev/null
+++ b/.github/actions/setup-docs-builder/action.yaml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Docs Builder
+description: 'Install the tools needed to build the documentation in docs'
+runs:
+ using: "composite"
+ steps:
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-rustup
+ - name: Setup uv
+ uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+ - name: Install Python dependencies
+ shell: bash
+ run: uv sync --package datafusion-docs
+ - name: Install Graphviz
+ shell: bash
+ run: |
+ set -x
+ sudo apt-get update
+ sudo apt-get install -y graphviz
+ - name: Install cargo-depgraph
+ uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
+ with:
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 292a710508..783ec8f26a 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -26,12 +26,17 @@ on:
name: Deploy DataFusion site
+# run the publishing workflows one at a time.
+concurrency:
+ group: publish-asf-site
+ cancel-in-progress: false
+
jobs:
build-docs:
permissions:
contents: write
name: Build docs
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
steps:
- name: Checkout docs sources
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -42,20 +47,8 @@ jobs:
ref: asf-site
path: asf-site
- - name: Setup uv
- uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
-
- - name: Install dependencies
- run: uv sync --package datafusion-docs
- - name: Install Graphviz
- run: |
- set -x
- sudo apt-get update
- sudo apt-get install -y graphviz
- - name: Install cargo-depgraph
- uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
- with:
+ - name: Setup docs builder
+ uses: ./.github/actions/setup-docs-builder
- name: Build docs
run: |
diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml
index 571995a108..f62e6c2768 100644
--- a/.github/workflows/docs_pr.yaml
+++ b/.github/workflows/docs_pr.yaml
@@ -51,20 +51,7 @@ jobs:
with:
submodules: true
fetch-depth: 1
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup-rustup
- - name: Setup uv
- uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
- - name: Install doc dependencies
- run: uv sync --package datafusion-docs
- - name: Install Graphviz
- run: |
- set -x
- sudo apt-get update
- sudo apt-get install -y graphviz
- - name: Install cargo-depgraph
- uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
- with:
+ - name: Setup docs builder
+ uses: ./.github/actions/setup-docs-builder
- name: Build docs html and check for warnings
run: ./ci/scripts/check_docs_html.sh # fails on errors
diff --git a/.github/workflows/docs_versions.yaml b/.github/workflows/docs_versions.yaml
new file mode 100644
index 0000000000..a22727847e
--- /dev/null
+++ b/.github/workflows/docs_versions.yaml
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Publishes the documentation for released versions to
+# https://datafusion.apache.org/versions/<version>/
+#
+# The list of versions is in docs/source/_static/versions.json
+#
+# The current development docs are published from `main` by docs.yaml.
+
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - .github/workflows/docs_versions.yaml
+ - docs/scripts/build_versions.sh
+ - docs/scripts/release/conf.py
+ - docs/source/_static/versions.json
+ - docs/pyproject.toml
+ - uv.lock
+
+name: Deploy DataFusion versioned docs
+
+# run the publishing workflows one at a time.
+concurrency:
+ group: publish-asf-site
+ cancel-in-progress: false
+
+jobs:
+ build-versions:
+ permissions:
+ contents: write
+ name: Build versioned docs
+ runs-on: ubuntu-slim
+ steps:
+ - name: Checkout docs sources
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+ - name: Checkout asf-site branch
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: asf-site
+ path: asf-site
+
+ - name: Setup docs builder
+ uses: ./.github/actions/setup-docs-builder
+
+ - name: Build versioned docs
+ run: |
+ set -x
+ docs/scripts/build_versions.sh --output docs/build/versions
+
+ - name: Copy & push the generated HTML
+ run: |
+ set -x
+ cd asf-site/
+ # versions/ mirrors the releases in versions.json; the current docs
+ # at the site root are published by docs.yaml
+ mkdir -p versions
+ # remove any old versions and replace with newly built ones
+ rsync -a --delete ../docs/build/versions/ versions/
+ git status --porcelain
+ if [ "$(git status --porcelain)" != "" ]; then
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+ git add --all
+ git commit -m 'Publish versioned docs triggered by ${{ github.sha }}'
+ git push
+ fi
diff --git a/dev/release/README.md b/dev/release/README.md
index 71714ae1d9..cc2501a14c 100644
--- a/dev/release/README.md
+++ b/dev/release/README.md
@@ -311,71 +311,27 @@ git push apache 55.2.0
#### Publish the versioned documentation
-**1. Build the documentation from the release tag** using the configuration
-overlay (`docs/scripts/release/conf.py`) from the main checkout. For example,
-for `55.0.0`, from the repository root:
+Open a PR against `main` that adds the release to
+`docs/source/_static/versions.json` immediately after the `Development (main)`
+entry mark it with `"preferred": true`.
-```shell
-export DATAFUSION_DOCS_OVERLAY="$PWD/docs/scripts/release"
-git fetch apache tag 55.0.0
-git worktree add --detach /tmp/datafusion-55.0.0 55.0.0
-cd /tmp/datafusion-55.0.0/docs
-DATAFUSION_DOCS_SOURCE="$PWD/source" DATAFUSION_DOCS_VERSION=55.0.0 \
- SPHINXOPTS="-W -c $DATAFUSION_DOCS_OVERLAY" \
- uv run --package datafusion-docs --with sphinx-sitemap ./build.sh
-cd -
-```
-
-Notes:
-
-1. You need to use an absolute path to the overlay in your current checkout.
-2. The `--with sphinx-sitemap` option supplies the sitemap extension for older tags.
-
-To preview the result, build the docs from your main checkout
-(`uv run --package datafusion-docs ./build.sh` in `docs/`), copy the release
-build in under `versions/<version>/`, and serve the tree over HTTP (the version
-picker does not work when the pages are opened directly as files):
-
-```shell
-mkdir -p docs/build/html/versions/55.0.0
-rsync -a /tmp/datafusion-55.0.0/docs/build/html/ docs/build/html/versions/55.0.0/
-python3 -m http.server --directory docs/build/html 8000
-```
-
-Then open http://localhost:8000/ and http://localhost:8000/versions/55.0.0/
-and check the version picker in the navigation bar.
+Once the PR is merged, the
+[Deploy DataFusion versioned docs](../../.github/workflows/docs_versions.yaml)
+workflow builds the documentation for each release and
+pushes the result to the `asf-site` branch.
-**2. Add the built site to the `asf-site` branch.** Open a PR against
-`asf-site` that adds the release under `versions/<version>/` (for example
-[#25881](https://github.com/apache/datafusion/pull/25881)):
+To preview a release locally before opening the PR, build the current site
+and then the release with `docs/scripts/build_versions.sh <version>` and serve
+`docs/build/html` over HTTP:
```shell
-git fetch apache asf-site
-git worktree add -b docs/publish-55.0.0 /tmp/datafusion-asf-site apache/asf-site
-rsync -a --exclude '/.buildinfo' \
- /tmp/datafusion-55.0.0/docs/build/html/ \
- /tmp/datafusion-asf-site/versions/55.0.0/
-cd /tmp/datafusion-asf-site
-git add versions/55.0.0
-git commit -m "Publish documentation for 55.0.0"
-git push origin docs/publish-55.0.0
+cd docs
+uv run --package datafusion-docs ./build.sh
+scripts/build_versions.sh 55.2.0
+python3 -m http.server --directory build/html 8000
```
-The release is live at `https://datafusion.apache.org/versions/55.0.0/` once
-the PR is merged.
-
-**3. Add the release to the version picker.** Open a PR against `main` that
-adds the release to `docs/source/_static/versions.json`. Put the new release
-directly after the `Development (main)` entry and move `"preferred": true` to it, so that
-it is treated as the current stable release. Only add releases that are already
-published, since the picker links to them directly.
-
-Finally, remove the temporary worktrees:
-
-```shell
-git worktree remove /tmp/datafusion-55.0.0
-git worktree remove /tmp/datafusion-asf-site
-```
+Then open http://localhost:8000/ and check the version picker in the navigation bar.
### 10. Publish on Crates.io
diff --git a/docs/README.md b/docs/README.md
index b55d770cdc..a94d2bc109 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -88,3 +88,8 @@ The Apache Software Foundation provides https://datafusion.apache.org/,
which serves content based on the configuration in
[.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml),
which specifies the target as https://datafusion.apache.org/.
+
+The documentation for previously released versions is served under
+`/versions/<version>/`. See the
+[release instructions](../dev/release/README.md#publish-the-versioned-documentation)
+for how it is published.
diff --git a/docs/scripts/build_versions.sh b/docs/scripts/build_versions.sh
new file mode 100755
index 0000000000..04636471a4
--- /dev/null
+++ b/docs/scripts/build_versions.sh
@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build the documentation for each release listed in docs/source/_static/versions.json
+#
+# The release is checked out into a temporary worktree, writing the result
+# to <output>/<version>/.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DOCS_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+REPO_DIR="$(cd "${DOCS_DIR}/.." && pwd)"
+OVERLAY_DIR="${SCRIPT_DIR}/release"
+MANIFEST="${DOCS_DIR}/source/_static/versions.json"
+
+# Where to fetch release tags from (forks usually do not contain release tags)
+UPSTREAM_URL="${DATAFUSION_UPSTREAM_URL:-https://github.com/apache/datafusion.git}"
+
+OUTPUT_DIR="${DOCS_DIR}/build/html/versions"
+WORK_DIR=""
+VERSIONS=()
+
+usage() {
+ cat <<USAGE
+Build the documentation for released DataFusion versions.
+
+Usage: $(basename "$0") [options] [VERSION ...]
+
+Builds the documentation for each VERSION from its git tag. With no VERSION,
+builds every release listed in ${MANIFEST#"${REPO_DIR}/"}.
+
+Options:
+ -o, --output DIR Write the documentation to DIR/<version>/
+ (default: docs/build/html/versions, so that the versions
+ appear under a site built with docs/build.sh)
+ -w, --work DIR Check the tags out under DIR/<version> instead of a
+ temporary directory
+ -h, --help Show this help message
+
+Environment:
+ DATAFUSION_UPSTREAM_URL Repository to fetch missing release tags from
+ (default: ${UPSTREAM_URL})
+
+Example: preview the site with the 55.0.0 docs under versions/55.0.0/
+ cd docs
+ uv run --package datafusion-docs ./build.sh
+ scripts/build_versions.sh 55.0.0
+ python3 -m http.server --directory build/html 8000
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -o|--output)
+ OUTPUT_DIR="$2"
+ shift 2
+ ;;
+ -w|--work)
+ WORK_DIR="$2"
+ shift 2
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ -*)
+ echo "Unknown option: $1" >&2
+ usage >&2
+ exit 1
+ ;;
+ *)
+ VERSIONS+=("$1")
+ shift
+ ;;
+ esac
+done
+
+for tool in git uv python3; do
+ if ! command -v "${tool}" >/dev/null 2>&1; then
+ echo "${tool} is required." >&2
+ exit 1
+ fi
+done
+
+# Releases listed in the version picker manifest, newest first.
+list_versions() {
+ python3 - "${MANIFEST}" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1]) as f:
+ for entry in json.load(f):
+ if entry["version"] != "main":
+ print(entry["version"])
+PY
+}
+
+if [[ ${#VERSIONS[@]} -eq 0 ]]; then
+ while IFS= read -r version; do
+ VERSIONS+=("${version}")
+ done < <(list_versions)
+fi
+
+if [[ ${#VERSIONS[@]} -eq 0 ]]; then
+ echo "No versions to build." >&2
+ exit 1
+fi
+
+TEMP_WORK_DIR=""
+if [[ -z "${WORK_DIR}" ]]; then
+ TEMP_WORK_DIR="$(mktemp -d -t datafusion-docs-versions.XXXXXX)"
+ WORK_DIR="${TEMP_WORK_DIR}"
+fi
+mkdir -p "${WORK_DIR}" "${OUTPUT_DIR}"
+WORK_DIR="$(cd "${WORK_DIR}" && pwd)"
+OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)"
+
+cleanup() {
+ if [[ -n "${TEMP_WORK_DIR}" ]]; then
+ for worktree in "${TEMP_WORK_DIR}"/*/; do
+ [[ -d "${worktree}" ]] || continue
+ git -C "${REPO_DIR}" worktree remove --force "${worktree}" >/dev/null 2>&1 || true
+ done
+ rm -rf "${TEMP_WORK_DIR}"
+ git -C "${REPO_DIR}" worktree prune
+ fi
+}
+trap cleanup EXIT
+
+# Make sure this checkout has the release tag, fetching it from upstream
+# otherwise (a shallow CI checkout stays shallow).
+fetch_tag() {
+ local version="$1"
+ if git -C "${REPO_DIR}" rev-parse -q --verify "refs/tags/${version}^{commit}" >/dev/null; then
+ return
+ fi
+ echo "Fetching tag ${version} from ${UPSTREAM_URL}"
+ local depth=()
+ if [[ "$(git -C "${REPO_DIR}" rev-parse --is-shallow-repository)" == "true" ]]; then
+ depth=(--depth 1)
+ fi
+ git -C "${REPO_DIR}" fetch "${depth[@]}" "${UPSTREAM_URL}" "refs/tags/${version}:refs/tags/${version}"
+}
+
+build_version() {
+ local version="$1"
+ local worktree="${WORK_DIR}/${version}"
+ local output="${OUTPUT_DIR}/${version}"
+
+ echo "==> Building documentation for ${version}"
+ fetch_tag "${version}"
+
+ if [[ -e "${worktree}" ]]; then
+ git -C "${REPO_DIR}" worktree remove --force "${worktree}" >/dev/null 2>&1 || rm -rf "${worktree}"
+ fi
+ git -C "${REPO_DIR}" worktree add --detach "${worktree}" "refs/tags/${version}"
+
+ # The workspace dependency graph is generated at build time and embedded in
+ # the contributor guide (docs/build.sh does the same for the current docs).
+ if [[ -x "${worktree}/docs/scripts/generate_dependency_graph.sh" ]]; then
+ "${worktree}/docs/scripts/generate_dependency_graph.sh"
+ fi
+
+ # Sphinx's incremental build state (doctrees, .buildinfo) is not part of the
+ # published site: keep the doctrees in the worktree, which is removed below.
+ rm -rf "${output}"
+ (
+ cd "${DOCS_DIR}"
+ DATAFUSION_DOCS_SOURCE="${worktree}/docs/source" \
+ DATAFUSION_DOCS_VERSION="${version}" \
+ uv run --package datafusion-docs \
+ sphinx-build -b html -W -c "${OVERLAY_DIR}" \
+ -d "${worktree}/docs/build/doctrees" \
+ "${worktree}/docs/source" "${output}"
+ )
+ rm -f "${output}/.buildinfo"
+
+ git -C "${REPO_DIR}" worktree remove --force "${worktree}"
+ echo "==> Wrote ${output}"
+}
+
+for version in "${VERSIONS[@]}"; do
+ build_version "${version}"
+done
diff --git a/docs/scripts/release/conf.py b/docs/scripts/release/conf.py
index a4281e1726..8b706b08cc 100644
--- a/docs/scripts/release/conf.py
+++ b/docs/scripts/release/conf.py
@@ -21,6 +21,8 @@ the release process builds docs with the following command from a main
checkout:
sphinx-build -c docs/scripts/release
+
+See docs/scripts/build_versions.sh for building past releases
"""
import os