Skip to content

Apache DataFusion SQL Query Engine

dev: include generated config and function docs in the local lint suite (#25525)

Kumar Ujjawal99af44a5e620
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index a2314098f8..73d1ff3de5 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -855,15 +855,9 @@ jobs:
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "20"
- - name: Check if configs.md has been modified
+ - name: Check generated config and function docs
run: |
- # If you encounter an error, run './dev/update_config_docs.sh' and commit
- ./dev/update_config_docs.sh
- git diff --exit-code
- - name: Check if any of the ***_functions.md has been modified
- run: |
- # If you encounter an error, run './dev/update_function_docs.sh' and commit
- ./dev/update_function_docs.sh
+ ./ci/scripts/check_generated_docs.sh
git diff --exit-code
# This job ensures `datafusion-examples/README.md` stays in sync with the source code:
diff --git a/ci/scripts/check_generated_docs.sh b/ci/scripts/check_generated_docs.sh
new file mode 100755
index 0000000000..fc9d31c745
--- /dev/null
+++ b/ci/scripts/check_generated_docs.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Regenerates the configuration and function documentation pages and checks
+# that the committed pages match, the same way the "check configs.md and
+# ***_functions.md is up-to-date" job does. With `--write`, replaces the pages
+# with the generated ones.
+
+set -euo pipefail
+
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+source "${SCRIPT_DIR}/utils/git.sh"
+
+CONFIG_DOCS_DIR="docs/source/user-guide"
+FUNCTION_DOCS_DIR="docs/source/user-guide/sql"
+
+MODE="check"
+ALLOW_DIRTY=0
+
+usage() {
+ cat >&2 <<USAGE
+Usage: $0 [--write] [--allow-dirty]
+
+Checks that docs/source/user-guide/configs.md and the aggregate, scalar, and window
+function pages under docs/source/user-guide/sql/ match the output of
+dev/update_config_docs.sh and dev/update_function_docs.sh.
+--write Replace the pages with the generated ones (requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+USAGE
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+cd "${ROOT_DIR}"
+
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
+fi
+
+if ! command -v npx >/dev/null 2>&1; then
+ echo "[${SCRIPT_NAME}] npx is required to run the prettier check. Install Node.js (e.g., brew install node) and re-run." >&2
+ exit 1
+fi
+
+# One scratch directory beneath each documentation directory, so Prettier finds
+# the same configuration as for the committed pages.
+SCRATCH_DIRS=()
+cleanup() {
+ rm -rf ${SCRATCH_DIRS[@]+"${SCRATCH_DIRS[@]}"}
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+CONFIG_SCRATCH="$(mktemp -d "${ROOT_DIR}/${CONFIG_DOCS_DIR}/.config-docs-check.XXXXXX")"
+SCRATCH_DIRS+=("${CONFIG_SCRATCH}")
+FUNCTION_SCRATCH="$(mktemp -d "${ROOT_DIR}/${FUNCTION_DOCS_DIR}/.function-docs-check.XXXXXX")"
+SCRATCH_DIRS+=("${FUNCTION_SCRATCH}")
+
+./dev/update_config_docs.sh --output-dir "${CONFIG_SCRATCH}"
+./dev/update_function_docs.sh --output-dir "${FUNCTION_SCRATCH}"
+
+GENERATED=(
+ "${CONFIG_SCRATCH}/configs.md"
+ "${FUNCTION_SCRATCH}/aggregate_functions.md"
+ "${FUNCTION_SCRATCH}/scalar_functions.md"
+ "${FUNCTION_SCRATCH}/window_functions.md"
+)
+COMMITTED=(
+ "${CONFIG_DOCS_DIR}/configs.md"
+ "${FUNCTION_DOCS_DIR}/aggregate_functions.md"
+ "${FUNCTION_DOCS_DIR}/scalar_functions.md"
+ "${FUNCTION_DOCS_DIR}/window_functions.md"
+)
+
+if [[ "$MODE" == "write" ]]; then
+ for i in "${!COMMITTED[@]}"; do
+ cp "${GENERATED[$i]}" "${COMMITTED[$i]}" || {
+ echo "[${SCRIPT_NAME}] failed to copy the generated page to ${COMMITTED[$i]}" >&2
+ exit 1
+ }
+ echo "✅ ${COMMITTED[$i]} updated."
+ done
+ exit 0
+fi
+
+stale_count=0
+for i in "${!COMMITTED[@]}"; do
+ diff_status=0
+ diff -u -L "${COMMITTED[$i]} (committed)" -L "${COMMITTED[$i]} (generated)" \
+ "${COMMITTED[$i]}" "${GENERATED[$i]}" > "${GENERATED[$i]}.diff" || diff_status=$?
+ case "${diff_status}" in
+ 0)
+ echo "✅ ${COMMITTED[$i]} is up-to-date."
+ ;;
+ 1)
+ stale_count=$((stale_count + 1))
+ echo ""
+ echo "❌ ${COMMITTED[$i]} is out of date."
+ echo "------------------------------------------------------------"
+ cat "${GENERATED[$i]}.diff"
+ echo "------------------------------------------------------------"
+ ;;
+ *)
+ echo "❌ diff exited with status ${diff_status} while comparing ${COMMITTED[$i]}; no comparison result." >&2
+ exit "${diff_status}"
+ ;;
+ esac
+done
+
+if [[ ${stale_count} -gt 0 ]]; then
+ echo ""
+ echo "${stale_count} generated page(s) out of date. To update them, run:"
+ echo ""
+ echo " ./ci/scripts/check_generated_docs.sh --write"
+ exit 1
+fi
diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh
index 77bed06457..74d5fc16d7 100755
--- a/dev/rust_lint.sh
+++ b/dev/rust_lint.sh
@@ -130,6 +130,7 @@ declare -a WRITE_STEPS=(
"ci/scripts/typos_check.sh|true"
"ci/scripts/doc_prettier_check.sh|true"
"ci/scripts/check_examples_docs.sh|true"
+ "ci/scripts/check_generated_docs.sh|true"
)
declare -a READONLY_STEPS=(
diff --git a/dev/update_config_docs.sh b/dev/update_config_docs.sh
index df40c65210..5bba39625f 100755
--- a/dev/update_config_docs.sh
+++ b/dev/update_config_docs.sh
@@ -26,7 +26,39 @@ cd "${ROOT_DIR}"
# Load centralized tool versions
source "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh"
-TARGET_FILE="docs/source/user-guide/configs.md"
+# `--output-dir DIR` writes the generated page into DIR instead of
+# docs/source/user-guide. A relative DIR is taken from the repository root.
+OUTPUT_DIR="docs/source/user-guide"
+
+usage() {
+ cat >&2 <<USAGE
+Usage: $0 [--output-dir DIR]
+
+Regenerates docs/source/user-guide/configs.md.
+--output-dir DIR Write the generated page into DIR instead of docs/source/user-guide (relative to the repository root).
+USAGE
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --output-dir)
+ [[ $# -ge 2 ]] || usage
+ OUTPUT_DIR="$2"
+ shift
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+mkdir -p "${OUTPUT_DIR}"
+TARGET_FILE="${OUTPUT_DIR}/configs.md"
PRINT_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_config_docs"
PRINT_RUNTIME_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_runtime_config_docs"
diff --git a/dev/update_function_docs.sh b/dev/update_function_docs.sh
index 005122c33e..9bbe857a53 100755
--- a/dev/update_function_docs.sh
+++ b/dev/update_function_docs.sh
@@ -26,7 +26,39 @@ cd "${ROOT_DIR}"
# Load centralized tool versions
source "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh"
-TARGET_FILE="docs/source/user-guide/sql/aggregate_functions.md"
+# `--output-dir DIR` writes the generated pages into DIR instead of
+# docs/source/user-guide/sql. A relative DIR is taken from the repository root.
+OUTPUT_DIR="docs/source/user-guide/sql"
+
+usage() {
+ cat >&2 <<USAGE
+Usage: $0 [--output-dir DIR]
+
+Regenerates the aggregate, scalar, and window function pages under docs/source/user-guide/sql.
+--output-dir DIR Write the generated pages into DIR instead of docs/source/user-guide/sql (relative to the repository root).
+USAGE
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --output-dir)
+ [[ $# -ge 2 ]] || usage
+ OUTPUT_DIR="$2"
+ shift
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+mkdir -p "${OUTPUT_DIR}"
+TARGET_FILE="${OUTPUT_DIR}/aggregate_functions.md"
PRINT_AGGREGATE_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- aggregate"
echo "Inserting header"
@@ -120,7 +152,7 @@ npx "prettier@${PRETTIER_VERSION}" --write "$TARGET_FILE"
echo "'$TARGET_FILE' successfully updated!"
-TARGET_FILE="docs/source/user-guide/sql/scalar_functions.md"
+TARGET_FILE="${OUTPUT_DIR}/scalar_functions.md"
PRINT_SCALAR_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- scalar"
echo "Inserting header"
@@ -164,7 +196,7 @@ npx "prettier@${PRETTIER_VERSION}" --write "$TARGET_FILE"
echo "'$TARGET_FILE' successfully updated!"
-TARGET_FILE="docs/source/user-guide/sql/window_functions.md"
+TARGET_FILE="${OUTPUT_DIR}/window_functions.md"
PRINT_WINDOW_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- window"
echo "Inserting header"
diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md
index 6983c84da3..89dc7969b3 100644
--- a/docs/source/contributor-guide/testing.md
+++ b/docs/source/contributor-guide/testing.md
@@ -304,6 +304,22 @@ the README:
./ci/scripts/check_examples_docs.sh --write
```
+## Config and Function Docs Check
+
+[`configs.md`], [`aggregate_functions.md`], [`scalar_functions.md`], and
+[`window_functions.md`] are generated by [`dev/update_config_docs.sh`] and
+[`dev/update_function_docs.sh`]. To check that they are up to date, run the
+[`ci/scripts/check_generated_docs.sh`] script, which also runs as part of
+`./dev/rust_lint.sh`. Run it with `--write` to update the pages.
+
+[`configs.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/configs.md
+[`aggregate_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/aggregate_functions.md
+[`scalar_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/scalar_functions.md
+[`window_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/window_functions.md
+[`dev/update_config_docs.sh`]: https://github.com/apache/datafusion/blob/main/dev/update_config_docs.sh
+[`dev/update_function_docs.sh`]: https://github.com/apache/datafusion/blob/main/dev/update_function_docs.sh
+[`ci/scripts/check_generated_docs.sh`]: https://github.com/apache/datafusion/blob/main/ci/scripts/check_generated_docs.sh
+
## Benchmarks
### Criterion Benchmarks