Apache DataFusion SQL Query Engine
dev: include generated config and function docs in the local lint suite (#25525)
Kumar Ujjawal99af44a5e620
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex a2314098f8..73d1ff3de5 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -855,15 +855,9 @@ jobs:- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0with:node-version: "20"- - name: Check if configs.md has been modified+ - name: Check generated config and function docsrun: |- # If you encounter an error, run './dev/update_config_docs.sh' and commit- ./dev/update_config_docs.sh- git diff --exit-code- - name: Check if any of the ***_functions.md has been modified- run: |- # If you encounter an error, run './dev/update_function_docs.sh' and commit- ./dev/update_function_docs.sh+ ./ci/scripts/check_generated_docs.shgit diff --exit-code# This job ensures `datafusion-examples/README.md` stays in sync with the source code:diff --git a/ci/scripts/check_generated_docs.sh b/ci/scripts/check_generated_docs.shnew file mode 100755index 0000000000..fc9d31c745--- /dev/null+++ b/ci/scripts/check_generated_docs.sh@@ -0,0 +1,152 @@+#!/usr/bin/env bash+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Regenerates the configuration and function documentation pages and checks+# that the committed pages match, the same way the "check configs.md and+# ***_functions.md is up-to-date" job does. With `--write`, replaces the pages+# with the generated ones.++set -euo pipefail++SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"++source "${SCRIPT_DIR}/utils/git.sh"++CONFIG_DOCS_DIR="docs/source/user-guide"+FUNCTION_DOCS_DIR="docs/source/user-guide/sql"++MODE="check"+ALLOW_DIRTY=0++usage() {+ cat >&2 <<USAGE+Usage: $0 [--write] [--allow-dirty]++Checks that docs/source/user-guide/configs.md and the aggregate, scalar, and window+function pages under docs/source/user-guide/sql/ match the output of+dev/update_config_docs.sh and dev/update_function_docs.sh.+--write Replace the pages with the generated ones (requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+USAGE+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++cd "${ROOT_DIR}"++if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1+fi++if ! command -v npx >/dev/null 2>&1; then+ echo "[${SCRIPT_NAME}] npx is required to run the prettier check. Install Node.js (e.g., brew install node) and re-run." >&2+ exit 1+fi++# One scratch directory beneath each documentation directory, so Prettier finds+# the same configuration as for the committed pages.+SCRATCH_DIRS=()+cleanup() {+ rm -rf ${SCRATCH_DIRS[@]+"${SCRATCH_DIRS[@]}"}+}+trap cleanup EXIT+trap 'exit 130' INT+trap 'exit 143' TERM+CONFIG_SCRATCH="$(mktemp -d "${ROOT_DIR}/${CONFIG_DOCS_DIR}/.config-docs-check.XXXXXX")"+SCRATCH_DIRS+=("${CONFIG_SCRATCH}")+FUNCTION_SCRATCH="$(mktemp -d "${ROOT_DIR}/${FUNCTION_DOCS_DIR}/.function-docs-check.XXXXXX")"+SCRATCH_DIRS+=("${FUNCTION_SCRATCH}")++./dev/update_config_docs.sh --output-dir "${CONFIG_SCRATCH}"+./dev/update_function_docs.sh --output-dir "${FUNCTION_SCRATCH}"++GENERATED=(+ "${CONFIG_SCRATCH}/configs.md"+ "${FUNCTION_SCRATCH}/aggregate_functions.md"+ "${FUNCTION_SCRATCH}/scalar_functions.md"+ "${FUNCTION_SCRATCH}/window_functions.md"+)+COMMITTED=(+ "${CONFIG_DOCS_DIR}/configs.md"+ "${FUNCTION_DOCS_DIR}/aggregate_functions.md"+ "${FUNCTION_DOCS_DIR}/scalar_functions.md"+ "${FUNCTION_DOCS_DIR}/window_functions.md"+)++if [[ "$MODE" == "write" ]]; then+ for i in "${!COMMITTED[@]}"; do+ cp "${GENERATED[$i]}" "${COMMITTED[$i]}" || {+ echo "[${SCRIPT_NAME}] failed to copy the generated page to ${COMMITTED[$i]}" >&2+ exit 1+ }+ echo "✅ ${COMMITTED[$i]} updated."+ done+ exit 0+fi++stale_count=0+for i in "${!COMMITTED[@]}"; do+ diff_status=0+ diff -u -L "${COMMITTED[$i]} (committed)" -L "${COMMITTED[$i]} (generated)" \+ "${COMMITTED[$i]}" "${GENERATED[$i]}" > "${GENERATED[$i]}.diff" || diff_status=$?+ case "${diff_status}" in+ 0)+ echo "✅ ${COMMITTED[$i]} is up-to-date."+ ;;+ 1)+ stale_count=$((stale_count + 1))+ echo ""+ echo "❌ ${COMMITTED[$i]} is out of date."+ echo "------------------------------------------------------------"+ cat "${GENERATED[$i]}.diff"+ echo "------------------------------------------------------------"+ ;;+ *)+ echo "❌ diff exited with status ${diff_status} while comparing ${COMMITTED[$i]}; no comparison result." >&2+ exit "${diff_status}"+ ;;+ esac+done++if [[ ${stale_count} -gt 0 ]]; then+ echo ""+ echo "${stale_count} generated page(s) out of date. To update them, run:"+ echo ""+ echo " ./ci/scripts/check_generated_docs.sh --write"+ exit 1+fidiff --git a/dev/rust_lint.sh b/dev/rust_lint.shindex 77bed06457..74d5fc16d7 100755--- a/dev/rust_lint.sh+++ b/dev/rust_lint.sh@@ -130,6 +130,7 @@ declare -a WRITE_STEPS=("ci/scripts/typos_check.sh|true""ci/scripts/doc_prettier_check.sh|true""ci/scripts/check_examples_docs.sh|true"+ "ci/scripts/check_generated_docs.sh|true")declare -a READONLY_STEPS=(diff --git a/dev/update_config_docs.sh b/dev/update_config_docs.shindex df40c65210..5bba39625f 100755--- a/dev/update_config_docs.sh+++ b/dev/update_config_docs.sh@@ -26,7 +26,39 @@ cd "${ROOT_DIR}"# Load centralized tool versionssource "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh"-TARGET_FILE="docs/source/user-guide/configs.md"+# `--output-dir DIR` writes the generated page into DIR instead of+# docs/source/user-guide. A relative DIR is taken from the repository root.+OUTPUT_DIR="docs/source/user-guide"++usage() {+ cat >&2 <<USAGE+Usage: $0 [--output-dir DIR]++Regenerates docs/source/user-guide/configs.md.+--output-dir DIR Write the generated page into DIR instead of docs/source/user-guide (relative to the repository root).+USAGE+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --output-dir)+ [[ $# -ge 2 ]] || usage+ OUTPUT_DIR="$2"+ shift+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++mkdir -p "${OUTPUT_DIR}"+TARGET_FILE="${OUTPUT_DIR}/configs.md"PRINT_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_config_docs"PRINT_RUNTIME_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_runtime_config_docs"diff --git a/dev/update_function_docs.sh b/dev/update_function_docs.shindex 005122c33e..9bbe857a53 100755--- a/dev/update_function_docs.sh+++ b/dev/update_function_docs.sh@@ -26,7 +26,39 @@ cd "${ROOT_DIR}"# Load centralized tool versionssource "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh"-TARGET_FILE="docs/source/user-guide/sql/aggregate_functions.md"+# `--output-dir DIR` writes the generated pages into DIR instead of+# docs/source/user-guide/sql. A relative DIR is taken from the repository root.+OUTPUT_DIR="docs/source/user-guide/sql"++usage() {+ cat >&2 <<USAGE+Usage: $0 [--output-dir DIR]++Regenerates the aggregate, scalar, and window function pages under docs/source/user-guide/sql.+--output-dir DIR Write the generated pages into DIR instead of docs/source/user-guide/sql (relative to the repository root).+USAGE+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --output-dir)+ [[ $# -ge 2 ]] || usage+ OUTPUT_DIR="$2"+ shift+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++mkdir -p "${OUTPUT_DIR}"+TARGET_FILE="${OUTPUT_DIR}/aggregate_functions.md"PRINT_AGGREGATE_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- aggregate"echo "Inserting header"@@ -120,7 +152,7 @@ npx "prettier@${PRETTIER_VERSION}" --write "$TARGET_FILE"echo "'$TARGET_FILE' successfully updated!"-TARGET_FILE="docs/source/user-guide/sql/scalar_functions.md"+TARGET_FILE="${OUTPUT_DIR}/scalar_functions.md"PRINT_SCALAR_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- scalar"echo "Inserting header"@@ -164,7 +196,7 @@ npx "prettier@${PRETTIER_VERSION}" --write "$TARGET_FILE"echo "'$TARGET_FILE' successfully updated!"-TARGET_FILE="docs/source/user-guide/sql/window_functions.md"+TARGET_FILE="${OUTPUT_DIR}/window_functions.md"PRINT_WINDOW_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- window"echo "Inserting header"diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.mdindex 6983c84da3..89dc7969b3 100644--- a/docs/source/contributor-guide/testing.md+++ b/docs/source/contributor-guide/testing.md@@ -304,6 +304,22 @@ the README:./ci/scripts/check_examples_docs.sh --write```+## Config and Function Docs Check++[`configs.md`], [`aggregate_functions.md`], [`scalar_functions.md`], and+[`window_functions.md`] are generated by [`dev/update_config_docs.sh`] and+[`dev/update_function_docs.sh`]. To check that they are up to date, run the+[`ci/scripts/check_generated_docs.sh`] script, which also runs as part of+`./dev/rust_lint.sh`. Run it with `--write` to update the pages.++[`configs.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/configs.md+[`aggregate_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/aggregate_functions.md+[`scalar_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/scalar_functions.md+[`window_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/window_functions.md+[`dev/update_config_docs.sh`]: https://github.com/apache/datafusion/blob/main/dev/update_config_docs.sh+[`dev/update_function_docs.sh`]: https://github.com/apache/datafusion/blob/main/dev/update_function_docs.sh+[`ci/scripts/check_generated_docs.sh`]: https://github.com/apache/datafusion/blob/main/ci/scripts/check_generated_docs.sh+## Benchmarks### Criterion Benchmarks