Skip to content

Apache DataFusion SQL Query Engine

dev: include examples README checks in the local lint suite (#25466)

Kumar Ujjawal879c8fbfa11b
diff --git a/ci/scripts/check_examples_docs.sh b/ci/scripts/check_examples_docs.sh
index 62308b323b..d86edb1671 100755
--- a/ci/scripts/check_examples_docs.sh
+++ b/ci/scripts/check_examples_docs.sh
@@ -17,61 +17,119 @@
# specific language governing permissions and limitations
# under the License.
-# Generates documentation for DataFusion examples using the Rust-based
-# documentation generator and verifies that the committed README.md
-# is up to date.
-#
-# The README is generated from documentation comments in:
-# datafusion-examples/examples/<group>/main.rs
-#
-# This script is intended to be run in CI to ensure that example
-# documentation stays in sync with the code.
-#
-# To update the README locally, run this script and replace README.md
-# with the generated output.
+# Generates the examples README with the Rust generator and checks that the
+# committed `datafusion-examples/README.md` matches it, the same way the
+# "check example README is up-to-date" job does. With `--write`, replaces the
+# README with the generated one.
set -euo pipefail
-ROOT_DIR="$(git rev-parse --show-toplevel)"
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+source "${SCRIPT_DIR}/utils/tool_versions.sh"
+source "${SCRIPT_DIR}/utils/git.sh"
+
+EXAMPLES_DIR="${ROOT_DIR}/datafusion-examples"
+README="${EXAMPLES_DIR}/README.md"
+
+MODE="check"
+ALLOW_DIRTY=0
+
+usage() {
+ cat >&2 <<USAGE
+Usage: $0 [--write] [--allow-dirty]
+
+Checks that datafusion-examples/README.md matches the output of the examples-docs generator.
+--write Replace the README with the generated one (requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+USAGE
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
-# Load centralized tool versions
-source "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh"
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
+fi
+
+if ! command -v npx >/dev/null 2>&1; then
+ echo "[${SCRIPT_NAME}] npx is required to run the prettier check. Install Node.js (e.g., brew install node) and re-run." >&2
+ exit 1
+fi
+
+# The scratch directory sits beneath the examples directory so Prettier finds
+# the same configuration as for the committed README.
+SCRATCH_DIR="$(mktemp -d "${EXAMPLES_DIR}/.examples-docs-check.XXXXXX")"
+trap 'rm -rf "${SCRATCH_DIR}"' EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+README_NEW="${SCRATCH_DIR}/README.md"
+README_DIFF="${SCRATCH_DIR}/README.diff"
-EXAMPLES_DIR="$ROOT_DIR/datafusion-examples"
-README="$EXAMPLES_DIR/README.md"
-README_NEW="$EXAMPLES_DIR/README-NEW.md"
+cd "${ROOT_DIR}"
echo "▶ Generating examples README (Rust generator)…"
cargo run --quiet \
- --manifest-path "$EXAMPLES_DIR/Cargo.toml" \
+ --manifest-path "${EXAMPLES_DIR}/Cargo.toml" \
--bin examples-docs \
- > "$README_NEW"
+ > "${README_NEW}"
echo "▶ Formatting generated README with prettier ${PRETTIER_VERSION}…"
npx "prettier@${PRETTIER_VERSION}" \
--parser markdown \
- --write "$README_NEW"
+ --write "${README_NEW}"
-echo "▶ Comparing generated README with committed version…"
-
-if ! diff -u "$README" "$README_NEW" > /tmp/examples-readme.diff; then
- echo ""
- echo "❌ Examples README is out of date."
- echo ""
- echo "The examples documentation is generated automatically from:"
- echo " - datafusion-examples/examples/<group>/main.rs"
- echo ""
- echo "To update the README locally, run:"
- echo ""
- echo " cargo run --bin examples-docs \\"
- echo " | npx prettier@${PRETTIER_VERSION} --parser markdown --write \\"
- echo " > datafusion-examples/README.md"
- echo ""
- echo "Diff:"
- echo "------------------------------------------------------------"
- cat /tmp/examples-readme.diff
- echo "------------------------------------------------------------"
- exit 1
+if [[ "$MODE" == "write" ]]; then
+ cp "${README_NEW}" "${README}"
+ echo "✅ Examples README updated."
+ exit 0
fi
-echo "✅ Examples README is up-to-date."
+echo "▶ Comparing generated README with committed version…"
+diff_status=0
+diff -u "${README}" "${README_NEW}" > "${README_DIFF}" || diff_status=$?
+
+case "${diff_status}" in
+ 0)
+ echo "✅ Examples README is up-to-date."
+ ;;
+ 1)
+ echo ""
+ echo "❌ Examples README is out of date."
+ echo ""
+ echo "The examples documentation is generated automatically from:"
+ echo " - datafusion-examples/examples/<group>/main.rs"
+ echo ""
+ echo "To update the README, run:"
+ echo ""
+ echo " ./ci/scripts/check_examples_docs.sh --write"
+ echo ""
+ echo "Diff:"
+ echo "------------------------------------------------------------"
+ cat "${README_DIFF}"
+ echo "------------------------------------------------------------"
+ exit 1
+ ;;
+ *)
+ echo "❌ diff exited with status ${diff_status} while comparing the README; no comparison result." >&2
+ exit "${diff_status}"
+ ;;
+esac
diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh
index 7e17dea9db..a4f9e2f6fc 100755
--- a/dev/rust_lint.sh
+++ b/dev/rust_lint.sh
@@ -129,6 +129,7 @@ declare -a WRITE_STEPS=(
"ci/scripts/license_header.sh|true"
"ci/scripts/typos_check.sh|true"
"ci/scripts/doc_prettier_check.sh|true"
+ "ci/scripts/check_examples_docs.sh|true"
)
declare -a READONLY_STEPS=(
diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md
index 37143fd57d..6371ea8c65 100644
--- a/docs/source/contributor-guide/testing.md
+++ b/docs/source/contributor-guide/testing.md
@@ -275,6 +275,19 @@ To run either check on its own:
[`dev/depcheck`]: https://github.com/apache/datafusion/tree/main/dev/depcheck
[cargo-machete]: https://github.com/bnjbvr/cargo-machete
+## Examples README Check
+
+`datafusion-examples/README.md` is generated from the documentation comments in
+`datafusion-examples/examples/<group>/main.rs`. `ci/scripts/check_examples_docs.sh`
+regenerates it and fails if the committed file differs. `./dev/rust_lint.sh`
+runs it and needs `cargo` and `npx`. To run the check on its own, or to update
+the README:
+
+```shell
+./ci/scripts/check_examples_docs.sh
+./ci/scripts/check_examples_docs.sh --write
+```
+
## Benchmarks
### Criterion Benchmarks