Apache DataFusion SQL Query Engine
dev: include examples README checks in the local lint suite (#25466)
Kumar Ujjawal879c8fbfa11b
diff --git a/ci/scripts/check_examples_docs.sh b/ci/scripts/check_examples_docs.shindex 62308b323b..d86edb1671 100755--- a/ci/scripts/check_examples_docs.sh+++ b/ci/scripts/check_examples_docs.sh@@ -17,61 +17,119 @@# specific language governing permissions and limitations# under the License.-# Generates documentation for DataFusion examples using the Rust-based-# documentation generator and verifies that the committed README.md-# is up to date.-#-# The README is generated from documentation comments in:-# datafusion-examples/examples/<group>/main.rs-#-# This script is intended to be run in CI to ensure that example-# documentation stays in sync with the code.-#-# To update the README locally, run this script and replace README.md-# with the generated output.+# Generates the examples README with the Rust generator and checks that the+# committed `datafusion-examples/README.md` matches it, the same way the+# "check example README is up-to-date" job does. With `--write`, replaces the+# README with the generated one.set -euo pipefail-ROOT_DIR="$(git rev-parse --show-toplevel)"+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"++source "${SCRIPT_DIR}/utils/tool_versions.sh"+source "${SCRIPT_DIR}/utils/git.sh"++EXAMPLES_DIR="${ROOT_DIR}/datafusion-examples"+README="${EXAMPLES_DIR}/README.md"++MODE="check"+ALLOW_DIRTY=0++usage() {+ cat >&2 <<USAGE+Usage: $0 [--write] [--allow-dirty]++Checks that datafusion-examples/README.md matches the output of the examples-docs generator.+--write Replace the README with the generated one (requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+USAGE+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done-# Load centralized tool versions-source "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh"+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1+fi++if ! command -v npx >/dev/null 2>&1; then+ echo "[${SCRIPT_NAME}] npx is required to run the prettier check. Install Node.js (e.g., brew install node) and re-run." >&2+ exit 1+fi++# The scratch directory sits beneath the examples directory so Prettier finds+# the same configuration as for the committed README.+SCRATCH_DIR="$(mktemp -d "${EXAMPLES_DIR}/.examples-docs-check.XXXXXX")"+trap 'rm -rf "${SCRATCH_DIR}"' EXIT+trap 'exit 130' INT+trap 'exit 143' TERM+README_NEW="${SCRATCH_DIR}/README.md"+README_DIFF="${SCRATCH_DIR}/README.diff"-EXAMPLES_DIR="$ROOT_DIR/datafusion-examples"-README="$EXAMPLES_DIR/README.md"-README_NEW="$EXAMPLES_DIR/README-NEW.md"+cd "${ROOT_DIR}"echo "▶ Generating examples README (Rust generator)…"cargo run --quiet \- --manifest-path "$EXAMPLES_DIR/Cargo.toml" \+ --manifest-path "${EXAMPLES_DIR}/Cargo.toml" \--bin examples-docs \- > "$README_NEW"+ > "${README_NEW}"echo "▶ Formatting generated README with prettier ${PRETTIER_VERSION}…"npx "prettier@${PRETTIER_VERSION}" \--parser markdown \- --write "$README_NEW"+ --write "${README_NEW}"-echo "▶ Comparing generated README with committed version…"--if ! diff -u "$README" "$README_NEW" > /tmp/examples-readme.diff; then- echo ""- echo "❌ Examples README is out of date."- echo ""- echo "The examples documentation is generated automatically from:"- echo " - datafusion-examples/examples/<group>/main.rs"- echo ""- echo "To update the README locally, run:"- echo ""- echo " cargo run --bin examples-docs \\"- echo " | npx prettier@${PRETTIER_VERSION} --parser markdown --write \\"- echo " > datafusion-examples/README.md"- echo ""- echo "Diff:"- echo "------------------------------------------------------------"- cat /tmp/examples-readme.diff- echo "------------------------------------------------------------"- exit 1+if [[ "$MODE" == "write" ]]; then+ cp "${README_NEW}" "${README}"+ echo "✅ Examples README updated."+ exit 0fi-echo "✅ Examples README is up-to-date."+echo "▶ Comparing generated README with committed version…"+diff_status=0+diff -u "${README}" "${README_NEW}" > "${README_DIFF}" || diff_status=$?++case "${diff_status}" in+ 0)+ echo "✅ Examples README is up-to-date."+ ;;+ 1)+ echo ""+ echo "❌ Examples README is out of date."+ echo ""+ echo "The examples documentation is generated automatically from:"+ echo " - datafusion-examples/examples/<group>/main.rs"+ echo ""+ echo "To update the README, run:"+ echo ""+ echo " ./ci/scripts/check_examples_docs.sh --write"+ echo ""+ echo "Diff:"+ echo "------------------------------------------------------------"+ cat "${README_DIFF}"+ echo "------------------------------------------------------------"+ exit 1+ ;;+ *)+ echo "❌ diff exited with status ${diff_status} while comparing the README; no comparison result." >&2+ exit "${diff_status}"+ ;;+esacdiff --git a/dev/rust_lint.sh b/dev/rust_lint.shindex 7e17dea9db..a4f9e2f6fc 100755--- a/dev/rust_lint.sh+++ b/dev/rust_lint.sh@@ -129,6 +129,7 @@ declare -a WRITE_STEPS=("ci/scripts/license_header.sh|true""ci/scripts/typos_check.sh|true""ci/scripts/doc_prettier_check.sh|true"+ "ci/scripts/check_examples_docs.sh|true")declare -a READONLY_STEPS=(diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.mdindex 37143fd57d..6371ea8c65 100644--- a/docs/source/contributor-guide/testing.md+++ b/docs/source/contributor-guide/testing.md@@ -275,6 +275,19 @@ To run either check on its own:[`dev/depcheck`]: https://github.com/apache/datafusion/tree/main/dev/depcheck[cargo-machete]: https://github.com/bnjbvr/cargo-machete+## Examples README Check++`datafusion-examples/README.md` is generated from the documentation comments in+`datafusion-examples/examples/<group>/main.rs`. `ci/scripts/check_examples_docs.sh`+regenerates it and fails if the committed file differs. `./dev/rust_lint.sh`+runs it and needs `cargo` and `npx`. To run the check on its own, or to update+the README:++```shell+./ci/scripts/check_examples_docs.sh+./ci/scripts/check_examples_docs.sh --write+```+## Benchmarks### Criterion Benchmarks