Skip to content

Apache DataFusion SQL Query Engine

dev: Add a script to auto fix all lint violations (#19560)

Yongting You6bf5e98b59e5
diff --git a/ci/scripts/doc_prettier_check.sh b/ci/scripts/doc_prettier_check.sh
index d94a0d1c96..f074ab9283 100755
--- a/ci/scripts/doc_prettier_check.sh
+++ b/ci/scripts/doc_prettier_check.sh
@@ -17,41 +17,68 @@
# specific language governing permissions and limitations
# under the License.
-SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
-
-MODE="--check"
-ACTION="Checking"
-if [ $# -gt 0 ]; then
- if [ "$1" = "--write" ]; then
- MODE="--write"
- ACTION="Formatting"
- else
- echo "Usage: $0 [--write]" >&2
- exit 1
- fi
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+PRETTIER_VERSION="2.7.1"
+PRETTIER_TARGETS=(
+ '{datafusion,datafusion-cli,datafusion-examples,dev,docs}/**/*.md'
+ '!datafusion/CHANGELOG.md'
+ README.md
+ CONTRIBUTING.md
+)
+
+source "${SCRIPT_DIR}/utils/git.sh"
+
+MODE="check"
+ALLOW_DIRTY=0
+
+usage() {
+ cat >&2 <<EOF
+Usage: $SCRIPT_NAME [--write] [--allow-dirty]
+
+Runs prettier@${PRETTIER_VERSION} over markdown docs.
+--write Run with \`--write\` to format files (requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+EOF
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
fi
-echo "$SCRIPT_PATH: $ACTION documents with prettier"
+echo "[${SCRIPT_NAME}] prettier@${PRETTIER_VERSION} ${MODE}"
# Ensure `npx` is available
if ! command -v npx >/dev/null 2>&1; then
echo "npx is required to run the prettier check. Install Node.js (e.g., brew install node) and re-run." >&2
exit 1
fi
-
-# Ignore subproject CHANGELOG.md because it is machine generated
-npx [email protected] $MODE \
- '{datafusion,datafusion-cli,datafusion-examples,dev,docs}/**/*.md' \
- '!datafusion/CHANGELOG.md' \
- README.md \
- CONTRIBUTING.md
-status=$?
-
-if [ $status -ne 0 ]; then
- if [ "$MODE" = "--check" ]; then
- echo "Prettier check failed. Re-run with --write (e.g., ./ci/scripts/doc_prettier_check.sh --write) to format files, commit the changes, and re-run the check." >&2
- else
- echo "Prettier format failed. Files may have been modified; commit any changes and re-run." >&2
- fi
- exit $status
+
+PRETTIER_MODE=(--check)
+if [[ "$MODE" == "write" ]]; then
+ PRETTIER_MODE=(--write)
fi
+
+# Ignore subproject CHANGELOG.md because it is machine generated
+npx "prettier@${PRETTIER_VERSION}" "${PRETTIER_MODE[@]}" "${PRETTIER_TARGETS[@]}"
diff --git a/ci/scripts/license_header.sh b/ci/scripts/license_header.sh
index 5345728f9c..7ab8c96375 100755
--- a/ci/scripts/license_header.sh
+++ b/ci/scripts/license_header.sh
@@ -17,6 +17,62 @@
# specific language governing permissions and limitations
# under the License.
-# Check Apache license header
-set -ex
-hawkeye check --config licenserc.toml
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+
+source "${SCRIPT_DIR}/utils/git.sh"
+
+MODE="check"
+ALLOW_DIRTY=0
+HAWKEYE_CONFIG="licenserc.toml"
+
+usage() {
+ cat >&2 <<EOF
+Usage: $SCRIPT_NAME [--write] [--allow-dirty]
+
+Checks Apache license headers with \`hawkeye check --config $HAWKEYE_CONFIG\`.
+--write Run \`hawkeye format --config $HAWKEYE_CONFIG\` to auto-add/fix headers (requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+EOF
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
+fi
+
+if [[ "$MODE" == "write" ]]; then
+ echo "[${SCRIPT_NAME}] \`hawkeye format --config ${HAWKEYE_CONFIG}\`"
+ if ! hawkeye format --config "${HAWKEYE_CONFIG}"; then
+ status=$?
+ # hawkeye returns exit code 1 when it applies fixes; treat that as success.
+ if [[ $status -eq 1 ]]; then
+ echo "[${SCRIPT_NAME}] hawkeye format applied fixes (exit 1 treated as success)"
+ else
+ exit $status
+ fi
+ fi
+else
+ echo "[${SCRIPT_NAME}] \`hawkeye check --config ${HAWKEYE_CONFIG}\`"
+ hawkeye check --config "${HAWKEYE_CONFIG}"
+fi
diff --git a/ci/scripts/rust_clippy.sh b/ci/scripts/rust_clippy.sh
index aa994bc2b8..f8b5c0852f 100755
--- a/ci/scripts/rust_clippy.sh
+++ b/ci/scripts/rust_clippy.sh
@@ -17,5 +17,60 @@
# specific language governing permissions and limitations
# under the License.
-set -ex
-cargo clippy --all-targets --workspace --features avro,integration-tests,extended_tests -- -D warnings
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+CLIPPY_FEATURES="avro,integration-tests,extended_tests"
+CLIPPY_ARGS=(--all-targets --workspace --features "$CLIPPY_FEATURES")
+CLIPPY_LINT_ARGS=(-- -D warnings)
+
+source "${SCRIPT_DIR}/utils/git.sh"
+
+MODE="check"
+ALLOW_DIRTY=0
+
+usage() {
+ cat >&2 <<EOF
+Usage: $SCRIPT_NAME [--write] [--allow-dirty]
+
+Runs \`cargo clippy\` to lint.
+--write Run \`cargo clippy --fix\` to apply fixes for clippy lints (requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted or staged changes.
+EOF
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
+fi
+
+CLIPPY_CMD=(cargo clippy)
+if [[ "$MODE" == "write" ]]; then
+ CLIPPY_CMD+=(--fix)
+ if [[ $ALLOW_DIRTY -eq 1 ]]; then
+ CLIPPY_CMD+=(--allow-dirty --allow-staged)
+ fi
+fi
+CLIPPY_CMD+=("${CLIPPY_ARGS[@]}" "${CLIPPY_LINT_ARGS[@]}")
+
+echo "[${SCRIPT_NAME}] \`${CLIPPY_CMD[*]}\`"
+"${CLIPPY_CMD[@]}"
diff --git a/ci/scripts/rust_docs.sh b/ci/scripts/rust_docs.sh
index e90bfdf8bc..91cc305f51 100755
--- a/ci/scripts/rust_docs.sh
+++ b/ci/scripts/rust_docs.sh
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+# Note: cargo doc does not support an auto-fix mode; this script runs the check-only build.
set -ex
export RUSTDOCFLAGS="-D warnings"
cargo doc --document-private-items --no-deps --workspace
diff --git a/ci/scripts/rust_fmt.sh b/ci/scripts/rust_fmt.sh
index 9d8325877a..16c87cea5e 100755
--- a/ci/scripts/rust_fmt.sh
+++ b/ci/scripts/rust_fmt.sh
@@ -17,5 +17,52 @@
# specific language governing permissions and limitations
# under the License.
-set -ex
-cargo fmt --all -- --check
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+source "${SCRIPT_DIR}/utils/git.sh"
+
+MODE="check"
+ALLOW_DIRTY=0
+
+usage() {
+ cat >&2 <<EOF
+Usage: $0 [--write] [--allow-dirty]
+
+Runs \`cargo fmt --all -- --check\` by default to verify Rust formatting.
+--write Run \`cargo fmt --all\` to auto-fix formatting (requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+EOF
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
+fi
+
+if [[ "$MODE" == "write" ]]; then
+ echo "[${SCRIPT_NAME}] \`cargo fmt --all\`"
+ cargo fmt --all
+else
+ echo "[${SCRIPT_NAME}] \`cargo fmt --all -- --check\`"
+ cargo fmt --all -- --check
+fi
diff --git a/ci/scripts/rust_toml_fmt.sh b/ci/scripts/rust_toml_fmt.sh
index 393ad55f41..b1f9373fbf 100755
--- a/ci/scripts/rust_toml_fmt.sh
+++ b/ci/scripts/rust_toml_fmt.sh
@@ -17,8 +17,53 @@
# specific language governing permissions and limitations
# under the License.
-# Run `taplo format` with flag `--check` in dry run to check formatting
-# without overwritng the file. If any error occur, you may want to
-# rerun `taplo format` to fix the formatting automatically.
-set -ex
-taplo format --check
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+
+source "${SCRIPT_DIR}/utils/git.sh"
+
+MODE="check"
+ALLOW_DIRTY=0
+
+usage() {
+ cat >&2 <<EOF
+Usage: $0 [--write] [--allow-dirty]
+
+Runs \`taplo format --check\` by default to verify TOML formatting.
+--write Run \`taplo format\` to auto-fix formatting (best-effort; requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+EOF
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
+fi
+
+if [[ "$MODE" == "write" ]]; then
+ echo "[${SCRIPT_NAME}] \`taplo format\`"
+ taplo format
+else
+ echo "[${SCRIPT_NAME}] \`taplo format --check\`"
+ taplo format --check
+fi
diff --git a/ci/scripts/typos_check.sh b/ci/scripts/typos_check.sh
index a3a4a89321..a567c7b44e 100755
--- a/ci/scripts/typos_check.sh
+++ b/ci/scripts/typos_check.sh
@@ -17,7 +17,54 @@
# specific language governing permissions and limitations
# under the License.
-set -ex
-# To use this script, you must have installed `typos`, for example:
-# cargo install typos-cli --locked --version 1.37.0
-typos --config typos.toml
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+TYPOS_CONFIG="typos.toml"
+
+source "${SCRIPT_DIR}/utils/git.sh"
+
+MODE="check"
+ALLOW_DIRTY=0
+
+usage() {
+ cat >&2 <<EOF
+Usage: $SCRIPT_NAME [--write] [--allow-dirty]
+
+Runs \`typos --config ${TYPOS_CONFIG}\` by default to check spelling.
+--write Run \`typos --write-changes --config ${TYPOS_CONFIG}\` to auto-fix spelling issues (requires a clean git worktree, no uncommitted changes).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+EOF
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
+ require_clean_work_tree "$SCRIPT_NAME" || exit 1
+fi
+
+if [[ "$MODE" == "write" ]]; then
+ echo "[${SCRIPT_NAME}] \`typos --write-changes --config ${TYPOS_CONFIG}\`"
+ typos --write-changes --config "${TYPOS_CONFIG}"
+else
+ echo "[${SCRIPT_NAME}] \`typos --config ${TYPOS_CONFIG}\`"
+ typos --config "${TYPOS_CONFIG}"
+fi
diff --git a/ci/scripts/utils/git.sh b/ci/scripts/utils/git.sh
new file mode 100644
index 0000000000..b5baecda75
--- /dev/null
+++ b/ci/scripts/utils/git.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Ensure the repository is clean before auto-fixing files.
+require_clean_work_tree() {
+ local caller="${1:-script}"
+ if [[ -n "$(git status --porcelain)" ]]; then
+ echo "[$caller] Uncommitted changes detected. Commit or stash them, or re-run with --allow-dirty." >&2
+ return 1
+ fi
+}
diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh
index 21d4611846..43d29bd881 100755
--- a/dev/rust_lint.sh
+++ b/dev/rust_lint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -23,30 +23,103 @@
# Note: The installed checking tools (e.g., taplo) are not guaranteed to match
# the CI versions for simplicity, there might be some minor differences. Check
# `.github/workflows` for the CI versions.
+#
+#
+#
+# For each lint scripts:
+#
+# By default, they run in check mode:
+# ./ci/scripts/rust_fmt.sh
+#
+# With `--write`, scripts perform best-effort auto fixes:
+# ./ci/scripts/rust_fmt.sh --write
+#
+# The `--write` flag assumes a clean git repository (no uncommitted changes); to force
+# auto fixes even if there are unstaged changes, use `--allow-dirty`:
+# ./ci/scripts/rust_fmt.sh --write --allow-dirty
+#
+# New scripts can use `rust_fmt.sh` as a reference.
+
+set -euo pipefail
+
+usage() {
+ cat >&2 <<EOF
+Usage: $0 [--write] [--allow-dirty]
+
+Runs the local Rust lint suite similar to CI.
+--write Run formatters, clippy and other non-functional checks in best-effort write/fix mode (requires a clean git worktree, no uncommitted changes; some checks are test-only and ignore this flag).
+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
+EOF
+ exit 1
+}
+
+ensure_tool() {
+ local cmd="$1"
+ local install_cmd="$2"
+ if ! command -v "$cmd" &> /dev/null; then
+ echo "Installing $cmd using: $install_cmd"
+ eval "$install_cmd"
+ fi
+}
+
+MODE="check"
+ALLOW_DIRTY=0
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --write)
+ MODE="write"
+ ;;
+ --allow-dirty)
+ ALLOW_DIRTY=1
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+
+ensure_tool "taplo" "cargo install taplo-cli --locked"
+ensure_tool "hawkeye" "cargo install hawkeye --locked"
+ensure_tool "typos" "cargo install typos-cli --locked"
+
+run_step() {
+ local name="$1"
+ shift
+ echo "[${SCRIPT_NAME}] Running ${name}"
+ "$@"
+}
+
+declare -a WRITE_STEPS=(
+ "ci/scripts/rust_fmt.sh|true"
+ "ci/scripts/rust_clippy.sh|true"
+ "ci/scripts/rust_toml_fmt.sh|true"
+ "ci/scripts/license_header.sh|true"
+ "ci/scripts/typos_check.sh|true"
+ "ci/scripts/doc_prettier_check.sh|true"
+)
+
+declare -a READONLY_STEPS=(
+ "ci/scripts/rust_docs.sh|false"
+)
-# For `.toml` format checking
-set -e
-if ! command -v taplo &> /dev/null; then
- echo "Installing taplo using cargo"
- cargo install taplo-cli
-fi
-
-# For Apache licence header checking
-if ! command -v hawkeye &> /dev/null; then
- echo "Installing hawkeye using cargo"
- cargo install hawkeye --locked
-fi
-
-# For spelling checks
-if ! command -v typos &> /dev/null; then
- echo "Installing typos using cargo"
- cargo install typos-cli --locked
-fi
-
-ci/scripts/rust_fmt.sh
-ci/scripts/rust_clippy.sh
-ci/scripts/rust_toml_fmt.sh
-ci/scripts/rust_docs.sh
-ci/scripts/license_header.sh
-ci/scripts/typos_check.sh
-ci/scripts/doc_prettier_check.sh
+for entry in "${WRITE_STEPS[@]}" "${READONLY_STEPS[@]}"; do
+ IFS='|' read -r script_path supports_write <<<"$entry"
+ script_name="$(basename "$script_path")"
+ args=()
+ if [[ "$supports_write" == "true" && "$MODE" == "write" ]]; then
+ args+=(--write)
+ [[ $ALLOW_DIRTY -eq 1 ]] && args+=(--allow-dirty)
+ fi
+ if [[ ${#args[@]} -gt 0 ]]; then
+ run_step "$script_name" "$script_path" "${args[@]}"
+ else
+ run_step "$script_name" "$script_path"
+ fi
+done