Apache DataFusion SQL Query Engine
dev: Add a script to auto fix all lint violations (#19560)
Yongting You6bf5e98b59e5
diff --git a/ci/scripts/doc_prettier_check.sh b/ci/scripts/doc_prettier_check.shindex d94a0d1c96..f074ab9283 100755--- a/ci/scripts/doc_prettier_check.sh+++ b/ci/scripts/doc_prettier_check.sh@@ -17,41 +17,68 @@# specific language governing permissions and limitations# under the License.-SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"--MODE="--check"-ACTION="Checking"-if [ $# -gt 0 ]; then- if [ "$1" = "--write" ]; then- MODE="--write"- ACTION="Formatting"- else- echo "Usage: $0 [--write]" >&2- exit 1- fi+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+PRETTIER_VERSION="2.7.1"+PRETTIER_TARGETS=(+ '{datafusion,datafusion-cli,datafusion-examples,dev,docs}/**/*.md'+ '!datafusion/CHANGELOG.md'+ README.md+ CONTRIBUTING.md+)++source "${SCRIPT_DIR}/utils/git.sh"++MODE="check"+ALLOW_DIRTY=0++usage() {+ cat >&2 <<EOF+Usage: $SCRIPT_NAME [--write] [--allow-dirty]++Runs prettier@${PRETTIER_VERSION} over markdown docs.+--write Run with \`--write\` to format files (requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+EOF+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1fi-echo "$SCRIPT_PATH: $ACTION documents with prettier"+echo "[${SCRIPT_NAME}] prettier@${PRETTIER_VERSION} ${MODE}"# Ensure `npx` is availableif ! command -v npx >/dev/null 2>&1; thenecho "npx is required to run the prettier check. Install Node.js (e.g., brew install node) and re-run." >&2exit 1fi--# Ignore subproject CHANGELOG.md because it is machine generated-npx [email protected] $MODE \- '{datafusion,datafusion-cli,datafusion-examples,dev,docs}/**/*.md' \- '!datafusion/CHANGELOG.md' \- README.md \- CONTRIBUTING.md-status=$?--if [ $status -ne 0 ]; then- if [ "$MODE" = "--check" ]; then- echo "Prettier check failed. Re-run with --write (e.g., ./ci/scripts/doc_prettier_check.sh --write) to format files, commit the changes, and re-run the check." >&2- else- echo "Prettier format failed. Files may have been modified; commit any changes and re-run." >&2- fi- exit $status++PRETTIER_MODE=(--check)+if [[ "$MODE" == "write" ]]; then+ PRETTIER_MODE=(--write)fi++# Ignore subproject CHANGELOG.md because it is machine generated+npx "prettier@${PRETTIER_VERSION}" "${PRETTIER_MODE[@]}" "${PRETTIER_TARGETS[@]}"diff --git a/ci/scripts/license_header.sh b/ci/scripts/license_header.shindex 5345728f9c..7ab8c96375 100755--- a/ci/scripts/license_header.sh+++ b/ci/scripts/license_header.sh@@ -17,6 +17,62 @@# specific language governing permissions and limitations# under the License.-# Check Apache license header-set -ex-hawkeye check --config licenserc.toml+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"++source "${SCRIPT_DIR}/utils/git.sh"++MODE="check"+ALLOW_DIRTY=0+HAWKEYE_CONFIG="licenserc.toml"++usage() {+ cat >&2 <<EOF+Usage: $SCRIPT_NAME [--write] [--allow-dirty]++Checks Apache license headers with \`hawkeye check --config $HAWKEYE_CONFIG\`.+--write Run \`hawkeye format --config $HAWKEYE_CONFIG\` to auto-add/fix headers (requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+EOF+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1+fi++if [[ "$MODE" == "write" ]]; then+ echo "[${SCRIPT_NAME}] \`hawkeye format --config ${HAWKEYE_CONFIG}\`"+ if ! hawkeye format --config "${HAWKEYE_CONFIG}"; then+ status=$?+ # hawkeye returns exit code 1 when it applies fixes; treat that as success.+ if [[ $status -eq 1 ]]; then+ echo "[${SCRIPT_NAME}] hawkeye format applied fixes (exit 1 treated as success)"+ else+ exit $status+ fi+ fi+else+ echo "[${SCRIPT_NAME}] \`hawkeye check --config ${HAWKEYE_CONFIG}\`"+ hawkeye check --config "${HAWKEYE_CONFIG}"+fidiff --git a/ci/scripts/rust_clippy.sh b/ci/scripts/rust_clippy.shindex aa994bc2b8..f8b5c0852f 100755--- a/ci/scripts/rust_clippy.sh+++ b/ci/scripts/rust_clippy.sh@@ -17,5 +17,60 @@# specific language governing permissions and limitations# under the License.-set -ex-cargo clippy --all-targets --workspace --features avro,integration-tests,extended_tests -- -D warnings+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+CLIPPY_FEATURES="avro,integration-tests,extended_tests"+CLIPPY_ARGS=(--all-targets --workspace --features "$CLIPPY_FEATURES")+CLIPPY_LINT_ARGS=(-- -D warnings)++source "${SCRIPT_DIR}/utils/git.sh"++MODE="check"+ALLOW_DIRTY=0++usage() {+ cat >&2 <<EOF+Usage: $SCRIPT_NAME [--write] [--allow-dirty]++Runs \`cargo clippy\` to lint.+--write Run \`cargo clippy --fix\` to apply fixes for clippy lints (requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted or staged changes.+EOF+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1+fi++CLIPPY_CMD=(cargo clippy)+if [[ "$MODE" == "write" ]]; then+ CLIPPY_CMD+=(--fix)+ if [[ $ALLOW_DIRTY -eq 1 ]]; then+ CLIPPY_CMD+=(--allow-dirty --allow-staged)+ fi+fi+CLIPPY_CMD+=("${CLIPPY_ARGS[@]}" "${CLIPPY_LINT_ARGS[@]}")++echo "[${SCRIPT_NAME}] \`${CLIPPY_CMD[*]}\`"+"${CLIPPY_CMD[@]}"diff --git a/ci/scripts/rust_docs.sh b/ci/scripts/rust_docs.shindex e90bfdf8bc..91cc305f51 100755--- a/ci/scripts/rust_docs.sh+++ b/ci/scripts/rust_docs.sh@@ -17,6 +17,7 @@# specific language governing permissions and limitations# under the License.+# Note: cargo doc does not support an auto-fix mode; this script runs the check-only build.set -exexport RUSTDOCFLAGS="-D warnings"cargo doc --document-private-items --no-deps --workspacediff --git a/ci/scripts/rust_fmt.sh b/ci/scripts/rust_fmt.shindex 9d8325877a..16c87cea5e 100755--- a/ci/scripts/rust_fmt.sh+++ b/ci/scripts/rust_fmt.sh@@ -17,5 +17,52 @@# specific language governing permissions and limitations# under the License.-set -ex-cargo fmt --all -- --check+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+source "${SCRIPT_DIR}/utils/git.sh"++MODE="check"+ALLOW_DIRTY=0++usage() {+ cat >&2 <<EOF+Usage: $0 [--write] [--allow-dirty]++Runs \`cargo fmt --all -- --check\` by default to verify Rust formatting.+--write Run \`cargo fmt --all\` to auto-fix formatting (requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+EOF+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1+fi++if [[ "$MODE" == "write" ]]; then+ echo "[${SCRIPT_NAME}] \`cargo fmt --all\`"+ cargo fmt --all+else+ echo "[${SCRIPT_NAME}] \`cargo fmt --all -- --check\`"+ cargo fmt --all -- --check+fidiff --git a/ci/scripts/rust_toml_fmt.sh b/ci/scripts/rust_toml_fmt.shindex 393ad55f41..b1f9373fbf 100755--- a/ci/scripts/rust_toml_fmt.sh+++ b/ci/scripts/rust_toml_fmt.sh@@ -17,8 +17,53 @@# specific language governing permissions and limitations# under the License.-# Run `taplo format` with flag `--check` in dry run to check formatting-# without overwritng the file. If any error occur, you may want to-# rerun `taplo format` to fix the formatting automatically.-set -ex-taplo format --check+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"++source "${SCRIPT_DIR}/utils/git.sh"++MODE="check"+ALLOW_DIRTY=0++usage() {+ cat >&2 <<EOF+Usage: $0 [--write] [--allow-dirty]++Runs \`taplo format --check\` by default to verify TOML formatting.+--write Run \`taplo format\` to auto-fix formatting (best-effort; requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+EOF+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1+fi++if [[ "$MODE" == "write" ]]; then+ echo "[${SCRIPT_NAME}] \`taplo format\`"+ taplo format+else+ echo "[${SCRIPT_NAME}] \`taplo format --check\`"+ taplo format --check+fidiff --git a/ci/scripts/typos_check.sh b/ci/scripts/typos_check.shindex a3a4a89321..a567c7b44e 100755--- a/ci/scripts/typos_check.sh+++ b/ci/scripts/typos_check.sh@@ -17,7 +17,54 @@# specific language governing permissions and limitations# under the License.-set -ex-# To use this script, you must have installed `typos`, for example:-# cargo install typos-cli --locked --version 1.37.0-typos --config typos.toml+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+TYPOS_CONFIG="typos.toml"++source "${SCRIPT_DIR}/utils/git.sh"++MODE="check"+ALLOW_DIRTY=0++usage() {+ cat >&2 <<EOF+Usage: $SCRIPT_NAME [--write] [--allow-dirty]++Runs \`typos --config ${TYPOS_CONFIG}\` by default to check spelling.+--write Run \`typos --write-changes --config ${TYPOS_CONFIG}\` to auto-fix spelling issues (requires a clean git worktree, no uncommitted changes).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+EOF+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then+ require_clean_work_tree "$SCRIPT_NAME" || exit 1+fi++if [[ "$MODE" == "write" ]]; then+ echo "[${SCRIPT_NAME}] \`typos --write-changes --config ${TYPOS_CONFIG}\`"+ typos --write-changes --config "${TYPOS_CONFIG}"+else+ echo "[${SCRIPT_NAME}] \`typos --config ${TYPOS_CONFIG}\`"+ typos --config "${TYPOS_CONFIG}"+fidiff --git a/ci/scripts/utils/git.sh b/ci/scripts/utils/git.shnew file mode 100644index 0000000000..b5baecda75--- /dev/null+++ b/ci/scripts/utils/git.sh@@ -0,0 +1,27 @@+#!/usr/bin/env bash+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Ensure the repository is clean before auto-fixing files.+require_clean_work_tree() {+ local caller="${1:-script}"+ if [[ -n "$(git status --porcelain)" ]]; then+ echo "[$caller] Uncommitted changes detected. Commit or stash them, or re-run with --allow-dirty." >&2+ return 1+ fi+}diff --git a/dev/rust_lint.sh b/dev/rust_lint.shindex 21d4611846..43d29bd881 100755--- a/dev/rust_lint.sh+++ b/dev/rust_lint.sh@@ -1,4 +1,4 @@-#!/bin/bash+#!/usr/bin/env bash# Licensed to the Apache Software Foundation (ASF) under one# or more contributor license agreements. See the NOTICE file@@ -23,30 +23,103 @@# Note: The installed checking tools (e.g., taplo) are not guaranteed to match# the CI versions for simplicity, there might be some minor differences. Check# `.github/workflows` for the CI versions.+#+#+#+# For each lint scripts:+#+# By default, they run in check mode:+# ./ci/scripts/rust_fmt.sh+#+# With `--write`, scripts perform best-effort auto fixes:+# ./ci/scripts/rust_fmt.sh --write+#+# The `--write` flag assumes a clean git repository (no uncommitted changes); to force+# auto fixes even if there are unstaged changes, use `--allow-dirty`:+# ./ci/scripts/rust_fmt.sh --write --allow-dirty+#+# New scripts can use `rust_fmt.sh` as a reference.++set -euo pipefail++usage() {+ cat >&2 <<EOF+Usage: $0 [--write] [--allow-dirty]++Runs the local Rust lint suite similar to CI.+--write Run formatters, clippy and other non-functional checks in best-effort write/fix mode (requires a clean git worktree, no uncommitted changes; some checks are test-only and ignore this flag).+--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.+EOF+ exit 1+}++ensure_tool() {+ local cmd="$1"+ local install_cmd="$2"+ if ! command -v "$cmd" &> /dev/null; then+ echo "Installing $cmd using: $install_cmd"+ eval "$install_cmd"+ fi+}++MODE="check"+ALLOW_DIRTY=0++while [[ $# -gt 0 ]]; do+ case "$1" in+ --write)+ MODE="write"+ ;;+ --allow-dirty)+ ALLOW_DIRTY=1+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"++ensure_tool "taplo" "cargo install taplo-cli --locked"+ensure_tool "hawkeye" "cargo install hawkeye --locked"+ensure_tool "typos" "cargo install typos-cli --locked"++run_step() {+ local name="$1"+ shift+ echo "[${SCRIPT_NAME}] Running ${name}"+ "$@"+}++declare -a WRITE_STEPS=(+ "ci/scripts/rust_fmt.sh|true"+ "ci/scripts/rust_clippy.sh|true"+ "ci/scripts/rust_toml_fmt.sh|true"+ "ci/scripts/license_header.sh|true"+ "ci/scripts/typos_check.sh|true"+ "ci/scripts/doc_prettier_check.sh|true"+)++declare -a READONLY_STEPS=(+ "ci/scripts/rust_docs.sh|false"+)-# For `.toml` format checking-set -e-if ! command -v taplo &> /dev/null; then- echo "Installing taplo using cargo"- cargo install taplo-cli-fi--# For Apache licence header checking-if ! command -v hawkeye &> /dev/null; then- echo "Installing hawkeye using cargo"- cargo install hawkeye --locked-fi--# For spelling checks-if ! command -v typos &> /dev/null; then- echo "Installing typos using cargo"- cargo install typos-cli --locked-fi--ci/scripts/rust_fmt.sh-ci/scripts/rust_clippy.sh-ci/scripts/rust_toml_fmt.sh-ci/scripts/rust_docs.sh-ci/scripts/license_header.sh-ci/scripts/typos_check.sh-ci/scripts/doc_prettier_check.sh+for entry in "${WRITE_STEPS[@]}" "${READONLY_STEPS[@]}"; do+ IFS='|' read -r script_path supports_write <<<"$entry"+ script_name="$(basename "$script_path")"+ args=()+ if [[ "$supports_write" == "true" && "$MODE" == "write" ]]; then+ args+=(--write)+ [[ $ALLOW_DIRTY -eq 1 ]] && args+=(--allow-dirty)+ fi+ if [[ ${#args[@]} -gt 0 ]]; then+ run_step "$script_name" "$script_path" "${args[@]}"+ else+ run_step "$script_name" "$script_path"+ fi+done