Apache DataFusion SQL Query Engine
dev: include dependency checks in the local lint suite (#25399)
Kumar Ujjawal62f039f0d958
diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.ymlindex 6cab01c2a5..4d6549d941 100644--- a/.github/workflows/dependencies.yml+++ b/.github/workflows/dependencies.yml@@ -51,9 +51,7 @@ jobs:with:rust-version: stable- name: Check dependencies- working-directory: dev/depcheck- run: |- cargo run --locked+ run: bash ci/scripts/check_circular_dependencies.shdetect-unused-dependencies:name: Detect Unused Dependencies@@ -62,9 +60,13 @@ jobs:image: amd64/ruststeps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - name: Load tool versions+ run: |+ . ci/scripts/utils/tool_versions.sh+ echo "CARGO_MACHETE_VERSION=${CARGO_MACHETE_VERSION}" >> "$GITHUB_ENV"- name: Install cargo-macheteuses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12with:- tool: [email protected]+ tool: cargo-machete@${{ env.CARGO_MACHETE_VERSION }}- name: Detect unused dependencies- run: cargo machete --with-metadata+ run: bash ci/scripts/check_unused_dependencies.shdiff --git a/ci/scripts/check_circular_dependencies.sh b/ci/scripts/check_circular_dependencies.shnew file mode 100755index 0000000000..7cdc97b0d6--- /dev/null+++ b/ci/scripts/check_circular_dependencies.sh@@ -0,0 +1,34 @@+#!/usr/bin/env bash+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Checks for circular dependencies between DataFusion crates with+# `dev/depcheck`, the same way the "Circular Dependency Check" job does.++set -euo pipefail++SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"++# depcheck is outside the root workspace and finds the root manifest from its+# own directory, so run Cargo from there.+cd "${ROOT_DIR}/dev/depcheck"++echo "[${SCRIPT_NAME}] \`cargo run --locked\` in dev/depcheck"+cargo run --lockeddiff --git a/ci/scripts/check_unused_dependencies.sh b/ci/scripts/check_unused_dependencies.shnew file mode 100755index 0000000000..0221d50dfd--- /dev/null+++ b/ci/scripts/check_unused_dependencies.sh@@ -0,0 +1,39 @@+#!/usr/bin/env bash+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Detects unused dependencies with `cargo machete`, the same way the+# "Detect Unused Dependencies" job does.++set -euo pipefail++SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"++source "${SCRIPT_DIR}/utils/tool_versions.sh"++if ! command -v cargo-machete &> /dev/null; then+ echo "[${SCRIPT_NAME}] cargo-machete is required. Install it with: cargo install cargo-machete --locked --version ^${CARGO_MACHETE_VERSION}" >&2+ exit 1+fi++cd "${ROOT_DIR}"++echo "[${SCRIPT_NAME}] \`cargo machete --with-metadata\`"+cargo machete --with-metadatadiff --git a/ci/scripts/utils/tool_versions.sh b/ci/scripts/utils/tool_versions.shindex 237b18b62e..b0a514edd0 100644--- a/ci/scripts/utils/tool_versions.sh+++ b/ci/scripts/utils/tool_versions.sh@@ -22,3 +22,4 @@PRETTIER_VERSION="2.7.1"LYCHEE_VERSION="0.23.0"+CARGO_MACHETE_VERSION="0.9"diff --git a/dev/rust_lint.sh b/dev/rust_lint.shindex 5862302d25..7e17dea9db 100755--- a/dev/rust_lint.sh+++ b/dev/rust_lint.sh@@ -113,6 +113,7 @@ ensure_tool "hawkeye" "cargo install hawkeye --locked"ensure_tool "typos" "cargo install typos-cli --locked"ensure_tool "lychee" "cargo install lychee --locked --version ${LYCHEE_VERSION}"ensure_tool "cargo-audit" "cargo install cargo-audit --locked"+ensure_tool "cargo-machete" "cargo install cargo-machete --locked --version ^${CARGO_MACHETE_VERSION}"run_step() {local name="$1"@@ -135,6 +136,8 @@ declare -a READONLY_STEPS=("ci/scripts/check_asf_yaml_status_checks.py|false""ci/scripts/markdown_link_check.sh|false""ci/scripts/security_audit.sh|false"+ "ci/scripts/check_circular_dependencies.sh|false"+ "ci/scripts/check_unused_dependencies.sh|false""ci/scripts/rust_docs.sh|false")diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.mdindex e50e0942d0..a4332352fa 100644--- a/docs/source/contributor-guide/testing.md+++ b/docs/source/contributor-guide/testing.md@@ -254,6 +254,26 @@ The audit fetches the RustSec advisory database. A new advisory or a different[cargo-audit]: https://github.com/rustsec/rustsec/blob/main/cargo-audit/README.md+## Dependency Checks++CI runs two dependency checks, and `./dev/rust_lint.sh` runs both:++- `ci/scripts/check_circular_dependencies.sh` builds and runs [`dev/depcheck`],+ which fails on dependency cycles between DataFusion crates.+- `ci/scripts/check_unused_dependencies.sh` runs `cargo machete --with-metadata`+ from the repository root. The lint suite installs [cargo-machete] with the+ version in `ci/scripts/utils/tool_versions.sh` if it is missing.++To run either check on its own:++```shell+./ci/scripts/check_circular_dependencies.sh+./ci/scripts/check_unused_dependencies.sh+```++[`dev/depcheck`]: https://github.com/apache/datafusion/tree/main/dev/depcheck+[cargo-machete]: https://github.com/bnjbvr/cargo-machete+## Benchmarks### Criterion Benchmarks