Skip to content

Apache DataFusion SQL Query Engine

dev: include dependency checks in the local lint suite (#25399)

Kumar Ujjawal62f039f0d958
diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
index 6cab01c2a5..4d6549d941 100644
--- a/.github/workflows/dependencies.yml
+++ b/.github/workflows/dependencies.yml
@@ -51,9 +51,7 @@ jobs:
with:
rust-version: stable
- name: Check dependencies
- working-directory: dev/depcheck
- run: |
- cargo run --locked
+ run: bash ci/scripts/check_circular_dependencies.sh
detect-unused-dependencies:
name: Detect Unused Dependencies
@@ -62,9 +60,13 @@ jobs:
image: amd64/rust
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Load tool versions
+ run: |
+ . ci/scripts/utils/tool_versions.sh
+ echo "CARGO_MACHETE_VERSION=${CARGO_MACHETE_VERSION}" >> "$GITHUB_ENV"
- name: Install cargo-machete
uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12
with:
+ tool: cargo-machete@${{ env.CARGO_MACHETE_VERSION }}
- name: Detect unused dependencies
- run: cargo machete --with-metadata
+ run: bash ci/scripts/check_unused_dependencies.sh
diff --git a/ci/scripts/check_circular_dependencies.sh b/ci/scripts/check_circular_dependencies.sh
new file mode 100755
index 0000000000..7cdc97b0d6
--- /dev/null
+++ b/ci/scripts/check_circular_dependencies.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Checks for circular dependencies between DataFusion crates with
+# `dev/depcheck`, the same way the "Circular Dependency Check" job does.
+
+set -euo pipefail
+
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+# depcheck is outside the root workspace and finds the root manifest from its
+# own directory, so run Cargo from there.
+cd "${ROOT_DIR}/dev/depcheck"
+
+echo "[${SCRIPT_NAME}] \`cargo run --locked\` in dev/depcheck"
+cargo run --locked
diff --git a/ci/scripts/check_unused_dependencies.sh b/ci/scripts/check_unused_dependencies.sh
new file mode 100755
index 0000000000..0221d50dfd
--- /dev/null
+++ b/ci/scripts/check_unused_dependencies.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Detects unused dependencies with `cargo machete`, the same way the
+# "Detect Unused Dependencies" job does.
+
+set -euo pipefail
+
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+source "${SCRIPT_DIR}/utils/tool_versions.sh"
+
+if ! command -v cargo-machete &> /dev/null; then
+ echo "[${SCRIPT_NAME}] cargo-machete is required. Install it with: cargo install cargo-machete --locked --version ^${CARGO_MACHETE_VERSION}" >&2
+ exit 1
+fi
+
+cd "${ROOT_DIR}"
+
+echo "[${SCRIPT_NAME}] \`cargo machete --with-metadata\`"
+cargo machete --with-metadata
diff --git a/ci/scripts/utils/tool_versions.sh b/ci/scripts/utils/tool_versions.sh
index 237b18b62e..b0a514edd0 100644
--- a/ci/scripts/utils/tool_versions.sh
+++ b/ci/scripts/utils/tool_versions.sh
@@ -22,3 +22,4 @@
PRETTIER_VERSION="2.7.1"
LYCHEE_VERSION="0.23.0"
+CARGO_MACHETE_VERSION="0.9"
diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh
index 5862302d25..7e17dea9db 100755
--- a/dev/rust_lint.sh
+++ b/dev/rust_lint.sh
@@ -113,6 +113,7 @@ ensure_tool "hawkeye" "cargo install hawkeye --locked"
ensure_tool "typos" "cargo install typos-cli --locked"
ensure_tool "lychee" "cargo install lychee --locked --version ${LYCHEE_VERSION}"
ensure_tool "cargo-audit" "cargo install cargo-audit --locked"
+ensure_tool "cargo-machete" "cargo install cargo-machete --locked --version ^${CARGO_MACHETE_VERSION}"
run_step() {
local name="$1"
@@ -135,6 +136,8 @@ declare -a READONLY_STEPS=(
"ci/scripts/check_asf_yaml_status_checks.py|false"
"ci/scripts/markdown_link_check.sh|false"
"ci/scripts/security_audit.sh|false"
+ "ci/scripts/check_circular_dependencies.sh|false"
+ "ci/scripts/check_unused_dependencies.sh|false"
"ci/scripts/rust_docs.sh|false"
)
diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md
index e50e0942d0..a4332352fa 100644
--- a/docs/source/contributor-guide/testing.md
+++ b/docs/source/contributor-guide/testing.md
@@ -254,6 +254,26 @@ The audit fetches the RustSec advisory database. A new advisory or a different
[cargo-audit]: https://github.com/rustsec/rustsec/blob/main/cargo-audit/README.md
+## Dependency Checks
+
+CI runs two dependency checks, and `./dev/rust_lint.sh` runs both:
+
+- `ci/scripts/check_circular_dependencies.sh` builds and runs [`dev/depcheck`],
+ which fails on dependency cycles between DataFusion crates.
+- `ci/scripts/check_unused_dependencies.sh` runs `cargo machete --with-metadata`
+ from the repository root. The lint suite installs [cargo-machete] with the
+ version in `ci/scripts/utils/tool_versions.sh` if it is missing.
+
+To run either check on its own:
+
+```shell
+./ci/scripts/check_circular_dependencies.sh
+./ci/scripts/check_unused_dependencies.sh
+```
+
+[`dev/depcheck`]: https://github.com/apache/datafusion/tree/main/dev/depcheck
+[cargo-machete]: https://github.com/bnjbvr/cargo-machete
+
## Benchmarks
### Criterion Benchmarks