Skip to content

Apache DataFusion SQL Query Engine

feat(CI): Introduce `xtask` to keep local test and CI commands in sync (#24083)

Yongting Youb9a005367720
diff --git a/.cargo/config.toml b/.cargo/config.toml
new file mode 100644
index 0000000000..9dae805eb1
--- /dev/null
+++ b/.cargo/config.toml
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[alias]
+xtask = "run --quiet --locked --package datafusion-xtask --"
+ci-step = "run --quiet --locked --package datafusion-xtask -- ci step"
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 7a943088e2..7839547980 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -15,8 +15,21 @@
# specific language governing permissions and limitations
# under the License.
-# For some actions, we use Runs-On to run them on ASF infrastructure: https://datafusion.apache.org/contributor-guide/#ci-runners
-
+# # CI runners
+#
+# Some actions use Runs-On to run on ASF infrastructure:
+# https://datafusion.apache.org/contributor-guide/#ci-runners
+#
+# # `cargo xtask`
+#
+# Individual CI commands are implemented with `cargo xtask` to make them easier
+# to reproduce locally.
+#
+# See `xtask/README.md` for details about DataFusion's `xtask` tooling.
+#
+# It is being rolled out incrementally across all workflows.
+# Track progress at https://github.com/apache/datafusion/issues/24487.
+#
name: Rust
concurrency:
@@ -63,11 +76,8 @@ jobs:
shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs
save-if: ${{ github.ref_name == 'main' }}
- name: Prepare cargo build
- run: |
- # Adding `--locked` here to assert that the `Cargo.lock` file is up to
- # date with the manifest. When this fails, please make sure to commit
- # the changes to `Cargo.lock` after building with the updated manifest.
- cargo check --profile ci --workspace --all-targets --features integration-tests --locked
+ # `--locked` asserts that Cargo.lock is up to date with the manifest.
+ run: cargo xtask ci step check workspace
# Check datafusion-common features
#
@@ -86,12 +96,12 @@ jobs:
with:
rust-version: stable
- name: Check datafusion-common (default features)
- run: cargo check --profile ci --all-targets -p datafusion-common
+ run: cargo xtask ci step check datafusion-common default
#
# Note: Only check libraries (not --all-targets) to cover end user APIs
#
- name: Check datafusion-common (no-default-features)
- run: cargo check --profile ci --no-default-features -p datafusion-common
+ run: cargo xtask ci step check datafusion-common no-default
# Note: don't check other feature flags as datafusion-common is not typically used standalone
# Check datafusion-substrait features
@@ -116,20 +126,20 @@ jobs:
save-if: false # set in linux-test
shared-key: "amd-ci"
- name: Check datafusion-substrait (default features)
- run: cargo check --profile ci --all-targets -p datafusion-substrait
+ run: cargo xtask ci step check datafusion-substrait default
#
# Note: Only check libraries (not --all-targets) to cover end user APIs
#
- name: Check datafusion-substrait (no-default-features)
- run: cargo check --profile ci --no-default-features -p datafusion-substrait
+ run: cargo xtask ci step check datafusion-substrait no-default
- name: Check datafusion-substrait (physical)
- run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=physical
+ run: cargo xtask ci step check datafusion-substrait physical
- name: Install cmake
run: |
# note the builder setup runs apt-get update / installs protobuf compiler
apt-get install -y cmake
- name: Check datafusion-substrait (protoc)
- run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=protoc
+ run: cargo xtask ci step check datafusion-substrait protoc
# Check datafusion-proto features
#
@@ -149,18 +159,18 @@ jobs:
with:
rust-version: stable
- name: Check datafusion-proto (default features)
- run: cargo check --profile ci --all-targets -p datafusion-proto
+ run: cargo xtask ci step check datafusion-proto default
#
# Note: Only check libraries (not --all-targets) to cover end user APIs
#
- name: Check datafusion-proto (no-default-features)
- run: cargo check --profile ci --no-default-features -p datafusion-proto
+ run: cargo xtask ci step check datafusion-proto no-default
- name: Check datafusion-proto (json)
- run: cargo check --profile ci --no-default-features -p datafusion-proto --features=json
+ run: cargo xtask ci step check datafusion-proto json
- name: Check datafusion-proto (parquet)
- run: cargo check --profile ci --no-default-features -p datafusion-proto --features=parquet
+ run: cargo xtask ci step check datafusion-proto parquet
- name: Check datafusion-proto (avro)
- run: cargo check --profile ci --no-default-features -p datafusion-proto --features=avro
+ run: cargo xtask ci step check datafusion-proto avro
# Check datafusion-ffi features
#
@@ -180,7 +190,7 @@ jobs:
with:
rust-version: stable
- name: Check datafusion-ffi (no-default-features)
- run: cargo check --profile ci --no-default-features -p datafusion-ffi
+ run: cargo xtask ci step check datafusion-ffi no-default
# Check datafusion crate features
@@ -206,48 +216,48 @@ jobs:
save-if: false # set in linux-test
shared-key: "amd-ci"
- name: Check datafusion (default features)
- run: cargo check --profile ci --all-targets -p datafusion
+ run: cargo xtask ci step check datafusion default
#
# Note: Only check libraries (not --all-targets) to cover end user APIs
#
- name: Check datafusion (no-default-features)
- run: cargo check --profile ci --no-default-features -p datafusion
+ run: cargo xtask ci step check datafusion no-default
- name: Check datafusion (nested_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=nested_expressions
+ run: cargo xtask ci step check datafusion nested_expressions
- name: Check datafusion (array_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=array_expressions
+ run: cargo xtask ci step check datafusion array_expressions
- name: Check datafusion (avro)
- run: cargo check --profile ci --no-default-features -p datafusion --features=avro
+ run: cargo xtask ci step check datafusion avro
- name: Check datafusion (backtrace)
- run: cargo check --profile ci --no-default-features -p datafusion --features=backtrace
+ run: cargo xtask ci step check datafusion backtrace
- name: Check datafusion (compression)
- run: cargo check --profile ci --no-default-features -p datafusion --features=compression
+ run: cargo xtask ci step check datafusion compression
- name: Check datafusion (crypto_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=crypto_expressions
+ run: cargo xtask ci step check datafusion crypto_expressions
- name: Check datafusion (datetime_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=datetime_expressions
+ run: cargo xtask ci step check datafusion datetime_expressions
- name: Check datafusion (encoding_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=encoding_expressions
+ run: cargo xtask ci step check datafusion encoding_expressions
- name: Check datafusion (force_hash_collisions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=force_hash_collisions
+ run: cargo xtask ci step check datafusion force_hash_collisions
- name: Check datafusion (math_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=math_expressions
+ run: cargo xtask ci step check datafusion math_expressions
- name: Check datafusion (parquet)
- run: cargo check --profile ci --no-default-features -p datafusion --features=parquet
+ run: cargo xtask ci step check datafusion parquet
- name: Check datafusion (regex_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=regex_expressions
+ run: cargo xtask ci step check datafusion regex_expressions
- name: Check datafusion (recursive_protection)
- run: cargo check --profile ci --no-default-features -p datafusion --features=recursive_protection
+ run: cargo xtask ci step check datafusion recursive_protection
- name: Check datafusion (serde)
- run: cargo check --profile ci --no-default-features -p datafusion --features=serde
+ run: cargo xtask ci step check datafusion serde
- name: Check datafusion (sql)
- run: cargo check --profile ci --no-default-features -p datafusion --features=sql
+ run: cargo xtask ci step check datafusion sql
- name: Check datafusion (string_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=string_expressions
+ run: cargo xtask ci step check datafusion string_expressions
- name: Check datafusion (unicode_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion --features=unicode_expressions
+ run: cargo xtask ci step check datafusion unicode_expressions
- name: Check parquet encryption (parquet_encryption)
- run: cargo check --profile ci --no-default-features -p datafusion --features=parquet_encryption
+ run: cargo xtask ci step check datafusion parquet_encryption
# Check datafusion-functions crate features
#
@@ -266,26 +276,26 @@ jobs:
with:
rust-version: stable
- name: Check datafusion-functions (default features)
- run: cargo check --profile ci --all-targets -p datafusion-functions
+ run: cargo xtask ci step check datafusion-functions default
#
# Note: Only check libraries (not --all-targets) to cover end user APIs
#
- name: Check datafusion-functions (no-default-features)
- run: cargo check --profile ci --no-default-features -p datafusion-functions
+ run: cargo xtask ci step check datafusion-functions no-default
- name: Check datafusion-functions (crypto_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=crypto_expressions
+ run: cargo xtask ci step check datafusion-functions crypto_expressions
- name: Check datafusion-functions (datetime_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=datetime_expressions
+ run: cargo xtask ci step check datafusion-functions datetime_expressions
- name: Check datafusion-functions (encoding_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=encoding_expressions
+ run: cargo xtask ci step check datafusion-functions encoding_expressions
- name: Check datafusion-functions (math_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=math_expressions
+ run: cargo xtask ci step check datafusion-functions math_expressions
- name: Check datafusion-functions (regex_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=regex_expressions
+ run: cargo xtask ci step check datafusion-functions regex_expressions
- name: Check datafusion-functions (string_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=string_expressions
+ run: cargo xtask ci step check datafusion-functions string_expressions
- name: Check datafusion-functions (unicode_expressions)
- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=unicode_expressions
+ run: cargo xtask ci step check datafusion-functions unicode_expressions
# Check datafusion-spark crate features
#
@@ -347,19 +357,7 @@ jobs:
- name: Run tests (excluding doctests and datafusion-cli)
env:
RUST_BACKTRACE: 1
- run: |
- cargo llvm-cov \
- --profile ci \
- --exclude datafusion-examples \
- --exclude ffi_example_table_provider \
- --exclude datafusion-cli \
- --workspace \
- --lib \
- --tests \
- --bins \
- --features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \
- --codecov \
- --output-path target/codecov.json
+ run: cargo xtask ci step test workspace
- parallel:
- name: Verify Working Directory Clean
run: git diff --exit-code
@@ -405,7 +403,7 @@ jobs:
AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword
TEST_STORAGE_INTEGRATION: 1
AWS_ALLOW_HTTP: true
- run: cargo test --features backtrace --profile ci -p datafusion-cli --lib --tests --bins
+ run: cargo xtask ci step test cli
- name: Verify Working Directory Clean
run: git diff --exit-code
@@ -458,7 +456,7 @@ jobs:
with:
rust-version: stable
- name: Run doctests
- run: cargo test --profile ci --doc --features avro,json
+ run: cargo xtask ci step test doctest
- name: Verify Working Directory Clean
run: git diff --exit-code
@@ -532,11 +530,8 @@ jobs:
mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data
- name: Verify that benchmark queries return expected results
run: |
- # increase stack size to fix stack overflow
- export RUST_MIN_STACK=20971520
- export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`
- cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1
- INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests
+ cargo xtask ci step test benchmark-plan
+ cargo xtask ci step test benchmark-sqllogic
- name: Verify Working Directory Clean
run: git diff --exit-code
@@ -571,9 +566,7 @@ jobs:
with:
rust-version: stable
- name: Run sqllogictest
- run: |
- cd datafusion/sqllogictest
- PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --features backtrace --profile ci --features=postgres --test sqllogictests
+ run: cargo xtask ci step test postgres
env:
# use postgres for the host here because we have specified a container for the job
POSTGRES_HOST: postgres
@@ -600,7 +593,7 @@ jobs:
# command cannot be run for all the .slt files. Run it for just one that works (limit.slt)
# until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed
# and this command can be run without filters.
- run: cargo test -p datafusion-sqllogictest --test sqllogictests --features substrait -- --substrait-round-trip limit.slt
+ run: cargo xtask ci step test substrait
# Temporarily commenting out the Windows flow, the reason is enormously slow running build
# Waiting for new Windows 2025 github runner
@@ -638,7 +631,7 @@ jobs:
uses: ./.github/actions/setup-macos-aarch64-builder
- name: Run datafusion-ffi tests
shell: bash
- run: cargo test --profile ci -p datafusion-ffi --lib --tests --features integration-tests
+ run: cargo xtask ci step test ffi
vendor:
name: Verify Vendored Code
diff --git a/Cargo.lock b/Cargo.lock
index 44cef5185d..f1fbb5b580 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2748,6 +2748,13 @@ dependencies = [
"wasm-bindgen-test",
]
+[[package]]
+name = "datafusion-xtask"
+version = "0.1.0"
+dependencies = [
+ "insta",
+]
+
[[package]]
name = "deranged"
version = "0.5.8"
diff --git a/Cargo.toml b/Cargo.toml
index f9b2cac5c5..d529f551e9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -66,6 +66,7 @@ members = [
"benchmarks",
"datafusion/macros",
"datafusion/doc",
+ "xtask",
]
exclude = ["dev/depcheck"]
resolver = "2"
diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml
new file mode 100644
index 0000000000..ea4a63aa9f
--- /dev/null
+++ b/xtask/Cargo.toml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-xtask"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+publish = false
+
+[dev-dependencies]
+insta = { workspace = true }
+
+[lints]
+workspace = true
diff --git a/xtask/README.md b/xtask/README.md
new file mode 100644
index 0000000000..9d8da45626
--- /dev/null
+++ b/xtask/README.md
@@ -0,0 +1,42 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# DataFusion xtask
+
+This directory contains DataFusion's project automation implemented using the
+[`xtask` pattern](https://github.com/matklad/cargo-xtask).
+
+## Usage
+
+Run the following command from the repository root for the available commands,
+arguments, and examples:
+
+```shell
+cargo xtask help
+```
+
+## CI runner
+
+`cargo xtask ci [args]` domain is used for organizing commands used in GitHub CI.
+
+For example, `cargo xtask ci step test workspace` runs DataFusion's default CI test suite. Developers can run the same command to reproduce that GitHub Actions step locally. Append `--explain` to display the underlying command without running it.
+
+The goal is to keep local development and GitHub CI runs in sync. Each CI step command is defined once in this in-repository Rust binary and is invoked the same way locally and in GitHub Actions.
+
+Tracking issue: https://github.com/apache/datafusion/issues/24487
diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rs
new file mode 100644
index 0000000000..f40cb61818
--- /dev/null
+++ b/xtask/src/ci_steps.rs
@@ -0,0 +1,793 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! CI steps that can be inspected or run through `cargo xtask`.
+
+use crate::Result;
+use std::env;
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+type StepResult<T> = std::result::Result<T, StepError>;
+type StepRunner = fn(&StepContext, &[String]) -> StepResult<CiCommand>;
+
+const CI_COMMAND: &str = "cargo xtask ci step";
+const CI_SHORTCUT: &str = "cargo ci-step";
+const TPCH_DATA_DIR: &str = "datafusion/sqllogictest/test_files/tpch/data";
+
+/// Metadata and implementation for one CI step.
+///
+/// Adding an entry to `CI_STEPS` makes the step dispatchable and includes its
+/// usage and description in the generated help text.
+struct StepInfo {
+ command: &'static str,
+ help_usage: &'static str,
+ help_examples: &'static [&'static str],
+ help_description: &'static str,
+ error_message: &'static str,
+ runner: StepRunner,
+}
+
+static CI_STEPS: &[StepInfo] = &[
+ StepInfo {
+ command: "check",
+ help_usage: "check <workspace|package> [default|no-default|feature] [--explain]",
+ help_examples: &["check workspace", "check datafusion default"],
+ help_description: "Check workspace or package compilation",
+ error_message: "Cargo check step failed",
+ runner: StepContext::run_check,
+ },
+ StepInfo {
+ command: "test",
+ help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait> [--explain]",
+ help_examples: &["test cli", "test workspace"],
+ help_description: "Run a CI test suite",
+ error_message: "Cargo test step failed",
+ runner: StepContext::run_test,
+ },
+];
+
+pub(crate) fn help() -> String {
+ let mut help = format!(
+ "DataFusion CI commands\n\nUsage:\n {CI_COMMAND} <step-name> [args] [--explain]\n\nExamples:\n",
+ );
+ for step in CI_STEPS {
+ if let Some(example) = step.help_examples.first() {
+ help.push_str(&format!(" {CI_COMMAND} {example}\n"));
+ }
+ }
+ help.push_str(&format!(" {CI_COMMAND} test workspace --explain\n"));
+
+ let command_width = CI_STEPS
+ .iter()
+ .map(|step| step.command.len())
+ .max()
+ .unwrap_or_default();
+ help.push_str("\nAvailable steps:\n");
+ for step in CI_STEPS {
+ help.push_str(&format!(
+ " {:command_width$} {}\n",
+ step.command, step.help_description
+ ));
+ }
+
+ help.push_str(&format!(
+ "\nShortcut:\n # `{CI_SHORTCUT}` is short for `{CI_COMMAND}`.\n {CI_SHORTCUT} check workspace\n\nFor more details:\n {CI_COMMAND} check --help\n"
+ ));
+ help
+}
+
+fn step_help(step: &StepInfo) -> String {
+ let mut help = format!(
+ "DataFusion CI command: {}\n\n{}\n\nUsage:\n {CI_COMMAND} {}\n\nExamples:\n",
+ step.command, step.help_description, step.help_usage,
+ );
+ for example in step.help_examples {
+ help.push_str(&format!(" {CI_COMMAND} {example}\n"));
+ }
+ if let Some(example) = step.help_examples.first() {
+ help.push_str(&format!(
+ "\nUse '--explain' to show the full command:\n {CI_COMMAND} {example} --explain\n"
+ ));
+ }
+ help
+}
+
+fn find_step(command: &str) -> Result<&'static StepInfo> {
+ CI_STEPS
+ .iter()
+ .find(|step| step.command == command)
+ .ok_or_else(|| format!("unknown CI step `{command}`"))
+}
+
+pub(crate) fn is_help_arg(arg: &str) -> bool {
+ matches!(arg, "help" | "-h" | "--help")
+}
+
+pub(crate) fn run(root: &Path, args: &[String]) -> Result<()> {
+ StepContext::new(root).run(args)
+}
+
+struct StepContext {
+ root: PathBuf,
+ env_reader: fn(&str) -> Result<String>,
+}
+
+impl StepContext {
+ fn new(root: &Path) -> Self {
+ Self {
+ root: root.to_path_buf(),
+ env_reader: required_env,
+ }
+ }
+
+ fn run(&self, args: &[String]) -> Result<()> {
+ match args {
+ [] => {
+ print!("{}", help());
+ Ok(())
+ }
+ [help_arg] if is_help_arg(help_arg) => {
+ print!("{}", help());
+ Ok(())
+ }
+ [step_name, help_arg] if is_help_arg(help_arg) => {
+ print!("{}", step_help(find_step(step_name)?));
+ Ok(())
+ }
+ step_args => {
+ let (action, step, command) = self.ci_step(step_args)?;
+ match action {
+ StepAction::Execute => command.execute(step.error_message),
+ StepAction::Explain => {
+ command.explain();
+ Ok(())
+ }
+ }
+ }
+ }
+ }
+
+ /// Parses a CI step into an action and its complete command description.
+ fn ci_step(
+ &self,
+ args: &[String],
+ ) -> Result<(StepAction, &'static StepInfo, CiCommand)> {
+ let (action, args) = match args.split_last() {
+ Some((arg, args)) if arg == "--explain" => (StepAction::Explain, args),
+ _ => (StepAction::Execute, args),
+ };
+ let Some((step, args)) = args.split_first() else {
+ return Err(format!("missing CI step\n\n{}", help()));
+ };
+
+ let step = find_step(step)?;
+ let command = (step.runner)(self, args).map_err(|error| error.render(step))?;
+ Ok((action, step, command))
+ }
+
+ /// Implementation for `cargo xtask ci step check [args]`
+ fn run_check(&self, args: &[String]) -> StepResult<CiCommand> {
+ let args = args.iter().map(String::as_str).collect::<Vec<_>>();
+ let mut command = self.cargo();
+ command.args(["check", "--profile", "ci"]);
+
+ match args.as_slice() {
+ ["workspace"] => {
+ command.args([
+ "--workspace",
+ "--all-targets",
+ "--features",
+ "integration-tests",
+ "--locked",
+ ]);
+ }
+ [package, "default"] => {
+ command.args(["--all-targets", "-p", package]);
+ }
+ [package, "no-default"] => {
+ command.args(["--no-default-features", "-p", package]);
+ }
+ [package, feature] => {
+ command.args([
+ "--no-default-features",
+ "-p",
+ package,
+ "--features",
+ feature,
+ ]);
+ }
+ _ => return Err(StepError::Usage),
+ }
+
+ Ok(command)
+ }
+
+ /// Implementation for `cargo xtask ci step test [args]`
+ fn run_test(&self, args: &[String]) -> StepResult<CiCommand> {
+ let [variant] = args else {
+ return Err(StepError::Usage);
+ };
+
+ let mut command = self.cargo();
+ match variant.as_str() {
+ "workspace" => {
+ command.args([
+ "llvm-cov",
+ "--profile",
+ "ci",
+ "--exclude",
+ "datafusion-examples",
+ "--exclude",
+ "ffi_example_table_provider",
+ "--exclude",
+ "datafusion-cli",
+ "--workspace",
+ "--lib",
+ "--tests",
+ "--bins",
+ "--features",
+ "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait",
+ "--codecov",
+ "--output-path",
+ "target/codecov.json",
+ ]);
+ }
+ "cli" => {
+ command.args([
+ "test",
+ "--features",
+ "backtrace",
+ "--profile",
+ "ci",
+ "-p",
+ "datafusion-cli",
+ "--lib",
+ "--tests",
+ "--bins",
+ ]);
+ }
+ "doctest" => {
+ command.args([
+ "test",
+ "--profile",
+ "ci",
+ "--doc",
+ "--features",
+ "avro,json",
+ ]);
+ }
+ "ffi" => {
+ command.args([
+ "test",
+ "--profile",
+ "ci",
+ "-p",
+ "datafusion-ffi",
+ "--lib",
+ "--tests",
+ "--features",
+ "integration-tests",
+ ]);
+ }
+ "benchmark-plan" => {
+ command
+ .args([
+ "test",
+ "plan_q",
+ "--package",
+ "datafusion-benchmarks",
+ "--profile",
+ "ci",
+ "--features=ci",
+ "--",
+ "--test-threads=1",
+ ])
+ .env("RUST_MIN_STACK", "20971520")
+ .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR));
+ }
+ "benchmark-sqllogic" => {
+ command
+ .args([
+ "test",
+ "--features",
+ "backtrace,parquet_encryption,substrait",
+ "--profile",
+ "ci",
+ "--package",
+ "datafusion-sqllogictest",
+ "--test",
+ "sqllogictests",
+ ])
+ .env("RUST_MIN_STACK", "20971520")
+ .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR))
+ .env("INCLUDE_TPCH", "true");
+ }
+ "postgres" => {
+ let host = self
+ .required_env("POSTGRES_HOST")
+ .map_err(StepError::Message)?;
+ let port = self
+ .required_env("POSTGRES_PORT")
+ .map_err(StepError::Message)?;
+ let uri = format!("postgresql://postgres:postgres@{host}:{port}/db_test");
+ command
+ .args([
+ "test",
+ "--features",
+ "backtrace",
+ "--profile",
+ "ci",
+ "--features=postgres",
+ "--test",
+ "sqllogictests",
+ ])
+ .current_dir(self.root.join("datafusion/sqllogictest"))
+ .env("PG_COMPAT", "true")
+ .env("PG_URI", uri);
+ }
+ "substrait" => {
+ command.args([
+ "test",
+ "-p",
+ "datafusion-sqllogictest",
+ "--test",
+ "sqllogictests",
+ "--features",
+ "substrait",
+ "--",
+ "--substrait-round-trip",
+ "limit.slt",
+ ]);
+ }
+ _ => return Err(StepError::Usage),
+ }
+
+ Ok(command)
+ }
+
+ fn cargo(&self) -> CiCommand {
+ CiCommand::new("cargo", &self.root)
+ }
+
+ fn required_env(&self, name: &str) -> Result<String> {
+ (self.env_reader)(name)
+ }
+}
+
+#[derive(Debug, PartialEq)]
+enum StepAction {
+ Execute,
+ Explain,
+}
+
+#[derive(Debug)]
+enum StepError {
+ Usage,
+ Message(String),
+}
+
+impl StepError {
+ fn render(self, step: &StepInfo) -> String {
+ match self {
+ Self::Usage => format!("usage: {CI_COMMAND} {}", step.help_usage),
+ Self::Message(message) => message,
+ }
+ }
+}
+
+/// Command shared by invocation and `--explain`
+struct CiCommand {
+ command: Command,
+}
+
+impl CiCommand {
+ fn new(program: impl AsRef<OsStr>, current_dir: impl AsRef<Path>) -> Self {
+ let mut command = Command::new(program);
+ command.current_dir(current_dir);
+ Self { command }
+ }
+
+ fn args<I, S>(&mut self, args: I) -> &mut Self
+ where
+ I: IntoIterator<Item = S>,
+ S: AsRef<OsStr>,
+ {
+ self.command.args(args);
+ self
+ }
+
+ fn env<K, V>(&mut self, key: K, value: V) -> &mut Self
+ where
+ K: AsRef<OsStr>,
+ V: AsRef<OsStr>,
+ {
+ self.command.env(key, value);
+ self
+ }
+
+ fn current_dir(&mut self, directory: impl AsRef<Path>) -> &mut Self {
+ self.command.current_dir(directory);
+ self
+ }
+
+ fn explain(&self) {
+ println!("{}", self.full_command());
+ }
+
+ fn execute(mut self, error_message: &str) -> Result<()> {
+ println!("+ {}", self.full_command());
+ let status = self
+ .command
+ .stdin(Stdio::inherit())
+ .stdout(Stdio::inherit())
+ .stderr(Stdio::inherit())
+ .status()
+ .map_err(|error| format!("{error_message}: {error}"))?;
+
+ if status.success() {
+ Ok(())
+ } else {
+ Err(format!("{error_message}: {status}"))
+ }
+ }
+
+ fn full_command(&self) -> String {
+ let mut lines = Vec::new();
+
+ if let Some(current_dir) = self.command.get_current_dir() {
+ lines.push(format!("cd {} &&", shell_quote(current_dir.as_os_str())));
+ }
+
+ for (key, value) in self.command.get_envs() {
+ let Some(value) = value else {
+ continue;
+ };
+ lines.push(format!("{}={}", shell_quote(key), shell_quote(value)));
+ }
+
+ lines.extend(shell_command_lines(
+ self.command.get_program(),
+ self.command.get_args(),
+ ));
+ lines.join(" \\\n")
+ }
+}
+
+/// Groups option-value pairs so the result reads like a hand-written command.
+fn shell_command_lines<'a>(
+ program: &OsStr,
+ args: impl Iterator<Item = &'a OsStr>,
+) -> Vec<String> {
+ let args = args.collect::<Vec<_>>();
+ let mut lines = vec![shell_quote(program)];
+ let mut index = 0;
+
+ // Keep command names and leading positional arguments together, such as
+ // `cargo check` and `cargo test plan_q`.
+ while index < args.len() && !is_option(args[index]) {
+ lines[0].push(' ');
+ lines[0].push_str(&shell_quote(args[index]));
+ index += 1;
+ }
+
+ while index < args.len() {
+ let arg = args[index];
+ let mut line = shell_quote(arg);
+ index += 1;
+
+ if arg == "--" {
+ while index < args.len() {
+ line.push(' ');
+ line.push_str(&shell_quote(args[index]));
+ index += 1;
+ }
+ } else if !arg.to_string_lossy().contains('=')
+ && index < args.len()
+ && !is_option(args[index])
+ {
+ line.push(' ');
+ line.push_str(&shell_quote(args[index]));
+ index += 1;
+ }
+
+ lines.push(line);
+ }
+
+ lines
+}
+
+fn is_option(value: &OsStr) -> bool {
+ value.to_string_lossy().starts_with('-')
+}
+
+/// Quotes one shell word when it contains characters that need protection.
+fn shell_quote(value: &OsStr) -> String {
+ let value = value.to_string_lossy();
+ if !value.is_empty()
+ && value.bytes().all(|byte| {
+ byte.is_ascii_alphanumeric()
+ || matches!(
+ byte,
+ b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-'
+ )
+ })
+ {
+ value.into_owned()
+ } else {
+ format!("'{}'", value.replace('\'', "'\"'\"'"))
+ }
+}
+
+fn required_env(name: &str) -> Result<String> {
+ env::var(name)
+ .map_err(|_| format!("required environment variable `{name}` is not set"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn args(args: &[&str]) -> Vec<String> {
+ args.iter().map(|arg| (*arg).to_string()).collect()
+ }
+
+ fn test_env(name: &str) -> Result<String> {
+ match name {
+ "POSTGRES_HOST" => Ok("postgres.example".to_string()),
+ "POSTGRES_PORT" => Ok("15432".to_string()),
+ _ => Err(format!("unexpected environment variable `{name}`")),
+ }
+ }
+
+ fn context() -> StepContext {
+ StepContext {
+ root: PathBuf::from("/workspace"),
+ env_reader: test_env,
+ }
+ }
+
+ #[test]
+ fn explain_builds_the_same_command_as_execute() {
+ let context = context();
+ let (execute_action, _, execute_command) = context
+ .ci_step(&args(&["check", "datafusion", "default"]))
+ .unwrap();
+ let (explain_action, _, explain_command) = context
+ .ci_step(&args(&["check", "datafusion", "default", "--explain"]))
+ .unwrap();
+
+ assert_eq!(execute_action, StepAction::Execute);
+ assert_eq!(explain_action, StepAction::Explain);
+ assert_eq!(
+ execute_command.full_command(),
+ explain_command.full_command()
+ );
+ }
+
+ #[test]
+ fn explain_flag_must_be_last() {
+ let error = context()
+ .ci_step(&args(&["test", "--explain", "workspace"]))
+ .err()
+ .unwrap();
+
+ assert_eq!(
+ error,
+ "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait> [--explain]"
+ );
+ }
+
+ // Test all available commands, ensure their executed command is expected
+ #[test]
+ fn explain_all_available_ci_commands() {
+ struct Cmd {
+ // Actual command to run
+ command: &'static [&'static str],
+ // Full explain output
+ expected: fn(&str),
+ }
+
+ let commands = [
+ Cmd {
+ command: &["check", "workspace"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo check \
+--profile ci \
+--workspace \
+--all-targets \
+--features integration-tests \
+--locked
+")
+ },
+ },
+ Cmd {
+ command: &["check", "datafusion", "default"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo check \
+--profile ci \
+--all-targets \
+-p datafusion
+")
+ },
+ },
+ Cmd {
+ command: &["check", "datafusion", "no-default"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo check \
+--profile ci \
+--no-default-features \
+-p datafusion
+")
+ },
+ },
+ Cmd {
+ command: &["check", "datafusion", "parquet"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo check \
+--profile ci \
+--no-default-features \
+-p datafusion \
+--features parquet
+")
+ },
+ },
+ Cmd {
+ command: &["test", "workspace"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo llvm-cov \
+--profile ci \
+--exclude datafusion-examples \
+--exclude ffi_example_table_provider \
+--exclude datafusion-cli \
+--workspace \
+--lib \
+--tests \
+--bins \
+--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \
+--codecov \
+--output-path target/codecov.json
+")
+ },
+ },
+ Cmd {
+ command: &["test", "cli"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo test \
+--features backtrace \
+--profile ci \
+-p datafusion-cli \
+--lib \
+--tests \
+--bins
+")
+ },
+ },
+ Cmd {
+ command: &["test", "doctest"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo test \
+--profile ci \
+--doc \
+--features avro,json
+")
+ },
+ },
+ Cmd {
+ command: &["test", "ffi"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo test \
+--profile ci \
+-p datafusion-ffi \
+--lib \
+--tests \
+--features integration-tests
+")
+ },
+ },
+ Cmd {
+ command: &["test", "benchmark-plan"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+RUST_MIN_STACK=20971520 \
+TPCH_DATA=/workspace/datafusion/sqllogictest/test_files/tpch/data \
+cargo test plan_q \
+--package datafusion-benchmarks \
+--profile ci \
+--features=ci \
+-- --test-threads=1
+")
+ },
+ },
+ Cmd {
+ command: &["test", "benchmark-sqllogic"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+INCLUDE_TPCH=true \
+RUST_MIN_STACK=20971520 \
+TPCH_DATA=/workspace/datafusion/sqllogictest/test_files/tpch/data \
+cargo test \
+--features backtrace,parquet_encryption,substrait \
+--profile ci \
+--package datafusion-sqllogictest \
+--test sqllogictests
+")
+ },
+ },
+ Cmd {
+ command: &["test", "postgres"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace/datafusion/sqllogictest && \
+PG_COMPAT=true \
+PG_URI=postgresql://postgres:[email protected]:15432/db_test \
+cargo test \
+--features backtrace \
+--profile ci \
+--features=postgres \
+--test sqllogictests
+")
+ },
+ },
+ Cmd {
+ command: &["test", "substrait"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo test \
+-p datafusion-sqllogictest \
+--test sqllogictests \
+--features substrait \
+-- --substrait-round-trip limit.slt
+")
+ },
+ },
+ ];
+
+ let context = context();
+ for Cmd { command, expected } in commands {
+ let explain_args = command
+ .iter()
+ .copied()
+ .chain(std::iter::once("--explain"))
+ .collect::<Vec<_>>();
+ let (action, _, actual) = context.ci_step(&args(&explain_args)).unwrap();
+
+ assert_eq!(action, StepAction::Explain);
+ expected(&actual.full_command());
+ }
+ }
+}
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
new file mode 100644
index 0000000000..c714a1da34
--- /dev/null
+++ b/xtask/src/main.rs
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! # DataFusion repository development tools
+//!
+//! ## Example
+//!
+//! Run the default CI test suite locally:
+//!
+//! ```sh
+//! cargo xtask ci step test workspace
+//! ```
+//!
+//! Implementing the command inside `xtask` can make it easier to maintain. For example,
+//! the local test runs and CI runs can keep in sync.
+//!
+//! ## Mechanism
+//!
+//! `xtask` is an internal Rust binary in this workspace. The `cargo xtask` alias
+//! builds and runs that binary, forwarding the remaining arguments to it.
+//!
+//! There is no external dependency required.
+//!
+//! ## Reference
+//!
+//! The [`xtask` convention](https://github.com/matklad/cargo-xtask) keeps project
+//! automation in ordinary Rust code. It is used by projects such as
+//! [rust-analyzer](https://github.com/rust-lang/rust-analyzer/tree/master/xtask).
+//! Cargo uses the same pattern for several
+//! [`xtask-*` maintenance commands](https://github.com/rust-lang/cargo/blob/master/.cargo/config.toml).
+//!
+//! # Supported Commands
+//!
+//! ## CI steps
+//!
+//! A CI step is one command used by DataFusion's CI that can also be run
+//! locally. `--explain` prints the complete shell command without executing it.
+//!
+//! Use `cargo xtask ci step help` to list the available steps and
+//! `cargo xtask ci step <step-name> help` for step-specific arguments.
+//!
+//! # Examples
+//!
+//! Show all available CI steps:
+//!
+//! ```sh
+//! cargo xtask help
+//! ```
+//!
+//! Show the arguments and examples for the `test` step:
+//!
+//! ```sh
+//! cargo xtask ci step test help
+//! ```
+//!
+//! Run the `cli` variant of the `test` step:
+//!
+//! ```sh
+//! cargo xtask ci step test cli
+//! ```
+//!
+//! Print that step's shell command without running it:
+//!
+//! ```sh
+//! cargo xtask ci step test cli --explain
+//! ```
+
+mod ci_steps;
+
+use std::env;
+use std::path::PathBuf;
+
+type Result<T> = std::result::Result<T, String>;
+
+fn main() {
+ let args = env::args().skip(1).collect::<Vec<_>>();
+ let result = Xtask::new().and_then(|xtask| xtask.run(&args));
+
+ if let Err(error) = result {
+ eprintln!("error: {error}");
+ std::process::exit(1);
+ }
+}
+
+struct Xtask {
+ root: PathBuf,
+}
+
+impl Xtask {
+ fn new() -> Result<Self> {
+ let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+ let root = manifest_dir
+ .parent()
+ .ok_or_else(|| {
+ format!(
+ "could not find workspace root from {}",
+ manifest_dir.display()
+ )
+ })?
+ .to_path_buf();
+ Ok(Self { root })
+ }
+
+ fn run(&self, args: &[String]) -> Result<()> {
+ match args {
+ [] => {
+ print!("{}", ci_steps::help());
+ Ok(())
+ }
+ [help_arg] if ci_steps::is_help_arg(help_arg) => {
+ print!("{}", ci_steps::help());
+ Ok(())
+ }
+ [ci, step, step_args @ ..] if ci == "ci" && step == "step" => {
+ ci_steps::run(&self.root, step_args)
+ }
+ _ => Err(format!("unknown command\n\n{}", ci_steps::help())),
+ }
+ }
+}