Apache DataFusion SQL Query Engine
feat(CI): Introduce `xtask` to keep local test and CI commands in sync (#24083)
Yongting Youb9a005367720
diff --git a/.cargo/config.toml b/.cargo/config.tomlnew file mode 100644index 0000000000..9dae805eb1--- /dev/null+++ b/.cargo/config.toml@@ -0,0 +1,20 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++[alias]+xtask = "run --quiet --locked --package datafusion-xtask --"+ci-step = "run --quiet --locked --package datafusion-xtask -- ci step"diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 7a943088e2..7839547980 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -15,8 +15,21 @@# specific language governing permissions and limitations# under the License.-# For some actions, we use Runs-On to run them on ASF infrastructure: https://datafusion.apache.org/contributor-guide/#ci-runners-+# # CI runners+#+# Some actions use Runs-On to run on ASF infrastructure:+# https://datafusion.apache.org/contributor-guide/#ci-runners+#+# # `cargo xtask`+#+# Individual CI commands are implemented with `cargo xtask` to make them easier+# to reproduce locally.+#+# See `xtask/README.md` for details about DataFusion's `xtask` tooling.+#+# It is being rolled out incrementally across all workflows.+# Track progress at https://github.com/apache/datafusion/issues/24487.+#name: Rustconcurrency:@@ -63,11 +76,8 @@ jobs:shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobssave-if: ${{ github.ref_name == 'main' }}- name: Prepare cargo build- run: |- # Adding `--locked` here to assert that the `Cargo.lock` file is up to- # date with the manifest. When this fails, please make sure to commit- # the changes to `Cargo.lock` after building with the updated manifest.- cargo check --profile ci --workspace --all-targets --features integration-tests --locked+ # `--locked` asserts that Cargo.lock is up to date with the manifest.+ run: cargo xtask ci step check workspace# Check datafusion-common features#@@ -86,12 +96,12 @@ jobs:with:rust-version: stable- name: Check datafusion-common (default features)- run: cargo check --profile ci --all-targets -p datafusion-common+ run: cargo xtask ci step check datafusion-common default## Note: Only check libraries (not --all-targets) to cover end user APIs#- name: Check datafusion-common (no-default-features)- run: cargo check --profile ci --no-default-features -p datafusion-common+ run: cargo xtask ci step check datafusion-common no-default# Note: don't check other feature flags as datafusion-common is not typically used standalone# Check datafusion-substrait features@@ -116,20 +126,20 @@ jobs:save-if: false # set in linux-testshared-key: "amd-ci"- name: Check datafusion-substrait (default features)- run: cargo check --profile ci --all-targets -p datafusion-substrait+ run: cargo xtask ci step check datafusion-substrait default## Note: Only check libraries (not --all-targets) to cover end user APIs#- name: Check datafusion-substrait (no-default-features)- run: cargo check --profile ci --no-default-features -p datafusion-substrait+ run: cargo xtask ci step check datafusion-substrait no-default- name: Check datafusion-substrait (physical)- run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=physical+ run: cargo xtask ci step check datafusion-substrait physical- name: Install cmakerun: |# note the builder setup runs apt-get update / installs protobuf compilerapt-get install -y cmake- name: Check datafusion-substrait (protoc)- run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=protoc+ run: cargo xtask ci step check datafusion-substrait protoc# Check datafusion-proto features#@@ -149,18 +159,18 @@ jobs:with:rust-version: stable- name: Check datafusion-proto (default features)- run: cargo check --profile ci --all-targets -p datafusion-proto+ run: cargo xtask ci step check datafusion-proto default## Note: Only check libraries (not --all-targets) to cover end user APIs#- name: Check datafusion-proto (no-default-features)- run: cargo check --profile ci --no-default-features -p datafusion-proto+ run: cargo xtask ci step check datafusion-proto no-default- name: Check datafusion-proto (json)- run: cargo check --profile ci --no-default-features -p datafusion-proto --features=json+ run: cargo xtask ci step check datafusion-proto json- name: Check datafusion-proto (parquet)- run: cargo check --profile ci --no-default-features -p datafusion-proto --features=parquet+ run: cargo xtask ci step check datafusion-proto parquet- name: Check datafusion-proto (avro)- run: cargo check --profile ci --no-default-features -p datafusion-proto --features=avro+ run: cargo xtask ci step check datafusion-proto avro# Check datafusion-ffi features#@@ -180,7 +190,7 @@ jobs:with:rust-version: stable- name: Check datafusion-ffi (no-default-features)- run: cargo check --profile ci --no-default-features -p datafusion-ffi+ run: cargo xtask ci step check datafusion-ffi no-default# Check datafusion crate features@@ -206,48 +216,48 @@ jobs:save-if: false # set in linux-testshared-key: "amd-ci"- name: Check datafusion (default features)- run: cargo check --profile ci --all-targets -p datafusion+ run: cargo xtask ci step check datafusion default## Note: Only check libraries (not --all-targets) to cover end user APIs#- name: Check datafusion (no-default-features)- run: cargo check --profile ci --no-default-features -p datafusion+ run: cargo xtask ci step check datafusion no-default- name: Check datafusion (nested_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=nested_expressions+ run: cargo xtask ci step check datafusion nested_expressions- name: Check datafusion (array_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=array_expressions+ run: cargo xtask ci step check datafusion array_expressions- name: Check datafusion (avro)- run: cargo check --profile ci --no-default-features -p datafusion --features=avro+ run: cargo xtask ci step check datafusion avro- name: Check datafusion (backtrace)- run: cargo check --profile ci --no-default-features -p datafusion --features=backtrace+ run: cargo xtask ci step check datafusion backtrace- name: Check datafusion (compression)- run: cargo check --profile ci --no-default-features -p datafusion --features=compression+ run: cargo xtask ci step check datafusion compression- name: Check datafusion (crypto_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=crypto_expressions+ run: cargo xtask ci step check datafusion crypto_expressions- name: Check datafusion (datetime_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=datetime_expressions+ run: cargo xtask ci step check datafusion datetime_expressions- name: Check datafusion (encoding_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=encoding_expressions+ run: cargo xtask ci step check datafusion encoding_expressions- name: Check datafusion (force_hash_collisions)- run: cargo check --profile ci --no-default-features -p datafusion --features=force_hash_collisions+ run: cargo xtask ci step check datafusion force_hash_collisions- name: Check datafusion (math_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=math_expressions+ run: cargo xtask ci step check datafusion math_expressions- name: Check datafusion (parquet)- run: cargo check --profile ci --no-default-features -p datafusion --features=parquet+ run: cargo xtask ci step check datafusion parquet- name: Check datafusion (regex_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=regex_expressions+ run: cargo xtask ci step check datafusion regex_expressions- name: Check datafusion (recursive_protection)- run: cargo check --profile ci --no-default-features -p datafusion --features=recursive_protection+ run: cargo xtask ci step check datafusion recursive_protection- name: Check datafusion (serde)- run: cargo check --profile ci --no-default-features -p datafusion --features=serde+ run: cargo xtask ci step check datafusion serde- name: Check datafusion (sql)- run: cargo check --profile ci --no-default-features -p datafusion --features=sql+ run: cargo xtask ci step check datafusion sql- name: Check datafusion (string_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=string_expressions+ run: cargo xtask ci step check datafusion string_expressions- name: Check datafusion (unicode_expressions)- run: cargo check --profile ci --no-default-features -p datafusion --features=unicode_expressions+ run: cargo xtask ci step check datafusion unicode_expressions- name: Check parquet encryption (parquet_encryption)- run: cargo check --profile ci --no-default-features -p datafusion --features=parquet_encryption+ run: cargo xtask ci step check datafusion parquet_encryption# Check datafusion-functions crate features#@@ -266,26 +276,26 @@ jobs:with:rust-version: stable- name: Check datafusion-functions (default features)- run: cargo check --profile ci --all-targets -p datafusion-functions+ run: cargo xtask ci step check datafusion-functions default## Note: Only check libraries (not --all-targets) to cover end user APIs#- name: Check datafusion-functions (no-default-features)- run: cargo check --profile ci --no-default-features -p datafusion-functions+ run: cargo xtask ci step check datafusion-functions no-default- name: Check datafusion-functions (crypto_expressions)- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=crypto_expressions+ run: cargo xtask ci step check datafusion-functions crypto_expressions- name: Check datafusion-functions (datetime_expressions)- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=datetime_expressions+ run: cargo xtask ci step check datafusion-functions datetime_expressions- name: Check datafusion-functions (encoding_expressions)- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=encoding_expressions+ run: cargo xtask ci step check datafusion-functions encoding_expressions- name: Check datafusion-functions (math_expressions)- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=math_expressions+ run: cargo xtask ci step check datafusion-functions math_expressions- name: Check datafusion-functions (regex_expressions)- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=regex_expressions+ run: cargo xtask ci step check datafusion-functions regex_expressions- name: Check datafusion-functions (string_expressions)- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=string_expressions+ run: cargo xtask ci step check datafusion-functions string_expressions- name: Check datafusion-functions (unicode_expressions)- run: cargo check --profile ci --no-default-features -p datafusion-functions --features=unicode_expressions+ run: cargo xtask ci step check datafusion-functions unicode_expressions# Check datafusion-spark crate features#@@ -347,19 +357,7 @@ jobs:- name: Run tests (excluding doctests and datafusion-cli)env:RUST_BACKTRACE: 1- run: |- cargo llvm-cov \- --profile ci \- --exclude datafusion-examples \- --exclude ffi_example_table_provider \- --exclude datafusion-cli \- --workspace \- --lib \- --tests \- --bins \- --features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \- --codecov \- --output-path target/codecov.json+ run: cargo xtask ci step test workspace- parallel:- name: Verify Working Directory Cleanrun: git diff --exit-code@@ -405,7 +403,7 @@ jobs:AWS_SECRET_ACCESS_KEY: TEST-DataFusionPasswordTEST_STORAGE_INTEGRATION: 1AWS_ALLOW_HTTP: true- run: cargo test --features backtrace --profile ci -p datafusion-cli --lib --tests --bins+ run: cargo xtask ci step test cli- name: Verify Working Directory Cleanrun: git diff --exit-code@@ -458,7 +456,7 @@ jobs:with:rust-version: stable- name: Run doctests- run: cargo test --profile ci --doc --features avro,json+ run: cargo xtask ci step test doctest- name: Verify Working Directory Cleanrun: git diff --exit-code@@ -532,11 +530,8 @@ jobs:mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data- name: Verify that benchmark queries return expected resultsrun: |- # increase stack size to fix stack overflow- export RUST_MIN_STACK=20971520- export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`- cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1- INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests+ cargo xtask ci step test benchmark-plan+ cargo xtask ci step test benchmark-sqllogic- name: Verify Working Directory Cleanrun: git diff --exit-code@@ -571,9 +566,7 @@ jobs:with:rust-version: stable- name: Run sqllogictest- run: |- cd datafusion/sqllogictest- PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --features backtrace --profile ci --features=postgres --test sqllogictests+ run: cargo xtask ci step test postgresenv:# use postgres for the host here because we have specified a container for the jobPOSTGRES_HOST: postgres@@ -600,7 +593,7 @@ jobs:# command cannot be run for all the .slt files. Run it for just one that works (limit.slt)# until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed# and this command can be run without filters.- run: cargo test -p datafusion-sqllogictest --test sqllogictests --features substrait -- --substrait-round-trip limit.slt+ run: cargo xtask ci step test substrait# Temporarily commenting out the Windows flow, the reason is enormously slow running build# Waiting for new Windows 2025 github runner@@ -638,7 +631,7 @@ jobs:uses: ./.github/actions/setup-macos-aarch64-builder- name: Run datafusion-ffi testsshell: bash- run: cargo test --profile ci -p datafusion-ffi --lib --tests --features integration-tests+ run: cargo xtask ci step test ffivendor:name: Verify Vendored Codediff --git a/Cargo.lock b/Cargo.lockindex 44cef5185d..f1fbb5b580 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2748,6 +2748,13 @@ dependencies = ["wasm-bindgen-test",]+[[package]]+name = "datafusion-xtask"+version = "0.1.0"+dependencies = [+ "insta",+]+[[package]]name = "deranged"version = "0.5.8"diff --git a/Cargo.toml b/Cargo.tomlindex f9b2cac5c5..d529f551e9 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -66,6 +66,7 @@ members = ["benchmarks","datafusion/macros","datafusion/doc",+ "xtask",]exclude = ["dev/depcheck"]resolver = "2"diff --git a/xtask/Cargo.toml b/xtask/Cargo.tomlnew file mode 100644index 0000000000..ea4a63aa9f--- /dev/null+++ b/xtask/Cargo.toml@@ -0,0 +1,29 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++[package]+name = "datafusion-xtask"+version = "0.1.0"+edition.workspace = true+license.workspace = true+publish = false++[dev-dependencies]+insta = { workspace = true }++[lints]+workspace = truediff --git a/xtask/README.md b/xtask/README.mdnew file mode 100644index 0000000000..9d8da45626--- /dev/null+++ b/xtask/README.md@@ -0,0 +1,42 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# DataFusion xtask++This directory contains DataFusion's project automation implemented using the+[`xtask` pattern](https://github.com/matklad/cargo-xtask).++## Usage++Run the following command from the repository root for the available commands,+arguments, and examples:++```shell+cargo xtask help+```++## CI runner++`cargo xtask ci [args]` domain is used for organizing commands used in GitHub CI.++For example, `cargo xtask ci step test workspace` runs DataFusion's default CI test suite. Developers can run the same command to reproduce that GitHub Actions step locally. Append `--explain` to display the underlying command without running it.++The goal is to keep local development and GitHub CI runs in sync. Each CI step command is defined once in this in-repository Rust binary and is invoked the same way locally and in GitHub Actions.++Tracking issue: https://github.com/apache/datafusion/issues/24487diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rsnew file mode 100644index 0000000000..f40cb61818--- /dev/null+++ b/xtask/src/ci_steps.rs@@ -0,0 +1,793 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! CI steps that can be inspected or run through `cargo xtask`.++use crate::Result;+use std::env;+use std::ffi::OsStr;+use std::path::{Path, PathBuf};+use std::process::{Command, Stdio};++type StepResult<T> = std::result::Result<T, StepError>;+type StepRunner = fn(&StepContext, &[String]) -> StepResult<CiCommand>;++const CI_COMMAND: &str = "cargo xtask ci step";+const CI_SHORTCUT: &str = "cargo ci-step";+const TPCH_DATA_DIR: &str = "datafusion/sqllogictest/test_files/tpch/data";++/// Metadata and implementation for one CI step.+///+/// Adding an entry to `CI_STEPS` makes the step dispatchable and includes its+/// usage and description in the generated help text.+struct StepInfo {+ command: &'static str,+ help_usage: &'static str,+ help_examples: &'static [&'static str],+ help_description: &'static str,+ error_message: &'static str,+ runner: StepRunner,+}++static CI_STEPS: &[StepInfo] = &[+ StepInfo {+ command: "check",+ help_usage: "check <workspace|package> [default|no-default|feature] [--explain]",+ help_examples: &["check workspace", "check datafusion default"],+ help_description: "Check workspace or package compilation",+ error_message: "Cargo check step failed",+ runner: StepContext::run_check,+ },+ StepInfo {+ command: "test",+ help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait> [--explain]",+ help_examples: &["test cli", "test workspace"],+ help_description: "Run a CI test suite",+ error_message: "Cargo test step failed",+ runner: StepContext::run_test,+ },+];++pub(crate) fn help() -> String {+ let mut help = format!(+ "DataFusion CI commands\n\nUsage:\n {CI_COMMAND} <step-name> [args] [--explain]\n\nExamples:\n",+ );+ for step in CI_STEPS {+ if let Some(example) = step.help_examples.first() {+ help.push_str(&format!(" {CI_COMMAND} {example}\n"));+ }+ }+ help.push_str(&format!(" {CI_COMMAND} test workspace --explain\n"));++ let command_width = CI_STEPS+ .iter()+ .map(|step| step.command.len())+ .max()+ .unwrap_or_default();+ help.push_str("\nAvailable steps:\n");+ for step in CI_STEPS {+ help.push_str(&format!(+ " {:command_width$} {}\n",+ step.command, step.help_description+ ));+ }++ help.push_str(&format!(+ "\nShortcut:\n # `{CI_SHORTCUT}` is short for `{CI_COMMAND}`.\n {CI_SHORTCUT} check workspace\n\nFor more details:\n {CI_COMMAND} check --help\n"+ ));+ help+}++fn step_help(step: &StepInfo) -> String {+ let mut help = format!(+ "DataFusion CI command: {}\n\n{}\n\nUsage:\n {CI_COMMAND} {}\n\nExamples:\n",+ step.command, step.help_description, step.help_usage,+ );+ for example in step.help_examples {+ help.push_str(&format!(" {CI_COMMAND} {example}\n"));+ }+ if let Some(example) = step.help_examples.first() {+ help.push_str(&format!(+ "\nUse '--explain' to show the full command:\n {CI_COMMAND} {example} --explain\n"+ ));+ }+ help+}++fn find_step(command: &str) -> Result<&'static StepInfo> {+ CI_STEPS+ .iter()+ .find(|step| step.command == command)+ .ok_or_else(|| format!("unknown CI step `{command}`"))+}++pub(crate) fn is_help_arg(arg: &str) -> bool {+ matches!(arg, "help" | "-h" | "--help")+}++pub(crate) fn run(root: &Path, args: &[String]) -> Result<()> {+ StepContext::new(root).run(args)+}++struct StepContext {+ root: PathBuf,+ env_reader: fn(&str) -> Result<String>,+}++impl StepContext {+ fn new(root: &Path) -> Self {+ Self {+ root: root.to_path_buf(),+ env_reader: required_env,+ }+ }++ fn run(&self, args: &[String]) -> Result<()> {+ match args {+ [] => {+ print!("{}", help());+ Ok(())+ }+ [help_arg] if is_help_arg(help_arg) => {+ print!("{}", help());+ Ok(())+ }+ [step_name, help_arg] if is_help_arg(help_arg) => {+ print!("{}", step_help(find_step(step_name)?));+ Ok(())+ }+ step_args => {+ let (action, step, command) = self.ci_step(step_args)?;+ match action {+ StepAction::Execute => command.execute(step.error_message),+ StepAction::Explain => {+ command.explain();+ Ok(())+ }+ }+ }+ }+ }++ /// Parses a CI step into an action and its complete command description.+ fn ci_step(+ &self,+ args: &[String],+ ) -> Result<(StepAction, &'static StepInfo, CiCommand)> {+ let (action, args) = match args.split_last() {+ Some((arg, args)) if arg == "--explain" => (StepAction::Explain, args),+ _ => (StepAction::Execute, args),+ };+ let Some((step, args)) = args.split_first() else {+ return Err(format!("missing CI step\n\n{}", help()));+ };++ let step = find_step(step)?;+ let command = (step.runner)(self, args).map_err(|error| error.render(step))?;+ Ok((action, step, command))+ }++ /// Implementation for `cargo xtask ci step check [args]`+ fn run_check(&self, args: &[String]) -> StepResult<CiCommand> {+ let args = args.iter().map(String::as_str).collect::<Vec<_>>();+ let mut command = self.cargo();+ command.args(["check", "--profile", "ci"]);++ match args.as_slice() {+ ["workspace"] => {+ command.args([+ "--workspace",+ "--all-targets",+ "--features",+ "integration-tests",+ "--locked",+ ]);+ }+ [package, "default"] => {+ command.args(["--all-targets", "-p", package]);+ }+ [package, "no-default"] => {+ command.args(["--no-default-features", "-p", package]);+ }+ [package, feature] => {+ command.args([+ "--no-default-features",+ "-p",+ package,+ "--features",+ feature,+ ]);+ }+ _ => return Err(StepError::Usage),+ }++ Ok(command)+ }++ /// Implementation for `cargo xtask ci step test [args]`+ fn run_test(&self, args: &[String]) -> StepResult<CiCommand> {+ let [variant] = args else {+ return Err(StepError::Usage);+ };++ let mut command = self.cargo();+ match variant.as_str() {+ "workspace" => {+ command.args([+ "llvm-cov",+ "--profile",+ "ci",+ "--exclude",+ "datafusion-examples",+ "--exclude",+ "ffi_example_table_provider",+ "--exclude",+ "datafusion-cli",+ "--workspace",+ "--lib",+ "--tests",+ "--bins",+ "--features",+ "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait",+ "--codecov",+ "--output-path",+ "target/codecov.json",+ ]);+ }+ "cli" => {+ command.args([+ "test",+ "--features",+ "backtrace",+ "--profile",+ "ci",+ "-p",+ "datafusion-cli",+ "--lib",+ "--tests",+ "--bins",+ ]);+ }+ "doctest" => {+ command.args([+ "test",+ "--profile",+ "ci",+ "--doc",+ "--features",+ "avro,json",+ ]);+ }+ "ffi" => {+ command.args([+ "test",+ "--profile",+ "ci",+ "-p",+ "datafusion-ffi",+ "--lib",+ "--tests",+ "--features",+ "integration-tests",+ ]);+ }+ "benchmark-plan" => {+ command+ .args([+ "test",+ "plan_q",+ "--package",+ "datafusion-benchmarks",+ "--profile",+ "ci",+ "--features=ci",+ "--",+ "--test-threads=1",+ ])+ .env("RUST_MIN_STACK", "20971520")+ .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR));+ }+ "benchmark-sqllogic" => {+ command+ .args([+ "test",+ "--features",+ "backtrace,parquet_encryption,substrait",+ "--profile",+ "ci",+ "--package",+ "datafusion-sqllogictest",+ "--test",+ "sqllogictests",+ ])+ .env("RUST_MIN_STACK", "20971520")+ .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR))+ .env("INCLUDE_TPCH", "true");+ }+ "postgres" => {+ let host = self+ .required_env("POSTGRES_HOST")+ .map_err(StepError::Message)?;+ let port = self+ .required_env("POSTGRES_PORT")+ .map_err(StepError::Message)?;+ let uri = format!("postgresql://postgres:postgres@{host}:{port}/db_test");+ command+ .args([+ "test",+ "--features",+ "backtrace",+ "--profile",+ "ci",+ "--features=postgres",+ "--test",+ "sqllogictests",+ ])+ .current_dir(self.root.join("datafusion/sqllogictest"))+ .env("PG_COMPAT", "true")+ .env("PG_URI", uri);+ }+ "substrait" => {+ command.args([+ "test",+ "-p",+ "datafusion-sqllogictest",+ "--test",+ "sqllogictests",+ "--features",+ "substrait",+ "--",+ "--substrait-round-trip",+ "limit.slt",+ ]);+ }+ _ => return Err(StepError::Usage),+ }++ Ok(command)+ }++ fn cargo(&self) -> CiCommand {+ CiCommand::new("cargo", &self.root)+ }++ fn required_env(&self, name: &str) -> Result<String> {+ (self.env_reader)(name)+ }+}++#[derive(Debug, PartialEq)]+enum StepAction {+ Execute,+ Explain,+}++#[derive(Debug)]+enum StepError {+ Usage,+ Message(String),+}++impl StepError {+ fn render(self, step: &StepInfo) -> String {+ match self {+ Self::Usage => format!("usage: {CI_COMMAND} {}", step.help_usage),+ Self::Message(message) => message,+ }+ }+}++/// Command shared by invocation and `--explain`+struct CiCommand {+ command: Command,+}++impl CiCommand {+ fn new(program: impl AsRef<OsStr>, current_dir: impl AsRef<Path>) -> Self {+ let mut command = Command::new(program);+ command.current_dir(current_dir);+ Self { command }+ }++ fn args<I, S>(&mut self, args: I) -> &mut Self+ where+ I: IntoIterator<Item = S>,+ S: AsRef<OsStr>,+ {+ self.command.args(args);+ self+ }++ fn env<K, V>(&mut self, key: K, value: V) -> &mut Self+ where+ K: AsRef<OsStr>,+ V: AsRef<OsStr>,+ {+ self.command.env(key, value);+ self+ }++ fn current_dir(&mut self, directory: impl AsRef<Path>) -> &mut Self {+ self.command.current_dir(directory);+ self+ }++ fn explain(&self) {+ println!("{}", self.full_command());+ }++ fn execute(mut self, error_message: &str) -> Result<()> {+ println!("+ {}", self.full_command());+ let status = self+ .command+ .stdin(Stdio::inherit())+ .stdout(Stdio::inherit())+ .stderr(Stdio::inherit())+ .status()+ .map_err(|error| format!("{error_message}: {error}"))?;++ if status.success() {+ Ok(())+ } else {+ Err(format!("{error_message}: {status}"))+ }+ }++ fn full_command(&self) -> String {+ let mut lines = Vec::new();++ if let Some(current_dir) = self.command.get_current_dir() {+ lines.push(format!("cd {} &&", shell_quote(current_dir.as_os_str())));+ }++ for (key, value) in self.command.get_envs() {+ let Some(value) = value else {+ continue;+ };+ lines.push(format!("{}={}", shell_quote(key), shell_quote(value)));+ }++ lines.extend(shell_command_lines(+ self.command.get_program(),+ self.command.get_args(),+ ));+ lines.join(" \\\n")+ }+}++/// Groups option-value pairs so the result reads like a hand-written command.+fn shell_command_lines<'a>(+ program: &OsStr,+ args: impl Iterator<Item = &'a OsStr>,+) -> Vec<String> {+ let args = args.collect::<Vec<_>>();+ let mut lines = vec![shell_quote(program)];+ let mut index = 0;++ // Keep command names and leading positional arguments together, such as+ // `cargo check` and `cargo test plan_q`.+ while index < args.len() && !is_option(args[index]) {+ lines[0].push(' ');+ lines[0].push_str(&shell_quote(args[index]));+ index += 1;+ }++ while index < args.len() {+ let arg = args[index];+ let mut line = shell_quote(arg);+ index += 1;++ if arg == "--" {+ while index < args.len() {+ line.push(' ');+ line.push_str(&shell_quote(args[index]));+ index += 1;+ }+ } else if !arg.to_string_lossy().contains('=')+ && index < args.len()+ && !is_option(args[index])+ {+ line.push(' ');+ line.push_str(&shell_quote(args[index]));+ index += 1;+ }++ lines.push(line);+ }++ lines+}++fn is_option(value: &OsStr) -> bool {+ value.to_string_lossy().starts_with('-')+}++/// Quotes one shell word when it contains characters that need protection.+fn shell_quote(value: &OsStr) -> String {+ let value = value.to_string_lossy();+ if !value.is_empty()+ && value.bytes().all(|byte| {+ byte.is_ascii_alphanumeric()+ || matches!(+ byte,+ b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-'+ )+ })+ {+ value.into_owned()+ } else {+ format!("'{}'", value.replace('\'', "'\"'\"'"))+ }+}++fn required_env(name: &str) -> Result<String> {+ env::var(name)+ .map_err(|_| format!("required environment variable `{name}` is not set"))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn args(args: &[&str]) -> Vec<String> {+ args.iter().map(|arg| (*arg).to_string()).collect()+ }++ fn test_env(name: &str) -> Result<String> {+ match name {+ "POSTGRES_HOST" => Ok("postgres.example".to_string()),+ "POSTGRES_PORT" => Ok("15432".to_string()),+ _ => Err(format!("unexpected environment variable `{name}`")),+ }+ }++ fn context() -> StepContext {+ StepContext {+ root: PathBuf::from("/workspace"),+ env_reader: test_env,+ }+ }++ #[test]+ fn explain_builds_the_same_command_as_execute() {+ let context = context();+ let (execute_action, _, execute_command) = context+ .ci_step(&args(&["check", "datafusion", "default"]))+ .unwrap();+ let (explain_action, _, explain_command) = context+ .ci_step(&args(&["check", "datafusion", "default", "--explain"]))+ .unwrap();++ assert_eq!(execute_action, StepAction::Execute);+ assert_eq!(explain_action, StepAction::Explain);+ assert_eq!(+ execute_command.full_command(),+ explain_command.full_command()+ );+ }++ #[test]+ fn explain_flag_must_be_last() {+ let error = context()+ .ci_step(&args(&["test", "--explain", "workspace"]))+ .err()+ .unwrap();++ assert_eq!(+ error,+ "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait> [--explain]"+ );+ }++ // Test all available commands, ensure their executed command is expected+ #[test]+ fn explain_all_available_ci_commands() {+ struct Cmd {+ // Actual command to run+ command: &'static [&'static str],+ // Full explain output+ expected: fn(&str),+ }++ let commands = [+ Cmd {+ command: &["check", "workspace"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo check \+--profile ci \+--workspace \+--all-targets \+--features integration-tests \+--locked+")+ },+ },+ Cmd {+ command: &["check", "datafusion", "default"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo check \+--profile ci \+--all-targets \+-p datafusion+")+ },+ },+ Cmd {+ command: &["check", "datafusion", "no-default"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo check \+--profile ci \+--no-default-features \+-p datafusion+")+ },+ },+ Cmd {+ command: &["check", "datafusion", "parquet"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo check \+--profile ci \+--no-default-features \+-p datafusion \+--features parquet+")+ },+ },+ Cmd {+ command: &["test", "workspace"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo llvm-cov \+--profile ci \+--exclude datafusion-examples \+--exclude ffi_example_table_provider \+--exclude datafusion-cli \+--workspace \+--lib \+--tests \+--bins \+--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \+--codecov \+--output-path target/codecov.json+")+ },+ },+ Cmd {+ command: &["test", "cli"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo test \+--features backtrace \+--profile ci \+-p datafusion-cli \+--lib \+--tests \+--bins+")+ },+ },+ Cmd {+ command: &["test", "doctest"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo test \+--profile ci \+--doc \+--features avro,json+")+ },+ },+ Cmd {+ command: &["test", "ffi"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo test \+--profile ci \+-p datafusion-ffi \+--lib \+--tests \+--features integration-tests+")+ },+ },+ Cmd {+ command: &["test", "benchmark-plan"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+RUST_MIN_STACK=20971520 \+TPCH_DATA=/workspace/datafusion/sqllogictest/test_files/tpch/data \+cargo test plan_q \+--package datafusion-benchmarks \+--profile ci \+--features=ci \+-- --test-threads=1+")+ },+ },+ Cmd {+ command: &["test", "benchmark-sqllogic"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+INCLUDE_TPCH=true \+RUST_MIN_STACK=20971520 \+TPCH_DATA=/workspace/datafusion/sqllogictest/test_files/tpch/data \+cargo test \+--features backtrace,parquet_encryption,substrait \+--profile ci \+--package datafusion-sqllogictest \+--test sqllogictests+")+ },+ },+ Cmd {+ command: &["test", "postgres"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace/datafusion/sqllogictest && \+PG_COMPAT=true \+PG_URI=postgresql://postgres:[email protected]:15432/db_test \+cargo test \+--features backtrace \+--profile ci \+--features=postgres \+--test sqllogictests+")+ },+ },+ Cmd {+ command: &["test", "substrait"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo test \+-p datafusion-sqllogictest \+--test sqllogictests \+--features substrait \+-- --substrait-round-trip limit.slt+")+ },+ },+ ];++ let context = context();+ for Cmd { command, expected } in commands {+ let explain_args = command+ .iter()+ .copied()+ .chain(std::iter::once("--explain"))+ .collect::<Vec<_>>();+ let (action, _, actual) = context.ci_step(&args(&explain_args)).unwrap();++ assert_eq!(action, StepAction::Explain);+ expected(&actual.full_command());+ }+ }+}diff --git a/xtask/src/main.rs b/xtask/src/main.rsnew file mode 100644index 0000000000..c714a1da34--- /dev/null+++ b/xtask/src/main.rs@@ -0,0 +1,134 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! # DataFusion repository development tools+//!+//! ## Example+//!+//! Run the default CI test suite locally:+//!+//! ```sh+//! cargo xtask ci step test workspace+//! ```+//!+//! Implementing the command inside `xtask` can make it easier to maintain. For example,+//! the local test runs and CI runs can keep in sync.+//!+//! ## Mechanism+//!+//! `xtask` is an internal Rust binary in this workspace. The `cargo xtask` alias+//! builds and runs that binary, forwarding the remaining arguments to it.+//!+//! There is no external dependency required.+//!+//! ## Reference+//!+//! The [`xtask` convention](https://github.com/matklad/cargo-xtask) keeps project+//! automation in ordinary Rust code. It is used by projects such as+//! [rust-analyzer](https://github.com/rust-lang/rust-analyzer/tree/master/xtask).+//! Cargo uses the same pattern for several+//! [`xtask-*` maintenance commands](https://github.com/rust-lang/cargo/blob/master/.cargo/config.toml).+//!+//! # Supported Commands+//!+//! ## CI steps+//!+//! A CI step is one command used by DataFusion's CI that can also be run+//! locally. `--explain` prints the complete shell command without executing it.+//!+//! Use `cargo xtask ci step help` to list the available steps and+//! `cargo xtask ci step <step-name> help` for step-specific arguments.+//!+//! # Examples+//!+//! Show all available CI steps:+//!+//! ```sh+//! cargo xtask help+//! ```+//!+//! Show the arguments and examples for the `test` step:+//!+//! ```sh+//! cargo xtask ci step test help+//! ```+//!+//! Run the `cli` variant of the `test` step:+//!+//! ```sh+//! cargo xtask ci step test cli+//! ```+//!+//! Print that step's shell command without running it:+//!+//! ```sh+//! cargo xtask ci step test cli --explain+//! ```++mod ci_steps;++use std::env;+use std::path::PathBuf;++type Result<T> = std::result::Result<T, String>;++fn main() {+ let args = env::args().skip(1).collect::<Vec<_>>();+ let result = Xtask::new().and_then(|xtask| xtask.run(&args));++ if let Err(error) = result {+ eprintln!("error: {error}");+ std::process::exit(1);+ }+}++struct Xtask {+ root: PathBuf,+}++impl Xtask {+ fn new() -> Result<Self> {+ let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));+ let root = manifest_dir+ .parent()+ .ok_or_else(|| {+ format!(+ "could not find workspace root from {}",+ manifest_dir.display()+ )+ })?+ .to_path_buf();+ Ok(Self { root })+ }++ fn run(&self, args: &[String]) -> Result<()> {+ match args {+ [] => {+ print!("{}", ci_steps::help());+ Ok(())+ }+ [help_arg] if ci_steps::is_help_arg(help_arg) => {+ print!("{}", ci_steps::help());+ Ok(())+ }+ [ci, step, step_args @ ..] if ci == "ci" && step == "step" => {+ ci_steps::run(&self.root, step_args)+ }+ _ => Err(format!("unknown command\n\n{}", ci_steps::help())),+ }+ }+}