Skip to content

Apache DataFusion SQL Query Engine

refactor: Centralize CI command definitions in xtask (#26765)

Yongting You0e314cc3c904
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 4db0dcacee..bbee5900cd 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -462,8 +462,6 @@ jobs:
save-if: ${{ github.ref_name == 'main' }}
shared-key: "amd-ci"
- name: Run tests (excluding doctests and datafusion-cli)
- env:
- RUST_BACKTRACE: 1
run: cargo xtask ci step test workspace
- name: Run SQLLogicTests with coverage
env:
@@ -533,13 +531,6 @@ jobs:
ci/scripts/retry timeout 120 docker pull "$RUSTFS_IMAGE" \
|| echo "::warning::Could not pre-pull $RUSTFS_IMAGE, the tests will pull it themselves"
- name: Run tests (excluding doctests)
- env:
- RUST_BACKTRACE: 1
- AWS_ENDPOINT: http://127.0.0.1:9000
- AWS_ACCESS_KEY_ID: TEST-DataFusionLogin
- AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword
- TEST_STORAGE_INTEGRATION: 1
- AWS_ALLOW_HTTP: true
run: cargo xtask ci step test cli
- name: Verify Working Directory Clean
run: git diff --exit-code
diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rs
index 5184a5cfbf..c6b605d36e 100644
--- a/xtask/src/ci_steps.rs
+++ b/xtask/src/ci_steps.rs
@@ -5,14 +5,12 @@
//! CI steps that can be inspected or run through `cargo xtask`.
use crate::Result;
-use std::env;
-use std::ffi::OsStr;
+use crate::command::{CommandSpec, EnvReader, system_env};
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
type StepResult<T> = std::result::Result<T, StepError>;
-type StepRunner = fn(&StepContext, &[String]) -> StepResult<CiCommand>;
+type StepRunner = fn(&StepContext, &[String]) -> StepResult<CommandSpec>;
const CI_COMMAND: &str = "cargo xtask ci step";
const CI_SHORTCUT: &str = "cargo ci-step";
@@ -29,7 +27,6 @@ struct StepInfo {
help_usage: &'static str,
help_examples: &'static [&'static str],
help_description: &'static str,
- error_message: &'static str,
runner: StepRunner,
}
@@ -39,7 +36,6 @@ static CI_STEPS: &[StepInfo] = &[
help_usage: "check <workspace|package> [default|no-default|feature] [--explain]",
help_examples: &["check workspace", "check datafusion default"],
help_description: "Check workspace or package compilation",
- error_message: "Cargo check step failed",
runner: StepContext::run_check,
},
StepInfo {
@@ -47,7 +43,6 @@ static CI_STEPS: &[StepInfo] = &[
help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",
help_examples: &["test cli", "test workspace"],
help_description: "Run a CI test suite",
- error_message: "Cargo test step failed",
runner: StepContext::run_test,
},
];
@@ -117,14 +112,14 @@ pub(crate) fn run(root: &Path, args: &[String]) -> Result<()> {
struct StepContext {
root: PathBuf,
- env_reader: fn(&str) -> Result<String>,
+ env_reader: EnvReader,
}
impl StepContext {
fn new(root: &Path) -> Self {
Self {
root: root.to_path_buf(),
- env_reader: required_env,
+ env_reader: system_env,
}
}
@@ -143,9 +138,9 @@ impl StepContext {
Ok(())
}
step_args => {
- let (action, step, command) = self.ci_step(step_args)?;
+ let (action, command) = self.ci_step(step_args)?;
match action {
- StepAction::Execute => command.execute(step.error_message),
+ StepAction::Execute => command.execute(),
StepAction::Explain => {
command.explain();
Ok(())
@@ -156,10 +151,7 @@ impl StepContext {
}
/// Parses a CI step into an action and its complete command description.
- fn ci_step(
- &self,
- args: &[String],
- ) -> Result<(StepAction, &'static StepInfo, CiCommand)> {
+ fn ci_step(&self, args: &[String]) -> Result<(StepAction, CommandSpec)> {
let (action, args) = match args.split_last() {
Some((arg, args)) if arg == "--explain" => (StepAction::Explain, args),
_ => (StepAction::Execute, args),
@@ -170,18 +162,23 @@ impl StepContext {
let step = find_step(step)?;
let command = (step.runner)(self, args).map_err(|error| error.render(step))?;
- Ok((action, step, command))
+ Ok((action, command))
}
/// Implementation for `cargo xtask ci step check [args]`
- fn run_check(&self, args: &[String]) -> StepResult<CiCommand> {
+ fn run_check(&self, args: &[String]) -> StepResult<CommandSpec> {
let args = args.iter().map(String::as_str).collect::<Vec<_>>();
- let mut command = self.cargo();
- command.args(["check", "--profile", "ci"]);
+ let mut builder = CommandSpec::builder(self.env_reader);
+ builder
+ .current_dir(&self.root)
+ .failure_message("CI check step failed");
match args.as_slice() {
["workspace"] => {
- command.args([
+ builder.program("cargo").args([
+ "check",
+ "--profile",
+ "ci",
"--workspace",
"--all-targets",
"--features",
@@ -190,13 +187,30 @@ impl StepContext {
]);
}
[package, "default"] => {
- command.args(["--all-targets", "-p", package]);
+ builder.program("cargo").args([
+ "check",
+ "--profile",
+ "ci",
+ "--all-targets",
+ "-p",
+ package,
+ ]);
}
[package, "no-default"] => {
- command.args(["--no-default-features", "-p", package]);
+ builder.program("cargo").args([
+ "check",
+ "--profile",
+ "ci",
+ "--no-default-features",
+ "-p",
+ package,
+ ]);
}
[package, feature] => {
- command.args([
+ builder.program("cargo").args([
+ "check",
+ "--profile",
+ "ci",
"--no-default-features",
"-p",
package,
@@ -207,19 +221,23 @@ impl StepContext {
_ => return Err(StepError::Usage),
}
- Ok(command)
+ builder.build().map_err(StepError::InvalidCommand)
}
/// Implementation for `cargo xtask ci step test [args]`
- fn run_test(&self, args: &[String]) -> StepResult<CiCommand> {
+ fn run_test(&self, args: &[String]) -> StepResult<CommandSpec> {
let [variant] = args else {
return Err(StepError::Usage);
};
- let mut command = self.cargo();
+ let mut cmd_builder = CommandSpec::builder(self.env_reader);
+ cmd_builder
+ .current_dir(&self.root)
+ .failure_message("CI test step failed");
+
match variant.as_str() {
"workspace" | "slt-coverage" => {
- command.args([
+ cmd_builder.program("cargo").args([
"llvm-cov",
"--profile",
"ci",
@@ -232,36 +250,54 @@ impl StepContext {
"--workspace",
]);
let output_path = if variant == "slt-coverage" {
- command.args(["--test", "sqllogictests"]);
+ cmd_builder.args(["--test", "sqllogictests"]);
"target/codecov-slt.json"
} else {
- command.args(["--lib", "--tests", "--bins"]);
+ cmd_builder.args(["--lib", "--tests", "--bins"]);
"target/codecov.json"
};
- command.args([
- "--features",
- "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait",
- "--codecov",
- "--output-path",
- output_path,
- ]);
+ cmd_builder
+ .args([
+ "--features",
+ "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait",
+ "--codecov",
+ "--output-path",
+ output_path,
+ ])
+ .env("RUST_BACKTRACE", "1")
+ .requires_executable(
+ "cargo-llvm-cov",
+ "install it with `cargo install cargo-llvm-cov --locked`",
+ );
}
"cli" => {
- command.args([
- "test",
- "--features",
- "backtrace",
- "--profile",
- "ci",
- "-p",
- "datafusion-cli",
- "--lib",
- "--tests",
- "--bins",
- ]);
+ cmd_builder.doc(
+ "Run CLI tests with storage integration enabled. The AWS credentials\n\
+ shown below are fixed test values, not secrets.",
+ );
+ cmd_builder
+ .program("cargo")
+ .args([
+ "test",
+ "--features",
+ "backtrace",
+ "--profile",
+ "ci",
+ "-p",
+ "datafusion-cli",
+ "--lib",
+ "--tests",
+ "--bins",
+ ])
+ .env("RUST_BACKTRACE", "1")
+ .env("AWS_ENDPOINT", "http://127.0.0.1:9000")
+ .env("AWS_ACCESS_KEY_ID", "TEST-DataFusionLogin")
+ .env("AWS_SECRET_ACCESS_KEY", "TEST-DataFusionPassword")
+ .env("TEST_STORAGE_INTEGRATION", "1")
+ .env("AWS_ALLOW_HTTP", "true");
}
"doctest" => {
- command.args([
+ cmd_builder.program("cargo").args([
"test",
"--profile",
"ci",
@@ -271,7 +307,7 @@ impl StepContext {
]);
}
"ffi" => {
- command.args([
+ cmd_builder.program("cargo").args([
"test",
"--profile",
"ci",
@@ -286,18 +322,19 @@ impl StepContext {
]);
}
"microbenchmarks" => {
- // The SQL benchmark harness needs separately generated datasets
- // and is covered by the benchmark result verification job.
- // Select only bench targets (not lib/bin unit tests), and tell
- // Criterion to execute each benchmark once instead of timing it.
- // Keep debug assertions and overflow checks enabled without the
- // cost of unoptimized execution on the larger benchmark datasets.
- command.env(
+ cmd_builder.doc(
+ "Run each Criterion benchmark once without timing it, selecting only\n\
+ bench targets (not lib/bin unit tests). Keep debug assertions and\n\
+ overflow checks enabled while optimizing execution for larger datasets.\n\
+ The SQL benchmark harness needs separately generated datasets and\n\
+ is covered by the benchmark result verification job.",
+ );
+ cmd_builder.env(
"CLICKBENCH_DATA_PATH",
self.root
.join("datafusion/core/tests/data/clickbench_hits_10.parquet"),
);
- command.args([
+ cmd_builder.program("cargo").args([
"test",
"--profile",
"ci-bench",
@@ -318,7 +355,9 @@ impl StepContext {
]);
}
"benchmark-plan" => {
- command
+ let tpch_data = self.root.join(TPCH_DATA_DIR);
+ cmd_builder
+ .program("cargo")
.args([
"test",
"plan_q",
@@ -331,10 +370,16 @@ impl StepContext {
"--test-threads=1",
])
.env("RUST_MIN_STACK", "20971520")
- .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR));
+ .env("TPCH_DATA", tpch_data.as_os_str())
+ .requires_path(
+ tpch_data,
+ "generate the TPC-H fixture data before running this step",
+ );
}
"benchmark-sqllogic" => {
- command
+ let tpch_data = self.root.join(TPCH_DATA_DIR);
+ cmd_builder
+ .program("cargo")
.args([
"test",
"--features",
@@ -347,18 +392,16 @@ impl StepContext {
"sqllogictests",
])
.env("RUST_MIN_STACK", "20971520")
- .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR))
- .env("INCLUDE_TPCH", "true");
+ .env("TPCH_DATA", tpch_data.as_os_str())
+ .env("INCLUDE_TPCH", "true")
+ .requires_path(
+ tpch_data,
+ "generate the TPC-H fixture data before running this step",
+ );
}
"postgres" => {
- let host = self
- .required_env("POSTGRES_HOST")
- .map_err(StepError::Message)?;
- let port = self
- .required_env("POSTGRES_PORT")
- .map_err(StepError::Message)?;
- let uri = format!("postgresql://postgres:postgres@{host}:{port}/db_test");
- command
+ cmd_builder
+ .program("cargo")
.args([
"test",
"--features",
@@ -371,10 +414,13 @@ impl StepContext {
])
.current_dir(self.root.join("datafusion/sqllogictest"))
.env("PG_COMPAT", "true")
- .env("PG_URI", uri);
+ .env_template(
+ "PG_URI",
+ "postgresql://postgres:postgres@{POSTGRES_HOST}:{POSTGRES_PORT}/db_test",
+ );
}
"substrait" | "substrait-optimized" => {
- command.args([
+ cmd_builder.program("cargo").args([
"test",
"-p",
"datafusion-sqllogictest",
@@ -386,15 +432,18 @@ impl StepContext {
"--substrait-round-trip",
]);
if variant == "substrait-optimized" {
- command.args(["--substrait-optimize", "joins_conditionless.slt"]);
+ cmd_builder.args(["--substrait-optimize", "joins_conditionless.slt"]);
} else {
- command.args(["limit.slt"]);
+ cmd_builder.args(["limit.slt"]);
}
}
"extended" => {
- // Tests behind the `extended_tests` feature. The `workspace`
- // step runs everything else.
- command
+ cmd_builder.doc(
+ "Run tests behind the `extended_tests` feature. The `workspace`\n\
+ step runs the remaining tests.",
+ );
+ cmd_builder
+ .program("cargo")
.args([
"test",
"--profile",
@@ -409,9 +458,12 @@ impl StepContext {
.env("RUST_BACKTRACE", "1");
}
"spill-pool-fuzz" => {
- // Run more random scenarios of the spill pool fuzzer than the
- // `workspace` step does (about a minute).
- command
+ cmd_builder.doc(
+ "Run more random spill pool fuzzer scenarios than the `workspace`\n\
+ step. This takes about a minute.",
+ );
+ cmd_builder
+ .program("cargo")
.args([
"test",
"--profile",
@@ -427,10 +479,12 @@ impl StepContext {
.env("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", "1000");
}
"hash-collisions" => {
- // Check answers are correct when hash values collide. The CI job
- // runs from the `datafusion` directory with the builder setup,
- // which sets `RUST_BACKTRACE=1`.
- command
+ cmd_builder.doc(
+ "Check that results stay correct when hash values collide.\n\
+ Run from the `datafusion` directory with `RUST_BACKTRACE=1`.",
+ );
+ cmd_builder
+ .program("cargo")
.args([
"test",
"--profile",
@@ -452,9 +506,11 @@ impl StepContext {
.env("RUST_BACKTRACE", "1");
}
"sqlite" => {
- // The CI job skips the builder setup because backtraces make this
- // suite expensive, so `RUST_BACKTRACE` is deliberately not set.
- command.args([
+ cmd_builder.doc(
+ "Run SQL logic tests including the SQLite suite. This step does not\n\
+ set `RUST_BACKTRACE` because backtraces make the suite expensive.",
+ );
+ cmd_builder.program("cargo").args([
"test",
"--features",
"backtrace,parquet_encryption",
@@ -469,15 +525,7 @@ impl StepContext {
_ => return Err(StepError::Usage),
}
- Ok(command)
- }
-
- fn cargo(&self) -> CiCommand {
- CiCommand::new("cargo", &self.root)
- }
-
- fn required_env(&self, name: &str) -> Result<String> {
- (self.env_reader)(name)
+ cmd_builder.build().map_err(StepError::InvalidCommand)
}
}
@@ -490,169 +538,24 @@ enum StepAction {
#[derive(Debug)]
enum StepError {
Usage,
- Message(String),
+ InvalidCommand(String),
}
impl StepError {
fn render(self, step: &StepInfo) -> String {
match self {
Self::Usage => format!("usage: {CI_COMMAND} {}", step.help_usage),
- Self::Message(message) => message,
- }
- }
-}
-
-/// Command shared by invocation and `--explain`
-struct CiCommand {
- command: Command,
-}
-
-impl CiCommand {
- fn new(program: impl AsRef<OsStr>, current_dir: impl AsRef<Path>) -> Self {
- let mut command = Command::new(program);
- command.current_dir(current_dir);
- Self { command }
- }
-
- fn args<I, S>(&mut self, args: I) -> &mut Self
- where
- I: IntoIterator<Item = S>,
- S: AsRef<OsStr>,
- {
- self.command.args(args);
- self
- }
-
- fn env<K, V>(&mut self, key: K, value: V) -> &mut Self
- where
- K: AsRef<OsStr>,
- V: AsRef<OsStr>,
- {
- self.command.env(key, value);
- self
- }
-
- fn current_dir(&mut self, directory: impl AsRef<Path>) -> &mut Self {
- self.command.current_dir(directory);
- self
- }
-
- fn explain(&self) {
- println!("{}", self.full_command());
- }
-
- fn execute(mut self, error_message: &str) -> Result<()> {
- println!("+ {}", self.full_command());
- let status = self
- .command
- .stdin(Stdio::inherit())
- .stdout(Stdio::inherit())
- .stderr(Stdio::inherit())
- .status()
- .map_err(|error| format!("{error_message}: {error}"))?;
-
- if status.success() {
- Ok(())
- } else {
- Err(format!("{error_message}: {status}"))
- }
- }
-
- fn full_command(&self) -> String {
- let mut lines = Vec::new();
-
- if let Some(current_dir) = self.command.get_current_dir() {
- lines.push(format!("cd {} &&", shell_quote(current_dir.as_os_str())));
- }
-
- for (key, value) in self.command.get_envs() {
- let Some(value) = value else {
- continue;
- };
- lines.push(format!("{}={}", shell_quote(key), shell_quote(value)));
- }
-
- lines.extend(shell_command_lines(
- self.command.get_program(),
- self.command.get_args(),
- ));
- lines.join(" \\\n")
- }
-}
-
-/// Groups option-value pairs so the result reads like a hand-written command.
-fn shell_command_lines<'a>(
- program: &OsStr,
- args: impl Iterator<Item = &'a OsStr>,
-) -> Vec<String> {
- let args = args.collect::<Vec<_>>();
- let mut lines = vec![shell_quote(program)];
- let mut index = 0;
-
- // Keep command names and leading positional arguments together, such as
- // `cargo check` and `cargo test plan_q`.
- while index < args.len() && !is_option(args[index]) {
- lines[0].push(' ');
- lines[0].push_str(&shell_quote(args[index]));
- index += 1;
- }
-
- while index < args.len() {
- let arg = args[index];
- let mut line = shell_quote(arg);
- index += 1;
-
- if arg == "--" {
- while index < args.len() {
- line.push(' ');
- line.push_str(&shell_quote(args[index]));
- index += 1;
+ Self::InvalidCommand(message) => {
+ format!("invalid CI command definition: {message}")
}
- } else if !arg.to_string_lossy().contains('=')
- && index < args.len()
- && !is_option(args[index])
- {
- line.push(' ');
- line.push_str(&shell_quote(args[index]));
- index += 1;
}
-
- lines.push(line);
- }
-
- lines
-}
-
-fn is_option(value: &OsStr) -> bool {
- value.to_string_lossy().starts_with('-')
-}
-
-/// Quotes one shell word when it contains characters that need protection.
-fn shell_quote(value: &OsStr) -> String {
- let value = value.to_string_lossy();
- if !value.is_empty()
- && value.bytes().all(|byte| {
- byte.is_ascii_alphanumeric()
- || matches!(
- byte,
- b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-'
- )
- })
- {
- value.into_owned()
- } else {
- format!("'{}'", value.replace('\'', "'\"'\"'"))
}
}
-fn required_env(name: &str) -> Result<String> {
- env::var(name)
- .map_err(|_| format!("required environment variable `{name}` is not set"))
-}
-
#[cfg(test)]
mod tests {
use super::*;
+ use std::ffi::OsStr;
const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";
@@ -678,10 +581,10 @@ mod tests {
#[test]
fn explain_builds_the_same_command_as_execute() {
let context = context();
- let (execute_action, _, execute_command) = context
+ let (execute_action, execute_command) = context
.ci_step(&args(&["check", "datafusion", "default"]))
.unwrap();
- let (explain_action, _, explain_command) = context
+ let (explain_action, explain_command) = context
.ci_step(&args(&["check", "datafusion", "default", "--explain"]))
.unwrap();
@@ -829,7 +732,9 @@ cargo check \
command: &["test", "workspace"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Requires executable `cargo-llvm-cov` (install it with `cargo install cargo-llvm-cov --locked`)
cd /workspace && \
+RUST_BACKTRACE=1 \
cargo llvm-cov \
--profile ci \
--exclude datafusion-examples \
@@ -849,7 +754,9 @@ cargo llvm-cov \
command: &["test", "slt-coverage"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Requires executable `cargo-llvm-cov` (install it with `cargo install cargo-llvm-cov --locked`)
cd /workspace && \
+RUST_BACKTRACE=1 \
cargo llvm-cov \
--profile ci \
--exclude datafusion-examples \
@@ -867,7 +774,16 @@ cargo llvm-cov \
command: &["test", "cli"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Run CLI tests with storage integration enabled. The AWS credentials
+# shown below are fixed test values, not secrets.
+
cd /workspace && \
+AWS_ACCESS_KEY_ID=TEST-DataFusionLogin \
+AWS_ALLOW_HTTP=true \
+AWS_ENDPOINT=http://127.0.0.1:9000 \
+AWS_SECRET_ACCESS_KEY=TEST-DataFusionPassword \
+RUST_BACKTRACE=1 \
+TEST_STORAGE_INTEGRATION=1 \
cargo test \
--features backtrace \
--profile ci \
@@ -909,6 +825,12 @@ cargo test \
command: &["test", "microbenchmarks"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Run each Criterion benchmark once without timing it, selecting only
+# bench targets (not lib/bin unit tests). Keep debug assertions and
+# overflow checks enabled while optimizing execution for larger datasets.
+# The SQL benchmark harness needs separately generated datasets and
+# is covered by the benchmark result verification job.
+
cd /workspace && \
CLICKBENCH_DATA_PATH=/workspace/datafusion/core/tests/data/clickbench_hits_10.parquet \
cargo test \
@@ -928,6 +850,7 @@ cargo test \
command: &["test", "benchmark-plan"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Requires path `/workspace/datafusion/sqllogictest/test_files/tpch/data` (generate the TPC-H fixture data before running this step)
cd /workspace && \
RUST_MIN_STACK=20971520 \
TPCH_DATA=/workspace/datafusion/sqllogictest/test_files/tpch/data \
@@ -943,6 +866,7 @@ cargo test plan_q \
command: &["test", "benchmark-sqllogic"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Requires path `/workspace/datafusion/sqllogictest/test_files/tpch/data` (generate the TPC-H fixture data before running this step)
cd /workspace && \
INCLUDE_TPCH=true \
RUST_MIN_STACK=20971520 \
@@ -959,6 +883,8 @@ cargo test \
command: &["test", "postgres"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Requires environment variable `POSTGRES_HOST`
+# Requires environment variable `POSTGRES_PORT`
cd /workspace/datafusion/sqllogictest && \
PG_COMPAT=true \
PG_URI=postgresql://postgres:[email protected]:15432/db_test \
@@ -1000,6 +926,9 @@ cargo test \
command: &["test", "extended"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Run tests behind the `extended_tests` feature. The `workspace`
+# step runs the remaining tests.
+
cd /workspace && \
RUST_BACKTRACE=1 \
cargo test \
@@ -1014,6 +943,9 @@ cargo test \
command: &["test", "spill-pool-fuzz"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Run more random spill pool fuzzer scenarios than the `workspace`
+# step. This takes about a minute.
+
cd /workspace && \
DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS=1000 \
RUST_BACKTRACE=1 \
@@ -1029,6 +961,9 @@ cargo test \
command: &["test", "hash-collisions"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Check that results stay correct when hash values collide.
+# Run from the `datafusion` directory with `RUST_BACKTRACE=1`.
+
cd /workspace/datafusion && \
RUST_BACKTRACE=1 \
cargo test \
@@ -1048,6 +983,9 @@ cargo test \
command: &["test", "sqlite"],
expected: |actual| {
insta::assert_snapshot!(actual, @r"
+# Run SQL logic tests including the SQLite suite. This step does not
+# set `RUST_BACKTRACE` because backtraces make the suite expensive.
+
cd /workspace && \
cargo test \
--features backtrace,parquet_encryption \
@@ -1066,10 +1004,10 @@ cargo test \
.copied()
.chain(std::iter::once("--explain"))
.collect::<Vec<_>>();
- let (action, _, actual) = context.ci_step(&args(&explain_args)).unwrap();
+ let (action, actual) = context.ci_step(&args(&explain_args)).unwrap();
assert_eq!(action, StepAction::Explain);
- expected(&actual.full_command());
+ expected(&actual.explanation());
}
}
}
diff --git a/xtask/src/command.rs b/xtask/src/command.rs
new file mode 100644
index 0000000000..0bfa3f5bde
--- /dev/null
+++ b/xtask/src/command.rs
@@ -0,0 +1,685 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+//! Complete, inspectable definitions of external commands.
+//!
+//! CI code only describes what a command needs: program, arguments, working
+//! directory, non-secret environment, prerequisites, and failure context.
+//! This module owns the mechanics of resolving that definition, validating it,
+//! rendering it for `--explain`, and executing it. Keeping those paths together
+//! ensures that local and CI invocations cannot drift from the command being shown.
+
+use crate::Result;
+use std::collections::{BTreeMap, BTreeSet};
+use std::env;
+use std::ffi::{OsStr, OsString};
+use std::fmt::Write as _;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+pub(crate) type EnvReader = fn(&str) -> Result<String>;
+
+/// Access to environment inputs required while resolving a command.
+struct HostEnvironment {
+ reader: EnvReader,
+}
+
+impl HostEnvironment {
+ fn new(reader: EnvReader) -> Self {
+ Self { reader }
+ }
+
+ fn required(&self, name: &str) -> Result<String> {
+ (self.reader)(name)
+ }
+}
+
+enum EnvTemplatePart {
+ Literal(&'static str),
+ Required(&'static str),
+}
+
+struct EnvTemplate {
+ parts: Vec<EnvTemplatePart>,
+}
+
+impl EnvTemplate {
+ fn parse(mut template: &'static str) -> Result<Self> {
+ let original = template;
+ let mut parts = Vec::new();
+ while let Some(start) = template.find('{') {
+ let (literal, after_literal) = template.split_at(start);
+ if literal.contains('}') {
+ return Err(format!("invalid environment template `{original}`"));
+ }
+ if !literal.is_empty() {
+ parts.push(EnvTemplatePart::Literal(literal));
+ }
+ let after_open = &after_literal[1..];
+ let Some(end) = after_open.find('}') else {
+ return Err(format!("invalid environment template `{original}`"));
+ };
+ let (name, after_name) = after_open.split_at(end);
+ if name.is_empty()
+ || !name.bytes().all(|byte| {
+ byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_'
+ })
+ {
+ return Err(format!("invalid environment name in template `{original}`"));
+ }
+ parts.push(EnvTemplatePart::Required(name));
+ template = &after_name[1..];
+ }
+ if template.contains('}') {
+ return Err(format!("invalid environment template `{original}`"));
+ }
+ if !template.is_empty() {
+ parts.push(EnvTemplatePart::Literal(template));
+ }
+ Ok(Self { parts })
+ }
+
+ fn resolve(&self, environment: &HostEnvironment) -> Result<OsString> {
+ let mut value = String::new();
+ for part in &self.parts {
+ match part {
+ EnvTemplatePart::Literal(literal) => value.push_str(literal),
+ EnvTemplatePart::Required(name) => {
+ value.push_str(&environment.required(name)?)
+ }
+ }
+ }
+ Ok(value.into())
+ }
+
+ fn display(&self, environment: &HostEnvironment) -> OsString {
+ let mut value = String::new();
+ for part in &self.parts {
+ match part {
+ EnvTemplatePart::Literal(literal) => value.push_str(literal),
+ EnvTemplatePart::Required(name) => match environment.required(name) {
+ Ok(resolved) => value.push_str(&resolved),
+ Err(_) => {
+ write!(value, "<{name}>").ok();
+ }
+ },
+ }
+ }
+ value.into()
+ }
+
+ fn required_env(&self) -> impl Iterator<Item = &'static str> + '_ {
+ self.parts.iter().filter_map(|part| match part {
+ EnvTemplatePart::Required(name) => Some(*name),
+ EnvTemplatePart::Literal(_) => None,
+ })
+ }
+}
+
+enum EnvValue {
+ Literal(OsString),
+ Template(EnvTemplate),
+}
+
+enum PendingEnvValue {
+ Literal(OsString),
+ Template(&'static str),
+}
+
+impl EnvValue {
+ fn resolve(&self, environment: &HostEnvironment) -> Result<OsString> {
+ match self {
+ Self::Literal(value) => Ok(value.clone()),
+ Self::Template(template) => template.resolve(environment),
+ }
+ }
+
+ fn display(&self, environment: &HostEnvironment) -> OsString {
+ match self {
+ Self::Literal(value) => value.clone(),
+ Self::Template(template) => template.display(environment),
+ }
+ }
+}
+
+/// A tool or path that must exist before a command can run.
+///
+/// Execution checks each prerequisite before starting the process.
+/// If one is missing, it returns an error containing the `help` hint.
+///
+/// # Example
+/// A coverage command needs `cargo-llvm-cov` in `PATH`.
+/// Its `help` hint tells the user how to install it:
+///
+/// ```text
+/// cargo install cargo-llvm-cov --locked
+/// ```
+///
+/// `--explain` lists this prerequisite even when the tool is missing.
+enum Requirement {
+ Executable { name: OsString, help: &'static str },
+ Path { path: PathBuf, help: &'static str },
+}
+
+impl Requirement {
+ fn explain(&self) -> String {
+ match self {
+ Self::Executable { name, help } => {
+ format!("executable `{}` ({help})", name.to_string_lossy())
+ }
+ Self::Path { path, help } => {
+ format!("path `{}` ({help})", path.display())
+ }
+ }
+ }
+
+ fn validate(&self) -> Result<()> {
+ match self {
+ Self::Executable { name, help } if !executable_exists(name) => Err(format!(
+ "required executable `{}` was not found in PATH; {help}",
+ name.to_string_lossy()
+ )),
+ Self::Path { path, help } if !path.exists() => Err(format!(
+ "required path `{}` does not exist; {help}",
+ path.display()
+ )),
+ _ => Ok(()),
+ }
+ }
+}
+
+/// A complete recipe for an external command, shared by `--explain` and execution.
+///
+/// Example coverage command, run from the repository root:
+///
+/// ```text
+/// RUST_BACKTRACE=1 cargo llvm-cov
+/// ```
+///
+/// Prerequisite: `cargo-llvm-cov` must be available in `PATH`.
+///
+/// - `--explain` prints the documentation, prerequisites, and command.
+/// - Execution checks the prerequisites, then launches the process.
+/// Failures are reported with this step's context.
+pub(crate) struct CommandSpec {
+ /// Optional explanation, printed as shell comments before the command.
+ doc: Option<&'static str>,
+ /// Executable to launch, e.g. `cargo`.
+ program: OsString,
+ /// Arguments in order, e.g. `["llvm-cov"]`.
+ args: Vec<OsString>,
+ /// Working directory, e.g. the repository root.
+ current_dir: PathBuf,
+ /// Child process environment overrides, e.g. `RUST_BACKTRACE=1`.
+ env: BTreeMap<OsString, EnvValue>,
+ /// Prerequisites checked before launch, e.g. `cargo-llvm-cov` in `PATH`.
+ requirements: Vec<Requirement>,
+ /// Reads host variables used by templates in `env`.
+ environment: HostEnvironment,
+ /// Error prefix identifying the step, e.g. `CI test step failed`.
+ failure_message: &'static str,
+}
+
+/// Collects a command definition before validating its required fields.
+pub(crate) struct CommandSpecBuilder {
+ doc: Option<&'static str>,
+ program: Option<OsString>,
+ args: Vec<OsString>,
+ current_dir: Option<PathBuf>,
+ env: BTreeMap<OsString, PendingEnvValue>,
+ requirements: Vec<Requirement>,
+ environment: HostEnvironment,
+ failure_message: Option<&'static str>,
+}
+
+impl CommandSpec {
+ pub(crate) fn builder(env_reader: EnvReader) -> CommandSpecBuilder {
+ CommandSpecBuilder {
+ doc: None,
+ program: None,
+ args: Vec::new(),
+ current_dir: None,
+ env: BTreeMap::new(),
+ requirements: Vec::new(),
+ environment: HostEnvironment::new(env_reader),
+ failure_message: None,
+ }
+ }
+}
+
+impl CommandSpecBuilder {
+ pub(crate) fn doc(&mut self, doc: &'static str) -> &mut Self {
+ self.doc = Some(doc);
+ self
+ }
+
+ pub(crate) fn program(&mut self, program: impl AsRef<OsStr>) -> &mut Self {
+ self.program = Some(program.as_ref().to_owned());
+ self
+ }
+
+ pub(crate) fn args<I, S>(&mut self, args: I) -> &mut Self
+ where
+ I: IntoIterator<Item = S>,
+ S: AsRef<OsStr>,
+ {
+ self.args
+ .extend(args.into_iter().map(|arg| arg.as_ref().to_owned()));
+ self
+ }
+
+ pub(crate) fn env<K, V>(&mut self, key: K, value: V) -> &mut Self
+ where
+ K: AsRef<OsStr>,
+ V: AsRef<OsStr>,
+ {
+ // Command definitions must not pass secrets here: values are displayed
+ // by both `--explain` and the execution log.
+ self.env.insert(
+ key.as_ref().to_owned(),
+ PendingEnvValue::Literal(value.as_ref().to_owned()),
+ );
+ self
+ }
+
+ pub(crate) fn env_template<K>(&mut self, key: K, template: &'static str) -> &mut Self
+ where
+ K: AsRef<OsStr>,
+ {
+ // Templates are inspectable command semantics, so resolved values are
+ // subject to the same non-secret restriction as literal values.
+ self.env
+ .insert(key.as_ref().to_owned(), PendingEnvValue::Template(template));
+ self
+ }
+
+ pub(crate) fn requires_executable(
+ &mut self,
+ name: impl AsRef<OsStr>,
+ help: &'static str,
+ ) -> &mut Self {
+ self.requirements.push(Requirement::Executable {
+ name: name.as_ref().to_owned(),
+ help,
+ });
+ self
+ }
+
+ pub(crate) fn requires_path(
+ &mut self,
+ path: impl AsRef<Path>,
+ help: &'static str,
+ ) -> &mut Self {
+ self.requirements.push(Requirement::Path {
+ path: path.as_ref().to_path_buf(),
+ help,
+ });
+ self
+ }
+
+ pub(crate) fn current_dir(&mut self, directory: impl AsRef<Path>) -> &mut Self {
+ self.current_dir = Some(directory.as_ref().to_path_buf());
+ self
+ }
+
+ pub(crate) fn failure_message(&mut self, message: &'static str) -> &mut Self {
+ self.failure_message = Some(message);
+ self
+ }
+
+ /// Finalizes the command only after its structural definition is complete.
+ pub(crate) fn build(self) -> Result<CommandSpec> {
+ let program = self
+ .program
+ .filter(|program| !program.is_empty())
+ .ok_or_else(|| "command program is required".to_string())?;
+ let current_dir = self
+ .current_dir
+ .filter(|directory| !directory.as_os_str().is_empty())
+ .ok_or_else(|| "command working directory is required".to_string())?;
+ let failure_message = self
+ .failure_message
+ .filter(|message| !message.trim().is_empty())
+ .ok_or_else(|| "command failure message is required".to_string())?;
+
+ let env =
+ self.env
+ .into_iter()
+ .map(|(key, value)| {
+ if key.is_empty() {
+ return Err("command environment variable name cannot be empty"
+ .to_string());
+ }
+ let value = match value {
+ PendingEnvValue::Literal(value) => EnvValue::Literal(value),
+ PendingEnvValue::Template(template) => {
+ EnvValue::Template(EnvTemplate::parse(template)?)
+ }
+ };
+ Ok((key, value))
+ })
+ .collect::<Result<_>>()?;
+
+ Ok(CommandSpec {
+ doc: self.doc,
+ program,
+ args: self.args,
+ current_dir,
+ env,
+ requirements: self.requirements,
+ environment: self.environment,
+ failure_message,
+ })
+ }
+}
+
+impl CommandSpec {
+ pub(crate) fn explain(&self) {
+ println!("{}", self.explanation());
+ }
+
+ pub(crate) fn explanation(&self) -> String {
+ let mut lines = Vec::new();
+ if let Some(doc) = self.doc {
+ lines.extend(doc.lines().map(|line| format!("# {line}")));
+ lines.push(String::new());
+ }
+ lines.extend(
+ self.requirement_descriptions()
+ .into_iter()
+ .map(|requirement| format!("# Requires {requirement}")),
+ );
+ lines.push(self.full_command());
+ lines.join("\n")
+ }
+
+ pub(crate) fn execute(self) -> Result<()> {
+ self.validate_requirements()
+ .map_err(|error| format!("{}: {error}", self.failure_message))?;
+ let env = self
+ .resolve_env()
+ .map_err(|error| format!("{}: {error}", self.failure_message))?;
+ println!("+ {}", self.render_command(&env));
+
+ let status = Command::new(&self.program)
+ .args(&self.args)
+ .current_dir(&self.current_dir)
+ .envs(env)
+ .stdin(Stdio::inherit())
+ .stdout(Stdio::inherit())
+ .stderr(Stdio::inherit())
+ .status()
+ .map_err(|error| format!("{}: {error}", self.failure_message))?;
+
+ if status.success() {
+ Ok(())
+ } else {
+ Err(format!("{}: {status}", self.failure_message))
+ }
+ }
+
+ pub(crate) fn full_command(&self) -> String {
+ let env = self
+ .env
+ .iter()
+ .map(|(key, value)| (key.clone(), value.display(&self.environment)))
+ .collect();
+ self.render_command(&env)
+ }
+
+ fn requirement_descriptions(&self) -> Vec<String> {
+ let mut descriptions = self
+ .requirements
+ .iter()
+ .map(Requirement::explain)
+ .collect::<Vec<_>>();
+ let mut required_env = BTreeSet::new();
+ for value in self.env.values() {
+ if let EnvValue::Template(template) = value {
+ required_env.extend(template.required_env());
+ }
+ }
+ descriptions.extend(
+ required_env
+ .into_iter()
+ .map(|name| format!("environment variable `{name}`")),
+ );
+ descriptions
+ }
+
+ fn validate_requirements(&self) -> Result<()> {
+ for requirement in &self.requirements {
+ requirement.validate()?;
+ }
+ Ok(())
+ }
+
+ fn resolve_env(&self) -> Result<BTreeMap<OsString, OsString>> {
+ self.env
+ .iter()
+ .map(|(key, value)| {
+ value
+ .resolve(&self.environment)
+ .map(|value| (key.clone(), value))
+ })
+ .collect()
+ }
+
+ fn render_command(&self, env: &BTreeMap<OsString, OsString>) -> String {
+ let mut lines = vec![format!(
+ "cd {} &&",
+ shell_quote(self.current_dir.as_os_str())
+ )];
+
+ lines.extend(
+ env.iter().map(|(key, value)| {
+ format!("{}={}", shell_quote(key), shell_quote(value))
+ }),
+ );
+ lines.extend(shell_command_lines(&self.program, self.args.iter()));
+ lines.join(" \\\n")
+ }
+}
+
+pub(crate) fn system_env(name: &str) -> Result<String> {
+ env::var(name)
+ .map_err(|_| format!("required environment variable `{name}` is not set"))
+}
+
+fn executable_exists(name: &OsStr) -> bool {
+ let path = Path::new(name);
+ if path.components().count() > 1 {
+ return is_executable_file(path);
+ }
+
+ let Some(search_path) = env::var_os("PATH") else {
+ return false;
+ };
+
+ env::split_paths(&search_path).any(|directory| executable_in(&directory, name))
+}
+
+fn executable_in(directory: &Path, name: &OsStr) -> bool {
+ let candidate = directory.join(name);
+ if is_executable_file(&candidate) {
+ return true;
+ }
+
+ #[cfg(windows)]
+ {
+ if Path::new(name).extension().is_some() {
+ return false;
+ }
+ let extensions = env::var_os("PATHEXT")
+ .unwrap_or_else(|| OsString::from(".COM;.EXE;.BAT;.CMD"));
+ return extensions.to_string_lossy().split(';').any(|extension| {
+ is_executable_file(
+ &candidate.with_extension(extension.trim_start_matches('.')),
+ )
+ });
+ }
+
+ #[cfg(not(windows))]
+ false
+}
+
+fn is_executable_file(path: &Path) -> bool {
+ let Ok(metadata) = path.metadata() else {
+ return false;
+ };
+ if !metadata.is_file() {
+ return false;
+ }
+
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ metadata.permissions().mode() & 0o111 != 0
+ }
+
+ #[cfg(not(unix))]
+ true
+}
+
+/// Groups option-value pairs so the result reads like a hand-written command.
+fn shell_command_lines<'a>(
+ program: &OsStr,
+ args: impl Iterator<Item = &'a OsString>,
+) -> Vec<String> {
+ let args = args.map(OsString::as_os_str).collect::<Vec<_>>();
+ let mut lines = vec![shell_quote(program)];
+ let mut index = 0;
+
+ // Keep command names and leading positional arguments together, such as
+ // `cargo check` and `cargo test plan_q`.
+ while index < args.len() && !is_option(args[index]) {
+ lines[0].push(' ');
+ lines[0].push_str(&shell_quote(args[index]));
+ index += 1;
+ }
+
+ while index < args.len() {
+ let arg = args[index];
+ let mut line = shell_quote(arg);
+ index += 1;
+
+ if arg == "--" {
+ while index < args.len() {
+ line.push(' ');
+ line.push_str(&shell_quote(args[index]));
+ index += 1;
+ }
+ } else if !arg.to_string_lossy().contains('=')
+ && index < args.len()
+ && !is_option(args[index])
+ {
+ line.push(' ');
+ line.push_str(&shell_quote(args[index]));
+ index += 1;
+ }
+
+ lines.push(line);
+ }
+
+ lines
+}
+
+fn is_option(value: &OsStr) -> bool {
+ value.to_string_lossy().starts_with('-')
+}
+
+/// Quotes one shell word when it contains characters that need protection.
+fn shell_quote(value: &OsStr) -> String {
+ let value = value.to_string_lossy();
+ if !value.is_empty()
+ && value.bytes().all(|byte| {
+ byte.is_ascii_alphanumeric()
+ || matches!(
+ byte,
+ b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-'
+ )
+ })
+ {
+ value.into_owned()
+ } else {
+ format!("'{}'", value.replace('\'', "'\"'\"'"))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn missing_env(name: &str) -> Result<String> {
+ Err(format!("missing `{name}`"))
+ }
+
+ fn command_builder() -> CommandSpecBuilder {
+ let mut builder = CommandSpec::builder(missing_env);
+ builder
+ .program("tool")
+ .current_dir("/workspace")
+ .failure_message("tool failed");
+ builder
+ }
+
+ #[test]
+ fn explain_lists_prerequisites_without_resolving_environment() {
+ let mut builder = command_builder();
+ builder
+ .env_template("ENDPOINT", "{HOST}:{PORT}")
+ .requires_path(
+ "/workspace/input",
+ "generate the input before running this command",
+ );
+ let command = builder.build().unwrap();
+
+ insta::assert_snapshot!(command.explanation(), @r"
+# Requires path `/workspace/input` (generate the input before running this command)
+# Requires environment variable `HOST`
+# Requires environment variable `PORT`
+cd /workspace && \
+ENDPOINT='<HOST>:<PORT>' \
+tool
+");
+ }
+
+ #[test]
+ fn missing_prerequisite_is_reported_with_failure_context() {
+ let mut builder = command_builder();
+ builder.requires_executable(
+ "datafusion-xtask-command-that-does-not-exist",
+ "install the test tool",
+ );
+ let command = builder.build().unwrap();
+
+ assert_eq!(
+ command.execute().unwrap_err(),
+ "tool failed: required executable `datafusion-xtask-command-that-does-not-exist` was not found in PATH; install the test tool"
+ );
+ }
+
+ #[test]
+ fn missing_environment_is_reported_with_failure_context() {
+ let mut builder = command_builder();
+ builder.env_template("ENDPOINT", "{HOST}:{PORT}");
+ let command = builder.build().unwrap();
+
+ assert_eq!(
+ command.execute().unwrap_err(),
+ "tool failed: missing `HOST`"
+ );
+ }
+
+ #[test]
+ fn build_validates_required_fields_and_templates() {
+ let error = CommandSpec::builder(missing_env).build().err().unwrap();
+ assert_eq!(error, "command program is required");
+
+ let mut builder = command_builder();
+ builder.env_template("ENDPOINT", "{lowercase}");
+ let error = builder.build().err().unwrap();
+ assert_eq!(error, "invalid environment name in template `{lowercase}`");
+ }
+}
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index 3f1fc7b5c2..f69b392f69 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -67,6 +67,7 @@
//! ```
mod ci_steps;
+mod command;
use std::env;
use std::path::PathBuf;