Apache DataFusion SQL Query Engine
refactor: Centralize CI command definitions in xtask (#26765)
Yongting You0e314cc3c904
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 4db0dcacee..bbee5900cd 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -462,8 +462,6 @@ jobs:save-if: ${{ github.ref_name == 'main' }}shared-key: "amd-ci"- name: Run tests (excluding doctests and datafusion-cli)- env:- RUST_BACKTRACE: 1run: cargo xtask ci step test workspace- name: Run SQLLogicTests with coverageenv:@@ -533,13 +531,6 @@ jobs:ci/scripts/retry timeout 120 docker pull "$RUSTFS_IMAGE" \|| echo "::warning::Could not pre-pull $RUSTFS_IMAGE, the tests will pull it themselves"- name: Run tests (excluding doctests)- env:- RUST_BACKTRACE: 1- AWS_ENDPOINT: http://127.0.0.1:9000- AWS_ACCESS_KEY_ID: TEST-DataFusionLogin- AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword- TEST_STORAGE_INTEGRATION: 1- AWS_ALLOW_HTTP: truerun: cargo xtask ci step test cli- name: Verify Working Directory Cleanrun: git diff --exit-codediff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rsindex 5184a5cfbf..c6b605d36e 100644--- a/xtask/src/ci_steps.rs+++ b/xtask/src/ci_steps.rs@@ -5,14 +5,12 @@//! CI steps that can be inspected or run through `cargo xtask`.use crate::Result;-use std::env;-use std::ffi::OsStr;+use crate::command::{CommandSpec, EnvReader, system_env};use std::fmt::Write as _;use std::path::{Path, PathBuf};-use std::process::{Command, Stdio};type StepResult<T> = std::result::Result<T, StepError>;-type StepRunner = fn(&StepContext, &[String]) -> StepResult<CiCommand>;+type StepRunner = fn(&StepContext, &[String]) -> StepResult<CommandSpec>;const CI_COMMAND: &str = "cargo xtask ci step";const CI_SHORTCUT: &str = "cargo ci-step";@@ -29,7 +27,6 @@ struct StepInfo {help_usage: &'static str,help_examples: &'static [&'static str],help_description: &'static str,- error_message: &'static str,runner: StepRunner,}@@ -39,7 +36,6 @@ static CI_STEPS: &[StepInfo] = &[help_usage: "check <workspace|package> [default|no-default|feature] [--explain]",help_examples: &["check workspace", "check datafusion default"],help_description: "Check workspace or package compilation",- error_message: "Cargo check step failed",runner: StepContext::run_check,},StepInfo {@@ -47,7 +43,6 @@ static CI_STEPS: &[StepInfo] = &[help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",help_examples: &["test cli", "test workspace"],help_description: "Run a CI test suite",- error_message: "Cargo test step failed",runner: StepContext::run_test,},];@@ -117,14 +112,14 @@ pub(crate) fn run(root: &Path, args: &[String]) -> Result<()> {struct StepContext {root: PathBuf,- env_reader: fn(&str) -> Result<String>,+ env_reader: EnvReader,}impl StepContext {fn new(root: &Path) -> Self {Self {root: root.to_path_buf(),- env_reader: required_env,+ env_reader: system_env,}}@@ -143,9 +138,9 @@ impl StepContext {Ok(())}step_args => {- let (action, step, command) = self.ci_step(step_args)?;+ let (action, command) = self.ci_step(step_args)?;match action {- StepAction::Execute => command.execute(step.error_message),+ StepAction::Execute => command.execute(),StepAction::Explain => {command.explain();Ok(())@@ -156,10 +151,7 @@ impl StepContext {}/// Parses a CI step into an action and its complete command description.- fn ci_step(- &self,- args: &[String],- ) -> Result<(StepAction, &'static StepInfo, CiCommand)> {+ fn ci_step(&self, args: &[String]) -> Result<(StepAction, CommandSpec)> {let (action, args) = match args.split_last() {Some((arg, args)) if arg == "--explain" => (StepAction::Explain, args),_ => (StepAction::Execute, args),@@ -170,18 +162,23 @@ impl StepContext {let step = find_step(step)?;let command = (step.runner)(self, args).map_err(|error| error.render(step))?;- Ok((action, step, command))+ Ok((action, command))}/// Implementation for `cargo xtask ci step check [args]`- fn run_check(&self, args: &[String]) -> StepResult<CiCommand> {+ fn run_check(&self, args: &[String]) -> StepResult<CommandSpec> {let args = args.iter().map(String::as_str).collect::<Vec<_>>();- let mut command = self.cargo();- command.args(["check", "--profile", "ci"]);+ let mut builder = CommandSpec::builder(self.env_reader);+ builder+ .current_dir(&self.root)+ .failure_message("CI check step failed");match args.as_slice() {["workspace"] => {- command.args([+ builder.program("cargo").args([+ "check",+ "--profile",+ "ci","--workspace","--all-targets","--features",@@ -190,13 +187,30 @@ impl StepContext {]);}[package, "default"] => {- command.args(["--all-targets", "-p", package]);+ builder.program("cargo").args([+ "check",+ "--profile",+ "ci",+ "--all-targets",+ "-p",+ package,+ ]);}[package, "no-default"] => {- command.args(["--no-default-features", "-p", package]);+ builder.program("cargo").args([+ "check",+ "--profile",+ "ci",+ "--no-default-features",+ "-p",+ package,+ ]);}[package, feature] => {- command.args([+ builder.program("cargo").args([+ "check",+ "--profile",+ "ci","--no-default-features","-p",package,@@ -207,19 +221,23 @@ impl StepContext {_ => return Err(StepError::Usage),}- Ok(command)+ builder.build().map_err(StepError::InvalidCommand)}/// Implementation for `cargo xtask ci step test [args]`- fn run_test(&self, args: &[String]) -> StepResult<CiCommand> {+ fn run_test(&self, args: &[String]) -> StepResult<CommandSpec> {let [variant] = args else {return Err(StepError::Usage);};- let mut command = self.cargo();+ let mut cmd_builder = CommandSpec::builder(self.env_reader);+ cmd_builder+ .current_dir(&self.root)+ .failure_message("CI test step failed");+match variant.as_str() {"workspace" | "slt-coverage" => {- command.args([+ cmd_builder.program("cargo").args(["llvm-cov","--profile","ci",@@ -232,36 +250,54 @@ impl StepContext {"--workspace",]);let output_path = if variant == "slt-coverage" {- command.args(["--test", "sqllogictests"]);+ cmd_builder.args(["--test", "sqllogictests"]);"target/codecov-slt.json"} else {- command.args(["--lib", "--tests", "--bins"]);+ cmd_builder.args(["--lib", "--tests", "--bins"]);"target/codecov.json"};- command.args([- "--features",- "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait",- "--codecov",- "--output-path",- output_path,- ]);+ cmd_builder+ .args([+ "--features",+ "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait",+ "--codecov",+ "--output-path",+ output_path,+ ])+ .env("RUST_BACKTRACE", "1")+ .requires_executable(+ "cargo-llvm-cov",+ "install it with `cargo install cargo-llvm-cov --locked`",+ );}"cli" => {- command.args([- "test",- "--features",- "backtrace",- "--profile",- "ci",- "-p",- "datafusion-cli",- "--lib",- "--tests",- "--bins",- ]);+ cmd_builder.doc(+ "Run CLI tests with storage integration enabled. The AWS credentials\n\+ shown below are fixed test values, not secrets.",+ );+ cmd_builder+ .program("cargo")+ .args([+ "test",+ "--features",+ "backtrace",+ "--profile",+ "ci",+ "-p",+ "datafusion-cli",+ "--lib",+ "--tests",+ "--bins",+ ])+ .env("RUST_BACKTRACE", "1")+ .env("AWS_ENDPOINT", "http://127.0.0.1:9000")+ .env("AWS_ACCESS_KEY_ID", "TEST-DataFusionLogin")+ .env("AWS_SECRET_ACCESS_KEY", "TEST-DataFusionPassword")+ .env("TEST_STORAGE_INTEGRATION", "1")+ .env("AWS_ALLOW_HTTP", "true");}"doctest" => {- command.args([+ cmd_builder.program("cargo").args(["test","--profile","ci",@@ -271,7 +307,7 @@ impl StepContext {]);}"ffi" => {- command.args([+ cmd_builder.program("cargo").args(["test","--profile","ci",@@ -286,18 +322,19 @@ impl StepContext {]);}"microbenchmarks" => {- // The SQL benchmark harness needs separately generated datasets- // and is covered by the benchmark result verification job.- // Select only bench targets (not lib/bin unit tests), and tell- // Criterion to execute each benchmark once instead of timing it.- // Keep debug assertions and overflow checks enabled without the- // cost of unoptimized execution on the larger benchmark datasets.- command.env(+ cmd_builder.doc(+ "Run each Criterion benchmark once without timing it, selecting only\n\+ bench targets (not lib/bin unit tests). Keep debug assertions and\n\+ overflow checks enabled while optimizing execution for larger datasets.\n\+ The SQL benchmark harness needs separately generated datasets and\n\+ is covered by the benchmark result verification job.",+ );+ cmd_builder.env("CLICKBENCH_DATA_PATH",self.root.join("datafusion/core/tests/data/clickbench_hits_10.parquet"),);- command.args([+ cmd_builder.program("cargo").args(["test","--profile","ci-bench",@@ -318,7 +355,9 @@ impl StepContext {]);}"benchmark-plan" => {- command+ let tpch_data = self.root.join(TPCH_DATA_DIR);+ cmd_builder+ .program("cargo").args(["test","plan_q",@@ -331,10 +370,16 @@ impl StepContext {"--test-threads=1",]).env("RUST_MIN_STACK", "20971520")- .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR));+ .env("TPCH_DATA", tpch_data.as_os_str())+ .requires_path(+ tpch_data,+ "generate the TPC-H fixture data before running this step",+ );}"benchmark-sqllogic" => {- command+ let tpch_data = self.root.join(TPCH_DATA_DIR);+ cmd_builder+ .program("cargo").args(["test","--features",@@ -347,18 +392,16 @@ impl StepContext {"sqllogictests",]).env("RUST_MIN_STACK", "20971520")- .env("TPCH_DATA", self.root.join(TPCH_DATA_DIR))- .env("INCLUDE_TPCH", "true");+ .env("TPCH_DATA", tpch_data.as_os_str())+ .env("INCLUDE_TPCH", "true")+ .requires_path(+ tpch_data,+ "generate the TPC-H fixture data before running this step",+ );}"postgres" => {- let host = self- .required_env("POSTGRES_HOST")- .map_err(StepError::Message)?;- let port = self- .required_env("POSTGRES_PORT")- .map_err(StepError::Message)?;- let uri = format!("postgresql://postgres:postgres@{host}:{port}/db_test");- command+ cmd_builder+ .program("cargo").args(["test","--features",@@ -371,10 +414,13 @@ impl StepContext {]).current_dir(self.root.join("datafusion/sqllogictest")).env("PG_COMPAT", "true")- .env("PG_URI", uri);+ .env_template(+ "PG_URI",+ "postgresql://postgres:postgres@{POSTGRES_HOST}:{POSTGRES_PORT}/db_test",+ );}"substrait" | "substrait-optimized" => {- command.args([+ cmd_builder.program("cargo").args(["test","-p","datafusion-sqllogictest",@@ -386,15 +432,18 @@ impl StepContext {"--substrait-round-trip",]);if variant == "substrait-optimized" {- command.args(["--substrait-optimize", "joins_conditionless.slt"]);+ cmd_builder.args(["--substrait-optimize", "joins_conditionless.slt"]);} else {- command.args(["limit.slt"]);+ cmd_builder.args(["limit.slt"]);}}"extended" => {- // Tests behind the `extended_tests` feature. The `workspace`- // step runs everything else.- command+ cmd_builder.doc(+ "Run tests behind the `extended_tests` feature. The `workspace`\n\+ step runs the remaining tests.",+ );+ cmd_builder+ .program("cargo").args(["test","--profile",@@ -409,9 +458,12 @@ impl StepContext {.env("RUST_BACKTRACE", "1");}"spill-pool-fuzz" => {- // Run more random scenarios of the spill pool fuzzer than the- // `workspace` step does (about a minute).- command+ cmd_builder.doc(+ "Run more random spill pool fuzzer scenarios than the `workspace`\n\+ step. This takes about a minute.",+ );+ cmd_builder+ .program("cargo").args(["test","--profile",@@ -427,10 +479,12 @@ impl StepContext {.env("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", "1000");}"hash-collisions" => {- // Check answers are correct when hash values collide. The CI job- // runs from the `datafusion` directory with the builder setup,- // which sets `RUST_BACKTRACE=1`.- command+ cmd_builder.doc(+ "Check that results stay correct when hash values collide.\n\+ Run from the `datafusion` directory with `RUST_BACKTRACE=1`.",+ );+ cmd_builder+ .program("cargo").args(["test","--profile",@@ -452,9 +506,11 @@ impl StepContext {.env("RUST_BACKTRACE", "1");}"sqlite" => {- // The CI job skips the builder setup because backtraces make this- // suite expensive, so `RUST_BACKTRACE` is deliberately not set.- command.args([+ cmd_builder.doc(+ "Run SQL logic tests including the SQLite suite. This step does not\n\+ set `RUST_BACKTRACE` because backtraces make the suite expensive.",+ );+ cmd_builder.program("cargo").args(["test","--features","backtrace,parquet_encryption",@@ -469,15 +525,7 @@ impl StepContext {_ => return Err(StepError::Usage),}- Ok(command)- }-- fn cargo(&self) -> CiCommand {- CiCommand::new("cargo", &self.root)- }-- fn required_env(&self, name: &str) -> Result<String> {- (self.env_reader)(name)+ cmd_builder.build().map_err(StepError::InvalidCommand)}}@@ -490,169 +538,24 @@ enum StepAction {#[derive(Debug)]enum StepError {Usage,- Message(String),+ InvalidCommand(String),}impl StepError {fn render(self, step: &StepInfo) -> String {match self {Self::Usage => format!("usage: {CI_COMMAND} {}", step.help_usage),- Self::Message(message) => message,- }- }-}--/// Command shared by invocation and `--explain`-struct CiCommand {- command: Command,-}--impl CiCommand {- fn new(program: impl AsRef<OsStr>, current_dir: impl AsRef<Path>) -> Self {- let mut command = Command::new(program);- command.current_dir(current_dir);- Self { command }- }-- fn args<I, S>(&mut self, args: I) -> &mut Self- where- I: IntoIterator<Item = S>,- S: AsRef<OsStr>,- {- self.command.args(args);- self- }-- fn env<K, V>(&mut self, key: K, value: V) -> &mut Self- where- K: AsRef<OsStr>,- V: AsRef<OsStr>,- {- self.command.env(key, value);- self- }-- fn current_dir(&mut self, directory: impl AsRef<Path>) -> &mut Self {- self.command.current_dir(directory);- self- }-- fn explain(&self) {- println!("{}", self.full_command());- }-- fn execute(mut self, error_message: &str) -> Result<()> {- println!("+ {}", self.full_command());- let status = self- .command- .stdin(Stdio::inherit())- .stdout(Stdio::inherit())- .stderr(Stdio::inherit())- .status()- .map_err(|error| format!("{error_message}: {error}"))?;-- if status.success() {- Ok(())- } else {- Err(format!("{error_message}: {status}"))- }- }-- fn full_command(&self) -> String {- let mut lines = Vec::new();-- if let Some(current_dir) = self.command.get_current_dir() {- lines.push(format!("cd {} &&", shell_quote(current_dir.as_os_str())));- }-- for (key, value) in self.command.get_envs() {- let Some(value) = value else {- continue;- };- lines.push(format!("{}={}", shell_quote(key), shell_quote(value)));- }-- lines.extend(shell_command_lines(- self.command.get_program(),- self.command.get_args(),- ));- lines.join(" \\\n")- }-}--/// Groups option-value pairs so the result reads like a hand-written command.-fn shell_command_lines<'a>(- program: &OsStr,- args: impl Iterator<Item = &'a OsStr>,-) -> Vec<String> {- let args = args.collect::<Vec<_>>();- let mut lines = vec![shell_quote(program)];- let mut index = 0;-- // Keep command names and leading positional arguments together, such as- // `cargo check` and `cargo test plan_q`.- while index < args.len() && !is_option(args[index]) {- lines[0].push(' ');- lines[0].push_str(&shell_quote(args[index]));- index += 1;- }-- while index < args.len() {- let arg = args[index];- let mut line = shell_quote(arg);- index += 1;-- if arg == "--" {- while index < args.len() {- line.push(' ');- line.push_str(&shell_quote(args[index]));- index += 1;+ Self::InvalidCommand(message) => {+ format!("invalid CI command definition: {message}")}- } else if !arg.to_string_lossy().contains('=')- && index < args.len()- && !is_option(args[index])- {- line.push(' ');- line.push_str(&shell_quote(args[index]));- index += 1;}-- lines.push(line);- }-- lines-}--fn is_option(value: &OsStr) -> bool {- value.to_string_lossy().starts_with('-')-}--/// Quotes one shell word when it contains characters that need protection.-fn shell_quote(value: &OsStr) -> String {- let value = value.to_string_lossy();- if !value.is_empty()- && value.bytes().all(|byte| {- byte.is_ascii_alphanumeric()- || matches!(- byte,- b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-'- )- })- {- value.into_owned()- } else {- format!("'{}'", value.replace('\'', "'\"'\"'"))}}-fn required_env(name: &str) -> Result<String> {- env::var(name)- .map_err(|_| format!("required environment variable `{name}` is not set"))-}-#[cfg(test)]mod tests {use super::*;+ use std::ffi::OsStr;const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";@@ -678,10 +581,10 @@ mod tests {#[test]fn explain_builds_the_same_command_as_execute() {let context = context();- let (execute_action, _, execute_command) = context+ let (execute_action, execute_command) = context.ci_step(&args(&["check", "datafusion", "default"])).unwrap();- let (explain_action, _, explain_command) = context+ let (explain_action, explain_command) = context.ci_step(&args(&["check", "datafusion", "default", "--explain"])).unwrap();@@ -829,7 +732,9 @@ cargo check \command: &["test", "workspace"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Requires executable `cargo-llvm-cov` (install it with `cargo install cargo-llvm-cov --locked`)cd /workspace && \+RUST_BACKTRACE=1 \cargo llvm-cov \--profile ci \--exclude datafusion-examples \@@ -849,7 +754,9 @@ cargo llvm-cov \command: &["test", "slt-coverage"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Requires executable `cargo-llvm-cov` (install it with `cargo install cargo-llvm-cov --locked`)cd /workspace && \+RUST_BACKTRACE=1 \cargo llvm-cov \--profile ci \--exclude datafusion-examples \@@ -867,7 +774,16 @@ cargo llvm-cov \command: &["test", "cli"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Run CLI tests with storage integration enabled. The AWS credentials+# shown below are fixed test values, not secrets.+cd /workspace && \+AWS_ACCESS_KEY_ID=TEST-DataFusionLogin \+AWS_ALLOW_HTTP=true \+AWS_ENDPOINT=http://127.0.0.1:9000 \+AWS_SECRET_ACCESS_KEY=TEST-DataFusionPassword \+RUST_BACKTRACE=1 \+TEST_STORAGE_INTEGRATION=1 \cargo test \--features backtrace \--profile ci \@@ -909,6 +825,12 @@ cargo test \command: &["test", "microbenchmarks"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Run each Criterion benchmark once without timing it, selecting only+# bench targets (not lib/bin unit tests). Keep debug assertions and+# overflow checks enabled while optimizing execution for larger datasets.+# The SQL benchmark harness needs separately generated datasets and+# is covered by the benchmark result verification job.+cd /workspace && \CLICKBENCH_DATA_PATH=/workspace/datafusion/core/tests/data/clickbench_hits_10.parquet \cargo test \@@ -928,6 +850,7 @@ cargo test \command: &["test", "benchmark-plan"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Requires path `/workspace/datafusion/sqllogictest/test_files/tpch/data` (generate the TPC-H fixture data before running this step)cd /workspace && \RUST_MIN_STACK=20971520 \TPCH_DATA=/workspace/datafusion/sqllogictest/test_files/tpch/data \@@ -943,6 +866,7 @@ cargo test plan_q \command: &["test", "benchmark-sqllogic"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Requires path `/workspace/datafusion/sqllogictest/test_files/tpch/data` (generate the TPC-H fixture data before running this step)cd /workspace && \INCLUDE_TPCH=true \RUST_MIN_STACK=20971520 \@@ -959,6 +883,8 @@ cargo test \command: &["test", "postgres"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Requires environment variable `POSTGRES_HOST`+# Requires environment variable `POSTGRES_PORT`cd /workspace/datafusion/sqllogictest && \PG_COMPAT=true \PG_URI=postgresql://postgres:[email protected]:15432/db_test \@@ -1000,6 +926,9 @@ cargo test \command: &["test", "extended"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Run tests behind the `extended_tests` feature. The `workspace`+# step runs the remaining tests.+cd /workspace && \RUST_BACKTRACE=1 \cargo test \@@ -1014,6 +943,9 @@ cargo test \command: &["test", "spill-pool-fuzz"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Run more random spill pool fuzzer scenarios than the `workspace`+# step. This takes about a minute.+cd /workspace && \DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS=1000 \RUST_BACKTRACE=1 \@@ -1029,6 +961,9 @@ cargo test \command: &["test", "hash-collisions"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Check that results stay correct when hash values collide.+# Run from the `datafusion` directory with `RUST_BACKTRACE=1`.+cd /workspace/datafusion && \RUST_BACKTRACE=1 \cargo test \@@ -1048,6 +983,9 @@ cargo test \command: &["test", "sqlite"],expected: |actual| {insta::assert_snapshot!(actual, @r"+# Run SQL logic tests including the SQLite suite. This step does not+# set `RUST_BACKTRACE` because backtraces make the suite expensive.+cd /workspace && \cargo test \--features backtrace,parquet_encryption \@@ -1066,10 +1004,10 @@ cargo test \.copied().chain(std::iter::once("--explain")).collect::<Vec<_>>();- let (action, _, actual) = context.ci_step(&args(&explain_args)).unwrap();+ let (action, actual) = context.ci_step(&args(&explain_args)).unwrap();assert_eq!(action, StepAction::Explain);- expected(&actual.full_command());+ expected(&actual.explanation());}}}diff --git a/xtask/src/command.rs b/xtask/src/command.rsnew file mode 100644index 0000000000..0bfa3f5bde--- /dev/null+++ b/xtask/src/command.rs@@ -0,0 +1,685 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++//! Complete, inspectable definitions of external commands.+//!+//! CI code only describes what a command needs: program, arguments, working+//! directory, non-secret environment, prerequisites, and failure context.+//! This module owns the mechanics of resolving that definition, validating it,+//! rendering it for `--explain`, and executing it. Keeping those paths together+//! ensures that local and CI invocations cannot drift from the command being shown.++use crate::Result;+use std::collections::{BTreeMap, BTreeSet};+use std::env;+use std::ffi::{OsStr, OsString};+use std::fmt::Write as _;+use std::path::{Path, PathBuf};+use std::process::{Command, Stdio};++pub(crate) type EnvReader = fn(&str) -> Result<String>;++/// Access to environment inputs required while resolving a command.+struct HostEnvironment {+ reader: EnvReader,+}++impl HostEnvironment {+ fn new(reader: EnvReader) -> Self {+ Self { reader }+ }++ fn required(&self, name: &str) -> Result<String> {+ (self.reader)(name)+ }+}++enum EnvTemplatePart {+ Literal(&'static str),+ Required(&'static str),+}++struct EnvTemplate {+ parts: Vec<EnvTemplatePart>,+}++impl EnvTemplate {+ fn parse(mut template: &'static str) -> Result<Self> {+ let original = template;+ let mut parts = Vec::new();+ while let Some(start) = template.find('{') {+ let (literal, after_literal) = template.split_at(start);+ if literal.contains('}') {+ return Err(format!("invalid environment template `{original}`"));+ }+ if !literal.is_empty() {+ parts.push(EnvTemplatePart::Literal(literal));+ }+ let after_open = &after_literal[1..];+ let Some(end) = after_open.find('}') else {+ return Err(format!("invalid environment template `{original}`"));+ };+ let (name, after_name) = after_open.split_at(end);+ if name.is_empty()+ || !name.bytes().all(|byte| {+ byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_'+ })+ {+ return Err(format!("invalid environment name in template `{original}`"));+ }+ parts.push(EnvTemplatePart::Required(name));+ template = &after_name[1..];+ }+ if template.contains('}') {+ return Err(format!("invalid environment template `{original}`"));+ }+ if !template.is_empty() {+ parts.push(EnvTemplatePart::Literal(template));+ }+ Ok(Self { parts })+ }++ fn resolve(&self, environment: &HostEnvironment) -> Result<OsString> {+ let mut value = String::new();+ for part in &self.parts {+ match part {+ EnvTemplatePart::Literal(literal) => value.push_str(literal),+ EnvTemplatePart::Required(name) => {+ value.push_str(&environment.required(name)?)+ }+ }+ }+ Ok(value.into())+ }++ fn display(&self, environment: &HostEnvironment) -> OsString {+ let mut value = String::new();+ for part in &self.parts {+ match part {+ EnvTemplatePart::Literal(literal) => value.push_str(literal),+ EnvTemplatePart::Required(name) => match environment.required(name) {+ Ok(resolved) => value.push_str(&resolved),+ Err(_) => {+ write!(value, "<{name}>").ok();+ }+ },+ }+ }+ value.into()+ }++ fn required_env(&self) -> impl Iterator<Item = &'static str> + '_ {+ self.parts.iter().filter_map(|part| match part {+ EnvTemplatePart::Required(name) => Some(*name),+ EnvTemplatePart::Literal(_) => None,+ })+ }+}++enum EnvValue {+ Literal(OsString),+ Template(EnvTemplate),+}++enum PendingEnvValue {+ Literal(OsString),+ Template(&'static str),+}++impl EnvValue {+ fn resolve(&self, environment: &HostEnvironment) -> Result<OsString> {+ match self {+ Self::Literal(value) => Ok(value.clone()),+ Self::Template(template) => template.resolve(environment),+ }+ }++ fn display(&self, environment: &HostEnvironment) -> OsString {+ match self {+ Self::Literal(value) => value.clone(),+ Self::Template(template) => template.display(environment),+ }+ }+}++/// A tool or path that must exist before a command can run.+///+/// Execution checks each prerequisite before starting the process.+/// If one is missing, it returns an error containing the `help` hint.+///+/// # Example+/// A coverage command needs `cargo-llvm-cov` in `PATH`.+/// Its `help` hint tells the user how to install it:+///+/// ```text+/// cargo install cargo-llvm-cov --locked+/// ```+///+/// `--explain` lists this prerequisite even when the tool is missing.+enum Requirement {+ Executable { name: OsString, help: &'static str },+ Path { path: PathBuf, help: &'static str },+}++impl Requirement {+ fn explain(&self) -> String {+ match self {+ Self::Executable { name, help } => {+ format!("executable `{}` ({help})", name.to_string_lossy())+ }+ Self::Path { path, help } => {+ format!("path `{}` ({help})", path.display())+ }+ }+ }++ fn validate(&self) -> Result<()> {+ match self {+ Self::Executable { name, help } if !executable_exists(name) => Err(format!(+ "required executable `{}` was not found in PATH; {help}",+ name.to_string_lossy()+ )),+ Self::Path { path, help } if !path.exists() => Err(format!(+ "required path `{}` does not exist; {help}",+ path.display()+ )),+ _ => Ok(()),+ }+ }+}++/// A complete recipe for an external command, shared by `--explain` and execution.+///+/// Example coverage command, run from the repository root:+///+/// ```text+/// RUST_BACKTRACE=1 cargo llvm-cov+/// ```+///+/// Prerequisite: `cargo-llvm-cov` must be available in `PATH`.+///+/// - `--explain` prints the documentation, prerequisites, and command.+/// - Execution checks the prerequisites, then launches the process.+/// Failures are reported with this step's context.+pub(crate) struct CommandSpec {+ /// Optional explanation, printed as shell comments before the command.+ doc: Option<&'static str>,+ /// Executable to launch, e.g. `cargo`.+ program: OsString,+ /// Arguments in order, e.g. `["llvm-cov"]`.+ args: Vec<OsString>,+ /// Working directory, e.g. the repository root.+ current_dir: PathBuf,+ /// Child process environment overrides, e.g. `RUST_BACKTRACE=1`.+ env: BTreeMap<OsString, EnvValue>,+ /// Prerequisites checked before launch, e.g. `cargo-llvm-cov` in `PATH`.+ requirements: Vec<Requirement>,+ /// Reads host variables used by templates in `env`.+ environment: HostEnvironment,+ /// Error prefix identifying the step, e.g. `CI test step failed`.+ failure_message: &'static str,+}++/// Collects a command definition before validating its required fields.+pub(crate) struct CommandSpecBuilder {+ doc: Option<&'static str>,+ program: Option<OsString>,+ args: Vec<OsString>,+ current_dir: Option<PathBuf>,+ env: BTreeMap<OsString, PendingEnvValue>,+ requirements: Vec<Requirement>,+ environment: HostEnvironment,+ failure_message: Option<&'static str>,+}++impl CommandSpec {+ pub(crate) fn builder(env_reader: EnvReader) -> CommandSpecBuilder {+ CommandSpecBuilder {+ doc: None,+ program: None,+ args: Vec::new(),+ current_dir: None,+ env: BTreeMap::new(),+ requirements: Vec::new(),+ environment: HostEnvironment::new(env_reader),+ failure_message: None,+ }+ }+}++impl CommandSpecBuilder {+ pub(crate) fn doc(&mut self, doc: &'static str) -> &mut Self {+ self.doc = Some(doc);+ self+ }++ pub(crate) fn program(&mut self, program: impl AsRef<OsStr>) -> &mut Self {+ self.program = Some(program.as_ref().to_owned());+ self+ }++ pub(crate) fn args<I, S>(&mut self, args: I) -> &mut Self+ where+ I: IntoIterator<Item = S>,+ S: AsRef<OsStr>,+ {+ self.args+ .extend(args.into_iter().map(|arg| arg.as_ref().to_owned()));+ self+ }++ pub(crate) fn env<K, V>(&mut self, key: K, value: V) -> &mut Self+ where+ K: AsRef<OsStr>,+ V: AsRef<OsStr>,+ {+ // Command definitions must not pass secrets here: values are displayed+ // by both `--explain` and the execution log.+ self.env.insert(+ key.as_ref().to_owned(),+ PendingEnvValue::Literal(value.as_ref().to_owned()),+ );+ self+ }++ pub(crate) fn env_template<K>(&mut self, key: K, template: &'static str) -> &mut Self+ where+ K: AsRef<OsStr>,+ {+ // Templates are inspectable command semantics, so resolved values are+ // subject to the same non-secret restriction as literal values.+ self.env+ .insert(key.as_ref().to_owned(), PendingEnvValue::Template(template));+ self+ }++ pub(crate) fn requires_executable(+ &mut self,+ name: impl AsRef<OsStr>,+ help: &'static str,+ ) -> &mut Self {+ self.requirements.push(Requirement::Executable {+ name: name.as_ref().to_owned(),+ help,+ });+ self+ }++ pub(crate) fn requires_path(+ &mut self,+ path: impl AsRef<Path>,+ help: &'static str,+ ) -> &mut Self {+ self.requirements.push(Requirement::Path {+ path: path.as_ref().to_path_buf(),+ help,+ });+ self+ }++ pub(crate) fn current_dir(&mut self, directory: impl AsRef<Path>) -> &mut Self {+ self.current_dir = Some(directory.as_ref().to_path_buf());+ self+ }++ pub(crate) fn failure_message(&mut self, message: &'static str) -> &mut Self {+ self.failure_message = Some(message);+ self+ }++ /// Finalizes the command only after its structural definition is complete.+ pub(crate) fn build(self) -> Result<CommandSpec> {+ let program = self+ .program+ .filter(|program| !program.is_empty())+ .ok_or_else(|| "command program is required".to_string())?;+ let current_dir = self+ .current_dir+ .filter(|directory| !directory.as_os_str().is_empty())+ .ok_or_else(|| "command working directory is required".to_string())?;+ let failure_message = self+ .failure_message+ .filter(|message| !message.trim().is_empty())+ .ok_or_else(|| "command failure message is required".to_string())?;++ let env =+ self.env+ .into_iter()+ .map(|(key, value)| {+ if key.is_empty() {+ return Err("command environment variable name cannot be empty"+ .to_string());+ }+ let value = match value {+ PendingEnvValue::Literal(value) => EnvValue::Literal(value),+ PendingEnvValue::Template(template) => {+ EnvValue::Template(EnvTemplate::parse(template)?)+ }+ };+ Ok((key, value))+ })+ .collect::<Result<_>>()?;++ Ok(CommandSpec {+ doc: self.doc,+ program,+ args: self.args,+ current_dir,+ env,+ requirements: self.requirements,+ environment: self.environment,+ failure_message,+ })+ }+}++impl CommandSpec {+ pub(crate) fn explain(&self) {+ println!("{}", self.explanation());+ }++ pub(crate) fn explanation(&self) -> String {+ let mut lines = Vec::new();+ if let Some(doc) = self.doc {+ lines.extend(doc.lines().map(|line| format!("# {line}")));+ lines.push(String::new());+ }+ lines.extend(+ self.requirement_descriptions()+ .into_iter()+ .map(|requirement| format!("# Requires {requirement}")),+ );+ lines.push(self.full_command());+ lines.join("\n")+ }++ pub(crate) fn execute(self) -> Result<()> {+ self.validate_requirements()+ .map_err(|error| format!("{}: {error}", self.failure_message))?;+ let env = self+ .resolve_env()+ .map_err(|error| format!("{}: {error}", self.failure_message))?;+ println!("+ {}", self.render_command(&env));++ let status = Command::new(&self.program)+ .args(&self.args)+ .current_dir(&self.current_dir)+ .envs(env)+ .stdin(Stdio::inherit())+ .stdout(Stdio::inherit())+ .stderr(Stdio::inherit())+ .status()+ .map_err(|error| format!("{}: {error}", self.failure_message))?;++ if status.success() {+ Ok(())+ } else {+ Err(format!("{}: {status}", self.failure_message))+ }+ }++ pub(crate) fn full_command(&self) -> String {+ let env = self+ .env+ .iter()+ .map(|(key, value)| (key.clone(), value.display(&self.environment)))+ .collect();+ self.render_command(&env)+ }++ fn requirement_descriptions(&self) -> Vec<String> {+ let mut descriptions = self+ .requirements+ .iter()+ .map(Requirement::explain)+ .collect::<Vec<_>>();+ let mut required_env = BTreeSet::new();+ for value in self.env.values() {+ if let EnvValue::Template(template) = value {+ required_env.extend(template.required_env());+ }+ }+ descriptions.extend(+ required_env+ .into_iter()+ .map(|name| format!("environment variable `{name}`")),+ );+ descriptions+ }++ fn validate_requirements(&self) -> Result<()> {+ for requirement in &self.requirements {+ requirement.validate()?;+ }+ Ok(())+ }++ fn resolve_env(&self) -> Result<BTreeMap<OsString, OsString>> {+ self.env+ .iter()+ .map(|(key, value)| {+ value+ .resolve(&self.environment)+ .map(|value| (key.clone(), value))+ })+ .collect()+ }++ fn render_command(&self, env: &BTreeMap<OsString, OsString>) -> String {+ let mut lines = vec![format!(+ "cd {} &&",+ shell_quote(self.current_dir.as_os_str())+ )];++ lines.extend(+ env.iter().map(|(key, value)| {+ format!("{}={}", shell_quote(key), shell_quote(value))+ }),+ );+ lines.extend(shell_command_lines(&self.program, self.args.iter()));+ lines.join(" \\\n")+ }+}++pub(crate) fn system_env(name: &str) -> Result<String> {+ env::var(name)+ .map_err(|_| format!("required environment variable `{name}` is not set"))+}++fn executable_exists(name: &OsStr) -> bool {+ let path = Path::new(name);+ if path.components().count() > 1 {+ return is_executable_file(path);+ }++ let Some(search_path) = env::var_os("PATH") else {+ return false;+ };++ env::split_paths(&search_path).any(|directory| executable_in(&directory, name))+}++fn executable_in(directory: &Path, name: &OsStr) -> bool {+ let candidate = directory.join(name);+ if is_executable_file(&candidate) {+ return true;+ }++ #[cfg(windows)]+ {+ if Path::new(name).extension().is_some() {+ return false;+ }+ let extensions = env::var_os("PATHEXT")+ .unwrap_or_else(|| OsString::from(".COM;.EXE;.BAT;.CMD"));+ return extensions.to_string_lossy().split(';').any(|extension| {+ is_executable_file(+ &candidate.with_extension(extension.trim_start_matches('.')),+ )+ });+ }++ #[cfg(not(windows))]+ false+}++fn is_executable_file(path: &Path) -> bool {+ let Ok(metadata) = path.metadata() else {+ return false;+ };+ if !metadata.is_file() {+ return false;+ }++ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ metadata.permissions().mode() & 0o111 != 0+ }++ #[cfg(not(unix))]+ true+}++/// Groups option-value pairs so the result reads like a hand-written command.+fn shell_command_lines<'a>(+ program: &OsStr,+ args: impl Iterator<Item = &'a OsString>,+) -> Vec<String> {+ let args = args.map(OsString::as_os_str).collect::<Vec<_>>();+ let mut lines = vec![shell_quote(program)];+ let mut index = 0;++ // Keep command names and leading positional arguments together, such as+ // `cargo check` and `cargo test plan_q`.+ while index < args.len() && !is_option(args[index]) {+ lines[0].push(' ');+ lines[0].push_str(&shell_quote(args[index]));+ index += 1;+ }++ while index < args.len() {+ let arg = args[index];+ let mut line = shell_quote(arg);+ index += 1;++ if arg == "--" {+ while index < args.len() {+ line.push(' ');+ line.push_str(&shell_quote(args[index]));+ index += 1;+ }+ } else if !arg.to_string_lossy().contains('=')+ && index < args.len()+ && !is_option(args[index])+ {+ line.push(' ');+ line.push_str(&shell_quote(args[index]));+ index += 1;+ }++ lines.push(line);+ }++ lines+}++fn is_option(value: &OsStr) -> bool {+ value.to_string_lossy().starts_with('-')+}++/// Quotes one shell word when it contains characters that need protection.+fn shell_quote(value: &OsStr) -> String {+ let value = value.to_string_lossy();+ if !value.is_empty()+ && value.bytes().all(|byte| {+ byte.is_ascii_alphanumeric()+ || matches!(+ byte,+ b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-'+ )+ })+ {+ value.into_owned()+ } else {+ format!("'{}'", value.replace('\'', "'\"'\"'"))+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn missing_env(name: &str) -> Result<String> {+ Err(format!("missing `{name}`"))+ }++ fn command_builder() -> CommandSpecBuilder {+ let mut builder = CommandSpec::builder(missing_env);+ builder+ .program("tool")+ .current_dir("/workspace")+ .failure_message("tool failed");+ builder+ }++ #[test]+ fn explain_lists_prerequisites_without_resolving_environment() {+ let mut builder = command_builder();+ builder+ .env_template("ENDPOINT", "{HOST}:{PORT}")+ .requires_path(+ "/workspace/input",+ "generate the input before running this command",+ );+ let command = builder.build().unwrap();++ insta::assert_snapshot!(command.explanation(), @r"+# Requires path `/workspace/input` (generate the input before running this command)+# Requires environment variable `HOST`+# Requires environment variable `PORT`+cd /workspace && \+ENDPOINT='<HOST>:<PORT>' \+tool+");+ }++ #[test]+ fn missing_prerequisite_is_reported_with_failure_context() {+ let mut builder = command_builder();+ builder.requires_executable(+ "datafusion-xtask-command-that-does-not-exist",+ "install the test tool",+ );+ let command = builder.build().unwrap();++ assert_eq!(+ command.execute().unwrap_err(),+ "tool failed: required executable `datafusion-xtask-command-that-does-not-exist` was not found in PATH; install the test tool"+ );+ }++ #[test]+ fn missing_environment_is_reported_with_failure_context() {+ let mut builder = command_builder();+ builder.env_template("ENDPOINT", "{HOST}:{PORT}");+ let command = builder.build().unwrap();++ assert_eq!(+ command.execute().unwrap_err(),+ "tool failed: missing `HOST`"+ );+ }++ #[test]+ fn build_validates_required_fields_and_templates() {+ let error = CommandSpec::builder(missing_env).build().err().unwrap();+ assert_eq!(error, "command program is required");++ let mut builder = command_builder();+ builder.env_template("ENDPOINT", "{lowercase}");+ let error = builder.build().err().unwrap();+ assert_eq!(error, "invalid environment name in template `{lowercase}`");+ }+}diff --git a/xtask/src/main.rs b/xtask/src/main.rsindex 3f1fc7b5c2..f69b392f69 100644--- a/xtask/src/main.rs+++ b/xtask/src/main.rs@@ -67,6 +67,7 @@//! ```mod ci_steps;+mod command;use std::env;use std::path::PathBuf;