Skip to content

Apache DataFusion SQL Query Engine

20.9 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Complete, inspectable definitions of external commands.
//!
//! CI code only describes what a command needs: program, arguments, working
//! directory, non-secret environment, prerequisites, and failure context.
//! This module owns the mechanics of resolving that definition, validating it,
//! rendering it for `--explain`, and executing it. Keeping those paths together
//! ensures that local and CI invocations cannot drift from the command being shown.

use crate::Result;
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::ffi::{OsStr, OsString};
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

pub(crate) type EnvReader = fn(&str) -> Result<String>;

/// Access to environment inputs required while resolving a command.
struct HostEnvironment {
    reader: EnvReader,
}

impl HostEnvironment {
    fn new(reader: EnvReader) -> Self {
        Self { reader }
    }

    fn required(&self, name: &str) -> Result<String> {
        (self.reader)(name)
    }
}

enum EnvTemplatePart {
    Literal(&'static str),
    Required(&'static str),
}

struct EnvTemplate {
    parts: Vec<EnvTemplatePart>,
}

impl EnvTemplate {
    fn parse(mut template: &'static str) -> Result<Self> {
        let original = template;
        let mut parts = Vec::new();
        while let Some(start) = template.find('{') {
            let (literal, after_literal) = template.split_at(start);
            if literal.contains('}') {
                return Err(format!("invalid environment template `{original}`"));
            }
            if !literal.is_empty() {
                parts.push(EnvTemplatePart::Literal(literal));
            }
            let after_open = &after_literal[1..];
            let Some(end) = after_open.find('}') else {
                return Err(format!("invalid environment template `{original}`"));
            };
            let (name, after_name) = after_open.split_at(end);
            if name.is_empty()
                || !name.bytes().all(|byte| {
                    byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_'
                })
            {
                return Err(format!("invalid environment name in template `{original}`"));
            }
            parts.push(EnvTemplatePart::Required(name));
            template = &after_name[1..];
        }
        if template.contains('}') {
            return Err(format!("invalid environment template `{original}`"));
        }
        if !template.is_empty() {
            parts.push(EnvTemplatePart::Literal(template));
        }
        Ok(Self { parts })
    }

    fn resolve(&self, environment: &HostEnvironment) -> Result<OsString> {
        let mut value = String::new();
        for part in &self.parts {
            match part {
                EnvTemplatePart::Literal(literal) => value.push_str(literal),
                EnvTemplatePart::Required(name) => {
                    value.push_str(&environment.required(name)?)
                }
            }
        }
        Ok(value.into())
    }

    fn display(&self, environment: &HostEnvironment) -> OsString {
        let mut value = String::new();
        for part in &self.parts {
            match part {
                EnvTemplatePart::Literal(literal) => value.push_str(literal),
                EnvTemplatePart::Required(name) => match environment.required(name) {
                    Ok(resolved) => value.push_str(&resolved),
                    Err(_) => {
                        write!(value, "<{name}>").ok();
                    }
                },
            }
        }
        value.into()
    }

    fn required_env(&self) -> impl Iterator<Item = &'static str> + '_ {
        self.parts.iter().filter_map(|part| match part {
            EnvTemplatePart::Required(name) => Some(*name),
            EnvTemplatePart::Literal(_) => None,
        })
    }
}

enum EnvValue {
    Literal(OsString),
    Template(EnvTemplate),
}

enum PendingEnvValue {
    Literal(OsString),
    Template(&'static str),
}

impl EnvValue {
    fn resolve(&self, environment: &HostEnvironment) -> Result<OsString> {
        match self {
            Self::Literal(value) => Ok(value.clone()),
            Self::Template(template) => template.resolve(environment),
        }
    }

    fn display(&self, environment: &HostEnvironment) -> OsString {
        match self {
            Self::Literal(value) => value.clone(),
            Self::Template(template) => template.display(environment),
        }
    }
}

/// A tool or path that must exist before a command can run.
///
/// Execution checks each prerequisite before starting the process.
/// If one is missing, it returns an error containing the `help` hint.
///
/// # Example
/// A coverage command needs `cargo-llvm-cov` in `PATH`.
/// Its `help` hint tells the user how to install it:
///
/// ```text
/// cargo install cargo-llvm-cov --locked
/// ```
///
/// `--explain` lists this prerequisite even when the tool is missing.
enum Requirement {
    Executable { name: OsString, help: &'static str },
    Path { path: PathBuf, help: &'static str },
}

impl Requirement {
    fn explain(&self) -> String {
        match self {
            Self::Executable { name, help } => {
                format!("executable `{}` ({help})", name.to_string_lossy())
            }
            Self::Path { path, help } => {
                format!("path `{}` ({help})", path.display())
            }
        }
    }

    fn validate(&self) -> Result<()> {
        match self {
            Self::Executable { name, help } if !executable_exists(name) => Err(format!(
                "required executable `{}` was not found in PATH; {help}",
                name.to_string_lossy()
            )),
            Self::Path { path, help } if !path.exists() => Err(format!(
                "required path `{}` does not exist; {help}",
                path.display()
            )),
            _ => Ok(()),
        }
    }
}

/// A complete recipe for an external command, shared by `--explain` and execution.
///
/// Example coverage command, run from the repository root:
///
/// ```text
/// RUST_BACKTRACE=1 cargo llvm-cov
/// ```
///
/// Prerequisite: `cargo-llvm-cov` must be available in `PATH`.
///
/// - `--explain` prints the documentation, prerequisites, and command.
/// - Execution checks the prerequisites, then launches the process.
///   Failures are reported with this step's context.
pub(crate) struct CommandSpec {
    /// Optional explanation, printed as shell comments before the command.
    doc: Option<&'static str>,
    /// Executable to launch, e.g. `cargo`.
    program: OsString,
    /// Arguments in order, e.g. `["llvm-cov"]`.
    args: Vec<OsString>,
    /// Working directory, e.g. the repository root.
    current_dir: PathBuf,
    /// Child process environment overrides, e.g. `RUST_BACKTRACE=1`.
    env: BTreeMap<OsString, EnvValue>,
    /// Prerequisites checked before launch, e.g. `cargo-llvm-cov` in `PATH`.
    requirements: Vec<Requirement>,
    /// Reads host variables used by templates in `env`.
    environment: HostEnvironment,
    /// Error prefix identifying the step, e.g. `CI test step failed`.
    failure_message: &'static str,
}

/// Collects a command definition before validating its required fields.
pub(crate) struct CommandSpecBuilder {
    doc: Option<&'static str>,
    program: Option<OsString>,
    args: Vec<OsString>,
    current_dir: Option<PathBuf>,
    env: BTreeMap<OsString, PendingEnvValue>,
    requirements: Vec<Requirement>,
    environment: HostEnvironment,
    failure_message: Option<&'static str>,
}

impl CommandSpec {
    pub(crate) fn builder(env_reader: EnvReader) -> CommandSpecBuilder {
        CommandSpecBuilder {
            doc: None,
            program: None,
            args: Vec::new(),
            current_dir: None,
            env: BTreeMap::new(),
            requirements: Vec::new(),
            environment: HostEnvironment::new(env_reader),
            failure_message: None,
        }
    }
}

impl CommandSpecBuilder {
    pub(crate) fn doc(&mut self, doc: &'static str) -> &mut Self {
        self.doc = Some(doc);
        self
    }

    pub(crate) fn program(&mut self, program: impl AsRef<OsStr>) -> &mut Self {
        self.program = Some(program.as_ref().to_owned());
        self
    }

    pub(crate) fn args<I, S>(&mut self, args: I) -> &mut Self
    where
        I: IntoIterator<Item = S>,
        S: AsRef<OsStr>,
    {
        self.args
            .extend(args.into_iter().map(|arg| arg.as_ref().to_owned()));
        self
    }

    pub(crate) fn env<K, V>(&mut self, key: K, value: V) -> &mut Self
    where
        K: AsRef<OsStr>,
        V: AsRef<OsStr>,
    {
        // Command definitions must not pass secrets here: values are displayed
        // by both `--explain` and the execution log.
        self.env.insert(
            key.as_ref().to_owned(),
            PendingEnvValue::Literal(value.as_ref().to_owned()),
        );
        self
    }

    pub(crate) fn env_template<K>(&mut self, key: K, template: &'static str) -> &mut Self
    where
        K: AsRef<OsStr>,
    {
        // Templates are inspectable command semantics, so resolved values are
        // subject to the same non-secret restriction as literal values.
        self.env
            .insert(key.as_ref().to_owned(), PendingEnvValue::Template(template));
        self
    }

    pub(crate) fn requires_executable(
        &mut self,
        name: impl AsRef<OsStr>,
        help: &'static str,
    ) -> &mut Self {
        self.requirements.push(Requirement::Executable {
            name: name.as_ref().to_owned(),
            help,
        });
        self
    }

    pub(crate) fn requires_path(
        &mut self,
        path: impl AsRef<Path>,
        help: &'static str,
    ) -> &mut Self {
        self.requirements.push(Requirement::Path {
            path: path.as_ref().to_path_buf(),
            help,
        });
        self
    }

    pub(crate) fn current_dir(&mut self, directory: impl AsRef<Path>) -> &mut Self {
        self.current_dir = Some(directory.as_ref().to_path_buf());
        self
    }

    pub(crate) fn failure_message(&mut self, message: &'static str) -> &mut Self {
        self.failure_message = Some(message);
        self
    }

    /// Finalizes the command only after its structural definition is complete.
    pub(crate) fn build(self) -> Result<CommandSpec> {
        let program = self
            .program
            .filter(|program| !program.is_empty())
            .ok_or_else(|| "command program is required".to_string())?;
        let current_dir = self
            .current_dir
            .filter(|directory| !directory.as_os_str().is_empty())
            .ok_or_else(|| "command working directory is required".to_string())?;
        let failure_message = self
            .failure_message
            .filter(|message| !message.trim().is_empty())
            .ok_or_else(|| "command failure message is required".to_string())?;

        let env =
            self.env
                .into_iter()
                .map(|(key, value)| {
                    if key.is_empty() {
                        return Err("command environment variable name cannot be empty"
                            .to_string());
                    }
                    let value = match value {
                        PendingEnvValue::Literal(value) => EnvValue::Literal(value),
                        PendingEnvValue::Template(template) => {
                            EnvValue::Template(EnvTemplate::parse(template)?)
                        }
                    };
                    Ok((key, value))
                })
                .collect::<Result<_>>()?;

        Ok(CommandSpec {
            doc: self.doc,
            program,
            args: self.args,
            current_dir,
            env,
            requirements: self.requirements,
            environment: self.environment,
            failure_message,
        })
    }
}

impl CommandSpec {
    pub(crate) fn explain(&self) {
        println!("{}", self.explanation());
    }

    pub(crate) fn explanation(&self) -> String {
        let mut lines = Vec::new();
        if let Some(doc) = self.doc {
            lines.extend(doc.lines().map(|line| format!("# {line}")));
            lines.push(String::new());
        }
        lines.extend(
            self.requirement_descriptions()
                .into_iter()
                .map(|requirement| format!("# Requires {requirement}")),
        );
        lines.push(self.full_command());
        lines.join("\n")
    }

    pub(crate) fn execute(self) -> Result<()> {
        self.validate_requirements()
            .map_err(|error| format!("{}: {error}", self.failure_message))?;
        let env = self
            .resolve_env()
            .map_err(|error| format!("{}: {error}", self.failure_message))?;
        println!("+ {}", self.render_command(&env));

        let status = Command::new(&self.program)
            .args(&self.args)
            .current_dir(&self.current_dir)
            .envs(env)
            .stdin(Stdio::inherit())
            .stdout(Stdio::inherit())
            .stderr(Stdio::inherit())
            .status()
            .map_err(|error| format!("{}: {error}", self.failure_message))?;

        if status.success() {
            Ok(())
        } else {
            Err(format!("{}: {status}", self.failure_message))
        }
    }

    pub(crate) fn full_command(&self) -> String {
        let env = self
            .env
            .iter()
            .map(|(key, value)| (key.clone(), value.display(&self.environment)))
            .collect();
        self.render_command(&env)
    }

    fn requirement_descriptions(&self) -> Vec<String> {
        let mut descriptions = self
            .requirements
            .iter()
            .map(Requirement::explain)
            .collect::<Vec<_>>();
        let mut required_env = BTreeSet::new();
        for value in self.env.values() {
            if let EnvValue::Template(template) = value {
                required_env.extend(template.required_env());
            }
        }
        descriptions.extend(
            required_env
                .into_iter()
                .map(|name| format!("environment variable `{name}`")),
        );
        descriptions
    }

    fn validate_requirements(&self) -> Result<()> {
        for requirement in &self.requirements {
            requirement.validate()?;
        }
        Ok(())
    }

    fn resolve_env(&self) -> Result<BTreeMap<OsString, OsString>> {
        self.env
            .iter()
            .map(|(key, value)| {
                value
                    .resolve(&self.environment)
                    .map(|value| (key.clone(), value))
            })
            .collect()
    }

    fn render_command(&self, env: &BTreeMap<OsString, OsString>) -> String {
        let mut lines = vec![format!(
            "cd {} &&",
            shell_quote(self.current_dir.as_os_str())
        )];

        lines.extend(
            env.iter().map(|(key, value)| {
                format!("{}={}", shell_quote(key), shell_quote(value))
            }),
        );
        lines.extend(shell_command_lines(&self.program, self.args.iter()));
        lines.join(" \\\n")
    }
}

pub(crate) fn system_env(name: &str) -> Result<String> {
    env::var(name)
        .map_err(|_| format!("required environment variable `{name}` is not set"))
}

fn executable_exists(name: &OsStr) -> bool {
    let path = Path::new(name);
    if path.components().count() > 1 {
        return is_executable_file(path);
    }

    let Some(search_path) = env::var_os("PATH") else {
        return false;
    };

    env::split_paths(&search_path).any(|directory| executable_in(&directory, name))
}

fn executable_in(directory: &Path, name: &OsStr) -> bool {
    let candidate = directory.join(name);
    if is_executable_file(&candidate) {
        return true;
    }

    #[cfg(windows)]
    {
        if Path::new(name).extension().is_some() {
            return false;
        }
        let extensions = env::var_os("PATHEXT")
            .unwrap_or_else(|| OsString::from(".COM;.EXE;.BAT;.CMD"));
        return extensions.to_string_lossy().split(';').any(|extension| {
            is_executable_file(
                &candidate.with_extension(extension.trim_start_matches('.')),
            )
        });
    }

    #[cfg(not(windows))]
    false
}

fn is_executable_file(path: &Path) -> bool {
    let Ok(metadata) = path.metadata() else {
        return false;
    };
    if !metadata.is_file() {
        return false;
    }

    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        metadata.permissions().mode() & 0o111 != 0
    }

    #[cfg(not(unix))]
    true
}

/// Groups option-value pairs so the result reads like a hand-written command.
fn shell_command_lines<'a>(
    program: &OsStr,
    args: impl Iterator<Item = &'a OsString>,
) -> Vec<String> {
    let args = args.map(OsString::as_os_str).collect::<Vec<_>>();
    let mut lines = vec![shell_quote(program)];
    let mut index = 0;

    // Keep command names and leading positional arguments together, such as
    // `cargo check` and `cargo test plan_q`.
    while index < args.len() && !is_option(args[index]) {
        lines[0].push(' ');
        lines[0].push_str(&shell_quote(args[index]));
        index += 1;
    }

    while index < args.len() {
        let arg = args[index];
        let mut line = shell_quote(arg);
        index += 1;

        if arg == "--" {
            while index < args.len() {
                line.push(' ');
                line.push_str(&shell_quote(args[index]));
                index += 1;
            }
        } else if !arg.to_string_lossy().contains('=')
            && index < args.len()
            && !is_option(args[index])
        {
            line.push(' ');
            line.push_str(&shell_quote(args[index]));
            index += 1;
        }

        lines.push(line);
    }

    lines
}

fn is_option(value: &OsStr) -> bool {
    value.to_string_lossy().starts_with('-')
}

/// Quotes one shell word when it contains characters that need protection.
fn shell_quote(value: &OsStr) -> String {
    let value = value.to_string_lossy();
    if !value.is_empty()
        && value.bytes().all(|byte| {
            byte.is_ascii_alphanumeric()
                || matches!(
                    byte,
                    b'_' | b'@' | b'%' | b'+' | b'=' | b':' | b',' | b'.' | b'/' | b'-'
                )
        })
    {
        value.into_owned()
    } else {
        format!("'{}'", value.replace('\'', "'\"'\"'"))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn missing_env(name: &str) -> Result<String> {
        Err(format!("missing `{name}`"))
    }

    fn command_builder() -> CommandSpecBuilder {
        let mut builder = CommandSpec::builder(missing_env);
        builder
            .program("tool")
            .current_dir("/workspace")
            .failure_message("tool failed");
        builder
    }

    #[test]
    fn explain_lists_prerequisites_without_resolving_environment() {
        let mut builder = command_builder();
        builder
            .env_template("ENDPOINT", "{HOST}:{PORT}")
            .requires_path(
                "/workspace/input",
                "generate the input before running this command",
            );
        let command = builder.build().unwrap();

        insta::assert_snapshot!(command.explanation(), @r"
# Requires path `/workspace/input` (generate the input before running this command)
# Requires environment variable `HOST`
# Requires environment variable `PORT`
cd /workspace && \
ENDPOINT='<HOST>:<PORT>' \
tool
");
    }

    #[test]
    fn missing_prerequisite_is_reported_with_failure_context() {
        let mut builder = command_builder();
        builder.requires_executable(
            "datafusion-xtask-command-that-does-not-exist",
            "install the test tool",
        );
        let command = builder.build().unwrap();

        assert_eq!(
            command.execute().unwrap_err(),
            "tool failed: required executable `datafusion-xtask-command-that-does-not-exist` was not found in PATH; install the test tool"
        );
    }

    #[test]
    fn missing_environment_is_reported_with_failure_context() {
        let mut builder = command_builder();
        builder.env_template("ENDPOINT", "{HOST}:{PORT}");
        let command = builder.build().unwrap();

        assert_eq!(
            command.execute().unwrap_err(),
            "tool failed: missing `HOST`"
        );
    }

    #[test]
    fn build_validates_required_fields_and_templates() {
        let error = CommandSpec::builder(missing_env).build().err().unwrap();
        assert_eq!(error, "command program is required");

        let mut builder = command_builder();
        builder.env_template("ENDPOINT", "{lowercase}");
        let error = builder.build().err().unwrap();
        assert_eq!(error, "invalid environment name in template `{lowercase}`");
    }
}