Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows
//! Concurrent execution of shell commands with prefixed-line output.
//!
//! Foreground concurrent groups (from `HookStep::Concurrent`) spawn every
//! command at once and combine their output into a single terminal stream,
//! each line prefixed with its command's colored label. Prefixed lines keep
//! full scrollback intact for debugging failures and work identically under
//! a TTY or pipe (CI logs).
//!
//! ## Execution model
//!
//! Children share the caller's foreground process group with closed stdin and
//! piped stdout/stderr. A single Unix poll loop owns all output pipes and renders
//! complete labeled lines. Direct child waits run independently of output EOF.
//!
//! Commands retain EOF semantics, including output from surviving descendants,
//! regardless of ordinary exit status. Cancellation drains queued bytes and
//! releases pipes even when an unowned descendant retains a write end.
//! Once direct waits finish, the signal listener releases its command lease.
//! Ctrl-C reaches the native job through the kernel. Each child handles it its
//! own way; wt's receipt of the key cancels the group once they exit, including
//! any remaining output wait. PID-targeted SIGTERM reaches owned direct children.
//!
//! Every admitted child runs to completion. Spawn failures occupy the same
//! per-command outcome slots as child exits; the pipeline owns identity and
//! failure policy. Cancellation alone stops admission and drains owned children.
use shared_child::SharedChild;
#[cfg(not(unix))]
use std::io::Read;
use std::io::Write;
use std::path::Path;
use std::process::Stdio;
use std::sync::Arc;
use std::sync::mpsc;
#[cfg(not(unix))]
use std::sync::mpsc::Sender;
use std::thread;
use std::time::Instant;
use anyhow::Context;
use worktrunk::command_log::log_command;
#[cfg(unix)]
use worktrunk::git::ErrorExt;
use worktrunk::git::WorktrunkError;
#[cfg(unix)]
use worktrunk::shell_exec::TerminalInterrupt;
use worktrunk::shell_exec::{
ShellConfig, apply_cd_directive_env, scrub_directive_env_vars, scrub_git_discovery_env_vars,
wait_foreground_child,
};
#[cfg(unix)]
use worktrunk::signal_forwarder::ForegroundSignals;
use worktrunk::styling::stderr;
use worktrunk::trace::CommandTrace;
use super::handlers::DirectivePassthrough;
/// One command in a concurrent group.
pub struct ConcurrentCommand<'a> {
/// Short label used as the line prefix (e.g., the command name).
pub label: &'a str,
/// Fully-expanded shell command string.
pub expanded: &'a str,
/// Child's working directory.
pub working_dir: &'a Path,
/// Optional label for `commands.jsonl` tracing.
pub log_label: Option<&'a str>,
/// Directive file env vars to pass through to the child. See
/// `DirectivePassthrough`: the CD directive file is the only one passed
/// through — every child scrubs the rest.
pub directives: &'a DirectivePassthrough,
/// Scrub inherited git-discovery vars (`GIT_DIR`/`GIT_WORK_TREE`/…) from the
/// child. `true` for hooks (they operate on the worktree wt targets), `false`
/// for aliases (they keep wt's inherited context). See issue #3373.
pub scrub_git_discovery: bool,
}
/// Run every command concurrently and return each per-child result in input
/// order. `Err(WorktrunkError::ChildProcessExited { .. })` signals a non-zero
/// exit; a spawn failure occupies that command's result slot. Outer errors
/// come from executor setup, output collection, or cancellation.
///
/// When the `WORKTRUNK_TEST_SERIAL_CONCURRENT` env var is set, commands run
/// sequentially in input order — same prefix-line output path, just one child
/// at a time. Tests use this to pin deterministic interleaving for snapshots.
pub fn run_concurrent_commands(
cmds: &[ConcurrentCommand<'_>],
) -> anyhow::Result<Vec<anyhow::Result<()>>> {
if cmds.is_empty() {
return Ok(Vec::new());
}
let prefix_width = cmds.iter().map(|c| c.label.len()).max().unwrap_or(0);
let shell = ShellConfig::get()?;
if std::env::var_os("WORKTRUNK_TEST_SERIAL_CONCURRENT").is_some() {
return run_serial_with_prefix(shell, cmds, prefix_width);
}
// Install before spawn so startup signals remain queued.
#[cfg(unix)]
let signals = ForegroundSignals::install(TerminalInterrupt::Cancels)?;
let mut children = Vec::with_capacity(cmds.len());
let mut outcomes = (0..cmds.len()).map(|_| Ok(())).collect::<Vec<_>>();
for (index, cmd) in cmds.iter().enumerate() {
match spawn_child(
shell,
index,
cmd,
#[cfg(unix)]
&signals,
) {
Ok(spawned) => children.push((index, spawned)),
Err(error) => {
#[cfg(unix)]
if error.interrupt_signal().is_some() {
// Drain admitted children through the normal cancellation owner.
drain_children(children, cmds, prefix_width, signals)?;
return Err(error);
}
outcomes[index] = Err(error);
}
}
}
for (index, outcome) in drain_children(
children,
cmds,
prefix_width,
#[cfg(unix)]
signals,
)? {
outcomes[index] = outcome;
}
Ok(outcomes)
}
/// Wait independently of output EOF. Cancellation releases remaining pipes
/// after all direct waits; ordinary exits preserve descendant output to EOF.
#[cfg(unix)]
fn drain_children(
mut children: Vec<(usize, SpawnedChild)>,
cmds: &[ConcurrentCommand<'_>],
prefix_width: usize,
signals: ForegroundSignals,
) -> anyhow::Result<Vec<(usize, anyhow::Result<()>)>> {
use nix::poll::{PollFd, PollFlags, PollTimeout, poll};
use std::io::ErrorKind;
use std::os::fd::AsFd;
// One wake pair serves the whole group, regardless of its number of pipes.
// Allocate it and prepare every pipe before starting any waiter.
let setup = (|| {
let (control, wake) = worktrunk::shell_exec::socket_pair()?;
control.set_nonblocking(true)?;
wake.set_nonblocking(true)?;
let mut pipes = Vec::with_capacity(children.len() * 2);
for (index, child) in &children {
let prefix = render_prefix(*index, cmds[*index].label, prefix_width);
if let Some(stdout) = child.child.take_stdout() {
pipes.push(OutputPipe::new(prefix.clone(), stdout)?);
}
if let Some(stderr) = child.child.take_stderr() {
pipes.push(OutputPipe::new(prefix, stderr)?);
}
}
Ok::<_, std::io::Error>((control, wake, pipes))
})();
let (control, wake, mut pipes) = match setup {
Ok(prepared) => prepared,
Err(error) => {
for (_, child) in &mut children {
let _ = child.child.kill();
let _ = child.child.wait();
child.trace.fail(&error);
}
return Err(error).context("Failed to prepare concurrent command output");
}
};
let processes: Vec<_> = children
.iter()
.map(|(_, child)| child.child.clone())
.collect();
let (tx, rx) = mpsc::channel();
let notify = WakeSender {
tx,
wake: Arc::new(wake),
};
let signal_notify = notify.clone();
let mut signal_thread = signals.forward_to_children(processes.clone(), move || {
signal_notify.wake();
});
let child_count = children.len();
let mut outcomes = Vec::with_capacity(child_count);
let mut output_error = None;
thread::scope(|scope| {
// Observe every exit independently: a later child's interruption must
// cancel the operation while an earlier child is still cleaning up.
for (index, child) in children {
let notify = notify.clone();
let cmd = &cmds[index];
scope.spawn(move || notify.send(index, collect_outcome(child, cmd)));
}
loop {
let result = (|| {
// Consume the wake before the queue: a message racing the queue
// read must leave a byte to wake the next poll.
let mut wake_bytes = [0; 256];
match nix::unistd::read(&control, &mut wake_bytes).map_err(std::io::Error::from) {
Ok(_) => {}
Err(error)
if matches!(
error.kind(),
ErrorKind::WouldBlock | ErrorKind::Interrupted
) => {}
Err(error) => return Err(error),
}
while let Ok(outcome) = rx.try_recv() {
outcomes.push(outcome);
}
if outcomes.len() == child_count {
// The execution lease ends here. A Ctrl-C received at any
// point stops waiting for descendants that hold the pipes.
signal_thread.finish_children();
}
if outcomes.len() == child_count && signal_thread.interrupt_signal().is_some() {
for pipe in &mut pipes {
pipe.finish_available()?;
}
pipes.clear();
}
if outcomes.len() == child_count && pipes.is_empty() {
return Ok(true);
}
let ready = {
let mut fds = pipes
.iter()
.map(|pipe| PollFd::new(pipe.stream.as_fd(), PollFlags::POLLIN))
.collect::<Vec<_>>();
fds.push(PollFd::new(control.as_fd(), PollFlags::POLLIN));
match poll(&mut fds, PollTimeout::NONE) {
Ok(_) => {}
Err(nix::errno::Errno::EINTR) => return Ok(false),
Err(error) => return Err(error.into()),
}
fds.into_iter()
.take(pipes.len())
.map(|fd| fd.revents().unwrap_or(PollFlags::empty()))
.collect::<Vec<_>>()
};
// Read one chunk per ready pipe so a continuous writer cannot
// starve completion messages or another child's output.
for (pipe, ready) in pipes.iter_mut().zip(ready) {
if !ready.is_empty() && !pipe.read_ready()? {
pipe.closed = true;
}
}
pipes.retain(|pipe| !pipe.closed);
Ok::<_, std::io::Error>(false)
})();
match result {
Ok(true) => break,
Ok(false) => {}
Err(error) => {
// Closing pipes releases writers; waiters still reap every
// owned direct child before we return the I/O failure.
pipes.clear();
for child in &processes {
let _ = child.kill();
}
output_error = Some(error);
break;
}
}
}
});
let cancelled = signal_thread
.stop()
.context("Failed to stop concurrent signal listener")?;
if let Some(error) = output_error {
return Err(error).context("Failed to read concurrent command output");
}
if let Some(signal) = cancelled {
return Err(WorktrunkError::Interrupted { signal, hint: None }.into());
}
outcomes.sort_unstable_by_key(|(index, _)| *index);
Ok(outcomes)
}
#[cfg(unix)]
#[derive(Clone)]
struct WakeSender {
tx: mpsc::Sender<(usize, anyhow::Result<()>)>,
wake: Arc<std::os::unix::net::UnixStream>,
}
#[cfg(unix)]
impl WakeSender {
fn send(&self, index: usize, outcome: anyhow::Result<()>) {
if self.tx.send((index, outcome)).is_ok() {
self.wake();
}
}
fn wake(&self) {
worktrunk::shell_exec::pipe::wake(&*self.wake);
}
}
#[cfg(unix)]
struct OutputPipe {
prefix: String,
stream: worktrunk::shell_exec::pipe::PipeReader,
pending: Vec<u8>,
closed: bool,
}
#[cfg(unix)]
impl OutputPipe {
fn new(prefix: String, stream: impl Into<std::os::fd::OwnedFd>) -> std::io::Result<Self> {
let stream = worktrunk::shell_exec::pipe::PipeReader::new(stream, None, None)?;
Ok(Self {
prefix,
stream,
pending: Vec::new(),
closed: false,
})
}
fn output(&mut self, bytes: &[u8]) {
for part in bytes.split_inclusive(|byte| *byte == b'\n') {
self.pending.extend_from_slice(part);
if part.ends_with(b"\n") {
self.flush_line();
}
}
}
fn flush_line(&mut self) {
let line = worktrunk::shell_exec::output_line(&self.pending);
writeln!(stderr().lock(), "{}{}", self.prefix, line).ok();
self.pending.clear();
}
fn read_ready(&mut self) -> std::io::Result<bool> {
use std::io::ErrorKind;
let mut bytes = [0; 8192];
match self.stream.read_ready(&mut bytes) {
Ok(0) => {
if !self.pending.is_empty() {
self.flush_line();
}
Ok(false)
}
Ok(count) => {
self.output(&bytes[..count]);
Ok(true)
}
Err(error)
if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::Interrupted) =>
{
Ok(true)
}
Err(error) => Err(error),
}
}
/// Snapshot queued bytes once: draining until WouldBlock can run forever
/// when an unowned descendant continuously writes without newlines.
fn finish_available(&mut self) -> std::io::Result<()> {
use std::io::ErrorKind;
self.stream.cancel()?;
let mut bytes = [0; 8192];
loop {
match self.stream.read_ready(&mut bytes) {
Ok(0) => break,
Ok(count) => self.output(&bytes[..count]),
Err(error) if error.kind() == ErrorKind::Interrupted => continue,
Err(error) => return Err(error),
}
}
if !self.pending.is_empty() {
self.flush_line();
}
Ok(())
}
}
/// Platforms without Unix poll retain blocking readers and direct-child waits.
#[cfg(not(unix))]
fn drain_children(
children: Vec<(usize, SpawnedChild)>,
cmds: &[ConcurrentCommand<'_>],
prefix_width: usize,
) -> anyhow::Result<Vec<(usize, anyhow::Result<()>)>> {
let (tx, rx) = mpsc::channel::<Event>();
let mut readers = Vec::new();
for (index, child) in &children {
let label = cmds[*index].label.to_string();
if let Some(stdout) = child.child.take_stdout() {
readers.push(spawn_reader(*index, label.clone(), stdout, tx.clone()));
}
if let Some(stderr) = child.child.take_stderr() {
readers.push(spawn_reader(*index, label, stderr, tx.clone()));
}
}
drop(tx);
let mut output_error = None;
let outcomes = thread::scope(|scope| {
let waiter = scope.spawn(move || {
children
.into_iter()
.map(|(index, child)| (index, collect_outcome(child, &cmds[index])))
.collect()
});
for event in rx {
match event {
Event::Line(labeled) => {
let prefix = render_prefix(labeled.index, &labeled.label, prefix_width);
writeln!(stderr().lock(), "{}{}", prefix, labeled.line).ok();
}
Event::ReaderClosed(Err(error)) => {
output_error.get_or_insert(error);
}
Event::ReaderClosed(Ok(())) => {}
}
}
waiter
.join()
.map_err(|_| anyhow::anyhow!("concurrent child waiter panicked"))
})?;
for reader in readers {
reader
.join()
.map_err(|_| anyhow::anyhow!("concurrent output reader panicked"))?;
}
if let Some(error) = output_error {
return Err(error).context("Failed to read concurrent command output");
}
Ok(outcomes)
}
fn run_serial_with_prefix(
shell: &ShellConfig,
cmds: &[ConcurrentCommand<'_>],
prefix_width: usize,
) -> anyhow::Result<Vec<anyhow::Result<()>>> {
let mut outcomes = Vec::with_capacity(cmds.len());
for (index, cmd) in cmds.iter().enumerate() {
#[cfg(unix)]
let signals = ForegroundSignals::install(TerminalInterrupt::Cancels)?;
match spawn_child(
shell,
index,
cmd,
#[cfg(unix)]
&signals,
) {
Ok(spawned) => {
for (_, outcome) in drain_children(
vec![(index, spawned)],
cmds,
prefix_width,
#[cfg(unix)]
signals,
)? {
outcomes.push(outcome);
}
}
Err(error) => {
#[cfg(unix)]
if error.interrupt_signal().is_some() {
return Err(error);
}
outcomes.push(Err(error));
}
}
}
Ok(outcomes)
}
struct SpawnedChild {
child: Arc<SharedChild>,
cmd_str: String,
log_label: Option<String>,
started_at: Instant,
/// `[wt-trace]` record for this child, captured at spawn time and resolved
/// in `collect_outcome`. Held across the output-draining window so the
/// recorded duration is the full spawn → wait span, not just the wait.
trace: CommandTrace,
}
fn spawn_child(
shell: &ShellConfig,
index: usize,
cmd: &ConcurrentCommand<'_>,
#[cfg(unix)] signals: &ForegroundSignals,
) -> anyhow::Result<SpawnedChild> {
// Siblings cannot share interactive stdin, but each retains native access
// to /dev/tty through the shell-established foreground group.
let mut command = shell.command(cmd.expanded);
command
.current_dir(cmd.working_dir)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
// User hooks discover their repo from the cwd wt sets, not an inherited
// GIT_DIR/GIT_WORK_TREE (issue #3373). Aliases keep the inherited context.
if cmd.scrub_git_discovery {
scrub_git_discovery_env_vars(&mut command);
}
// Scrub all directive env vars, then re-add the CD passthrough.
scrub_directive_env_vars(&mut command);
if let Some(path) = &cmd.directives.cd_file {
apply_cd_directive_env(&mut command, path);
}
tracing::debug!(
command = %cmd.expanded,
shell = %shell.name,
"$ {} (concurrent #{index}, shell: {})",
cmd.expanded,
shell.name
);
// Start the trace just before spawning so its duration brackets the real
// spawn → wait span (the child keeps running while we drain its output).
let mut trace = CommandTrace::new(None, cmd.expanded);
#[cfg(unix)]
let spawned = signals.spawn(&mut command);
#[cfg(not(unix))]
let spawned =
worktrunk::shell_exec::spawn_shared_child(&mut command).map_err(anyhow::Error::from);
let child = match spawned {
Ok(child) => Arc::new(child),
Err(e) => {
trace.fail(e.as_ref());
return Err(e);
}
};
Ok(SpawnedChild {
child,
cmd_str: cmd.expanded.to_string(),
log_label: cmd.log_label.map(str::to_string),
started_at: Instant::now(),
trace,
})
}
#[cfg(not(unix))]
fn send_line(index: usize, label: &str, bytes: &[u8], tx: &Sender<Event>) {
let _ = tx.send(Event::Line(LabeledLine {
index,
label: label.to_owned(),
line: worktrunk::shell_exec::output_line(bytes).into_owned(),
}));
}
#[cfg(not(unix))]
fn spawn_reader<R: Read + Send + 'static>(
index: usize,
label: String,
stream: R,
tx: Sender<Event>,
) -> thread::JoinHandle<()> {
use std::io::{BufRead, BufReader};
thread::spawn(move || {
let mut reader = BufReader::new(stream);
let mut pending = Vec::new();
let result = (|| {
while reader.read_until(b'\n', &mut pending)? != 0 {
send_line(index, &label, &pending, &tx);
pending.clear();
}
Ok(())
})();
let _ = tx.send(Event::ReaderClosed(result));
})
}
fn collect_outcome(spawned: SpawnedChild, cmd: &ConcurrentCommand<'_>) -> anyhow::Result<()> {
let SpawnedChild {
child,
cmd_str,
log_label,
started_at,
mut trace,
} = spawned;
let status = match wait_foreground_child(&child) {
Ok(status) => status,
Err(e) => {
let _ = child.kill();
let _ = child.wait();
trace.fail(&e);
return Err(e)
.with_context(|| format!("failed to wait for concurrent command '{}'", cmd.label));
}
};
trace.complete(status.success());
let duration = started_at.elapsed();
let outcome = WorktrunkError::from_child_status(&status, None);
if let Some(label) = log_label {
log_command(&label, &cmd_str, outcome.exit_code(), Some(duration));
}
if status.success() {
Ok(())
} else {
Err(outcome.into())
}
}
#[cfg(not(unix))]
enum Event {
Line(LabeledLine),
ReaderClosed(std::io::Result<()>),
}
#[cfg(not(unix))]
struct LabeledLine {
index: usize,
label: String,
line: String,
}
fn render_prefix(index: usize, label: &str, width: usize) -> String {
use anstyle::{AnsiColor, Color, Style};
let palette = [
AnsiColor::Cyan,
AnsiColor::Magenta,
AnsiColor::Yellow,
AnsiColor::Green,
AnsiColor::Blue,
AnsiColor::BrightCyan,
AnsiColor::BrightMagenta,
AnsiColor::BrightYellow,
];
let style = Style::new()
.fg_color(Some(Color::Ansi(palette[index % palette.len()])))
.bold();
format!("{style}{label:<width$}{style:#} │ ")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[cfg(unix)]
fn test_real_concurrent_process_failure_descriptions() {
use worktrunk::git::ErrorExt;
for (script, description, code) in [
("exit 7", "exit code 7", 7),
("kill -9 $$", "killed by signal 9", 137),
] {
let cwd = worktrunk::testing::tempdir().unwrap();
let directives = DirectivePassthrough::default();
let mut outcomes = run_concurrent_commands(&[ConcurrentCommand {
label: "check",
expanded: script,
working_dir: cwd.path(),
log_label: None,
directives: &directives,
scrub_git_discovery: false,
}])
.unwrap();
let error = outcomes.pop().unwrap().unwrap_err();
assert_eq!(error.to_string(), description);
assert_eq!(error.exit_code(), Some(code));
}
}
#[test]
fn test_concurrent_spawn_failure_retains_member_outcomes_and_io_source() {
let cwd = worktrunk::testing::tempdir().unwrap();
let missing = cwd.path().join("missing");
let directives = DirectivePassthrough::default();
let specs =
[cwd.path(), missing.as_path(), cwd.path()].map(|working_dir| ConcurrentCommand {
label: "check",
expanded: "printf completed > completed",
working_dir,
log_label: None,
directives: &directives,
scrub_git_discovery: false,
});
insta::allow_duplicates! {
for serial in [false, true] {
let outcomes = if serial {
run_serial_with_prefix(ShellConfig::get().unwrap(), &specs, "check".len())
} else {
run_concurrent_commands(&specs)
}
.unwrap();
assert_eq!(outcomes.len(), 3);
let mut outcomes = outcomes.into_iter();
outcomes.next().unwrap().unwrap();
let error = outcomes.next().unwrap().unwrap_err();
outcomes.next().unwrap().unwrap();
assert_eq!(
std::fs::read_to_string(cwd.path().join("completed")).unwrap(),
"completed"
);
#[cfg(unix)]
{
use worktrunk::git::ErrorExt;
let path = worktrunk::path::format_path_for_display(&missing);
let detail = error.display_message().replace(&path, "_MISSING_");
insta::assert_snapshot!(detail, @"Failed to execute sh @ _MISSING_: No such file or directory (os error 2)");
}
let preserved = error.downcast_ref::<std::io::Error>().unwrap();
let native = error.root_cause().downcast_ref::<std::io::Error>().unwrap();
assert!(native.raw_os_error().is_some());
assert_eq!(preserved.kind(), native.kind());
}
}
}
#[test]
fn test_empty_cmds_returns_empty() {
let outcomes = run_concurrent_commands(&[]).expect("no spawn should happen");
assert!(outcomes.is_empty());
}
}