Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows
feat: Handle concurrent dev-detach builds
Maximilian Roos9b91401d8599
diff --git a/.gitignore b/.gitignoreindex dab7be284..f829c04d5 100644--- a/.gitignore+++ b/.gitignore@@ -2,3 +2,4 @@# Lychee link checker cache.lycheecache+test-concurrent.shdiff --git a/Cargo.lock b/Cargo.lockindex bf0c30080..ab7c57710 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -812,6 +812,16 @@ version = "0.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"+[[package]]+name = "fs2"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"+dependencies = [+ "libc",+ "winapi",+]+[[package]]name = "futures-core"version = "0.3.31"@@ -2656,6 +2666,7 @@ dependencies = ["directories","env_logger","etcetera",+ "fs2","indexmap","insta","insta-cmd",diff --git a/Cargo.toml b/Cargo.tomlindex 409fd96d9..aed547cfa 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -81,6 +81,7 @@ toml = "0.9"criterion = "0.7"regex = "1.12"portable-pty = "0.9"+fs2 = "0.4"[[bench]]name = "completion"diff --git a/ISSUE.md b/ISSUE.mdnew file mode 100644index 000000000..72ed2050e--- /dev/null+++ b/ISSUE.md@@ -0,0 +1,667 @@+# Concurrent Test Failures: SIGKILL During Cargo Lock Contention++## Executive Summary++When running integration tests concurrently (8-10 parallel `cargo test` processes), approximately 29% of test runs fail with processes being killed by SIGKILL (signal 9). The failures occur during `cargo run -p dev-detach` invocations that are blocking on cargo's package cache lock. We implemented a workaround that eliminates the problem by avoiding concurrent cargo invocations, but **we don't understand what is sending SIGKILL or why**.++## Environment++- **OS**: macOS (Darwin 25.0.0)+- **Hardware**: Apple Silicon (aarch64-apple-darwin)+- **Cargo**: 1.90.0 (840b83a10 2025-07-30)+- **Rustc**: 1.90.0 (1159e78c4 2025-09-14)+- **Test Framework**: libtest (standard `cargo test`, NOT nextest)+- **Process Limits**:+ - `kern.maxproc: 12000`+ - `kern.maxprocperuid: 8000`+ - `ulimit -u: 8000`+ - `ulimit -n: unlimited` (file descriptors)++## Goals++1. **Primary**: Understand what is sending SIGKILL to cargo processes waiting on locks+2. **Secondary**: Determine if this is expected behavior or a bug+3. **Tertiary**: Understand if there's a better solution than avoiding concurrent cargo invocations++## System Architecture++### Test Infrastructure++Our integration tests need to execute shell scripts in isolated environments. We use a helper binary called `dev-detach` to isolate processes from the controlling terminal (prevents PTY-related hangs).++**Original implementation** (`tests/common/shell.rs`):++```rust+fn build_shell_command(repo: &TestRepo, shell: &str, script: &str) -> Command {+ // Get absolute path to workspace Cargo.toml (tests run from workspace root)+ let manifest = std::env::current_dir()+ .expect("Failed to get current directory")+ .join("Cargo.toml");++ let mut cmd = Command::new("cargo");+ cmd.args(["run", "--manifest-path"])+ .arg(&manifest)+ .args(["-p", "dev-detach", "--"]);++ // ... configure shell and arguments ...++ cmd+}+```++This function is called **hundreds of times** during test execution (each shell test invokes it multiple times).++### The dev-detach Binary++`tests/helpers/dev-detach/src/main.rs`:++```rust+#[cfg(unix)]+use nix::unistd::{execvp, setsid};+use std::ffi::CString;++fn main() {+ #[cfg(unix)]+ {+ // Become a new session leader with no controlling terminal+ if let Err(e) = setsid() {+ eprintln!("dev-detach: setsid failed: {}", e);+ process::exit(1);+ }++ // Get command and arguments from our argv+ let args: Vec<String> = env::args().skip(1).collect();+ let prog = CString::new(args[0].clone()).unwrap();+ let cargs: Vec<CString> = args+ .iter()+ .map(|a| CString::new(a.as_str()).unwrap())+ .collect();++ // Replace this process with the target command+ let _ = execvp(prog.as_c_str(), &cargs);+ eprintln!("dev-detach: execvp failed");+ process::exit(127);+ }+}+```++**Key point**: This is a simple wrapper that calls `setsid()` then `execvp()`. It doesn't interact with cargo locks.++## Observed Behavior++### Test Execution Pattern++When running `cargo test` with high concurrency:++```bash+# 8 parallel test processes, 3 iterations = 24 total runs+PARALLEL_RUNS=8+ITERATIONS=3++for iter in 1..3; do+ for i in 1..8; do+ cargo test integration_tests::e2e_shell -- --test-threads=8 &+ done+ wait+done+```++### Failure Rate++**Without fix**: 7/24 failures (29%)+**With fix**: 0/50 failures (0%) across 10 parallel × 5 iterations++### Actual Failure Examples++#### Example 1: SIGKILL During Lock Wait++```+thread 'integration_tests::e2e_shell_post_start::test_bash_post_create_blocks' panicked at tests/common/shell.rs:112:9:+Shell script failed (killed by signal 9 (SIGKILL)):+Command: cargo run --manifest-path <workspace>/Cargo.toml -p dev-detach -- bash [shell-flags...] -c <script>+stdout:+stderr: Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on build directory+ Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.52s+ Running `/Users/maximilian/workspace/worktrunk.concurrent-tests/target/debug/dev-detach bash --noprofile --norc -c '+ export PATH="/Users/maximilian/workspace/worktrunk.concurrent-tests/target/debug:$PATH"+```++**Critical observation**: The stderr shows:+1. Multiple "Blocking waiting for file lock" messages+2. Successfully completes: "Finished `dev` profile"+3. Starts executing: "Running `/path/to/dev-detach ...`"+4. Then gets SIGKILL++#### Example 2: SIGKILL Before Build Completion++```+thread 'integration_tests::e2e_shell::test_e2e_switch_and_remove_roundtrip::case_2' panicked at tests/common/shell.rs:112:9:+Shell script failed (killed by signal 9 (SIGKILL)):+Command: cargo run --manifest-path <workspace>/Cargo.toml -p dev-detach -- fish [shell-flags...] -c <script>+stdout:+stderr: Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on build directory+```++**Critical observation**: No "Finished" message - the process was killed **while waiting for the lock**.++### CPU and Resource Usage++During concurrent execution (without fix):++```bash+# From test run observations:+- CPU usage: 909% (9+ cores saturated)+- Active cargo processes: ~20-30 concurrent+- Shell processes: ~50-80 concurrent (including dev-detach, bash, fish, zsh)+```++## What We Tried++### Attempt 1: Using get_cargo_bin() with Once Guard++**Hypothesis**: `insta_cmd::get_cargo_bin()` might invoke cargo every time. Let's call it once and cache the result.++```rust+use std::sync::Once;+static BUILD_DEV_DETACH: Once = Once::new();++fn ensure_dev_detach_built() {+ BUILD_DEV_DETACH.call_once(|| {+ let status = Command::new("cargo")+ .args(["build", "-p", "dev-detach"])+ .status()+ .expect("Failed to run cargo build for dev-detach");++ if !status.success() {+ panic!("Failed to build dev-detach binary");+ }+ });+}++fn build_shell_command(repo: &TestRepo, shell: &str, script: &str) -> Command {+ ensure_dev_detach_built();+ let mut cmd = Command::new(get_cargo_bin("dev-detach"));+ // ...+}+```++**Result**: Failures reduced from 29% to 12.5% (7/24 → 3/24), but still failing.++**Analysis**: Each test process has its own `Once` instance, so we still had 8 concurrent builds at startup.++### Attempt 2: Using OnceLock to Cache Binary Path++**Hypothesis**: `get_cargo_bin()` itself might be invoking cargo. Let's cache the PathBuf.++```rust+use std::sync::OnceLock;+static DEV_DETACH_BIN: OnceLock<PathBuf> = OnceLock::new();++fn get_dev_detach_bin() -> &'static PathBuf {+ DEV_DETACH_BIN.get_or_init(|| {+ let status = Command::new("cargo")+ .args(["build", "-p", "dev-detach"])+ .status()+ .expect("Failed to run cargo build");++ if !status.success() {+ panic!("Failed to build dev-detach binary");+ }++ get_cargo_bin("dev-detach")+ })+}+```++**Result**: Still ~12-29% failure rate.++**Analysis**: Each test process still has independent static storage, so 8 concurrent builds still occur.++### Attempt 3: Direct Path Construction + Pre-build (THE FIX)++**Hypothesis**: Avoid ALL cargo invocations during test execution by:+1. Building dev-detach once before running tests+2. Using direct path construction instead of calling any cargo-related functions++```rust+/// Get path to dev-detach binary.+/// Assumes the binary has been built (e.g., by `cargo build -p dev-detach` before tests).+fn get_dev_detach_bin() -> PathBuf {+ // Construct path manually: target/debug/dev-detach+ let manifest_dir = std::env::current_dir().expect("Failed to get current directory");+ manifest_dir.join("target/debug/dev-detach")+}++fn build_shell_command(repo: &TestRepo, shell: &str, script: &str) -> Command {+ // Use pre-built dev-detach binary (no cargo invocation)+ let mut cmd = Command::new(get_dev_detach_bin());+ // ...+}+```++**Pre-build in test script**:+```bash+# Build dev-detach once before running tests+cargo build -p dev-detach --quiet++# Now run concurrent tests+for iter in 1..5; do+ for i in 1..10; do+ cargo test integration_tests::e2e_shell -- --test-threads=8 &+ done+ wait+done+```++**Result**: ✅ **100% success rate** - 50/50 runs passed (10 parallel × 5 iterations)++**Verification**:+```bash+$ cargo build -p dev-detach --quiet && cargo test --test integration -- --test-threads=12+...+test result: ok. 321 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 13.81s+```++## Key Findings++### 1. SIGKILL Timing is Inconsistent++From our failure logs, SIGKILL happens at two different points:++**Point A**: While waiting for lock (no "Finished" message)+```+ Blocking waiting for file lock on build directory+[Process killed here - no "Finished" message]+```++**Point B**: After successful build, during execution+```+ Blocking waiting for file lock on build directory+ Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.52s+ Running `/path/to/dev-detach bash ...`+[Process killed here - mid-execution]+```++### 2. No Evidence of Timeout Configuration++We checked:+- ❌ No `RUST_TEST_TIME_*` environment variables set+- ❌ No `.nextest.toml` configuration (and we're not using nextest anyway)+- ❌ No timeout flags in `cargo test --help`+- ❌ libtest has `--report-time` and `--ensure-time`, but these don't kill processes+- ❌ No cargo lock timeout documented in cargo source or documentation++### 3. System Limits are Not Exhausted++```bash+$ ulimit -a+-u: processes 8000+-n: file descriptors unlimited++$ sysctl kern.maxproc kern.maxprocperuid+kern.maxproc: 12000+kern.maxprocperuid: 8000++# During test execution:+$ ps aux | grep -E "bash|zsh|fish|dev-detach" | wc -l+53++$ ps aux | grep -c "[c]argo"+21+```++We're using ~74 processes out of 8000 limit - well within bounds.++### 4. No OOM Killer Evidence++```bash+$ dmesg 2>&1 | grep -i "killed\|oom"+[No output - dmesg not available on macOS by default]+```++macOS doesn't have a traditional OOM killer like Linux. Process termination under memory pressure is handled differently.++## Our Assumptions (UNPROVEN)++### Assumption 1: macOS is Sending SIGKILL++**Reasoning**:+- SIGKILL (signal 9) cannot be caught or blocked by the process+- It's typically sent by the kernel or system management+- User processes rarely send SIGKILL (they use SIGTERM first)++**Unproven**: We haven't traced which process/kernel subsystem sends the signal.++**How to verify**:+- Use `sudo dtruss -p <cargo_pid>` to trace system calls+- Monitor Console.app for system messages during failures+- Check if cargo itself has watchdog timers in source code++### Assumption 2: High Lock Contention Triggers System Intervention++**Reasoning**:+- Failures correlate with "Blocking waiting for file lock" messages+- Eliminating concurrent cargo invocations eliminates SIGKILL+- macOS might detect processes in prolonged wait states as "stuck"++**Unproven**: We don't know macOS's specific heuristics for killing processes.++**How to verify**:+- Research macOS process management policies+- Check if macOS has configurable thresholds for process wait states+- Look for Apple documentation on system resource management++### Assumption 3: Cargo Lock Implementation Doesn't Have Timeouts++**Reasoning**:+- No timeout configuration found in cargo documentation+- `cargo build --help` has no timeout flags+- Cargo source code (github.com/rust-lang/cargo) would show timeout logic++**Unproven**: We haven't audited cargo's source code for lock timeout mechanisms.++**How to verify**:+- Search cargo source for lock timeout implementations+- Check cargo issue tracker for lock-related SIGKILL reports+- Review cargo's `util/config` module for hidden timeout settings++### Assumption 4: The Problem is Lock-Specific, Not Load-Specific++**Reasoning**:+- The fix (pre-building) eliminates concurrent cargo invocations+- Pre-building doesn't reduce total system load (tests still run concurrently)+- SIGKILL stops happening despite similar CPU/memory usage++**Counter-evidence**:+- We observed 909% CPU during original runs (high load)+- With fix, CPU usage is still high during test execution+- Maybe the difference is **duration** of high load, not peak?++**Unproven**: We haven't compared system metrics (CPU, I/O, memory pressure) between failing and passing runs.++**How to verify**:+- Monitor system pressure metrics during both scenarios+- Use `powermetrics` or Activity Monitor to compare resource usage+- Check if macOS's App Nap or other power management features are involved++## Open Questions++### Question 1: What Process/Subsystem Sends SIGKILL?++**What we need to know**:+- Is it the kernel?+- Is it cargo itself (internal watchdog)?+- Is it macOS's process management daemon (e.g., `kernel_task`, `launchd`)?+- Is it a resource governor (memory pressure, thermal throttling)?++**Research directions**:+1. Search for "macOS SIGKILL blocked processes" or "macOS kill waiting processes"+2. Look for cargo source code related to subprocess management and timeouts+3. Check Apple documentation on process lifecycle management+4. Search for similar reports in Rust/cargo communities+5. Look for macOS system calls that can trigger SIGKILL (`kill(2)`, `killpg(2)`)++### Question 2: Is This Expected macOS Behavior?++**What we need to know**:+- Does macOS have documented policies for killing processes in wait states?+- Are there system-wide timeout thresholds we're exceeding?+- Is this a feature or a bug?++**Research directions**:+1. Apple Developer Documentation on process management+2. Darwin kernel source code (open source) for process scheduler+3. XNU (kernel) documentation on process states and lifecycle+4. Search for "macOS kill blocking process" or "macOS I/O wait timeout"++### Question 3: Why is the Timing Inconsistent?++Some processes get killed while waiting for lock, others after build completion. Why?++**Hypotheses**:+- Different cargo subprocesses have different wait times+- macOS uses cumulative metrics (total time in wait state)+- Race condition in detection logic+- Different code paths have different watchdogs++**Research directions**:+1. Map cargo's subprocess tree during `cargo run`+2. Understand which process gets SIGKILL (parent cargo or child rustc?)+3. Check if cargo uses process groups and how SIGKILL propagates++### Question 4: Are There Hidden Cargo Configuration Options?++**What we need to know**:+- Does cargo have undocumented timeout settings?+- Are there environment variables we're not aware of?+- Does cargo have different behavior in "test mode"?++**Research directions**:+1. Review `cargo --list` for all subcommands and options+2. Search cargo source for `CARGO_*` environment variables+3. Check if `cargo test` sets special environment for child processes+4. Look for `.cargo/config.toml` options related to timeouts or locks++### Question 5: Why Does Lock Contention Occur at All?++**What we need to know**:+- What is cargo locking when building dev-detach?+- Why does building a simple helper binary require multiple locks?+- Can we configure cargo to use different lock granularity?++**From our observations**:+```+Blocking waiting for file lock on package cache+Blocking waiting for file lock on build directory+```++This suggests at least two separate locks. Why multiple locks for a simple build?++**Research directions**:+1. Understand cargo's lock hierarchy (package cache → dependency resolution → build directory)+2. Check if `cargo build -p` (workspace member build) has different locking than `cargo build`+3. Research cargo's concurrent build architecture+4. Look for cargo RFC/documentation on lock implementation++### Question 6: Could This Be Related to `insta_cmd`?++**What we need to know**:+- What does `insta_cmd::get_cargo_bin()` actually do?+- Does it invoke cargo, or just locate the binary?+- Could it be interacting with cargo in unexpected ways?++**Research directions**:+1. Review insta_cmd source code for `get_cargo_bin()` implementation+2. Check if it caches binary locations+3. See if it has any lock files or state that could conflict++## Detailed Code Context++### Complete Test Execution Flow++1. **Test harness** runs `cargo test integration_tests::e2e_shell`+2. **Test function** (e.g., `test_bash_e2e_switch_existing_worktree`) calls:+ ```rust+ let output = execute_shell_script(&repo, "bash", &script);+ ```+3. **`execute_shell_script`** (in `tests/common/shell.rs:59`) calls:+ ```rust+ let mut cmd = build_shell_command(repo, shell, script);+ let output = cmd.current_dir(repo.root_path()).output().unwrap();+ ```+4. **`build_shell_command`** (original version) constructs:+ ```rust+ Command::new("cargo")+ .args(["run", "--manifest-path", &manifest, "-p", "dev-detach", "--"])+ .arg(shell)+ .args([shell_flags...])+ .arg("-c")+ .arg(script)+ ```+5. **Cargo** receives this and must:+ - Acquire package cache lock+ - Check if dev-detach needs rebuilding+ - Acquire build directory lock+ - Build dev-detach (if needed)+ - Execute dev-detach with remaining arguments++When 8-10 test processes do this concurrently, step 5 becomes a bottleneck.++### Signal Handling Code++We added signal number reporting to diagnose the issue:++```rust+/// Convert signal number to human-readable name+#[cfg(unix)]+fn signal_name(sig: i32) -> &'static str {+ match sig {+ 1 => "SIGHUP",+ 2 => "SIGINT",+ 3 => "SIGQUIT",+ 6 => "SIGABRT",+ 9 => "SIGKILL",+ 11 => "SIGSEGV",+ 13 => "SIGPIPE",+ 15 => "SIGTERM",+ _ => "UNKNOWN",+ }+}++// In execute_shell_script:+if !output.status.success() {+ let exit_info = match output.status.code() {+ Some(code) => format!("exit code {}", code),+ None => {+ #[cfg(unix)]+ {+ use std::os::unix::process::ExitStatusExt;+ match output.status.signal() {+ Some(sig) => format!("killed by signal {} ({})", sig, signal_name(sig)),+ None => "killed by signal (unknown)".to_string(),+ }+ }+ #[cfg(not(unix))]+ {+ "killed by signal".to_string()+ }+ }+ };+ panic!("Shell script failed ({}):\n...", exit_info);+}+```++This confirmed all failures were **signal 9 (SIGKILL)**, not other signals like SIGTERM or SIGPIPE.++## What Success Looks Like++The workaround successfully eliminates the problem:++```bash+$ cargo build -p dev-detach --quiet+$ cargo test --test integration -- --test-threads=12+...+test result: ok. 321 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 13.81s++# Stress test: 10 parallel × 5 iterations = 50 runs+$ for iter in 1..5; do+ for i in 1..10; do+ cargo test integration_tests::e2e_shell -- --test-threads=8 &+ done+ wait+done++Result: ✅ All 50 runs passed (0 failures)+```++## Priority Research Questions++**Highest Priority (Critical to Understanding)**:++1. **Who sends SIGKILL?** - Trace the actual source of the signal+2. **Is there a cargo lock timeout?** - Audit cargo source code or find documentation+3. **Does macOS have process wait state policies?** - Find Apple documentation on process lifecycle++**Medium Priority (Nice to Know)**:++4. Why is timing inconsistent (kill during wait vs. after build)?+5. What exactly triggers the decision to send SIGKILL?+6. Are there configuration options to adjust thresholds?++**Lower Priority (Optimization)**:++7. Could cargo's locking be more granular for workspace members?+8. Is there a way to pre-warm cargo's cache to avoid lock contention?+9. Should we file a bug report with cargo or macOS?++## Potential Alternative Solutions++If we understood the root cause, we might be able to:++1. **Configure a timeout** (if one exists) to be longer+2. **Adjust system policies** (if macOS has configurable thresholds)+3. **Use cargo's jobserver** to coordinate builds (if that's relevant)+4. **File a bug report** if this is unintended behavior+5. **Document the workaround** as best practice for test infrastructure++## Current Workaround (Implemented)++```rust+// In tests/common/shell.rs:+fn get_dev_detach_bin() -> PathBuf {+ std::env::current_dir()+ .expect("Failed to get current directory")+ .join("target/debug/dev-detach")+}++fn build_shell_command(repo: &TestRepo, shell: &str, script: &str) -> Command {+ let mut cmd = Command::new(get_dev_detach_bin());+ // ... rest of setup+}+```++**Requirements**:+- Must run `cargo build -p dev-detach` before running tests+- Binary must exist at `target/debug/dev-detach`++**Advantages**:+- ✅ 100% reliable (no failures in 50+ runs)+- ✅ Faster test execution (no cargo overhead)+- ✅ Simple implementation++**Disadvantages**:+- ❌ Requires explicit pre-build step+- ❌ Won't automatically rebuild if source changes+- ❌ Doesn't solve the underlying mystery++## Summary++We have a **reliable workaround** but **no understanding of root cause**. The key mystery is: **What sends SIGKILL to cargo processes waiting on locks, and why?**++The evidence strongly suggests it's not:+- Process/FD limits (we're well within limits)+- OOM killer (no memory pressure observed)+- Test framework timeouts (libtest doesn't kill processes)+- Cargo configuration (no timeout options found)++The most likely explanation is **macOS system-level process management** responding to prolonged wait states under high load, but we have no concrete evidence or documentation to support this.++**We need research into**:+1. macOS process lifecycle policies+2. Cargo's subprocess and lock management+3. Darwin/XNU kernel behavior for I/O-blocked processes+4. Similar reports in the Rust/cargo community++The goal is to determine if this is:+- **Expected behavior** we should work around (as we have)+- **A bug** in cargo's lock management+- **A system limitation** we can configure+- **Something else** we haven't considereddiff --git a/tests/common/shell.rs b/tests/common/shell.rsindex ab8cf4fb0..534ae1b72 100644--- a/tests/common/shell.rs+++ b/tests/common/shell.rs@@ -1,7 +1,62 @@use super::{TestRepo, wt_command};+use fs2::FileExt;use insta_cmd::get_cargo_bin;+use std::fs::OpenOptions;+use std::path::PathBuf;use std::process::Command;+/// Get path to dev-detach binary, building it once if needed.+/// Uses file locking to ensure only one concurrent build across test processes.+fn get_dev_detach_bin() -> PathBuf {+ let manifest_dir = std::env::current_dir().expect("Failed to get current directory");+ let bin_path = manifest_dir.join("target/debug/dev-detach");++ // Lock file ensures only one process builds at a time+ let lock_path = manifest_dir.join("target/.dev-detach.lock");+ let lock_file = OpenOptions::new()+ .create(true)+ .truncate(false)+ .write(true)+ .open(&lock_path)+ .expect("Failed to create lock file");++ // Acquire exclusive lock (blocks if another process is building)+ lock_file.lock_exclusive().expect("Failed to acquire lock");++ // While holding lock: build if binary doesn't exist+ if !bin_path.exists() {+ let status = Command::new("cargo")+ .args(["build", "-p", "dev-detach", "--quiet"])+ .status()+ .expect("Failed to execute cargo build");++ if !status.success() {+ panic!("Failed to build dev-detach binary");+ }+ }++ // Lock is automatically released when lock_file is dropped+ drop(lock_file);++ bin_path+}++/// Convert signal number to human-readable name+#[cfg(unix)]+fn signal_name(sig: i32) -> &'static str {+ match sig {+ 1 => "SIGHUP",+ 2 => "SIGINT",+ 3 => "SIGQUIT",+ 6 => "SIGABRT",+ 9 => "SIGKILL",+ 11 => "SIGSEGV",+ 13 => "SIGPIPE",+ 15 => "SIGTERM",+ _ => "UNKNOWN",+ }+}+/// Map shell display names to actual binaries.pub fn get_shell_binary(shell: &str) -> &str {match shell {@@ -13,18 +68,10 @@ pub fn get_shell_binary(shell: &str) -> &str {}/// Build a command to execute a shell script via dev-detach.-/// Uses `cargo run --manifest-path <workspace>/Cargo.toml -p dev-detach` so cargo can-/// find the workspace manifest while the shell executes in the test repo directory.+/// Uses pre-built binary to avoid cargo lock contention during concurrent test execution.fn build_shell_command(repo: &TestRepo, shell: &str, script: &str) -> Command {- // Get absolute path to workspace Cargo.toml (tests run from workspace root)- let manifest = std::env::current_dir()- .expect("Failed to get current directory")- .join("Cargo.toml");-- let mut cmd = Command::new("cargo");- cmd.args(["run", "--manifest-path"])- .arg(&manifest)- .args(["-p", "dev-detach", "--"]);+ // Use pre-built dev-detach binary (no cargo invocation)+ let mut cmd = Command::new(get_dev_detach_bin());repo.clean_cli_env(&mut cmd);// Prevent user shell config from leaking into tests@@ -78,10 +125,23 @@ pub fn execute_shell_script(repo: &TestRepo, shell: &str, script: &str) -> Strinif !output.status.success() {let exit_info = match output.status.code() {Some(code) => format!("exit code {}", code),- None => "killed by signal".to_string(),+ None => {+ #[cfg(unix)]+ {+ use std::os::unix::process::ExitStatusExt;+ match output.status.signal() {+ Some(sig) => format!("killed by signal {} ({})", sig, signal_name(sig)),+ None => "killed by signal (unknown)".to_string(),+ }+ }+ #[cfg(not(unix))]+ {+ "killed by signal".to_string()+ }+ }};panic!(- "Shell script failed ({}):\nCommand: cargo run --manifest-path <workspace>/Cargo.toml -p dev-detach -- {} [shell-flags...] -c <script>\nstdout: {}\nstderr: {}",+ "Shell script failed ({}):\nCommand: dev-detach {} [shell-flags...] -c <script>\nstdout: {}\nstderr: {}",exit_info,shell,String::from_utf8_lossy(&output.stdout),