Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows
perf(prune): parallelize removals on the scan lock's read side (#3631)
Maximilian Roose91b5fa88580
diff --git a/benches/CLAUDE.md b/benches/CLAUDE.mdindex 468867eaa..2581c11e8 100644--- a/benches/CLAUDE.md+++ b/benches/CLAUDE.md@@ -198,7 +198,7 @@ worth measuring at CI cadence.## Recording `wt remove` / `wt step prune` staging-The removal commands interleave serial per-target work with parallel scans and+The removal commands interleave per-target work with parallel scans anddetached background processes; a single e2e number hides which phase moved.Record them in two layers:@@ -217,7 +217,7 @@ because its ~15 GiB fixture must never build on a hosted CI runner):|---------|----------|------------------|| `prune_e2e/dry_run_probe_cold` | ~150 ms | full parallel scan, probes re-run (`.git/wt/cache/` cleared; git's own caches stay warm — the "first prune after fetching main" shape) || `prune_e2e/dry_run_warm` | ~60 ms | steady-state re-scan, probes hit sha_cache |-| `prune_e2e/live` | ~600 ms | probe-cold scan + serial removal of the 8 candidates (worktree candidates ~100 ms each — mostly the fsmonitor-daemon stop — branch-only ~25 ms, under the scan write lock, reusing scan-time plans) |+| `prune_e2e/live` | ~400 ms | probe-cold scan + parallel removal of the 8 candidates (worktree candidates ~145 ms each — mostly the fsmonitor-daemon stop — branch-only ~50 ms, concurrent on the removal worker pool under the scan lock's read side, reusing scan-time plans) || `prune_real_repo/dry_run_warm` | ~0.25–0.8 s | steady-state scan of 72 items (36 worktrees + 36 branches) at 331k-commit scale || `prune_real_repo/dry_run_probe_cold` | ~0.6–1 s | the same 72-item scan with probes re-running at real cost (statuses stay stat-warm) || `first_output/remove` | ~86 ms | single-target validation up to first output (`benches/time_to_first_output.rs`) |@@ -233,14 +233,19 @@ alone; a live one consumes the candidates). Expected one-shots on the`merge-base --is-ancestor` ~40 ms and `merge-tree --write-tree` ~130 ms (vs4–25 ms synthetic, where shallow history walks bottom out atsubprocess-spawn cost)-- **live ~2.9 s wall** — all 24 removals serialize under the scan write lock- inside the `prune-scan` window: each of the 12 worktree candidates- ~155–210 ms (fsmonitor-served status ~20 ms, daemon stop ~57 ms, metadata- prune ~11 ms, fresh ref snapshot ~40 ms, CAS delete ~7 ms), branch-only- candidates ~40–100 ms+- **live ~0.6 s wall** — all 24 removals run concurrently on the removal+ worker pool inside the `prune-scan` window (read side of the scan lock):+ the 12 worktree candidates ~240–290 ms each, branch-only ~100–135 ms.+ Contention inflates the per-span numbers — run serially, a worktree+ candidate's chain is ~155–210 ms (fsmonitor-served status ~20 ms, daemon+ stop ~57 ms, metadata prune ~11 ms, fresh ref snapshot ~40 ms, CAS delete+ ~7 ms) — but the wall collapses to the scan plus the slowest straggler.+ When candidate branches are packed (post-`gc`), the CAS deletes serialize+ on `packed-refs.lock`, putting a floor under the branch-deletion tailThis is the "prune takes many seconds" experience users report: worktree-count × stat-cold statuses bounds the scan, and removals extend it serially.+count × stat-cold statuses bounds the scan, and the removal tail extends it+by roughly one candidate's chain.The synthetic fixture can't show it — its statuses are milliseconds — soscale-sensitive changes need a one-shot on `prune-real` (or`wt-perf timeline -- -C <repo> step prune --dry-run` on a real repo) alongside@@ -262,10 +267,12 @@ silently timed.check region), one `prune-check:<ref>` per scanned item, and one`prune-remove:<label>` per removed candidate; `wt remove` emits`internal-sweep` around its end-of-command janitor. The `prune-remove` spans-sit *inside* the `prune-scan` window on the live path — each removal takes the-scan lock's write side, so a span covers the wait for in-flight checks to-drain *plus* the removal itself: read it as "how long this removal stalled the-run", not as pure removal work.+sit *inside* the `prune-scan` window on the live path and overlap each other —+removals execute concurrently on the worker pool, holding the scan lock's read+side. A span covers any wait for the lock plus the removal itself; the+exceptional candidates that take the write side (hook-bearing, `--foreground`,+metadata-pruning — `removal_needs_write` in `src/commands/step/prune.rs`) also+wait for every in-flight check and removal to drain first.```bashcargo run -p wt-perf -- setup prune-4-8 --path /tmp/prune-repodiff --git a/src/commands/hook_plan.rs b/src/commands/hook_plan.rsindex 5e6988960..65c8dd6fb 100644--- a/src/commands/hook_plan.rs+++ b/src/commands/hook_plan.rs@@ -316,6 +316,17 @@ impl ApprovedHookPlan {self.entries.is_empty()}+ /// True when the frozen selection carries any commands for `anchor` under+ /// one of `hook_types`. `wt step prune` routes such candidates onto its+ /// serialized removal path — a foreground hook body is an arbitrary+ /// command with streamed output, so it doesn't join the parallel removal+ /// fan-out.+ pub fn has_hooks_for(&self, anchor: &Path, hook_types: &[HookType]) -> bool {+ hook_types+ .iter()+ .any(|ht| !self.lookup(*ht, anchor).is_empty())+ }+/// The frozen selection for `(hook_type, anchor)`, by exact match.////// Every covered gate anchors at the identical path its executor passes:diff --git a/src/commands/step/prune.rs b/src/commands/step/prune.rsindex b943eeafa..a13d4348c 100644--- a/src/commands/step/prune.rs+++ b/src/commands/step/prune.rs@@ -1,9 +1,23 @@//! `wt step prune` — remove worktrees and branches integrated into the default branch.+//!+//! Live-path concurrency: candidate checks fan out on the rayon pool and+//! stream results to the main thread, which queues per-candidate jobs+//! (removals and skip lines) in scan-completion order onto a worker pool+//! sized like rayon's ([`RemovalJob`]). Checks and hook-free removals hold+//! the read side of [`RemovalContext::check_lock`] and run concurrently; the+//! exceptional removals serialize on the write side+//! ([`removal_needs_write`]). One FIFO queue carrying both removals and skip+//! lines means a single worker (`RAYON_NUM_THREADS=1`) reproduces the serial+//! total order the deterministic-output tests pin. The first failing removal+//! flips an abort flag that drains the remaining queue unexecuted; the+//! current worktree is removed last, after the fan-out, because its removal+//! cd's the shell to the primary.use std::collections::HashSet;use std::fs;use std::path::{Path, PathBuf};use std::sync::RwLock;+use std::sync::atomic::{AtomicBool, Ordering};use std::time::Duration;use anyhow::Context;@@ -91,6 +105,26 @@ impl Candidate {/// (its removal cd's the shell to the primary), with its scan-time plan.type DeferredCurrent = (Candidate, Option<RemoveResult>);+/// One unit of work for the removal workers, queued in scan-completion order.+///+/// Skip lines ride the same queue as removals so that with a single worker+/// (`RAYON_NUM_THREADS=1`) prune's per-candidate output keeps one total+/// order — the property the deterministic-output tests pin. With more+/// workers, whole lines from different candidates interleave freely: a skip+/// is a single line, and removal output may span lines but names its branch+/// on each (one `eprintln!` writes one line under one stderr lock).+enum RemovalJob {+ /// Execute a removal via [`try_remove`] and report the outcome.+ Remove {+ candidate: Candidate,+ /// Boxed to keep the variants near parity+ /// (`clippy::large_enum_variant` — `RemoveResult` is large).+ plan: Box<Option<RemoveResult>>,+ },+ /// Print an already-formatted skip line.+ PrintSkip(String),+}+#[derive(Clone, Copy)]enum CandidateKind {Current,@@ -192,24 +226,81 @@ fn prune_summary(candidates: &[Candidate]) -> String {parts.join(", ")}-/// Loop-invariant context for [`try_remove`]: every field is identical at all-/// three call sites in [`step_prune`] (only the `Candidate` varies). Built once-/// and passed by reference.+/// Loop-invariant context for [`try_remove`]: every field is identical at+/// both call sites in [`step_prune`] — the removal workers and the deferred+/// current worktree (only the `Candidate` varies). Built once and passed by+/// reference.struct RemovalContext<'a> {repo: &'a Repository,foreground: bool,hook_plan: &'a ApprovedHookPlan,worktrees: &'a [WorktreeInfo],snapshot: &'a RefSnapshot,- /// Serializes the parallel check readers against the removal writer.- /// `try_remove` takes the write guard while the background scan workers- /// hold a read guard around `integration_reason` and- /// `prepare_worktree_removal` — load-bearing for the Windows `.git/config`- /// race (rename-fallback rewrites it via lockfile+rename, readers fan out- /// child git processes that read it).+ /// Coordinates the parallel workers (scan checks and removals, both on+ /// the read side) against the few removals that need exclusivity (write+ /// side — see [`removal_needs_write`]).+ ///+ /// The lock exists for the Windows `.git/config` race: git rewrites+ /// config via lockfile + atomic rename, and a concurrent reader's plain+ /// `fopen` fails on the rename (#2801). Historically every removal held+ /// the write side because branch deletion was `git branch -D`, which+ /// rewrites `.git/config` (it drops the `[branch "<name>"]` section —+ /// even when none exists). The removal chain has since moved to the CAS+ /// `git update-ref -d`, and neither it nor `git worktree prune` /+ /// `git worktree remove` (the rename-failure fallback) touches+ /// `.git/config` — verified empirically by inode-watching `.git/config`+ /// across each command — so hook-free removals only spawn readers of+ /// shared repo state and can run concurrently. (`git branch -D` remains+ /// reachable only via `delete_branch_if_safe`'s force arm, which prune+ /// never uses, and its snapshot-miss arm, unreachable here because the+ /// chain captures the snapshot immediately before consulting it.)check_lock: &'a RwLock<()>,}+/// Which removals must hold the write side of [`RemovalContext::check_lock`]+/// instead of joining the parallel (read-side) fan-out:+///+/// - **Hook-bearing worktree candidates** — the `pre-remove` body runs+/// foreground here, and hook bodies are arbitrary commands (`git branch+/// -D`, `git config`, anything), so it keeps the exclusion every removal+/// had before removals parallelized; the write side also keeps the hook+/// stream and announce lines from interleaving with other candidates'+/// output. (`post-remove`/`post-switch` pipelines spawn detached and+/// always ran outside the lock.)+/// - **`--foreground` worktree candidates** — the foreground path runs a TTY+/// trash-cleanup spinner, and concurrent spinners would fight over the+/// cursor.+/// - **`StaleDetached` and plan-less `Prunable` candidates** — both call+/// `prune_worktrees()` fail-hard, so they don't race sibling prunes.+/// (The fail-soft prune inside `stage_worktree_removal` stays on the read+/// side: concurrent `git worktree prune` runs are idempotent and exit 0 —+/// verified empirically — and a swallowed failure only leaves stale+/// metadata for the next prune, never data loss. The scan's+/// `prepare_worktree_removal` can also prune under the read guard when a+/// Linked item's directory vanished mid-scan; `check_one` `.ok()`s that+/// prepare, so an error there just deselects the candidate.)+/// - **The current worktree** — deferred until after the fan-out drains and+/// run alone; write for uniformity with its post-switch hooks.+fn removal_needs_write(+ candidate: &Candidate,+ plan: Option<&RemoveResult>,+ ctx: &RemovalContext<'_>,+) -> bool {+ match candidate.kind {+ CandidateKind::Current | CandidateKind::StaleDetached => true,+ CandidateKind::Other | CandidateKind::BranchOnly => match plan {+ None => true,+ Some(RemoveResult::RemovedWorktree { worktree_path, .. }) => {+ ctx.foreground+ || ctx+ .hook_plan+ .has_hooks_for(worktree_path, &[HookType::PreRemove, HookType::PostRemove])+ }+ Some(RemoveResult::BranchOnly { .. }) => false,+ },+ }+}+/// Try to remove a candidate immediately. Returns Ok(true) if removed,/// Ok(false) if skipped (preparation error), Err on execution error.///@@ -224,11 +315,23 @@ fn try_remove(ctx: &RemovalContext<'_>,) -> anyhow::Result<bool> {let _span = Span::new(format!("prune-remove:{}", candidate.label));- // The guard protects `()` — there is no shared state to corrupt, so a- // poisoned lock is meaningless here. Recover the guard rather than- // `.expect()`-ing: a panic elsewhere should surface as itself, not as a- // cascade of secondary poison panics on every later removal/reader.- let _write = ctx.check_lock.write().unwrap_or_else(|e| e.into_inner());+ // Read side for the parallel default, write side for the exclusive cases+ // (see `removal_needs_write`). The guards protect `()` — there is no+ // shared state to corrupt, so a poisoned lock is meaningless here.+ // Recover the guard rather than `.expect()`-ing: a panic elsewhere should+ // surface as itself, not as a cascade of secondary poison panics on every+ // later removal/reader.+ let (_read, _write) = if removal_needs_write(candidate, plan.as_ref(), ctx) {+ (+ None,+ Some(ctx.check_lock.write().unwrap_or_else(|e| e.into_inner())),+ )+ } else {+ (+ Some(ctx.check_lock.read().unwrap_or_else(|e| e.into_inner())),+ None,+ )+ };if matches!(candidate.kind, CandidateKind::StaleDetached) {ctx.repo.prune_worktrees()?;@@ -254,8 +357,9 @@ fn try_remove(},};let mut announcer = HookAnnouncer::new(ctx.repo, true);- // `SynchronousForNonCurrent`: prune keeps the rename-failure fallback's- // `.git/config` rewrite serialized with its integration-check readers.+ // `SynchronousForNonCurrent`: a rename-failure fallback completes inline,+ // so the candidate counts as removed only once the worktree and branch+ // are actually gone — before the final summary prints.handle_remove_output(&plan,ctx.foreground,@@ -293,8 +397,8 @@ struct CheckOutcome {reason: Option<IntegrationReason>,/// Removal plan from `prepare_worktree_removal` — the same gate `wt/// remove` uses, computed here on the parallel scan so `try_remove`- /// doesn't re-derive it (a `git status` per worktree) under the write- /// lock. `None` means not removable (dirty, locked, primary — filtered+ /// doesn't re-derive it (a `git status` per worktree) at removal time.+ /// `None` means not removable (dirty, locked, primary — filtered/// silently, never reported as "younger than") — except for `Prunable`/// items, which are always removable but plan in `try_remove`: preparing/// them calls `prune_worktrees()`, a mutation that must stay serialized.@@ -309,8 +413,9 @@ struct CheckOutcome {}/// One check item's full parallel work: integration + removability + age.-/// Held under the check-lock read guard at the call site to serialize against-/// `try_remove` rewriting `.git/config` on the Windows rename-fallback path.+/// Held under the check-lock read guard at the call site so it never overlaps+/// a write-side removal (hook-bearing or metadata-pruning candidates — see+/// [`removal_needs_write`]).#[allow(clippy::too_many_arguments)]fn check_one(item: &CheckItem,@@ -795,7 +900,7 @@ pub fn step_prune(if dry_run {let check_lock = RwLock::new(());let scan_span = Span::new("prune-scan");- let mut dry_run_info: Vec<(Candidate, DryRunInfo)> = std::thread::scope(|s| {+ let dry_run_info: Vec<(Candidate, DryRunInfo)> = std::thread::scope(|s| {let (tx, rx) = chan::unbounded::<(usize, anyhow::Result<CheckOutcome>)>();// Pre-shadow with references so `move` on s.spawn moves only `tx`// (so it's dropped when the spawn ends and `rx` can terminate),@@ -864,7 +969,6 @@ pub fn step_prune(anyhow::Ok(info)})?;drop(scan_span);- dry_run_info.sort_by_key(|(c, _)| c.check_idx);return render_dry_run(dry_run_info, skipped_young, min_age, format);}@@ -958,11 +1062,17 @@ pub fn step_prune(snapshot: &snapshot,check_lock: &check_lock,};-- // Streaming live path: scans run in parallel and the main thread acts on- // each result as it arrives — print "Skipped (younger than X)" or call- // `try_remove` immediately for positives. The current worktree is the one- // exception: its removal cd's to the primary, so defer it until last.+ // Flipped by the first failing removal: the rest of the queue drains+ // without executing (matching the serial loop's abort-on-first-error),+ // and the error propagates after the workers finish.+ let abort = AtomicBool::new(false);++ // Streaming live path: scans run in parallel and the main thread queues a+ // job for each result as it arrives — a "Skipped ..." line or a removal.+ // Removals execute concurrently on the worker pool (read side of+ // `check_lock`; the exceptional candidates take the write side — see+ // `removal_needs_write`). The current worktree is the one exception to+ // the fan-out: its removal cd's to the primary, so defer it until last.let scan_span = Span::new("prune-scan");let (removed, deferred_current) = std::thread::scope(|s| -> anyhow::Result<(Vec<Candidate>, Option<DeferredCurrent>)> {@@ -970,7 +1080,7 @@ pub fn step_prune(// Pre-shadow with references so `move` on s.spawn moves only `tx`// (so it's dropped when the spawn ends and `rx` can terminate),// while the heavy state stays borrowed and remains usable by the- // main thread's removal calls.+ // main thread.let repo_ref = &repo;let snapshot_ref = &snapshot;let check_items_ref = &check_items;@@ -997,10 +1107,65 @@ pub fn step_prune(});});- let mut removed: Vec<Candidate> = Vec::new();+ // Removal workers, sized like the scan's rayon pool so+ // `RAYON_NUM_THREADS=1` serializes removals too (deterministic+ // output for tests). Results flow back on `done_rx`; the channel+ // closes once every worker has drained the job queue and exited,+ // so draining it below also waits out all in-flight printing.+ let (job_tx, job_rx) = chan::unbounded::<RemovalJob>();+ let (done_tx, done_rx) = chan::unbounded::<(Candidate, anyhow::Result<bool>)>();+ let abort_ref = &abort;+ let removal_ctx_ref = &removal_ctx;+ // Empty check_items → zero workers, correctly: no jobs can queue.+ let workers = rayon::current_num_threads().min(check_items.len());+ for _ in 0..workers {+ let job_rx = job_rx.clone();+ let done_tx = done_tx.clone();+ s.spawn(move || {+ for job in job_rx {+ match job {+ RemovalJob::Remove { candidate, plan } => {+ // Skip (not print, not remove) once a sibling+ // failed; queued skip lines below still print+ // — they were discovered before the failure.+ if abort_ref.load(Ordering::Relaxed) {+ continue;+ }+ let result = try_remove(&candidate, *plan, removal_ctx_ref)+ .with_context(|| candidate.removal_context());+ if result.is_err() {+ abort_ref.store(true, Ordering::Relaxed);+ }+ if done_tx.send((candidate, result)).is_err() {+ return;+ }+ }+ RemovalJob::PrintSkip(line) => {+ // Hold the read side so a skip line can't+ // land inside a write-side removal's+ // exclusive output window (spinner, hook+ // stream).+ let _read = removal_ctx_ref+ .check_lock+ .read()+ .unwrap_or_else(|e| e.into_inner());+ eprintln!("{line}");+ }+ }+ }+ });+ }+ drop(job_rx);+ drop(done_tx);+let mut deferred_current: Option<DeferredCurrent> = None;for (idx, outcome) in &rx {- let outcome = outcome.context("checking branch integration")?;+ // A check error fails the whole run — flip `abort` so the+ // workers drain their queue without executing more removals+ // (whose results this early return would silently drop).+ let outcome = outcome+ .context("checking branch integration")+ .inspect_err(|_| abort_ref.store(true, Ordering::Relaxed))?;let Some(_reason) = outcome.reason else {continue;};@@ -1013,12 +1178,11 @@ pub fn step_prune(if let Some(age) = outcome.age&& age < min_age_duration{- eprintln!(- "{}",- info_message(cformat!(- "Skipped <bold>{label}</> (younger than {min_age})"- ))- );+ let line = info_message(cformat!(+ "Skipped <bold>{label}</> (younger than {min_age})"+ ))+ .to_string();+ let _ = job_tx.send(RemovalJob::PrintSkip(line));skipped_young.push(label);continue;}@@ -1035,10 +1199,10 @@ pub fn step_prune(CandidateKind::BranchOnly | CandidateKind::StaleDetached => false,};if needs_approval {- eprintln!(- "{}",+ let line =info_message(cformat!("Skipped <bold>{label}</> (approval required)"))- );+ .to_string();+ let _ = job_tx.send(RemovalJob::PrintSkip(line));let differs = path.as_deref().is_some_and(|wt_path| {let candidate_bytes =std::fs::read(wt_path.join(".config").join("wt.toml")).ok();@@ -1057,18 +1221,47 @@ pub fn step_prune(};if matches!(candidate.kind, CandidateKind::Current) {deferred_current = Some((candidate, outcome.plan));- } else if try_remove(&candidate, outcome.plan, &removal_ctx)- .with_context(|| candidate.removal_context())?- {- removed.push(candidate);+ } else {+ let _ = job_tx.send(RemovalJob::Remove {+ candidate,+ plan: Box::new(outcome.plan),+ });}}+ drop(job_tx);++ let mut removed: Vec<Candidate> = Vec::new();+ let mut first_err: Option<anyhow::Error> = None;+ for (candidate, result) in &done_rx {+ match result {+ Ok(true) => removed.push(candidate),+ Ok(false) => {}+ Err(err) if first_err.is_none() => first_err = Some(err),+ // Concurrent failures (e.g. one Ctrl-C killing every+ // in-flight child) all carry the same story; report the+ // first and keep the rest out of the terminal.+ Err(err) => {+ tracing::debug!(+ error = %err,+ "additional removal failure for {}: {err:#}",+ candidate.label+ );+ }+ }+ }+ if let Some(err) = first_err {+ return Err(err);+ }Ok((removed, deferred_current))},)?;drop(scan_span);let mut removed = removed;+ // Deterministic order for `--format=json` regardless of which worker+ // finished first (the dry-run path sorts the same way); the deferred+ // current worktree stays last.+ removed.sort_by_key(|c| c.check_idx);// Remove deferred current worktree last (cd-to-primary happens here)if let Some((current, plan)) = deferred_current&& try_remove(¤t, plan, &removal_ctx).with_context(|| current.removal_context())?diff --git a/src/git/remove.rs b/src/git/remove.rsindex cd080cefb..c80fae8ef 100644--- a/src/git/remove.rs+++ b/src/git/remove.rs@@ -483,6 +483,13 @@ pub fn delete_branch_if_safe(/// failed → propagate the original error) by re-checking with `rev-parse/// --verify --quiet`, which has a structured exit code (0 = present, 1 =/// absent) rather than relying on locale-sensitive error-message text.+///+/// A `packed-refs.lock` that stays contended past git's ~1 s retry budget+/// (concurrent deletes of packed branches — `wt step prune`'s parallel+/// removals — on slow ref-store I/O) also fails the CAS with the ref+/// present, and reads as `RetainedRaced` even though the tip never moved.+/// Accepted: it is fail-closed, empirically unobserved at 24-way concurrency+/// on local disks, and the next prune collects the branch.fn cas_delete_branch_outcome(repo: &Repository,branch_name: &str,diff --git a/src/output/handlers.rs b/src/output/handlers.rsindex d069f5a0d..fc451f21d 100644--- a/src/output/handlers.rs+++ b/src/output/handlers.rs@@ -103,9 +103,9 @@ pub enum BackgroundFallbackMode {/// `wt remove`, `wt merge`, and the picker.Detached,/// Run the fallback removal and branch deletion synchronously for a- /// non-current worktree. `wt step prune` uses this so each iteration's- /// removal completes before the next candidate is considered and before- /// the final summary is printed.+ /// non-current worktree. `wt step prune` uses this so a candidate's+ /// removal is complete by the time it is reported removed — before the+ /// final summary prints, never outliving the prune process.SynchronousForNonCurrent,}@@ -148,8 +148,8 @@ fn spawn_background_removal(/// is no longer checked out in any worktree), and the background command is just `rm -rf`./// If rename fails (cross-filesystem, permissions, Windows file locking), either returns the/// legacy `git worktree remove` command with branch deletion deferred to the background, or-/// runs that fallback synchronously for non-current worktrees when the caller needs to keep-/// the fallback's `.git/config` rewrite serialized with other git readers.+/// runs that fallback synchronously for non-current worktrees when the caller needs the+/// removal complete before it reports success (`wt step prune`).////// The caller is responsible for spawning detached plans in the background.fn execute_instant_removal_or_fallback(@@ -1024,8 +1024,7 @@ pub fn execute_user_command(command: &str, display_path: Option<&Path>) -> anyho/// rename-into-trash fast path fails behaves: every caller but `wt step prune`/// passes [`BackgroundFallbackMode::Detached`] (spawn the legacy `git worktree/// remove`); prune passes [`BackgroundFallbackMode::SynchronousForNonCurrent`]-/// to keep the fallback's `.git/config` rewrite serialized with its parallel-/// integration-check readers.+/// so a candidate's removal is complete by the time prune reports it removed.pub fn handle_remove_output(result: &RemoveResult,foreground: bool,diff --git a/tests/integration_tests/step_prune.rs b/tests/integration_tests/step_prune.rsindex 963363902..6fcd6f605 100644--- a/tests/integration_tests/step_prune.rs+++ b/tests/integration_tests/step_prune.rs@@ -1338,24 +1338,26 @@ fn test_prune_runs_pre_remove_hook(mut repo: TestRepo) {assert!(!wt_path.exists(), "the merged worktree should be removed");}-/// Regression test for serialized `wt step prune` fallback removals.+/// Canary for `wt step prune` removals overlapping `.git/config` readers,+/// and regression test for synchronous fallback completion.///-/// B-prime (the `check_lock` RwLock in `src/commands/step/prune.rs`)-/// serializes the parallel `integration_reason` `.git/config` readers against-/// the config-rewriting branch deletion writer. That includes the-/// cross-filesystem / `.gitmodules` / Windows-file-lock fallback: prune runs-/// the non-current fallback removal and branch deletion synchronously under-/// the write guard instead of spawning a detached `git worktree remove && git-/// branch -d`. A branch with a `[branch "<name>"]` section makes deletion-/// rewrite `.git/config` via lockfile+rename.+/// Prune's hook-free removals — the rename-failure fallback included — run+/// concurrently with the parallel `integration_reason` readers on the read+/// side of `check_lock` (`src/commands/step/prune.rs`). That is safe because+/// the chain's branch deletion is a CAS `git update-ref -d`, which never+/// rewrites `.git/config`; a deletion mechanism that rewrites config via+/// lockfile+rename (as `git branch -D` does — the original Windows race,+/// #2801) would collide with those readers again. Each branch here gets a+/// `[branch "<name>"]` section so any such regression has a section to+/// rewrite, and the assertion watches for the Windows+/// `unable to access '.git/config'` failure.///-/// This forces that fallback for one non-current integrated worktree (by-/// pre-blocking its staged path, like+/// The fallback must also complete synchronously+/// (`SynchronousForNonCurrent`): this forces it for one non-current+/// integrated worktree (by pre-blocking its staged path, like/// `test_remove_background_fallback_on_rename_failure`) while several other-/// integrated worktrees keep the parallel integration-check fan-out running,-/// so the config-rewriting branch deletion overlaps live `.git/config`-/// readers. After the fix, removal and deletion run synchronously inside-/// `try_remove`, so `wt step prune` cannot exit before both finish — the+/// integrated worktrees keep the parallel fan-out running. `wt step prune`+/// cannot exit before the fallback removal and branch deletion finish — the/// regression assertion (`blocked` worktree and branch gone the instant prune/// returns) holds on every platform.///@@ -1374,11 +1376,11 @@ fn test_prune_fallback_config_race_canary(mut repo: TestRepo) {// Several integrated worktrees → a real parallel integration-check// fan-out. `add_worktree` puts each branch at `main` HEAD, so all are// same-commit integrated and will be pruned. Each branch gets a- // `[branch "<name>"]` section so its `git branch -d` rewrites- // `.git/config` — the racing write. (No remote needed: `git branch -d`- // removes the section regardless of whether `origin` resolves, and the- // same-commit local check yields "integrated" before upstream is- // consulted.)+ // `[branch "<name>"]` section so a config-rewriting deletion (a+ // regression from the CAS `update-ref -d` back toward `git branch -d`,+ // which removes the section via lockfile+rename) has a racing write to+ // make. (No remote needed: the same-commit local check yields+ // "integrated" before upstream is consulted.)let names: Vec<String> = (0..6).map(|i| format!("merged-canary-{i}")).collect();for name in &names {repo.add_worktree(name);@@ -1392,8 +1394,8 @@ fn test_prune_fallback_config_race_canary(mut repo: TestRepo) {// Force the fallback for one *non-current* worktree by pre-creating a// file at its computed staged path so `std::fs::rename(worktree → trash)`- // fails. Pick one in the middle so integration checks for later refs would- // still be in flight if fallback branch deletion escaped the write guard.+ // fails. Pick one in the middle so integration checks for later refs are+ // still in flight while the fallback runs.let blocked = names[3].clone();let blocked_wt_path = repo.worktree_path(&blocked).to_path_buf();let trash_dir = crate::common::resolve_git_common_dir(repo.root_path()).join("wt/trash");@@ -1508,6 +1510,275 @@ fn test_prune_skips_prunable_candidate_whose_preparation_fails(mut repo: TestRep);}+/// Hook-free removals execute concurrently (the read side of `check_lock`),+/// not one at a time.+///+/// Two integrated orphan branches are removed through a `git` shim whose+/// `update-ref -d` arms barrier on each other: each records that it started,+/// then waits for the *other* branch's deletion to start before proceeding.+/// The barrier only resolves if both removals are in flight at once — under+/// serialized removals the first deletion would wait out the shim's 15 s+/// timeout and drop a sentinel file, which the test asserts absent. Causally+/// driven, so it runs at barrier speed when concurrency works; the timeout is+/// only the safety net. Unix-only for the same `CreateProcess` shim reason as+/// the canary above.+#[cfg(unix)]+#[rstest]+fn test_prune_removals_run_concurrently(repo: TestRepo) {+ repo.commit("initial");++ // Orphan branches at main HEAD: same-commit integrated, no worktree, so+ // each becomes a hook-free BranchOnly candidate on the parallel path.+ repo.create_branch("para-a");+ repo.create_branch("para-b");++ let mut cmd = repo.wt_command();+ // The removal pool is sized from the rayon thread count; pin it to two so+ // the barrier can resolve even on a single-core runner (the workers block+ // in subprocess waits, so two threads don't need two CPUs).+ cmd.env("RAYON_NUM_THREADS", "2");+ let git_wrapper_dir = repo.home_path().join("git-wrapper");+ std::fs::create_dir_all(&git_wrapper_dir).unwrap();+ write_barrier_git_wrapper(&git_wrapper_dir, &which::which("git").unwrap());+ prepend_path(&mut cmd, &git_wrapper_dir);+ let barrier_dir = repo.home_path().join("barrier");+ std::fs::create_dir_all(&barrier_dir).unwrap();+ cmd.env("WT_TEST_BARRIER_DIR", &barrier_dir);++ let output = cmd+ .args(["step", "prune", "--yes", "--min-age=0s"])+ .output()+ .unwrap();+ let stderr = String::from_utf8_lossy(&output.stderr);++ assert!(output.status.success(), "prune should succeed:\n{stderr}");+ for name in ["para-a", "para-b"] {+ assert!(+ barrier_dir.join(format!("started-{name}")).exists(),+ "the shim never fired for {name} — its CAS delete was not exercised"+ );+ }+ let timeouts: Vec<String> = std::fs::read_dir(&barrier_dir)+ .unwrap()+ .filter_map(|e| e.ok())+ .map(|e| e.file_name().to_string_lossy().into_owned())+ .filter(|n| n.starts_with("timeout-"))+ .collect();+ assert!(+ timeouts.is_empty(),+ "a deletion waited out the barrier — removals ran serially: {timeouts:?}"+ );+ let branches = repo.git_output(&["branch", "--format=%(refname:short)"]);+ for name in ["para-a", "para-b"] {+ assert!(+ !branches.lines().any(|branch| branch == name),+ "{name} should have been deleted; branches:\n{branches}"+ );+ }+}++/// The first failing removal aborts the rest of the queue.+///+/// Three integrated orphan branches scan in sorted order (`abort-a` first)+/// with a single worker (`RAYON_NUM_THREADS=1`), so the jobs run FIFO. A+/// `git` shim makes `abort-a`'s CAS delete fail *after* deleting the ref+/// (so `cas_delete_branch_outcome`'s re-check finds it gone and propagates+/// the error rather than reporting `RetainedRaced`). The failure must flip+/// the abort flag: the queued `abort-b`/`abort-c` removals never execute,+/// their branches survive, no summary prints, and prune exits non-zero —+/// the serial loop's abort-on-first-error, preserved across the fan-out.+/// Unix-only for the same `CreateProcess` shim reason as the canary above.+#[cfg(unix)]+#[rstest]+fn test_prune_removal_failure_aborts_remaining_queue(repo: TestRepo) {+ repo.commit("initial");++ repo.create_branch("abort-a");+ repo.create_branch("abort-b");+ repo.create_branch("abort-c");++ let mut cmd = repo.wt_command();+ cmd.env("RAYON_NUM_THREADS", "1"); // FIFO: abort-a dispatches first+ let git_wrapper_dir = repo.home_path().join("git-wrapper");+ std::fs::create_dir_all(&git_wrapper_dir).unwrap();+ write_failing_branch_delete_wrapper(&git_wrapper_dir, &which::which("git").unwrap());+ prepend_path(&mut cmd, &git_wrapper_dir);+ cmd.env("WT_TEST_FAIL_DELETE_BRANCH", "abort-a");++ let output = cmd+ .args(["step", "prune", "--yes", "--min-age=0s"])+ .output()+ .unwrap();+ let stderr = String::from_utf8_lossy(&output.stderr);++ assert!(+ !output.status.success(),+ "a failed removal must fail the run:\n{stderr}"+ );+ assert!(+ stderr.contains("removing branch abort-a"),+ "the error should carry the failing candidate's context:\n{stderr}"+ );+ assert!(+ !stderr.contains("Pruned "),+ "no summary after an aborted run:\n{stderr}"+ );+ let branches = repo.git_output(&["branch", "--format=%(refname:short)"]);+ assert!(+ !branches.lines().any(|b| b == "abort-a"),+ "the shim deleted abort-a's ref; branches:\n{branches}"+ );+ for name in ["abort-b", "abort-c"] {+ assert!(+ branches.lines().any(|b| b == name),+ "{name} was queued behind the failure and must survive; branches:\n{branches}"+ );+ }+}++/// Concurrent failures: the first error is reported, the rest stay quiet.+///+/// Two failing removals rendezvous inside the shim (the barrier from+/// `test_prune_removals_run_concurrently`) so both are in flight before+/// either error lands — exercising the drain's duplicate-failure arm, which+/// logs at debug rather than printing a second error.+#[cfg(unix)]+#[rstest]+fn test_prune_concurrent_removal_failures_report_first(repo: TestRepo) {+ repo.commit("initial");++ repo.create_branch("dupe-a");+ repo.create_branch("dupe-b");++ let mut cmd = repo.wt_command();+ cmd.env("RAYON_NUM_THREADS", "2"); // both jobs in flight at once+ let git_wrapper_dir = repo.home_path().join("git-wrapper");+ std::fs::create_dir_all(&git_wrapper_dir).unwrap();+ write_barrier_failing_delete_wrapper(&git_wrapper_dir, &which::which("git").unwrap());+ prepend_path(&mut cmd, &git_wrapper_dir);+ let barrier_dir = repo.home_path().join("barrier");+ std::fs::create_dir_all(&barrier_dir).unwrap();+ cmd.env("WT_TEST_BARRIER_DIR", &barrier_dir);++ let output = cmd+ .args(["step", "prune", "--yes", "--min-age=0s"])+ .output()+ .unwrap();+ let stderr = String::from_utf8_lossy(&output.stderr);++ assert!(+ !output.status.success(),+ "failed removals must fail the run:\n{stderr}"+ );+ for name in ["dupe-a", "dupe-b"] {+ assert!(+ barrier_dir.join(format!("started-{name}")).exists(),+ "the shim never fired for {name}"+ );+ }+ // Exactly one candidate's failure reaches the terminal.+ assert_eq!(+ stderr.matches("removing branch dupe-").count(),+ 1,+ "exactly one of the concurrent failures should be reported:\n{stderr}"+ );+}++/// A `git` shim that deletes `refs/heads/$WT_TEST_FAIL_DELETE_BRANCH` for+/// real and then reports failure when prune's CAS delete targets it —+/// making `cas_delete_branch_outcome` propagate an error (ref gone on+/// re-check) instead of `RetainedRaced` (ref still present).+#[cfg(unix)]+fn write_failing_branch_delete_wrapper(dir: &std::path::Path, real_git: &std::path::Path) {+ use std::os::unix::fs::PermissionsExt;++ let real_git = shell_escape::unix::escape(real_git.to_string_lossy());+ let script = format!(+ r#"#!/bin/sh+if [ "$1" = "update-ref" ] && [ "$2" = "-d" ] && [ "$3" = "refs/heads/$WT_TEST_FAIL_DELETE_BRANCH" ]; then+ {real_git} update-ref -d "$3" || true+ exit 1+fi+exec {real_git} "$@"+"#+ );+ let path = dir.join("git");+ std::fs::write(&path, script).unwrap();+ let mut permissions = std::fs::metadata(&path).unwrap().permissions();+ permissions.set_mode(0o755);+ std::fs::set_permissions(&path, permissions).unwrap();+}++/// The barrier shim (see `write_barrier_git_wrapper`) with a failing tail:+/// both `dupe-{a,b}` deletions rendezvous, delete their ref for real, then+/// report failure — two concurrent removal errors.+#[cfg(unix)]+fn write_barrier_failing_delete_wrapper(dir: &std::path::Path, real_git: &std::path::Path) {+ use std::os::unix::fs::PermissionsExt;++ let real_git = shell_escape::unix::escape(real_git.to_string_lossy());+ let script = format!(+ r#"#!/bin/sh+case "$1 $2 $3" in+ "update-ref -d refs/heads/dupe-a") own=dupe-a; other=dupe-b ;;+ "update-ref -d refs/heads/dupe-b") own=dupe-b; other=dupe-a ;;+ *) exec {real_git} "$@" ;;+esac+: > "$WT_TEST_BARRIER_DIR/started-$own"+i=0+while [ ! -e "$WT_TEST_BARRIER_DIR/started-$other" ]; do+ i=$((i+1))+ if [ "$i" -gt 300 ]; then+ break+ fi+ sleep 0.05+done+{real_git} update-ref -d "$3" || true+exit 1+"#+ );+ let path = dir.join("git");+ std::fs::write(&path, script).unwrap();+ let mut permissions = std::fs::metadata(&path).unwrap().permissions();+ permissions.set_mode(0o755);+ std::fs::set_permissions(&path, permissions).unwrap();+}++/// A `git` shim whose `update-ref -d refs/heads/para-{a,b}` arms rendezvous+/// with each other (see `test_prune_removals_run_concurrently`); everything+/// else passes through to the real git.+#[cfg(unix)]+fn write_barrier_git_wrapper(dir: &std::path::Path, real_git: &std::path::Path) {+ use std::os::unix::fs::PermissionsExt;++ let real_git = shell_escape::unix::escape(real_git.to_string_lossy());+ let script = format!(+ r#"#!/bin/sh+case "$1 $2 $3" in+ "update-ref -d refs/heads/para-a") own=para-a; other=para-b ;;+ "update-ref -d refs/heads/para-b") own=para-b; other=para-a ;;+ *) exec {real_git} "$@" ;;+esac+: > "$WT_TEST_BARRIER_DIR/started-$own"+i=0+while [ ! -e "$WT_TEST_BARRIER_DIR/started-$other" ]; do+ i=$((i+1))+ if [ "$i" -gt 300 ]; then+ : > "$WT_TEST_BARRIER_DIR/timeout-$own"+ break+ fi+ sleep 0.05+done+exec {real_git} "$@"+"#+ );+ let path = dir.join("git");+ std::fs::write(&path, script).unwrap();+ let mut permissions = std::fs::metadata(&path).unwrap().permissions();+ permissions.set_mode(0o755);+ std::fs::set_permissions(&path, permissions).unwrap();+}+/// A `git` shim that fails every `git worktree prune` and passes everything/// else through to the real git.#[cfg(unix)]