Skip to content

Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows

perf(prune): parallelize removals on the scan lock's read side (#3631)

Maximilian Roose91b5fa88580
diff --git a/benches/CLAUDE.md b/benches/CLAUDE.md
index 468867eaa..2581c11e8 100644
--- a/benches/CLAUDE.md
+++ b/benches/CLAUDE.md
@@ -198,7 +198,7 @@ worth measuring at CI cadence.
## Recording `wt remove` / `wt step prune` staging
-The removal commands interleave serial per-target work with parallel scans and
+The removal commands interleave per-target work with parallel scans and
detached background processes; a single e2e number hides which phase moved.
Record them in two layers:
@@ -217,7 +217,7 @@ because its ~15 GiB fixture must never build on a hosted CI runner):
|---------|----------|------------------|
| `prune_e2e/dry_run_probe_cold` | ~150 ms | full parallel scan, probes re-run (`.git/wt/cache/` cleared; git's own caches stay warm — the "first prune after fetching main" shape) |
| `prune_e2e/dry_run_warm` | ~60 ms | steady-state re-scan, probes hit sha_cache |
-| `prune_e2e/live` | ~600 ms | probe-cold scan + serial removal of the 8 candidates (worktree candidates ~100 ms each — mostly the fsmonitor-daemon stop — branch-only ~25 ms, under the scan write lock, reusing scan-time plans) |
+| `prune_e2e/live` | ~400 ms | probe-cold scan + parallel removal of the 8 candidates (worktree candidates ~145 ms each — mostly the fsmonitor-daemon stop — branch-only ~50 ms, concurrent on the removal worker pool under the scan lock's read side, reusing scan-time plans) |
| `prune_real_repo/dry_run_warm` | ~0.25–0.8 s | steady-state scan of 72 items (36 worktrees + 36 branches) at 331k-commit scale |
| `prune_real_repo/dry_run_probe_cold` | ~0.6–1 s | the same 72-item scan with probes re-running at real cost (statuses stay stat-warm) |
| `first_output/remove` | ~86 ms | single-target validation up to first output (`benches/time_to_first_output.rs`) |
@@ -233,14 +233,19 @@ alone; a live one consumes the candidates). Expected one-shots on the
`merge-base --is-ancestor` ~40 ms and `merge-tree --write-tree` ~130 ms (vs
4–25 ms synthetic, where shallow history walks bottom out at
subprocess-spawn cost)
-- **live ~2.9 s wall** — all 24 removals serialize under the scan write lock
- inside the `prune-scan` window: each of the 12 worktree candidates
- ~155–210 ms (fsmonitor-served status ~20 ms, daemon stop ~57 ms, metadata
- prune ~11 ms, fresh ref snapshot ~40 ms, CAS delete ~7 ms), branch-only
- candidates ~40–100 ms
+- **live ~0.6 s wall** — all 24 removals run concurrently on the removal
+ worker pool inside the `prune-scan` window (read side of the scan lock):
+ the 12 worktree candidates ~240–290 ms each, branch-only ~100–135 ms.
+ Contention inflates the per-span numbers — run serially, a worktree
+ candidate's chain is ~155–210 ms (fsmonitor-served status ~20 ms, daemon
+ stop ~57 ms, metadata prune ~11 ms, fresh ref snapshot ~40 ms, CAS delete
+ ~7 ms) — but the wall collapses to the scan plus the slowest straggler.
+ When candidate branches are packed (post-`gc`), the CAS deletes serialize
+ on `packed-refs.lock`, putting a floor under the branch-deletion tail
This is the "prune takes many seconds" experience users report: worktree
-count × stat-cold statuses bounds the scan, and removals extend it serially.
+count × stat-cold statuses bounds the scan, and the removal tail extends it
+by roughly one candidate's chain.
The synthetic fixture can't show it — its statuses are milliseconds — so
scale-sensitive changes need a one-shot on `prune-real` (or
`wt-perf timeline -- -C <repo> step prune --dry-run` on a real repo) alongside
@@ -262,10 +267,12 @@ silently timed.
check region), one `prune-check:<ref>` per scanned item, and one
`prune-remove:<label>` per removed candidate; `wt remove` emits
`internal-sweep` around its end-of-command janitor. The `prune-remove` spans
-sit *inside* the `prune-scan` window on the live path — each removal takes the
-scan lock's write side, so a span covers the wait for in-flight checks to
-drain *plus* the removal itself: read it as "how long this removal stalled the
-run", not as pure removal work.
+sit *inside* the `prune-scan` window on the live path and overlap each other —
+removals execute concurrently on the worker pool, holding the scan lock's read
+side. A span covers any wait for the lock plus the removal itself; the
+exceptional candidates that take the write side (hook-bearing, `--foreground`,
+metadata-pruning — `removal_needs_write` in `src/commands/step/prune.rs`) also
+wait for every in-flight check and removal to drain first.
```bash
cargo run -p wt-perf -- setup prune-4-8 --path /tmp/prune-repo
diff --git a/src/commands/hook_plan.rs b/src/commands/hook_plan.rs
index 5e6988960..65c8dd6fb 100644
--- a/src/commands/hook_plan.rs
+++ b/src/commands/hook_plan.rs
@@ -316,6 +316,17 @@ impl ApprovedHookPlan {
self.entries.is_empty()
}
+ /// True when the frozen selection carries any commands for `anchor` under
+ /// one of `hook_types`. `wt step prune` routes such candidates onto its
+ /// serialized removal path — a foreground hook body is an arbitrary
+ /// command with streamed output, so it doesn't join the parallel removal
+ /// fan-out.
+ pub fn has_hooks_for(&self, anchor: &Path, hook_types: &[HookType]) -> bool {
+ hook_types
+ .iter()
+ .any(|ht| !self.lookup(*ht, anchor).is_empty())
+ }
+
/// The frozen selection for `(hook_type, anchor)`, by exact match.
///
/// Every covered gate anchors at the identical path its executor passes:
diff --git a/src/commands/step/prune.rs b/src/commands/step/prune.rs
index b943eeafa..a13d4348c 100644
--- a/src/commands/step/prune.rs
+++ b/src/commands/step/prune.rs
@@ -1,9 +1,23 @@
//! `wt step prune` — remove worktrees and branches integrated into the default branch.
+//!
+//! Live-path concurrency: candidate checks fan out on the rayon pool and
+//! stream results to the main thread, which queues per-candidate jobs
+//! (removals and skip lines) in scan-completion order onto a worker pool
+//! sized like rayon's ([`RemovalJob`]). Checks and hook-free removals hold
+//! the read side of [`RemovalContext::check_lock`] and run concurrently; the
+//! exceptional removals serialize on the write side
+//! ([`removal_needs_write`]). One FIFO queue carrying both removals and skip
+//! lines means a single worker (`RAYON_NUM_THREADS=1`) reproduces the serial
+//! total order the deterministic-output tests pin. The first failing removal
+//! flips an abort flag that drains the remaining queue unexecuted; the
+//! current worktree is removed last, after the fan-out, because its removal
+//! cd's the shell to the primary.
use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::RwLock;
+use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use anyhow::Context;
@@ -91,6 +105,26 @@ impl Candidate {
/// (its removal cd's the shell to the primary), with its scan-time plan.
type DeferredCurrent = (Candidate, Option<RemoveResult>);
+/// One unit of work for the removal workers, queued in scan-completion order.
+///
+/// Skip lines ride the same queue as removals so that with a single worker
+/// (`RAYON_NUM_THREADS=1`) prune's per-candidate output keeps one total
+/// order — the property the deterministic-output tests pin. With more
+/// workers, whole lines from different candidates interleave freely: a skip
+/// is a single line, and removal output may span lines but names its branch
+/// on each (one `eprintln!` writes one line under one stderr lock).
+enum RemovalJob {
+ /// Execute a removal via [`try_remove`] and report the outcome.
+ Remove {
+ candidate: Candidate,
+ /// Boxed to keep the variants near parity
+ /// (`clippy::large_enum_variant` — `RemoveResult` is large).
+ plan: Box<Option<RemoveResult>>,
+ },
+ /// Print an already-formatted skip line.
+ PrintSkip(String),
+}
+
#[derive(Clone, Copy)]
enum CandidateKind {
Current,
@@ -192,24 +226,81 @@ fn prune_summary(candidates: &[Candidate]) -> String {
parts.join(", ")
}
-/// Loop-invariant context for [`try_remove`]: every field is identical at all
-/// three call sites in [`step_prune`] (only the `Candidate` varies). Built once
-/// and passed by reference.
+/// Loop-invariant context for [`try_remove`]: every field is identical at
+/// both call sites in [`step_prune`] — the removal workers and the deferred
+/// current worktree (only the `Candidate` varies). Built once and passed by
+/// reference.
struct RemovalContext<'a> {
repo: &'a Repository,
foreground: bool,
hook_plan: &'a ApprovedHookPlan,
worktrees: &'a [WorktreeInfo],
snapshot: &'a RefSnapshot,
- /// Serializes the parallel check readers against the removal writer.
- /// `try_remove` takes the write guard while the background scan workers
- /// hold a read guard around `integration_reason` and
- /// `prepare_worktree_removal` — load-bearing for the Windows `.git/config`
- /// race (rename-fallback rewrites it via lockfile+rename, readers fan out
- /// child git processes that read it).
+ /// Coordinates the parallel workers (scan checks and removals, both on
+ /// the read side) against the few removals that need exclusivity (write
+ /// side — see [`removal_needs_write`]).
+ ///
+ /// The lock exists for the Windows `.git/config` race: git rewrites
+ /// config via lockfile + atomic rename, and a concurrent reader's plain
+ /// `fopen` fails on the rename (#2801). Historically every removal held
+ /// the write side because branch deletion was `git branch -D`, which
+ /// rewrites `.git/config` (it drops the `[branch "<name>"]` section —
+ /// even when none exists). The removal chain has since moved to the CAS
+ /// `git update-ref -d`, and neither it nor `git worktree prune` /
+ /// `git worktree remove` (the rename-failure fallback) touches
+ /// `.git/config` — verified empirically by inode-watching `.git/config`
+ /// across each command — so hook-free removals only spawn readers of
+ /// shared repo state and can run concurrently. (`git branch -D` remains
+ /// reachable only via `delete_branch_if_safe`'s force arm, which prune
+ /// never uses, and its snapshot-miss arm, unreachable here because the
+ /// chain captures the snapshot immediately before consulting it.)
check_lock: &'a RwLock<()>,
}
+/// Which removals must hold the write side of [`RemovalContext::check_lock`]
+/// instead of joining the parallel (read-side) fan-out:
+///
+/// - **Hook-bearing worktree candidates** — the `pre-remove` body runs
+/// foreground here, and hook bodies are arbitrary commands (`git branch
+/// -D`, `git config`, anything), so it keeps the exclusion every removal
+/// had before removals parallelized; the write side also keeps the hook
+/// stream and announce lines from interleaving with other candidates'
+/// output. (`post-remove`/`post-switch` pipelines spawn detached and
+/// always ran outside the lock.)
+/// - **`--foreground` worktree candidates** — the foreground path runs a TTY
+/// trash-cleanup spinner, and concurrent spinners would fight over the
+/// cursor.
+/// - **`StaleDetached` and plan-less `Prunable` candidates** — both call
+/// `prune_worktrees()` fail-hard, so they don't race sibling prunes.
+/// (The fail-soft prune inside `stage_worktree_removal` stays on the read
+/// side: concurrent `git worktree prune` runs are idempotent and exit 0 —
+/// verified empirically — and a swallowed failure only leaves stale
+/// metadata for the next prune, never data loss. The scan's
+/// `prepare_worktree_removal` can also prune under the read guard when a
+/// Linked item's directory vanished mid-scan; `check_one` `.ok()`s that
+/// prepare, so an error there just deselects the candidate.)
+/// - **The current worktree** — deferred until after the fan-out drains and
+/// run alone; write for uniformity with its post-switch hooks.
+fn removal_needs_write(
+ candidate: &Candidate,
+ plan: Option<&RemoveResult>,
+ ctx: &RemovalContext<'_>,
+) -> bool {
+ match candidate.kind {
+ CandidateKind::Current | CandidateKind::StaleDetached => true,
+ CandidateKind::Other | CandidateKind::BranchOnly => match plan {
+ None => true,
+ Some(RemoveResult::RemovedWorktree { worktree_path, .. }) => {
+ ctx.foreground
+ || ctx
+ .hook_plan
+ .has_hooks_for(worktree_path, &[HookType::PreRemove, HookType::PostRemove])
+ }
+ Some(RemoveResult::BranchOnly { .. }) => false,
+ },
+ }
+}
+
/// Try to remove a candidate immediately. Returns Ok(true) if removed,
/// Ok(false) if skipped (preparation error), Err on execution error.
///
@@ -224,11 +315,23 @@ fn try_remove(
ctx: &RemovalContext<'_>,
) -> anyhow::Result<bool> {
let _span = Span::new(format!("prune-remove:{}", candidate.label));
- // The guard protects `()` — there is no shared state to corrupt, so a
- // poisoned lock is meaningless here. Recover the guard rather than
- // `.expect()`-ing: a panic elsewhere should surface as itself, not as a
- // cascade of secondary poison panics on every later removal/reader.
- let _write = ctx.check_lock.write().unwrap_or_else(|e| e.into_inner());
+ // Read side for the parallel default, write side for the exclusive cases
+ // (see `removal_needs_write`). The guards protect `()` — there is no
+ // shared state to corrupt, so a poisoned lock is meaningless here.
+ // Recover the guard rather than `.expect()`-ing: a panic elsewhere should
+ // surface as itself, not as a cascade of secondary poison panics on every
+ // later removal/reader.
+ let (_read, _write) = if removal_needs_write(candidate, plan.as_ref(), ctx) {
+ (
+ None,
+ Some(ctx.check_lock.write().unwrap_or_else(|e| e.into_inner())),
+ )
+ } else {
+ (
+ Some(ctx.check_lock.read().unwrap_or_else(|e| e.into_inner())),
+ None,
+ )
+ };
if matches!(candidate.kind, CandidateKind::StaleDetached) {
ctx.repo.prune_worktrees()?;
@@ -254,8 +357,9 @@ fn try_remove(
},
};
let mut announcer = HookAnnouncer::new(ctx.repo, true);
- // `SynchronousForNonCurrent`: prune keeps the rename-failure fallback's
- // `.git/config` rewrite serialized with its integration-check readers.
+ // `SynchronousForNonCurrent`: a rename-failure fallback completes inline,
+ // so the candidate counts as removed only once the worktree and branch
+ // are actually gone — before the final summary prints.
handle_remove_output(
&plan,
ctx.foreground,
@@ -293,8 +397,8 @@ struct CheckOutcome {
reason: Option<IntegrationReason>,
/// Removal plan from `prepare_worktree_removal` — the same gate `wt
/// remove` uses, computed here on the parallel scan so `try_remove`
- /// doesn't re-derive it (a `git status` per worktree) under the write
- /// lock. `None` means not removable (dirty, locked, primary — filtered
+ /// doesn't re-derive it (a `git status` per worktree) at removal time.
+ /// `None` means not removable (dirty, locked, primary — filtered
/// silently, never reported as "younger than") — except for `Prunable`
/// items, which are always removable but plan in `try_remove`: preparing
/// them calls `prune_worktrees()`, a mutation that must stay serialized.
@@ -309,8 +413,9 @@ struct CheckOutcome {
}
/// One check item's full parallel work: integration + removability + age.
-/// Held under the check-lock read guard at the call site to serialize against
-/// `try_remove` rewriting `.git/config` on the Windows rename-fallback path.
+/// Held under the check-lock read guard at the call site so it never overlaps
+/// a write-side removal (hook-bearing or metadata-pruning candidates — see
+/// [`removal_needs_write`]).
#[allow(clippy::too_many_arguments)]
fn check_one(
item: &CheckItem,
@@ -795,7 +900,7 @@ pub fn step_prune(
if dry_run {
let check_lock = RwLock::new(());
let scan_span = Span::new("prune-scan");
- let mut dry_run_info: Vec<(Candidate, DryRunInfo)> = std::thread::scope(|s| {
+ let dry_run_info: Vec<(Candidate, DryRunInfo)> = std::thread::scope(|s| {
let (tx, rx) = chan::unbounded::<(usize, anyhow::Result<CheckOutcome>)>();
// Pre-shadow with references so `move` on s.spawn moves only `tx`
// (so it's dropped when the spawn ends and `rx` can terminate),
@@ -864,7 +969,6 @@ pub fn step_prune(
anyhow::Ok(info)
})?;
drop(scan_span);
- dry_run_info.sort_by_key(|(c, _)| c.check_idx);
return render_dry_run(dry_run_info, skipped_young, min_age, format);
}
@@ -958,11 +1062,17 @@ pub fn step_prune(
snapshot: &snapshot,
check_lock: &check_lock,
};
-
- // Streaming live path: scans run in parallel and the main thread acts on
- // each result as it arrives — print "Skipped (younger than X)" or call
- // `try_remove` immediately for positives. The current worktree is the one
- // exception: its removal cd's to the primary, so defer it until last.
+ // Flipped by the first failing removal: the rest of the queue drains
+ // without executing (matching the serial loop's abort-on-first-error),
+ // and the error propagates after the workers finish.
+ let abort = AtomicBool::new(false);
+
+ // Streaming live path: scans run in parallel and the main thread queues a
+ // job for each result as it arrives — a "Skipped ..." line or a removal.
+ // Removals execute concurrently on the worker pool (read side of
+ // `check_lock`; the exceptional candidates take the write side — see
+ // `removal_needs_write`). The current worktree is the one exception to
+ // the fan-out: its removal cd's to the primary, so defer it until last.
let scan_span = Span::new("prune-scan");
let (removed, deferred_current) = std::thread::scope(
|s| -> anyhow::Result<(Vec<Candidate>, Option<DeferredCurrent>)> {
@@ -970,7 +1080,7 @@ pub fn step_prune(
// Pre-shadow with references so `move` on s.spawn moves only `tx`
// (so it's dropped when the spawn ends and `rx` can terminate),
// while the heavy state stays borrowed and remains usable by the
- // main thread's removal calls.
+ // main thread.
let repo_ref = &repo;
let snapshot_ref = &snapshot;
let check_items_ref = &check_items;
@@ -997,10 +1107,65 @@ pub fn step_prune(
});
});
- let mut removed: Vec<Candidate> = Vec::new();
+ // Removal workers, sized like the scan's rayon pool so
+ // `RAYON_NUM_THREADS=1` serializes removals too (deterministic
+ // output for tests). Results flow back on `done_rx`; the channel
+ // closes once every worker has drained the job queue and exited,
+ // so draining it below also waits out all in-flight printing.
+ let (job_tx, job_rx) = chan::unbounded::<RemovalJob>();
+ let (done_tx, done_rx) = chan::unbounded::<(Candidate, anyhow::Result<bool>)>();
+ let abort_ref = &abort;
+ let removal_ctx_ref = &removal_ctx;
+ // Empty check_items → zero workers, correctly: no jobs can queue.
+ let workers = rayon::current_num_threads().min(check_items.len());
+ for _ in 0..workers {
+ let job_rx = job_rx.clone();
+ let done_tx = done_tx.clone();
+ s.spawn(move || {
+ for job in job_rx {
+ match job {
+ RemovalJob::Remove { candidate, plan } => {
+ // Skip (not print, not remove) once a sibling
+ // failed; queued skip lines below still print
+ // — they were discovered before the failure.
+ if abort_ref.load(Ordering::Relaxed) {
+ continue;
+ }
+ let result = try_remove(&candidate, *plan, removal_ctx_ref)
+ .with_context(|| candidate.removal_context());
+ if result.is_err() {
+ abort_ref.store(true, Ordering::Relaxed);
+ }
+ if done_tx.send((candidate, result)).is_err() {
+ return;
+ }
+ }
+ RemovalJob::PrintSkip(line) => {
+ // Hold the read side so a skip line can't
+ // land inside a write-side removal's
+ // exclusive output window (spinner, hook
+ // stream).
+ let _read = removal_ctx_ref
+ .check_lock
+ .read()
+ .unwrap_or_else(|e| e.into_inner());
+ eprintln!("{line}");
+ }
+ }
+ }
+ });
+ }
+ drop(job_rx);
+ drop(done_tx);
+
let mut deferred_current: Option<DeferredCurrent> = None;
for (idx, outcome) in &rx {
- let outcome = outcome.context("checking branch integration")?;
+ // A check error fails the whole run — flip `abort` so the
+ // workers drain their queue without executing more removals
+ // (whose results this early return would silently drop).
+ let outcome = outcome
+ .context("checking branch integration")
+ .inspect_err(|_| abort_ref.store(true, Ordering::Relaxed))?;
let Some(_reason) = outcome.reason else {
continue;
};
@@ -1013,12 +1178,11 @@ pub fn step_prune(
if let Some(age) = outcome.age
&& age < min_age_duration
{
- eprintln!(
- "{}",
- info_message(cformat!(
- "Skipped <bold>{label}</> (younger than {min_age})"
- ))
- );
+ let line = info_message(cformat!(
+ "Skipped <bold>{label}</> (younger than {min_age})"
+ ))
+ .to_string();
+ let _ = job_tx.send(RemovalJob::PrintSkip(line));
skipped_young.push(label);
continue;
}
@@ -1035,10 +1199,10 @@ pub fn step_prune(
CandidateKind::BranchOnly | CandidateKind::StaleDetached => false,
};
if needs_approval {
- eprintln!(
- "{}",
+ let line =
info_message(cformat!("Skipped <bold>{label}</> (approval required)"))
- );
+ .to_string();
+ let _ = job_tx.send(RemovalJob::PrintSkip(line));
let differs = path.as_deref().is_some_and(|wt_path| {
let candidate_bytes =
std::fs::read(wt_path.join(".config").join("wt.toml")).ok();
@@ -1057,18 +1221,47 @@ pub fn step_prune(
};
if matches!(candidate.kind, CandidateKind::Current) {
deferred_current = Some((candidate, outcome.plan));
- } else if try_remove(&candidate, outcome.plan, &removal_ctx)
- .with_context(|| candidate.removal_context())?
- {
- removed.push(candidate);
+ } else {
+ let _ = job_tx.send(RemovalJob::Remove {
+ candidate,
+ plan: Box::new(outcome.plan),
+ });
}
}
+ drop(job_tx);
+
+ let mut removed: Vec<Candidate> = Vec::new();
+ let mut first_err: Option<anyhow::Error> = None;
+ for (candidate, result) in &done_rx {
+ match result {
+ Ok(true) => removed.push(candidate),
+ Ok(false) => {}
+ Err(err) if first_err.is_none() => first_err = Some(err),
+ // Concurrent failures (e.g. one Ctrl-C killing every
+ // in-flight child) all carry the same story; report the
+ // first and keep the rest out of the terminal.
+ Err(err) => {
+ tracing::debug!(
+ error = %err,
+ "additional removal failure for {}: {err:#}",
+ candidate.label
+ );
+ }
+ }
+ }
+ if let Some(err) = first_err {
+ return Err(err);
+ }
Ok((removed, deferred_current))
},
)?;
drop(scan_span);
let mut removed = removed;
+ // Deterministic order for `--format=json` regardless of which worker
+ // finished first (the dry-run path sorts the same way); the deferred
+ // current worktree stays last.
+ removed.sort_by_key(|c| c.check_idx);
// Remove deferred current worktree last (cd-to-primary happens here)
if let Some((current, plan)) = deferred_current
&& try_remove(&current, plan, &removal_ctx).with_context(|| current.removal_context())?
diff --git a/src/git/remove.rs b/src/git/remove.rs
index cd080cefb..c80fae8ef 100644
--- a/src/git/remove.rs
+++ b/src/git/remove.rs
@@ -483,6 +483,13 @@ pub fn delete_branch_if_safe(
/// failed → propagate the original error) by re-checking with `rev-parse
/// --verify --quiet`, which has a structured exit code (0 = present, 1 =
/// absent) rather than relying on locale-sensitive error-message text.
+///
+/// A `packed-refs.lock` that stays contended past git's ~1 s retry budget
+/// (concurrent deletes of packed branches — `wt step prune`'s parallel
+/// removals — on slow ref-store I/O) also fails the CAS with the ref
+/// present, and reads as `RetainedRaced` even though the tip never moved.
+/// Accepted: it is fail-closed, empirically unobserved at 24-way concurrency
+/// on local disks, and the next prune collects the branch.
fn cas_delete_branch_outcome(
repo: &Repository,
branch_name: &str,
diff --git a/src/output/handlers.rs b/src/output/handlers.rs
index d069f5a0d..fc451f21d 100644
--- a/src/output/handlers.rs
+++ b/src/output/handlers.rs
@@ -103,9 +103,9 @@ pub enum BackgroundFallbackMode {
/// `wt remove`, `wt merge`, and the picker.
Detached,
/// Run the fallback removal and branch deletion synchronously for a
- /// non-current worktree. `wt step prune` uses this so each iteration's
- /// removal completes before the next candidate is considered and before
- /// the final summary is printed.
+ /// non-current worktree. `wt step prune` uses this so a candidate's
+ /// removal is complete by the time it is reported removed — before the
+ /// final summary prints, never outliving the prune process.
SynchronousForNonCurrent,
}
@@ -148,8 +148,8 @@ fn spawn_background_removal(
/// is no longer checked out in any worktree), and the background command is just `rm -rf`.
/// If rename fails (cross-filesystem, permissions, Windows file locking), either returns the
/// legacy `git worktree remove` command with branch deletion deferred to the background, or
-/// runs that fallback synchronously for non-current worktrees when the caller needs to keep
-/// the fallback's `.git/config` rewrite serialized with other git readers.
+/// runs that fallback synchronously for non-current worktrees when the caller needs the
+/// removal complete before it reports success (`wt step prune`).
///
/// The caller is responsible for spawning detached plans in the background.
fn execute_instant_removal_or_fallback(
@@ -1024,8 +1024,7 @@ pub fn execute_user_command(command: &str, display_path: Option<&Path>) -> anyho
/// rename-into-trash fast path fails behaves: every caller but `wt step prune`
/// passes [`BackgroundFallbackMode::Detached`] (spawn the legacy `git worktree
/// remove`); prune passes [`BackgroundFallbackMode::SynchronousForNonCurrent`]
-/// to keep the fallback's `.git/config` rewrite serialized with its parallel
-/// integration-check readers.
+/// so a candidate's removal is complete by the time prune reports it removed.
pub fn handle_remove_output(
result: &RemoveResult,
foreground: bool,
diff --git a/tests/integration_tests/step_prune.rs b/tests/integration_tests/step_prune.rs
index 963363902..6fcd6f605 100644
--- a/tests/integration_tests/step_prune.rs
+++ b/tests/integration_tests/step_prune.rs
@@ -1338,24 +1338,26 @@ fn test_prune_runs_pre_remove_hook(mut repo: TestRepo) {
assert!(!wt_path.exists(), "the merged worktree should be removed");
}
-/// Regression test for serialized `wt step prune` fallback removals.
+/// Canary for `wt step prune` removals overlapping `.git/config` readers,
+/// and regression test for synchronous fallback completion.
///
-/// B-prime (the `check_lock` RwLock in `src/commands/step/prune.rs`)
-/// serializes the parallel `integration_reason` `.git/config` readers against
-/// the config-rewriting branch deletion writer. That includes the
-/// cross-filesystem / `.gitmodules` / Windows-file-lock fallback: prune runs
-/// the non-current fallback removal and branch deletion synchronously under
-/// the write guard instead of spawning a detached `git worktree remove && git
-/// branch -d`. A branch with a `[branch "<name>"]` section makes deletion
-/// rewrite `.git/config` via lockfile+rename.
+/// Prune's hook-free removals — the rename-failure fallback included — run
+/// concurrently with the parallel `integration_reason` readers on the read
+/// side of `check_lock` (`src/commands/step/prune.rs`). That is safe because
+/// the chain's branch deletion is a CAS `git update-ref -d`, which never
+/// rewrites `.git/config`; a deletion mechanism that rewrites config via
+/// lockfile+rename (as `git branch -D` does — the original Windows race,
+/// #2801) would collide with those readers again. Each branch here gets a
+/// `[branch "<name>"]` section so any such regression has a section to
+/// rewrite, and the assertion watches for the Windows
+/// `unable to access '.git/config'` failure.
///
-/// This forces that fallback for one non-current integrated worktree (by
-/// pre-blocking its staged path, like
+/// The fallback must also complete synchronously
+/// (`SynchronousForNonCurrent`): this forces it for one non-current
+/// integrated worktree (by pre-blocking its staged path, like
/// `test_remove_background_fallback_on_rename_failure`) while several other
-/// integrated worktrees keep the parallel integration-check fan-out running,
-/// so the config-rewriting branch deletion overlaps live `.git/config`
-/// readers. After the fix, removal and deletion run synchronously inside
-/// `try_remove`, so `wt step prune` cannot exit before both finish — the
+/// integrated worktrees keep the parallel fan-out running. `wt step prune`
+/// cannot exit before the fallback removal and branch deletion finish — the
/// regression assertion (`blocked` worktree and branch gone the instant prune
/// returns) holds on every platform.
///
@@ -1374,11 +1376,11 @@ fn test_prune_fallback_config_race_canary(mut repo: TestRepo) {
// Several integrated worktrees → a real parallel integration-check
// fan-out. `add_worktree` puts each branch at `main` HEAD, so all are
// same-commit integrated and will be pruned. Each branch gets a
- // `[branch "<name>"]` section so its `git branch -d` rewrites
- // `.git/config` — the racing write. (No remote needed: `git branch -d`
- // removes the section regardless of whether `origin` resolves, and the
- // same-commit local check yields "integrated" before upstream is
- // consulted.)
+ // `[branch "<name>"]` section so a config-rewriting deletion (a
+ // regression from the CAS `update-ref -d` back toward `git branch -d`,
+ // which removes the section via lockfile+rename) has a racing write to
+ // make. (No remote needed: the same-commit local check yields
+ // "integrated" before upstream is consulted.)
let names: Vec<String> = (0..6).map(|i| format!("merged-canary-{i}")).collect();
for name in &names {
repo.add_worktree(name);
@@ -1392,8 +1394,8 @@ fn test_prune_fallback_config_race_canary(mut repo: TestRepo) {
// Force the fallback for one *non-current* worktree by pre-creating a
// file at its computed staged path so `std::fs::rename(worktree → trash)`
- // fails. Pick one in the middle so integration checks for later refs would
- // still be in flight if fallback branch deletion escaped the write guard.
+ // fails. Pick one in the middle so integration checks for later refs are
+ // still in flight while the fallback runs.
let blocked = names[3].clone();
let blocked_wt_path = repo.worktree_path(&blocked).to_path_buf();
let trash_dir = crate::common::resolve_git_common_dir(repo.root_path()).join("wt/trash");
@@ -1508,6 +1510,275 @@ fn test_prune_skips_prunable_candidate_whose_preparation_fails(mut repo: TestRep
);
}
+/// Hook-free removals execute concurrently (the read side of `check_lock`),
+/// not one at a time.
+///
+/// Two integrated orphan branches are removed through a `git` shim whose
+/// `update-ref -d` arms barrier on each other: each records that it started,
+/// then waits for the *other* branch's deletion to start before proceeding.
+/// The barrier only resolves if both removals are in flight at once — under
+/// serialized removals the first deletion would wait out the shim's 15 s
+/// timeout and drop a sentinel file, which the test asserts absent. Causally
+/// driven, so it runs at barrier speed when concurrency works; the timeout is
+/// only the safety net. Unix-only for the same `CreateProcess` shim reason as
+/// the canary above.
+#[cfg(unix)]
+#[rstest]
+fn test_prune_removals_run_concurrently(repo: TestRepo) {
+ repo.commit("initial");
+
+ // Orphan branches at main HEAD: same-commit integrated, no worktree, so
+ // each becomes a hook-free BranchOnly candidate on the parallel path.
+ repo.create_branch("para-a");
+ repo.create_branch("para-b");
+
+ let mut cmd = repo.wt_command();
+ // The removal pool is sized from the rayon thread count; pin it to two so
+ // the barrier can resolve even on a single-core runner (the workers block
+ // in subprocess waits, so two threads don't need two CPUs).
+ cmd.env("RAYON_NUM_THREADS", "2");
+ let git_wrapper_dir = repo.home_path().join("git-wrapper");
+ std::fs::create_dir_all(&git_wrapper_dir).unwrap();
+ write_barrier_git_wrapper(&git_wrapper_dir, &which::which("git").unwrap());
+ prepend_path(&mut cmd, &git_wrapper_dir);
+ let barrier_dir = repo.home_path().join("barrier");
+ std::fs::create_dir_all(&barrier_dir).unwrap();
+ cmd.env("WT_TEST_BARRIER_DIR", &barrier_dir);
+
+ let output = cmd
+ .args(["step", "prune", "--yes", "--min-age=0s"])
+ .output()
+ .unwrap();
+ let stderr = String::from_utf8_lossy(&output.stderr);
+
+ assert!(output.status.success(), "prune should succeed:\n{stderr}");
+ for name in ["para-a", "para-b"] {
+ assert!(
+ barrier_dir.join(format!("started-{name}")).exists(),
+ "the shim never fired for {name} — its CAS delete was not exercised"
+ );
+ }
+ let timeouts: Vec<String> = std::fs::read_dir(&barrier_dir)
+ .unwrap()
+ .filter_map(|e| e.ok())
+ .map(|e| e.file_name().to_string_lossy().into_owned())
+ .filter(|n| n.starts_with("timeout-"))
+ .collect();
+ assert!(
+ timeouts.is_empty(),
+ "a deletion waited out the barrier — removals ran serially: {timeouts:?}"
+ );
+ let branches = repo.git_output(&["branch", "--format=%(refname:short)"]);
+ for name in ["para-a", "para-b"] {
+ assert!(
+ !branches.lines().any(|branch| branch == name),
+ "{name} should have been deleted; branches:\n{branches}"
+ );
+ }
+}
+
+/// The first failing removal aborts the rest of the queue.
+///
+/// Three integrated orphan branches scan in sorted order (`abort-a` first)
+/// with a single worker (`RAYON_NUM_THREADS=1`), so the jobs run FIFO. A
+/// `git` shim makes `abort-a`'s CAS delete fail *after* deleting the ref
+/// (so `cas_delete_branch_outcome`'s re-check finds it gone and propagates
+/// the error rather than reporting `RetainedRaced`). The failure must flip
+/// the abort flag: the queued `abort-b`/`abort-c` removals never execute,
+/// their branches survive, no summary prints, and prune exits non-zero —
+/// the serial loop's abort-on-first-error, preserved across the fan-out.
+/// Unix-only for the same `CreateProcess` shim reason as the canary above.
+#[cfg(unix)]
+#[rstest]
+fn test_prune_removal_failure_aborts_remaining_queue(repo: TestRepo) {
+ repo.commit("initial");
+
+ repo.create_branch("abort-a");
+ repo.create_branch("abort-b");
+ repo.create_branch("abort-c");
+
+ let mut cmd = repo.wt_command();
+ cmd.env("RAYON_NUM_THREADS", "1"); // FIFO: abort-a dispatches first
+ let git_wrapper_dir = repo.home_path().join("git-wrapper");
+ std::fs::create_dir_all(&git_wrapper_dir).unwrap();
+ write_failing_branch_delete_wrapper(&git_wrapper_dir, &which::which("git").unwrap());
+ prepend_path(&mut cmd, &git_wrapper_dir);
+ cmd.env("WT_TEST_FAIL_DELETE_BRANCH", "abort-a");
+
+ let output = cmd
+ .args(["step", "prune", "--yes", "--min-age=0s"])
+ .output()
+ .unwrap();
+ let stderr = String::from_utf8_lossy(&output.stderr);
+
+ assert!(
+ !output.status.success(),
+ "a failed removal must fail the run:\n{stderr}"
+ );
+ assert!(
+ stderr.contains("removing branch abort-a"),
+ "the error should carry the failing candidate's context:\n{stderr}"
+ );
+ assert!(
+ !stderr.contains("Pruned "),
+ "no summary after an aborted run:\n{stderr}"
+ );
+ let branches = repo.git_output(&["branch", "--format=%(refname:short)"]);
+ assert!(
+ !branches.lines().any(|b| b == "abort-a"),
+ "the shim deleted abort-a's ref; branches:\n{branches}"
+ );
+ for name in ["abort-b", "abort-c"] {
+ assert!(
+ branches.lines().any(|b| b == name),
+ "{name} was queued behind the failure and must survive; branches:\n{branches}"
+ );
+ }
+}
+
+/// Concurrent failures: the first error is reported, the rest stay quiet.
+///
+/// Two failing removals rendezvous inside the shim (the barrier from
+/// `test_prune_removals_run_concurrently`) so both are in flight before
+/// either error lands — exercising the drain's duplicate-failure arm, which
+/// logs at debug rather than printing a second error.
+#[cfg(unix)]
+#[rstest]
+fn test_prune_concurrent_removal_failures_report_first(repo: TestRepo) {
+ repo.commit("initial");
+
+ repo.create_branch("dupe-a");
+ repo.create_branch("dupe-b");
+
+ let mut cmd = repo.wt_command();
+ cmd.env("RAYON_NUM_THREADS", "2"); // both jobs in flight at once
+ let git_wrapper_dir = repo.home_path().join("git-wrapper");
+ std::fs::create_dir_all(&git_wrapper_dir).unwrap();
+ write_barrier_failing_delete_wrapper(&git_wrapper_dir, &which::which("git").unwrap());
+ prepend_path(&mut cmd, &git_wrapper_dir);
+ let barrier_dir = repo.home_path().join("barrier");
+ std::fs::create_dir_all(&barrier_dir).unwrap();
+ cmd.env("WT_TEST_BARRIER_DIR", &barrier_dir);
+
+ let output = cmd
+ .args(["step", "prune", "--yes", "--min-age=0s"])
+ .output()
+ .unwrap();
+ let stderr = String::from_utf8_lossy(&output.stderr);
+
+ assert!(
+ !output.status.success(),
+ "failed removals must fail the run:\n{stderr}"
+ );
+ for name in ["dupe-a", "dupe-b"] {
+ assert!(
+ barrier_dir.join(format!("started-{name}")).exists(),
+ "the shim never fired for {name}"
+ );
+ }
+ // Exactly one candidate's failure reaches the terminal.
+ assert_eq!(
+ stderr.matches("removing branch dupe-").count(),
+ 1,
+ "exactly one of the concurrent failures should be reported:\n{stderr}"
+ );
+}
+
+/// A `git` shim that deletes `refs/heads/$WT_TEST_FAIL_DELETE_BRANCH` for
+/// real and then reports failure when prune's CAS delete targets it —
+/// making `cas_delete_branch_outcome` propagate an error (ref gone on
+/// re-check) instead of `RetainedRaced` (ref still present).
+#[cfg(unix)]
+fn write_failing_branch_delete_wrapper(dir: &std::path::Path, real_git: &std::path::Path) {
+ use std::os::unix::fs::PermissionsExt;
+
+ let real_git = shell_escape::unix::escape(real_git.to_string_lossy());
+ let script = format!(
+ r#"#!/bin/sh
+if [ "$1" = "update-ref" ] && [ "$2" = "-d" ] && [ "$3" = "refs/heads/$WT_TEST_FAIL_DELETE_BRANCH" ]; then
+ {real_git} update-ref -d "$3" || true
+ exit 1
+fi
+exec {real_git} "$@"
+"#
+ );
+ let path = dir.join("git");
+ std::fs::write(&path, script).unwrap();
+ let mut permissions = std::fs::metadata(&path).unwrap().permissions();
+ permissions.set_mode(0o755);
+ std::fs::set_permissions(&path, permissions).unwrap();
+}
+
+/// The barrier shim (see `write_barrier_git_wrapper`) with a failing tail:
+/// both `dupe-{a,b}` deletions rendezvous, delete their ref for real, then
+/// report failure — two concurrent removal errors.
+#[cfg(unix)]
+fn write_barrier_failing_delete_wrapper(dir: &std::path::Path, real_git: &std::path::Path) {
+ use std::os::unix::fs::PermissionsExt;
+
+ let real_git = shell_escape::unix::escape(real_git.to_string_lossy());
+ let script = format!(
+ r#"#!/bin/sh
+case "$1 $2 $3" in
+ "update-ref -d refs/heads/dupe-a") own=dupe-a; other=dupe-b ;;
+ "update-ref -d refs/heads/dupe-b") own=dupe-b; other=dupe-a ;;
+ *) exec {real_git} "$@" ;;
+esac
+: > "$WT_TEST_BARRIER_DIR/started-$own"
+i=0
+while [ ! -e "$WT_TEST_BARRIER_DIR/started-$other" ]; do
+ i=$((i+1))
+ if [ "$i" -gt 300 ]; then
+ break
+ fi
+ sleep 0.05
+done
+{real_git} update-ref -d "$3" || true
+exit 1
+"#
+ );
+ let path = dir.join("git");
+ std::fs::write(&path, script).unwrap();
+ let mut permissions = std::fs::metadata(&path).unwrap().permissions();
+ permissions.set_mode(0o755);
+ std::fs::set_permissions(&path, permissions).unwrap();
+}
+
+/// A `git` shim whose `update-ref -d refs/heads/para-{a,b}` arms rendezvous
+/// with each other (see `test_prune_removals_run_concurrently`); everything
+/// else passes through to the real git.
+#[cfg(unix)]
+fn write_barrier_git_wrapper(dir: &std::path::Path, real_git: &std::path::Path) {
+ use std::os::unix::fs::PermissionsExt;
+
+ let real_git = shell_escape::unix::escape(real_git.to_string_lossy());
+ let script = format!(
+ r#"#!/bin/sh
+case "$1 $2 $3" in
+ "update-ref -d refs/heads/para-a") own=para-a; other=para-b ;;
+ "update-ref -d refs/heads/para-b") own=para-b; other=para-a ;;
+ *) exec {real_git} "$@" ;;
+esac
+: > "$WT_TEST_BARRIER_DIR/started-$own"
+i=0
+while [ ! -e "$WT_TEST_BARRIER_DIR/started-$other" ]; do
+ i=$((i+1))
+ if [ "$i" -gt 300 ]; then
+ : > "$WT_TEST_BARRIER_DIR/timeout-$own"
+ break
+ fi
+ sleep 0.05
+done
+exec {real_git} "$@"
+"#
+ );
+ let path = dir.join("git");
+ std::fs::write(&path, script).unwrap();
+ let mut permissions = std::fs::metadata(&path).unwrap().permissions();
+ permissions.set_mode(0o755);
+ std::fs::set_permissions(&path, permissions).unwrap();
+}
+
/// A `git` shim that fails every `git worktree prune` and passes everything
/// else through to the real git.
#[cfg(unix)]