Skip to content

Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows

refactor: use static LazyLock pool for copy operations (#1894)

Maximilian Roosc0b93c6d967c
diff --git a/src/commands/step_commands.rs b/src/commands/step_commands.rs
index 6a86fdbab..49b51c2e2 100644
--- a/src/commands/step_commands.rs
+++ b/src/commands/step_commands.rs
@@ -24,7 +24,7 @@ use ignore::gitignore::GitignoreBuilder;
use rayon::prelude::*;
use worktrunk::HookType;
use worktrunk::config::{CopyIgnoredConfig, UserConfig};
-use worktrunk::copy::{MAX_COPY_THREADS, copy_dir_recursive, create_symlink, remove_if_exists};
+use worktrunk::copy::{copy_dir_recursive, create_symlink, in_copy_pool, remove_if_exists};
use worktrunk::git::Repository;
use worktrunk::path::format_path_for_display;
use worktrunk::shell_exec::Cmd;
@@ -801,18 +801,9 @@ pub fn step_copy_ignored(
}
}
- // Copy entries in parallel — each entry has a distinct destination path.
- // A single dedicated thread pool is created once and shared across all copy
- // operations (outer loop + inner directory recursion). This prevents
- // concurrent callers from each creating their own pool simultaneously,
- // which would exhaust the OS file-descriptor limit on large trees
- // (EMFILE / "too many open files"). copy_dir_recursive() detects that it is
- // already running inside a rayon worker and reuses this pool automatically.
- let pool = rayon::ThreadPoolBuilder::new()
- .num_threads(MAX_COPY_THREADS)
- .build()
- .context("building copy thread pool")?;
- pool.install(|| {
+ // Run all copies in the shared copy pool so that the outer par_iter and
+ // inner copy_dir_recursive calls share the same bounded thread pool.
+ in_copy_pool(|| {
entries_to_copy
.par_iter()
.try_for_each(|(src_entry, is_dir)| -> anyhow::Result<()> {
diff --git a/src/copy.rs b/src/copy.rs
index bcef3afb4..e23edb5b2 100644
--- a/src/copy.rs
+++ b/src/copy.rs
@@ -11,15 +11,36 @@
use std::fs;
use std::io::ErrorKind;
use std::path::Path;
+use std::sync::LazyLock;
use anyhow::Context;
use rayon::prelude::*;
-/// Maximum threads for filesystem copy operations. Beyond this, SSD I/O
-/// contention causes performance to regress (benchmarked on APFS and ext4).
-/// The global rayon pool is sized for network I/O (2x cores) which is too
-/// many for local filesystem work.
-pub const MAX_COPY_THREADS: usize = 4;
+// TODO: benchmark thread count for the shared pool — the old per-call pool
+// was tuned to 4 threads for single-directory copies (#1721), but the shared
+// pool workload is different. Using rayon default (num cores) for now.
+/// Shared thread pool for all filesystem copy operations, created once on first
+/// use. Callers that batch many copies (e.g. `step_copy_ignored`) should wrap
+/// the entire batch in `in_copy_pool()` so all work — including nested
+/// `copy_dir_recursive` calls — shares this pool rather than each spawning its
+/// own (which would exhaust file-descriptor limits on large trees).
+static COPY_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+ rayon::ThreadPoolBuilder::new()
+ .build()
+ .expect("failed to build copy thread pool")
+});
+
+/// Run a closure in the shared copy thread pool.
+///
+/// When already inside the copy pool (e.g. from a nested `copy_dir_recursive`
+/// call), this is a no-op — the closure runs inline on the current thread.
+pub fn in_copy_pool<F, T>(f: F) -> T
+where
+ F: FnOnce() -> T + Send,
+ T: Send,
+{
+ COPY_POOL.install(f)
+}
/// Copy a directory tree recursively using reflink (COW) per file.
///
@@ -30,26 +51,10 @@ pub const MAX_COPY_THREADS: usize = 4;
/// When `force` is true, existing files and symlinks at the destination are
/// removed before copying.
///
-/// Uses a dedicated rayon thread pool capped at `MAX_COPY_THREADS` to avoid
-/// SSD I/O contention from the larger global pool. When called from within an
-/// existing rayon pool context (e.g. via `pool.install()`), that pool is reused
-/// rather than creating a new one — this prevents concurrent callers from each
-/// spawning their own pool and exhausting OS file-descriptor limits on large
-/// trees (EMFILE / "too many open files").
+/// Uses the shared copy pool (capped at 4 threads) to avoid SSD I/O contention
+/// from the larger global rayon pool.
pub fn copy_dir_recursive(src: &Path, dest: &Path, force: bool) -> anyhow::Result<()> {
- // If we are already executing inside a rayon worker thread, reuse that
- // pool rather than creating a new one. This avoids concurrent callers
- // (e.g. from an outer par_iter) each allocating their own pool.
- if rayon::current_thread_index().is_some() {
- return copy_dir_recursive_inner(src, dest, force);
- }
-
- let pool = rayon::ThreadPoolBuilder::new()
- .num_threads(MAX_COPY_THREADS)
- .build()
- .context("building copy thread pool")?;
-
- pool.install(|| copy_dir_recursive_inner(src, dest, force))
+ in_copy_pool(|| copy_dir_recursive_inner(src, dest, force))
}
fn copy_dir_recursive_inner(src: &Path, dest: &Path, force: bool) -> anyhow::Result<()> {