Skip to content

Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows

refactor: simplify copy pool to single-level parallelism (#1931)

Maximilian Roosbaec5b12fd7d
diff --git a/src/commands/step_commands.rs b/src/commands/step_commands.rs
index 0f89ae6fa..6c181f001 100644
--- a/src/commands/step_commands.rs
+++ b/src/commands/step_commands.rs
@@ -15,7 +15,6 @@
use std::fs;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use anyhow::Context;
@@ -24,7 +23,7 @@ use ignore::gitignore::GitignoreBuilder;
use rayon::prelude::*;
use worktrunk::HookType;
use worktrunk::config::{CopyIgnoredConfig, UserConfig};
-use worktrunk::copy::{copy_dir_recursive, copy_pool_install, create_symlink, remove_if_exists};
+use worktrunk::copy::{copy_dir_recursive, copy_leaf};
use worktrunk::git::Repository;
use worktrunk::path::format_path_for_display;
use worktrunk::shell_exec::Cmd;
@@ -770,7 +769,6 @@ pub fn step_copy_ignored(
}
let verbose = verbosity();
- let copied_count = AtomicUsize::new(0);
// Show entries in verbose or dry-run mode
if verbose >= 1 || dry_run {
@@ -800,80 +798,44 @@ pub fn step_copy_ignored(
}
}
- copy_pool_install(|| {
- entries_to_copy
- .par_iter()
- .try_for_each(|(src_entry, is_dir)| -> anyhow::Result<()> {
- // Paths from git ls-files are always under source_path
- let relative = src_entry
- .strip_prefix(&source_path)
- .expect("git ls-files path under worktree");
- let dest_entry = dest_path.join(relative);
-
- if *is_dir {
- copy_dir_recursive(src_entry, &dest_entry, force).with_context(|| {
- format!("copying directory {}", format_path_for_display(relative))
- })?;
- copied_count.fetch_add(1, Ordering::Relaxed);
- } else {
- if let Some(parent) = dest_entry.parent() {
- fs::create_dir_all(parent).with_context(|| {
- format!(
- "creating directory for {}",
- format_path_for_display(relative)
- )
- })?;
- }
- if force {
- remove_if_exists(&dest_entry)?;
- }
- // Check if source is a symlink — preserve it instead of following it
- let display_path = format_path_for_display(relative);
- let source_is_symlink = fs::symlink_metadata(src_entry)
- .context(format!("reading metadata for {display_path}"))?
- .file_type()
- .is_symlink();
- if source_is_symlink {
- // Skip existing symlinks for idempotent hook usage
- if dest_entry.symlink_metadata().is_err() {
- let target = fs::read_link(src_entry)
- .context(format!("reading symlink {display_path}"))?;
- create_symlink(&target, src_entry, &dest_entry)?;
- copied_count.fetch_add(1, Ordering::Relaxed);
- }
- } else {
- // Skip existing entries (files or symlinks) for idempotent hook usage.
- // Check symlink_metadata (not exists()) because exists() follows symlinks
- // and returns false for broken ones, which would cause reflink_or_copy to
- // fail with ENOENT on some platforms when copying through the broken symlink.
- if dest_entry.symlink_metadata().is_err() {
- match reflink_copy::reflink_or_copy(src_entry, &dest_entry) {
- Ok(_) => {
- copied_count.fetch_add(1, Ordering::Relaxed);
- }
- Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
- Err(e) => {
- return Err(anyhow::Error::from(e)
- .context(format!("copying {display_path}")));
- }
- }
- }
- }
- }
- Ok(())
- })
- })?;
+ let mut copied_count = 0usize;
+ for (src_entry, is_dir) in &entries_to_copy {
+ let relative = src_entry
+ .strip_prefix(&source_path)
+ .expect("git ls-files path under worktree");
+ let dest_entry = dest_path.join(relative);
+
+ if *is_dir {
+ copied_count +=
+ copy_dir_recursive(src_entry, &dest_entry, force).with_context(|| {
+ format!("copying directory {}", format_path_for_display(relative))
+ })?;
+ } else {
+ if let Some(parent) = dest_entry.parent() {
+ fs::create_dir_all(parent).with_context(|| {
+ format!(
+ "creating directory for {}",
+ format_path_for_display(relative)
+ )
+ })?;
+ }
+ let is_symlink = fs::symlink_metadata(src_entry)
+ .with_context(|| {
+ format!("reading metadata for {}", format_path_for_display(relative))
+ })?
+ .file_type()
+ .is_symlink();
+ if copy_leaf(src_entry, &dest_entry, is_symlink, force)? {
+ copied_count += 1;
+ }
+ }
+ }
// Show summary
- let copied_count = copied_count.load(Ordering::Relaxed);
- let entry_word = if copied_count == 1 {
- "entry"
- } else {
- "entries"
- };
+ let file_word = if copied_count == 1 { "file" } else { "files" };
eprintln!(
"{}",
- success_message(format!("Copied {copied_count} {entry_word}"))
+ success_message(format!("Copied {copied_count} {file_word}"))
);
Ok(())
@@ -1782,19 +1744,6 @@ pub fn step_relocate(
mod tests {
use super::*;
- #[test]
- fn test_remove_if_exists_nonexistent() {
- // NotFound is silently ignored
- assert!(remove_if_exists(Path::new("/nonexistent/file")).is_ok());
- }
-
- #[test]
- fn test_remove_if_exists_not_a_file() {
- // Trying to remove a directory with remove_file produces a non-NotFound error
- let dir = std::env::temp_dir();
- assert!(remove_if_exists(&dir).is_err());
- }
-
#[test]
fn test_move_entry_file() {
let tmp = tempfile::tempdir().unwrap();
diff --git a/src/copy.rs b/src/copy.rs
index 118dffba7..919b2c3f0 100644
--- a/src/copy.rs
+++ b/src/copy.rs
@@ -1,48 +1,35 @@
-//! Recursive directory copying with reflink (COW) and rayon parallelism.
+//! Directory copying with reflink (COW) and rayon parallelism.
//!
//! Copies directory trees file-by-file using `reflink_or_copy` which uses
//! copy-on-write clones where the filesystem supports them (APFS, btrfs, XFS),
//! falling back to regular copies otherwise.
//!
-//! All copy I/O runs in a dedicated 4-thread pool rather than the global rayon
+//! All copy I/O runs on a dedicated 4-thread pool rather than the global rayon
//! pool (which is sized at 2× CPU cores for network I/O) to avoid saturating
-//! the CPU on a background operation.
+//! the CPU on a background operation. Directory trees are walked iteratively
+//! (no recursion) then copied in a single parallel pass.
//!
//! Callers that want low-priority I/O (e.g. `step_copy_ignored`) should call
//! [`lower_process_priority`] before starting work.
use std::fs;
use std::io::ErrorKind;
-use std::path::Path;
+use std::path::{Path, PathBuf};
use std::sync::LazyLock;
+use std::sync::atomic::{AtomicUsize, Ordering};
use anyhow::Context;
use rayon::prelude::*;
/// Capped at 4 threads to avoid saturating the CPU — the global rayon pool is
/// much larger (2× CPU cores, tuned for network I/O in `wt list`).
-///
-/// The 8 MiB stack matches the default on Linux/macOS; Windows defaults to
-/// ~2 MiB, which can overflow under concurrent reflink/copy work (observed as
-/// a worker-thread stack overflow in `test_copy_ignored_many_directories_no_emfile`
-/// on Windows CI).
static COPY_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
rayon::ThreadPoolBuilder::new()
.num_threads(4)
- .stack_size(8 * 1024 * 1024)
.build()
.expect("failed to build copy thread pool")
});
-/// Run a closure on the dedicated copy thread pool.
-///
-/// Use this to wrap outer `par_iter` loops that call [`copy_dir_recursive`],
-/// so all copy parallelism shares the same 4 threads instead of blocking
-/// global-pool threads while waiting for copy workers.
-pub fn copy_pool_install<R: Send>(f: impl FnOnce() -> R + Send) -> R {
- COPY_POOL.install(f)
-}
-
/// Lower the current process's scheduling priority so copy I/O doesn't
/// compete with interactive foreground work.
///
@@ -62,84 +49,118 @@ pub fn lower_process_priority() {
}
}
-/// Copy a directory tree recursively using reflink (COW) per file.
+/// Copy a single file or symlink, using reflink (COW) when possible.
///
-/// Handles regular files, directories, and symlinks. Non-regular files (sockets,
-/// FIFOs) are silently skipped. Existing entries at the destination are skipped
-/// for idempotent usage.
+/// Returns `true` if the entry was copied, `false` if skipped (destination
+/// already exists). When `force` is true, existing entries are removed before
+/// copying.
+pub fn copy_leaf(src: &Path, dest: &Path, is_symlink: bool, force: bool) -> anyhow::Result<bool> {
+ if force {
+ remove_if_exists(dest)?;
+ }
+ // Use symlink_metadata (not exists()) because exists() follows symlinks
+ // and returns false for broken ones.
+ if dest.symlink_metadata().is_ok() {
+ return Ok(false);
+ }
+
+ if is_symlink {
+ let target =
+ fs::read_link(src).with_context(|| format!("reading symlink {}", src.display()))?;
+ create_symlink(&target, src, dest)?;
+ } else {
+ match reflink_copy::reflink_or_copy(src, dest) {
+ Ok(_) => {}
+ Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(false),
+ Err(e) => {
+ return Err(anyhow::Error::from(e).context(format!("copying {}", src.display())));
+ }
+ }
+ }
+ Ok(true)
+}
+
+/// A leaf item (file or symlink) collected during the directory walk.
+struct CopyLeaf {
+ src: PathBuf,
+ dest: PathBuf,
+ is_symlink: bool,
+}
+
+/// Copy a directory tree using reflink (COW) per file.
+///
+/// Walks the tree iteratively (no recursion), then copies all files and
+/// symlinks in parallel on a dedicated 4-thread pool. Non-regular files
+/// (sockets, FIFOs) are silently skipped. Existing entries at the destination
+/// are skipped for idempotent usage.
///
/// When `force` is true, existing files and symlinks at the destination are
/// removed before copying.
///
-/// Uses a dedicated 4-thread pool. Nested calls (recursive directories) skip
-/// pool entry and run inline on the current worker thread.
-pub fn copy_dir_recursive(src: &Path, dest: &Path, force: bool) -> anyhow::Result<()> {
- COPY_POOL.install(|| copy_dir_recursive_inner(src, dest, force))
-}
-
-fn copy_dir_recursive_inner(src: &Path, dest: &Path, force: bool) -> anyhow::Result<()> {
- fs::create_dir_all(dest).with_context(|| format!("creating directory {}", dest.display()))?;
+/// Returns the number of files actually copied (excludes skipped entries).
+pub fn copy_dir_recursive(src: &Path, dest: &Path, force: bool) -> anyhow::Result<usize> {
+ // Phase 1: Walk directories iteratively, creating dest dirs and collecting leaves.
+ let mut leaves = Vec::new();
+ let mut dir_stack = vec![(src.to_path_buf(), dest.to_path_buf())];
+ #[cfg(unix)]
+ let mut dirs_for_perms: Vec<(PathBuf, PathBuf)> = Vec::new();
- let entries: Vec<_> = fs::read_dir(src)?.collect::<Result<Vec<_>, _>>()?;
+ while let Some((src_dir, dest_dir)) = dir_stack.pop() {
+ fs::create_dir_all(&dest_dir)
+ .with_context(|| format!("creating directory {}", dest_dir.display()))?;
+ #[cfg(unix)]
+ dirs_for_perms.push((src_dir.clone(), dest_dir.clone()));
- entries.into_par_iter().try_for_each(|entry| {
- let file_type = entry.file_type()?;
- let src_path = entry.path();
- let dest_path = dest.join(entry.file_name());
+ let entries: Vec<_> = fs::read_dir(&src_dir)?.collect::<Result<Vec<_>, _>>()?;
+ for entry in entries {
+ let file_type = entry.file_type()?;
+ let src_path = entry.path();
+ let dest_path = dest_dir.join(entry.file_name());
- if file_type.is_symlink() {
- if force {
- remove_if_exists(&dest_path)?;
- }
- // Use symlink_metadata to detect broken symlinks (exists() follows symlinks
- // and returns false for broken ones, causing EEXIST on the next symlink call)
- if dest_path.symlink_metadata().is_err() {
- let target = fs::read_link(&src_path)
- .with_context(|| format!("reading symlink {}", src_path.display()))?;
- create_symlink(&target, &src_path, &dest_path)?;
- }
- } else if file_type.is_dir() {
- copy_dir_recursive_inner(&src_path, &dest_path, force)?;
- } else if !file_type.is_file() {
- log::debug!("skipping non-regular file: {}", src_path.display());
- } else {
- if force {
- remove_if_exists(&dest_path)?;
- }
- // Check symlink_metadata (not exists()) because exists() follows symlinks
- // and returns false for broken ones, which would cause reflink_or_copy to
- // fail with ENOENT on some platforms when copying through the broken symlink.
- if dest_path.symlink_metadata().is_err() {
- match reflink_copy::reflink_or_copy(&src_path, &dest_path) {
- Ok(_) => {}
- Err(e) if e.kind() == ErrorKind::AlreadyExists => {}
- Err(e) => {
- return Err(anyhow::Error::from(e)
- .context(format!("copying {}", src_path.display())));
- }
- }
+ if file_type.is_dir() {
+ dir_stack.push((src_path, dest_path));
+ } else if file_type.is_file() || file_type.is_symlink() {
+ leaves.push(CopyLeaf {
+ src: src_path,
+ dest: dest_path,
+ is_symlink: file_type.is_symlink(),
+ });
+ } else {
+ log::debug!("skipping non-regular file: {}", src_path.display());
}
}
- Ok(())
+ }
+
+ // Phase 2: Copy all leaves in parallel.
+ let copied = AtomicUsize::new(0);
+ COPY_POOL.install(|| {
+ leaves
+ .par_iter()
+ .try_for_each(|leaf| -> anyhow::Result<()> {
+ if copy_leaf(&leaf.src, &leaf.dest, leaf.is_symlink, force)? {
+ copied.fetch_add(1, Ordering::Relaxed);
+ }
+ Ok(())
+ })
})?;
- // Preserve source directory permissions AFTER copying contents.
- // Must be done after the loop — if the source lacks write permission (e.g., 0o555),
+ // Phase 3: Preserve source directory permissions AFTER copying contents.
+ // Must be done after copying — if the source lacks write permission (e.g., 0o555),
// setting it before copying would make the destination read-only and fail the copies.
#[cfg(unix)]
- {
- let src_perms = fs::metadata(src)
- .with_context(|| format!("reading permissions for {}", src.display()))?
+ for (src_dir, dest_dir) in &dirs_for_perms {
+ let src_perms = fs::metadata(src_dir)
+ .with_context(|| format!("reading permissions for {}", src_dir.display()))?
.permissions();
- fs::set_permissions(dest, src_perms)
- .with_context(|| format!("setting permissions on {}", dest.display()))?;
+ fs::set_permissions(dest_dir, src_perms)
+ .with_context(|| format!("setting permissions on {}", dest_dir.display()))?;
}
- Ok(())
+ Ok(copied.into_inner())
}
/// Remove a file, ignoring "not found" errors.
-pub fn remove_if_exists(path: &Path) -> anyhow::Result<()> {
+fn remove_if_exists(path: &Path) -> anyhow::Result<()> {
if let Err(e) = fs::remove_file(path) {
anyhow::ensure!(e.kind() == ErrorKind::NotFound, e);
}
@@ -147,7 +168,7 @@ pub fn remove_if_exists(path: &Path) -> anyhow::Result<()> {
}
/// Create a symlink, handling platform differences.
-pub fn create_symlink(target: &Path, src_path: &Path, dest_path: &Path) -> anyhow::Result<()> {
+fn create_symlink(target: &Path, src_path: &Path, dest_path: &Path) -> anyhow::Result<()> {
#[cfg(unix)]
{
let _ = src_path; // Used on Windows to determine symlink type
@@ -172,3 +193,21 @@ pub fn create_symlink(target: &Path, src_path: &Path, dest_path: &Path) -> anyho
}
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_remove_if_exists_nonexistent() {
+ // NotFound is silently ignored
+ assert!(remove_if_exists(Path::new("/nonexistent/file")).is_ok());
+ }
+
+ #[test]
+ fn test_remove_if_exists_not_a_file() {
+ // Trying to remove a directory with remove_file produces a non-NotFound error
+ let dir = std::env::temp_dir();
+ assert!(remove_if_exists(&dir).is_err());
+ }
+}
diff --git a/tests/integration_tests/step_copy_ignored.rs b/tests/integration_tests/step_copy_ignored.rs
index ef6c2c229..d6d29961b 100644
--- a/tests/integration_tests/step_copy_ignored.rs
+++ b/tests/integration_tests/step_copy_ignored.rs
@@ -28,8 +28,8 @@ fn run_copy_ignored(repo: &TestRepo, feature_path: &Path) -> std::process::Outpu
fn run_copy_ignored_single_entry(repo: &TestRepo, feature_path: &Path) {
let output = run_copy_ignored(repo, feature_path);
assert!(
- String::from_utf8_lossy(&output.stderr).contains("Copied 1 entry"),
- "expected one copied entry: {}",
+ String::from_utf8_lossy(&output.stderr).contains("Copied 1 file"),
+ "expected one copied file: {}",
String::from_utf8_lossy(&output.stderr)
);
}
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snap
index 78d2707ff..d719f70ca 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snap
@@ -37,4 +37,4 @@ exit_code: 0
----- stdout -----
----- stderr -----
-✓ Copied 1 entry
+✓ Copied 1 file
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snap
index f92bae2e8..f353b39f9 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snap
@@ -44,4 +44,4 @@ exit_code: 0
----- stdout -----
----- stderr -----
-✓ Copied 2 entries
+✓ Copied 2 files
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snap
index 6410bfeba..4cd223f73 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snap
@@ -45,4 +45,4 @@ exit_code: 0
----- stdout -----
----- stderr -----
-✓ Copied 1 entry
+✓ Copied 1 file
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snap
index 6410bfeba..4cd223f73 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snap
@@ -45,4 +45,4 @@ exit_code: 0
----- stdout -----
----- stderr -----
-✓ Copied 1 entry
+✓ Copied 1 file
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snap
index 3e8b7d126..a5f82c6cd 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snap
@@ -46,4 +46,4 @@ exit_code: 0
----- stdout -----
----- stderr -----
-✓ Copied 1 entry
+✓ Copied 1 file
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snap
index 31ff608c7..7a5abc263 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snap
@@ -44,4 +44,4 @@ exit_code: 0
----- stdout -----
----- stderr -----
-✓ Copied 1 entry
+✓ Copied 1 file
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snap
index 84d101b70..e91cac9af 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snap
@@ -46,4 +46,4 @@ exit_code: 0
----- stdout -----
----- stderr -----
-✓ Copied 1 entry
+✓ Copied 1 file
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snap
index bcdf82e78..d456b94fa 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snap
@@ -47,4 +47,4 @@ exit_code: 0
----- stderr -----
○ Copying 1 entry:
  .env (file)
-✓ Copied 1 entry
+✓ Copied 1 file
diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snap
index e436470d2..85815aac4 100644
--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snap
+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snap
@@ -47,4 +47,4 @@ exit_code: 0
----- stderr -----
○ Copying 1 entry:
  target (dir)
-✓ Copied 1 entry
+✓ Copied 1 file