Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows
refactor: simplify copy pool to single-level parallelism (#1931)
Maximilian Roosbaec5b12fd7d
diff --git a/src/commands/step_commands.rs b/src/commands/step_commands.rsindex 0f89ae6fa..6c181f001 100644--- a/src/commands/step_commands.rs+++ b/src/commands/step_commands.rs@@ -15,7 +15,6 @@use std::fs;use std::io::ErrorKind;use std::path::{Path, PathBuf};-use std::sync::atomic::{AtomicUsize, Ordering};use std::time::Duration;use anyhow::Context;@@ -24,7 +23,7 @@ use ignore::gitignore::GitignoreBuilder;use rayon::prelude::*;use worktrunk::HookType;use worktrunk::config::{CopyIgnoredConfig, UserConfig};-use worktrunk::copy::{copy_dir_recursive, copy_pool_install, create_symlink, remove_if_exists};+use worktrunk::copy::{copy_dir_recursive, copy_leaf};use worktrunk::git::Repository;use worktrunk::path::format_path_for_display;use worktrunk::shell_exec::Cmd;@@ -770,7 +769,6 @@ pub fn step_copy_ignored(}let verbose = verbosity();- let copied_count = AtomicUsize::new(0);// Show entries in verbose or dry-run modeif verbose >= 1 || dry_run {@@ -800,80 +798,44 @@ pub fn step_copy_ignored(}}- copy_pool_install(|| {- entries_to_copy- .par_iter()- .try_for_each(|(src_entry, is_dir)| -> anyhow::Result<()> {- // Paths from git ls-files are always under source_path- let relative = src_entry- .strip_prefix(&source_path)- .expect("git ls-files path under worktree");- let dest_entry = dest_path.join(relative);-- if *is_dir {- copy_dir_recursive(src_entry, &dest_entry, force).with_context(|| {- format!("copying directory {}", format_path_for_display(relative))- })?;- copied_count.fetch_add(1, Ordering::Relaxed);- } else {- if let Some(parent) = dest_entry.parent() {- fs::create_dir_all(parent).with_context(|| {- format!(- "creating directory for {}",- format_path_for_display(relative)- )- })?;- }- if force {- remove_if_exists(&dest_entry)?;- }- // Check if source is a symlink — preserve it instead of following it- let display_path = format_path_for_display(relative);- let source_is_symlink = fs::symlink_metadata(src_entry)- .context(format!("reading metadata for {display_path}"))?- .file_type()- .is_symlink();- if source_is_symlink {- // Skip existing symlinks for idempotent hook usage- if dest_entry.symlink_metadata().is_err() {- let target = fs::read_link(src_entry)- .context(format!("reading symlink {display_path}"))?;- create_symlink(&target, src_entry, &dest_entry)?;- copied_count.fetch_add(1, Ordering::Relaxed);- }- } else {- // Skip existing entries (files or symlinks) for idempotent hook usage.- // Check symlink_metadata (not exists()) because exists() follows symlinks- // and returns false for broken ones, which would cause reflink_or_copy to- // fail with ENOENT on some platforms when copying through the broken symlink.- if dest_entry.symlink_metadata().is_err() {- match reflink_copy::reflink_or_copy(src_entry, &dest_entry) {- Ok(_) => {- copied_count.fetch_add(1, Ordering::Relaxed);- }- Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}- Err(e) => {- return Err(anyhow::Error::from(e)- .context(format!("copying {display_path}")));- }- }- }- }- }- Ok(())- })- })?;+ let mut copied_count = 0usize;+ for (src_entry, is_dir) in &entries_to_copy {+ let relative = src_entry+ .strip_prefix(&source_path)+ .expect("git ls-files path under worktree");+ let dest_entry = dest_path.join(relative);++ if *is_dir {+ copied_count +=+ copy_dir_recursive(src_entry, &dest_entry, force).with_context(|| {+ format!("copying directory {}", format_path_for_display(relative))+ })?;+ } else {+ if let Some(parent) = dest_entry.parent() {+ fs::create_dir_all(parent).with_context(|| {+ format!(+ "creating directory for {}",+ format_path_for_display(relative)+ )+ })?;+ }+ let is_symlink = fs::symlink_metadata(src_entry)+ .with_context(|| {+ format!("reading metadata for {}", format_path_for_display(relative))+ })?+ .file_type()+ .is_symlink();+ if copy_leaf(src_entry, &dest_entry, is_symlink, force)? {+ copied_count += 1;+ }+ }+ }// Show summary- let copied_count = copied_count.load(Ordering::Relaxed);- let entry_word = if copied_count == 1 {- "entry"- } else {- "entries"- };+ let file_word = if copied_count == 1 { "file" } else { "files" };eprintln!("{}",- success_message(format!("Copied {copied_count} {entry_word}"))+ success_message(format!("Copied {copied_count} {file_word}")));Ok(())@@ -1782,19 +1744,6 @@ pub fn step_relocate(mod tests {use super::*;- #[test]- fn test_remove_if_exists_nonexistent() {- // NotFound is silently ignored- assert!(remove_if_exists(Path::new("/nonexistent/file")).is_ok());- }-- #[test]- fn test_remove_if_exists_not_a_file() {- // Trying to remove a directory with remove_file produces a non-NotFound error- let dir = std::env::temp_dir();- assert!(remove_if_exists(&dir).is_err());- }-#[test]fn test_move_entry_file() {let tmp = tempfile::tempdir().unwrap();diff --git a/src/copy.rs b/src/copy.rsindex 118dffba7..919b2c3f0 100644--- a/src/copy.rs+++ b/src/copy.rs@@ -1,48 +1,35 @@-//! Recursive directory copying with reflink (COW) and rayon parallelism.+//! Directory copying with reflink (COW) and rayon parallelism.//!//! Copies directory trees file-by-file using `reflink_or_copy` which uses//! copy-on-write clones where the filesystem supports them (APFS, btrfs, XFS),//! falling back to regular copies otherwise.//!-//! All copy I/O runs in a dedicated 4-thread pool rather than the global rayon+//! All copy I/O runs on a dedicated 4-thread pool rather than the global rayon//! pool (which is sized at 2× CPU cores for network I/O) to avoid saturating-//! the CPU on a background operation.+//! the CPU on a background operation. Directory trees are walked iteratively+//! (no recursion) then copied in a single parallel pass.//!//! Callers that want low-priority I/O (e.g. `step_copy_ignored`) should call//! [`lower_process_priority`] before starting work.use std::fs;use std::io::ErrorKind;-use std::path::Path;+use std::path::{Path, PathBuf};use std::sync::LazyLock;+use std::sync::atomic::{AtomicUsize, Ordering};use anyhow::Context;use rayon::prelude::*;/// Capped at 4 threads to avoid saturating the CPU — the global rayon pool is/// much larger (2× CPU cores, tuned for network I/O in `wt list`).-///-/// The 8 MiB stack matches the default on Linux/macOS; Windows defaults to-/// ~2 MiB, which can overflow under concurrent reflink/copy work (observed as-/// a worker-thread stack overflow in `test_copy_ignored_many_directories_no_emfile`-/// on Windows CI).static COPY_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {rayon::ThreadPoolBuilder::new().num_threads(4)- .stack_size(8 * 1024 * 1024).build().expect("failed to build copy thread pool")});-/// Run a closure on the dedicated copy thread pool.-///-/// Use this to wrap outer `par_iter` loops that call [`copy_dir_recursive`],-/// so all copy parallelism shares the same 4 threads instead of blocking-/// global-pool threads while waiting for copy workers.-pub fn copy_pool_install<R: Send>(f: impl FnOnce() -> R + Send) -> R {- COPY_POOL.install(f)-}-/// Lower the current process's scheduling priority so copy I/O doesn't/// compete with interactive foreground work.///@@ -62,84 +49,118 @@ pub fn lower_process_priority() {}}-/// Copy a directory tree recursively using reflink (COW) per file.+/// Copy a single file or symlink, using reflink (COW) when possible.///-/// Handles regular files, directories, and symlinks. Non-regular files (sockets,-/// FIFOs) are silently skipped. Existing entries at the destination are skipped-/// for idempotent usage.+/// Returns `true` if the entry was copied, `false` if skipped (destination+/// already exists). When `force` is true, existing entries are removed before+/// copying.+pub fn copy_leaf(src: &Path, dest: &Path, is_symlink: bool, force: bool) -> anyhow::Result<bool> {+ if force {+ remove_if_exists(dest)?;+ }+ // Use symlink_metadata (not exists()) because exists() follows symlinks+ // and returns false for broken ones.+ if dest.symlink_metadata().is_ok() {+ return Ok(false);+ }++ if is_symlink {+ let target =+ fs::read_link(src).with_context(|| format!("reading symlink {}", src.display()))?;+ create_symlink(&target, src, dest)?;+ } else {+ match reflink_copy::reflink_or_copy(src, dest) {+ Ok(_) => {}+ Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(false),+ Err(e) => {+ return Err(anyhow::Error::from(e).context(format!("copying {}", src.display())));+ }+ }+ }+ Ok(true)+}++/// A leaf item (file or symlink) collected during the directory walk.+struct CopyLeaf {+ src: PathBuf,+ dest: PathBuf,+ is_symlink: bool,+}++/// Copy a directory tree using reflink (COW) per file.+///+/// Walks the tree iteratively (no recursion), then copies all files and+/// symlinks in parallel on a dedicated 4-thread pool. Non-regular files+/// (sockets, FIFOs) are silently skipped. Existing entries at the destination+/// are skipped for idempotent usage.////// When `force` is true, existing files and symlinks at the destination are/// removed before copying.///-/// Uses a dedicated 4-thread pool. Nested calls (recursive directories) skip-/// pool entry and run inline on the current worker thread.-pub fn copy_dir_recursive(src: &Path, dest: &Path, force: bool) -> anyhow::Result<()> {- COPY_POOL.install(|| copy_dir_recursive_inner(src, dest, force))-}--fn copy_dir_recursive_inner(src: &Path, dest: &Path, force: bool) -> anyhow::Result<()> {- fs::create_dir_all(dest).with_context(|| format!("creating directory {}", dest.display()))?;+/// Returns the number of files actually copied (excludes skipped entries).+pub fn copy_dir_recursive(src: &Path, dest: &Path, force: bool) -> anyhow::Result<usize> {+ // Phase 1: Walk directories iteratively, creating dest dirs and collecting leaves.+ let mut leaves = Vec::new();+ let mut dir_stack = vec![(src.to_path_buf(), dest.to_path_buf())];+ #[cfg(unix)]+ let mut dirs_for_perms: Vec<(PathBuf, PathBuf)> = Vec::new();- let entries: Vec<_> = fs::read_dir(src)?.collect::<Result<Vec<_>, _>>()?;+ while let Some((src_dir, dest_dir)) = dir_stack.pop() {+ fs::create_dir_all(&dest_dir)+ .with_context(|| format!("creating directory {}", dest_dir.display()))?;+ #[cfg(unix)]+ dirs_for_perms.push((src_dir.clone(), dest_dir.clone()));- entries.into_par_iter().try_for_each(|entry| {- let file_type = entry.file_type()?;- let src_path = entry.path();- let dest_path = dest.join(entry.file_name());+ let entries: Vec<_> = fs::read_dir(&src_dir)?.collect::<Result<Vec<_>, _>>()?;+ for entry in entries {+ let file_type = entry.file_type()?;+ let src_path = entry.path();+ let dest_path = dest_dir.join(entry.file_name());- if file_type.is_symlink() {- if force {- remove_if_exists(&dest_path)?;- }- // Use symlink_metadata to detect broken symlinks (exists() follows symlinks- // and returns false for broken ones, causing EEXIST on the next symlink call)- if dest_path.symlink_metadata().is_err() {- let target = fs::read_link(&src_path)- .with_context(|| format!("reading symlink {}", src_path.display()))?;- create_symlink(&target, &src_path, &dest_path)?;- }- } else if file_type.is_dir() {- copy_dir_recursive_inner(&src_path, &dest_path, force)?;- } else if !file_type.is_file() {- log::debug!("skipping non-regular file: {}", src_path.display());- } else {- if force {- remove_if_exists(&dest_path)?;- }- // Check symlink_metadata (not exists()) because exists() follows symlinks- // and returns false for broken ones, which would cause reflink_or_copy to- // fail with ENOENT on some platforms when copying through the broken symlink.- if dest_path.symlink_metadata().is_err() {- match reflink_copy::reflink_or_copy(&src_path, &dest_path) {- Ok(_) => {}- Err(e) if e.kind() == ErrorKind::AlreadyExists => {}- Err(e) => {- return Err(anyhow::Error::from(e)- .context(format!("copying {}", src_path.display())));- }- }+ if file_type.is_dir() {+ dir_stack.push((src_path, dest_path));+ } else if file_type.is_file() || file_type.is_symlink() {+ leaves.push(CopyLeaf {+ src: src_path,+ dest: dest_path,+ is_symlink: file_type.is_symlink(),+ });+ } else {+ log::debug!("skipping non-regular file: {}", src_path.display());}}- Ok(())+ }++ // Phase 2: Copy all leaves in parallel.+ let copied = AtomicUsize::new(0);+ COPY_POOL.install(|| {+ leaves+ .par_iter()+ .try_for_each(|leaf| -> anyhow::Result<()> {+ if copy_leaf(&leaf.src, &leaf.dest, leaf.is_symlink, force)? {+ copied.fetch_add(1, Ordering::Relaxed);+ }+ Ok(())+ })})?;- // Preserve source directory permissions AFTER copying contents.- // Must be done after the loop — if the source lacks write permission (e.g., 0o555),+ // Phase 3: Preserve source directory permissions AFTER copying contents.+ // Must be done after copying — if the source lacks write permission (e.g., 0o555),// setting it before copying would make the destination read-only and fail the copies.#[cfg(unix)]- {- let src_perms = fs::metadata(src)- .with_context(|| format!("reading permissions for {}", src.display()))?+ for (src_dir, dest_dir) in &dirs_for_perms {+ let src_perms = fs::metadata(src_dir)+ .with_context(|| format!("reading permissions for {}", src_dir.display()))?.permissions();- fs::set_permissions(dest, src_perms)- .with_context(|| format!("setting permissions on {}", dest.display()))?;+ fs::set_permissions(dest_dir, src_perms)+ .with_context(|| format!("setting permissions on {}", dest_dir.display()))?;}- Ok(())+ Ok(copied.into_inner())}/// Remove a file, ignoring "not found" errors.-pub fn remove_if_exists(path: &Path) -> anyhow::Result<()> {+fn remove_if_exists(path: &Path) -> anyhow::Result<()> {if let Err(e) = fs::remove_file(path) {anyhow::ensure!(e.kind() == ErrorKind::NotFound, e);}@@ -147,7 +168,7 @@ pub fn remove_if_exists(path: &Path) -> anyhow::Result<()> {}/// Create a symlink, handling platform differences.-pub fn create_symlink(target: &Path, src_path: &Path, dest_path: &Path) -> anyhow::Result<()> {+fn create_symlink(target: &Path, src_path: &Path, dest_path: &Path) -> anyhow::Result<()> {#[cfg(unix)]{let _ = src_path; // Used on Windows to determine symlink type@@ -172,3 +193,21 @@ pub fn create_symlink(target: &Path, src_path: &Path, dest_path: &Path) -> anyho}Ok(())}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_remove_if_exists_nonexistent() {+ // NotFound is silently ignored+ assert!(remove_if_exists(Path::new("/nonexistent/file")).is_ok());+ }++ #[test]+ fn test_remove_if_exists_not_a_file() {+ // Trying to remove a directory with remove_file produces a non-NotFound error+ let dir = std::env::temp_dir();+ assert!(remove_if_exists(&dir).is_err());+ }+}diff --git a/tests/integration_tests/step_copy_ignored.rs b/tests/integration_tests/step_copy_ignored.rsindex ef6c2c229..d6d29961b 100644--- a/tests/integration_tests/step_copy_ignored.rs+++ b/tests/integration_tests/step_copy_ignored.rs@@ -28,8 +28,8 @@ fn run_copy_ignored(repo: &TestRepo, feature_path: &Path) -> std::process::Outpufn run_copy_ignored_single_entry(repo: &TestRepo, feature_path: &Path) {let output = run_copy_ignored(repo, feature_path);assert!(- String::from_utf8_lossy(&output.stderr).contains("Copied 1 entry"),- "expected one copied entry: {}",+ String::from_utf8_lossy(&output.stderr).contains("Copied 1 file"),+ "expected one copied file: {}",String::from_utf8_lossy(&output.stderr));}diff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snapindex 78d2707ff..d719f70ca 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_bare_repo.snap@@ -37,4 +37,4 @@ exit_code: 0----- stdout ---------- stderr ------✓ Copied 1 entry+✓ Copied 1 filediff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snapindex f92bae2e8..f353b39f9 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_default_copies_all.snap@@ -44,4 +44,4 @@ exit_code: 0----- stdout ---------- stderr ------[32m✓[39m [32mCopied 2 entries[39m+[32m✓[39m [32mCopied 2 files[39mdiff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snapindex 6410bfeba..4cd223f73 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_no_existing.snap@@ -45,4 +45,4 @@ exit_code: 0----- stdout ---------- stderr ------[32m✓[39m [32mCopied 1 entry[39m+[32m✓[39m [32mCopied 1 file[39mdiff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snapindex 6410bfeba..4cd223f73 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_force_overwrites.snap@@ -45,4 +45,4 @@ exit_code: 0----- stdout ---------- stderr ------[32m✓[39m [32mCopied 1 entry[39m+[32m✓[39m [32mCopied 1 file[39mdiff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snapindex 3e8b7d126..a5f82c6cd 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_from_flag.snap@@ -46,4 +46,4 @@ exit_code: 0----- stdout ---------- stderr ------[32m✓[39m [32mCopied 1 entry[39m+[32m✓[39m [32mCopied 1 file[39mdiff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snapindex 31ff608c7..7a5abc263 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_skips_nested_worktrees.snap@@ -44,4 +44,4 @@ exit_code: 0----- stdout ---------- stderr ------[32m✓[39m [32mCopied 1 entry[39m+[32m✓[39m [32mCopied 1 file[39mdiff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snapindex 84d101b70..e91cac9af 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_to_flag.snap@@ -46,4 +46,4 @@ exit_code: 0----- stdout ---------- stderr ------[32m✓[39m [32mCopied 1 entry[39m+[32m✓[39m [32mCopied 1 file[39mdiff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snapindex bcdf82e78..d456b94fa 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose.snap@@ -47,4 +47,4 @@ exit_code: 0----- stderr -----[2m○[22m Copying 1 entry:[107m [0m .env (file)-[32m✓[39m [32mCopied 1 entry[39m+[32m✓[39m [32mCopied 1 file[39mdiff --git a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snap b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snapindex e436470d2..85815aac4 100644--- a/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snap+++ b/tests/snapshots/integration__integration_tests__step_copy_ignored__copy_ignored_verbose_directory.snap@@ -47,4 +47,4 @@ exit_code: 0----- stderr -----[2m○[22m Copying 1 entry:[107m [0m target (dir)-[32m✓[39m [32mCopied 1 entry[39m+[32m✓[39m [32mCopied 1 file[39m