A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.
Rust
Merge origin/main and preserve scan admission integration
Harold Huntd448eaa1392d
diff --cc crates/disktree-app/src/main.rsindex 591ce5a,761e331..4af9de7--- a/crates/disktree-app/src/main.rs+++ b/crates/disktree-app/src/main.rs@@@ -156,23 -180,25 +186,29 @@@ fn run() -> Result<()>Ok(())}- fn parse_args() -> Result<Args> {- parse_args_from(std::env::args().skip(1))- }-- fn parse_args_from(arguments: impl Iterator<Item = String>) -> Result<Args> {+ /// Read the command line, program name already skipped.+ fn parse_args(+ mut args: impl Iterator<Item = std::ffi::OsString>,+ ) -> Result<Args> {let mut root: Option<PathBuf> = None;let mut options = ScanOptions::default();+ if cfg!(target_os = "macos") {+ options.threads.max_threads = 8;+ options.threads.adaptive = true;+ }let mut depth = 3_u32;let mut disk = false;- let mut args = arguments;+ // `std::env::args` panics on a name that is not Unicode, and a path is+ // any name: a restart as administrator hands the root back exactly as+ // it was, so the caller passes `args_os`.+ let text = |value: Option<std::ffi::OsString>, need: &str| {+ value+ .and_then(|value| value.into_string().ok())+ .with_context(|| need.to_owned())+ };while let Some(arg) = args.next() {- match arg.as_str() {+ match arg.to_str().unwrap_or_default() {"-h" | "--help" => {println!("{USAGE}");std::process::exit(0);@@@ -193,44 -219,8 +229,45 @@@"--depth must be 1 to 6");}+ "--scan-threads" => {- let value =- args.next().context("--scan-threads needs a number")?;++ let value = text(args.next(), "--scan-threads needs a number")?;+ options.threads.max_threads = value+ .parse()+ .context("--scan-threads needs a positive integer")?;+ anyhow::ensure!(+ options.threads.max_threads > 0,+ "--scan-threads must be positive"+ );+ }+ "--fixed-threads" => options.threads.adaptive = false,+ "--thread-throughput-percent" => {- let value = args- .next()- .context("--thread-throughput-percent needs a number")?;++ let value = text(++ args.next(),++ "--thread-throughput-percent needs a number",++ )?;+ let percent: u8 = value+ .parse()+ .context("throughput percent must be 1 to 100")?;+ anyhow::ensure!(+ (1..=100).contains(&percent),+ "throughput percent must be 1 to 100"+ );+ options.threads.retained_throughput =+ f64::from(percent) / 100.0;+ }+ "--thread-system-cpu-percent" => {- let value = args- .next()- .context("--thread-system-cpu-percent needs a number")?;++ let value = text(++ args.next(),++ "--thread-system-cpu-percent needs a number",++ )?;+ let percent: u8 =+ value.parse().context("CPU percent must be 0 to 100")?;+ anyhow::ensure!(percent <= 100, "CPU percent must be 0 to 100");+ options.threads.system_cpu_limit =+ (percent > 0).then(|| f64::from(percent) / 100.0);+ }"--metric" => {- let value = args.next().context("--metric needs a value")?;+ let value = text(args.next(), "--metric needs a value")?;options.metric = match value.as_str() {"files" => disktree_core::tree::Metric::Files,"bytes" | "size" => disktree_core::tree::Metric::Bytes,@@@ -313,60 -303,3 +350,84 @@@ mod console}}}++#[cfg(test)]+mod argument_tests {+ use super::*;+- fn arguments(extra: &[&str]) -> impl Iterator<Item = String> {++ fn arguments(extra: &[&str]) -> impl Iterator<Item = std::ffi::OsString> {+ extra+ .iter()+ .copied()+ .chain(std::iter::once("."))- .map(str::to_owned)++ .map(std::ffi::OsString::from)+ }++ #[test]+ fn adaptive_budget_and_fixed_override_are_explicit() {- let args = parse_args_from(arguments(&[])).expect("defaults");++ let args = parse_args(arguments(&[])).expect("defaults");+ assert_eq!(+ args.options.threads.max_threads,+ if cfg!(target_os = "macos") {+ 8+ } else {+ usize::MAX+ }+ );+ assert_eq!(args.options.threads.adaptive, cfg!(target_os = "macos"));- let args = parse_args_from(arguments(&[++ let args = parse_args(arguments(&[+ "--scan-threads",+ "4",+ "--fixed-threads",+ "--thread-throughput-percent",+ "85",+ "--thread-system-cpu-percent",+ "0",+ ]))+ .expect("custom");+ assert_eq!(args.options.threads.max_threads, 4);+ assert!(!args.options.threads.adaptive);+ assert!(+ (args.options.threads.retained_throughput - 0.85).abs()+ < f64::EPSILON+ );+ assert_eq!(args.options.threads.system_cpu_limit, None);+ }+++ // APFS rejects these filename bytes; Linux filesystems permit them.++ #[cfg(target_os = "linux")]++ #[test]++ fn worker_options_preserve_a_non_unicode_root() {++ use std::os::unix::ffi::OsStringExt as _;++++ let temp = tempfile::TempDir::new().expect("tempdir");++ let root = temp++ .path()++ .join(std::ffi::OsString::from_vec(vec![b'r', 0xff]));++ std::fs::create_dir(&root).expect("directory");++ let args = parse_args(++ [++ std::ffi::OsString::from("--scan-threads"),++ std::ffi::OsString::from("4"),++ root.clone().into_os_string(),++ ]++ .into_iter(),++ )++ .expect("native path");++ assert_eq!(args.root, dunce::canonicalize(root).expect("root"));++ assert_eq!(args.options.threads.max_threads, 4);++ }+++ #[test]+ fn invalid_worker_budgets_are_rejected() {+ for extra in [+ vec!["--scan-threads", "0"],+ vec!["--scan-threads", "-1"],+ vec!["--thread-throughput-percent", "0"],+ vec!["--thread-throughput-percent", "101"],+ vec!["--thread-system-cpu-percent", "101"],+ ] {- assert!(parse_args_from(arguments(&extra)).is_err());++ assert!(parse_args(arguments(&extra)).is_err());+ }+ }+}diff --cc crates/disktree-core/src/lib.rsindex 2ff2297,9d2ee86..b280239--- a/crates/disktree-core/src/lib.rs+++ b/crates/disktree-core/src/lib.rs@@@ -10,9 -10,10 +10,11 @@@ pub mod classifypub mod export;pub mod filter;pub mod insights;+ #[cfg(windows)]+ mod mft;pub mod removal;pub mod scan;+pub mod scan_threads;pub mod size;pub mod space;pub mod tree;diff --cc crates/disktree-core/src/scan.rsindex c3c6bf3,191f280..d62ce5e--- a/crates/disktree-core/src/scan.rs+++ b/crates/disktree-core/src/scan.rs@@@ -28,8 -28,7 +28,8 @@@ use std::threaduse rayon::Scope;use rustc_hash::FxHashSet;+use crate::scan_threads::{Admission, ControllerGuard, ScanThreads};- use crate::tree::{Metric, Node, NodeKind, aggregate};+ use crate::tree::{Metric, Node, NodeKind, Seen, aggregate, aggregate_deduped};/// Errors kept verbatim before the list is truncated; the count keeps rising.const MAX_ERROR_DETAIL: usize = 50;@@@ -120,12 -105,11 +120,15 @@@ pub struct ScanSnapshot}impl ScanProgress {+ pub(crate) fn entries(&self) -> u64 {+ self.files.load(Ordering::Relaxed) + self.dirs.load(Ordering::Relaxed)+ }+- fn count_file(&self, bytes: u64) {- self.files.fetch_add(1, Ordering::Relaxed);+ /// What a directory's listing found (see `Tally`), or what a reader+ /// that measures in bulk has come across so far.+ pub(crate) fn add(&self, files: u64, dirs: u64, bytes: u64) {+ self.files.fetch_add(files, Ordering::Relaxed);+ self.dirs.fetch_add(dirs, Ordering::Relaxed);self.bytes.fetch_add(bytes, Ordering::Relaxed);}@@@ -723,41 -809,14 +835,53 @@@ fn scan_blocking(root: &Path, context:lock(&context.visited_dirs).insert(key);}- let root_dir = Arc::new(PendingDir::new(root.to_path_buf(), None, 0));- let mut threads = context.options.threads.normalized();- threads.max_threads = threads.max_threads.min(rayon::current_num_threads());- context- .progress- .threads- .store(threads.max_threads, Ordering::Relaxed);- context- .progress- .threads_settled- .store(true, Ordering::Relaxed);- context- .progress- .thread_tuning_complete- .store(!threads.adaptive, Ordering::Relaxed);- if !threads.adaptive && threads.max_threads == rayon::current_num_threads()- {- // Pool-sized fixed scans need no admission lock or ready queue. Keep- // existing core callers on Rayon's direct scheduling path.- rayon::scope(|scope| {- dispatch_fixed(scope, &root_dir, context);- });- } else {- let admission = Arc::new(Admission::new(root_dir, threads.max_threads));- // The controller owns no Rayon worker and is joined before finishing- // the tree, where entry throughput no longer describes useful work.- let _controller = ControllerGuard::spawn(- threads,- Arc::clone(&admission),- Arc::clone(&context.progress),- )?;- rayon::scope(|scope| {- dispatch(scope, admission.start(), context, &admission);- });- }+ let root_dir = Arc::new(PendingDir::new(+ root.to_path_buf(),+ file_name(root),+ None,+ 0,+ ));+ let context: &WalkContext = context;- WALK_POOL.install(|| rayon::scope(|scope| walk(scope, &root_dir, context)));++ WALK_POOL.install(|| {++ let mut threads = context.options.threads.normalized();++ threads.max_threads =++ threads.max_threads.min(rayon::current_num_threads());++ context++ .progress++ .threads++ .store(threads.max_threads, Ordering::Relaxed);++ context++ .progress++ .threads_settled++ .store(true, Ordering::Relaxed);++ context++ .progress++ .thread_tuning_complete++ .store(!threads.adaptive, Ordering::Relaxed);++ if !threads.adaptive++ && threads.max_threads == rayon::current_num_threads()++ {++ // Pool-sized fixed scans need no admission lock or ready queue. Keep++ // existing core callers on Rayon's direct scheduling path.++ rayon::scope(|scope| {++ dispatch_fixed(scope, &root_dir, context);++ });++ } else {++ let admission =++ Arc::new(Admission::new(root_dir, threads.max_threads));++ // The controller owns no Rayon worker and is joined before finishing++ // the tree, where entry throughput no longer describes useful work.++ let _controller = ControllerGuard::spawn(++ threads,++ Arc::clone(&admission),++ Arc::clone(&context.progress),++ )?;++ rayon::scope(|scope| {++ dispatch(scope, admission.start(), context, &admission);++ });++ }++ Ok::<(), io::Error>(())++ })?;let node = lock(&context.root).take();let node = node.ok_or_else(|| {@@@ -766,59 -825,74 +890,113 @@@Ok(finish_tree(node, &context.options))}- fn dispatch_fixed(- scope: &Scope<'_>,+ /// The walk's workers, and the file table reader's. Listing directories+ /// contends in the file system, and past some number of threads more of+ /// them only queue: on Windows a home directory of 3.4 million files+ /// walked in 6.5 s on 16 threads and 8.1 s on all 24. That cap was+ /// measured on one 24-thread NTFS laptop; another disk or CPU may want+ /// another. Elsewhere the default count. A pool of its own on every+ /// platform all the same: work the UI hands the global pool, such as+ /// re-ranking a tree, must not queue behind a scan's blocking reads.+ static WALK_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {+ let threads = if cfg!(windows) {+ rayon::current_num_threads().min(16)+ } else {+ rayon::current_num_threads()+ };+ rayon::ThreadPoolBuilder::new()+ .num_threads(threads)+ .thread_name(|index| format!("disktree-walk-{index}"))+ .build()+ .expect("a thread pool")+ });++ /// Entries counted before the shared progress counters are touched. Every+ /// worker adding to the same counters per entry keeps one cache line+ /// bouncing between cores; per directory, or per this many entries in a+ /// wide one so the meter still moves, is invisible to a reader.+ const TALLY_EVERY: u64 = 1024;++ /// What one directory's listing has found so far, for the progress meter.+ #[derive(Default)]+ struct Tally {+ files: u64,+ dirs: u64,+ bytes: u64,+ }++ impl Tally {+ const fn add(&mut self, node: &Node) {+ if node.is_dir() {+ // A subtree taken whole from an earlier scan.+ self.files += node.files;+ self.dirs += node.dirs;+ } else {+ // A link counts as a file on the meter, as it always has,+ // though the tree does not count it.+ self.files += 1;+ }+ self.bytes += node.bytes;+ }++ fn flush(&mut self, progress: &ScanProgress) {+ progress.add(self.files, self.dirs, self.bytes);+ *self = Self::default();+ }+ }++ /// Read one directory, spawn a task per subdirectory, then report completion.+ ///+ /// Flat tasks inside the scope: the worker stack never grows with tree depth.-fn walk<'scope>(++fn dispatch_fixed<'scope>(+ scope: &Scope<'scope>,dir: &Arc<PendingDir>,- context: &Arc<WalkContext>,+ context: &'scope WalkContext,) {+ for child in walk(dir, context) {- let context = Arc::clone(context);+ scope.spawn(move |scope| {- dispatch_fixed(scope, &child, &context);++ dispatch_fixed(scope, &child, context);+ });+ }+}++/// Read one directory, spawn a task per subdirectory, then report completion.+///+/// Flat tasks inside the scope: the worker stack never grows with tree depth.- fn dispatch(- scope: &Scope<'_>,++fn dispatch<'scope>(++ scope: &Scope<'scope>,+ jobs: Vec<Arc<PendingDir>>,- context: &Arc<WalkContext>,++ context: &'scope WalkContext,+ admission: &Arc<Admission<Arc<PendingDir>>>,+) {+ for dir in jobs {- let context = Arc::clone(context);+ let admission = Arc::clone(admission);+ scope.spawn(move |scope| {+ let mut next = Some(dir);+ while let Some(dir) = next.take() {- let children = walk(&dir, &context);++ let children = walk(&dir, context);+ let mut jobs = admission.complete(children);+ // Continue locally into one ready descendant; only surplus+ // jobs are offered to other workers. A running directory is+ // never split, and a retirement returns from this loop.+ next = jobs.pop();- dispatch(scope, jobs, &context, &admission);++ dispatch(scope, jobs, context, &admission);+ }+ });+ }+}+- fn walk(- dir: &Arc<PendingDir>,- context: &Arc<WalkContext>,- ) -> Vec<Arc<PendingDir>> {++fn walk(dir: &Arc<PendingDir>, context: &WalkContext) -> Vec<Arc<PendingDir>> {+ if context.cancelled() {+ // Queued jobs still own a pending token even when no I/O is needed.+ signal_done(dir, context);+ return Vec::new();+ }let mut subdirs: Vec<Arc<PendingDir>> = Vec::new();let mut leaves: Vec<Node> = Vec::new();+ let mut tally = Tally::default();- match list(&dir.path) {+ match list(&dir.path, context.volume.get().copied()) {Ok(entries) => {for entry in entries {if context.cancelled() {@@@ -855,20 -939,30 +1043,32 @@@.options.max_depth.is_none_or(|max_depth| dir.depth < max_depth);+ // Sized once for the leaves and every subdirectory to come, before any+ // subdirectory task can push its node: otherwise that first push+ // doubles a list just sized to the leaves, and the slack stays in the+ // finished tree. The leaves are copied, not moved: the list they grew+ // in carries up to twice their size in slack, and moving it in whole+ // made a home directory walk a second slower (measured, 5 interleaved+ // pairs).+ let spawned = if descend { subdirs.len() } else { 0 };+ {+ let mut partial = lock(&dir.partial);+ partial.children.reserve_exact(leaves.len() + spawned);+ partial.children.extend(leaves);+ }+ let mut children = Vec::new();if descend {for subdir in subdirs {if context.cancelled() {break;}dir.pending.fetch_add(1, Ordering::AcqRel);- scope.spawn(move |scope| walk(scope, &subdir, context));+ children.push(subdir);}}- lock(&dir.children).extend(leaves);signal_done(dir, context);+ children}/// Report that one task for `dir` is done: either its own scan, or one of its@@@ -1113,53 -1195,6 +1301,58 @@@ mod tests})}+ #[test]+ fn fixed_admission_preserves_a_wide_deep_tree() {+ let temp = TempDir::new().expect("tempdir");+ for index in 0..96 {+ write(temp.path(), &format!("d{index}/nested/file"), index + 1);+ }+ for threads in [1, 2, 4, 8, usize::MAX] {+ let mut settings = options();+ settings.threads.max_threads = threads;+ let tree = scan_dir(temp.path(), &settings);+ assert_eq!(tree.files, 96);+ assert_eq!(tree.dirs, 193);+ assert_eq!(tree.bytes, (1..=96).sum::<u64>());+ }+ }++ #[test]+ fn cancelled_queued_jobs_release_parent_completion_tokens() {+ let temp = TempDir::new().expect("tempdir");+ for index in 0..128 {+ write(temp.path(), &format!("d{index}/file"), 1);+ }+ let progress = Arc::new(ScanProgress::default());+ let context = Arc::new(WalkContext {+ known: None,+ options: options(),+ progress: Arc::clone(&progress),+ root_device: Mutex::new(None),+ foreign_mounts: OnceLock::new(),+ never_scanned: OnceLock::new(),+ visited_dirs: Mutex::new(FxHashSet::default()),+ root: Mutex::new(None),++ volume: OnceLock::new(),+ });- let root =- Arc::new(PendingDir::new(temp.path().to_path_buf(), None, 0));++ let root = Arc::new(PendingDir::new(++ temp.path().to_path_buf(),++ file_name(temp.path()),++ None,++ 0,++ ));+ let queued = walk(&root, &context);+ assert_eq!(queued.len(), 128);+ assert!(lock(&context.root).is_none());+ progress.cancel();+ for dir in queued {+ assert!(walk(&dir, &context).is_empty());+ }+ let tree = lock(&context.root).take().expect("root completed");+ assert_eq!(tree.children.len(), 128);+ assert!(tree.children.iter().all(|child| child.children.is_empty()));+ }+#[test]fn totals_are_summed_bottom_up() {let temp = TempDir::new().expect("tempdir");