Skip to content

A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.

Rust

Merge origin/main and preserve scan admission integration

Harold Huntd448eaa1392d
diff --cc crates/disktree-app/src/main.rs
index 591ce5a,761e331..4af9de7
--- a/crates/disktree-app/src/main.rs
+++ b/crates/disktree-app/src/main.rs
@@@ -156,23 -180,25 +186,29 @@@ fn run() -> Result<()>
Ok(())
}
- fn parse_args() -> Result<Args> {
- parse_args_from(std::env::args().skip(1))
- }
-
- fn parse_args_from(arguments: impl Iterator<Item = String>) -> Result<Args> {
+ /// Read the command line, program name already skipped.
+ fn parse_args(
+ mut args: impl Iterator<Item = std::ffi::OsString>,
+ ) -> Result<Args> {
let mut root: Option<PathBuf> = None;
let mut options = ScanOptions::default();
+ if cfg!(target_os = "macos") {
+ options.threads.max_threads = 8;
+ options.threads.adaptive = true;
+ }
let mut depth = 3_u32;
let mut disk = false;
- let mut args = arguments;
+ // `std::env::args` panics on a name that is not Unicode, and a path is
+ // any name: a restart as administrator hands the root back exactly as
+ // it was, so the caller passes `args_os`.
+ let text = |value: Option<std::ffi::OsString>, need: &str| {
+ value
+ .and_then(|value| value.into_string().ok())
+ .with_context(|| need.to_owned())
+ };
while let Some(arg) = args.next() {
- match arg.as_str() {
+ match arg.to_str().unwrap_or_default() {
"-h" | "--help" => {
println!("{USAGE}");
std::process::exit(0);
@@@ -193,44 -219,8 +229,45 @@@
"--depth must be 1 to 6"
);
}
+ "--scan-threads" => {
- let value =
- args.next().context("--scan-threads needs a number")?;
++ let value = text(args.next(), "--scan-threads needs a number")?;
+ options.threads.max_threads = value
+ .parse()
+ .context("--scan-threads needs a positive integer")?;
+ anyhow::ensure!(
+ options.threads.max_threads > 0,
+ "--scan-threads must be positive"
+ );
+ }
+ "--fixed-threads" => options.threads.adaptive = false,
+ "--thread-throughput-percent" => {
- let value = args
- .next()
- .context("--thread-throughput-percent needs a number")?;
++ let value = text(
++ args.next(),
++ "--thread-throughput-percent needs a number",
++ )?;
+ let percent: u8 = value
+ .parse()
+ .context("throughput percent must be 1 to 100")?;
+ anyhow::ensure!(
+ (1..=100).contains(&percent),
+ "throughput percent must be 1 to 100"
+ );
+ options.threads.retained_throughput =
+ f64::from(percent) / 100.0;
+ }
+ "--thread-system-cpu-percent" => {
- let value = args
- .next()
- .context("--thread-system-cpu-percent needs a number")?;
++ let value = text(
++ args.next(),
++ "--thread-system-cpu-percent needs a number",
++ )?;
+ let percent: u8 =
+ value.parse().context("CPU percent must be 0 to 100")?;
+ anyhow::ensure!(percent <= 100, "CPU percent must be 0 to 100");
+ options.threads.system_cpu_limit =
+ (percent > 0).then(|| f64::from(percent) / 100.0);
+ }
"--metric" => {
- let value = args.next().context("--metric needs a value")?;
+ let value = text(args.next(), "--metric needs a value")?;
options.metric = match value.as_str() {
"files" => disktree_core::tree::Metric::Files,
"bytes" | "size" => disktree_core::tree::Metric::Bytes,
@@@ -313,60 -303,3 +350,84 @@@ mod console
}
}
}
+
+#[cfg(test)]
+mod argument_tests {
+ use super::*;
+
- fn arguments(extra: &[&str]) -> impl Iterator<Item = String> {
++ fn arguments(extra: &[&str]) -> impl Iterator<Item = std::ffi::OsString> {
+ extra
+ .iter()
+ .copied()
+ .chain(std::iter::once("."))
- .map(str::to_owned)
++ .map(std::ffi::OsString::from)
+ }
+
+ #[test]
+ fn adaptive_budget_and_fixed_override_are_explicit() {
- let args = parse_args_from(arguments(&[])).expect("defaults");
++ let args = parse_args(arguments(&[])).expect("defaults");
+ assert_eq!(
+ args.options.threads.max_threads,
+ if cfg!(target_os = "macos") {
+ 8
+ } else {
+ usize::MAX
+ }
+ );
+ assert_eq!(args.options.threads.adaptive, cfg!(target_os = "macos"));
- let args = parse_args_from(arguments(&[
++ let args = parse_args(arguments(&[
+ "--scan-threads",
+ "4",
+ "--fixed-threads",
+ "--thread-throughput-percent",
+ "85",
+ "--thread-system-cpu-percent",
+ "0",
+ ]))
+ .expect("custom");
+ assert_eq!(args.options.threads.max_threads, 4);
+ assert!(!args.options.threads.adaptive);
+ assert!(
+ (args.options.threads.retained_throughput - 0.85).abs()
+ < f64::EPSILON
+ );
+ assert_eq!(args.options.threads.system_cpu_limit, None);
+ }
+
++ // APFS rejects these filename bytes; Linux filesystems permit them.
++ #[cfg(target_os = "linux")]
++ #[test]
++ fn worker_options_preserve_a_non_unicode_root() {
++ use std::os::unix::ffi::OsStringExt as _;
++
++ let temp = tempfile::TempDir::new().expect("tempdir");
++ let root = temp
++ .path()
++ .join(std::ffi::OsString::from_vec(vec![b'r', 0xff]));
++ std::fs::create_dir(&root).expect("directory");
++ let args = parse_args(
++ [
++ std::ffi::OsString::from("--scan-threads"),
++ std::ffi::OsString::from("4"),
++ root.clone().into_os_string(),
++ ]
++ .into_iter(),
++ )
++ .expect("native path");
++ assert_eq!(args.root, dunce::canonicalize(root).expect("root"));
++ assert_eq!(args.options.threads.max_threads, 4);
++ }
++
+ #[test]
+ fn invalid_worker_budgets_are_rejected() {
+ for extra in [
+ vec!["--scan-threads", "0"],
+ vec!["--scan-threads", "-1"],
+ vec!["--thread-throughput-percent", "0"],
+ vec!["--thread-throughput-percent", "101"],
+ vec!["--thread-system-cpu-percent", "101"],
+ ] {
- assert!(parse_args_from(arguments(&extra)).is_err());
++ assert!(parse_args(arguments(&extra)).is_err());
+ }
+ }
+}
diff --cc crates/disktree-core/src/lib.rs
index 2ff2297,9d2ee86..b280239
--- a/crates/disktree-core/src/lib.rs
+++ b/crates/disktree-core/src/lib.rs
@@@ -10,9 -10,10 +10,11 @@@ pub mod classify
pub mod export;
pub mod filter;
pub mod insights;
+ #[cfg(windows)]
+ mod mft;
pub mod removal;
pub mod scan;
+pub mod scan_threads;
pub mod size;
pub mod space;
pub mod tree;
diff --cc crates/disktree-core/src/scan.rs
index c3c6bf3,191f280..d62ce5e
--- a/crates/disktree-core/src/scan.rs
+++ b/crates/disktree-core/src/scan.rs
@@@ -28,8 -28,7 +28,8 @@@ use std::thread
use rayon::Scope;
use rustc_hash::FxHashSet;
+use crate::scan_threads::{Admission, ControllerGuard, ScanThreads};
- use crate::tree::{Metric, Node, NodeKind, aggregate};
+ use crate::tree::{Metric, Node, NodeKind, Seen, aggregate, aggregate_deduped};
/// Errors kept verbatim before the list is truncated; the count keeps rising.
const MAX_ERROR_DETAIL: usize = 50;
@@@ -120,12 -105,11 +120,15 @@@ pub struct ScanSnapshot
}
impl ScanProgress {
+ pub(crate) fn entries(&self) -> u64 {
+ self.files.load(Ordering::Relaxed) + self.dirs.load(Ordering::Relaxed)
+ }
+
- fn count_file(&self, bytes: u64) {
- self.files.fetch_add(1, Ordering::Relaxed);
+ /// What a directory's listing found (see `Tally`), or what a reader
+ /// that measures in bulk has come across so far.
+ pub(crate) fn add(&self, files: u64, dirs: u64, bytes: u64) {
+ self.files.fetch_add(files, Ordering::Relaxed);
+ self.dirs.fetch_add(dirs, Ordering::Relaxed);
self.bytes.fetch_add(bytes, Ordering::Relaxed);
}
@@@ -723,41 -809,14 +835,53 @@@ fn scan_blocking(root: &Path, context:
lock(&context.visited_dirs).insert(key);
}
- let root_dir = Arc::new(PendingDir::new(root.to_path_buf(), None, 0));
- let mut threads = context.options.threads.normalized();
- threads.max_threads = threads.max_threads.min(rayon::current_num_threads());
- context
- .progress
- .threads
- .store(threads.max_threads, Ordering::Relaxed);
- context
- .progress
- .threads_settled
- .store(true, Ordering::Relaxed);
- context
- .progress
- .thread_tuning_complete
- .store(!threads.adaptive, Ordering::Relaxed);
- if !threads.adaptive && threads.max_threads == rayon::current_num_threads()
- {
- // Pool-sized fixed scans need no admission lock or ready queue. Keep
- // existing core callers on Rayon's direct scheduling path.
- rayon::scope(|scope| {
- dispatch_fixed(scope, &root_dir, context);
- });
- } else {
- let admission = Arc::new(Admission::new(root_dir, threads.max_threads));
- // The controller owns no Rayon worker and is joined before finishing
- // the tree, where entry throughput no longer describes useful work.
- let _controller = ControllerGuard::spawn(
- threads,
- Arc::clone(&admission),
- Arc::clone(&context.progress),
- )?;
- rayon::scope(|scope| {
- dispatch(scope, admission.start(), context, &admission);
- });
- }
+ let root_dir = Arc::new(PendingDir::new(
+ root.to_path_buf(),
+ file_name(root),
+ None,
+ 0,
+ ));
+ let context: &WalkContext = context;
- WALK_POOL.install(|| rayon::scope(|scope| walk(scope, &root_dir, context)));
++ WALK_POOL.install(|| {
++ let mut threads = context.options.threads.normalized();
++ threads.max_threads =
++ threads.max_threads.min(rayon::current_num_threads());
++ context
++ .progress
++ .threads
++ .store(threads.max_threads, Ordering::Relaxed);
++ context
++ .progress
++ .threads_settled
++ .store(true, Ordering::Relaxed);
++ context
++ .progress
++ .thread_tuning_complete
++ .store(!threads.adaptive, Ordering::Relaxed);
++ if !threads.adaptive
++ && threads.max_threads == rayon::current_num_threads()
++ {
++ // Pool-sized fixed scans need no admission lock or ready queue. Keep
++ // existing core callers on Rayon's direct scheduling path.
++ rayon::scope(|scope| {
++ dispatch_fixed(scope, &root_dir, context);
++ });
++ } else {
++ let admission =
++ Arc::new(Admission::new(root_dir, threads.max_threads));
++ // The controller owns no Rayon worker and is joined before finishing
++ // the tree, where entry throughput no longer describes useful work.
++ let _controller = ControllerGuard::spawn(
++ threads,
++ Arc::clone(&admission),
++ Arc::clone(&context.progress),
++ )?;
++ rayon::scope(|scope| {
++ dispatch(scope, admission.start(), context, &admission);
++ });
++ }
++ Ok::<(), io::Error>(())
++ })?;
let node = lock(&context.root).take();
let node = node.ok_or_else(|| {
@@@ -766,59 -825,74 +890,113 @@@
Ok(finish_tree(node, &context.options))
}
- fn dispatch_fixed(
- scope: &Scope<'_>,
+ /// The walk's workers, and the file table reader's. Listing directories
+ /// contends in the file system, and past some number of threads more of
+ /// them only queue: on Windows a home directory of 3.4 million files
+ /// walked in 6.5 s on 16 threads and 8.1 s on all 24. That cap was
+ /// measured on one 24-thread NTFS laptop; another disk or CPU may want
+ /// another. Elsewhere the default count. A pool of its own on every
+ /// platform all the same: work the UI hands the global pool, such as
+ /// re-ranking a tree, must not queue behind a scan's blocking reads.
+ static WALK_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+ let threads = if cfg!(windows) {
+ rayon::current_num_threads().min(16)
+ } else {
+ rayon::current_num_threads()
+ };
+ rayon::ThreadPoolBuilder::new()
+ .num_threads(threads)
+ .thread_name(|index| format!("disktree-walk-{index}"))
+ .build()
+ .expect("a thread pool")
+ });
+
+ /// Entries counted before the shared progress counters are touched. Every
+ /// worker adding to the same counters per entry keeps one cache line
+ /// bouncing between cores; per directory, or per this many entries in a
+ /// wide one so the meter still moves, is invisible to a reader.
+ const TALLY_EVERY: u64 = 1024;
+
+ /// What one directory's listing has found so far, for the progress meter.
+ #[derive(Default)]
+ struct Tally {
+ files: u64,
+ dirs: u64,
+ bytes: u64,
+ }
+
+ impl Tally {
+ const fn add(&mut self, node: &Node) {
+ if node.is_dir() {
+ // A subtree taken whole from an earlier scan.
+ self.files += node.files;
+ self.dirs += node.dirs;
+ } else {
+ // A link counts as a file on the meter, as it always has,
+ // though the tree does not count it.
+ self.files += 1;
+ }
+ self.bytes += node.bytes;
+ }
+
+ fn flush(&mut self, progress: &ScanProgress) {
+ progress.add(self.files, self.dirs, self.bytes);
+ *self = Self::default();
+ }
+ }
+
+ /// Read one directory, spawn a task per subdirectory, then report completion.
+ ///
+ /// Flat tasks inside the scope: the worker stack never grows with tree depth.
-fn walk<'scope>(
++fn dispatch_fixed<'scope>(
+ scope: &Scope<'scope>,
dir: &Arc<PendingDir>,
- context: &Arc<WalkContext>,
+ context: &'scope WalkContext,
) {
+ for child in walk(dir, context) {
- let context = Arc::clone(context);
+ scope.spawn(move |scope| {
- dispatch_fixed(scope, &child, &context);
++ dispatch_fixed(scope, &child, context);
+ });
+ }
+}
+
+/// Read one directory, spawn a task per subdirectory, then report completion.
+///
+/// Flat tasks inside the scope: the worker stack never grows with tree depth.
- fn dispatch(
- scope: &Scope<'_>,
++fn dispatch<'scope>(
++ scope: &Scope<'scope>,
+ jobs: Vec<Arc<PendingDir>>,
- context: &Arc<WalkContext>,
++ context: &'scope WalkContext,
+ admission: &Arc<Admission<Arc<PendingDir>>>,
+) {
+ for dir in jobs {
- let context = Arc::clone(context);
+ let admission = Arc::clone(admission);
+ scope.spawn(move |scope| {
+ let mut next = Some(dir);
+ while let Some(dir) = next.take() {
- let children = walk(&dir, &context);
++ let children = walk(&dir, context);
+ let mut jobs = admission.complete(children);
+ // Continue locally into one ready descendant; only surplus
+ // jobs are offered to other workers. A running directory is
+ // never split, and a retirement returns from this loop.
+ next = jobs.pop();
- dispatch(scope, jobs, &context, &admission);
++ dispatch(scope, jobs, context, &admission);
+ }
+ });
+ }
+}
+
- fn walk(
- dir: &Arc<PendingDir>,
- context: &Arc<WalkContext>,
- ) -> Vec<Arc<PendingDir>> {
++fn walk(dir: &Arc<PendingDir>, context: &WalkContext) -> Vec<Arc<PendingDir>> {
+ if context.cancelled() {
+ // Queued jobs still own a pending token even when no I/O is needed.
+ signal_done(dir, context);
+ return Vec::new();
+ }
let mut subdirs: Vec<Arc<PendingDir>> = Vec::new();
let mut leaves: Vec<Node> = Vec::new();
+ let mut tally = Tally::default();
- match list(&dir.path) {
+ match list(&dir.path, context.volume.get().copied()) {
Ok(entries) => {
for entry in entries {
if context.cancelled() {
@@@ -855,20 -939,30 +1043,32 @@@
.options
.max_depth
.is_none_or(|max_depth| dir.depth < max_depth);
+ // Sized once for the leaves and every subdirectory to come, before any
+ // subdirectory task can push its node: otherwise that first push
+ // doubles a list just sized to the leaves, and the slack stays in the
+ // finished tree. The leaves are copied, not moved: the list they grew
+ // in carries up to twice their size in slack, and moving it in whole
+ // made a home directory walk a second slower (measured, 5 interleaved
+ // pairs).
+ let spawned = if descend { subdirs.len() } else { 0 };
+ {
+ let mut partial = lock(&dir.partial);
+ partial.children.reserve_exact(leaves.len() + spawned);
+ partial.children.extend(leaves);
+ }
+ let mut children = Vec::new();
if descend {
for subdir in subdirs {
if context.cancelled() {
break;
}
dir.pending.fetch_add(1, Ordering::AcqRel);
- scope.spawn(move |scope| walk(scope, &subdir, context));
+ children.push(subdir);
}
}
- lock(&dir.children).extend(leaves);
signal_done(dir, context);
+ children
}
/// Report that one task for `dir` is done: either its own scan, or one of its
@@@ -1113,53 -1195,6 +1301,58 @@@ mod tests
})
}
+ #[test]
+ fn fixed_admission_preserves_a_wide_deep_tree() {
+ let temp = TempDir::new().expect("tempdir");
+ for index in 0..96 {
+ write(temp.path(), &format!("d{index}/nested/file"), index + 1);
+ }
+ for threads in [1, 2, 4, 8, usize::MAX] {
+ let mut settings = options();
+ settings.threads.max_threads = threads;
+ let tree = scan_dir(temp.path(), &settings);
+ assert_eq!(tree.files, 96);
+ assert_eq!(tree.dirs, 193);
+ assert_eq!(tree.bytes, (1..=96).sum::<u64>());
+ }
+ }
+
+ #[test]
+ fn cancelled_queued_jobs_release_parent_completion_tokens() {
+ let temp = TempDir::new().expect("tempdir");
+ for index in 0..128 {
+ write(temp.path(), &format!("d{index}/file"), 1);
+ }
+ let progress = Arc::new(ScanProgress::default());
+ let context = Arc::new(WalkContext {
+ known: None,
+ options: options(),
+ progress: Arc::clone(&progress),
+ root_device: Mutex::new(None),
+ foreign_mounts: OnceLock::new(),
+ never_scanned: OnceLock::new(),
+ visited_dirs: Mutex::new(FxHashSet::default()),
+ root: Mutex::new(None),
++ volume: OnceLock::new(),
+ });
- let root =
- Arc::new(PendingDir::new(temp.path().to_path_buf(), None, 0));
++ let root = Arc::new(PendingDir::new(
++ temp.path().to_path_buf(),
++ file_name(temp.path()),
++ None,
++ 0,
++ ));
+ let queued = walk(&root, &context);
+ assert_eq!(queued.len(), 128);
+ assert!(lock(&context.root).is_none());
+ progress.cancel();
+ for dir in queued {
+ assert!(walk(&dir, &context).is_empty());
+ }
+ let tree = lock(&context.root).take().expect("root completed");
+ assert_eq!(tree.children.len(), 128);
+ assert!(tree.children.iter().all(|child| child.children.is_empty()));
+ }
+
#[test]
fn totals_are_summed_bottom_up() {
let temp = TempDir::new().expect("tempdir");