A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.
Rust
core: memoized widening and name filtering
Tobi Lutke23b9e5d21b88
diff --git a/crates/disktree-core/src/filter.rs b/crates/disktree-core/src/filter.rsnew file mode 100644index 0000000..24c1a7a--- /dev/null+++ b/crates/disktree-core/src/filter.rs@@ -0,0 +1,249 @@+//! Typeahead filtering: which parts of a tree match a name.+//!+//! A node whose name contains the needle (ignoring ASCII case) matches, and+//! is kept whole: everything under a matching directory goes with it. Its+//! ancestors are kept only partly, and sized by what matched beneath them,+//! so a filtered treemap shows exactly the matches, at their true relative+//! sizes, in the places they live.++use rustc_hash::FxHashMap;++use crate::tree::{Metric, Node};++/// How a node takes part in a filtered view.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Keep {+ /// It matches: drawn as usual, with everything beneath it.+ Whole,+ /// It holds matches: drawn with only those, at their size.+ Partial { bytes: u64, files: u64 },+}++/// The outcome of filtering a subtree by name.+#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct Matches {+ /// The needle, lowercased.+ pub needle: String,+ /// Absolute crumbs of the subtree that was searched.+ pub base: Vec<usize>,+ /// Keyed by absolute crumbs. Only matches and their ancestors appear;+ /// a match's own descendants are implied.+ pub keep: FxHashMap<Vec<usize>, Keep>,+ /// Topmost matches: a match inside a match is not counted again.+ pub count: usize,+ pub bytes: u64,+ pub files: u64,+}++impl Matches {+ /// How the node at `crumbs` takes part: `None` when it is filtered out.+ /// Anything outside the searched subtree, and anything beneath a match,+ /// is kept whole.+ pub fn keep(&self, crumbs: &[usize]) -> Option<Keep> {+ if !crumbs.starts_with(&self.base) {+ return Some(Keep::Whole);+ }+ for length in self.base.len()..=crumbs.len() {+ match self.keep.get(&crumbs[..length]) {+ Some(Keep::Whole) => return Some(Keep::Whole),+ Some(partial) if length == crumbs.len() => {+ return Some(*partial);+ }+ // Only the base may be absent: it holds the matches but is+ // not recorded, being where the search started.+ None if length > self.base.len() => return None,+ _ => {}+ }+ }+ // The base itself holds the matches.+ Some(Keep::Partial {+ bytes: self.bytes,+ files: self.files,+ })+ }++ /// The value a kept node is laid out by.+ pub const fn value(keep: Keep, node: &Node, metric: Metric) -> u64 {+ match (keep, metric) {+ (Keep::Whole, _) => node.value(metric),+ (Keep::Partial { bytes, .. }, Metric::Bytes) => bytes,+ (Keep::Partial { files, .. }, Metric::Files) => files,+ }+ }+}++/// Search `node`, found at absolute `base`, for names containing `needle`.+/// `None` for an empty needle: nothing is filtered.+pub fn filter(node: &Node, base: &[usize], needle: &str) -> Option<Matches> {+ let needle = needle.trim().to_ascii_lowercase();+ if needle.is_empty() {+ return None;+ }+ let mut matches = Matches {+ needle,+ base: base.to_vec(),+ ..Matches::default()+ };+ let mut crumbs = base.to_vec();+ let (bytes, files) = visit(node, &mut crumbs, &mut matches);+ matches.bytes = bytes;+ matches.files = files;+ Some(matches)+}++/// Returns the bytes and files that matched at or beneath `node`'s+/// children, recording what to keep.+fn visit(+ node: &Node,+ crumbs: &mut Vec<usize>,+ matches: &mut Matches,+) -> (u64, u64) {+ let mut total = (0, 0);+ for (index, child) in node.children.iter().enumerate() {+ crumbs.push(index);+ if contains_ignoring_case(&child.name, &matches.needle) {+ matches.keep.insert(crumbs.clone(), Keep::Whole);+ matches.count += 1;+ total.0 += child.bytes;+ total.1 += child.files;+ } else if !child.children.is_empty() {+ let (bytes, files) = visit(child, crumbs, matches);+ if bytes > 0 || files > 0 {+ matches+ .keep+ .insert(crumbs.clone(), Keep::Partial { bytes, files });+ total.0 += bytes;+ total.1 += files;+ }+ }+ crumbs.pop();+ }+ total+}++/// Substring search ignoring ASCII case, without allocating: a filter runs+/// over every name in view on every keystroke.+fn contains_ignoring_case(haystack: &str, lower_needle: &str) -> bool {+ let (hay, needle) = (haystack.as_bytes(), lower_needle.as_bytes());+ if needle.is_empty() {+ return true;+ }+ if needle.len() > hay.len() {+ return false;+ }+ hay.windows(needle.len()).any(|window| {+ window+ .iter()+ .zip(needle)+ .all(|(left, right)| left.to_ascii_lowercase() == *right)+ })+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::tree::{NodeKind, aggregate};++ fn file(name: &str, bytes: u64) -> Node {+ Node::entry(name, NodeKind::File, bytes)+ }++ fn dir(name: &str, children: Vec<Node>) -> Node {+ let mut node = Node::directory(name);+ node.children = children;+ node+ }++ fn tree() -> Node {+ let mut root = dir(+ "root",+ vec![+ dir(+ "src",+ vec![+ dir("App", vec![file("main.rs", 10)]),+ file("notes", 5),+ ],+ ),+ dir(+ "apps",+ vec![file("x", 100), dir("apple", vec![file("y", 7)])],+ ),+ file("readme", 1),+ ],+ );+ aggregate(&mut root, Metric::Bytes);+ root+ }++ fn crumbs_of(root: &Node, names: &[&str]) -> Vec<usize> {+ let mut node = root;+ names+ .iter()+ .map(|name| {+ let index = node+ .children+ .iter()+ .position(|child| &*child.name == *name)+ .expect("present");+ node = &node.children[index];+ index+ })+ .collect()+ }++ #[test]+ fn matches_are_kept_whole_and_their_ancestors_by_what_matched() {+ let root = tree();+ let found = filter(&root, &[], "APP").expect("a needle");+ assert_eq!(found.count, 2, "src/App and apps; apple is inside apps");+ assert_eq!(found.bytes, 10 + 107);+ assert_eq!(found.keep(&crumbs_of(&root, &["apps"])), Some(Keep::Whole));+ assert_eq!(+ found.keep(&crumbs_of(&root, &["apps", "x"])),+ Some(Keep::Whole),+ "inside a match"+ );+ assert_eq!(+ found.keep(&crumbs_of(&root, &["src"])),+ Some(Keep::Partial {+ bytes: 10,+ files: 1+ })+ );+ assert_eq!(found.keep(&crumbs_of(&root, &["src", "notes"])), None);+ assert_eq!(found.keep(&crumbs_of(&root, &["readme"])), None);+ }++ #[test]+ fn a_search_below_the_root_leaves_the_rest_alone() {+ let root = tree();+ let src = crumbs_of(&root, &["src"]);+ let found = filter(root.resolve(&src).expect("src"), &src, "main")+ .expect("needle");+ assert_eq!(found.keep(&crumbs_of(&root, &["apps"])), Some(Keep::Whole));+ assert_eq!(found.keep(&crumbs_of(&root, &["src", "notes"])), None);+ assert_eq!(+ found.keep(&src),+ Some(Keep::Partial {+ bytes: 10,+ files: 1+ })+ );+ }++ #[test]+ fn an_empty_needle_filters_nothing() {+ assert!(filter(&tree(), &[], " ").is_none());+ let found = filter(&tree(), &[], "zzz").expect("needle");+ assert_eq!(found.count, 0);+ assert!(found.keep.is_empty());+ }++ #[test]+ fn case_is_ignored_without_allocating() {+ assert!(contains_ignoring_case("Cargo.TOML", "toml"));+ assert!(!contains_ignoring_case("ab", "abc"));+ assert!(contains_ignoring_case("x", ""));+ }+}diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rsindex 27a8b55..d7cdb24 100644--- a/crates/disktree-core/src/lib.rs+++ b/crates/disktree-core/src/lib.rs@@ -6,6 +6,7 @@//! be built and tested without GPUI, a display, or a GPU.pub mod classify;+pub mod filter;pub mod insights;pub mod removal;pub mod scan;diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rsindex 8cdec56..be5f2ff 100644--- a/crates/disktree-core/src/scan.rs+++ b/crates/disktree-core/src/scan.rs@@ -156,11 +156,31 @@ pub struct ScanHandle {result: Receiver<io::Result<Node>>,}+/// A subtree already measured, reused by a wider scan instead of walked+/// again: widening from `~` to `/` only reads what is outside `~`.+#[derive(Clone, Debug)]+pub struct Known {+ /// Where it is. Compared with the walk's own paths, so give it in the+ /// same form as the root (both canonical).+ pub path: PathBuf,+ pub tree: Arc<Node>,+}+impl ScanHandle {/// Start walking `root` on a worker thread.pub fn spawn(root: PathBuf, options: ScanOptions) -> Self {+ Self::spawn_with(root, options, None)+ }++ /// Start walking `root`, reusing `known` where the walk reaches it.+ pub fn spawn_with(+ root: PathBuf,+ options: ScanOptions,+ known: Option<Known>,+ ) -> Self {let progress = Arc::new(ScanProgress::default());let context = Arc::new(WalkContext {+ known,options,progress: Arc::clone(&progress),root_device: Mutex::new(None),@@ -208,6 +228,7 @@ impl ScanHandle {pub fn scan(root: &Path, options: ScanOptions) -> io::Result<Node> {let progress = Arc::new(ScanProgress::default());let context = Arc::new(WalkContext {+ known: None,options,progress: Arc::clone(&progress),root_device: Mutex::new(None),@@ -221,6 +242,8 @@ pub fn scan(root: &Path, options: ScanOptions) -> io::Result<Node> {}struct WalkContext {+ /// A subtree to reuse rather than walk.+ known: Option<Known>,options: ScanOptions,progress: Arc<ScanProgress>,/// Device of the root, resolved once: the `one_filesystem` fallback@@ -274,6 +297,20 @@ impl WalkContext {}if file_type.is_dir() {+ // Memoized: the subtree a narrower scan already measured is+ // taken whole, before any volume rule, since it was measured+ // under the same rules.+ if let Some(known) = &self.known+ && known.path == path+ {+ let tree = (*known.tree).clone();+ self.progress.files.fetch_add(tree.files, Ordering::Relaxed);+ self.progress.bytes.fetch_add(tree.bytes, Ordering::Relaxed);+ self.progress.dirs.fetch_add(tree.dirs, Ordering::Relaxed);+ let mut tree = tree;+ tree.name = name;+ return Classified::Entry(tree);+ }if self.options.one_filesystem&& let Some(foreign) = self.foreign_mounts.get(){@@ -906,6 +943,7 @@ mod tests {let readable = fs::read_dir(&locked).map(|mut it| it.next().is_some());let progress = Arc::new(ScanProgress::default());let context = Arc::new(WalkContext {+ known: None,options: options(),progress: Arc::clone(&progress),root_device: Mutex::new(None),@@ -958,6 +996,44 @@ mod tests {assert_eq!(snapshot.errors, 0);}+ #[test]+ fn a_wider_scan_reuses_the_subtree_it_already_knows() {+ let temp = tempfile::tempdir().expect("tempdir");+ let root = temp.path().canonicalize().expect("canonical");+ let inner = root.join("inner");+ fs::create_dir_all(inner.join("deep")).expect("mkdir");+ fs::write(inner.join("deep/a.bin"), vec![0_u8; 4096]).expect("write");+ fs::write(root.join("outside.bin"), vec![0_u8; 8192]).expect("write");+ let known = scan(&inner, options()).expect("inner scan");++ // Changed on disk after the inner scan: a memoized subtree must not+ // see it, which is how this test knows the walk skipped it.+ fs::write(inner.join("deep/b.bin"), vec![0_u8; 4096]).expect("write");++ let handle = ScanHandle::spawn_with(+ root,+ options(),+ Some(Known {+ path: inner,+ tree: Arc::new(known.clone()),+ }),+ );+ let tree = loop {+ if let Some(outcome) = handle.poll() {+ break outcome.expect("wide scan");+ }+ thread::sleep(std::time::Duration::from_millis(2));+ };+ let reused = tree.child_named("inner").expect("inner is in the tree");+ assert_eq!(reused.files, known.files, "b.bin was never read");+ assert_eq!(reused.bytes, known.bytes);+ assert!(+ tree.child_named("outside.bin").is_some(),+ "the rest was walked"+ );+ assert_eq!(tree.files, known.files + 1);+ }+/// A real whole-disk scan, run by hand: `cargo test -p disktree-core/// -- --ignored --nocapture whole_disk`. Prints what it found, so the/// volume rules can be checked against this machine's mounts.@@ -984,5 +1060,16 @@ mod tests {let skipped =crate::space::foreign_mounts_for(&root).unwrap_or_default();println!("left out: {skipped:?}");+ for needle in ["c", "cache", "node_modules", "zzzz"] {+ let started = std::time::Instant::now();+ let found =+ crate::filter::filter(&tree, &[], needle).expect("needle");+ println!(+ "filter {needle:?}: {} matches, {} in {:.1?}",+ found.count,+ crate::size::human_bytes(found.bytes),+ started.elapsed()+ );+ }}}diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rsindex 1ad9ae2..997882f 100644--- a/crates/disktree-core/src/tree.rs+++ b/crates/disktree-core/src/tree.rs@@ -144,6 +144,11 @@ impl Node {}}+ /// The child with this name, if there is one.+ pub fn child_named(&self, name: &str) -> Option<&Self> {+ self.children.iter().find(|child| &*child.name == name)+ }+pub fn child(&self, index: usize) -> Option<&Self> {self.children.get(index)}diff --git a/crates/disktree-core/src/treemap.rs b/crates/disktree-core/src/treemap.rsindex 6dfa020..598b559 100644--- a/crates/disktree-core/src/treemap.rs+++ b/crates/disktree-core/src/treemap.rs@@ -10,6 +10,7 @@//! view applies its own pan and zoom when painting, so re-layout is only needed//! when the tree, the viewport size, or the requested depth changes.+use crate::filter::{Keep, Matches};use crate::tree::{Metric, Node};/// An axis-aligned rectangle in viewport pixels.@@ -149,22 +150,49 @@ pub fn layout(area: Rect,metric: Metric,options: &LayoutOptions,+) -> Vec<Tile> {+ layout_filtered(root, root_crumbs, area, metric, options, None)+}++/// [`layout`], showing only what `filter` keeps: its matches, at the size+/// of what matched, inside ancestors sized the same way.+pub fn layout_filtered(+ root: &Node,+ root_crumbs: &[usize],+ area: Rect,+ metric: Metric,+ options: &LayoutOptions,+ filter: Option<&Matches>,) -> Vec<Tile> {let mut tiles = Vec::new();let mut crumbs = root_crumbs.to_vec();- place_children(root, area, metric, options, 0, &mut crumbs, &mut tiles);+ let place = Placement {+ metric,+ options,+ filter,+ };+ place_children(root, area, &place, 0, &mut crumbs, &mut tiles);tiles}+/// What stays the same for every level of one layout.+struct Placement<'a> {+ metric: Metric,+ options: &'a LayoutOptions,+ /// Set while the level being placed is inside a filtered region; a+ /// match clears it for everything beneath.+ filter: Option<&'a Matches>,+}+fn place_children(node: &Node,area: Rect,- metric: Metric,- options: &LayoutOptions,+ place: &Placement<'_>,depth: u32,crumbs: &mut Vec<usize>,out: &mut Vec<Tile>,) {+ let (metric, options) = (place.metric, place.options);if node.children.is_empty() || area.w <= 0.0 || area.h <= 0.0 {return;}@@ -175,7 +203,18 @@ fn place_children(.children.iter().enumerate()- .map(|(index, child)| (index, child.value(metric) as f64))+ .filter_map(|(index, child)| {+ let value = match place.filter {+ None => child.value(metric),+ Some(filter) => {+ crumbs.push(index);+ let keep = filter.keep(crumbs);+ crumbs.pop();+ Matches::value(keep?, child, metric)+ }+ };+ Some((index, value as f64))+ }).filter(|(_, value)| *value > 0.0).collect();if ranked.is_empty() {@@ -243,15 +282,14 @@ fn place_children(rect.w,rect.bottom() - header.bottom(),);- place_children(- child,- body,- metric,- options,- depth + 1,- crumbs,- out,- );+ // Beneath a match everything is shown; above one, only matches.+ let inner = Placement {+ filter: place+ .filter+ .filter(|filter| filter.keep(crumbs) != Some(Keep::Whole)),+ ..*place+ };+ place_children(child, body, &inner, depth + 1, crumbs, out);}crumbs.pop();}@@ -627,4 +665,61 @@ mod tests {assert_eq!(found.crumbs(), &[0, 0]);assert!(hit(&tiles, -50.0, -50.0).is_none());}++ #[test]+ fn a_filtered_layout_shows_only_the_matches_at_their_size() {+ use crate::filter::filter;+ use crate::tree::{NodeKind, aggregate};++ let file = |name: &str, bytes| Node::entry(name, NodeKind::File, bytes);+ let mut src = Node::directory("src");+ src.children = vec![file("big_test.rs", 300), file("other.rs", 700)];+ let mut root = Node::directory("root");+ root.children =+ vec![src, file("unit_test.txt", 100), file("huge.iso", 5000)];+ aggregate(&mut root, Metric::Bytes);++ let matches = filter(&root, &[], "test").expect("needle");+ let area = Rect::new(0.0, 0.0, 400.0, 400.0);+ let options = LayoutOptions {+ padding: 0.0,+ padding_outer: 0.0,+ ..LayoutOptions::default()+ };+ let tiles = layout_filtered(+ &root,+ &[],+ area,+ Metric::Bytes,+ &options,+ Some(&matches),+ );+ let names: Vec<String> = tiles+ .iter()+ .filter_map(|tile| root.resolve(tile.crumbs()))+ .map(|node| node.name.to_string())+ .collect();+ assert!(names.contains(&"big_test.rs".to_string()));+ assert!(names.contains(&"unit_test.txt".to_string()));+ assert!(+ !names+ .iter()+ .any(|name| name == "huge.iso" || name == "other.rs")+ );++ // src is sized by its 300 matched bytes, not its 1000: three times+ // the 100-byte match beside it.+ let area_of = |name: &str| {+ tiles+ .iter()+ .find(|tile| {+ root.resolve(tile.crumbs())+ .is_some_and(|node| &*node.name == name)+ })+ .map(|tile| tile.rect.w * tile.rect.h)+ .expect("drawn")+ };+ let ratio = area_of("src") / area_of("unit_test.txt");+ assert!((ratio - 3.0).abs() < 0.05, "ratio {ratio}");+ }}