Skip to content

A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.

Rust

core: memoized widening and name filtering

Tobi Lutke23b9e5d21b88
diff --git a/crates/disktree-core/src/filter.rs b/crates/disktree-core/src/filter.rs
new file mode 100644
index 0000000..24c1a7a
--- /dev/null
+++ b/crates/disktree-core/src/filter.rs
@@ -0,0 +1,249 @@
+//! Typeahead filtering: which parts of a tree match a name.
+//!
+//! A node whose name contains the needle (ignoring ASCII case) matches, and
+//! is kept whole: everything under a matching directory goes with it. Its
+//! ancestors are kept only partly, and sized by what matched beneath them,
+//! so a filtered treemap shows exactly the matches, at their true relative
+//! sizes, in the places they live.
+
+use rustc_hash::FxHashMap;
+
+use crate::tree::{Metric, Node};
+
+/// How a node takes part in a filtered view.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Keep {
+ /// It matches: drawn as usual, with everything beneath it.
+ Whole,
+ /// It holds matches: drawn with only those, at their size.
+ Partial { bytes: u64, files: u64 },
+}
+
+/// The outcome of filtering a subtree by name.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Matches {
+ /// The needle, lowercased.
+ pub needle: String,
+ /// Absolute crumbs of the subtree that was searched.
+ pub base: Vec<usize>,
+ /// Keyed by absolute crumbs. Only matches and their ancestors appear;
+ /// a match's own descendants are implied.
+ pub keep: FxHashMap<Vec<usize>, Keep>,
+ /// Topmost matches: a match inside a match is not counted again.
+ pub count: usize,
+ pub bytes: u64,
+ pub files: u64,
+}
+
+impl Matches {
+ /// How the node at `crumbs` takes part: `None` when it is filtered out.
+ /// Anything outside the searched subtree, and anything beneath a match,
+ /// is kept whole.
+ pub fn keep(&self, crumbs: &[usize]) -> Option<Keep> {
+ if !crumbs.starts_with(&self.base) {
+ return Some(Keep::Whole);
+ }
+ for length in self.base.len()..=crumbs.len() {
+ match self.keep.get(&crumbs[..length]) {
+ Some(Keep::Whole) => return Some(Keep::Whole),
+ Some(partial) if length == crumbs.len() => {
+ return Some(*partial);
+ }
+ // Only the base may be absent: it holds the matches but is
+ // not recorded, being where the search started.
+ None if length > self.base.len() => return None,
+ _ => {}
+ }
+ }
+ // The base itself holds the matches.
+ Some(Keep::Partial {
+ bytes: self.bytes,
+ files: self.files,
+ })
+ }
+
+ /// The value a kept node is laid out by.
+ pub const fn value(keep: Keep, node: &Node, metric: Metric) -> u64 {
+ match (keep, metric) {
+ (Keep::Whole, _) => node.value(metric),
+ (Keep::Partial { bytes, .. }, Metric::Bytes) => bytes,
+ (Keep::Partial { files, .. }, Metric::Files) => files,
+ }
+ }
+}
+
+/// Search `node`, found at absolute `base`, for names containing `needle`.
+/// `None` for an empty needle: nothing is filtered.
+pub fn filter(node: &Node, base: &[usize], needle: &str) -> Option<Matches> {
+ let needle = needle.trim().to_ascii_lowercase();
+ if needle.is_empty() {
+ return None;
+ }
+ let mut matches = Matches {
+ needle,
+ base: base.to_vec(),
+ ..Matches::default()
+ };
+ let mut crumbs = base.to_vec();
+ let (bytes, files) = visit(node, &mut crumbs, &mut matches);
+ matches.bytes = bytes;
+ matches.files = files;
+ Some(matches)
+}
+
+/// Returns the bytes and files that matched at or beneath `node`'s
+/// children, recording what to keep.
+fn visit(
+ node: &Node,
+ crumbs: &mut Vec<usize>,
+ matches: &mut Matches,
+) -> (u64, u64) {
+ let mut total = (0, 0);
+ for (index, child) in node.children.iter().enumerate() {
+ crumbs.push(index);
+ if contains_ignoring_case(&child.name, &matches.needle) {
+ matches.keep.insert(crumbs.clone(), Keep::Whole);
+ matches.count += 1;
+ total.0 += child.bytes;
+ total.1 += child.files;
+ } else if !child.children.is_empty() {
+ let (bytes, files) = visit(child, crumbs, matches);
+ if bytes > 0 || files > 0 {
+ matches
+ .keep
+ .insert(crumbs.clone(), Keep::Partial { bytes, files });
+ total.0 += bytes;
+ total.1 += files;
+ }
+ }
+ crumbs.pop();
+ }
+ total
+}
+
+/// Substring search ignoring ASCII case, without allocating: a filter runs
+/// over every name in view on every keystroke.
+fn contains_ignoring_case(haystack: &str, lower_needle: &str) -> bool {
+ let (hay, needle) = (haystack.as_bytes(), lower_needle.as_bytes());
+ if needle.is_empty() {
+ return true;
+ }
+ if needle.len() > hay.len() {
+ return false;
+ }
+ hay.windows(needle.len()).any(|window| {
+ window
+ .iter()
+ .zip(needle)
+ .all(|(left, right)| left.to_ascii_lowercase() == *right)
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::tree::{NodeKind, aggregate};
+
+ fn file(name: &str, bytes: u64) -> Node {
+ Node::entry(name, NodeKind::File, bytes)
+ }
+
+ fn dir(name: &str, children: Vec<Node>) -> Node {
+ let mut node = Node::directory(name);
+ node.children = children;
+ node
+ }
+
+ fn tree() -> Node {
+ let mut root = dir(
+ "root",
+ vec![
+ dir(
+ "src",
+ vec![
+ dir("App", vec![file("main.rs", 10)]),
+ file("notes", 5),
+ ],
+ ),
+ dir(
+ "apps",
+ vec![file("x", 100), dir("apple", vec![file("y", 7)])],
+ ),
+ file("readme", 1),
+ ],
+ );
+ aggregate(&mut root, Metric::Bytes);
+ root
+ }
+
+ fn crumbs_of(root: &Node, names: &[&str]) -> Vec<usize> {
+ let mut node = root;
+ names
+ .iter()
+ .map(|name| {
+ let index = node
+ .children
+ .iter()
+ .position(|child| &*child.name == *name)
+ .expect("present");
+ node = &node.children[index];
+ index
+ })
+ .collect()
+ }
+
+ #[test]
+ fn matches_are_kept_whole_and_their_ancestors_by_what_matched() {
+ let root = tree();
+ let found = filter(&root, &[], "APP").expect("a needle");
+ assert_eq!(found.count, 2, "src/App and apps; apple is inside apps");
+ assert_eq!(found.bytes, 10 + 107);
+ assert_eq!(found.keep(&crumbs_of(&root, &["apps"])), Some(Keep::Whole));
+ assert_eq!(
+ found.keep(&crumbs_of(&root, &["apps", "x"])),
+ Some(Keep::Whole),
+ "inside a match"
+ );
+ assert_eq!(
+ found.keep(&crumbs_of(&root, &["src"])),
+ Some(Keep::Partial {
+ bytes: 10,
+ files: 1
+ })
+ );
+ assert_eq!(found.keep(&crumbs_of(&root, &["src", "notes"])), None);
+ assert_eq!(found.keep(&crumbs_of(&root, &["readme"])), None);
+ }
+
+ #[test]
+ fn a_search_below_the_root_leaves_the_rest_alone() {
+ let root = tree();
+ let src = crumbs_of(&root, &["src"]);
+ let found = filter(root.resolve(&src).expect("src"), &src, "main")
+ .expect("needle");
+ assert_eq!(found.keep(&crumbs_of(&root, &["apps"])), Some(Keep::Whole));
+ assert_eq!(found.keep(&crumbs_of(&root, &["src", "notes"])), None);
+ assert_eq!(
+ found.keep(&src),
+ Some(Keep::Partial {
+ bytes: 10,
+ files: 1
+ })
+ );
+ }
+
+ #[test]
+ fn an_empty_needle_filters_nothing() {
+ assert!(filter(&tree(), &[], " ").is_none());
+ let found = filter(&tree(), &[], "zzz").expect("needle");
+ assert_eq!(found.count, 0);
+ assert!(found.keep.is_empty());
+ }
+
+ #[test]
+ fn case_is_ignored_without_allocating() {
+ assert!(contains_ignoring_case("Cargo.TOML", "toml"));
+ assert!(!contains_ignoring_case("ab", "abc"));
+ assert!(contains_ignoring_case("x", ""));
+ }
+}
diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rs
index 27a8b55..d7cdb24 100644
--- a/crates/disktree-core/src/lib.rs
+++ b/crates/disktree-core/src/lib.rs
@@ -6,6 +6,7 @@
//! be built and tested without GPUI, a display, or a GPU.
pub mod classify;
+pub mod filter;
pub mod insights;
pub mod removal;
pub mod scan;
diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs
index 8cdec56..be5f2ff 100644
--- a/crates/disktree-core/src/scan.rs
+++ b/crates/disktree-core/src/scan.rs
@@ -156,11 +156,31 @@ pub struct ScanHandle {
result: Receiver<io::Result<Node>>,
}
+/// A subtree already measured, reused by a wider scan instead of walked
+/// again: widening from `~` to `/` only reads what is outside `~`.
+#[derive(Clone, Debug)]
+pub struct Known {
+ /// Where it is. Compared with the walk's own paths, so give it in the
+ /// same form as the root (both canonical).
+ pub path: PathBuf,
+ pub tree: Arc<Node>,
+}
+
impl ScanHandle {
/// Start walking `root` on a worker thread.
pub fn spawn(root: PathBuf, options: ScanOptions) -> Self {
+ Self::spawn_with(root, options, None)
+ }
+
+ /// Start walking `root`, reusing `known` where the walk reaches it.
+ pub fn spawn_with(
+ root: PathBuf,
+ options: ScanOptions,
+ known: Option<Known>,
+ ) -> Self {
let progress = Arc::new(ScanProgress::default());
let context = Arc::new(WalkContext {
+ known,
options,
progress: Arc::clone(&progress),
root_device: Mutex::new(None),
@@ -208,6 +228,7 @@ impl ScanHandle {
pub fn scan(root: &Path, options: ScanOptions) -> io::Result<Node> {
let progress = Arc::new(ScanProgress::default());
let context = Arc::new(WalkContext {
+ known: None,
options,
progress: Arc::clone(&progress),
root_device: Mutex::new(None),
@@ -221,6 +242,8 @@ pub fn scan(root: &Path, options: ScanOptions) -> io::Result<Node> {
}
struct WalkContext {
+ /// A subtree to reuse rather than walk.
+ known: Option<Known>,
options: ScanOptions,
progress: Arc<ScanProgress>,
/// Device of the root, resolved once: the `one_filesystem` fallback
@@ -274,6 +297,20 @@ impl WalkContext {
}
if file_type.is_dir() {
+ // Memoized: the subtree a narrower scan already measured is
+ // taken whole, before any volume rule, since it was measured
+ // under the same rules.
+ if let Some(known) = &self.known
+ && known.path == path
+ {
+ let tree = (*known.tree).clone();
+ self.progress.files.fetch_add(tree.files, Ordering::Relaxed);
+ self.progress.bytes.fetch_add(tree.bytes, Ordering::Relaxed);
+ self.progress.dirs.fetch_add(tree.dirs, Ordering::Relaxed);
+ let mut tree = tree;
+ tree.name = name;
+ return Classified::Entry(tree);
+ }
if self.options.one_filesystem
&& let Some(foreign) = self.foreign_mounts.get()
{
@@ -906,6 +943,7 @@ mod tests {
let readable = fs::read_dir(&locked).map(|mut it| it.next().is_some());
let progress = Arc::new(ScanProgress::default());
let context = Arc::new(WalkContext {
+ known: None,
options: options(),
progress: Arc::clone(&progress),
root_device: Mutex::new(None),
@@ -958,6 +996,44 @@ mod tests {
assert_eq!(snapshot.errors, 0);
}
+ #[test]
+ fn a_wider_scan_reuses_the_subtree_it_already_knows() {
+ let temp = tempfile::tempdir().expect("tempdir");
+ let root = temp.path().canonicalize().expect("canonical");
+ let inner = root.join("inner");
+ fs::create_dir_all(inner.join("deep")).expect("mkdir");
+ fs::write(inner.join("deep/a.bin"), vec![0_u8; 4096]).expect("write");
+ fs::write(root.join("outside.bin"), vec![0_u8; 8192]).expect("write");
+ let known = scan(&inner, options()).expect("inner scan");
+
+ // Changed on disk after the inner scan: a memoized subtree must not
+ // see it, which is how this test knows the walk skipped it.
+ fs::write(inner.join("deep/b.bin"), vec![0_u8; 4096]).expect("write");
+
+ let handle = ScanHandle::spawn_with(
+ root,
+ options(),
+ Some(Known {
+ path: inner,
+ tree: Arc::new(known.clone()),
+ }),
+ );
+ let tree = loop {
+ if let Some(outcome) = handle.poll() {
+ break outcome.expect("wide scan");
+ }
+ thread::sleep(std::time::Duration::from_millis(2));
+ };
+ let reused = tree.child_named("inner").expect("inner is in the tree");
+ assert_eq!(reused.files, known.files, "b.bin was never read");
+ assert_eq!(reused.bytes, known.bytes);
+ assert!(
+ tree.child_named("outside.bin").is_some(),
+ "the rest was walked"
+ );
+ assert_eq!(tree.files, known.files + 1);
+ }
+
/// A real whole-disk scan, run by hand: `cargo test -p disktree-core
/// -- --ignored --nocapture whole_disk`. Prints what it found, so the
/// volume rules can be checked against this machine's mounts.
@@ -984,5 +1060,16 @@ mod tests {
let skipped =
crate::space::foreign_mounts_for(&root).unwrap_or_default();
println!("left out: {skipped:?}");
+ for needle in ["c", "cache", "node_modules", "zzzz"] {
+ let started = std::time::Instant::now();
+ let found =
+ crate::filter::filter(&tree, &[], needle).expect("needle");
+ println!(
+ "filter {needle:?}: {} matches, {} in {:.1?}",
+ found.count,
+ crate::size::human_bytes(found.bytes),
+ started.elapsed()
+ );
+ }
}
}
diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rs
index 1ad9ae2..997882f 100644
--- a/crates/disktree-core/src/tree.rs
+++ b/crates/disktree-core/src/tree.rs
@@ -144,6 +144,11 @@ impl Node {
}
}
+ /// The child with this name, if there is one.
+ pub fn child_named(&self, name: &str) -> Option<&Self> {
+ self.children.iter().find(|child| &*child.name == name)
+ }
+
pub fn child(&self, index: usize) -> Option<&Self> {
self.children.get(index)
}
diff --git a/crates/disktree-core/src/treemap.rs b/crates/disktree-core/src/treemap.rs
index 6dfa020..598b559 100644
--- a/crates/disktree-core/src/treemap.rs
+++ b/crates/disktree-core/src/treemap.rs
@@ -10,6 +10,7 @@
//! view applies its own pan and zoom when painting, so re-layout is only needed
//! when the tree, the viewport size, or the requested depth changes.
+use crate::filter::{Keep, Matches};
use crate::tree::{Metric, Node};
/// An axis-aligned rectangle in viewport pixels.
@@ -149,22 +150,49 @@ pub fn layout(
area: Rect,
metric: Metric,
options: &LayoutOptions,
+) -> Vec<Tile> {
+ layout_filtered(root, root_crumbs, area, metric, options, None)
+}
+
+/// [`layout`], showing only what `filter` keeps: its matches, at the size
+/// of what matched, inside ancestors sized the same way.
+pub fn layout_filtered(
+ root: &Node,
+ root_crumbs: &[usize],
+ area: Rect,
+ metric: Metric,
+ options: &LayoutOptions,
+ filter: Option<&Matches>,
) -> Vec<Tile> {
let mut tiles = Vec::new();
let mut crumbs = root_crumbs.to_vec();
- place_children(root, area, metric, options, 0, &mut crumbs, &mut tiles);
+ let place = Placement {
+ metric,
+ options,
+ filter,
+ };
+ place_children(root, area, &place, 0, &mut crumbs, &mut tiles);
tiles
}
+/// What stays the same for every level of one layout.
+struct Placement<'a> {
+ metric: Metric,
+ options: &'a LayoutOptions,
+ /// Set while the level being placed is inside a filtered region; a
+ /// match clears it for everything beneath.
+ filter: Option<&'a Matches>,
+}
+
fn place_children(
node: &Node,
area: Rect,
- metric: Metric,
- options: &LayoutOptions,
+ place: &Placement<'_>,
depth: u32,
crumbs: &mut Vec<usize>,
out: &mut Vec<Tile>,
) {
+ let (metric, options) = (place.metric, place.options);
if node.children.is_empty() || area.w <= 0.0 || area.h <= 0.0 {
return;
}
@@ -175,7 +203,18 @@ fn place_children(
.children
.iter()
.enumerate()
- .map(|(index, child)| (index, child.value(metric) as f64))
+ .filter_map(|(index, child)| {
+ let value = match place.filter {
+ None => child.value(metric),
+ Some(filter) => {
+ crumbs.push(index);
+ let keep = filter.keep(crumbs);
+ crumbs.pop();
+ Matches::value(keep?, child, metric)
+ }
+ };
+ Some((index, value as f64))
+ })
.filter(|(_, value)| *value > 0.0)
.collect();
if ranked.is_empty() {
@@ -243,15 +282,14 @@ fn place_children(
rect.w,
rect.bottom() - header.bottom(),
);
- place_children(
- child,
- body,
- metric,
- options,
- depth + 1,
- crumbs,
- out,
- );
+ // Beneath a match everything is shown; above one, only matches.
+ let inner = Placement {
+ filter: place
+ .filter
+ .filter(|filter| filter.keep(crumbs) != Some(Keep::Whole)),
+ ..*place
+ };
+ place_children(child, body, &inner, depth + 1, crumbs, out);
}
crumbs.pop();
}
@@ -627,4 +665,61 @@ mod tests {
assert_eq!(found.crumbs(), &[0, 0]);
assert!(hit(&tiles, -50.0, -50.0).is_none());
}
+
+ #[test]
+ fn a_filtered_layout_shows_only_the_matches_at_their_size() {
+ use crate::filter::filter;
+ use crate::tree::{NodeKind, aggregate};
+
+ let file = |name: &str, bytes| Node::entry(name, NodeKind::File, bytes);
+ let mut src = Node::directory("src");
+ src.children = vec![file("big_test.rs", 300), file("other.rs", 700)];
+ let mut root = Node::directory("root");
+ root.children =
+ vec![src, file("unit_test.txt", 100), file("huge.iso", 5000)];
+ aggregate(&mut root, Metric::Bytes);
+
+ let matches = filter(&root, &[], "test").expect("needle");
+ let area = Rect::new(0.0, 0.0, 400.0, 400.0);
+ let options = LayoutOptions {
+ padding: 0.0,
+ padding_outer: 0.0,
+ ..LayoutOptions::default()
+ };
+ let tiles = layout_filtered(
+ &root,
+ &[],
+ area,
+ Metric::Bytes,
+ &options,
+ Some(&matches),
+ );
+ let names: Vec<String> = tiles
+ .iter()
+ .filter_map(|tile| root.resolve(tile.crumbs()))
+ .map(|node| node.name.to_string())
+ .collect();
+ assert!(names.contains(&"big_test.rs".to_string()));
+ assert!(names.contains(&"unit_test.txt".to_string()));
+ assert!(
+ !names
+ .iter()
+ .any(|name| name == "huge.iso" || name == "other.rs")
+ );
+
+ // src is sized by its 300 matched bytes, not its 1000: three times
+ // the 100-byte match beside it.
+ let area_of = |name: &str| {
+ tiles
+ .iter()
+ .find(|tile| {
+ root.resolve(tile.crumbs())
+ .is_some_and(|node| &*node.name == name)
+ })
+ .map(|tile| tile.rect.w * tile.rect.h)
+ .expect("drawn")
+ };
+ let ratio = area_of("src") / area_of("unit_test.txt");
+ assert!((ratio - 3.0).abs() < 0.05, "ratio {ratio}");
+ }
}