A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.
Rust
scan: stream live SSH progress and accept deeply nested trees
Tobi Lutke8d98a855e868
diff --git a/README.md b/README.mdindex 738411b..625ec18 100644--- a/README.md+++ b/README.md@@ -212,12 +212,21 @@ so outdated helpers reject them instead of silently scanning their home.The Linux/macOS helper streams a depth-first walk directly to stdout. Itnever constructs a tree or retains a directory's children: memory scales-with open ancestors (at most 256), not file count. Only pending hardlink+with open ancestors, not file count. Deep directories are not rejected+at an arbitrary depth; filesystem errors are reported as unreadable. Only pending hardlinkidentities are retained, capped at 65,536; ordinary files need no identitystorage. A resource limit fails explicitly instead of returning a partialtree. Symlinks are not followed and every nested mount is skipped, even abind mount or same-device subvolume. Select those as separate roots.+While indexing, the window shows live files, directories, measured bytes,+read errors, elapsed time and average throughput. Remote scans also show a+sampled recent directory and connection / helper setup / indexing / treemap+stages. Progress uses a single replaceable snapshot, not a growing event+queue; path updates are throttled to five per second. There is no guessed+percentage: the total is unknown until the walk finishes. Older helpers+still provide counters from their streamed records, without path samples.+One compressed SSH connection carries a bounded JSON request on stdin andcompact postorder records on stdout. The client keeps stdin open so thehelper exits on disconnect, including while scanning or blocked on output.diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rsindex 41d372d..5ad2871 100644--- a/crates/disktree-app/src/state.rs+++ b/crates/disktree-app/src/state.rs@@ -329,6 +329,7 @@ pub struct Disktree {pub remote: Option<String>,remote_disk: bool,pub remote_scan: Option<disktree_core::remote::RemoteScan>,+ pub remote_progress: disktree_core::remote_progress::Progress,remote_platform: String,pub remote_identity: Option<disktree_core::identity::HostIdentity>,pub home: Option<PathBuf>,@@ -524,6 +525,8 @@ impl Disktree {remote,remote_disk,remote_scan: None,+ remote_progress: disktree_core::remote_progress::Progress::default(+ ),remote_platform: String::new(),remote_identity: None,home,@@ -1236,6 +1239,8 @@ impl Disktree {let epoch = self.scan_epoch;self.progress = ScanSnapshot::default();self.scan_error = None;+ self.remote_progress =+ disktree_core::remote_progress::Progress::default();// A new scan starts at its root; where it was is one `<` away.if !self.crumbs.is_empty() {self.remember();@@ -1324,7 +1329,13 @@ impl Disktree {return false;}if let Some(scan) = &self.remote_scan {- let Some(outcome) = scan.poll() else {+ let outcome = scan.poll();+ self.remote_progress = scan.progress();+ self.progress.files = self.remote_progress.files;+ self.progress.dirs = self.remote_progress.dirs;+ self.progress.bytes = self.remote_progress.bytes;+ self.progress.errors = self.remote_progress.errors;+ let Some(outcome) = outcome else {cx.notify();return true;};diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rsindex e6f1e8d..8bc9fe5 100644--- a/crates/disktree-app/src/tests.rs+++ b/crates/disktree-app/src/tests.rs@@ -92,6 +92,34 @@ fn read<R>(view.read_with(cx, |app, _| f(app))}+#[gpui_kit::test]+fn remote_indexing_draws_live_activity_and_the_sampled_directory(+ cx: &mut TestAppContext,+) {+ cx.update(gpui_omarchy::init);+ let temp = fixture();+ let (view, cx) = view_over(temp.path(), cx);+ update(&view, cx, |app, _| {+ app.tree = None;+ app.remote = Some("fixture-only".into());+ app.scan_started = Some(std::time::Instant::now());+ app.progress.files = 1234;+ app.progress.bytes = 123_456;+ app.remote_progress.stage =+ disktree_core::remote_progress::Stage::Indexing;+ app.remote_progress.directory = Some(temp.path().join("junk/deeper"));+ });+ draw(cx);+ assert!(cx.debug_bounds("scan-activity").is_some());+ assert!(cx.debug_bounds("scan-directory").is_some());+ update(&view, cx, |app, _| {+ app.scan_error = Some("fixture scan failed".into());+ });+ draw(cx);+ assert!(cx.debug_bounds("scan-error").is_some());+ assert!(cx.debug_bounds("scan-reading").is_none());+}+#[gpui_kit::test]fn startup_chooser_never_scans_before_explicit_acceptance(cx: &mut TestAppContext,diff --git a/crates/disktree-app/src/ui.rs b/crates/disktree-app/src/ui.rsindex 3e21ed4..3f43a67 100644--- a/crates/disktree-app/src/ui.rs+++ b/crates/disktree-app/src/ui.rs@@ -85,7 +85,6 @@ pub mod size {/// A thin meter: share of the scan, the disk.pub const METER: Rems = Rems(0.3125);/// The meter on the scanning panel.- pub const SCANNING_METER: Rems = Rems(26.25);/// The Size | Files | Age choice in the settings row.pub const RANKING_CHOICE: Rems = Rems(11.0);/// The review screen's summary column.diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rsindex 78f1470..b9ea289 100644--- a/crates/disktree-app/src/views.rs+++ b/crates/disktree-app/src/views.rs@@ -2450,6 +2450,14 @@ fn scanning_panel(.text_size(text::BODY).child(error.clone()),)+ .children(app.remote_progress.directory.as_ref().map(|path| {+ div()+ .max_w(size::HELP)+ .truncate()+ .text_size(text::CAPTION)+ .text_color(theme.secondary)+ .child(format!("Last directory: {}", path.display()))+ })).child(widgets::hint("r", "retry scan", cx));}let progress = &app.progress;@@ -2479,8 +2487,11 @@ fn scanning_panel(.text_color(theme.bright).child(if cancelled {format!("Stopped reading {}", widgets::display_root(app))+ } else if app.remote.is_some() {+ format!("{} · {}", app.remote_progress.stage.label(),+ widgets::display_root(app))} else {- format!("Reading {}", widgets::display_root(app))+ format!("Indexing {}", widgets::display_root(app))}),).child(@@ -2509,16 +2520,18 @@ fn scanning_panel(.when(!cancelled, |panel| {panel.child(div()- .w(size::SCANNING_METER)- .child(widgets::meter_row(- "",- "",- progress_estimate(progress.files),- theme.accent,- cx,- )),+ .debug_selector(|| "scan-activity".into())+ .text_size(text::BODY)+ .text_color(theme.accent)+ .child(scan_activity(app)),)})+ .children(app.remote_progress.directory.as_ref().map(|path| {+ div().debug_selector(|| "scan-directory".into())+ .max_w(size::HELP).min_w_0().truncate()+ .text_size(text::CAPTION).text_color(theme.secondary)+ .child(format!("Recent directory: {}", path.display()))+ })).child(div().text_size(text::BODY)@@ -2526,7 +2539,7 @@ fn scanning_panel(.child(if cancelled {"Nothing is shown from a scan that did not finish."} else {- "Marking, zooming and the free-space meter all work as soon as it lands."+ "Total unknown until indexing finishes. Results stream live."}),)// A failed scan has nothing left to cancel either.@@ -2545,6 +2558,24 @@ fn scanning_panel(})}+fn scan_activity(app: &Disktree) -> String {+ let elapsed = app+ .scan_started+ .map_or(0.0, |start| start.elapsed().as_secs_f64());+ let phase = (elapsed * 8.0) as usize % 4;+ let spinner = ["◐", "◓", "◑", "◒"][phase];+ let rate = app.progress.files as f64 / elapsed.max(1.0);+ let quiet = app+ .remote_progress+ .updated+ .map_or(0, |at| at.elapsed().as_secs());+ if app.remote.is_some() && quiet >= 5 {+ format!("{spinner} {elapsed:.0}s elapsed · no new entries for {quiet}s")+ } else {+ format!("{spinner} {elapsed:.0}s elapsed · {rate:.0} files/s avg")+ }+}+fn find_field(app: &Disktree, theme: &Theme) -> Div {// What the text matches, said beside it as it is typed, and what Enter// and Escape will do with it.@@ -2648,18 +2679,6 @@ fn marks_ancestor(.next()}-/// A scan has no total to measure against, so the meter shows the work done-/// rather than pretending to know how far along it is.-fn progress_estimate(files: u64) -> f32 {- if files == 0 {- 0.06- } else {- // Asymptotic: the bar keeps moving while the walk continues.- let scaled = files as f32 / (files as f32 + 20_000.0);- (0.1 + scaled * 0.85).min(0.99)- }-}-// ── review ──────────────────────────────────────────────────────────────fn review(@@ -3375,7 +3394,7 @@ fn done(app: &Disktree, cx: &gpui_kit::Context<'_, Disktree>) -> Div {body = body.child(widgets::meter_row("re-scanning",format!("{} files", widgets::human_count(app.progress.files)),- progress_estimate(app.progress.files),+ 0.0,theme.accent,cx,));diff --git a/crates/disktree-core/src/deployment.rs b/crates/disktree-core/src/deployment.rsindex d856074..557f9aa 100644--- a/crates/disktree-core/src/deployment.rs+++ b/crates/disktree-core/src/deployment.rs@@ -51,8 +51,18 @@ pub fn deploy(archive: &Archive,input: &mut impl BufRead,output: &mut impl Write,+) -> io::Result<()> {+ deploy_with_progress(archive, input, output, |_| {})+}++pub(crate) fn deploy_with_progress(+ archive: &Archive,+ input: &mut impl BufRead,+ output: &mut impl Write,+ mut progress: impl FnMut(crate::remote_progress::Stage),) -> io::Result<()> {let hello = control_line(input)?;+ progress(crate::remote_progress::Stage::CheckingHelper);let parts: Vec<_> = hello.split_whitespace().collect();if parts.len() != 3 || parts[0] != "DISKTREE_BOOTSTRAP_V1" {return Err(io::Error::other("unexpected SSH deployment handshake"));@@ -77,6 +87,7 @@ pub fn deploy(output.flush()?;let mut state = control_line(input)?;if state == "DISKTREE_UPLOAD_V1" {+ progress(crate::remote_progress::Stage::UploadingHelper);output.write_all(&bytes)?;output.flush()?;state = control_line(input)?;diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rsindex b25f711..5b020a1 100644--- a/crates/disktree-core/src/lib.rs+++ b/crates/disktree-core/src/lib.rs@@ -18,6 +18,7 @@ mod macos;mod mft;pub mod payload;pub mod remote;+pub mod remote_progress;#[cfg(unix)]mod remote_walk;pub mod removal;diff --git a/crates/disktree-core/src/remote.rs b/crates/disktree-core/src/remote.rsindex 5608b77..cbe453b 100644--- a/crates/disktree-core/src/remote.rs+++ b/crates/disktree-core/src/remote.rs@@ -11,11 +11,13 @@ use std::sync::{Arc, Mutex, mpsc};use serde::{Deserialize, Serialize};+use crate::remote_progress::{INTERVAL, Progress, Shared, Stage, stage};use crate::scan::ScanOptions;#[cfg(not(unix))]use crate::scan::scan;use crate::space::{SpaceInfo, space_info, volume_root_for};use crate::tree::{Node, NodeKind, aggregate};+use std::time::Instant;const VERSION: u32 = 1;const MAX_LINE: u64 = 1 << 20;@@ -57,6 +59,9 @@ pub struct SessionRequest {pub operation: Operation,#[serde(default)]pub watch_disconnect: bool,+ // Opt-in keeps the postorder-only wire format for older clients.+ #[serde(default)]+ pub live_progress: bool,}/// The target's root candidates, without walking or probing any of them.@@ -166,7 +171,9 @@ pub fn serve_session(out: &mut impl Write,) -> io::Result<()> {match session.operation {- Operation::Scan(request) => serve_request(request, out),+ Operation::Scan(request) => {+ serve_request_with_progress(request, out, session.live_progress)+ }Operation::Discover { discover_version } => {if discover_version != VERSION {return Err(io::Error::other(@@ -190,6 +197,15 @@ pub fn serve_session(/// Scan a decoded request. The standalone helper supervises its lifetime.pub fn serve_request(request: Request, out: &mut impl Write) -> io::Result<()> {+ serve_request_with_progress(request, out, false)+}++fn serve_request_with_progress(+ request: Request,+ out: &mut impl Write,+ live_progress: bool,+) -> io::Result<()> {+ let _ = live_progress; // Only Unix has the streaming walker.let home = std::env::home_dir();let root = request.root@@ -218,7 +234,7 @@ pub fn serve_request(request: Request, out: &mut impl Write) -> io::Result<()> {)?;out.flush()?;#[cfg(unix)]- crate::remote_walk::stream(&root, &request.options, out)?;+ crate::remote_walk::stream(&root, &request.options, out, live_progress)?;#[cfg(not(unix))]write_node(out, &scan(&root, request.options)?)?;// An explicit terminator distinguishes a complete scan from a severed SSH@@ -257,16 +273,41 @@ fn write_node(out: &mut impl Write, node: &Node) -> io::Result<()> {}pub fn read_snapshot(input: &mut impl BufRead) -> io::Result<Snapshot> {+ read_snapshot_with_progress(input, &Shared::default())+}++fn read_snapshot_with_progress(+ input: &mut impl BufRead,+ shared: &Shared,+) -> io::Result<Snapshot> {let header: Header = receive(input)?;if header.version != VERSION || !header.root.is_absolute() {return Err(io::Error::other("incompatible remote helper protocol"));}+ let mut progress = Progress {+ stage: Stage::Indexing,+ directory: Some(header.root.clone()),+ updated: Some(Instant::now()),+ ..Progress::default()+ };+ *shared.lock().expect("remote progress lock") = progress.clone();+ let mut published = Instant::now();let mut stack = Vec::<Node>::new();loop {let value: serde_json::Value = receive(input)?;if value == "done" {break;}+ if let Some(directory) = value.get("directory") {+ progress.directory = Some(+ serde_json::from_value(directory.clone())+ .map_err(json_error)?,+ );+ progress.updated = Some(Instant::now());+ *shared.lock().expect("remote progress lock") = progress.clone();+ published = Instant::now();+ continue;+ }let Record(name, kind, bytes, files, modified, read_error, count) =serde_json::from_value(value).map_err(json_error)?;// Names must remain single components when joined to marked paths.@@ -297,14 +338,36 @@ pub fn read_snapshot(input: &mut impl BufRead) -> io::Result<Snapshot> {if !kind.is_dir() && count != 0 {return Err(io::Error::other("remote leaf has children"));}- let children = stack.split_off(stack.len() - count);+ // The final root often owns every stacked node. Transfer that+ // allocation instead of briefly duplicating a wide tree's storage.+ let children = if count == stack.len() {+ std::mem::take(&mut stack)+ } else {+ stack.split_off(stack.len() - count)+ };let mut node = Node::entry(name, kind, bytes);node.files = files;node.modified = modified;node.read_error = read_error;node.children = children;stack.push(node);+ progress.dirs += u64::from(kind.is_dir());+ // Legacy helpers include direct totals in directory records too.+ // Count each leaf once, never charge its parent's totals again.+ if !kind.is_dir() {+ progress.files += files;+ progress.bytes += bytes;+ }+ progress.errors += u64::from(read_error);+ if published.elapsed() >= INTERVAL {+ progress.updated = Some(Instant::now());+ *shared.lock().expect("remote progress lock") = progress.clone();+ published = Instant::now();+ }}+ progress.stage = Stage::Finalizing;+ progress.updated = Some(Instant::now());+ *shared.lock().expect("remote progress lock") = progress;if stack.len() != 1 {return Err(io::Error::other("incomplete remote tree"));}@@ -369,8 +432,7 @@ fn ssh_error(host: &str, details: &str, fallback: &io::Error) -> io::Error {Run make install-scan from the checkout on the server.")} else {- "Check the server's SSH connection and disktree-scan installation."- .into()+ "No incomplete scan is shown. Retry, or choose another root.".into()};io::Error::other(format!("SSH {host}: {reason}\n\n{hint}"))}@@ -383,6 +445,7 @@ fn ssh_error(host: &str, details: &str, fallback: &io::Error) -> io::Error {pub struct RemoteJob<T> {child: Arc<Mutex<Child>>,result: mpsc::Receiver<io::Result<T>>,+ progress: Shared,}pub type RemoteScan = RemoteJob<Snapshot>;@@ -442,8 +505,9 @@ impl RemoteScan {host,Operation::Scan(request.clone()),archive,- move |input| {- let mut snapshot = read_snapshot(input)?;+ move |input, progress| {+ let mut snapshot =+ read_snapshot_with_progress(input, progress)?;if metric != crate::tree::Metric::Bytes {aggregate(&mut snapshot.tree, metric);}@@ -463,7 +527,7 @@ impl RemoteRoots {discover_version: VERSION,},archive,- read_roots,+ |input, _| read_roots(input),)}}@@ -476,6 +540,7 @@ impl<T: Send + 'static> RemoteJob<T> {archive: Option<crate::payload::Archive>,decode: impl FnOnce(&mut BufReader<std::process::ChildStdout>,+ &Shared,) -> io::Result<T>+ Send+ 'static,@@ -494,22 +559,29 @@ impl<T: Send + 'static> RemoteJob<T> {let worker = child.clone();let host = host.to_owned();let (tx, result) = mpsc::channel();+ let progress = Shared::default();+ let live = progress.clone();std::thread::spawn(move || {let mut input = BufReader::new(stdout);let response = (|| {if let Some(archive) = archive {- crate::deployment::deploy(- &archive, &mut input, &mut stdin,+ crate::deployment::deploy_with_progress(+ &archive,+ &mut input,+ &mut stdin,+ |value| stage(&live, value),)?;}+ stage(&live, Stage::Starting);send(&mut stdin,&SessionRequest {operation,watch_disconnect: true,+ live_progress: true,},)?;- decode(&mut input)+ decode(&mut input, &live)})();drop(stdin);if response.is_err() {@@ -541,7 +613,15 @@ impl<T: Send + 'static> RemoteJob<T> {};let _ = tx.send(outcome);});- Ok(Self { child, result })+ Ok(Self {+ child,+ result,+ progress,+ })+ }++ pub fn progress(&self) -> Progress {+ self.progress.lock().expect("remote progress lock").clone()}pub fn poll(&self) -> Option<io::Result<T>> {@@ -640,6 +720,7 @@ mod tests {discover_version: VERSION,},watch_disconnect: true,+ live_progress: true,};let bytes = serde_json::to_vec(&session).unwrap();assert!(serde_json::from_slice::<Request>(&bytes).is_err());@@ -778,6 +859,14 @@ mod tests {let home = tempfile::tempdir().unwrap();let data = tempfile::tempdir().unwrap();std::fs::write(data.path().join("keep"), b"hello").unwrap();+ // Exercise the shipped static helper beyond the removed 256-level+ // cutoff, through deployment and decoding, not just the walker.+ let mut deep = data.path().to_path_buf();+ for _ in 0..320 {+ deep.push("d");+ std::fs::create_dir(&deep).unwrap();+ }+ std::fs::write(deep.join("leaf"), b"world").unwrap();let cache = home.path().join("cache");let remote = format!("HOME={} XDG_CACHE_HOME={}; export HOME XDG_CACHE_HOME;\n{}",@@ -810,7 +899,9 @@ mod tests {).unwrap();let snapshot = wait(&handle).unwrap();- assert_eq!(snapshot.tree.bytes, 5);+ assert_eq!(snapshot.tree.bytes, 10);+ assert_eq!(snapshot.tree.files, 2);+ assert_eq!(handle.progress().files, 2);assert!(snapshot.header.identity.is_some());assert_eq!(std::fs::read(&cached).unwrap(), bytes);if attempt == 1 {@@ -851,7 +942,7 @@ mod tests {discover_version: VERSION,},Some(archive),- read_roots,+ |input, _| read_roots(input),).unwrap();let roots = wait(&job).unwrap();@@ -930,6 +1021,57 @@ mod tests {assert!(snapshot.tree.child_named(r"back\slash").is_some());}+ #[cfg(unix)]+ #[test]+ fn live_counters_are_available_before_the_tree_finishes() {+ use std::os::unix::net::UnixStream;+ let dir = tempfile::tempdir().unwrap();+ std::fs::write(dir.path().join("file"), b"hello").unwrap();+ let request = Request {+ root: Some(dir.path().into()),+ disk: false,+ options: ScanOptions {+ apparent_size: true,+ ..ScanOptions::default()+ },+ };+ let mut data = Vec::new();+ serve_request(request, &mut data).unwrap();+ let mut lines = data.split_inclusive(|byte| *byte == b'\n');+ let header = lines.next().unwrap();+ let leaf = lines.next().unwrap();+ let tail = lines.flatten().copied().collect::<Vec<_>>();+ let (mut output, input) = UnixStream::pair().unwrap();+ input+ .set_read_timeout(Some(std::time::Duration::from_secs(5)))+ .unwrap();+ let shared = Shared::default();+ let live = shared.clone();+ let worker = std::thread::spawn(move || {+ read_snapshot_with_progress(&mut BufReader::new(input), &live)+ });+ output.write_all(header).unwrap();+ output.write_all(leaf).unwrap();+ send(&mut output, &serde_json::json!({"directory": dir.path()}))+ .unwrap();+ let deadline = Instant::now() + std::time::Duration::from_secs(2);+ loop {+ let progress = shared.lock().unwrap().clone();+ if progress.files == 1 {+ assert_eq!(progress.bytes, 5);+ assert_eq!(progress.stage, Stage::Indexing);+ assert!(!worker.is_finished());+ break;+ }+ assert!(Instant::now() < deadline);+ std::thread::yield_now();+ }+ output.write_all(&tail).unwrap();+ let snapshot = worker.join().unwrap().unwrap();+ assert_eq!(snapshot.tree.bytes, 5);+ assert_eq!(shared.lock().unwrap().stage, Stage::Finalizing);+ }+#[test]fn protocol_roundtrip_and_truncation() {let dir = tempfile::tempdir().unwrap();@@ -947,6 +1089,7 @@ mod tests {let mut input = Vec::new();send(&mut input, &request).unwrap();serve(&mut input.as_slice(), &mut data).unwrap();+ assert!(!data.windows(11).any(|part| part == b"\"directory\""));let snapshot = read_snapshot(&mut data.as_slice()).unwrap();assert_eq!(snapshot.tree.bytes, 5);assert_eq!(snapshot.tree.files, 1);diff --git a/crates/disktree-core/src/remote_progress.rs b/crates/disktree-core/src/remote_progress.rsnew file mode 100644index 0000000..8a78fa8--- /dev/null+++ b/crates/disktree-core/src/remote_progress.rs@@ -0,0 +1,49 @@+//! One replaceable progress snapshot, never a queue of filesystem events.+use std::path::PathBuf;+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+pub enum Stage {+ #[default]+ Connecting,+ CheckingHelper,+ UploadingHelper,+ Starting,+ Indexing,+ Finalizing,+}++impl Stage {+ pub const fn label(self) -> &'static str {+ match self {+ Self::Connecting => "Connecting over SSH",+ Self::CheckingHelper => "Checking remote scanner",+ Self::UploadingHelper => "Uploading remote scanner",+ Self::Starting => "Starting remote scanner",+ Self::Indexing => "Indexing remotely",+ Self::Finalizing => "Building the treemap",+ }+ }+}++#[derive(Clone, Debug, Default)]+pub struct Progress {+ pub stage: Stage,+ pub files: u64,+ pub dirs: u64,+ pub bytes: u64,+ pub errors: u64,+ pub directory: Option<PathBuf>,+ pub updated: Option<Instant>,+}++/// Cap both wire heartbeats and UI publications at five per second.+pub(crate) const INTERVAL: Duration = Duration::from_millis(200);+pub(crate) type Shared = Arc<Mutex<Progress>>;++pub(crate) fn stage(shared: &Shared, value: Stage) {+ let mut progress = shared.lock().expect("remote progress lock");+ progress.stage = value;+ progress.updated = Some(Instant::now());+}diff --git a/crates/disktree-core/src/remote_walk.rs b/crates/disktree-core/src/remote_walk.rsindex 64c2667..b845070 100644--- a/crates/disktree-core/src/remote_walk.rs+++ b/crates/disktree-core/src/remote_walk.rs@@ -10,9 +10,7 @@ use rustc_hash::{FxHashMap, FxHashSet};use crate::remote::{Record, send};use crate::scan::{ScanOptions, measure, modified_seconds};-// Open directory iterators are a bounded resource too. Fail explicitly,-// rather than silently truncating an exceptionally deep or hardlinked tree.-const MAX_ANCESTORS: usize = 256;+// Only unresolved multiply linked files need a file-identity table.const MAX_PENDING_LINKS: usize = 65_536;struct Frame {@@ -35,6 +33,7 @@ pub fn stream(root: &Path,options: &ScanOptions,out: &mut impl Write,+ live_progress: bool,) -> io::Result<()> {if options.follow_links || !options.one_filesystem {return Err(io::Error::other(@@ -48,7 +47,7 @@ pub fn stream(})?;let mut excluded: FxHashSet<PathBuf> = boundaries.into_iter().collect();excluded.extend(crate::space::never_scanned(root, root));- stream_with_boundaries(root, options, &excluded, out)+ stream_with_boundaries(root, options, &excluded, out, live_progress)}fn stream_with_boundaries(@@ -56,17 +55,31 @@ fn stream_with_boundaries(options: &ScanOptions,excluded: &FxHashSet<PathBuf>,out: &mut impl Write,+ live_progress: bool,) -> io::Result<()> {let device = fs::metadata(root)?.dev();let mut frames = vec![Frame::open(root, "root".into())];// Ordinary single-link files need no identity storage. Forget a shared// inode once all its names were met, and bound unmatched identities.let mut links = FxHashMap::<(u64, u64), u64>::default();+ let mut directory = root.to_path_buf();+ let mut heartbeat = None::<std::time::Instant>;while let Some(frame) = frames.last_mut() {+ if live_progress+ && heartbeat.is_none_or(|last| {+ last.elapsed() >= crate::remote_progress::INTERVAL+ })+ {+ send(out, &serde_json::json!({"directory": directory}))?;+ // Flush on a timer even when few tree records fill the buffer.+ out.flush()?;+ heartbeat = Some(std::time::Instant::now());+ }let entry = match frame.entries.as_mut().and_then(Iterator::next) {None => {let frame = frames.pop().expect("frame checked above");send(out, &frame.record)?;+ directory.pop();continue;}Some(Err(_)) => {@@ -100,12 +113,8 @@ fn stream_with_boundaries(if options.max_depth.is_some_and(|limit| frames.len() > limit) {continue;}- if frames.len() >= MAX_ANCESTORS {- return Err(io::Error::other(- "remote scan nesting limit reached; scan a smaller root",- ));- }frames.last_mut().expect("parent exists").record.6 += 1;+ directory.clone_from(&path);frames.push(Frame::open(&path, name.to_owned()));continue;}@@ -131,9 +140,10 @@ fn stream_with_boundaries(}} else {if links.len() >= MAX_PENDING_LINKS {- return Err(io::Error::other(- "remote hardlink tracking limit reached; scan a smaller root",- ));+ return Err(io::Error::other(format!(+ "remote hardlink tracking limit reached at {}; scan a smaller root",+ path.display()+ )));}links.insert(identity, meta.nlink() - 1);}@@ -170,6 +180,7 @@ mod tests {options,&excluded.iter().cloned().collect(),&mut bytes,+ false,).unwrap();bytes@@ -234,6 +245,21 @@ mod tests {assert_eq!(rows[1].6, 1);}+ #[test]+ fn a_deep_directory_does_not_abort_the_entire_stream() {+ let dir = tempfile::tempdir().unwrap();+ let mut path = dir.path().to_path_buf();+ for _ in 0..320 {+ path.push("d");+ fs::create_dir(&path).unwrap();+ }+ fs::write(path.join("leaf"), b"found").unwrap();+ let rows = records(dir.path(), &ScanOptions::default(), &[]);+ assert_eq!(rows.iter().filter(|row| row.1 == 0).count(), 321);+ assert!(rows.iter().any(|row| row.0 == "leaf"));+ assert!(!rows.iter().any(|row| row.5));+ }+#[test]fn unsafe_traversal_options_fail_before_walking() {let dir = tempfile::tempdir().unwrap();@@ -247,7 +273,9 @@ mod tests {..ScanOptions::default()},] {- assert!(stream(dir.path(), &options, &mut Vec::new()).is_err());+ assert!(+ stream(dir.path(), &options, &mut Vec::new(), false).is_err()+ );}}}diff --git a/crates/disktree-scan/src/main.rs b/crates/disktree-scan/src/main.rsindex 802b43e..2b9ad57 100644--- a/crates/disktree-scan/src/main.rs+++ b/crates/disktree-scan/src/main.rs@@ -83,9 +83,6 @@ fn limit_resources() -> io::Result<()> {maximum: Some(0),},)?;- eprintln!(- "Remote scanner memory limit: 1 GiB address space; narrow the scan if exhausted."- );Ok(())}diff --git a/crates/disktree-scan/tests/stdio.rs b/crates/disktree-scan/tests/stdio.rsindex 9782a9b..6d9a441 100644--- a/crates/disktree-scan/tests/stdio.rs+++ b/crates/disktree-scan/tests/stdio.rs@@ -125,14 +125,28 @@ fn a_live_supervised_client_receives_a_complete_snapshot() {#[test]fn memory_and_core_limits_are_installed_before_reading_a_request() {let mut helper = supervised_helper();- let mut notice = String::new();- BufReader::new(helper.0.stderr.take().unwrap())- .read_line(&mut notice)- .unwrap();- assert!(notice.contains("memory limit"));- let limits =- std::fs::read_to_string(format!("/proc/{}/limits", helper.0.id()))- .unwrap();+ // No startup chatter on stderr: wait for the kernel limit itself+ // while the helper is still blocked waiting for its first request.+ let deadline =+ std::time::Instant::now() + std::time::Duration::from_secs(2);+ let limits = loop {+ let limits =+ std::fs::read_to_string(format!("/proc/{}/limits", helper.0.id()))+ .unwrap();+ if limits.lines().any(|line| {+ line.starts_with("Max address space")+ && line+ .split_whitespace()+ .nth(3)+ .and_then(|value| value.parse::<u64>().ok())+ .is_some_and(|value| value <= 1 << 30)+ }) {+ break limits;+ }+ assert!(std::time::Instant::now() < deadline);+ assert!(helper.0.try_wait().unwrap().is_none());+ std::thread::yield_now();+ };let address = limits.lines().find(|line| line.starts_with("Max address space"))@@ -165,6 +179,7 @@ fn wide_stream_has_bounded_resident_memory() {&serde_json::json!({"root": dir.path(), "disk": false,"options": ScanOptions::default(), "watch_disconnect": true,+ "live_progress": true,}),).unwrap();