Skip to content

A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.

Rust

scan: stream live SSH progress and accept deeply nested trees

Tobi Lutke8d98a855e868
diff --git a/README.md b/README.md
index 738411b..625ec18 100644
--- a/README.md
+++ b/README.md
@@ -212,12 +212,21 @@ so outdated helpers reject them instead of silently scanning their home.
The Linux/macOS helper streams a depth-first walk directly to stdout. It
never constructs a tree or retains a directory's children: memory scales
-with open ancestors (at most 256), not file count. Only pending hardlink
+with open ancestors, not file count. Deep directories are not rejected
+at an arbitrary depth; filesystem errors are reported as unreadable. Only pending hardlink
identities are retained, capped at 65,536; ordinary files need no identity
storage. A resource limit fails explicitly instead of returning a partial
tree. Symlinks are not followed and every nested mount is skipped, even a
bind mount or same-device subvolume. Select those as separate roots.
+While indexing, the window shows live files, directories, measured bytes,
+read errors, elapsed time and average throughput. Remote scans also show a
+sampled recent directory and connection / helper setup / indexing / treemap
+stages. Progress uses a single replaceable snapshot, not a growing event
+queue; path updates are throttled to five per second. There is no guessed
+percentage: the total is unknown until the walk finishes. Older helpers
+still provide counters from their streamed records, without path samples.
+
One compressed SSH connection carries a bounded JSON request on stdin and
compact postorder records on stdout. The client keeps stdin open so the
helper exits on disconnect, including while scanning or blocked on output.
diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs
index 41d372d..5ad2871 100644
--- a/crates/disktree-app/src/state.rs
+++ b/crates/disktree-app/src/state.rs
@@ -329,6 +329,7 @@ pub struct Disktree {
pub remote: Option<String>,
remote_disk: bool,
pub remote_scan: Option<disktree_core::remote::RemoteScan>,
+ pub remote_progress: disktree_core::remote_progress::Progress,
remote_platform: String,
pub remote_identity: Option<disktree_core::identity::HostIdentity>,
pub home: Option<PathBuf>,
@@ -524,6 +525,8 @@ impl Disktree {
remote,
remote_disk,
remote_scan: None,
+ remote_progress: disktree_core::remote_progress::Progress::default(
+ ),
remote_platform: String::new(),
remote_identity: None,
home,
@@ -1236,6 +1239,8 @@ impl Disktree {
let epoch = self.scan_epoch;
self.progress = ScanSnapshot::default();
self.scan_error = None;
+ self.remote_progress =
+ disktree_core::remote_progress::Progress::default();
// A new scan starts at its root; where it was is one `<` away.
if !self.crumbs.is_empty() {
self.remember();
@@ -1324,7 +1329,13 @@ impl Disktree {
return false;
}
if let Some(scan) = &self.remote_scan {
- let Some(outcome) = scan.poll() else {
+ let outcome = scan.poll();
+ self.remote_progress = scan.progress();
+ self.progress.files = self.remote_progress.files;
+ self.progress.dirs = self.remote_progress.dirs;
+ self.progress.bytes = self.remote_progress.bytes;
+ self.progress.errors = self.remote_progress.errors;
+ let Some(outcome) = outcome else {
cx.notify();
return true;
};
diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs
index e6f1e8d..8bc9fe5 100644
--- a/crates/disktree-app/src/tests.rs
+++ b/crates/disktree-app/src/tests.rs
@@ -92,6 +92,34 @@ fn read<R>(
view.read_with(cx, |app, _| f(app))
}
+#[gpui_kit::test]
+fn remote_indexing_draws_live_activity_and_the_sampled_directory(
+ cx: &mut TestAppContext,
+) {
+ cx.update(gpui_omarchy::init);
+ let temp = fixture();
+ let (view, cx) = view_over(temp.path(), cx);
+ update(&view, cx, |app, _| {
+ app.tree = None;
+ app.remote = Some("fixture-only".into());
+ app.scan_started = Some(std::time::Instant::now());
+ app.progress.files = 1234;
+ app.progress.bytes = 123_456;
+ app.remote_progress.stage =
+ disktree_core::remote_progress::Stage::Indexing;
+ app.remote_progress.directory = Some(temp.path().join("junk/deeper"));
+ });
+ draw(cx);
+ assert!(cx.debug_bounds("scan-activity").is_some());
+ assert!(cx.debug_bounds("scan-directory").is_some());
+ update(&view, cx, |app, _| {
+ app.scan_error = Some("fixture scan failed".into());
+ });
+ draw(cx);
+ assert!(cx.debug_bounds("scan-error").is_some());
+ assert!(cx.debug_bounds("scan-reading").is_none());
+}
+
#[gpui_kit::test]
fn startup_chooser_never_scans_before_explicit_acceptance(
cx: &mut TestAppContext,
diff --git a/crates/disktree-app/src/ui.rs b/crates/disktree-app/src/ui.rs
index 3e21ed4..3f43a67 100644
--- a/crates/disktree-app/src/ui.rs
+++ b/crates/disktree-app/src/ui.rs
@@ -85,7 +85,6 @@ pub mod size {
/// A thin meter: share of the scan, the disk.
pub const METER: Rems = Rems(0.3125);
/// The meter on the scanning panel.
- pub const SCANNING_METER: Rems = Rems(26.25);
/// The Size | Files | Age choice in the settings row.
pub const RANKING_CHOICE: Rems = Rems(11.0);
/// The review screen's summary column.
diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs
index 78f1470..b9ea289 100644
--- a/crates/disktree-app/src/views.rs
+++ b/crates/disktree-app/src/views.rs
@@ -2450,6 +2450,14 @@ fn scanning_panel(
.text_size(text::BODY)
.child(error.clone()),
)
+ .children(app.remote_progress.directory.as_ref().map(|path| {
+ div()
+ .max_w(size::HELP)
+ .truncate()
+ .text_size(text::CAPTION)
+ .text_color(theme.secondary)
+ .child(format!("Last directory: {}", path.display()))
+ }))
.child(widgets::hint("r", "retry scan", cx));
}
let progress = &app.progress;
@@ -2479,8 +2487,11 @@ fn scanning_panel(
.text_color(theme.bright)
.child(if cancelled {
format!("Stopped reading {}", widgets::display_root(app))
+ } else if app.remote.is_some() {
+ format!("{} · {}", app.remote_progress.stage.label(),
+ widgets::display_root(app))
} else {
- format!("Reading {}", widgets::display_root(app))
+ format!("Indexing {}", widgets::display_root(app))
}),
)
.child(
@@ -2509,16 +2520,18 @@ fn scanning_panel(
.when(!cancelled, |panel| {
panel.child(
div()
- .w(size::SCANNING_METER)
- .child(widgets::meter_row(
- "",
- "",
- progress_estimate(progress.files),
- theme.accent,
- cx,
- )),
+ .debug_selector(|| "scan-activity".into())
+ .text_size(text::BODY)
+ .text_color(theme.accent)
+ .child(scan_activity(app)),
)
})
+ .children(app.remote_progress.directory.as_ref().map(|path| {
+ div().debug_selector(|| "scan-directory".into())
+ .max_w(size::HELP).min_w_0().truncate()
+ .text_size(text::CAPTION).text_color(theme.secondary)
+ .child(format!("Recent directory: {}", path.display()))
+ }))
.child(
div()
.text_size(text::BODY)
@@ -2526,7 +2539,7 @@ fn scanning_panel(
.child(if cancelled {
"Nothing is shown from a scan that did not finish."
} else {
- "Marking, zooming and the free-space meter all work as soon as it lands."
+ "Total unknown until indexing finishes. Results stream live."
}),
)
// A failed scan has nothing left to cancel either.
@@ -2545,6 +2558,24 @@ fn scanning_panel(
})
}
+fn scan_activity(app: &Disktree) -> String {
+ let elapsed = app
+ .scan_started
+ .map_or(0.0, |start| start.elapsed().as_secs_f64());
+ let phase = (elapsed * 8.0) as usize % 4;
+ let spinner = ["◐", "◓", "◑", "◒"][phase];
+ let rate = app.progress.files as f64 / elapsed.max(1.0);
+ let quiet = app
+ .remote_progress
+ .updated
+ .map_or(0, |at| at.elapsed().as_secs());
+ if app.remote.is_some() && quiet >= 5 {
+ format!("{spinner} {elapsed:.0}s elapsed · no new entries for {quiet}s")
+ } else {
+ format!("{spinner} {elapsed:.0}s elapsed · {rate:.0} files/s avg")
+ }
+}
+
fn find_field(app: &Disktree, theme: &Theme) -> Div {
// What the text matches, said beside it as it is typed, and what Enter
// and Escape will do with it.
@@ -2648,18 +2679,6 @@ fn marks_ancestor(
.next()
}
-/// A scan has no total to measure against, so the meter shows the work done
-/// rather than pretending to know how far along it is.
-fn progress_estimate(files: u64) -> f32 {
- if files == 0 {
- 0.06
- } else {
- // Asymptotic: the bar keeps moving while the walk continues.
- let scaled = files as f32 / (files as f32 + 20_000.0);
- (0.1 + scaled * 0.85).min(0.99)
- }
-}
-
// ── review ──────────────────────────────────────────────────────────────
fn review(
@@ -3375,7 +3394,7 @@ fn done(app: &Disktree, cx: &gpui_kit::Context<'_, Disktree>) -> Div {
body = body.child(widgets::meter_row(
"re-scanning",
format!("{} files", widgets::human_count(app.progress.files)),
- progress_estimate(app.progress.files),
+ 0.0,
theme.accent,
cx,
));
diff --git a/crates/disktree-core/src/deployment.rs b/crates/disktree-core/src/deployment.rs
index d856074..557f9aa 100644
--- a/crates/disktree-core/src/deployment.rs
+++ b/crates/disktree-core/src/deployment.rs
@@ -51,8 +51,18 @@ pub fn deploy(
archive: &Archive,
input: &mut impl BufRead,
output: &mut impl Write,
+) -> io::Result<()> {
+ deploy_with_progress(archive, input, output, |_| {})
+}
+
+pub(crate) fn deploy_with_progress(
+ archive: &Archive,
+ input: &mut impl BufRead,
+ output: &mut impl Write,
+ mut progress: impl FnMut(crate::remote_progress::Stage),
) -> io::Result<()> {
let hello = control_line(input)?;
+ progress(crate::remote_progress::Stage::CheckingHelper);
let parts: Vec<_> = hello.split_whitespace().collect();
if parts.len() != 3 || parts[0] != "DISKTREE_BOOTSTRAP_V1" {
return Err(io::Error::other("unexpected SSH deployment handshake"));
@@ -77,6 +87,7 @@ pub fn deploy(
output.flush()?;
let mut state = control_line(input)?;
if state == "DISKTREE_UPLOAD_V1" {
+ progress(crate::remote_progress::Stage::UploadingHelper);
output.write_all(&bytes)?;
output.flush()?;
state = control_line(input)?;
diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rs
index b25f711..5b020a1 100644
--- a/crates/disktree-core/src/lib.rs
+++ b/crates/disktree-core/src/lib.rs
@@ -18,6 +18,7 @@ mod macos;
mod mft;
pub mod payload;
pub mod remote;
+pub mod remote_progress;
#[cfg(unix)]
mod remote_walk;
pub mod removal;
diff --git a/crates/disktree-core/src/remote.rs b/crates/disktree-core/src/remote.rs
index 5608b77..cbe453b 100644
--- a/crates/disktree-core/src/remote.rs
+++ b/crates/disktree-core/src/remote.rs
@@ -11,11 +11,13 @@ use std::sync::{Arc, Mutex, mpsc};
use serde::{Deserialize, Serialize};
+use crate::remote_progress::{INTERVAL, Progress, Shared, Stage, stage};
use crate::scan::ScanOptions;
#[cfg(not(unix))]
use crate::scan::scan;
use crate::space::{SpaceInfo, space_info, volume_root_for};
use crate::tree::{Node, NodeKind, aggregate};
+use std::time::Instant;
const VERSION: u32 = 1;
const MAX_LINE: u64 = 1 << 20;
@@ -57,6 +59,9 @@ pub struct SessionRequest {
pub operation: Operation,
#[serde(default)]
pub watch_disconnect: bool,
+ // Opt-in keeps the postorder-only wire format for older clients.
+ #[serde(default)]
+ pub live_progress: bool,
}
/// The target's root candidates, without walking or probing any of them.
@@ -166,7 +171,9 @@ pub fn serve_session(
out: &mut impl Write,
) -> io::Result<()> {
match session.operation {
- Operation::Scan(request) => serve_request(request, out),
+ Operation::Scan(request) => {
+ serve_request_with_progress(request, out, session.live_progress)
+ }
Operation::Discover { discover_version } => {
if discover_version != VERSION {
return Err(io::Error::other(
@@ -190,6 +197,15 @@ pub fn serve_session(
/// Scan a decoded request. The standalone helper supervises its lifetime.
pub fn serve_request(request: Request, out: &mut impl Write) -> io::Result<()> {
+ serve_request_with_progress(request, out, false)
+}
+
+fn serve_request_with_progress(
+ request: Request,
+ out: &mut impl Write,
+ live_progress: bool,
+) -> io::Result<()> {
+ let _ = live_progress; // Only Unix has the streaming walker.
let home = std::env::home_dir();
let root = request
.root
@@ -218,7 +234,7 @@ pub fn serve_request(request: Request, out: &mut impl Write) -> io::Result<()> {
)?;
out.flush()?;
#[cfg(unix)]
- crate::remote_walk::stream(&root, &request.options, out)?;
+ crate::remote_walk::stream(&root, &request.options, out, live_progress)?;
#[cfg(not(unix))]
write_node(out, &scan(&root, request.options)?)?;
// An explicit terminator distinguishes a complete scan from a severed SSH
@@ -257,16 +273,41 @@ fn write_node(out: &mut impl Write, node: &Node) -> io::Result<()> {
}
pub fn read_snapshot(input: &mut impl BufRead) -> io::Result<Snapshot> {
+ read_snapshot_with_progress(input, &Shared::default())
+}
+
+fn read_snapshot_with_progress(
+ input: &mut impl BufRead,
+ shared: &Shared,
+) -> io::Result<Snapshot> {
let header: Header = receive(input)?;
if header.version != VERSION || !header.root.is_absolute() {
return Err(io::Error::other("incompatible remote helper protocol"));
}
+ let mut progress = Progress {
+ stage: Stage::Indexing,
+ directory: Some(header.root.clone()),
+ updated: Some(Instant::now()),
+ ..Progress::default()
+ };
+ *shared.lock().expect("remote progress lock") = progress.clone();
+ let mut published = Instant::now();
let mut stack = Vec::<Node>::new();
loop {
let value: serde_json::Value = receive(input)?;
if value == "done" {
break;
}
+ if let Some(directory) = value.get("directory") {
+ progress.directory = Some(
+ serde_json::from_value(directory.clone())
+ .map_err(json_error)?,
+ );
+ progress.updated = Some(Instant::now());
+ *shared.lock().expect("remote progress lock") = progress.clone();
+ published = Instant::now();
+ continue;
+ }
let Record(name, kind, bytes, files, modified, read_error, count) =
serde_json::from_value(value).map_err(json_error)?;
// Names must remain single components when joined to marked paths.
@@ -297,14 +338,36 @@ pub fn read_snapshot(input: &mut impl BufRead) -> io::Result<Snapshot> {
if !kind.is_dir() && count != 0 {
return Err(io::Error::other("remote leaf has children"));
}
- let children = stack.split_off(stack.len() - count);
+ // The final root often owns every stacked node. Transfer that
+ // allocation instead of briefly duplicating a wide tree's storage.
+ let children = if count == stack.len() {
+ std::mem::take(&mut stack)
+ } else {
+ stack.split_off(stack.len() - count)
+ };
let mut node = Node::entry(name, kind, bytes);
node.files = files;
node.modified = modified;
node.read_error = read_error;
node.children = children;
stack.push(node);
+ progress.dirs += u64::from(kind.is_dir());
+ // Legacy helpers include direct totals in directory records too.
+ // Count each leaf once, never charge its parent's totals again.
+ if !kind.is_dir() {
+ progress.files += files;
+ progress.bytes += bytes;
+ }
+ progress.errors += u64::from(read_error);
+ if published.elapsed() >= INTERVAL {
+ progress.updated = Some(Instant::now());
+ *shared.lock().expect("remote progress lock") = progress.clone();
+ published = Instant::now();
+ }
}
+ progress.stage = Stage::Finalizing;
+ progress.updated = Some(Instant::now());
+ *shared.lock().expect("remote progress lock") = progress;
if stack.len() != 1 {
return Err(io::Error::other("incomplete remote tree"));
}
@@ -369,8 +432,7 @@ fn ssh_error(host: &str, details: &str, fallback: &io::Error) -> io::Error {
Run make install-scan from the checkout on the server."
)
} else {
- "Check the server's SSH connection and disktree-scan installation."
- .into()
+ "No incomplete scan is shown. Retry, or choose another root.".into()
};
io::Error::other(format!("SSH {host}: {reason}\n\n{hint}"))
}
@@ -383,6 +445,7 @@ fn ssh_error(host: &str, details: &str, fallback: &io::Error) -> io::Error {
pub struct RemoteJob<T> {
child: Arc<Mutex<Child>>,
result: mpsc::Receiver<io::Result<T>>,
+ progress: Shared,
}
pub type RemoteScan = RemoteJob<Snapshot>;
@@ -442,8 +505,9 @@ impl RemoteScan {
host,
Operation::Scan(request.clone()),
archive,
- move |input| {
- let mut snapshot = read_snapshot(input)?;
+ move |input, progress| {
+ let mut snapshot =
+ read_snapshot_with_progress(input, progress)?;
if metric != crate::tree::Metric::Bytes {
aggregate(&mut snapshot.tree, metric);
}
@@ -463,7 +527,7 @@ impl RemoteRoots {
discover_version: VERSION,
},
archive,
- read_roots,
+ |input, _| read_roots(input),
)
}
}
@@ -476,6 +540,7 @@ impl<T: Send + 'static> RemoteJob<T> {
archive: Option<crate::payload::Archive>,
decode: impl FnOnce(
&mut BufReader<std::process::ChildStdout>,
+ &Shared,
) -> io::Result<T>
+ Send
+ 'static,
@@ -494,22 +559,29 @@ impl<T: Send + 'static> RemoteJob<T> {
let worker = child.clone();
let host = host.to_owned();
let (tx, result) = mpsc::channel();
+ let progress = Shared::default();
+ let live = progress.clone();
std::thread::spawn(move || {
let mut input = BufReader::new(stdout);
let response = (|| {
if let Some(archive) = archive {
- crate::deployment::deploy(
- &archive, &mut input, &mut stdin,
+ crate::deployment::deploy_with_progress(
+ &archive,
+ &mut input,
+ &mut stdin,
+ |value| stage(&live, value),
)?;
}
+ stage(&live, Stage::Starting);
send(
&mut stdin,
&SessionRequest {
operation,
watch_disconnect: true,
+ live_progress: true,
},
)?;
- decode(&mut input)
+ decode(&mut input, &live)
})();
drop(stdin);
if response.is_err() {
@@ -541,7 +613,15 @@ impl<T: Send + 'static> RemoteJob<T> {
};
let _ = tx.send(outcome);
});
- Ok(Self { child, result })
+ Ok(Self {
+ child,
+ result,
+ progress,
+ })
+ }
+
+ pub fn progress(&self) -> Progress {
+ self.progress.lock().expect("remote progress lock").clone()
}
pub fn poll(&self) -> Option<io::Result<T>> {
@@ -640,6 +720,7 @@ mod tests {
discover_version: VERSION,
},
watch_disconnect: true,
+ live_progress: true,
};
let bytes = serde_json::to_vec(&session).unwrap();
assert!(serde_json::from_slice::<Request>(&bytes).is_err());
@@ -778,6 +859,14 @@ mod tests {
let home = tempfile::tempdir().unwrap();
let data = tempfile::tempdir().unwrap();
std::fs::write(data.path().join("keep"), b"hello").unwrap();
+ // Exercise the shipped static helper beyond the removed 256-level
+ // cutoff, through deployment and decoding, not just the walker.
+ let mut deep = data.path().to_path_buf();
+ for _ in 0..320 {
+ deep.push("d");
+ std::fs::create_dir(&deep).unwrap();
+ }
+ std::fs::write(deep.join("leaf"), b"world").unwrap();
let cache = home.path().join("cache");
let remote = format!(
"HOME={} XDG_CACHE_HOME={}; export HOME XDG_CACHE_HOME;\n{}",
@@ -810,7 +899,9 @@ mod tests {
)
.unwrap();
let snapshot = wait(&handle).unwrap();
- assert_eq!(snapshot.tree.bytes, 5);
+ assert_eq!(snapshot.tree.bytes, 10);
+ assert_eq!(snapshot.tree.files, 2);
+ assert_eq!(handle.progress().files, 2);
assert!(snapshot.header.identity.is_some());
assert_eq!(std::fs::read(&cached).unwrap(), bytes);
if attempt == 1 {
@@ -851,7 +942,7 @@ mod tests {
discover_version: VERSION,
},
Some(archive),
- read_roots,
+ |input, _| read_roots(input),
)
.unwrap();
let roots = wait(&job).unwrap();
@@ -930,6 +1021,57 @@ mod tests {
assert!(snapshot.tree.child_named(r"back\slash").is_some());
}
+ #[cfg(unix)]
+ #[test]
+ fn live_counters_are_available_before_the_tree_finishes() {
+ use std::os::unix::net::UnixStream;
+ let dir = tempfile::tempdir().unwrap();
+ std::fs::write(dir.path().join("file"), b"hello").unwrap();
+ let request = Request {
+ root: Some(dir.path().into()),
+ disk: false,
+ options: ScanOptions {
+ apparent_size: true,
+ ..ScanOptions::default()
+ },
+ };
+ let mut data = Vec::new();
+ serve_request(request, &mut data).unwrap();
+ let mut lines = data.split_inclusive(|byte| *byte == b'\n');
+ let header = lines.next().unwrap();
+ let leaf = lines.next().unwrap();
+ let tail = lines.flatten().copied().collect::<Vec<_>>();
+ let (mut output, input) = UnixStream::pair().unwrap();
+ input
+ .set_read_timeout(Some(std::time::Duration::from_secs(5)))
+ .unwrap();
+ let shared = Shared::default();
+ let live = shared.clone();
+ let worker = std::thread::spawn(move || {
+ read_snapshot_with_progress(&mut BufReader::new(input), &live)
+ });
+ output.write_all(header).unwrap();
+ output.write_all(leaf).unwrap();
+ send(&mut output, &serde_json::json!({"directory": dir.path()}))
+ .unwrap();
+ let deadline = Instant::now() + std::time::Duration::from_secs(2);
+ loop {
+ let progress = shared.lock().unwrap().clone();
+ if progress.files == 1 {
+ assert_eq!(progress.bytes, 5);
+ assert_eq!(progress.stage, Stage::Indexing);
+ assert!(!worker.is_finished());
+ break;
+ }
+ assert!(Instant::now() < deadline);
+ std::thread::yield_now();
+ }
+ output.write_all(&tail).unwrap();
+ let snapshot = worker.join().unwrap().unwrap();
+ assert_eq!(snapshot.tree.bytes, 5);
+ assert_eq!(shared.lock().unwrap().stage, Stage::Finalizing);
+ }
+
#[test]
fn protocol_roundtrip_and_truncation() {
let dir = tempfile::tempdir().unwrap();
@@ -947,6 +1089,7 @@ mod tests {
let mut input = Vec::new();
send(&mut input, &request).unwrap();
serve(&mut input.as_slice(), &mut data).unwrap();
+ assert!(!data.windows(11).any(|part| part == b"\"directory\""));
let snapshot = read_snapshot(&mut data.as_slice()).unwrap();
assert_eq!(snapshot.tree.bytes, 5);
assert_eq!(snapshot.tree.files, 1);
diff --git a/crates/disktree-core/src/remote_progress.rs b/crates/disktree-core/src/remote_progress.rs
new file mode 100644
index 0000000..8a78fa8
--- /dev/null
+++ b/crates/disktree-core/src/remote_progress.rs
@@ -0,0 +1,49 @@
+//! One replaceable progress snapshot, never a queue of filesystem events.
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum Stage {
+ #[default]
+ Connecting,
+ CheckingHelper,
+ UploadingHelper,
+ Starting,
+ Indexing,
+ Finalizing,
+}
+
+impl Stage {
+ pub const fn label(self) -> &'static str {
+ match self {
+ Self::Connecting => "Connecting over SSH",
+ Self::CheckingHelper => "Checking remote scanner",
+ Self::UploadingHelper => "Uploading remote scanner",
+ Self::Starting => "Starting remote scanner",
+ Self::Indexing => "Indexing remotely",
+ Self::Finalizing => "Building the treemap",
+ }
+ }
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct Progress {
+ pub stage: Stage,
+ pub files: u64,
+ pub dirs: u64,
+ pub bytes: u64,
+ pub errors: u64,
+ pub directory: Option<PathBuf>,
+ pub updated: Option<Instant>,
+}
+
+/// Cap both wire heartbeats and UI publications at five per second.
+pub(crate) const INTERVAL: Duration = Duration::from_millis(200);
+pub(crate) type Shared = Arc<Mutex<Progress>>;
+
+pub(crate) fn stage(shared: &Shared, value: Stage) {
+ let mut progress = shared.lock().expect("remote progress lock");
+ progress.stage = value;
+ progress.updated = Some(Instant::now());
+}
diff --git a/crates/disktree-core/src/remote_walk.rs b/crates/disktree-core/src/remote_walk.rs
index 64c2667..b845070 100644
--- a/crates/disktree-core/src/remote_walk.rs
+++ b/crates/disktree-core/src/remote_walk.rs
@@ -10,9 +10,7 @@ use rustc_hash::{FxHashMap, FxHashSet};
use crate::remote::{Record, send};
use crate::scan::{ScanOptions, measure, modified_seconds};
-// Open directory iterators are a bounded resource too. Fail explicitly,
-// rather than silently truncating an exceptionally deep or hardlinked tree.
-const MAX_ANCESTORS: usize = 256;
+// Only unresolved multiply linked files need a file-identity table.
const MAX_PENDING_LINKS: usize = 65_536;
struct Frame {
@@ -35,6 +33,7 @@ pub fn stream(
root: &Path,
options: &ScanOptions,
out: &mut impl Write,
+ live_progress: bool,
) -> io::Result<()> {
if options.follow_links || !options.one_filesystem {
return Err(io::Error::other(
@@ -48,7 +47,7 @@ pub fn stream(
})?;
let mut excluded: FxHashSet<PathBuf> = boundaries.into_iter().collect();
excluded.extend(crate::space::never_scanned(root, root));
- stream_with_boundaries(root, options, &excluded, out)
+ stream_with_boundaries(root, options, &excluded, out, live_progress)
}
fn stream_with_boundaries(
@@ -56,17 +55,31 @@ fn stream_with_boundaries(
options: &ScanOptions,
excluded: &FxHashSet<PathBuf>,
out: &mut impl Write,
+ live_progress: bool,
) -> io::Result<()> {
let device = fs::metadata(root)?.dev();
let mut frames = vec![Frame::open(root, "root".into())];
// Ordinary single-link files need no identity storage. Forget a shared
// inode once all its names were met, and bound unmatched identities.
let mut links = FxHashMap::<(u64, u64), u64>::default();
+ let mut directory = root.to_path_buf();
+ let mut heartbeat = None::<std::time::Instant>;
while let Some(frame) = frames.last_mut() {
+ if live_progress
+ && heartbeat.is_none_or(|last| {
+ last.elapsed() >= crate::remote_progress::INTERVAL
+ })
+ {
+ send(out, &serde_json::json!({"directory": directory}))?;
+ // Flush on a timer even when few tree records fill the buffer.
+ out.flush()?;
+ heartbeat = Some(std::time::Instant::now());
+ }
let entry = match frame.entries.as_mut().and_then(Iterator::next) {
None => {
let frame = frames.pop().expect("frame checked above");
send(out, &frame.record)?;
+ directory.pop();
continue;
}
Some(Err(_)) => {
@@ -100,12 +113,8 @@ fn stream_with_boundaries(
if options.max_depth.is_some_and(|limit| frames.len() > limit) {
continue;
}
- if frames.len() >= MAX_ANCESTORS {
- return Err(io::Error::other(
- "remote scan nesting limit reached; scan a smaller root",
- ));
- }
frames.last_mut().expect("parent exists").record.6 += 1;
+ directory.clone_from(&path);
frames.push(Frame::open(&path, name.to_owned()));
continue;
}
@@ -131,9 +140,10 @@ fn stream_with_boundaries(
}
} else {
if links.len() >= MAX_PENDING_LINKS {
- return Err(io::Error::other(
- "remote hardlink tracking limit reached; scan a smaller root",
- ));
+ return Err(io::Error::other(format!(
+ "remote hardlink tracking limit reached at {}; scan a smaller root",
+ path.display()
+ )));
}
links.insert(identity, meta.nlink() - 1);
}
@@ -170,6 +180,7 @@ mod tests {
options,
&excluded.iter().cloned().collect(),
&mut bytes,
+ false,
)
.unwrap();
bytes
@@ -234,6 +245,21 @@ mod tests {
assert_eq!(rows[1].6, 1);
}
+ #[test]
+ fn a_deep_directory_does_not_abort_the_entire_stream() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut path = dir.path().to_path_buf();
+ for _ in 0..320 {
+ path.push("d");
+ fs::create_dir(&path).unwrap();
+ }
+ fs::write(path.join("leaf"), b"found").unwrap();
+ let rows = records(dir.path(), &ScanOptions::default(), &[]);
+ assert_eq!(rows.iter().filter(|row| row.1 == 0).count(), 321);
+ assert!(rows.iter().any(|row| row.0 == "leaf"));
+ assert!(!rows.iter().any(|row| row.5));
+ }
+
#[test]
fn unsafe_traversal_options_fail_before_walking() {
let dir = tempfile::tempdir().unwrap();
@@ -247,7 +273,9 @@ mod tests {
..ScanOptions::default()
},
] {
- assert!(stream(dir.path(), &options, &mut Vec::new()).is_err());
+ assert!(
+ stream(dir.path(), &options, &mut Vec::new(), false).is_err()
+ );
}
}
}
diff --git a/crates/disktree-scan/src/main.rs b/crates/disktree-scan/src/main.rs
index 802b43e..2b9ad57 100644
--- a/crates/disktree-scan/src/main.rs
+++ b/crates/disktree-scan/src/main.rs
@@ -83,9 +83,6 @@ fn limit_resources() -> io::Result<()> {
maximum: Some(0),
},
)?;
- eprintln!(
- "Remote scanner memory limit: 1 GiB address space; narrow the scan if exhausted."
- );
Ok(())
}
diff --git a/crates/disktree-scan/tests/stdio.rs b/crates/disktree-scan/tests/stdio.rs
index 9782a9b..6d9a441 100644
--- a/crates/disktree-scan/tests/stdio.rs
+++ b/crates/disktree-scan/tests/stdio.rs
@@ -125,14 +125,28 @@ fn a_live_supervised_client_receives_a_complete_snapshot() {
#[test]
fn memory_and_core_limits_are_installed_before_reading_a_request() {
let mut helper = supervised_helper();
- let mut notice = String::new();
- BufReader::new(helper.0.stderr.take().unwrap())
- .read_line(&mut notice)
- .unwrap();
- assert!(notice.contains("memory limit"));
- let limits =
- std::fs::read_to_string(format!("/proc/{}/limits", helper.0.id()))
- .unwrap();
+ // No startup chatter on stderr: wait for the kernel limit itself
+ // while the helper is still blocked waiting for its first request.
+ let deadline =
+ std::time::Instant::now() + std::time::Duration::from_secs(2);
+ let limits = loop {
+ let limits =
+ std::fs::read_to_string(format!("/proc/{}/limits", helper.0.id()))
+ .unwrap();
+ if limits.lines().any(|line| {
+ line.starts_with("Max address space")
+ && line
+ .split_whitespace()
+ .nth(3)
+ .and_then(|value| value.parse::<u64>().ok())
+ .is_some_and(|value| value <= 1 << 30)
+ }) {
+ break limits;
+ }
+ assert!(std::time::Instant::now() < deadline);
+ assert!(helper.0.try_wait().unwrap().is_none());
+ std::thread::yield_now();
+ };
let address = limits
.lines()
.find(|line| line.starts_with("Max address space"))
@@ -165,6 +179,7 @@ fn wide_stream_has_bounded_resident_memory() {
&serde_json::json!({
"root": dir.path(), "disk": false,
"options": ScanOptions::default(), "watch_disconnect": true,
+ "live_progress": true,
}),
)
.unwrap();