Skip to content

A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.

Rust

chore: add the scan-worker benchmark behind the Power Efficiency table

Harold Hunt5a170a1b1802
diff --git a/crates/disktree-core/examples/scan_workers.rs b/crates/disktree-core/examples/scan_workers.rs
new file mode 100644
index 0000000..8b79ecc
--- /dev/null
+++ b/crates/disktree-core/examples/scan_workers.rs
@@ -0,0 +1,60 @@
+//! One scan of `ROOT` with a fixed pool of `WORKERS`, the way a Power
+//! Efficiency preset runs it, reported as one JSON line. Driven by
+//! `scripts/bench-scan-workers.py`, which times it and samples the machine.
+//!
+//! ```sh
+//! cargo run --release -p disktree-core --example scan_workers -- ROOT WORKERS
+//! ```
+
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use disktree_core::scan::{ScanHandle, ScanOptions};
+use disktree_core::scan_threads::ScanThreads;
+
+fn main() {
+ let mut args = std::env::args_os().skip(1);
+ let (Some(root), Some(workers)) = (args.next(), args.next()) else {
+ eprintln!("usage: scan_workers ROOT WORKERS");
+ std::process::exit(2);
+ };
+ let Some(workers) = workers
+ .to_str()
+ .and_then(|value| value.parse::<usize>().ok())
+ .filter(|&value| value > 0)
+ else {
+ eprintln!("WORKERS must be a positive integer");
+ std::process::exit(2);
+ };
+ let options = ScanOptions {
+ // What `PowerEfficiency::policy` sets: a fixed pool, no governor.
+ threads: ScanThreads {
+ max_threads: workers,
+ adaptive: false,
+ system_cpu_limit: None,
+ ..ScanThreads::default()
+ },
+ ..ScanOptions::default()
+ };
+ let start = Instant::now();
+ let handle = ScanHandle::spawn(PathBuf::from(root), options);
+ let node = loop {
+ if let Some(result) = handle.poll() {
+ match result {
+ Ok(node) => break node,
+ Err(error) => {
+ eprintln!("scan failed: {error}");
+ std::process::exit(1);
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(20));
+ };
+ let elapsed = start.elapsed().as_secs_f64();
+ let progress = handle.progress.snapshot();
+ println!(
+ "{{\"workers\":{workers},\"pool\":{},\"elapsed\":{elapsed},\
+ \"files\":{},\"dirs\":{},\"bytes\":{},\"errors\":{}}}",
+ progress.threads, node.files, node.dirs, node.bytes, progress.errors
+ );
+}
diff --git a/scripts/bench-scan-workers.py b/scripts/bench-scan-workers.py
new file mode 100755
index 0000000..66b2e32
--- /dev/null
+++ b/scripts/bench-scan-workers.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Time whole scans at several fixed worker counts and print the PR table.
+
+Each run is one `scan_workers` example process under `/usr/bin/time`, so CPU
+time is the scan's own user + system seconds. Whole-machine busy time is
+sampled every half second while it runs (macOS `host_statistics`, Linux
+`/proc/stat`). Rounds rotate the order so drift in the machine or the cache
+does not favour one count. The first scan is an untimed warm-up.
+
+ scripts/bench-scan-workers.py /System/Volumes/Data --workers 18 8 4 2
+
+Hold the machine quiet: a VM, an indexer or a build skews every column.
+"""
+
+import argparse
+import ctypes
+import fcntl
+import json
+import os
+import pathlib
+import platform
+import re
+import statistics
+import subprocess
+import sys
+import time
+
+REPO = pathlib.Path(__file__).resolve().parent.parent
+EXAMPLE = REPO / "target/release/examples/scan_workers"
+
+
+def cpu_ticks():
+ """(busy, total) ticks for the whole machine since boot."""
+ if platform.system() == "Darwin":
+ lib = ctypes.CDLL("/usr/lib/libSystem.B.dylib")
+ lib.mach_host_self.restype = ctypes.c_uint
+ ticks = (ctypes.c_uint * 4)()
+ count = ctypes.c_uint(4)
+ # HOST_CPU_LOAD_INFO: user, system, idle, nice.
+ if lib.host_statistics(lib.mach_host_self(), 3, ticks, ctypes.byref(count)):
+ raise OSError("host_statistics failed")
+ return sum(ticks) - ticks[2], sum(ticks)
+ fields = [int(x) for x in open("/proc/stat").readline().split()[1:]]
+ idle = fields[3] + fields[4]
+ return sum(fields) - idle, sum(fields)
+
+
+def percent(value):
+ return "n/a" if value is None else f"{value:.0f}%"
+
+
+def busy_percent(before, after):
+ busy, total = (a - b for a, b in zip(after, before))
+ return 100 * busy / total if total else None
+
+
+def cpu_seconds(time_output):
+ """User + system seconds from BSD or GNU `/usr/bin/time` output."""
+ user = re.search(r"([\d.]+) user|User time \(seconds\): ([\d.]+)", time_output)
+ system = re.search(r"([\d.]+) sys|System time \(seconds\): ([\d.]+)", time_output)
+ return sum(float(next(g for g in m.groups() if g)) for m in (user, system))
+
+
+def scan(root, workers):
+ flag = "-l" if platform.system() == "Darwin" else "-v"
+ before = cpu_ticks()
+ started = time.monotonic()
+ process = subprocess.Popen(
+ ["/usr/bin/time", flag, str(EXAMPLE), root, str(workers)],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ # Its own group, so an interrupt never leaves a scan running.
+ start_new_session=True,
+ )
+ try:
+ out, err = process.communicate()
+ except BaseException:
+ os.killpg(process.pid, 9)
+ raise
+ if process.returncode:
+ sys.exit(f"scan failed:\n{err}")
+ run = json.loads(out.strip().splitlines()[-1])
+ run["wall"] = time.monotonic() - started
+ run["cpu_s"] = cpu_seconds(err)
+ run["machine"] = busy_percent(before, cpu_ticks())
+ return run
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ parser.add_argument("root", help="directory or volume to scan")
+ parser.add_argument("--workers", type=int, nargs="+", default=[18, 8, 4, 2])
+ parser.add_argument("--rounds", type=int, default=3)
+ parser.add_argument("--out", type=pathlib.Path, help="write every run as JSON here")
+ parser.add_argument("--lock", help="flock this file for the whole block")
+ args = parser.parse_args()
+
+ subprocess.run(
+ ["cargo", "build", "--release", "-p", "disktree-core", "--example", "scan_workers"],
+ cwd=REPO,
+ check=True,
+ )
+ lock = open(args.lock, "w") if args.lock else None
+ if lock:
+ fcntl.flock(lock, fcntl.LOCK_EX)
+ cores = os.cpu_count() or 1
+ base = max(args.workers)
+ print(f"warm-up: {base} workers", file=sys.stderr)
+ scan(args.root, base)
+ runs = []
+ for round_ in range(args.rounds):
+ shift = round_ % len(args.workers)
+ order = args.workers[shift:] + args.workers[:shift]
+ if round_ % 2:
+ order.reverse()
+ for workers in order:
+ run = scan(args.root, workers)
+ run["round"] = round_ + 1
+ runs.append(run)
+ print(
+ f"round {round_ + 1}, {workers} workers: {run['elapsed']:.1f} s, "
+ f"{run['cpu_s']:.0f} CPU-s, machine {percent(run['machine'])}",
+ file=sys.stderr,
+ )
+ if args.out:
+ args.out.write_text(json.dumps(runs, indent=1))
+
+ def median(workers, key):
+ values = [r[key] for r in runs if r["workers"] == workers and r[key] is not None]
+ return statistics.median(values) if values else None
+
+ # The scan's own share of every core, which background load cannot move.
+ for run in runs:
+ run["share"] = 100 * run["cpu_s"] / (run["elapsed"] * cores)
+ b_time, b_cpu, b_share = (median(base, k) for k in ("elapsed", "cpu_s", "share"))
+ print(f"\nMedian of {args.rounds}; Δ against {base} workers; {cores} cores.\n")
+ print("| Workers | Scan time | Δ time | CPU time | Δ CPU time | Machine used | Δ machine | Whole machine |")
+ print("| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |")
+ for workers in sorted(args.workers, reverse=True):
+ t, c, s, m = (median(workers, k) for k in ("elapsed", "cpu_s", "share", "machine"))
+ first = workers == base
+ pct = lambda x, y: "baseline" if first else f"{(x - y) / y * 100:+.0f}%"
+ pts = "baseline" if first else f"{s - b_share:+.0f} pts"
+ print(
+ f"| {workers} | {t:.0f} s | {pct(t, b_time)} | {c:,.0f} CPU-s | {pct(c, b_cpu)} "
+ f"| {s:.0f}% | {pts} | {percent(m)} |"
+ )
+
+
+if __name__ == "__main__":
+ main()