Skip to content

A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.

Rust

feat(scan): read the NTFS master file table when elevated

Shishir Pai0b1725d40e19
diff --git a/crates/disktree-core/Cargo.toml b/crates/disktree-core/Cargo.toml
index a1eb131..84387db 100644
--- a/crates/disktree-core/Cargo.toml
+++ b/crates/disktree-core/Cargo.toml
@@ -22,7 +22,13 @@ rustix = { version = "1.1", features = ["fs"] }
# The Recycle Bin, through the shell's own file operation.
trash = { version = "5.2.9", default-features = false, features = ["coinit_apartmentthreaded"] }
winapi-util = "0.1.11"
-windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
+windows-sys = { version = "0.61", features = [
+ "Win32_Foundation",
+ "Win32_Storage_FileSystem",
+ "Win32_System_Com",
+ "Win32_UI_Shell",
+ "Win32_UI_WindowsAndMessaging",
+] }
# The system trash, through NSFileManager: what Finder's Move to Trash does,
# without asking to control Finder. Only this backend is used; the crate's
diff --git a/crates/disktree-core/src/access.rs b/crates/disktree-core/src/access.rs
index d19abc1..a33025d 100644
--- a/crates/disktree-core/src/access.rs
+++ b/crates/disktree-core/src/access.rs
@@ -5,7 +5,14 @@
//! with `EPERM`, silently. A disk tool that only counts those as errors
//! leaves the user guessing, so the app asks once whether access is granted
//! and, if not, says why folders were unreadable and where to fix it.
+//!
+//! On Windows the same question is whether disktree runs as an
+//! administrator: only then can it read a whole drive from its file table,
+//! several times faster than walking it and including folders a walk is
+//! refused. Without, it walks.
+use std::ffi::OsString;
+use std::io;
use std::path::Path;
/// System Settings, opened at Privacy & Security › Full Disk Access.
@@ -33,6 +40,47 @@ pub fn full_disk_access(home: &Path) -> Option<bool> {
}
}
+/// Whether this process runs as an administrator: `None` off Windows,
+/// where nothing a scan does depends on it.
+#[cfg(windows)]
+pub fn administrator() -> Option<bool> {
+ Some(crate::windows::elevated())
+}
+
+/// Whether this process runs as an administrator: `None` off Windows,
+/// where nothing a scan does depends on it.
+#[cfg(not(windows))]
+pub const fn administrator() -> Option<bool> {
+ None
+}
+
+/// Whether an administrator would read `root` from its file table: a
+/// whole NTFS drive. Always `false` off Windows.
+#[cfg(windows)]
+pub fn file_table_readable(root: &Path) -> bool {
+ crate::windows::file_table_readable(root)
+}
+
+/// Whether an administrator would read `root` from its file table: a
+/// whole NTFS drive. Always `false` off Windows.
+#[cfg(not(windows))]
+pub const fn file_table_readable(root: &Path) -> bool {
+ let _ = root;
+ false
+}
+
+/// Start this program again as an administrator, with `args`, through the
+/// UAC prompt. An error if the prompt is declined, or off Windows.
+pub fn restart_as_administrator(args: &[OsString]) -> io::Result<()> {
+ #[cfg(windows)]
+ return crate::windows::run_elevated(&std::env::current_exe()?, args);
+ #[cfg(not(windows))]
+ {
+ let _ = args;
+ Err(io::ErrorKind::Unsupported.into())
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rs
index 4d72191..9d2ee86 100644
--- a/crates/disktree-core/src/lib.rs
+++ b/crates/disktree-core/src/lib.rs
@@ -10,6 +10,8 @@ pub mod classify;
pub mod export;
pub mod filter;
pub mod insights;
+#[cfg(windows)]
+mod mft;
pub mod removal;
pub mod scan;
pub mod size;
diff --git a/crates/disktree-core/src/mft.rs b/crates/disktree-core/src/mft.rs
new file mode 100644
index 0000000..54ea26e
--- /dev/null
+++ b/crates/disktree-core/src/mft.rs
@@ -0,0 +1,1713 @@
+//! Measuring an NTFS volume from its master file table, the way `WizTree`
+//! does: one read of every file record instead of a listing per directory.
+//! On a disk of four million files the walk opens six hundred thousand
+//! directories; this reads five gigabytes in a few large requests.
+//!
+//! Only for a whole drive, `C:\`. The table costs the same to read whatever
+//! part of it is wanted, so for a small directory the walk is faster.
+//!
+//! The table is read as it is on disk. NTFS writes changed records back
+//! lazily, so changes other programs made in the last few seconds may not
+//! show yet. disktree flushes the volume after its own removals, so a
+//! rescan right after one sees what went. Flushing here instead would
+//! need a write-capable volume open on every scan, which Controlled
+//! Folder Access blocks and reports.
+//!
+//! Reading the volume needs an administrator. Anything this cannot do,
+//! from a refused open to a record layout it does not expect, returns
+//! `None` and the walk measures instead, so this is only ever a faster way
+//! to the same tree.
+
+use std::fs::{File, OpenOptions};
+use std::io;
+use std::os::windows::fs::{FileExt as _, OpenOptionsExt as _};
+use std::path::Path;
+
+use rayon::prelude::*;
+use windows_sys::Win32::Storage::FileSystem::{
+ FILE_ATTRIBUTE_HIDDEN, FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS,
+ FILE_ATTRIBUTE_RECALL_ON_OPEN, FILE_ATTRIBUTE_REPARSE_POINT,
+ FILE_FLAG_NO_BUFFERING, FILE_SHARE_DELETE, FILE_SHARE_READ,
+ FILE_SHARE_WRITE,
+};
+
+use crate::scan::{ScanOptions, ScanProgress};
+use crate::tree::{Node, NodeKind};
+use crate::windows::{Aligned, drive_letter};
+
+/// Most bytes read per call: large enough that the disk streams, small
+/// enough that many reads are outstanding at once.
+const CHUNK_BYTES: u64 = 16 << 20;
+
+/// Free bytes a read spans rather than stop and start another. Through
+/// `BitLocker`, 16 threads reading 1 MiB each moved 2 GB/s and 16 MiB
+/// each 4.2 GB/s: a request costs about 0.29 ms on top of 0.23 ms per
+/// MiB, as much as reading 1.3 MiB more, so a shorter free stretch is
+/// cheaper read than skipped. On a file of synthetic records with free
+/// stretches of about 600 records every 2000, reading through them took
+/// 1028 ms against 1209 ms for stopping at 512 records; on a table
+/// without free stretches the two plans are the same. An elevated scan of
+/// a 4-million-record `C:\` took 3.44 s at 1 MiB against 3.60 s at 4 MiB
+/// (median of 6 alternating runs).
+const GAP_BYTES: u64 = 1 << 20;
+
+/// The root directory's record number, fixed by the format.
+const ROOT: u32 = 5;
+
+/// Directory levels the table is followed down. A real tree is far
+/// shallower; a corrupt one can loop, a directory beneath itself, and past
+/// this the walk measures instead.
+const MOST_LEVELS: usize = 1024;
+
+/// Most bytes [`contents`] reads: the table's `$ATTRIBUTE_LIST` and
+/// `$BITMAP` are kilobytes to a few megabytes, and a length beyond this is
+/// a corrupt record, not something to allocate.
+const MOST_CONTENTS: u64 = 64 << 20;
+
+/// Records below this are the file system's own ($MFT, $Bitmap, ...): a
+/// directory listing never shows them, so neither does this.
+const FIRST_USER_RECORD: u32 = 16;
+
+const IN_USE: u16 = 0x1;
+const IS_DIRECTORY: u16 = 0x2;
+
+const STANDARD_INFORMATION: u32 = 0x10;
+const ATTRIBUTE_LIST: u32 = 0x20;
+const FILE_NAME: u32 = 0x30;
+const DATA: u32 = 0x80;
+const BITMAP: u32 = 0xB0;
+const REPARSE_POINT: u32 = 0xC0;
+const END: u32 = 0xFFFF_FFFF;
+
+/// The 8.3 alias of a name that has a long one: the same entry again.
+const DOS_NAMESPACE: u8 = 2;
+
+/// See `windows.rs`: links, junctions and mounted folders.
+const NAME_SURROGATE: u32 = 0x2000_0000;
+const EVICTED: u32 =
+ FILE_ATTRIBUTE_RECALL_ON_OPEN | FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS;
+
+/// What one base file record says.
+#[derive(Clone, Copy, Default)]
+struct Info {
+ in_use: bool,
+ directory: bool,
+ /// Bumped each time the record is reused: a reference carrying another
+ /// one names a file that is gone.
+ sequence: u16,
+ attributes: u32,
+ reparse_tag: u32,
+ modified: i64,
+ apparent: u64,
+ allocated: u64,
+ names: u8,
+}
+
+/// One name of a file: an entry in its parent directory. The name itself
+/// is `len` bytes at `at` in its chunk's text: millions of names in a few
+/// strings, not millions of allocations made only to be copied again.
+#[derive(Clone, Copy)]
+struct Entry {
+ parent: u32,
+ parent_sequence: u16,
+ child: u32,
+ chunk: u32,
+ at: u32,
+ len: u16,
+}
+
+impl Entry {
+ /// A placeholder a list is filled with before its entries are copied in.
+ const EMPTY: Self = Self {
+ parent: 0,
+ parent_sequence: 0,
+ child: 0,
+ chunk: 0,
+ at: 0,
+ len: 0,
+ };
+}
+
+/// What one chunk of records contributes besides its stretch of the
+/// table.
+#[derive(Default)]
+struct Parsed {
+ names: Vec<Entry>,
+ text: String,
+ /// Files among the records, and what their base records say they
+ /// hold, for the progress meter: `settle` corrects both at the end.
+ files: u64,
+ apparent: u64,
+ allocated: u64,
+ /// What extension records hold for their base record: `(base, base
+ /// sequence, apparent size, bytes allocated, reparse tag)`.
+ extra: Vec<(u32, u16, Option<u64>, u64, u32)>,
+ /// Names found in extension records, with their base's sequence: kept
+ /// apart since they are rare and must be checked against the base.
+ extra_names: Vec<(Entry, u16)>,
+}
+
+/// Where a stream lies on the volume: `(byte offset, byte length)` runs.
+type Runs = Vec<(u64, u64)>;
+
+struct Geometry {
+ cluster: u64,
+ record: usize,
+ mft_offset: u64,
+ /// The volume's size in bytes: no table can be larger.
+ volume: u64,
+}
+
+/// The tree under `root`, read from the volume's file table; `None` when
+/// the table cannot be read and the walk has to measure instead.
+pub fn scan(
+ root: &Path,
+ canonical: &Path,
+ options: &ScanOptions,
+ progress: &ScanProgress,
+) -> Option<io::Result<Node>> {
+ // A followed link can lead off this volume, or back into it a second
+ // time, and the table describes neither: the walk follows them.
+ if options.follow_links {
+ return None;
+ }
+ let path = format!(r"\\.\{}:", drive_letter(canonical)?);
+ let volume = open_volume(&path).ok()?;
+ let geometry = geometry(&volume)?;
+ let (runs, bitmap) = table_layout(&volume, &geometry)?;
+ let reads = plan_reads(
+ &runs,
+ &bitmap,
+ &geometry,
+ GAP_BYTES / geometry.record as u64,
+ );
+ let mut infos = table(&reads, geometry.record);
+ let parsed = read_records(
+ &path,
+ geometry.record,
+ &reads,
+ &mut infos,
+ options.apparent_size,
+ progress,
+ )?;
+ if progress.is_cancelled() {
+ return Some(Err(cancelled()));
+ }
+
+ let (mut names, texts) = merge(parsed, &mut infos);
+ if progress.is_cancelled() {
+ return Some(Err(cancelled()));
+ }
+ if !is_root_directory(&infos) {
+ return None;
+ }
+ names.par_sort_unstable_by_key(|name| name.parent);
+ let starts = starts(&names, infos.len());
+ let table = Table {
+ infos,
+ names,
+ texts,
+ starts,
+ options,
+ progress,
+ };
+ let tree = table.directory(ROOT, crate::scan::file_name(root), 0);
+ // Hundreds of megabytes, whose freeing the caller would otherwise
+ // wait out before it can finish the tree.
+ let Table {
+ infos,
+ names,
+ texts,
+ starts,
+ ..
+ } = table;
+ // When no thread can be made, the closure is dropped here instead,
+ // and they are freed in place.
+ let _ = std::thread::Builder::new()
+ .spawn(move || drop((infos, names, texts, starts)));
+ match tree {
+ Ok(_) | Err(Stop) if progress.is_cancelled() => Some(Err(cancelled())),
+ Ok(node) => Some(Ok(node)),
+ Err(Stop) => None,
+ }
+}
+
+/// Whether the root's record reads as an in-use directory; if not, the
+/// table is not what this expects and the walk measures instead.
+fn is_root_directory(infos: &[Info]) -> bool {
+ infos
+ .get(ROOT as usize)
+ .is_some_and(|root| root.in_use && root.directory)
+}
+
+pub fn cancelled() -> io::Error {
+ io::Error::new(io::ErrorKind::Interrupted, "the scan was cancelled")
+}
+
+/// Past the file cache: the table is read once and never again, and
+/// through the cache it is copied twice and evicts what the rest of the
+/// system is using.
+fn open_volume(path: &str) -> io::Result<File> {
+ OpenOptions::new()
+ .read(true)
+ .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
+ .custom_flags(FILE_FLAG_NO_BUFFERING)
+ .open(path)
+}
+
+/// Where each parent's entries start in `names`, sorted by parent:
+/// `starts[p]..starts[p + 1]`.
+fn starts(names: &[Entry], records: usize) -> Vec<u32> {
+ let mut starts = vec![0_u32; records + 1];
+ for name in names {
+ if let Some(count) = starts.get_mut(name.parent as usize + 1) {
+ *count += 1;
+ }
+ }
+ for index in 1..starts.len() {
+ starts[index] += starts[index - 1];
+ }
+ starts
+}
+
+fn geometry(volume: &File) -> Option<Geometry> {
+ geometry_of(&read_at(volume, 0, 512)?)
+}
+
+/// The sizes the boot sector gives, each checked: a corrupt one would
+/// otherwise overflow a shift or allocate without bound.
+fn geometry_of(boot: &[u8]) -> Option<Geometry> {
+ if boot.get(3..11)? != b"NTFS " {
+ return None;
+ }
+ let sector = u64::from(u16_at(boot, 0x0B)?);
+ let per_cluster = *boot.get(0x0D)?;
+ // Above 0x80 the count is a negative power of two.
+ let sectors = if per_cluster > 0x80 {
+ 1_u64.checked_shl(256 - u32::from(per_cluster))?
+ } else {
+ u64::from(per_cluster)
+ };
+ let cluster = sector.checked_mul(sectors)?;
+ let per_record = boot.get(0x40)?.cast_signed();
+ let record = if per_record < 0 {
+ 1_u64.checked_shl(u32::from(per_record.unsigned_abs()))?
+ } else {
+ cluster.checked_mul(u64::from(per_record.unsigned_abs()))?
+ };
+ let mft_offset = u64_at(boot, 0x30)?.checked_mul(cluster)?;
+ let volume = u64_at(boot, 0x28)?.checked_mul(sector)?;
+ let valid = sector.is_power_of_two()
+ && (512..=4096).contains(&sector)
+ && cluster.is_power_of_two()
+ && cluster <= 2 << 20
+ && record.is_power_of_two()
+ && (1024..=4096).contains(&record);
+ valid.then_some(Geometry {
+ cluster,
+ record: usize::try_from(record).ok()?,
+ mft_offset,
+ volume,
+ })
+}
+
+/// Where the file table lies on disk, as `(byte offset, byte length)` runs,
+/// and its `$BITMAP`: which records are in use. A table grows and never
+/// shrinks, so on a disk that has seen churn most of it can be free; only
+/// the used part is read.
+///
+/// Both come from the table's own first record, or from the records its
+/// `$ATTRIBUTE_LIST` names once it has outgrown that one.
+fn table_layout(volume: &File, geometry: &Geometry) -> Option<(Runs, Vec<u8>)> {
+ let mut records =
+ vec![read_record_at(volume, geometry, geometry.mft_offset)?];
+ if attributes(&records[0]).any(|attribute| attribute.kind == ATTRIBUTE_LIST)
+ {
+ let list = contents(volume, geometry, &records, ATTRIBUTE_LIST)?;
+ let (_, first) = stream(&records, DATA, geometry.cluster)?;
+ let mut numbers = Vec::new();
+ let mut at = 0;
+ while let (Some(length), Some(reference)) =
+ (u16_at(&list, at + 4), u64_at(&list, at + 0x10))
+ {
+ let number = reference & 0xFFFF_FFFF_FFFF;
+ if number != 0 && !numbers.contains(&number) {
+ numbers.push(number);
+ }
+ if length == 0 {
+ break;
+ }
+ at += usize::from(length);
+ }
+ // The records holding the rest of the table's attributes are
+ // early in it, inside what the first record already maps.
+ for number in numbers {
+ let offset = record_offset(&first, geometry, number)?;
+ records.push(read_record_at(volume, geometry, offset)?);
+ }
+ }
+
+ let (data, runs) = stream(&records, DATA, geometry.cluster)?;
+ let listed = runs
+ .iter()
+ .try_fold(0_u64, |sum, &(_, length)| sum.checked_add(length))?;
+ let record_bytes = geometry.record as u64;
+ // Record numbers are 32 bits here, and `plan_reads` counts records
+ // across runs whose ends `decode_runs` has checked fit. A table larger
+ // than its volume is corrupt, and `merge` would allocate a slot per
+ // record it claims.
+ if listed < u64_at(data.bytes, 0x28)?
+ || listed > geometry.volume
+ || listed / record_bytes > u64::from(u32::MAX)
+ || runs.iter().any(|(_, length)| length % record_bytes != 0)
+ {
+ return None;
+ }
+ let bitmap = contents(volume, geometry, &records, BITMAP)?;
+ Some((runs, bitmap))
+}
+
+/// The unnamed attribute `kind` across `records`: the piece that starts
+/// it, which carries its sizes, and the runs of every piece in order.
+fn stream(
+ records: &[Vec<u8>],
+ kind: u32,
+ cluster: u64,
+) -> Option<(Attribute<'_>, Runs)> {
+ let mut pieces: Vec<Attribute<'_>> = records
+ .iter()
+ .flat_map(|record| attributes(record))
+ .filter(|attribute| {
+ attribute.kind == kind && attribute.name_length == 0
+ })
+ .collect();
+ pieces.sort_by_key(|piece| {
+ if piece.non_resident {
+ u64_at(piece.bytes, 0x10).unwrap_or(u64::MAX)
+ } else {
+ 0
+ }
+ });
+ let mut runs = Vec::new();
+ for piece in pieces.iter().filter(|piece| piece.non_resident) {
+ runs.extend(decode_runs(piece.bytes, cluster)?);
+ }
+ let first = pieces.into_iter().next()?;
+ Some((first, runs))
+}
+
+/// All of the unnamed attribute `kind`'s value, wherever it is kept.
+/// `None` when what the runs hold falls short of its length: for the
+/// table's bitmap every record past the end would be skipped as free.
+fn contents(
+ volume: &File,
+ geometry: &Geometry,
+ records: &[Vec<u8>],
+ kind: u32,
+) -> Option<Vec<u8>> {
+ let (first, runs) = stream(records, kind, geometry.cluster)?;
+ if !first.non_resident {
+ return first.value().map(<[u8]>::to_vec);
+ }
+ let length = u64_at(first.bytes, 0x30)?;
+ let held = runs
+ .iter()
+ .try_fold(0_u64, |sum, &(_, size)| sum.checked_add(size))?;
+ if length > MOST_CONTENTS || held > MOST_CONTENTS {
+ return None;
+ }
+ let length = usize::try_from(length).ok()?;
+ // The table's bitmap grows with it, a few clusters at a time: hundreds
+ // of runs, which read one after another cost tens of milliseconds.
+ let pieces: Vec<Vec<u8>> = runs
+ .into_par_iter()
+ .map(|(offset, size)| {
+ read_at(volume, offset, usize::try_from(size).ok()?)
+ })
+ .collect::<Option<_>>()?;
+ let mut bytes = pieces.concat();
+ (bytes.len() >= length).then_some(())?;
+ bytes.truncate(length);
+ Some(bytes)
+}
+
+/// Where record `number` lies, given the runs that hold it.
+fn record_offset(
+ runs: &[(u64, u64)],
+ geometry: &Geometry,
+ number: u64,
+) -> Option<u64> {
+ let mut position = number.checked_mul(geometry.record as u64)?;
+ for &(offset, length) in runs {
+ if position < length {
+ return offset.checked_add(position);
+ }
+ position -= length;
+ }
+ None
+}
+
+/// One record at `offset`, fixed up.
+fn read_record_at(
+ volume: &File,
+ geometry: &Geometry,
+ offset: u64,
+) -> Option<Vec<u8>> {
+ let mut record = read_at(volume, offset, geometry.record)?;
+ fixup(&mut record)?;
+ Some(record)
+}
+
+/// The reads that cover every used record: `(byte offset, byte length,
+/// first record number)`. A free stretch of fewer than `gap_records` is
+/// read through, since one longer read costs less than two (see
+/// [`GAP_BYTES`]). Reads start and end on cluster boundaries, which a
+/// volume handle requires of them.
+fn plan_reads(
+ runs: &[(u64, u64)],
+ bitmap: &[u8],
+ geometry: &Geometry,
+ gap_records: u64,
+) -> Vec<(u64, u64, u64)> {
+ let used = |number: u64| {
+ bitmap
+ .get((number / 8) as usize)
+ .is_some_and(|byte| byte & (1 << (number % 8)) != 0)
+ };
+ let record = geometry.record as u64;
+ let align = (geometry.cluster / record).max(1);
+ let most = CHUNK_BYTES / record;
+ let mut reads = Vec::new();
+ let mut first = 0_u64;
+ for &(offset, length) in runs {
+ let count = length / record;
+ let mut at = 0;
+ while at < count {
+ if !used(first + at) {
+ at += 1;
+ continue;
+ }
+ let start = at - at % align;
+ let mut end = at + 1;
+ let mut probe = end;
+ while probe < count
+ && probe - start < most
+ && probe - end < gap_records
+ {
+ if used(first + probe) {
+ end = probe + 1;
+ }
+ probe += 1;
+ }
+ let end = end.next_multiple_of(align).min(count);
+ reads.push((
+ offset + start * record,
+ (end - start) * record,
+ first + start,
+ ));
+ at = end;
+ }
+ first += count;
+ }
+ reads
+}
+
+/// The runs of a non-resident attribute. Each run's end fits in a `u64`,
+/// which `plan_reads` relies on.
+fn decode_runs(attribute: &[u8], cluster: u64) -> Option<Runs> {
+ let mut at = usize::from(u16_at(attribute, 0x20)?);
+ let mut lcn = 0_i64;
+ let mut runs = Vec::new();
+ loop {
+ let header = *attribute.get(at)?;
+ if header == 0 {
+ return Some(runs);
+ }
+ let length_size = usize::from(header & 0xF);
+ let offset_size = usize::from(header >> 4);
+ let length = unsigned(attribute.get(at + 1..at + 1 + length_size)?);
+ let delta = attribute
+ .get(at + 1 + length_size..at + 1 + length_size + offset_size)?;
+ at += 1 + length_size + offset_size;
+ // No offset marks a sparse run, which a file table never has.
+ if offset_size == 0 || offset_size > 8 || length_size > 8 {
+ return None;
+ }
+ lcn = lcn.checked_add(signed(delta))?;
+ let offset = u64::try_from(lcn).ok()?.checked_mul(cluster)?;
+ let length = length.checked_mul(cluster)?;
+ offset.checked_add(length)?;
+ runs.push((offset, length));
+ }
+}
+
+/// A slot per record up to the last one read, each empty until a read
+/// fills it: records no read covers are free.
+fn table(reads: &[(u64, u64, u64)], record: usize) -> Vec<Info> {
+ let len = reads.last().map_or(0, |&(_, size, first)| {
+ usize::try_from(first + size / record as u64).unwrap_or(0)
+ });
+ let mut infos = Vec::new();
+ (0..len)
+ .into_par_iter()
+ .map(|_| Info::default())
+ .collect_into_vec(&mut infos);
+ infos
+}
+
+/// Read and parse every used record, each into its slot of `infos`.
+/// Reads run in parallel: one at a time leaves a solid-state disk idle
+/// between requests. One list of reads per thread, each with its own
+/// volume handle and buffer: requests on one handle opened without
+/// overlapped I/O run one at a time, and rayon's `map_init` would open a
+/// handle and zero a buffer per split, hundreds of times. Read `i` goes to
+/// list `i % threads`, so the lists move through the disk together.
+/// Results come back in read order, which `merge` relies on.
+///
+/// Records land in `infos` as they are parsed, while other reads are
+/// still on the disk: gathered into it after the last read, they cost a
+/// tenth of a second on the way to the tree.
+fn read_records(
+ path: &str,
+ record: usize,
+ reads: &[(u64, u64, u64)],
+ infos: &mut [Info],
+ apparent_size: bool,
+ progress: &ScanProgress,
+) -> Option<Vec<Parsed>> {
+ let threads = rayon::current_num_threads().max(1);
+ let mut stretches: Vec<Vec<&mut [Info]>> =
+ (0..threads).map(|_| Vec::new()).collect();
+ // `plan_reads` gives them in record order, none overlapping.
+ let mut rest = infos;
+ let mut done = 0;
+ for (index, &(_, size, first)) in reads.iter().enumerate() {
+ let skip = usize::try_from(first).ok()?.checked_sub(done)?;
+ let count = usize::try_from(size).ok()? / record;
+ let (stretch, tail) =
+ rest.get_mut(skip..)?.split_at_mut_checked(count)?;
+ stretches[index % threads].push(stretch);
+ rest = tail;
+ done += skip + count;
+ }
+ let lists = stretches
+ .into_par_iter()
+ .enumerate()
+ .map(|(list, stretches)| {
+ let volume = open_volume(path).ok()?;
+ let mut buffer = Aligned::default();
+ let mut parsed = Vec::new();
+ for ((chunk, &(offset, size, first)), stretch) in reads
+ .iter()
+ .enumerate()
+ .skip(list)
+ .step_by(threads)
+ .zip(stretches)
+ {
+ if progress.is_cancelled() {
+ break;
+ }
+ let buffer = buffer.bytes(usize::try_from(size).ok()?);
+ volume.seek_read_exact(buffer, offset).ok()?;
+ let chunk = parse_chunk(
+ buffer,
+ record,
+ first,
+ u32::try_from(chunk).ok()?,
+ stretch,
+ );
+ let bytes = if apparent_size {
+ chunk.apparent
+ } else {
+ chunk.allocated
+ };
+ progress.add(chunk.files, 0, bytes);
+ parsed.push(chunk);
+ }
+ Some(parsed)
+ })
+ .collect::<Option<Vec<_>>>()?;
+ // Interleave back: read `i` is item `i / threads` of list `i % threads`.
+ let mut lists: Vec<_> = lists.into_iter().map(Vec::into_iter).collect();
+ let mut parsed = Vec::with_capacity(reads.len());
+ for index in 0..reads.len() {
+ // Short only when cancelled, which the caller checks next.
+ let Some(chunk) = lists[index % threads].next() else {
+ break;
+ };
+ parsed.push(chunk);
+ }
+ Some(parsed)
+}
+
+/// `len` bytes at `offset`. Read as whole, aligned 4 KiB blocks: past the
+/// cache a volume only reads whole sectors into sector-aligned memory, and
+/// no disk has sectors larger.
+fn read_at(volume: &File, offset: u64, len: usize) -> Option<Vec<u8>> {
+ const BLOCK: u64 = 4096;
+ let start = offset - offset % BLOCK;
+ let end = offset
+ .checked_add(len as u64)?
+ .checked_next_multiple_of(BLOCK)?;
+ let mut buffer = Aligned::default();
+ let block = buffer.bytes(usize::try_from(end - start).ok()?);
+ volume.seek_read_exact(block, start).ok()?;
+ let at = usize::try_from(offset - start).ok()?;
+ block.get(at..at + len).map(<[u8]>::to_vec)
+}
+
+trait ReadExact {
+ fn seek_read_exact(&self, buffer: &mut [u8], offset: u64)
+ -> io::Result<()>;
+}
+
+impl ReadExact for File {
+ fn seek_read_exact(
+ &self,
+ mut buffer: &mut [u8],
+ mut offset: u64,
+ ) -> io::Result<()> {
+ while !buffer.is_empty() {
+ match self.seek_read(buffer, offset)? {
+ 0 => return Err(io::ErrorKind::UnexpectedEof.into()),
+ read => {
+ buffer = &mut buffer[read..];
+ offset += read as u64;
+ }
+ }
+ }
+ Ok(())
+ }
+}
+
+fn parse_chunk(
+ buffer: &mut [u8],
+ record: usize,
+ first: u64,
+ chunk: u32,
+ infos: &mut [Info],
+) -> Parsed {
+ // About a name per record, and a name about 20 bytes: grown by
+ // doubling instead, the lists are copied a dozen times per read.
+ let records = buffer.len() / record;
+ let mut out = Parsed {
+ names: Vec::with_capacity(records),
+ text: String::with_capacity(records * 24),
+ ..Parsed::default()
+ };
+ for ((index, bytes), slot) in
+ buffer.chunks_exact_mut(record).enumerate().zip(infos)
+ {
+ let Ok(number) = u32::try_from(first + index as u64) else {
+ break;
+ };
+ parse_record(bytes, number, chunk, &mut out, slot);
+ }
+ out
+}
+
+/// A base record's facts go to `slot`; an extension record's to `out`.
+fn parse_record(
+ bytes: &mut [u8],
+ number: u32,
+ chunk: u32,
+ out: &mut Parsed,
+ slot: &mut Info,
+) {
+ if bytes.get(..4) != Some(b"FILE") || fixup(bytes).is_none() {
+ return;
+ }
+ let Some(flags) = u16_at(bytes, 0x16) else {
+ return;
+ };
+ if flags & IN_USE == 0 {
+ return;
+ }
+ // A record holding attributes that did not fit in its base record.
+ let base = u64_at(bytes, 0x20).unwrap_or(0);
+ let (owner, base_sequence) = if base & REFERENCE == 0 {
+ (number, 0)
+ } else {
+ let Some(reference) = reference(base) else {
+ return;
+ };
+ reference
+ };
+ let mut info = Info {
+ in_use: true,
+ directory: flags & IS_DIRECTORY != 0,
+ sequence: u16_at(bytes, 0x10).unwrap_or(0),
+ ..Info::default()
+ };
+ let mut apparent = None;
+ let mut allocated = 0_u64;
+ // A reparse point's tag is also kept beside each name, which serves
+ // when the `$REPARSE_POINT` value is not in the record itself.
+ let mut name_tag = 0;
+ for attribute in attributes(bytes) {
+ match attribute.kind {
+ STANDARD_INFORMATION => {
+ if let Some(value) = attribute.value() {
+ info.modified = u64_at(value, 0x08).map_or(0, |ticks| {
+ crate::windows::unix_seconds(ticks.cast_signed())
+ });
+ info.attributes = u32_at(value, 0x20).unwrap_or(0);
+ }
+ }
+ FILE_NAME => {
+ let Some(value) = attribute.value() else {
+ continue;
+ };
+ if u32_at(value, 0x38).unwrap_or(0)
+ & FILE_ATTRIBUTE_REPARSE_POINT
+ != 0
+ {
+ name_tag = u32_at(value, 0x3C).unwrap_or(0);
+ }
+ let at = out.text.len();
+ if let Some((parent, parent_sequence)) =
+ file_name(value, &mut out.text)
+ {
+ let entry = Entry {
+ parent,
+ parent_sequence,
+ child: owner,
+ chunk,
+ at: u32::try_from(at).unwrap_or(u32::MAX),
+ len: u16::try_from(out.text.len() - at)
+ .unwrap_or(u16::MAX),
+ };
+ if base & REFERENCE == 0 {
+ out.names.push(entry);
+ info.names = info.names.saturating_add(1);
+ } else {
+ out.extra_names.push((entry, base_sequence));
+ }
+ }
+ }
+ DATA => {
+ // Every stream's clusters, named ones too: they go when the
+ // file does. A file Windows compressed (`compact /exe`)
+ // keeps its data in one and leaves the unnamed one empty.
+ let (length, spent) = attribute.sizes();
+ allocated = allocated.saturating_add(spent);
+ // Only the first piece of the unnamed stream has its length.
+ if attribute.name_length == 0 && attribute.starts_file() {
+ apparent = Some(length);
+ }
+ }
+ REPARSE_POINT => {
+ if let Some(value) = attribute.value() {
+ info.reparse_tag = u32_at(value, 0).unwrap_or(0);
+ }
+ }
+ _ => {}
+ }
+ }
+ if info.reparse_tag == 0 {
+ info.reparse_tag = name_tag;
+ }
+ if base & REFERENCE == 0 {
+ if !info.directory {
+ out.files += 1;
+ out.apparent = out.apparent.saturating_add(apparent.unwrap_or(0));
+ out.allocated = out.allocated.saturating_add(allocated);
+ }
+ info.apparent = apparent.unwrap_or(0);
+ info.allocated = allocated;
+ *slot = info;
+ } else if apparent.is_some() || allocated != 0 || info.reparse_tag != 0 {
+ out.extra.push((
+ owner,
+ base_sequence,
+ apparent,
+ allocated,
+ info.reparse_tag,
+ ));
+ }
+}
+
+/// The record number part of a file reference.
+const REFERENCE: u64 = 0xFFFF_FFFF_FFFF;
+
+/// A file reference: `(record number, sequence number)`.
+fn reference(value: u64) -> Option<(u32, u16)> {
+ let number = u32::try_from(value & REFERENCE).ok()?;
+ let sequence = u16::try_from(value >> 48).ok()?;
+ Some((number, sequence))
+}
+
+/// The parent reference of a `$FILE_NAME`, its name appended to `text`;
+/// or `None`, and nothing appended, for an 8.3 alias or a name no
+/// directory can hold, which as a path would name another file.
+fn file_name(value: &[u8], text: &mut String) -> Option<(u32, u16)> {
+ let parent = reference(u64_at(value, 0)?)?;
+ let length = usize::from(*value.get(0x40)?);
+ if *value.get(0x41)? == DOS_NAMESPACE {
+ return None;
+ }
+ let units = value
+ .get(0x42..0x42 + length * 2)?
+ .chunks_exact(2)
+ .map(|pair| u16::from_le_bytes([pair[0], pair[1]]));
+ let start = text.len();
+ text.extend(
+ char::decode_utf16(units)
+ .map(|unit| unit.unwrap_or(char::REPLACEMENT_CHARACTER)),
+ );
+ let name = &text[start..];
+ if name.is_empty()
+ || name == "."
+ || name == ".."
+ || name.contains(['\\', '/', '\0'])
+ {
+ text.truncate(start);
+ return None;
+ }
+ Some(parent)
+}
+
+/// Undo the update sequence: the last two bytes of every sector were
+/// swapped for a check value when the record was written.
+fn fixup(bytes: &mut [u8]) -> Option<()> {
+ // The update sequence always strides 512 bytes, whatever the sector.
+ const STRIDE: usize = 512;
+ let at = usize::from(u16_at(bytes, 4)?);
+ let count = usize::from(u16_at(bytes, 6)?);
+ if bytes.is_empty()
+ || !bytes.len().is_multiple_of(STRIDE)
+ || count != bytes.len() / STRIDE + 1
+ || at + 2 * count > bytes.len()
+ {
+ return None;
+ }
+ let check = [bytes[at], bytes[at + 1]];
+ for sector in 1..count {
+ let end = sector * STRIDE;
+ let replacement = [bytes[at + 2 * sector], bytes[at + 2 * sector + 1]];
+ let tail = &mut bytes[end - 2..end];
+ if tail != check {
+ return None;
+ }
+ tail.copy_from_slice(&replacement);
+ }
+ Some(())
+}
+
+struct Attribute<'a> {
+ kind: u32,
+ non_resident: bool,
+ name_length: u8,
+ flags: u16,
+ bytes: &'a [u8],
+}
+
+impl Attribute<'_> {
+ fn value(&self) -> Option<&[u8]> {
+ if self.non_resident {
+ return None;
+ }
+ let length = u32_at(self.bytes, 0x10)? as usize;
+ let at = usize::from(u16_at(self.bytes, 0x14)?);
+ self.bytes.get(at..at + length)
+ }
+
+ fn starts_file(&self) -> bool {
+ !self.non_resident || u64_at(self.bytes, 0x10) == Some(0)
+ }
+
+ /// `(length, bytes allocated)`. Data kept in the record costs nothing
+ /// beyond the record; a compressed or sparse stream's real cost is its
+ /// total allocation, which only the first piece carries.
+ fn sizes(&self) -> (u64, u64) {
+ if !self.non_resident {
+ return (self.value().map_or(0, |value| value.len() as u64), 0);
+ }
+ if !self.starts_file() {
+ return (0, 0);
+ }
+ let length = u64_at(self.bytes, 0x30).unwrap_or(0);
+ let packed = self.flags & 0x8001 != 0;
+ let allocated =
+ u64_at(self.bytes, if packed { 0x40 } else { 0x28 }).unwrap_or(0);
+ (length, allocated)
+ }
+}
+
+fn attributes(record: &[u8]) -> impl Iterator<Item = Attribute<'_>> {
+ let mut at = u16_at(record, 0x14).map_or(record.len(), usize::from);
+ std::iter::from_fn(move || {
+ let kind = u32_at(record, at)?;
+ if kind == END {
+ return None;
+ }
+ let length = u32_at(record, at + 4)? as usize;
+ let bytes = record.get(at..at.checked_add(length)?)?;
+ if length < 0x18 {
+ return None;
+ }
+ at += length;
+ Some(Attribute {
+ kind,
+ non_resident: bytes[8] != 0,
+ name_length: bytes[9],
+ flags: u16_at(bytes, 0x0C)?,
+ bytes,
+ })
+ })
+}
+
+/// Every chunk's names in one list, and what extension records add to
+/// their base records' slots in `infos`.
+fn merge(parsed: Vec<Parsed>, infos: &mut [Info]) -> (Vec<Entry>, Vec<String>) {
+ // Copied in parallel, a chunk per task, into a list filled first: one
+ // thread appending millions of names made every other one wait. Room
+ // for the names from extension records too, pushed after: any system
+ // drive has some, and the first would copy the whole list to grow it.
+ let total = parsed.iter().map(|chunk| chunk.names.len()).sum();
+ let extra_total: usize =
+ parsed.iter().map(|chunk| chunk.extra_names.len()).sum();
+ let mut names = Vec::with_capacity(total + extra_total);
+ names.par_extend(rayon::iter::repeat_n(Entry::EMPTY, total));
+ let mut pieces = Vec::with_capacity(parsed.len());
+ let mut rest = names.as_mut_slice();
+ for chunk in &parsed {
+ let (piece, tail) = rest.split_at_mut(chunk.names.len());
+ pieces.push((piece, chunk.names.as_slice()));
+ rest = tail;
+ }
+ pieces
+ .into_par_iter()
+ .for_each(|(piece, chunk)| piece.copy_from_slice(chunk));
+ let mut extra = Vec::new();
+ let mut extra_names = Vec::new();
+ let mut texts = Vec::with_capacity(parsed.len());
+ for chunk in parsed {
+ extra.extend(chunk.extra);
+ extra_names.extend(chunk.extra_names);
+ texts.push(chunk.text);
+ }
+ // A name in an extension record left behind by a file whose record
+ // has since been reused would list that file under the new one's
+ // number.
+ for (name, sequence) in extra_names {
+ let Some(info) = infos.get_mut(name.child as usize) else {
+ continue;
+ };
+ if info.in_use && info.sequence == sequence {
+ info.names = info.names.saturating_add(1);
+ names.push(name);
+ }
+ }
+ for (number, sequence, apparent, allocated, tag) in extra {
+ let Some(info) = infos.get_mut(number as usize) else {
+ continue;
+ };
+ // An extension record left behind by a file whose record has
+ // since been reused.
+ if !info.in_use || info.sequence != sequence {
+ continue;
+ }
+ info.allocated = info.allocated.saturating_add(allocated);
+ if let Some(apparent) = apparent {
+ info.apparent = apparent;
+ }
+ if info.reparse_tag == 0 {
+ info.reparse_tag = tag;
+ }
+ }
+ (names, texts)
+}
+
+/// Why [`Table::directory`] gave up: the scan was cancelled, or the tree
+/// runs deeper than [`MOST_LEVELS`].
+struct Stop;
+
+struct Table<'a> {
+ infos: Vec<Info>,
+ /// Sorted by parent; `starts[p]..starts[p + 1]` are `p`'s entries.
+ names: Vec<Entry>,
+ texts: Vec<String>,
+ starts: Vec<u32>,
+ options: &'a ScanOptions,
+ progress: &'a ScanProgress,
+}
+
+impl Table<'_> {
+ fn name(&self, entry: &Entry) -> &str {
+ let at = entry.at as usize;
+ self.texts
+ .get(entry.chunk as usize)
+ .and_then(|text| text.get(at..at + usize::from(entry.len)))
+ .unwrap_or_default()
+ }
+
+ fn entries(&self, parent: u32) -> &[Entry] {
+ let index = parent as usize;
+ match (self.starts.get(index), self.starts.get(index + 1)) {
+ (Some(&start), Some(&end)) => {
+ &self.names[start as usize..end as usize]
+ }
+ _ => &[],
+ }
+ }
+
+ /// Mirrors what the walk keeps: see `WalkContext::classify`.
+ fn directory(
+ &self,
+ number: u32,
+ name: Box<str>,
+ depth: usize,
+ ) -> Result<Node, Stop> {
+ if depth >= MOST_LEVELS || self.progress.is_cancelled() {
+ return Err(Stop);
+ }
+ let descend = self.options.max_depth.is_none_or(|max| depth < max);
+ let sequence =
+ self.infos.get(number as usize).map(|info| info.sequence);
+ let child = |entry: &Entry| {
+ // A root directory holds hundreds of thousands of files; a
+ // check per entry lets a cancel stop within one.
+ if self.progress.is_cancelled() {
+ return Some(Err(Stop));
+ }
+ // An entry naming a parent whose record has since been reused.
+ if Some(entry.parent_sequence) != sequence {
+ return None;
+ }
+ self.child(entry, depth, descend).transpose()
+ };
+ let entries = self.entries(number);
+ // See `tree::PARALLEL_LEVELS`: parallel only near the top.
+ let children = if depth < crate::tree::PARALLEL_LEVELS {
+ entries
+ .par_iter()
+ .filter_map(child)
+ .collect::<Result<_, _>>()?
+ } else {
+ // Sized up front: nearly every entry becomes a child, and a
+ // list grown by doubling copies its nodes over and over.
+ let mut children = Vec::with_capacity(entries.len());
+ for node in entries.iter().filter_map(child) {
+ children.push(node?);
+ }
+ children
+ };
+ let mut node = Node::directory(name);
+ node.children = children;
+ Ok(node)
+ }
+
+ /// The node for `entry`; `None` for one the walk would not list.
+ fn child(
+ &self,
+ entry: &Entry,
+ depth: usize,
+ descend: bool,
+ ) -> Result<Option<Node>, Stop> {
+ // The root is its own parent.
+ if entry.child == entry.parent || entry.child < FIRST_USER_RECORD {
+ return Ok(None);
+ }
+ let Some(info) = self.infos.get(entry.child as usize) else {
+ return Ok(None);
+ };
+ if !info.in_use {
+ return Ok(None);
+ }
+ let name = self.name(entry);
+ if !self.options.include_hidden
+ && (name.starts_with('.')
+ || info.attributes & FILE_ATTRIBUTE_HIDDEN != 0)
+ {
+ return Ok(None);
+ }
+ let link = info.attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0
+ && info.reparse_tag & NAME_SURROGATE != 0;
+ if info.directory && !link {
+ if info.attributes & EVICTED != 0 || !descend {
+ return Ok(None);
+ }
+ return self
+ .directory(entry.child, name.into(), depth + 1)
+ .map(Some);
+ }
+ let size = if self.options.apparent_size {
+ info.apparent
+ } else {
+ info.allocated
+ };
+ let kind = if link {
+ NodeKind::Symlink
+ } else {
+ NodeKind::File
+ };
+ let mut node = Node::entry(name, kind, size);
+ node.modified = info.modified;
+ if info.names > 1 {
+ node.inode = Some((0, u64::from(entry.child)));
+ }
+ Ok(Some(node))
+ }
+}
+
+fn u16_at(bytes: &[u8], at: usize) -> Option<u16> {
+ Some(u16::from_le_bytes(bytes.get(at..at + 2)?.try_into().ok()?))
+}
+
+fn u32_at(bytes: &[u8], at: usize) -> Option<u32> {
+ Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?))
+}
+
+fn u64_at(bytes: &[u8], at: usize) -> Option<u64> {
+ Some(u64::from_le_bytes(bytes.get(at..at + 8)?.try_into().ok()?))
+}
+
+fn unsigned(bytes: &[u8]) -> u64 {
+ bytes
+ .iter()
+ .rev()
+ .fold(0, |value, &byte| (value << 8) | u64::from(byte))
+}
+
+/// A little-endian two's complement number of any width up to eight bytes.
+fn signed(bytes: &[u8]) -> i64 {
+ let value = unsigned(bytes);
+ let bits = bytes.len() * 8;
+ if bits == 0 || bits >= 64 {
+ return value.cast_signed();
+ }
+ let shift = 64 - bits;
+ (value << shift).cast_signed() >> shift
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const RECORD: usize = 1024;
+ const USA_AT: usize = 0x30;
+ const CHECK: [u8; 2] = [0xAB, 0xCD];
+
+ fn resident(kind: u32, value: &[u8]) -> Vec<u8> {
+ let mut bytes = vec![0; 0x18];
+ bytes[0..4].copy_from_slice(&kind.to_le_bytes());
+ bytes[0x10..0x14].copy_from_slice(&(value.len() as u32).to_le_bytes());
+ bytes[0x14..0x16].copy_from_slice(&0x18_u16.to_le_bytes());
+ bytes.extend_from_slice(value);
+ bytes.resize(bytes.len().next_multiple_of(8), 0);
+ let length = bytes.len() as u32;
+ bytes[4..8].copy_from_slice(&length.to_le_bytes());
+ bytes
+ }
+
+ fn non_resident(kind: u32, allocated: u64, length: u64) -> Vec<u8> {
+ let mut bytes = vec![0; 0x48];
+ bytes[0..4].copy_from_slice(&kind.to_le_bytes());
+ bytes[4..8].copy_from_slice(&0x48_u32.to_le_bytes());
+ bytes[8] = 1;
+ bytes[0x20..0x22].copy_from_slice(&0x40_u16.to_le_bytes());
+ bytes[0x28..0x30].copy_from_slice(&allocated.to_le_bytes());
+ bytes[0x30..0x38].copy_from_slice(&length.to_le_bytes());
+ bytes
+ }
+
+ fn name(parent: u32, text: &str, namespace: u8) -> Vec<u8> {
+ let mut value = vec![0; 0x42];
+ value[0..8].copy_from_slice(&u64::from(parent).to_le_bytes());
+ let units: Vec<u16> = text.encode_utf16().collect();
+ value[0x40] = units.len() as u8;
+ value[0x41] = namespace;
+ for unit in units {
+ value.extend_from_slice(&unit.to_le_bytes());
+ }
+ resident(FILE_NAME, &value)
+ }
+
+ fn standard(attributes: u32) -> Vec<u8> {
+ let mut value = vec![0; 0x48];
+ value[0x20..0x24].copy_from_slice(&attributes.to_le_bytes());
+ resident(STANDARD_INFORMATION, &value)
+ }
+
+ /// A record as the disk holds it: sector tails swapped for the check
+ /// value, their real bytes kept in the update sequence array.
+ fn record(flags: u16, base: u64, attributes: &[Vec<u8>]) -> Vec<u8> {
+ let mut bytes = vec![0; RECORD];
+ bytes[0..4].copy_from_slice(b"FILE");
+ bytes[4..6].copy_from_slice(&(USA_AT as u16).to_le_bytes());
+ bytes[6..8].copy_from_slice(&3_u16.to_le_bytes());
+ bytes[0x14..0x16].copy_from_slice(&0x38_u16.to_le_bytes());
+ bytes[0x16..0x18].copy_from_slice(&flags.to_le_bytes());
+ bytes[0x20..0x28].copy_from_slice(&base.to_le_bytes());
+ let mut at = 0x38;
+ for attribute in attributes {
+ bytes[at..at + attribute.len()].copy_from_slice(attribute);
+ at += attribute.len();
+ }
+ bytes[at..at + 4].copy_from_slice(&END.to_le_bytes());
+ bytes[USA_AT..USA_AT + 2].copy_from_slice(&CHECK);
+ for sector in 1..3 {
+ let end = sector * 512;
+ let (tail, usa) = (end - 2, USA_AT + 2 * sector);
+ bytes.copy_within(tail..end, usa);
+ bytes[tail..end].copy_from_slice(&CHECK);
+ }
+ bytes
+ }
+
+ fn parse(bytes: &mut [u8], number: u32) -> (Parsed, Info) {
+ let mut out = Parsed::default();
+ let mut info = Info::default();
+ parse_record(bytes, number, 0, &mut out, &mut info);
+ (out, info)
+ }
+
+ #[test]
+ fn a_record_gives_its_name_size_and_attributes() {
+ let mut bytes = record(
+ IN_USE,
+ 0,
+ &[
+ standard(FILE_ATTRIBUTE_HIDDEN),
+ name(40, "REPORT~1.TXT", DOS_NAMESPACE),
+ name(40, "report for the board.txt", 1),
+ non_resident(DATA, 8192, 5000),
+ ],
+ );
+ let (out, info) = parse(&mut bytes, 77);
+
+ assert!(info.in_use);
+ assert!(!info.directory);
+ assert_eq!(info.attributes, FILE_ATTRIBUTE_HIDDEN);
+ assert_eq!((info.apparent, info.allocated), (5000, 8192));
+ // The 8.3 alias is the same entry again, not a second name.
+ let [entry] = &out.names[..] else {
+ panic!("one name: {}", out.names.len());
+ };
+ assert_eq!((entry.parent, entry.child), (40, 77));
+ let at = entry.at as usize;
+ assert_eq!(
+ &out.text[at..at + usize::from(entry.len)],
+ "report for the board.txt"
+ );
+ assert_eq!(out.files, 1);
+ }
+
+ #[test]
+ fn a_free_or_torn_record_contributes_nothing() {
+ let mut free = record(0, 0, &[name(5, "gone", 1)]);
+ assert!(parse(&mut free, 20).0.names.is_empty());
+
+ // A write that reached only one sector: its tail no longer holds
+ // the check value, and the record cannot be trusted.
+ let mut torn = record(IN_USE, 0, &[name(5, "half", 1)]);
+ torn[1022] ^= 0xFF;
+ let (out, info) = parse(&mut torn, 21);
+ assert!(!info.in_use && out.names.is_empty());
+ }
+
+ #[test]
+ fn a_name_left_in_a_reused_records_extension_is_dropped() {
+ // Record 0x5A is now a file at sequence 8. Record 300 is an
+ // extension left over from sequence 7; record 301 is current.
+ let mut base = record(IN_USE, 0, &[name(5, "current", 1)]);
+ base[0x10..0x12].copy_from_slice(&8_u16.to_le_bytes());
+ let mut stale = record(IN_USE, 0x5A | (7 << 48), &[name(5, "old", 1)]);
+ let mut fresh =
+ record(IN_USE, 0x5A | (8 << 48), &[name(5, "second", 1)]);
+ let mut out = Parsed::default();
+ let mut infos = vec![Info::default(); 400];
+ parse_record(&mut base, 0x5A, 0, &mut out, &mut infos[0x5A]);
+ parse_record(&mut stale, 300, 0, &mut out, &mut infos[300]);
+ parse_record(&mut fresh, 301, 0, &mut out, &mut infos[301]);
+
+ let (names, texts) = merge(vec![out], &mut infos);
+ let mut listed: Vec<&str> = names
+ .iter()
+ .map(|entry| {
+ let at = entry.at as usize;
+ &texts[0][at..at + usize::from(entry.len)]
+ })
+ .collect();
+ listed.sort_unstable();
+ assert_eq!(listed, ["current", "second"]);
+ // Nor is the stale name counted as a link.
+ assert_eq!(infos[0x5A].names, 2);
+ }
+
+ #[test]
+ fn a_root_that_is_not_an_in_use_directory_is_refused() {
+ let mut infos = vec![Info::default(); 8];
+ assert!(!is_root_directory(&infos));
+ infos[ROOT as usize] = Info {
+ in_use: true,
+ ..Info::default()
+ };
+ assert!(!is_root_directory(&infos));
+ infos[ROOT as usize].directory = true;
+ assert!(is_root_directory(&infos));
+ assert!(!is_root_directory(&infos[..3]));
+ }
+
+ #[test]
+ fn an_extension_record_adds_to_its_base_record() {
+ let mut bytes =
+ record(IN_USE, 0x5A | (7 << 48), &[non_resident(DATA, 4096, 100)]);
+ let (out, info) = parse(&mut bytes, 300);
+ assert!(!info.in_use);
+ assert_eq!(out.extra, [(0x5A, 7, Some(100), 4096, 0)]);
+ }
+
+ #[test]
+ fn runs_are_relative_and_can_go_backwards() {
+ let mut attribute = vec![0; 0x40];
+ attribute[0x20..0x22].copy_from_slice(&0x40_u16.to_le_bytes());
+ // 16 clusters at 256, then 8 clusters 128 back, at 128.
+ attribute.extend_from_slice(&[0x21, 0x10, 0x00, 0x01]);
+ attribute.extend_from_slice(&[0x21, 0x08, 0x80, 0xFF, 0x00]);
+ assert_eq!(
+ decode_runs(&attribute, 4096),
+ Some(vec![(256 * 4096, 16 * 4096), (128 * 4096, 8 * 4096)])
+ );
+ }
+
+ #[test]
+ fn reads_cover_used_records_on_cluster_boundaries() {
+ let geometry = Geometry {
+ cluster: 4096,
+ record: RECORD,
+ mft_offset: 0,
+ volume: 1 << 30,
+ };
+ let record = RECORD as u64;
+ let gap = GAP_BYTES / record;
+ let count = 4 * gap;
+ let runs = [(1 << 20, count * record)];
+ let mut bitmap = vec![0_u8; count as usize / 8];
+ let mut used = |number: u64| {
+ bitmap[number as usize / 8] |= 1 << (number % 8);
+ };
+ // One free record short of the gap is read through; the gap
+ // itself splits the reads.
+ let through = 10 + gap;
+ let split = through + 1 + gap;
+ for number in [5, 10, through, split] {
+ used(number);
+ }
+
+ let reads = plan_reads(&runs, &bitmap, &geometry, gap);
+
+ let end = (through + 1).next_multiple_of(4);
+ let start = split - split % 4;
+ assert_eq!(
+ reads,
+ [
+ ((1 << 20) + 4 * record, (end - 4) * record, 4),
+ ((1 << 20) + start * record, 4 * record, start),
+ ]
+ );
+ }
+
+ #[test]
+ fn each_read_parses_into_the_slots_of_its_own_records() {
+ // Records 0..96, each named for its number and sized 100 more,
+ // in an ordinary file: reading it is reading a table, but for the
+ // volume. Reads of 4 records, every third one skipped, handed out
+ // to 3 threads, so no thread's reads are adjacent.
+ let dir = tempfile::tempdir().expect("temp dir");
+ let path = dir.path().join("records");
+ let mut bytes = Vec::new();
+ for number in 0..96_u64 {
+ bytes.extend(record(
+ IN_USE,
+ 0,
+ &[
+ name(ROOT, &format!("f{number}"), 1),
+ non_resident(DATA, 4096, number + 100),
+ ],
+ ));
+ }
+ std::fs::write(&path, bytes).expect("records written");
+ let reads: Vec<_> = (0..24_u64)
+ .filter(|read| read % 3 != 1)
+ .map(|read| (read * 4096, 4096, read * 4))
+ .collect();
+ let mut infos = super::table(&reads, RECORD);
+ let progress = ScanProgress::default();
+ let pool = rayon::ThreadPoolBuilder::new()
+ .num_threads(3)
+ .build()
+ .expect("a pool");
+ let parsed = pool
+ .install(|| {
+ read_records(
+ path.to_str().expect("a UTF-8 path"),
+ RECORD,
+ &reads,
+ &mut infos,
+ false,
+ &progress,
+ )
+ })
+ .expect("the file reads");
+ let (names, texts) = merge(parsed, &mut infos);
+
+ for (number, info) in infos.iter().enumerate() {
+ let read = number / 4 % 3 != 1;
+ assert_eq!(info.in_use, read, "record {number}");
+ if read {
+ assert_eq!(info.apparent, number as u64 + 100);
+ }
+ }
+ assert_eq!(names.len(), 64);
+ for entry in &names {
+ let at = entry.at as usize;
+ let text =
+ &texts[entry.chunk as usize][at..][..usize::from(entry.len)];
+ assert_eq!(text, format!("f{}", entry.child));
+ }
+ assert_eq!(progress.snapshot().files, 64);
+ }
+
+ #[test]
+ fn a_malformed_update_sequence_is_refused_without_panicking() {
+ let valid = record(IN_USE, 0, &[]);
+ assert!(fixup(&mut valid.clone()).is_some());
+ // Counts that do not match 512-byte strides, one of which once
+ // made the stride under two bytes, and an array past the record.
+ for (count, at) in [(0, USA_AT), (2, USA_AT), (600, USA_AT), (3, 1020)]
+ {
+ let mut bytes = valid.clone();
+ bytes[4..6].copy_from_slice(&(at as u16).to_le_bytes());
+ bytes[6..8].copy_from_slice(&(count as u16).to_le_bytes());
+ assert!(fixup(&mut bytes).is_none(), "count {count} at {at}");
+ }
+ }
+
+ #[test]
+ fn a_boot_sector_with_impossible_sizes_is_refused() {
+ let boot = |sector: u16, per_cluster: u8, per_record: u8| {
+ let mut bytes = vec![0; 512];
+ bytes[3..11].copy_from_slice(b"NTFS ");
+ bytes[0x0B..0x0D].copy_from_slice(&sector.to_le_bytes());
+ bytes[0x0D] = per_cluster;
+ bytes[0x30..0x38].copy_from_slice(&4_u64.to_le_bytes());
+ bytes[0x40] = per_record;
+ bytes
+ };
+ // 512-byte sectors, 8 to a cluster, 1 KiB records (2^-(-10)).
+ let geometry = geometry_of(&boot(512, 8, 0xF6)).expect("valid");
+ assert_eq!(
+ (geometry.cluster, geometry.record, geometry.mft_offset),
+ (4096, 1024, 4 * 4096)
+ );
+ for (sector, per_cluster, per_record) in [
+ (513, 8, 0xF6), // sector not a power of two
+ (512, 0, 0xF6), // no sectors to a cluster
+ (512, 0x90, 0xF6), // a shift past 64 bits
+ (512, 0xF0, 0xF6), // a 32 MiB cluster
+ (512, 8, 0xBA), // a record of 2^70 bytes
+ (512, 8, 2), // an 8 KiB record
+ ] {
+ assert!(
+ geometry_of(&boot(sector, per_cluster, per_record)).is_none(),
+ "{sector} {per_cluster:#x} {per_record:#x}"
+ );
+ }
+ assert!(geometry_of(&[0; 16]).is_none());
+ }
+
+ #[test]
+ fn a_name_that_would_resolve_elsewhere_is_skipped() {
+ for text in ["", ".", "..", "a/b", "a\\b", "a\0b"] {
+ let mut bytes = record(IN_USE, 0, &[name(40, text, 1)]);
+ let (out, _) = parse(&mut bytes, 77);
+ assert!(out.names.is_empty() && out.text.is_empty(), "{text:?}");
+ }
+ }
+
+ /// A table of `(record, parent, parent sequence, name, info)` rows
+ /// beneath the root, record 5 at sequence 5.
+ fn table<'a>(
+ rows: &[(u32, u32, u16, &str, Info)],
+ options: &'a ScanOptions,
+ progress: &'a ScanProgress,
+ ) -> Table<'a> {
+ let mut infos = vec![Info::default(); 64];
+ infos[ROOT as usize] = Info {
+ in_use: true,
+ directory: true,
+ sequence: 5,
+ ..Info::default()
+ };
+ let mut text = String::new();
+ let mut names = Vec::new();
+ for &(child, parent, parent_sequence, name, info) in rows {
+ infos[child as usize] = info;
+ names.push(Entry {
+ parent,
+ parent_sequence,
+ child,
+ chunk: 0,
+ at: text.len() as u32,
+ len: name.len() as u16,
+ });
+ text.push_str(name);
+ }
+ names.sort_by_key(|entry| entry.parent);
+ let starts = starts(&names, infos.len());
+ Table {
+ infos,
+ names,
+ texts: vec![text],
+ starts,
+ options,
+ progress,
+ }
+ }
+
+ fn paths(node: &Node, prefix: &str, out: &mut Vec<(String, NodeKind)>) {
+ for child in &node.children {
+ let path = format!("{prefix}{}", child.name);
+ out.push((path.clone(), child.kind));
+ paths(child, &format!("{path}/"), out);
+ }
+ }
+
+ #[test]
+ fn the_table_keeps_what_the_walk_would_list() {
+ let file = Info {
+ in_use: true,
+ sequence: 1,
+ ..Info::default()
+ };
+ let dir = Info {
+ directory: true,
+ sequence: 3,
+ ..file
+ };
+ let rows = [
+ (20, 5, 5, "visible.txt", file),
+ (21, 5, 5, ".dotfile", file),
+ (
+ 22,
+ 5,
+ 5,
+ "hidden.txt",
+ Info {
+ attributes: FILE_ATTRIBUTE_HIDDEN,
+ ..file
+ },
+ ),
+ (
+ 23,
+ 5,
+ 5,
+ "evicted",
+ Info {
+ attributes: FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS,
+ ..dir
+ },
+ ),
+ // A mount point: a name surrogate, measured as a link.
+ (
+ 24,
+ 5,
+ 5,
+ "link",
+ Info {
+ attributes: FILE_ATTRIBUTE_REPARSE_POINT,
+ reparse_tag: 0xA000_0003,
+ ..dir
+ },
+ ),
+ (25, 5, 5, "sub", dir),
+ (26, 25, 3, "inner.txt", file),
+ (27, 25, 3, "deeper", dir),
+ (28, 27, 3, "deepest.txt", file),
+ // The file system's own records.
+ (10, 5, 5, "$system", file),
+ // Entries of an earlier directory in a reused record.
+ (29, 5, 4, "stale.txt", file),
+ (30, 25, 2, "stale-in-sub.txt", file),
+ (31, 5, 5, "linked.txt", Info { names: 2, ..file }),
+ (31, 25, 3, "linked.txt", Info { names: 2, ..file }),
+ ];
+ let everything = vec![
+ ".dotfile",
+ "hidden.txt",
+ "link",
+ "linked.txt",
+ "sub",
+ "sub/deeper",
+ "sub/deeper/deepest.txt",
+ "sub/inner.txt",
+ "sub/linked.txt",
+ "visible.txt",
+ ];
+ let shallow_visible = vec![
+ "link",
+ "linked.txt",
+ "sub",
+ "sub/inner.txt",
+ "sub/linked.txt",
+ "visible.txt",
+ ];
+ let progress = ScanProgress::default();
+ for (include_hidden, max_depth, expected) in
+ [(true, None, everything), (false, Some(1), shallow_visible)]
+ {
+ let options = ScanOptions {
+ include_hidden,
+ max_depth,
+ ..ScanOptions::default()
+ };
+ let Ok(root) = table(&rows, &options, &progress).directory(
+ ROOT,
+ "C:\\".into(),
+ 0,
+ ) else {
+ panic!("the table is not cut short");
+ };
+ let mut found = Vec::new();
+ paths(&root, "", &mut found);
+ found.sort_by(|left, right| left.0.cmp(&right.0));
+ let kind = |path: &str| match path {
+ "link" => NodeKind::Symlink,
+ "sub" | "sub/deeper" => NodeKind::Directory,
+ _ => NodeKind::File,
+ };
+ let expected: Vec<(String, NodeKind)> = expected
+ .into_iter()
+ .map(|path| (path.to_owned(), kind(path)))
+ .collect();
+ assert_eq!(found, expected, "include_hidden {include_hidden}");
+
+ // Only a file with a second name needs its identity kept.
+ let inode = |name: &str| root.child_named(name).unwrap().inode;
+ assert_eq!(inode("linked.txt"), Some((0, 31)));
+ assert_eq!(inode("visible.txt"), None);
+ }
+ }
+
+ #[test]
+ fn a_directory_beneath_itself_gives_up_rather_than_recurse() {
+ let dir = Info {
+ in_use: true,
+ directory: true,
+ sequence: 1,
+ ..Info::default()
+ };
+ let rows = [
+ (40, 5, 5, "loop", dir),
+ (41, 40, 1, "inner", dir),
+ (40, 41, 1, "again", dir),
+ ];
+ let options = ScanOptions::default();
+ let progress = ScanProgress::default();
+ let table = table(&rows, &options, &progress);
+ assert!(table.directory(ROOT, "C:\\".into(), 0).is_err());
+ assert!(!progress.is_cancelled());
+ }
+}
diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs
index f9abac6..2dcda50 100644
--- a/crates/disktree-core/src/scan.rs
+++ b/crates/disktree-core/src/scan.rs
@@ -8,8 +8,8 @@
//! `+1` sentinel, so a directory is only built once its own scan *and* all of
//! its subdirectory tasks have finished,
//! * children are handed to the parent as finished `Node`s and sizes are
-//! aggregated bottom-up in one serial pass, which is also where hardlinks are
-//! de-duplicated.
+//! aggregated bottom-up in one pass over the finished tree, which is also
+//! where hardlinks are de-duplicated.
//!
//! What is added on top of dust's approach: live progress counters the UI can
//! poll without locking, and cooperative cancellation so a re-scan can abandon
@@ -22,13 +22,13 @@ use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::mpsc::{self, Receiver, TryRecvError};
-use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError};
+use std::sync::{Arc, LazyLock, Mutex, MutexGuard, OnceLock, PoisonError};
use std::thread;
use rayon::Scope;
use rustc_hash::FxHashSet;
-use crate::tree::{Metric, Node, NodeKind, aggregate};
+use crate::tree::{Metric, Node, NodeKind, Seen, aggregate, aggregate_deduped};
/// Errors kept verbatim before the list is truncated; the count keeps rising.
const MAX_ERROR_DETAIL: usize = 50;
@@ -105,13 +105,21 @@ pub struct ScanSnapshot {
}
impl ScanProgress {
- fn count_file(&self, bytes: u64) {
- self.files.fetch_add(1, Ordering::Relaxed);
+ /// What a directory's listing found (see `Tally`), or what a reader
+ /// that measures in bulk has come across so far.
+ pub(crate) fn add(&self, files: u64, dirs: u64, bytes: u64) {
+ self.files.fetch_add(files, Ordering::Relaxed);
+ self.dirs.fetch_add(dirs, Ordering::Relaxed);
self.bytes.fetch_add(bytes, Ordering::Relaxed);
}
- fn count_dir(&self) {
- self.dirs.fetch_add(1, Ordering::Relaxed);
+ /// The totals of the finished tree, replacing a bulk reader's running
+ /// count: it met the whole volume, and the tree may be less of it.
+ #[cfg(windows)]
+ pub(crate) fn settle(&self, files: u64, dirs: u64, bytes: u64) {
+ self.files.store(files, Ordering::Relaxed);
+ self.dirs.store(dirs, Ordering::Relaxed);
+ self.bytes.store(bytes, Ordering::Relaxed);
}
fn record_error(&self, path: &Path, error: &io::Error) {
@@ -191,6 +199,7 @@ impl ScanHandle {
never_scanned: OnceLock::new(),
visited_dirs: Mutex::new(FxHashSet::default()),
root: Mutex::new(None),
+ volume: OnceLock::new(),
});
let (sender, result) = mpsc::channel();
@@ -240,6 +249,7 @@ pub fn scan(root: &Path, options: ScanOptions) -> io::Result<Node> {
never_scanned: OnceLock::new(),
visited_dirs: Mutex::new(FxHashSet::default()),
root: Mutex::new(None),
+ volume: OnceLock::new(),
});
let node = scan_blocking(root, &context)?;
progress.finish();
@@ -264,6 +274,10 @@ struct WalkContext {
visited_dirs: Mutex<FxHashSet<(u64, u64)>>,
/// Set by the root's `complete`, read after the scope joins.
root: Mutex<Option<Node>>,
+ /// Serial number of the root's volume, on Windows, for a walk that
+ /// cannot leave it: every listing takes it instead of asking. See
+ /// [`crate::windows::walk_volume`].
+ volume: OnceLock<u64>,
}
impl WalkContext {
@@ -281,33 +295,41 @@ impl WalkContext {
self.progress.is_cancelled()
}
- /// Decide what to do with an entry, and account for the work it implies.
- fn classify(&self, entry: &impl Listed) -> Classified {
- let path = entry.entry_path();
+ /// Decide what to do with an entry listed in `dir`.
+ ///
+ /// The entry's path is built only where it is used: most entries are
+ /// files, and a disk has millions of them. The name is moved out of
+ /// the entry last, so nothing copies it.
+ fn classify(&self, dir: &Path, entry: &mut impl Listed) -> Classified {
let listing = match entry.listing() {
Ok(listing) => listing,
Err(error) => {
- self.progress.record_error(&path, &error);
+ self.progress.record_error(&entry.entry_path(dir), &error);
return Classified::Skipped;
}
};
- let name = entry.display_name();
if !self.options.include_hidden
- && (name.starts_with('.') || entry.hidden())
+ && (entry.name().starts_with('.') || entry.hidden())
{
return Classified::Skipped;
}
let kind = match listing {
- Listing::Symlink => return self.classify_symlink(&path, name),
- Listing::Directory => return self.classify_dir(entry, path, name),
+ Listing::Symlink => {
+ let path = entry.entry_path(dir);
+ return self.classify_symlink(&path, entry.take_name());
+ }
+ Listing::Directory => {
+ let path = entry.entry_path(dir);
+ return self.classify_dir(entry, path);
+ }
Listing::Leaf(kind) => kind,
};
match entry.facts(self.options.apparent_size) {
- Ok(facts) => self.leaf(name, kind, &facts),
+ Ok(facts) => self.leaf(entry.take_name(), kind, &facts),
Err(error) => {
- self.progress.record_error(&path, &error);
+ self.progress.record_error(&entry.entry_path(dir), &error);
Classified::Skipped
}
}
@@ -315,9 +337,8 @@ impl WalkContext {
fn classify_dir(
&self,
- entry: &impl Listed,
+ entry: &mut impl Listed,
path: PathBuf,
- name: Box<str>,
) -> Classified {
if self
.never_scanned
@@ -344,12 +365,8 @@ impl WalkContext {
if let Some(known) = &self.known
&& known.path == path
{
- let tree = (*known.tree).clone();
- self.progress.files.fetch_add(tree.files, Ordering::Relaxed);
- self.progress.bytes.fetch_add(tree.bytes, Ordering::Relaxed);
- self.progress.dirs.fetch_add(tree.dirs, Ordering::Relaxed);
- let mut tree = tree;
- tree.name = name;
+ let mut tree = (*known.tree).clone();
+ tree.name = entry.take_name();
return Classified::Entry(tree);
}
if self.options.one_filesystem
@@ -374,8 +391,10 @@ impl WalkContext {
_ => {}
}
}
- self.progress.count_dir();
- Classified::Subdirectory(path)
+ Classified::Subdirectory {
+ path,
+ name: entry.take_name(),
+ }
}
fn classify_symlink(&self, path: &Path, name: Box<str>) -> Classified {
@@ -385,7 +404,6 @@ impl WalkContext {
return match fs::symlink_metadata(path) {
Ok(meta) => {
let facts = Facts::of(&meta, self.options.apparent_size);
- self.progress.count_file(facts.size);
Classified::Entry(leaf_node(
name,
NodeKind::Symlink,
@@ -422,8 +440,10 @@ impl WalkContext {
{
return Classified::Skipped;
}
- self.progress.count_dir();
- return Classified::Subdirectory(path.to_path_buf());
+ return Classified::Subdirectory {
+ path: path.to_path_buf(),
+ name,
+ };
}
let mut facts = Facts::of(&meta, self.options.apparent_size);
@@ -439,7 +459,6 @@ impl WalkContext {
kind: NodeKind,
facts: &Facts,
) -> Classified {
- self.progress.count_file(facts.size);
// A followed link reaches a file a second way, whatever its link
// count says.
let track = self.options.follow_links || facts.shared;
@@ -454,10 +473,14 @@ impl WalkContext {
/// would cost an open per file; [`crate::windows`] lists a directory with
/// both instead.
trait Listed {
- fn entry_path(&self) -> PathBuf;
+ /// The entry's path inside `dir`, the directory it was listed from.
+ fn entry_path(&self, dir: &Path) -> PathBuf;
/// Non-UTF-8 names are lossy for display. The scan still measures them
/// correctly; only the reported name is approximate.
- fn display_name(&self) -> Box<str>;
+ fn name(&self) -> &str;
+ /// The name, moved out for the tree: after this `name` is empty, and
+ /// `entry_path`, which must open the real file, is off limits.
+ fn take_name(&mut self) -> Box<str>;
fn listing(&self) -> io::Result<Listing>;
/// Size, identity and age of a leaf.
fn facts(&self, apparent_size: bool) -> io::Result<Facts>;
@@ -515,18 +538,29 @@ impl Facts {
}
}
+/// A standard directory entry with its name decoded once, up front.
#[cfg(not(windows))]
-impl Listed for DirEntry {
- fn entry_path(&self) -> PathBuf {
- self.path()
+struct Named {
+ entry: DirEntry,
+ name: Box<str>,
+}
+
+#[cfg(not(windows))]
+impl Listed for Named {
+ fn entry_path(&self, _dir: &Path) -> PathBuf {
+ self.entry.path()
}
- fn display_name(&self) -> Box<str> {
- self.file_name().to_string_lossy().into_owned().into()
+ fn name(&self) -> &str {
+ &self.name
+ }
+
+ fn take_name(&mut self) -> Box<str> {
+ std::mem::take(&mut self.name)
}
fn listing(&self) -> io::Result<Listing> {
- let file_type = self.file_type()?;
+ let file_type = self.entry.file_type()?;
Ok(if file_type.is_symlink() {
Listing::Symlink
} else if file_type.is_dir() {
@@ -539,11 +573,13 @@ impl Listed for DirEntry {
}
fn facts(&self, apparent_size: bool) -> io::Result<Facts> {
- self.metadata().map(|meta| Facts::of(&meta, apparent_size))
+ self.entry
+ .metadata()
+ .map(|meta| Facts::of(&meta, apparent_size))
}
fn directory(&self) -> io::Result<Directory> {
- self.metadata().map(|meta| Directory {
+ self.entry.metadata().map(|meta| Directory {
device: device_of(&meta),
evicted: is_dataless(&meta),
})
@@ -552,12 +588,16 @@ impl Listed for DirEntry {
#[cfg(windows)]
impl Listed for crate::windows::Entry {
- fn entry_path(&self) -> PathBuf {
- self.path()
+ fn entry_path(&self, dir: &Path) -> PathBuf {
+ self.path(dir)
}
- fn display_name(&self) -> Box<str> {
- self.file_name().to_string_lossy().into_owned().into()
+ fn name(&self) -> &str {
+ self.name()
+ }
+
+ fn take_name(&mut self) -> Box<str> {
+ self.take_name()
}
fn listing(&self) -> io::Result<Listing> {
@@ -600,19 +640,33 @@ impl Listed for crate::windows::Entry {
/// List a directory the way [`Listed`] describes.
#[cfg(not(windows))]
-fn list(path: &Path) -> io::Result<fs::ReadDir> {
- fs::read_dir(path)
+fn list(
+ path: &Path,
+ _volume: Option<u64>,
+) -> io::Result<impl Iterator<Item = io::Result<Named>>> {
+ Ok(fs::read_dir(path)?.map(|entry| {
+ entry.map(|entry| Named {
+ name: entry.file_name().into_string().map_or_else(
+ |raw| raw.to_string_lossy().into(),
+ String::into_boxed_str,
+ ),
+ entry,
+ })
+ }))
}
#[cfg(windows)]
-fn list(path: &Path) -> io::Result<crate::windows::ReadDir> {
- crate::windows::read_dir(path)
+fn list(
+ path: &Path,
+ volume: Option<u64>,
+) -> io::Result<crate::windows::ReadDir> {
+ crate::windows::read_dir(path, volume)
}
/// What a directory entry turned out to be.
enum Classified {
/// Descend into this directory on a new task.
- Subdirectory(PathBuf),
+ Subdirectory { path: PathBuf, name: Box<str> },
/// A leaf that contributes size.
Entry(Node),
/// Filtered out, unreadable, or a symlink we chose not to follow.
@@ -627,15 +681,25 @@ struct PendingDir {
/// Starts at 1 for the directory itself; one more per subdirectory task.
/// When it reaches zero the directory is complete.
pending: AtomicUsize,
- /// Files found directly here, and finished subdirectories handed back up.
- children: Mutex<Vec<Node>>,
+ /// The node so far, moved out whole once the directory is complete.
+ partial: Mutex<Partial>,
read_error: AtomicBool,
depth: usize,
}
+/// What a directory's node is made of before it is built: the name the
+/// listing gave it, and the files found directly in it plus the finished
+/// subdirectories handed back up.
+#[derive(Debug, Default)]
+struct Partial {
+ name: Box<str>,
+ children: Vec<Node>,
+}
+
impl PendingDir {
const fn new(
path: PathBuf,
+ name: Box<str>,
parent: Option<Arc<Self>>,
depth: usize,
) -> Self {
@@ -643,21 +707,25 @@ impl PendingDir {
path,
parent,
pending: AtomicUsize::new(1),
- children: Mutex::new(Vec::new()),
+ partial: Mutex::new(Partial {
+ name,
+ children: Vec::new(),
+ }),
read_error: AtomicBool::new(false),
depth,
}
}
/// Turn a completed directory into a node. Only called when `pending` has
- /// reached zero, so every child is already in `self.children`.
+ /// reached zero, so every child is already in `self.partial`.
///
/// `bytes` and `own_bytes` are left at zero on purpose: the walk cannot
/// know the aggregate, and [`crate::tree::aggregate`] derives both from the
/// children once every child is present.
fn build(&self) -> Node {
+ let partial = std::mem::take(&mut *lock(&self.partial));
Node {
- name: file_name(&self.path),
+ name: partial.name,
kind: NodeKind::Directory,
bytes: 0,
own_bytes: 0,
@@ -669,7 +737,7 @@ impl PendingDir {
modified: 0,
category: crate::classify::Category::Other,
reclaim: None,
- children: std::mem::take(&mut *lock(&self.children)),
+ children: partial.children,
}
}
}
@@ -682,23 +750,64 @@ fn scan_blocking(root: &Path, context: &Arc<WalkContext>) -> io::Result<Node> {
format!("{} is not a directory", root.display()),
));
}
- let canonical = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
- let never = crate::space::never_scanned(root, &canonical);
+ let resolved = root.canonicalize().ok();
+ let canonical = resolved.as_deref().unwrap_or(root);
+ let never = crate::space::never_scanned(root, canonical);
let _ = context.never_scanned.set(never.into_iter().collect());
if context.options.one_filesystem {
*lock(&context.root_device) = Some(device_of(&root_meta));
- if let Some(foreign) = crate::space::foreign_mounts_for(&canonical) {
+ if let Some(foreign) = crate::space::foreign_mounts_for(canonical) {
let _ = context.foreign_mounts.set(foreign.into_iter().collect());
}
}
+ // Reading the file table is far faster than any walk, where the
+ // volume allows it. On the walk's pool too: see `WALK_POOL`.
+ #[cfg(windows)]
+ {
+ let progress = &context.progress;
+ let read = WALK_POOL.install(|| {
+ crate::mft::scan(root, canonical, &context.options, progress)
+ });
+ if let Some(node) = read {
+ let node = node?;
+ // Cancelled after the reader's last check: the rescan that
+ // asked is waiting, and this tree is not wanted.
+ if progress.is_cancelled() {
+ return Err(crate::mft::cancelled());
+ }
+ let node = finish_tree(node, &context.options);
+ // The reader counted every file on the volume; the tree may
+ // hold fewer.
+ progress.settle(node.files, node.dirs, node.bytes);
+ return Ok(node);
+ }
+ // The reader gave up, or never started: the walk counts from nothing.
+ progress.settle(0, 0, 0);
+ // Asking once instead of per directory; see `walk_volume` for
+ // when a walk that does not follow links cannot leave the volume.
+ // Only on a resolved root: a mapped share's letter kept as typed
+ // looks like a local drive.
+ if !context.options.follow_links
+ && let Some(resolved) = &resolved
+ && let Some(serial) = crate::windows::walk_volume(resolved)
+ {
+ let _ = context.volume.set(serial);
+ }
+ }
if context.options.follow_links
&& let Some(key) = identity_of(root, &root_meta)
{
lock(&context.visited_dirs).insert(key);
}
- let root_dir = Arc::new(PendingDir::new(root.to_path_buf(), None, 0));
- rayon::scope(|scope| walk(scope, &root_dir, context));
+ let root_dir = Arc::new(PendingDir::new(
+ root.to_path_buf(),
+ file_name(root),
+ None,
+ 0,
+ ));
+ let context: &WalkContext = context;
+ WALK_POOL.install(|| rayon::scope(|scope| walk(scope, &root_dir, context)));
let node = lock(&context.root).take();
let node = node.ok_or_else(|| {
@@ -707,32 +816,101 @@ fn scan_blocking(root: &Path, context: &Arc<WalkContext>) -> io::Result<Node> {
Ok(finish_tree(node, &context.options))
}
+/// The walk's workers, and the file table reader's. Listing directories
+/// contends in the file system, and past some number of threads more of
+/// them only queue: on Windows a home directory of 3.4 million files
+/// walked in 6.5 s on 16 threads and 8.1 s on all 24. That cap was
+/// measured on one 24-thread NTFS laptop; another disk or CPU may want
+/// another. Elsewhere the default count. A pool of its own on every
+/// platform all the same: work the UI hands the global pool, such as
+/// re-ranking a tree, must not queue behind a scan's blocking reads.
+static WALK_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+ let threads = if cfg!(windows) {
+ rayon::current_num_threads().min(16)
+ } else {
+ rayon::current_num_threads()
+ };
+ rayon::ThreadPoolBuilder::new()
+ .num_threads(threads)
+ .thread_name(|index| format!("disktree-walk-{index}"))
+ .build()
+ .expect("a thread pool")
+});
+
+/// Entries counted before the shared progress counters are touched. Every
+/// worker adding to the same counters per entry keeps one cache line
+/// bouncing between cores; per directory, or per this many entries in a
+/// wide one so the meter still moves, is invisible to a reader.
+const TALLY_EVERY: u64 = 1024;
+
+/// What one directory's listing has found so far, for the progress meter.
+#[derive(Default)]
+struct Tally {
+ files: u64,
+ dirs: u64,
+ bytes: u64,
+}
+
+impl Tally {
+ const fn add(&mut self, node: &Node) {
+ if node.is_dir() {
+ // A subtree taken whole from an earlier scan.
+ self.files += node.files;
+ self.dirs += node.dirs;
+ } else {
+ // A link counts as a file on the meter, as it always has,
+ // though the tree does not count it.
+ self.files += 1;
+ }
+ self.bytes += node.bytes;
+ }
+
+ fn flush(&mut self, progress: &ScanProgress) {
+ progress.add(self.files, self.dirs, self.bytes);
+ *self = Self::default();
+ }
+}
+
/// Read one directory, spawn a task per subdirectory, then report completion.
///
/// Flat tasks inside the scope: the worker stack never grows with tree depth.
-fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {
+fn walk<'scope>(
+ scope: &Scope<'scope>,
+ dir: &Arc<PendingDir>,
+ context: &'scope WalkContext,
+) {
let mut subdirs: Vec<Arc<PendingDir>> = Vec::new();
let mut leaves: Vec<Node> = Vec::new();
+ let mut tally = Tally::default();
- match list(&dir.path) {
+ match list(&dir.path, context.volume.get().copied()) {
Ok(entries) => {
for entry in entries {
if context.cancelled() {
break;
}
match entry {
- Ok(entry) => match context.classify(&entry) {
- Classified::Subdirectory(path) => {
- let child = Arc::new(PendingDir::new(
- path,
- Some(Arc::clone(dir)),
- dir.depth + 1,
- ));
- subdirs.push(child);
+ Ok(mut entry) => {
+ match context.classify(&dir.path, &mut entry) {
+ Classified::Subdirectory { path, name } => {
+ tally.dirs += 1;
+ subdirs.push(Arc::new(PendingDir::new(
+ path,
+ name,
+ Some(Arc::clone(dir)),
+ dir.depth + 1,
+ )));
+ }
+ Classified::Entry(node) => {
+ tally.add(&node);
+ leaves.push(node);
+ }
+ Classified::Skipped => {}
+ }
+ if tally.files + tally.dirs >= TALLY_EVERY {
+ tally.flush(&context.progress);
}
- Classified::Entry(node) => leaves.push(node),
- Classified::Skipped => {}
- },
+ }
Err(error) => {
context.progress.record_error(&dir.path, &error);
}
@@ -744,6 +922,7 @@ fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {
dir.read_error.store(true, Ordering::Relaxed);
}
}
+ tally.flush(&context.progress);
// A depth-limited scan still measures what is directly in the directory,
// it just does not descend further.
@@ -751,20 +930,30 @@ fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {
.options
.max_depth
.is_none_or(|max_depth| dir.depth < max_depth);
+ // Sized once for the leaves and every subdirectory to come, before any
+ // subdirectory task can push its node: otherwise that first push
+ // doubles a list just sized to the leaves, and the slack stays in the
+ // finished tree. The leaves are copied, not moved: the list they grew
+ // in carries up to twice their size in slack, and moving it in whole
+ // made a home directory walk a second slower (measured, 5 interleaved
+ // pairs).
+ let spawned = if descend { subdirs.len() } else { 0 };
+ {
+ let mut partial = lock(&dir.partial);
+ partial.children.reserve_exact(leaves.len() + spawned);
+ partial.children.extend(leaves);
+ }
if descend {
for subdir in subdirs {
if context.cancelled() {
break;
}
dir.pending.fetch_add(1, Ordering::AcqRel);
- let subdir = Arc::clone(&subdir);
- let context = Arc::clone(context);
- scope.spawn(move |scope| walk(scope, &subdir, &context));
+ scope.spawn(move |scope| walk(scope, &subdir, context));
}
}
- lock(&dir.children).extend(leaves);
- signal_done(&Arc::clone(dir), context);
+ signal_done(dir, context);
}
/// Report that one task for `dir` is done: either its own scan, or one of its
@@ -773,44 +962,32 @@ fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {
/// This is the whole reason for the `+1` sentinel: a directory is only built
/// once its own scan *and* every subdirectory task has finished, no matter
/// which of them lands last.
-fn signal_done(dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {
+fn signal_done(dir: &Arc<PendingDir>, context: &WalkContext) {
if dir.pending.fetch_sub(1, Ordering::AcqRel) != 1 {
return;
}
let node = dir.build();
- let Some(parent) = dir.parent.clone() else {
+ let Some(parent) = &dir.parent else {
*lock(&context.root) = Some(node);
return;
};
- lock(&parent.children).push(node);
- signal_done(&parent, context);
+ lock(&parent.partial).children.push(node);
+ signal_done(parent, context);
}
-/// Charge a hardlinked file once, then derive every aggregate from the result.
-///
-/// Zeroing `own_bytes` rather than `bytes` is deliberate:
-/// [`crate::tree::aggregate`] recomputes totals from the direct contents, so a
-/// patched `bytes` would be overwritten.
+/// Charge a hardlinked file once, derive every aggregate from the result,
+/// then classify. On the walk's pool: the UI's own work on the global pool
+/// must not queue behind a scan finishing.
fn finish_tree(mut node: Node, options: &ScanOptions) -> Node {
- if options.dedup_hardlinks {
- let mut seen = FxHashSet::default();
- mark_duplicate_hardlinks(&mut node, &mut seen);
- }
- aggregate(&mut node, options.metric);
- crate::classify::classify(&mut node);
- node
-}
-
-fn mark_duplicate_hardlinks(node: &mut Node, seen: &mut FxHashSet<(u64, u64)>) {
- if !node.is_dir() {
- if node.inode.is_some_and(|key| !seen.insert(key)) {
- node.own_bytes = 0;
+ WALK_POOL.install(|| {
+ if options.dedup_hardlinks {
+ aggregate_deduped(&mut node, options.metric, &Seen::new());
+ } else {
+ aggregate(&mut node, options.metric);
}
- return;
- }
- for child in &mut node.children {
- mark_duplicate_hardlinks(child, seen);
- }
+ crate::classify::classify(&mut node);
+ });
+ node
}
/// A leaf. Its identity is kept only when `track` says the walk could meet
@@ -958,7 +1135,7 @@ fn kind_of(meta: &Metadata, file_type: fs::FileType) -> NodeKind {
/// The display name of a scanned root: its final component, or the path itself
/// for `/`.
-fn file_name(path: &Path) -> Box<str> {
+pub(crate) fn file_name(path: &Path) -> Box<str> {
path.file_name()
.map_or_else(
|| path.to_string_lossy().into_owned(),
@@ -969,7 +1146,7 @@ fn file_name(path: &Path) -> Box<str> {
/// A poisoned lock means another task panicked; the data is still a valid tree
/// prefix, and refusing to read it would turn one panic into two.
-fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+pub(crate) fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().unwrap_or_else(PoisonError::into_inner)
}
@@ -1269,6 +1446,7 @@ mod tests {
never_scanned: OnceLock::new(),
visited_dirs: Mutex::new(FxHashSet::default()),
root: Mutex::new(None),
+ volume: OnceLock::new(),
});
let tree = scan_blocking(root, &context).expect("scan still succeeds");
let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o700));
diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rs
index 997882f..6f8e93b 100644
--- a/crates/disktree-core/src/tree.rs
+++ b/crates/disktree-core/src/tree.rs
@@ -1,6 +1,11 @@
//! The scanned tree.
use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{LazyLock, Mutex, OnceLock};
+
+use rayon::prelude::*;
+use rustc_hash::FxHashSet;
use crate::classify::{Category, Reclaim};
@@ -235,7 +240,104 @@ impl Node {
/// de-duplication rewrites a leaf's weight, and every total above it —
/// including its parent's "direct" figure — follows without a second pass.
pub fn aggregate(node: &mut Node, metric: Metric) {
+ aggregate_at(node, metric, 0, None);
+}
+
+/// [`aggregate`], charging a hardlinked file once: a leaf whose identity
+/// `seen` already holds weighs nothing. Which of a file's names is charged
+/// is whichever a worker reaches first, so two scans of an unchanged tree
+/// can split it differently between folders; the totals are the same.
+pub(crate) fn aggregate_deduped(node: &mut Node, metric: Metric, seen: &Seen) {
+ aggregate_at(node, metric, 0, Some(seen));
+}
+
+/// Identities a finish pass has met.
+///
+/// A hash set of every file on a disk is millions of random writes into a
+/// table too big for any cache. Inode and file record numbers are small
+/// integers, though, so a key on the first volume met whose whole number
+/// is below the bitmap's size gets one bit instead: an 8 MiB bitmap the
+/// workers set without locking. The file table reader's record numbers
+/// fit, as do most inodes. Anything else goes into the hash set, whole:
+/// a Windows file id keeps the record's reuse count in its top 16 bits,
+/// and that is what tells a file from an older one on the same record,
+/// which a subtree kept from an earlier scan can still hold. The set is
+/// sharded for the walk's millions of those: 4.1 million ids took 150 ms
+/// across 64 locks and 1.2 s behind one, in a synthetic run.
+pub(crate) struct Seen {
+ /// The first volume met, which is the scanned root's in all but a scan
+ /// that leaves its volume.
+ volume: OnceLock<u64>,
+ /// Made on the first key that fits: most Unix trees, where only files
+ /// with more than one name carry an identity, never need it.
+ bits: LazyLock<Box<[AtomicU64]>>,
+ rest: Box<[Shard]>,
+}
+
+/// One lock's part of the hash set.
+type Shard = Mutex<FxHashSet<(u64, u64)>>;
+
+impl Seen {
+ /// Numbers the bitmap covers: sixty-four million, more files than a
+ /// desktop volume holds, for 8 MiB written once.
+ const BITS: u64 = 1 << 26;
+
+ /// Hash set shards, as a power of two.
+ const SHARD_BITS: u32 = 6;
+
+ pub(crate) fn new() -> Self {
+ Self {
+ volume: OnceLock::new(),
+ bits: LazyLock::new(|| {
+ std::iter::repeat_with(|| AtomicU64::new(0))
+ .take((Self::BITS / 64) as usize)
+ .collect()
+ }),
+ rest: std::iter::repeat_with(Mutex::default)
+ .take(1 << Self::SHARD_BITS)
+ .collect(),
+ }
+ }
+
+ /// Whether `key` is new.
+ fn insert(&self, key: (u64, u64)) -> bool {
+ let (volume, number) = key;
+ if *self.volume.get_or_init(|| volume) == volume && number < Self::BITS
+ {
+ let bit = 1 << (number % 64);
+ let word = &self.bits[(number / 64) as usize];
+ return word.fetch_or(bit, Ordering::Relaxed) & bit == 0;
+ }
+ // The top bits of a multiplicative hash, with another constant
+ // than the set's own hasher, so a shard's keys still spread over
+ // its table.
+ let shard = (number.wrapping_mul(0x9E37_79B9_7F4A_7C15)
+ >> (u64::BITS - Self::SHARD_BITS)) as usize;
+ crate::scan::lock(&self.rest[shard]).insert(key)
+ }
+}
+
+/// Levels whose subtrees are aggregated in parallel. A top level already
+/// splits a disk into enough work for every thread; below it a parallel
+/// frame per level would only cut the depth that fits in a worker's stack.
+pub(crate) const PARALLEL_LEVELS: usize = 4;
+
+fn aggregate_at(
+ node: &mut Node,
+ metric: Metric,
+ depth: usize,
+ seen: Option<&Seen>,
+) {
if !node.is_dir() {
+ // Every name of a file has the file's size, so one that weighs
+ // nothing need not be remembered to be charged once.
+ if node.own_bytes > 0
+ && let Some(seen) = seen
+ && let Some(key) = node.inode
+ && !seen.insert(key)
+ {
+ node.own_bytes = 0;
+ }
node.bytes = node.own_bytes;
node.files = node.own_files;
node.dirs = 0;
@@ -248,14 +350,23 @@ pub fn aggregate(node: &mut Node, metric: Metric) {
let mut own_files = 0;
let mut dirs: u64 = 1;
let mut modified = 0;
- for child in &mut node.children {
- aggregate(child, metric);
+ if depth < PARALLEL_LEVELS {
+ node.children
+ .par_iter_mut()
+ .for_each(|child| aggregate_at(child, metric, depth + 1, seen));
+ } else {
+ for child in &mut node.children {
+ aggregate_at(child, metric, depth + 1, seen);
+ }
+ }
+ for child in &node.children {
modified = modified.max(child.modified);
- bytes += child.bytes;
+ // Saturating: a corrupt volume's file table can claim any size.
+ bytes = child.bytes.saturating_add(bytes);
files += child.files;
dirs += child.dirs;
if !child.is_dir() {
- own_bytes += child.bytes;
+ own_bytes = child.bytes.saturating_add(own_bytes);
own_files += child.files;
}
}
@@ -267,8 +378,11 @@ pub fn aggregate(node: &mut Node, metric: Metric) {
node.modified = modified;
// Largest first: a treemap lays out big tiles best, and the order is what
- // makes "descend into the largest child" meaningful.
- node.children.sort_by(|left, right| {
+ // makes "descend into the largest child" meaningful. Unstable: names in
+ // one directory are distinct, and the only ties are names that decoded
+ // to the same lossy text, whose order does not matter, so the stable
+ // sort's scratch allocation buys nothing.
+ node.children.sort_unstable_by(|left, right| {
right
.value(metric)
.cmp(&left.value(metric))
@@ -300,6 +414,42 @@ mod tests {
Node::entry(name, NodeKind::File, bytes)
}
+ #[test]
+ fn seen_charges_an_identity_once_in_either_store() {
+ let seen = Seen::new();
+ // The first volume: the bitmap.
+ assert!(seen.insert((7, 42)));
+ assert!(!seen.insert((7, 42)));
+ assert!(seen.insert((7, 43)));
+ // A number past the bitmap, and another volume: the hash set.
+ let far = Seen::BITS + 42;
+ assert!(seen.insert((7, far)));
+ assert!(!seen.insert((7, far)));
+ assert!(seen.insert((8, 42)), "another volume's 42 is another file");
+ assert!(!seen.insert((8, 42)));
+ }
+
+ #[test]
+ fn identities_that_differ_only_in_their_top_bits_are_two_files() {
+ // Record 0x2A of one volume under two reuse counts, as a subtree kept
+ // from an earlier scan and a later file can hold, and a real second
+ // name of the newer one.
+ let older = (7, (1 << 48) | 0x2A);
+ let newer = (7, (2 << 48) | 0x2A);
+ let mut root = Node::directory("root");
+ for (name, bytes, key) in [
+ ("older", 4096, older),
+ ("newer", 8192, newer),
+ ("newer again", 8192, newer),
+ ] {
+ let mut file = leaf(name, bytes);
+ file.inode = Some(key);
+ root.children.push(file);
+ }
+ aggregate_deduped(&mut root, Metric::Bytes, &Seen::new());
+ assert_eq!(root.bytes, 4096 + 8192);
+ }
+
#[test]
fn aggregate_derives_totals_and_orders_children() {
let mut root = Node::directory("root");
diff --git a/crates/disktree-core/src/windows.rs b/crates/disktree-core/src/windows.rs
index 9e9468d..b036bdb 100644
--- a/crates/disktree-core/src/windows.rs
+++ b/crates/disktree-core/src/windows.rs
@@ -23,11 +23,10 @@ use std::os::windows::ffi::{OsStrExt as _, OsStringExt as _};
use std::os::windows::fs::{MetadataExt as _, OpenOptionsExt as _};
use std::os::windows::io::AsRawHandle as _;
use std::path::{Component, Path, PathBuf, Prefix};
-use std::sync::Arc;
use windows_sys::Win32::Foundation::{
ERROR_INVALID_FUNCTION, ERROR_INVALID_LEVEL, ERROR_INVALID_PARAMETER,
- ERROR_NO_MORE_FILES, ERROR_NOT_SUPPORTED, INVALID_HANDLE_VALUE,
+ ERROR_NO_MORE_FILES, ERROR_NOT_SUPPORTED, INVALID_HANDLE_VALUE, MAX_PATH,
};
use windows_sys::Win32::Storage::FileSystem::{
FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_HIDDEN,
@@ -36,7 +35,8 @@ use windows_sys::Win32::Storage::FileSystem::{
FILE_ID_EXTD_DIR_INFO, FILE_LIST_DIRECTORY, FILE_READ_ATTRIBUTES,
FileIdExtdDirectoryInfo, FindFirstVolumeW, FindNextVolumeW,
FindVolumeClose, GetDiskFreeSpaceExW, GetFileInformationByHandleEx,
- GetVolumePathNameW, GetVolumePathNamesForVolumeNameW, SYNCHRONIZE,
+ GetVolumeInformationW, GetVolumePathNameW,
+ GetVolumePathNamesForVolumeNameW, SYNCHRONIZE,
};
use crate::space::SpaceInfo;
@@ -73,10 +73,15 @@ pub enum Kind {
}
/// One directory entry, with what a measurement needs.
+///
+/// The name is decoded to text once, here, since that is what the tree
+/// keeps: the listing hands the walk one allocation per entry, not three.
#[derive(Debug)]
pub struct Entry {
- dir: Arc<Path>,
- name: OsString,
+ name: Box<str>,
+ /// The name as Windows spells it, only when `name` could not: an
+ /// unpaired surrogate is shown as U+FFFD but must still open.
+ exact: Option<Box<OsStr>>,
kind: Kind,
apparent: u64,
allocated: u64,
@@ -86,14 +91,29 @@ pub struct Entry {
}
impl Entry {
- pub fn path(&self) -> PathBuf {
- self.dir.join(&self.name)
+ /// The entry's path inside `dir`, the directory it was listed from.
+ pub fn path(&self, dir: &Path) -> PathBuf {
+ dir.join(self.file_name())
}
pub fn file_name(&self) -> &OsStr {
+ self.exact
+ .as_deref()
+ .unwrap_or_else(|| OsStr::new(&*self.name))
+ }
+
+ /// The name as text; see [`Self::file_name`] for what it may lose.
+ pub fn name(&self) -> &str {
&self.name
}
+ /// The name, moved out: the tree keeps it, so nothing copies it. The
+ /// entry's other accessors keep working; `name`, `file_name` and
+ /// `path` do not.
+ pub fn take_name(&mut self) -> Box<str> {
+ std::mem::take(&mut self.name)
+ }
+
pub const fn kind(&self) -> Kind {
self.kind
}
@@ -133,7 +153,7 @@ impl Entry {
self.identity
}
- fn from_std(dir: &Arc<Path>, entry: &fs::DirEntry) -> io::Result<Self> {
+ fn from_std(entry: &fs::DirEntry) -> io::Result<Self> {
let meta = entry.metadata()?;
let file_type = meta.file_type();
let kind = if file_type.is_symlink() {
@@ -143,9 +163,15 @@ impl Entry {
} else {
Kind::File
};
+ let (name, exact) = match entry.file_name().into_string() {
+ Ok(name) => (name.into_boxed_str(), None),
+ Err(raw) => {
+ (raw.to_string_lossy().into(), Some(raw.into_boxed_os_str()))
+ }
+ };
Ok(Self {
- dir: Arc::clone(dir),
- name: entry.file_name(),
+ name,
+ exact,
kind,
apparent: meta.len(),
allocated: meta.len(),
@@ -162,7 +188,6 @@ impl Entry {
/// and `..`.
#[derive(Debug)]
pub struct ReadDir {
- dir: Arc<Path>,
source: Source,
}
@@ -189,7 +214,12 @@ struct Records {
}
/// List `dir`, with each entry's allocation and file id.
-pub fn read_dir(dir: &Path) -> io::Result<ReadDir> {
+///
+/// `volume` is the serial number of the volume `dir` is on, when the
+/// caller knows it for every directory of a walk (see [`walk_volume`]):
+/// asking per directory is several requests to the file system on top of
+/// the listing. `None` asks.
+pub fn read_dir(dir: &Path, volume: Option<u64>) -> io::Result<ReadDir> {
let handle = OpenOptions::new()
.access_mode(FILE_LIST_DIRECTORY | FILE_READ_ATTRIBUTES | SYNCHRONIZE)
// Needed to open a directory at all. Without
@@ -197,9 +227,11 @@ pub fn read_dir(dir: &Path) -> io::Result<ReadDir> {
// a walk that chose to enter one wants.
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
.open(dir)?;
- let volume = winapi_util::file::information(&handle)
- .ok()
- .map(|info| info.volume_serial_number());
+ let volume = volume.or_else(|| {
+ winapi_util::file::information(&handle)
+ .ok()
+ .map(|info| info.volume_serial_number())
+ });
let mut records = Records {
handle,
volume,
@@ -207,17 +239,16 @@ pub fn read_dir(dir: &Path) -> io::Result<ReadDir> {
next: None,
done: false,
};
- let dir: Arc<Path> = Arc::from(dir);
// The first fill shows whether the file system implements this listing,
// before any entry has been handed out.
let source = match records.fill() {
Ok(()) => Source::Records(records),
Err(error) if unsupported(&error) => {
- Source::Std(Box::new(fs::read_dir(&dir)?))
+ Source::Std(Box::new(fs::read_dir(dir)?))
}
Err(error) => return Err(error),
};
- Ok(ReadDir { dir, source })
+ Ok(ReadDir { source })
}
impl Iterator for ReadDir {
@@ -227,7 +258,7 @@ impl Iterator for ReadDir {
let records = match &mut self.source {
Source::Std(listing) => {
return listing.next().map(|entry| {
- entry.and_then(|entry| Entry::from_std(&self.dir, &entry))
+ entry.and_then(|entry| Entry::from_std(&entry))
});
}
Source::Records(records) => records,
@@ -243,7 +274,7 @@ impl Iterator for ReadDir {
}
continue;
};
- match records.record(&self.dir, offset) {
+ match records.record(offset) {
Ok((entry, next)) => {
records.next = next;
if let Some(entry) = entry {
@@ -304,7 +335,6 @@ impl Records {
/// the buffer before it is used.
fn record(
&self,
- dir: &Arc<Path>,
offset: usize,
) -> io::Result<(Option<Entry>, Option<usize>)> {
let malformed =
@@ -333,11 +363,11 @@ impl Records {
_ => return Err(malformed()),
};
- let name: Vec<u16> = name_bytes
+ let units = name_bytes
.chunks_exact(2)
- .map(|pair| u16::from_ne_bytes([pair[0], pair[1]]))
- .collect();
- if name == [u16::from(b'.')] || name == [u16::from(b'.'); 2] {
+ .map(|pair| u16::from_ne_bytes([pair[0], pair[1]]));
+ let (name, exact) = decode_name(units);
+ if &*name == "." || &*name == ".." {
return Ok((None, next));
}
@@ -358,8 +388,8 @@ impl Records {
header[id_at..id_at + 16].try_into().expect("sixteen"),
);
let entry = Entry {
- dir: Arc::clone(dir),
- name: OsString::from_wide(&name),
+ name,
+ exact,
kind,
apparent: bytes_from(signed(offset_of!(
FILE_ID_EXTD_DIR_INFO,
@@ -380,6 +410,27 @@ impl Records {
}
}
+/// A UTF-16 name as text, decoded in one pass; nearly every name is ASCII,
+/// so one code unit is one byte and the text is allocated once, at its
+/// final size. An unpaired surrogate is replaced, and the exact name kept
+/// beside it so the path still opens.
+fn decode_name(
+ units: impl Iterator<Item = u16> + Clone,
+) -> (Box<str>, Option<Box<OsStr>>) {
+ let mut name = String::with_capacity(units.size_hint().0);
+ let mut lossy = false;
+ for unit in char::decode_utf16(units.clone()) {
+ name.push(unit.unwrap_or_else(|_| {
+ lossy = true;
+ char::REPLACEMENT_CHARACTER
+ }));
+ }
+ let exact = lossy.then(|| {
+ OsString::from_wide(&units.collect::<Vec<u16>>()).into_boxed_os_str()
+ });
+ (name.into_boxed_str(), exact)
+}
+
/// The part of a 128-bit file id that names a file by itself.
///
/// NTFS keeps the high half zero ([MS-FSCC] 2.1.10), so the low half is the
@@ -403,15 +454,35 @@ fn file_id(id: u128) -> Option<u64> {
///
/// [MS-FSCC]: https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-fscc/2d3333fe-fc98-4a6f-98a2-4bb805aff407
pub fn identity(path: &Path) -> Option<(u64, u64)> {
+ let info = information(path)?;
+ let index = info.file_index();
+ (index != 0 && index != u64::MAX)
+ .then(|| (info.volume_serial_number(), index))
+}
+
+/// Serial number of the volume every directory under `canonical` is on,
+/// for a walk that does not follow links. Only a local drive has one:
+/// mounted folders there are reparse points the walk takes for links,
+/// while a share's DFS links are not, and lead to other servers' volumes.
+/// `None` for a share, or when the serial cannot be read.
+pub fn walk_volume(canonical: &Path) -> Option<u64> {
+ let Some(Component::Prefix(prefix)) = canonical.components().next() else {
+ return None;
+ };
+ if !matches!(prefix.kind(), Prefix::Disk(_) | Prefix::VerbatimDisk(_)) {
+ return None;
+ }
+ information(canonical).map(|info| info.volume_serial_number())
+}
+
+/// What the file system says about `path`, following links.
+fn information(path: &Path) -> Option<winapi_util::file::Information> {
let file = OpenOptions::new()
.access_mode(FILE_READ_ATTRIBUTES)
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
.open(path)
.ok()?;
- let info = winapi_util::file::information(&file).ok()?;
- let index = info.file_index();
- (index != 0 && index != u64::MAX)
- .then(|| (info.volume_serial_number(), index))
+ winapi_util::file::information(&file).ok()
}
/// Capacity and free space of the volume holding `path`.
@@ -531,6 +602,119 @@ pub fn is_mount_point(path: &Path) -> bool {
&& volume_root(path).is_some_and(|root| same(&root, path))
}
+/// Whether this process runs as an administrator, with the elevated token
+/// UAC hands out: what reading a volume directly takes.
+pub fn elevated() -> bool {
+ // SAFETY: takes no arguments and only reads the process token.
+ unsafe { windows_sys::Win32::UI::Shell::IsUserAnAdmin() != 0 }
+}
+
+/// Whether `root` is a whole drive an administrator would read from its
+/// file table: a drive letter's root, formatted NTFS. Anything else, from
+/// a folder or a mounted folder to another file system, is walked either way.
+pub fn file_table_readable(root: &Path) -> bool {
+ let canonical = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
+ let Some(letter) = drive_letter(&canonical) else {
+ return false;
+ };
+ let volume: Vec<u16> = format!("{letter}:\\\0").encode_utf16().collect();
+ let mut name = [0_u16; MAX_PATH as usize + 1];
+ let length = u32::try_from(name.len()).unwrap_or(u32::MAX);
+ // SAFETY: `volume` is NUL-terminated and outlives the call, `name` is
+ // writable for the length passed, and the other outputs may be null.
+ let ok = unsafe {
+ GetVolumeInformationW(
+ volume.as_ptr(),
+ std::ptr::null_mut(),
+ 0,
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ name.as_mut_ptr(),
+ length,
+ )
+ };
+ let end = name.iter().position(|&unit| unit == 0).unwrap_or(0);
+ ok != 0 && String::from_utf16_lossy(&name[..end]) == "NTFS"
+}
+
+/// Start `program` with `args` as an administrator, through the UAC
+/// prompt. Returns once the prompt is answered; declining it is an error
+/// (`ERROR_CANCELLED`).
+pub fn run_elevated(program: &Path, args: &[OsString]) -> io::Result<()> {
+ let file = wide(program, false)?;
+ let mut line: Vec<u16> = Vec::new();
+ for arg in args {
+ if !line.is_empty() {
+ line.push(u16::from(b' '));
+ }
+ line.extend(quoted(arg));
+ }
+ line.push(0);
+ let verb: Vec<u16> = "runas\0".encode_utf16().collect();
+ // SAFETY: every string is NUL-terminated and outlives the call; a null
+ // window and directory are allowed.
+ let instance = unsafe {
+ windows_sys::Win32::UI::Shell::ShellExecuteW(
+ std::ptr::null_mut(),
+ verb.as_ptr(),
+ file.as_ptr(),
+ line.as_ptr(),
+ std::ptr::null(),
+ windows_sys::Win32::UI::WindowsAndMessaging::SW_SHOWNORMAL,
+ )
+ };
+ // Anything above 32 is success, by the function's own convention.
+ if instance as usize > 32 {
+ Ok(())
+ } else {
+ Err(io::Error::last_os_error())
+ }
+}
+
+/// `C` when `canonical` is the root of drive `C:`; `None` for anything
+/// else, a folder or a volume mounted in one.
+pub fn drive_letter(canonical: &Path) -> Option<char> {
+ let mut components = canonical.components();
+ let Some(Component::Prefix(prefix)) = components.next() else {
+ return None;
+ };
+ let (Prefix::Disk(letter) | Prefix::VerbatimDisk(letter)) = prefix.kind()
+ else {
+ return None;
+ };
+ (components.next() == Some(Component::RootDir)
+ && components.next().is_none())
+ .then_some(char::from(letter))
+}
+
+/// `arg` quoted so a program's C runtime splits it back out whole: inside
+/// quotes, backslashes are literal except before a quote, where they must
+/// be doubled, so `C:\` becomes `"C:\\"` and not `"C:\"`.
+fn quoted(arg: &OsStr) -> Vec<u16> {
+ let quote = u16::from(b'"');
+ let backslash = u16::from(b'\\');
+ let mut out = vec![quote];
+ let mut slashes = 0;
+ for unit in arg.encode_wide() {
+ if unit == backslash {
+ slashes += 1;
+ continue;
+ }
+ let run = if unit == quote {
+ 2 * slashes + 1
+ } else {
+ slashes
+ };
+ out.extend(std::iter::repeat_n(backslash, run));
+ slashes = 0;
+ out.push(unit);
+ }
+ out.extend(std::iter::repeat_n(backslash, 2 * slashes));
+ out.push(quote);
+ out
+}
+
/// Whether `left` and `right` name the same place, compared the way Windows
/// compares names: see [`folded`].
pub fn same(left: &Path, right: &Path) -> bool {
@@ -615,7 +799,7 @@ fn bytes_from(value: i64) -> u64 {
}
/// A `FILETIME` as Unix seconds, `0` before 1970.
-const fn unix_seconds(ticks: i64) -> i64 {
+pub const fn unix_seconds(ticks: i64) -> i64 {
if ticks <= EPOCH_TICKS {
0
} else {
@@ -623,6 +807,36 @@ const fn unix_seconds(ticks: i64) -> i64 {
}
}
+/// Bytes aligned to a page, for reads that bypass the file cache: those
+/// want memory aligned to the disk's sector, which a page always is.
+#[derive(Debug, Default)]
+pub struct Aligned {
+ pages: Vec<Page>,
+}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C, align(4096))]
+struct Page([u8; 4096]);
+
+impl Aligned {
+ /// `len` bytes, their contents whatever the last use left.
+ pub fn bytes(&mut self, len: usize) -> &mut [u8] {
+ let pages = len.div_ceil(size_of::<Page>());
+ if self.pages.len() < pages {
+ self.pages.resize(pages, Page([0; 4096]));
+ }
+ // SAFETY: `Page` is a `repr(C)` byte array, so the pages are one
+ // allocation of initialized bytes with no padding, and `len` fits
+ // in it; the borrow of `self` keeps it alive and unaliased.
+ unsafe {
+ std::slice::from_raw_parts_mut(
+ self.pages.as_mut_ptr().cast::<u8>(),
+ len,
+ )
+ }
+ }
+}
+
/// Make a directory junction for a test: unlike a symbolic link, one needs
/// no privilege to create. `false` when there is no `cmd` to make it with.
#[cfg(test)]
@@ -643,6 +857,17 @@ mod tests {
use super::*;
use tempfile::TempDir;
+ #[test]
+ fn quoted_arguments_survive_the_command_line() {
+ let quote =
+ |arg: &str| String::from_utf16(&quoted(OsStr::new(arg))).unwrap();
+ // A drive root's trailing backslash would otherwise escape the quote.
+ assert_eq!(quote(r"C:\"), r#""C:\\""#);
+ assert_eq!(quote(r"C:\Program Files\x"), r#""C:\Program Files\x""#);
+ assert_eq!(quote(r#"a\"b"#), r#""a\\\"b""#);
+ assert_eq!(quote("--metric"), r#""--metric""#);
+ }
+
#[test]
fn the_volume_list_has_the_system_drive() {
let drive =
@@ -657,7 +882,7 @@ mod tests {
}
fn listed(dir: &Path) -> Vec<Entry> {
- let mut entries: Vec<Entry> = read_dir(dir)
+ let mut entries: Vec<Entry> = read_dir(dir, None)
.expect("list")
.collect::<io::Result<_>>()
.expect("entries");
@@ -677,11 +902,40 @@ mod tests {
assert_eq!(names, ["data.bin", "sub"]);
assert_eq!(entries[0].kind(), Kind::File);
assert_eq!(entries[0].apparent(), 100_000);
- assert_eq!(entries[0].path(), temp.path().join("data.bin"));
+ assert_eq!(entries[0].path(temp.path()), temp.path().join("data.bin"));
assert_eq!(entries[1].kind(), Kind::Directory);
assert!(entries[0].modified() > 0, "written just now");
}
+ #[test]
+ fn a_name_that_is_not_text_is_shown_lossy_and_still_opens() {
+ use std::os::windows::ffi::OsStringExt as _;
+ let temp = TempDir::new().expect("tempdir");
+ // An unpaired surrogate: NTFS takes it, text cannot hold it.
+ let raw =
+ OsString::from_wide(&[u16::from(b'a'), 0xD800, u16::from(b'b')]);
+ fs::write(temp.path().join(&raw), b"x").expect("write");
+
+ let entries = listed(temp.path());
+ let [entry] = &entries[..] else {
+ panic!("one entry: {}", entries.len());
+ };
+ assert_eq!(entry.name(), "a\u{FFFD}b");
+ assert_eq!(entry.file_name(), raw);
+ assert!(fs::metadata(entry.path(temp.path())).is_ok());
+ }
+
+ #[test]
+ fn only_a_local_drive_gives_a_walk_one_volume() {
+ let temp = TempDir::new().expect("tempdir");
+ let canonical = temp.path().canonicalize().expect("canonical");
+ assert!(walk_volume(&canonical).is_some(), "{}", canonical.display());
+ // A share's DFS links lead to other volumes; nothing is asked.
+ for share in [r"\\server\share\dir", r"\\?\UNC\server\share\dir"] {
+ assert_eq!(walk_volume(Path::new(share)), None, "{share}");
+ }
+ }
+
#[test]
fn the_allocation_is_whole_clusters_of_the_data() {
let temp = TempDir::new().expect("tempdir");
@@ -746,11 +1000,15 @@ mod tests {
};
assert_eq!(kind("junction"), Some(Kind::Link));
assert_eq!(kind("target"), Some(Kind::Directory));
- let through: Vec<OsString> = listed(&junction)
+ let through: Vec<Box<str>> = listed(&junction)
.into_iter()
.map(|entry| entry.name)
.collect();
- assert_eq!(through, ["inside.bin"], "a walk that enters one follows");
+ assert_eq!(
+ &*through[0], "inside.bin",
+ "a walk that enters one follows"
+ );
+ assert_eq!(through.len(), 1);
}
#[test]
@@ -794,6 +1052,7 @@ mod tests {
drive.display()
);
assert!(!is_mount_point(temp.path()));
+ assert!(!file_table_readable(temp.path()), "a folder is walked");
assert!(!is_mount_point(drive), "a drive is a root, not a folder");
}