A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.
Rust
feat(scan): read the NTFS master file table when elevated
Shishir Pai0b1725d40e19
diff --git a/crates/disktree-core/Cargo.toml b/crates/disktree-core/Cargo.tomlindex a1eb131..84387db 100644--- a/crates/disktree-core/Cargo.toml+++ b/crates/disktree-core/Cargo.toml@@ -22,7 +22,13 @@ rustix = { version = "1.1", features = ["fs"] }# The Recycle Bin, through the shell's own file operation.trash = { version = "5.2.9", default-features = false, features = ["coinit_apartmentthreaded"] }winapi-util = "0.1.11"-windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }+windows-sys = { version = "0.61", features = [+ "Win32_Foundation",+ "Win32_Storage_FileSystem",+ "Win32_System_Com",+ "Win32_UI_Shell",+ "Win32_UI_WindowsAndMessaging",+] }# The system trash, through NSFileManager: what Finder's Move to Trash does,# without asking to control Finder. Only this backend is used; the crate'sdiff --git a/crates/disktree-core/src/access.rs b/crates/disktree-core/src/access.rsindex d19abc1..a33025d 100644--- a/crates/disktree-core/src/access.rs+++ b/crates/disktree-core/src/access.rs@@ -5,7 +5,14 @@//! with `EPERM`, silently. A disk tool that only counts those as errors//! leaves the user guessing, so the app asks once whether access is granted//! and, if not, says why folders were unreadable and where to fix it.+//!+//! On Windows the same question is whether disktree runs as an+//! administrator: only then can it read a whole drive from its file table,+//! several times faster than walking it and including folders a walk is+//! refused. Without, it walks.+use std::ffi::OsString;+use std::io;use std::path::Path;/// System Settings, opened at Privacy & Security › Full Disk Access.@@ -33,6 +40,47 @@ pub fn full_disk_access(home: &Path) -> Option<bool> {}}+/// Whether this process runs as an administrator: `None` off Windows,+/// where nothing a scan does depends on it.+#[cfg(windows)]+pub fn administrator() -> Option<bool> {+ Some(crate::windows::elevated())+}++/// Whether this process runs as an administrator: `None` off Windows,+/// where nothing a scan does depends on it.+#[cfg(not(windows))]+pub const fn administrator() -> Option<bool> {+ None+}++/// Whether an administrator would read `root` from its file table: a+/// whole NTFS drive. Always `false` off Windows.+#[cfg(windows)]+pub fn file_table_readable(root: &Path) -> bool {+ crate::windows::file_table_readable(root)+}++/// Whether an administrator would read `root` from its file table: a+/// whole NTFS drive. Always `false` off Windows.+#[cfg(not(windows))]+pub const fn file_table_readable(root: &Path) -> bool {+ let _ = root;+ false+}++/// Start this program again as an administrator, with `args`, through the+/// UAC prompt. An error if the prompt is declined, or off Windows.+pub fn restart_as_administrator(args: &[OsString]) -> io::Result<()> {+ #[cfg(windows)]+ return crate::windows::run_elevated(&std::env::current_exe()?, args);+ #[cfg(not(windows))]+ {+ let _ = args;+ Err(io::ErrorKind::Unsupported.into())+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rsindex 4d72191..9d2ee86 100644--- a/crates/disktree-core/src/lib.rs+++ b/crates/disktree-core/src/lib.rs@@ -10,6 +10,8 @@ pub mod classify;pub mod export;pub mod filter;pub mod insights;+#[cfg(windows)]+mod mft;pub mod removal;pub mod scan;pub mod size;diff --git a/crates/disktree-core/src/mft.rs b/crates/disktree-core/src/mft.rsnew file mode 100644index 0000000..54ea26e--- /dev/null+++ b/crates/disktree-core/src/mft.rs@@ -0,0 +1,1713 @@+//! Measuring an NTFS volume from its master file table, the way `WizTree`+//! does: one read of every file record instead of a listing per directory.+//! On a disk of four million files the walk opens six hundred thousand+//! directories; this reads five gigabytes in a few large requests.+//!+//! Only for a whole drive, `C:\`. The table costs the same to read whatever+//! part of it is wanted, so for a small directory the walk is faster.+//!+//! The table is read as it is on disk. NTFS writes changed records back+//! lazily, so changes other programs made in the last few seconds may not+//! show yet. disktree flushes the volume after its own removals, so a+//! rescan right after one sees what went. Flushing here instead would+//! need a write-capable volume open on every scan, which Controlled+//! Folder Access blocks and reports.+//!+//! Reading the volume needs an administrator. Anything this cannot do,+//! from a refused open to a record layout it does not expect, returns+//! `None` and the walk measures instead, so this is only ever a faster way+//! to the same tree.++use std::fs::{File, OpenOptions};+use std::io;+use std::os::windows::fs::{FileExt as _, OpenOptionsExt as _};+use std::path::Path;++use rayon::prelude::*;+use windows_sys::Win32::Storage::FileSystem::{+ FILE_ATTRIBUTE_HIDDEN, FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS,+ FILE_ATTRIBUTE_RECALL_ON_OPEN, FILE_ATTRIBUTE_REPARSE_POINT,+ FILE_FLAG_NO_BUFFERING, FILE_SHARE_DELETE, FILE_SHARE_READ,+ FILE_SHARE_WRITE,+};++use crate::scan::{ScanOptions, ScanProgress};+use crate::tree::{Node, NodeKind};+use crate::windows::{Aligned, drive_letter};++/// Most bytes read per call: large enough that the disk streams, small+/// enough that many reads are outstanding at once.+const CHUNK_BYTES: u64 = 16 << 20;++/// Free bytes a read spans rather than stop and start another. Through+/// `BitLocker`, 16 threads reading 1 MiB each moved 2 GB/s and 16 MiB+/// each 4.2 GB/s: a request costs about 0.29 ms on top of 0.23 ms per+/// MiB, as much as reading 1.3 MiB more, so a shorter free stretch is+/// cheaper read than skipped. On a file of synthetic records with free+/// stretches of about 600 records every 2000, reading through them took+/// 1028 ms against 1209 ms for stopping at 512 records; on a table+/// without free stretches the two plans are the same. An elevated scan of+/// a 4-million-record `C:\` took 3.44 s at 1 MiB against 3.60 s at 4 MiB+/// (median of 6 alternating runs).+const GAP_BYTES: u64 = 1 << 20;++/// The root directory's record number, fixed by the format.+const ROOT: u32 = 5;++/// Directory levels the table is followed down. A real tree is far+/// shallower; a corrupt one can loop, a directory beneath itself, and past+/// this the walk measures instead.+const MOST_LEVELS: usize = 1024;++/// Most bytes [`contents`] reads: the table's `$ATTRIBUTE_LIST` and+/// `$BITMAP` are kilobytes to a few megabytes, and a length beyond this is+/// a corrupt record, not something to allocate.+const MOST_CONTENTS: u64 = 64 << 20;++/// Records below this are the file system's own ($MFT, $Bitmap, ...): a+/// directory listing never shows them, so neither does this.+const FIRST_USER_RECORD: u32 = 16;++const IN_USE: u16 = 0x1;+const IS_DIRECTORY: u16 = 0x2;++const STANDARD_INFORMATION: u32 = 0x10;+const ATTRIBUTE_LIST: u32 = 0x20;+const FILE_NAME: u32 = 0x30;+const DATA: u32 = 0x80;+const BITMAP: u32 = 0xB0;+const REPARSE_POINT: u32 = 0xC0;+const END: u32 = 0xFFFF_FFFF;++/// The 8.3 alias of a name that has a long one: the same entry again.+const DOS_NAMESPACE: u8 = 2;++/// See `windows.rs`: links, junctions and mounted folders.+const NAME_SURROGATE: u32 = 0x2000_0000;+const EVICTED: u32 =+ FILE_ATTRIBUTE_RECALL_ON_OPEN | FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS;++/// What one base file record says.+#[derive(Clone, Copy, Default)]+struct Info {+ in_use: bool,+ directory: bool,+ /// Bumped each time the record is reused: a reference carrying another+ /// one names a file that is gone.+ sequence: u16,+ attributes: u32,+ reparse_tag: u32,+ modified: i64,+ apparent: u64,+ allocated: u64,+ names: u8,+}++/// One name of a file: an entry in its parent directory. The name itself+/// is `len` bytes at `at` in its chunk's text: millions of names in a few+/// strings, not millions of allocations made only to be copied again.+#[derive(Clone, Copy)]+struct Entry {+ parent: u32,+ parent_sequence: u16,+ child: u32,+ chunk: u32,+ at: u32,+ len: u16,+}++impl Entry {+ /// A placeholder a list is filled with before its entries are copied in.+ const EMPTY: Self = Self {+ parent: 0,+ parent_sequence: 0,+ child: 0,+ chunk: 0,+ at: 0,+ len: 0,+ };+}++/// What one chunk of records contributes besides its stretch of the+/// table.+#[derive(Default)]+struct Parsed {+ names: Vec<Entry>,+ text: String,+ /// Files among the records, and what their base records say they+ /// hold, for the progress meter: `settle` corrects both at the end.+ files: u64,+ apparent: u64,+ allocated: u64,+ /// What extension records hold for their base record: `(base, base+ /// sequence, apparent size, bytes allocated, reparse tag)`.+ extra: Vec<(u32, u16, Option<u64>, u64, u32)>,+ /// Names found in extension records, with their base's sequence: kept+ /// apart since they are rare and must be checked against the base.+ extra_names: Vec<(Entry, u16)>,+}++/// Where a stream lies on the volume: `(byte offset, byte length)` runs.+type Runs = Vec<(u64, u64)>;++struct Geometry {+ cluster: u64,+ record: usize,+ mft_offset: u64,+ /// The volume's size in bytes: no table can be larger.+ volume: u64,+}++/// The tree under `root`, read from the volume's file table; `None` when+/// the table cannot be read and the walk has to measure instead.+pub fn scan(+ root: &Path,+ canonical: &Path,+ options: &ScanOptions,+ progress: &ScanProgress,+) -> Option<io::Result<Node>> {+ // A followed link can lead off this volume, or back into it a second+ // time, and the table describes neither: the walk follows them.+ if options.follow_links {+ return None;+ }+ let path = format!(r"\\.\{}:", drive_letter(canonical)?);+ let volume = open_volume(&path).ok()?;+ let geometry = geometry(&volume)?;+ let (runs, bitmap) = table_layout(&volume, &geometry)?;+ let reads = plan_reads(+ &runs,+ &bitmap,+ &geometry,+ GAP_BYTES / geometry.record as u64,+ );+ let mut infos = table(&reads, geometry.record);+ let parsed = read_records(+ &path,+ geometry.record,+ &reads,+ &mut infos,+ options.apparent_size,+ progress,+ )?;+ if progress.is_cancelled() {+ return Some(Err(cancelled()));+ }++ let (mut names, texts) = merge(parsed, &mut infos);+ if progress.is_cancelled() {+ return Some(Err(cancelled()));+ }+ if !is_root_directory(&infos) {+ return None;+ }+ names.par_sort_unstable_by_key(|name| name.parent);+ let starts = starts(&names, infos.len());+ let table = Table {+ infos,+ names,+ texts,+ starts,+ options,+ progress,+ };+ let tree = table.directory(ROOT, crate::scan::file_name(root), 0);+ // Hundreds of megabytes, whose freeing the caller would otherwise+ // wait out before it can finish the tree.+ let Table {+ infos,+ names,+ texts,+ starts,+ ..+ } = table;+ // When no thread can be made, the closure is dropped here instead,+ // and they are freed in place.+ let _ = std::thread::Builder::new()+ .spawn(move || drop((infos, names, texts, starts)));+ match tree {+ Ok(_) | Err(Stop) if progress.is_cancelled() => Some(Err(cancelled())),+ Ok(node) => Some(Ok(node)),+ Err(Stop) => None,+ }+}++/// Whether the root's record reads as an in-use directory; if not, the+/// table is not what this expects and the walk measures instead.+fn is_root_directory(infos: &[Info]) -> bool {+ infos+ .get(ROOT as usize)+ .is_some_and(|root| root.in_use && root.directory)+}++pub fn cancelled() -> io::Error {+ io::Error::new(io::ErrorKind::Interrupted, "the scan was cancelled")+}++/// Past the file cache: the table is read once and never again, and+/// through the cache it is copied twice and evicts what the rest of the+/// system is using.+fn open_volume(path: &str) -> io::Result<File> {+ OpenOptions::new()+ .read(true)+ .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)+ .custom_flags(FILE_FLAG_NO_BUFFERING)+ .open(path)+}++/// Where each parent's entries start in `names`, sorted by parent:+/// `starts[p]..starts[p + 1]`.+fn starts(names: &[Entry], records: usize) -> Vec<u32> {+ let mut starts = vec![0_u32; records + 1];+ for name in names {+ if let Some(count) = starts.get_mut(name.parent as usize + 1) {+ *count += 1;+ }+ }+ for index in 1..starts.len() {+ starts[index] += starts[index - 1];+ }+ starts+}++fn geometry(volume: &File) -> Option<Geometry> {+ geometry_of(&read_at(volume, 0, 512)?)+}++/// The sizes the boot sector gives, each checked: a corrupt one would+/// otherwise overflow a shift or allocate without bound.+fn geometry_of(boot: &[u8]) -> Option<Geometry> {+ if boot.get(3..11)? != b"NTFS " {+ return None;+ }+ let sector = u64::from(u16_at(boot, 0x0B)?);+ let per_cluster = *boot.get(0x0D)?;+ // Above 0x80 the count is a negative power of two.+ let sectors = if per_cluster > 0x80 {+ 1_u64.checked_shl(256 - u32::from(per_cluster))?+ } else {+ u64::from(per_cluster)+ };+ let cluster = sector.checked_mul(sectors)?;+ let per_record = boot.get(0x40)?.cast_signed();+ let record = if per_record < 0 {+ 1_u64.checked_shl(u32::from(per_record.unsigned_abs()))?+ } else {+ cluster.checked_mul(u64::from(per_record.unsigned_abs()))?+ };+ let mft_offset = u64_at(boot, 0x30)?.checked_mul(cluster)?;+ let volume = u64_at(boot, 0x28)?.checked_mul(sector)?;+ let valid = sector.is_power_of_two()+ && (512..=4096).contains(§or)+ && cluster.is_power_of_two()+ && cluster <= 2 << 20+ && record.is_power_of_two()+ && (1024..=4096).contains(&record);+ valid.then_some(Geometry {+ cluster,+ record: usize::try_from(record).ok()?,+ mft_offset,+ volume,+ })+}++/// Where the file table lies on disk, as `(byte offset, byte length)` runs,+/// and its `$BITMAP`: which records are in use. A table grows and never+/// shrinks, so on a disk that has seen churn most of it can be free; only+/// the used part is read.+///+/// Both come from the table's own first record, or from the records its+/// `$ATTRIBUTE_LIST` names once it has outgrown that one.+fn table_layout(volume: &File, geometry: &Geometry) -> Option<(Runs, Vec<u8>)> {+ let mut records =+ vec![read_record_at(volume, geometry, geometry.mft_offset)?];+ if attributes(&records[0]).any(|attribute| attribute.kind == ATTRIBUTE_LIST)+ {+ let list = contents(volume, geometry, &records, ATTRIBUTE_LIST)?;+ let (_, first) = stream(&records, DATA, geometry.cluster)?;+ let mut numbers = Vec::new();+ let mut at = 0;+ while let (Some(length), Some(reference)) =+ (u16_at(&list, at + 4), u64_at(&list, at + 0x10))+ {+ let number = reference & 0xFFFF_FFFF_FFFF;+ if number != 0 && !numbers.contains(&number) {+ numbers.push(number);+ }+ if length == 0 {+ break;+ }+ at += usize::from(length);+ }+ // The records holding the rest of the table's attributes are+ // early in it, inside what the first record already maps.+ for number in numbers {+ let offset = record_offset(&first, geometry, number)?;+ records.push(read_record_at(volume, geometry, offset)?);+ }+ }++ let (data, runs) = stream(&records, DATA, geometry.cluster)?;+ let listed = runs+ .iter()+ .try_fold(0_u64, |sum, &(_, length)| sum.checked_add(length))?;+ let record_bytes = geometry.record as u64;+ // Record numbers are 32 bits here, and `plan_reads` counts records+ // across runs whose ends `decode_runs` has checked fit. A table larger+ // than its volume is corrupt, and `merge` would allocate a slot per+ // record it claims.+ if listed < u64_at(data.bytes, 0x28)?+ || listed > geometry.volume+ || listed / record_bytes > u64::from(u32::MAX)+ || runs.iter().any(|(_, length)| length % record_bytes != 0)+ {+ return None;+ }+ let bitmap = contents(volume, geometry, &records, BITMAP)?;+ Some((runs, bitmap))+}++/// The unnamed attribute `kind` across `records`: the piece that starts+/// it, which carries its sizes, and the runs of every piece in order.+fn stream(+ records: &[Vec<u8>],+ kind: u32,+ cluster: u64,+) -> Option<(Attribute<'_>, Runs)> {+ let mut pieces: Vec<Attribute<'_>> = records+ .iter()+ .flat_map(|record| attributes(record))+ .filter(|attribute| {+ attribute.kind == kind && attribute.name_length == 0+ })+ .collect();+ pieces.sort_by_key(|piece| {+ if piece.non_resident {+ u64_at(piece.bytes, 0x10).unwrap_or(u64::MAX)+ } else {+ 0+ }+ });+ let mut runs = Vec::new();+ for piece in pieces.iter().filter(|piece| piece.non_resident) {+ runs.extend(decode_runs(piece.bytes, cluster)?);+ }+ let first = pieces.into_iter().next()?;+ Some((first, runs))+}++/// All of the unnamed attribute `kind`'s value, wherever it is kept.+/// `None` when what the runs hold falls short of its length: for the+/// table's bitmap every record past the end would be skipped as free.+fn contents(+ volume: &File,+ geometry: &Geometry,+ records: &[Vec<u8>],+ kind: u32,+) -> Option<Vec<u8>> {+ let (first, runs) = stream(records, kind, geometry.cluster)?;+ if !first.non_resident {+ return first.value().map(<[u8]>::to_vec);+ }+ let length = u64_at(first.bytes, 0x30)?;+ let held = runs+ .iter()+ .try_fold(0_u64, |sum, &(_, size)| sum.checked_add(size))?;+ if length > MOST_CONTENTS || held > MOST_CONTENTS {+ return None;+ }+ let length = usize::try_from(length).ok()?;+ // The table's bitmap grows with it, a few clusters at a time: hundreds+ // of runs, which read one after another cost tens of milliseconds.+ let pieces: Vec<Vec<u8>> = runs+ .into_par_iter()+ .map(|(offset, size)| {+ read_at(volume, offset, usize::try_from(size).ok()?)+ })+ .collect::<Option<_>>()?;+ let mut bytes = pieces.concat();+ (bytes.len() >= length).then_some(())?;+ bytes.truncate(length);+ Some(bytes)+}++/// Where record `number` lies, given the runs that hold it.+fn record_offset(+ runs: &[(u64, u64)],+ geometry: &Geometry,+ number: u64,+) -> Option<u64> {+ let mut position = number.checked_mul(geometry.record as u64)?;+ for &(offset, length) in runs {+ if position < length {+ return offset.checked_add(position);+ }+ position -= length;+ }+ None+}++/// One record at `offset`, fixed up.+fn read_record_at(+ volume: &File,+ geometry: &Geometry,+ offset: u64,+) -> Option<Vec<u8>> {+ let mut record = read_at(volume, offset, geometry.record)?;+ fixup(&mut record)?;+ Some(record)+}++/// The reads that cover every used record: `(byte offset, byte length,+/// first record number)`. A free stretch of fewer than `gap_records` is+/// read through, since one longer read costs less than two (see+/// [`GAP_BYTES`]). Reads start and end on cluster boundaries, which a+/// volume handle requires of them.+fn plan_reads(+ runs: &[(u64, u64)],+ bitmap: &[u8],+ geometry: &Geometry,+ gap_records: u64,+) -> Vec<(u64, u64, u64)> {+ let used = |number: u64| {+ bitmap+ .get((number / 8) as usize)+ .is_some_and(|byte| byte & (1 << (number % 8)) != 0)+ };+ let record = geometry.record as u64;+ let align = (geometry.cluster / record).max(1);+ let most = CHUNK_BYTES / record;+ let mut reads = Vec::new();+ let mut first = 0_u64;+ for &(offset, length) in runs {+ let count = length / record;+ let mut at = 0;+ while at < count {+ if !used(first + at) {+ at += 1;+ continue;+ }+ let start = at - at % align;+ let mut end = at + 1;+ let mut probe = end;+ while probe < count+ && probe - start < most+ && probe - end < gap_records+ {+ if used(first + probe) {+ end = probe + 1;+ }+ probe += 1;+ }+ let end = end.next_multiple_of(align).min(count);+ reads.push((+ offset + start * record,+ (end - start) * record,+ first + start,+ ));+ at = end;+ }+ first += count;+ }+ reads+}++/// The runs of a non-resident attribute. Each run's end fits in a `u64`,+/// which `plan_reads` relies on.+fn decode_runs(attribute: &[u8], cluster: u64) -> Option<Runs> {+ let mut at = usize::from(u16_at(attribute, 0x20)?);+ let mut lcn = 0_i64;+ let mut runs = Vec::new();+ loop {+ let header = *attribute.get(at)?;+ if header == 0 {+ return Some(runs);+ }+ let length_size = usize::from(header & 0xF);+ let offset_size = usize::from(header >> 4);+ let length = unsigned(attribute.get(at + 1..at + 1 + length_size)?);+ let delta = attribute+ .get(at + 1 + length_size..at + 1 + length_size + offset_size)?;+ at += 1 + length_size + offset_size;+ // No offset marks a sparse run, which a file table never has.+ if offset_size == 0 || offset_size > 8 || length_size > 8 {+ return None;+ }+ lcn = lcn.checked_add(signed(delta))?;+ let offset = u64::try_from(lcn).ok()?.checked_mul(cluster)?;+ let length = length.checked_mul(cluster)?;+ offset.checked_add(length)?;+ runs.push((offset, length));+ }+}++/// A slot per record up to the last one read, each empty until a read+/// fills it: records no read covers are free.+fn table(reads: &[(u64, u64, u64)], record: usize) -> Vec<Info> {+ let len = reads.last().map_or(0, |&(_, size, first)| {+ usize::try_from(first + size / record as u64).unwrap_or(0)+ });+ let mut infos = Vec::new();+ (0..len)+ .into_par_iter()+ .map(|_| Info::default())+ .collect_into_vec(&mut infos);+ infos+}++/// Read and parse every used record, each into its slot of `infos`.+/// Reads run in parallel: one at a time leaves a solid-state disk idle+/// between requests. One list of reads per thread, each with its own+/// volume handle and buffer: requests on one handle opened without+/// overlapped I/O run one at a time, and rayon's `map_init` would open a+/// handle and zero a buffer per split, hundreds of times. Read `i` goes to+/// list `i % threads`, so the lists move through the disk together.+/// Results come back in read order, which `merge` relies on.+///+/// Records land in `infos` as they are parsed, while other reads are+/// still on the disk: gathered into it after the last read, they cost a+/// tenth of a second on the way to the tree.+fn read_records(+ path: &str,+ record: usize,+ reads: &[(u64, u64, u64)],+ infos: &mut [Info],+ apparent_size: bool,+ progress: &ScanProgress,+) -> Option<Vec<Parsed>> {+ let threads = rayon::current_num_threads().max(1);+ let mut stretches: Vec<Vec<&mut [Info]>> =+ (0..threads).map(|_| Vec::new()).collect();+ // `plan_reads` gives them in record order, none overlapping.+ let mut rest = infos;+ let mut done = 0;+ for (index, &(_, size, first)) in reads.iter().enumerate() {+ let skip = usize::try_from(first).ok()?.checked_sub(done)?;+ let count = usize::try_from(size).ok()? / record;+ let (stretch, tail) =+ rest.get_mut(skip..)?.split_at_mut_checked(count)?;+ stretches[index % threads].push(stretch);+ rest = tail;+ done += skip + count;+ }+ let lists = stretches+ .into_par_iter()+ .enumerate()+ .map(|(list, stretches)| {+ let volume = open_volume(path).ok()?;+ let mut buffer = Aligned::default();+ let mut parsed = Vec::new();+ for ((chunk, &(offset, size, first)), stretch) in reads+ .iter()+ .enumerate()+ .skip(list)+ .step_by(threads)+ .zip(stretches)+ {+ if progress.is_cancelled() {+ break;+ }+ let buffer = buffer.bytes(usize::try_from(size).ok()?);+ volume.seek_read_exact(buffer, offset).ok()?;+ let chunk = parse_chunk(+ buffer,+ record,+ first,+ u32::try_from(chunk).ok()?,+ stretch,+ );+ let bytes = if apparent_size {+ chunk.apparent+ } else {+ chunk.allocated+ };+ progress.add(chunk.files, 0, bytes);+ parsed.push(chunk);+ }+ Some(parsed)+ })+ .collect::<Option<Vec<_>>>()?;+ // Interleave back: read `i` is item `i / threads` of list `i % threads`.+ let mut lists: Vec<_> = lists.into_iter().map(Vec::into_iter).collect();+ let mut parsed = Vec::with_capacity(reads.len());+ for index in 0..reads.len() {+ // Short only when cancelled, which the caller checks next.+ let Some(chunk) = lists[index % threads].next() else {+ break;+ };+ parsed.push(chunk);+ }+ Some(parsed)+}++/// `len` bytes at `offset`. Read as whole, aligned 4 KiB blocks: past the+/// cache a volume only reads whole sectors into sector-aligned memory, and+/// no disk has sectors larger.+fn read_at(volume: &File, offset: u64, len: usize) -> Option<Vec<u8>> {+ const BLOCK: u64 = 4096;+ let start = offset - offset % BLOCK;+ let end = offset+ .checked_add(len as u64)?+ .checked_next_multiple_of(BLOCK)?;+ let mut buffer = Aligned::default();+ let block = buffer.bytes(usize::try_from(end - start).ok()?);+ volume.seek_read_exact(block, start).ok()?;+ let at = usize::try_from(offset - start).ok()?;+ block.get(at..at + len).map(<[u8]>::to_vec)+}++trait ReadExact {+ fn seek_read_exact(&self, buffer: &mut [u8], offset: u64)+ -> io::Result<()>;+}++impl ReadExact for File {+ fn seek_read_exact(+ &self,+ mut buffer: &mut [u8],+ mut offset: u64,+ ) -> io::Result<()> {+ while !buffer.is_empty() {+ match self.seek_read(buffer, offset)? {+ 0 => return Err(io::ErrorKind::UnexpectedEof.into()),+ read => {+ buffer = &mut buffer[read..];+ offset += read as u64;+ }+ }+ }+ Ok(())+ }+}++fn parse_chunk(+ buffer: &mut [u8],+ record: usize,+ first: u64,+ chunk: u32,+ infos: &mut [Info],+) -> Parsed {+ // About a name per record, and a name about 20 bytes: grown by+ // doubling instead, the lists are copied a dozen times per read.+ let records = buffer.len() / record;+ let mut out = Parsed {+ names: Vec::with_capacity(records),+ text: String::with_capacity(records * 24),+ ..Parsed::default()+ };+ for ((index, bytes), slot) in+ buffer.chunks_exact_mut(record).enumerate().zip(infos)+ {+ let Ok(number) = u32::try_from(first + index as u64) else {+ break;+ };+ parse_record(bytes, number, chunk, &mut out, slot);+ }+ out+}++/// A base record's facts go to `slot`; an extension record's to `out`.+fn parse_record(+ bytes: &mut [u8],+ number: u32,+ chunk: u32,+ out: &mut Parsed,+ slot: &mut Info,+) {+ if bytes.get(..4) != Some(b"FILE") || fixup(bytes).is_none() {+ return;+ }+ let Some(flags) = u16_at(bytes, 0x16) else {+ return;+ };+ if flags & IN_USE == 0 {+ return;+ }+ // A record holding attributes that did not fit in its base record.+ let base = u64_at(bytes, 0x20).unwrap_or(0);+ let (owner, base_sequence) = if base & REFERENCE == 0 {+ (number, 0)+ } else {+ let Some(reference) = reference(base) else {+ return;+ };+ reference+ };+ let mut info = Info {+ in_use: true,+ directory: flags & IS_DIRECTORY != 0,+ sequence: u16_at(bytes, 0x10).unwrap_or(0),+ ..Info::default()+ };+ let mut apparent = None;+ let mut allocated = 0_u64;+ // A reparse point's tag is also kept beside each name, which serves+ // when the `$REPARSE_POINT` value is not in the record itself.+ let mut name_tag = 0;+ for attribute in attributes(bytes) {+ match attribute.kind {+ STANDARD_INFORMATION => {+ if let Some(value) = attribute.value() {+ info.modified = u64_at(value, 0x08).map_or(0, |ticks| {+ crate::windows::unix_seconds(ticks.cast_signed())+ });+ info.attributes = u32_at(value, 0x20).unwrap_or(0);+ }+ }+ FILE_NAME => {+ let Some(value) = attribute.value() else {+ continue;+ };+ if u32_at(value, 0x38).unwrap_or(0)+ & FILE_ATTRIBUTE_REPARSE_POINT+ != 0+ {+ name_tag = u32_at(value, 0x3C).unwrap_or(0);+ }+ let at = out.text.len();+ if let Some((parent, parent_sequence)) =+ file_name(value, &mut out.text)+ {+ let entry = Entry {+ parent,+ parent_sequence,+ child: owner,+ chunk,+ at: u32::try_from(at).unwrap_or(u32::MAX),+ len: u16::try_from(out.text.len() - at)+ .unwrap_or(u16::MAX),+ };+ if base & REFERENCE == 0 {+ out.names.push(entry);+ info.names = info.names.saturating_add(1);+ } else {+ out.extra_names.push((entry, base_sequence));+ }+ }+ }+ DATA => {+ // Every stream's clusters, named ones too: they go when the+ // file does. A file Windows compressed (`compact /exe`)+ // keeps its data in one and leaves the unnamed one empty.+ let (length, spent) = attribute.sizes();+ allocated = allocated.saturating_add(spent);+ // Only the first piece of the unnamed stream has its length.+ if attribute.name_length == 0 && attribute.starts_file() {+ apparent = Some(length);+ }+ }+ REPARSE_POINT => {+ if let Some(value) = attribute.value() {+ info.reparse_tag = u32_at(value, 0).unwrap_or(0);+ }+ }+ _ => {}+ }+ }+ if info.reparse_tag == 0 {+ info.reparse_tag = name_tag;+ }+ if base & REFERENCE == 0 {+ if !info.directory {+ out.files += 1;+ out.apparent = out.apparent.saturating_add(apparent.unwrap_or(0));+ out.allocated = out.allocated.saturating_add(allocated);+ }+ info.apparent = apparent.unwrap_or(0);+ info.allocated = allocated;+ *slot = info;+ } else if apparent.is_some() || allocated != 0 || info.reparse_tag != 0 {+ out.extra.push((+ owner,+ base_sequence,+ apparent,+ allocated,+ info.reparse_tag,+ ));+ }+}++/// The record number part of a file reference.+const REFERENCE: u64 = 0xFFFF_FFFF_FFFF;++/// A file reference: `(record number, sequence number)`.+fn reference(value: u64) -> Option<(u32, u16)> {+ let number = u32::try_from(value & REFERENCE).ok()?;+ let sequence = u16::try_from(value >> 48).ok()?;+ Some((number, sequence))+}++/// The parent reference of a `$FILE_NAME`, its name appended to `text`;+/// or `None`, and nothing appended, for an 8.3 alias or a name no+/// directory can hold, which as a path would name another file.+fn file_name(value: &[u8], text: &mut String) -> Option<(u32, u16)> {+ let parent = reference(u64_at(value, 0)?)?;+ let length = usize::from(*value.get(0x40)?);+ if *value.get(0x41)? == DOS_NAMESPACE {+ return None;+ }+ let units = value+ .get(0x42..0x42 + length * 2)?+ .chunks_exact(2)+ .map(|pair| u16::from_le_bytes([pair[0], pair[1]]));+ let start = text.len();+ text.extend(+ char::decode_utf16(units)+ .map(|unit| unit.unwrap_or(char::REPLACEMENT_CHARACTER)),+ );+ let name = &text[start..];+ if name.is_empty()+ || name == "."+ || name == ".."+ || name.contains(['\\', '/', '\0'])+ {+ text.truncate(start);+ return None;+ }+ Some(parent)+}++/// Undo the update sequence: the last two bytes of every sector were+/// swapped for a check value when the record was written.+fn fixup(bytes: &mut [u8]) -> Option<()> {+ // The update sequence always strides 512 bytes, whatever the sector.+ const STRIDE: usize = 512;+ let at = usize::from(u16_at(bytes, 4)?);+ let count = usize::from(u16_at(bytes, 6)?);+ if bytes.is_empty()+ || !bytes.len().is_multiple_of(STRIDE)+ || count != bytes.len() / STRIDE + 1+ || at + 2 * count > bytes.len()+ {+ return None;+ }+ let check = [bytes[at], bytes[at + 1]];+ for sector in 1..count {+ let end = sector * STRIDE;+ let replacement = [bytes[at + 2 * sector], bytes[at + 2 * sector + 1]];+ let tail = &mut bytes[end - 2..end];+ if tail != check {+ return None;+ }+ tail.copy_from_slice(&replacement);+ }+ Some(())+}++struct Attribute<'a> {+ kind: u32,+ non_resident: bool,+ name_length: u8,+ flags: u16,+ bytes: &'a [u8],+}++impl Attribute<'_> {+ fn value(&self) -> Option<&[u8]> {+ if self.non_resident {+ return None;+ }+ let length = u32_at(self.bytes, 0x10)? as usize;+ let at = usize::from(u16_at(self.bytes, 0x14)?);+ self.bytes.get(at..at + length)+ }++ fn starts_file(&self) -> bool {+ !self.non_resident || u64_at(self.bytes, 0x10) == Some(0)+ }++ /// `(length, bytes allocated)`. Data kept in the record costs nothing+ /// beyond the record; a compressed or sparse stream's real cost is its+ /// total allocation, which only the first piece carries.+ fn sizes(&self) -> (u64, u64) {+ if !self.non_resident {+ return (self.value().map_or(0, |value| value.len() as u64), 0);+ }+ if !self.starts_file() {+ return (0, 0);+ }+ let length = u64_at(self.bytes, 0x30).unwrap_or(0);+ let packed = self.flags & 0x8001 != 0;+ let allocated =+ u64_at(self.bytes, if packed { 0x40 } else { 0x28 }).unwrap_or(0);+ (length, allocated)+ }+}++fn attributes(record: &[u8]) -> impl Iterator<Item = Attribute<'_>> {+ let mut at = u16_at(record, 0x14).map_or(record.len(), usize::from);+ std::iter::from_fn(move || {+ let kind = u32_at(record, at)?;+ if kind == END {+ return None;+ }+ let length = u32_at(record, at + 4)? as usize;+ let bytes = record.get(at..at.checked_add(length)?)?;+ if length < 0x18 {+ return None;+ }+ at += length;+ Some(Attribute {+ kind,+ non_resident: bytes[8] != 0,+ name_length: bytes[9],+ flags: u16_at(bytes, 0x0C)?,+ bytes,+ })+ })+}++/// Every chunk's names in one list, and what extension records add to+/// their base records' slots in `infos`.+fn merge(parsed: Vec<Parsed>, infos: &mut [Info]) -> (Vec<Entry>, Vec<String>) {+ // Copied in parallel, a chunk per task, into a list filled first: one+ // thread appending millions of names made every other one wait. Room+ // for the names from extension records too, pushed after: any system+ // drive has some, and the first would copy the whole list to grow it.+ let total = parsed.iter().map(|chunk| chunk.names.len()).sum();+ let extra_total: usize =+ parsed.iter().map(|chunk| chunk.extra_names.len()).sum();+ let mut names = Vec::with_capacity(total + extra_total);+ names.par_extend(rayon::iter::repeat_n(Entry::EMPTY, total));+ let mut pieces = Vec::with_capacity(parsed.len());+ let mut rest = names.as_mut_slice();+ for chunk in &parsed {+ let (piece, tail) = rest.split_at_mut(chunk.names.len());+ pieces.push((piece, chunk.names.as_slice()));+ rest = tail;+ }+ pieces+ .into_par_iter()+ .for_each(|(piece, chunk)| piece.copy_from_slice(chunk));+ let mut extra = Vec::new();+ let mut extra_names = Vec::new();+ let mut texts = Vec::with_capacity(parsed.len());+ for chunk in parsed {+ extra.extend(chunk.extra);+ extra_names.extend(chunk.extra_names);+ texts.push(chunk.text);+ }+ // A name in an extension record left behind by a file whose record+ // has since been reused would list that file under the new one's+ // number.+ for (name, sequence) in extra_names {+ let Some(info) = infos.get_mut(name.child as usize) else {+ continue;+ };+ if info.in_use && info.sequence == sequence {+ info.names = info.names.saturating_add(1);+ names.push(name);+ }+ }+ for (number, sequence, apparent, allocated, tag) in extra {+ let Some(info) = infos.get_mut(number as usize) else {+ continue;+ };+ // An extension record left behind by a file whose record has+ // since been reused.+ if !info.in_use || info.sequence != sequence {+ continue;+ }+ info.allocated = info.allocated.saturating_add(allocated);+ if let Some(apparent) = apparent {+ info.apparent = apparent;+ }+ if info.reparse_tag == 0 {+ info.reparse_tag = tag;+ }+ }+ (names, texts)+}++/// Why [`Table::directory`] gave up: the scan was cancelled, or the tree+/// runs deeper than [`MOST_LEVELS`].+struct Stop;++struct Table<'a> {+ infos: Vec<Info>,+ /// Sorted by parent; `starts[p]..starts[p + 1]` are `p`'s entries.+ names: Vec<Entry>,+ texts: Vec<String>,+ starts: Vec<u32>,+ options: &'a ScanOptions,+ progress: &'a ScanProgress,+}++impl Table<'_> {+ fn name(&self, entry: &Entry) -> &str {+ let at = entry.at as usize;+ self.texts+ .get(entry.chunk as usize)+ .and_then(|text| text.get(at..at + usize::from(entry.len)))+ .unwrap_or_default()+ }++ fn entries(&self, parent: u32) -> &[Entry] {+ let index = parent as usize;+ match (self.starts.get(index), self.starts.get(index + 1)) {+ (Some(&start), Some(&end)) => {+ &self.names[start as usize..end as usize]+ }+ _ => &[],+ }+ }++ /// Mirrors what the walk keeps: see `WalkContext::classify`.+ fn directory(+ &self,+ number: u32,+ name: Box<str>,+ depth: usize,+ ) -> Result<Node, Stop> {+ if depth >= MOST_LEVELS || self.progress.is_cancelled() {+ return Err(Stop);+ }+ let descend = self.options.max_depth.is_none_or(|max| depth < max);+ let sequence =+ self.infos.get(number as usize).map(|info| info.sequence);+ let child = |entry: &Entry| {+ // A root directory holds hundreds of thousands of files; a+ // check per entry lets a cancel stop within one.+ if self.progress.is_cancelled() {+ return Some(Err(Stop));+ }+ // An entry naming a parent whose record has since been reused.+ if Some(entry.parent_sequence) != sequence {+ return None;+ }+ self.child(entry, depth, descend).transpose()+ };+ let entries = self.entries(number);+ // See `tree::PARALLEL_LEVELS`: parallel only near the top.+ let children = if depth < crate::tree::PARALLEL_LEVELS {+ entries+ .par_iter()+ .filter_map(child)+ .collect::<Result<_, _>>()?+ } else {+ // Sized up front: nearly every entry becomes a child, and a+ // list grown by doubling copies its nodes over and over.+ let mut children = Vec::with_capacity(entries.len());+ for node in entries.iter().filter_map(child) {+ children.push(node?);+ }+ children+ };+ let mut node = Node::directory(name);+ node.children = children;+ Ok(node)+ }++ /// The node for `entry`; `None` for one the walk would not list.+ fn child(+ &self,+ entry: &Entry,+ depth: usize,+ descend: bool,+ ) -> Result<Option<Node>, Stop> {+ // The root is its own parent.+ if entry.child == entry.parent || entry.child < FIRST_USER_RECORD {+ return Ok(None);+ }+ let Some(info) = self.infos.get(entry.child as usize) else {+ return Ok(None);+ };+ if !info.in_use {+ return Ok(None);+ }+ let name = self.name(entry);+ if !self.options.include_hidden+ && (name.starts_with('.')+ || info.attributes & FILE_ATTRIBUTE_HIDDEN != 0)+ {+ return Ok(None);+ }+ let link = info.attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0+ && info.reparse_tag & NAME_SURROGATE != 0;+ if info.directory && !link {+ if info.attributes & EVICTED != 0 || !descend {+ return Ok(None);+ }+ return self+ .directory(entry.child, name.into(), depth + 1)+ .map(Some);+ }+ let size = if self.options.apparent_size {+ info.apparent+ } else {+ info.allocated+ };+ let kind = if link {+ NodeKind::Symlink+ } else {+ NodeKind::File+ };+ let mut node = Node::entry(name, kind, size);+ node.modified = info.modified;+ if info.names > 1 {+ node.inode = Some((0, u64::from(entry.child)));+ }+ Ok(Some(node))+ }+}++fn u16_at(bytes: &[u8], at: usize) -> Option<u16> {+ Some(u16::from_le_bytes(bytes.get(at..at + 2)?.try_into().ok()?))+}++fn u32_at(bytes: &[u8], at: usize) -> Option<u32> {+ Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?))+}++fn u64_at(bytes: &[u8], at: usize) -> Option<u64> {+ Some(u64::from_le_bytes(bytes.get(at..at + 8)?.try_into().ok()?))+}++fn unsigned(bytes: &[u8]) -> u64 {+ bytes+ .iter()+ .rev()+ .fold(0, |value, &byte| (value << 8) | u64::from(byte))+}++/// A little-endian two's complement number of any width up to eight bytes.+fn signed(bytes: &[u8]) -> i64 {+ let value = unsigned(bytes);+ let bits = bytes.len() * 8;+ if bits == 0 || bits >= 64 {+ return value.cast_signed();+ }+ let shift = 64 - bits;+ (value << shift).cast_signed() >> shift+}++#[cfg(test)]+mod tests {+ use super::*;++ const RECORD: usize = 1024;+ const USA_AT: usize = 0x30;+ const CHECK: [u8; 2] = [0xAB, 0xCD];++ fn resident(kind: u32, value: &[u8]) -> Vec<u8> {+ let mut bytes = vec![0; 0x18];+ bytes[0..4].copy_from_slice(&kind.to_le_bytes());+ bytes[0x10..0x14].copy_from_slice(&(value.len() as u32).to_le_bytes());+ bytes[0x14..0x16].copy_from_slice(&0x18_u16.to_le_bytes());+ bytes.extend_from_slice(value);+ bytes.resize(bytes.len().next_multiple_of(8), 0);+ let length = bytes.len() as u32;+ bytes[4..8].copy_from_slice(&length.to_le_bytes());+ bytes+ }++ fn non_resident(kind: u32, allocated: u64, length: u64) -> Vec<u8> {+ let mut bytes = vec![0; 0x48];+ bytes[0..4].copy_from_slice(&kind.to_le_bytes());+ bytes[4..8].copy_from_slice(&0x48_u32.to_le_bytes());+ bytes[8] = 1;+ bytes[0x20..0x22].copy_from_slice(&0x40_u16.to_le_bytes());+ bytes[0x28..0x30].copy_from_slice(&allocated.to_le_bytes());+ bytes[0x30..0x38].copy_from_slice(&length.to_le_bytes());+ bytes+ }++ fn name(parent: u32, text: &str, namespace: u8) -> Vec<u8> {+ let mut value = vec![0; 0x42];+ value[0..8].copy_from_slice(&u64::from(parent).to_le_bytes());+ let units: Vec<u16> = text.encode_utf16().collect();+ value[0x40] = units.len() as u8;+ value[0x41] = namespace;+ for unit in units {+ value.extend_from_slice(&unit.to_le_bytes());+ }+ resident(FILE_NAME, &value)+ }++ fn standard(attributes: u32) -> Vec<u8> {+ let mut value = vec![0; 0x48];+ value[0x20..0x24].copy_from_slice(&attributes.to_le_bytes());+ resident(STANDARD_INFORMATION, &value)+ }++ /// A record as the disk holds it: sector tails swapped for the check+ /// value, their real bytes kept in the update sequence array.+ fn record(flags: u16, base: u64, attributes: &[Vec<u8>]) -> Vec<u8> {+ let mut bytes = vec![0; RECORD];+ bytes[0..4].copy_from_slice(b"FILE");+ bytes[4..6].copy_from_slice(&(USA_AT as u16).to_le_bytes());+ bytes[6..8].copy_from_slice(&3_u16.to_le_bytes());+ bytes[0x14..0x16].copy_from_slice(&0x38_u16.to_le_bytes());+ bytes[0x16..0x18].copy_from_slice(&flags.to_le_bytes());+ bytes[0x20..0x28].copy_from_slice(&base.to_le_bytes());+ let mut at = 0x38;+ for attribute in attributes {+ bytes[at..at + attribute.len()].copy_from_slice(attribute);+ at += attribute.len();+ }+ bytes[at..at + 4].copy_from_slice(&END.to_le_bytes());+ bytes[USA_AT..USA_AT + 2].copy_from_slice(&CHECK);+ for sector in 1..3 {+ let end = sector * 512;+ let (tail, usa) = (end - 2, USA_AT + 2 * sector);+ bytes.copy_within(tail..end, usa);+ bytes[tail..end].copy_from_slice(&CHECK);+ }+ bytes+ }++ fn parse(bytes: &mut [u8], number: u32) -> (Parsed, Info) {+ let mut out = Parsed::default();+ let mut info = Info::default();+ parse_record(bytes, number, 0, &mut out, &mut info);+ (out, info)+ }++ #[test]+ fn a_record_gives_its_name_size_and_attributes() {+ let mut bytes = record(+ IN_USE,+ 0,+ &[+ standard(FILE_ATTRIBUTE_HIDDEN),+ name(40, "REPORT~1.TXT", DOS_NAMESPACE),+ name(40, "report for the board.txt", 1),+ non_resident(DATA, 8192, 5000),+ ],+ );+ let (out, info) = parse(&mut bytes, 77);++ assert!(info.in_use);+ assert!(!info.directory);+ assert_eq!(info.attributes, FILE_ATTRIBUTE_HIDDEN);+ assert_eq!((info.apparent, info.allocated), (5000, 8192));+ // The 8.3 alias is the same entry again, not a second name.+ let [entry] = &out.names[..] else {+ panic!("one name: {}", out.names.len());+ };+ assert_eq!((entry.parent, entry.child), (40, 77));+ let at = entry.at as usize;+ assert_eq!(+ &out.text[at..at + usize::from(entry.len)],+ "report for the board.txt"+ );+ assert_eq!(out.files, 1);+ }++ #[test]+ fn a_free_or_torn_record_contributes_nothing() {+ let mut free = record(0, 0, &[name(5, "gone", 1)]);+ assert!(parse(&mut free, 20).0.names.is_empty());++ // A write that reached only one sector: its tail no longer holds+ // the check value, and the record cannot be trusted.+ let mut torn = record(IN_USE, 0, &[name(5, "half", 1)]);+ torn[1022] ^= 0xFF;+ let (out, info) = parse(&mut torn, 21);+ assert!(!info.in_use && out.names.is_empty());+ }++ #[test]+ fn a_name_left_in_a_reused_records_extension_is_dropped() {+ // Record 0x5A is now a file at sequence 8. Record 300 is an+ // extension left over from sequence 7; record 301 is current.+ let mut base = record(IN_USE, 0, &[name(5, "current", 1)]);+ base[0x10..0x12].copy_from_slice(&8_u16.to_le_bytes());+ let mut stale = record(IN_USE, 0x5A | (7 << 48), &[name(5, "old", 1)]);+ let mut fresh =+ record(IN_USE, 0x5A | (8 << 48), &[name(5, "second", 1)]);+ let mut out = Parsed::default();+ let mut infos = vec![Info::default(); 400];+ parse_record(&mut base, 0x5A, 0, &mut out, &mut infos[0x5A]);+ parse_record(&mut stale, 300, 0, &mut out, &mut infos[300]);+ parse_record(&mut fresh, 301, 0, &mut out, &mut infos[301]);++ let (names, texts) = merge(vec![out], &mut infos);+ let mut listed: Vec<&str> = names+ .iter()+ .map(|entry| {+ let at = entry.at as usize;+ &texts[0][at..at + usize::from(entry.len)]+ })+ .collect();+ listed.sort_unstable();+ assert_eq!(listed, ["current", "second"]);+ // Nor is the stale name counted as a link.+ assert_eq!(infos[0x5A].names, 2);+ }++ #[test]+ fn a_root_that_is_not_an_in_use_directory_is_refused() {+ let mut infos = vec![Info::default(); 8];+ assert!(!is_root_directory(&infos));+ infos[ROOT as usize] = Info {+ in_use: true,+ ..Info::default()+ };+ assert!(!is_root_directory(&infos));+ infos[ROOT as usize].directory = true;+ assert!(is_root_directory(&infos));+ assert!(!is_root_directory(&infos[..3]));+ }++ #[test]+ fn an_extension_record_adds_to_its_base_record() {+ let mut bytes =+ record(IN_USE, 0x5A | (7 << 48), &[non_resident(DATA, 4096, 100)]);+ let (out, info) = parse(&mut bytes, 300);+ assert!(!info.in_use);+ assert_eq!(out.extra, [(0x5A, 7, Some(100), 4096, 0)]);+ }++ #[test]+ fn runs_are_relative_and_can_go_backwards() {+ let mut attribute = vec![0; 0x40];+ attribute[0x20..0x22].copy_from_slice(&0x40_u16.to_le_bytes());+ // 16 clusters at 256, then 8 clusters 128 back, at 128.+ attribute.extend_from_slice(&[0x21, 0x10, 0x00, 0x01]);+ attribute.extend_from_slice(&[0x21, 0x08, 0x80, 0xFF, 0x00]);+ assert_eq!(+ decode_runs(&attribute, 4096),+ Some(vec![(256 * 4096, 16 * 4096), (128 * 4096, 8 * 4096)])+ );+ }++ #[test]+ fn reads_cover_used_records_on_cluster_boundaries() {+ let geometry = Geometry {+ cluster: 4096,+ record: RECORD,+ mft_offset: 0,+ volume: 1 << 30,+ };+ let record = RECORD as u64;+ let gap = GAP_BYTES / record;+ let count = 4 * gap;+ let runs = [(1 << 20, count * record)];+ let mut bitmap = vec![0_u8; count as usize / 8];+ let mut used = |number: u64| {+ bitmap[number as usize / 8] |= 1 << (number % 8);+ };+ // One free record short of the gap is read through; the gap+ // itself splits the reads.+ let through = 10 + gap;+ let split = through + 1 + gap;+ for number in [5, 10, through, split] {+ used(number);+ }++ let reads = plan_reads(&runs, &bitmap, &geometry, gap);++ let end = (through + 1).next_multiple_of(4);+ let start = split - split % 4;+ assert_eq!(+ reads,+ [+ ((1 << 20) + 4 * record, (end - 4) * record, 4),+ ((1 << 20) + start * record, 4 * record, start),+ ]+ );+ }++ #[test]+ fn each_read_parses_into_the_slots_of_its_own_records() {+ // Records 0..96, each named for its number and sized 100 more,+ // in an ordinary file: reading it is reading a table, but for the+ // volume. Reads of 4 records, every third one skipped, handed out+ // to 3 threads, so no thread's reads are adjacent.+ let dir = tempfile::tempdir().expect("temp dir");+ let path = dir.path().join("records");+ let mut bytes = Vec::new();+ for number in 0..96_u64 {+ bytes.extend(record(+ IN_USE,+ 0,+ &[+ name(ROOT, &format!("f{number}"), 1),+ non_resident(DATA, 4096, number + 100),+ ],+ ));+ }+ std::fs::write(&path, bytes).expect("records written");+ let reads: Vec<_> = (0..24_u64)+ .filter(|read| read % 3 != 1)+ .map(|read| (read * 4096, 4096, read * 4))+ .collect();+ let mut infos = super::table(&reads, RECORD);+ let progress = ScanProgress::default();+ let pool = rayon::ThreadPoolBuilder::new()+ .num_threads(3)+ .build()+ .expect("a pool");+ let parsed = pool+ .install(|| {+ read_records(+ path.to_str().expect("a UTF-8 path"),+ RECORD,+ &reads,+ &mut infos,+ false,+ &progress,+ )+ })+ .expect("the file reads");+ let (names, texts) = merge(parsed, &mut infos);++ for (number, info) in infos.iter().enumerate() {+ let read = number / 4 % 3 != 1;+ assert_eq!(info.in_use, read, "record {number}");+ if read {+ assert_eq!(info.apparent, number as u64 + 100);+ }+ }+ assert_eq!(names.len(), 64);+ for entry in &names {+ let at = entry.at as usize;+ let text =+ &texts[entry.chunk as usize][at..][..usize::from(entry.len)];+ assert_eq!(text, format!("f{}", entry.child));+ }+ assert_eq!(progress.snapshot().files, 64);+ }++ #[test]+ fn a_malformed_update_sequence_is_refused_without_panicking() {+ let valid = record(IN_USE, 0, &[]);+ assert!(fixup(&mut valid.clone()).is_some());+ // Counts that do not match 512-byte strides, one of which once+ // made the stride under two bytes, and an array past the record.+ for (count, at) in [(0, USA_AT), (2, USA_AT), (600, USA_AT), (3, 1020)]+ {+ let mut bytes = valid.clone();+ bytes[4..6].copy_from_slice(&(at as u16).to_le_bytes());+ bytes[6..8].copy_from_slice(&(count as u16).to_le_bytes());+ assert!(fixup(&mut bytes).is_none(), "count {count} at {at}");+ }+ }++ #[test]+ fn a_boot_sector_with_impossible_sizes_is_refused() {+ let boot = |sector: u16, per_cluster: u8, per_record: u8| {+ let mut bytes = vec![0; 512];+ bytes[3..11].copy_from_slice(b"NTFS ");+ bytes[0x0B..0x0D].copy_from_slice(§or.to_le_bytes());+ bytes[0x0D] = per_cluster;+ bytes[0x30..0x38].copy_from_slice(&4_u64.to_le_bytes());+ bytes[0x40] = per_record;+ bytes+ };+ // 512-byte sectors, 8 to a cluster, 1 KiB records (2^-(-10)).+ let geometry = geometry_of(&boot(512, 8, 0xF6)).expect("valid");+ assert_eq!(+ (geometry.cluster, geometry.record, geometry.mft_offset),+ (4096, 1024, 4 * 4096)+ );+ for (sector, per_cluster, per_record) in [+ (513, 8, 0xF6), // sector not a power of two+ (512, 0, 0xF6), // no sectors to a cluster+ (512, 0x90, 0xF6), // a shift past 64 bits+ (512, 0xF0, 0xF6), // a 32 MiB cluster+ (512, 8, 0xBA), // a record of 2^70 bytes+ (512, 8, 2), // an 8 KiB record+ ] {+ assert!(+ geometry_of(&boot(sector, per_cluster, per_record)).is_none(),+ "{sector} {per_cluster:#x} {per_record:#x}"+ );+ }+ assert!(geometry_of(&[0; 16]).is_none());+ }++ #[test]+ fn a_name_that_would_resolve_elsewhere_is_skipped() {+ for text in ["", ".", "..", "a/b", "a\\b", "a\0b"] {+ let mut bytes = record(IN_USE, 0, &[name(40, text, 1)]);+ let (out, _) = parse(&mut bytes, 77);+ assert!(out.names.is_empty() && out.text.is_empty(), "{text:?}");+ }+ }++ /// A table of `(record, parent, parent sequence, name, info)` rows+ /// beneath the root, record 5 at sequence 5.+ fn table<'a>(+ rows: &[(u32, u32, u16, &str, Info)],+ options: &'a ScanOptions,+ progress: &'a ScanProgress,+ ) -> Table<'a> {+ let mut infos = vec![Info::default(); 64];+ infos[ROOT as usize] = Info {+ in_use: true,+ directory: true,+ sequence: 5,+ ..Info::default()+ };+ let mut text = String::new();+ let mut names = Vec::new();+ for &(child, parent, parent_sequence, name, info) in rows {+ infos[child as usize] = info;+ names.push(Entry {+ parent,+ parent_sequence,+ child,+ chunk: 0,+ at: text.len() as u32,+ len: name.len() as u16,+ });+ text.push_str(name);+ }+ names.sort_by_key(|entry| entry.parent);+ let starts = starts(&names, infos.len());+ Table {+ infos,+ names,+ texts: vec![text],+ starts,+ options,+ progress,+ }+ }++ fn paths(node: &Node, prefix: &str, out: &mut Vec<(String, NodeKind)>) {+ for child in &node.children {+ let path = format!("{prefix}{}", child.name);+ out.push((path.clone(), child.kind));+ paths(child, &format!("{path}/"), out);+ }+ }++ #[test]+ fn the_table_keeps_what_the_walk_would_list() {+ let file = Info {+ in_use: true,+ sequence: 1,+ ..Info::default()+ };+ let dir = Info {+ directory: true,+ sequence: 3,+ ..file+ };+ let rows = [+ (20, 5, 5, "visible.txt", file),+ (21, 5, 5, ".dotfile", file),+ (+ 22,+ 5,+ 5,+ "hidden.txt",+ Info {+ attributes: FILE_ATTRIBUTE_HIDDEN,+ ..file+ },+ ),+ (+ 23,+ 5,+ 5,+ "evicted",+ Info {+ attributes: FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS,+ ..dir+ },+ ),+ // A mount point: a name surrogate, measured as a link.+ (+ 24,+ 5,+ 5,+ "link",+ Info {+ attributes: FILE_ATTRIBUTE_REPARSE_POINT,+ reparse_tag: 0xA000_0003,+ ..dir+ },+ ),+ (25, 5, 5, "sub", dir),+ (26, 25, 3, "inner.txt", file),+ (27, 25, 3, "deeper", dir),+ (28, 27, 3, "deepest.txt", file),+ // The file system's own records.+ (10, 5, 5, "$system", file),+ // Entries of an earlier directory in a reused record.+ (29, 5, 4, "stale.txt", file),+ (30, 25, 2, "stale-in-sub.txt", file),+ (31, 5, 5, "linked.txt", Info { names: 2, ..file }),+ (31, 25, 3, "linked.txt", Info { names: 2, ..file }),+ ];+ let everything = vec![+ ".dotfile",+ "hidden.txt",+ "link",+ "linked.txt",+ "sub",+ "sub/deeper",+ "sub/deeper/deepest.txt",+ "sub/inner.txt",+ "sub/linked.txt",+ "visible.txt",+ ];+ let shallow_visible = vec![+ "link",+ "linked.txt",+ "sub",+ "sub/inner.txt",+ "sub/linked.txt",+ "visible.txt",+ ];+ let progress = ScanProgress::default();+ for (include_hidden, max_depth, expected) in+ [(true, None, everything), (false, Some(1), shallow_visible)]+ {+ let options = ScanOptions {+ include_hidden,+ max_depth,+ ..ScanOptions::default()+ };+ let Ok(root) = table(&rows, &options, &progress).directory(+ ROOT,+ "C:\\".into(),+ 0,+ ) else {+ panic!("the table is not cut short");+ };+ let mut found = Vec::new();+ paths(&root, "", &mut found);+ found.sort_by(|left, right| left.0.cmp(&right.0));+ let kind = |path: &str| match path {+ "link" => NodeKind::Symlink,+ "sub" | "sub/deeper" => NodeKind::Directory,+ _ => NodeKind::File,+ };+ let expected: Vec<(String, NodeKind)> = expected+ .into_iter()+ .map(|path| (path.to_owned(), kind(path)))+ .collect();+ assert_eq!(found, expected, "include_hidden {include_hidden}");++ // Only a file with a second name needs its identity kept.+ let inode = |name: &str| root.child_named(name).unwrap().inode;+ assert_eq!(inode("linked.txt"), Some((0, 31)));+ assert_eq!(inode("visible.txt"), None);+ }+ }++ #[test]+ fn a_directory_beneath_itself_gives_up_rather_than_recurse() {+ let dir = Info {+ in_use: true,+ directory: true,+ sequence: 1,+ ..Info::default()+ };+ let rows = [+ (40, 5, 5, "loop", dir),+ (41, 40, 1, "inner", dir),+ (40, 41, 1, "again", dir),+ ];+ let options = ScanOptions::default();+ let progress = ScanProgress::default();+ let table = table(&rows, &options, &progress);+ assert!(table.directory(ROOT, "C:\\".into(), 0).is_err());+ assert!(!progress.is_cancelled());+ }+}diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rsindex f9abac6..2dcda50 100644--- a/crates/disktree-core/src/scan.rs+++ b/crates/disktree-core/src/scan.rs@@ -8,8 +8,8 @@//! `+1` sentinel, so a directory is only built once its own scan *and* all of//! its subdirectory tasks have finished,//! * children are handed to the parent as finished `Node`s and sizes are-//! aggregated bottom-up in one serial pass, which is also where hardlinks are-//! de-duplicated.+//! aggregated bottom-up in one pass over the finished tree, which is also+//! where hardlinks are de-duplicated.//!//! What is added on top of dust's approach: live progress counters the UI can//! poll without locking, and cooperative cancellation so a re-scan can abandon@@ -22,13 +22,13 @@ use std::io;use std::path::{Path, PathBuf};use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};use std::sync::mpsc::{self, Receiver, TryRecvError};-use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError};+use std::sync::{Arc, LazyLock, Mutex, MutexGuard, OnceLock, PoisonError};use std::thread;use rayon::Scope;use rustc_hash::FxHashSet;-use crate::tree::{Metric, Node, NodeKind, aggregate};+use crate::tree::{Metric, Node, NodeKind, Seen, aggregate, aggregate_deduped};/// Errors kept verbatim before the list is truncated; the count keeps rising.const MAX_ERROR_DETAIL: usize = 50;@@ -105,13 +105,21 @@ pub struct ScanSnapshot {}impl ScanProgress {- fn count_file(&self, bytes: u64) {- self.files.fetch_add(1, Ordering::Relaxed);+ /// What a directory's listing found (see `Tally`), or what a reader+ /// that measures in bulk has come across so far.+ pub(crate) fn add(&self, files: u64, dirs: u64, bytes: u64) {+ self.files.fetch_add(files, Ordering::Relaxed);+ self.dirs.fetch_add(dirs, Ordering::Relaxed);self.bytes.fetch_add(bytes, Ordering::Relaxed);}- fn count_dir(&self) {- self.dirs.fetch_add(1, Ordering::Relaxed);+ /// The totals of the finished tree, replacing a bulk reader's running+ /// count: it met the whole volume, and the tree may be less of it.+ #[cfg(windows)]+ pub(crate) fn settle(&self, files: u64, dirs: u64, bytes: u64) {+ self.files.store(files, Ordering::Relaxed);+ self.dirs.store(dirs, Ordering::Relaxed);+ self.bytes.store(bytes, Ordering::Relaxed);}fn record_error(&self, path: &Path, error: &io::Error) {@@ -191,6 +199,7 @@ impl ScanHandle {never_scanned: OnceLock::new(),visited_dirs: Mutex::new(FxHashSet::default()),root: Mutex::new(None),+ volume: OnceLock::new(),});let (sender, result) = mpsc::channel();@@ -240,6 +249,7 @@ pub fn scan(root: &Path, options: ScanOptions) -> io::Result<Node> {never_scanned: OnceLock::new(),visited_dirs: Mutex::new(FxHashSet::default()),root: Mutex::new(None),+ volume: OnceLock::new(),});let node = scan_blocking(root, &context)?;progress.finish();@@ -264,6 +274,10 @@ struct WalkContext {visited_dirs: Mutex<FxHashSet<(u64, u64)>>,/// Set by the root's `complete`, read after the scope joins.root: Mutex<Option<Node>>,+ /// Serial number of the root's volume, on Windows, for a walk that+ /// cannot leave it: every listing takes it instead of asking. See+ /// [`crate::windows::walk_volume`].+ volume: OnceLock<u64>,}impl WalkContext {@@ -281,33 +295,41 @@ impl WalkContext {self.progress.is_cancelled()}- /// Decide what to do with an entry, and account for the work it implies.- fn classify(&self, entry: &impl Listed) -> Classified {- let path = entry.entry_path();+ /// Decide what to do with an entry listed in `dir`.+ ///+ /// The entry's path is built only where it is used: most entries are+ /// files, and a disk has millions of them. The name is moved out of+ /// the entry last, so nothing copies it.+ fn classify(&self, dir: &Path, entry: &mut impl Listed) -> Classified {let listing = match entry.listing() {Ok(listing) => listing,Err(error) => {- self.progress.record_error(&path, &error);+ self.progress.record_error(&entry.entry_path(dir), &error);return Classified::Skipped;}};- let name = entry.display_name();if !self.options.include_hidden- && (name.starts_with('.') || entry.hidden())+ && (entry.name().starts_with('.') || entry.hidden()){return Classified::Skipped;}let kind = match listing {- Listing::Symlink => return self.classify_symlink(&path, name),- Listing::Directory => return self.classify_dir(entry, path, name),+ Listing::Symlink => {+ let path = entry.entry_path(dir);+ return self.classify_symlink(&path, entry.take_name());+ }+ Listing::Directory => {+ let path = entry.entry_path(dir);+ return self.classify_dir(entry, path);+ }Listing::Leaf(kind) => kind,};match entry.facts(self.options.apparent_size) {- Ok(facts) => self.leaf(name, kind, &facts),+ Ok(facts) => self.leaf(entry.take_name(), kind, &facts),Err(error) => {- self.progress.record_error(&path, &error);+ self.progress.record_error(&entry.entry_path(dir), &error);Classified::Skipped}}@@ -315,9 +337,8 @@ impl WalkContext {fn classify_dir(&self,- entry: &impl Listed,+ entry: &mut impl Listed,path: PathBuf,- name: Box<str>,) -> Classified {if self.never_scanned@@ -344,12 +365,8 @@ impl WalkContext {if let Some(known) = &self.known&& known.path == path{- let tree = (*known.tree).clone();- self.progress.files.fetch_add(tree.files, Ordering::Relaxed);- self.progress.bytes.fetch_add(tree.bytes, Ordering::Relaxed);- self.progress.dirs.fetch_add(tree.dirs, Ordering::Relaxed);- let mut tree = tree;- tree.name = name;+ let mut tree = (*known.tree).clone();+ tree.name = entry.take_name();return Classified::Entry(tree);}if self.options.one_filesystem@@ -374,8 +391,10 @@ impl WalkContext {_ => {}}}- self.progress.count_dir();- Classified::Subdirectory(path)+ Classified::Subdirectory {+ path,+ name: entry.take_name(),+ }}fn classify_symlink(&self, path: &Path, name: Box<str>) -> Classified {@@ -385,7 +404,6 @@ impl WalkContext {return match fs::symlink_metadata(path) {Ok(meta) => {let facts = Facts::of(&meta, self.options.apparent_size);- self.progress.count_file(facts.size);Classified::Entry(leaf_node(name,NodeKind::Symlink,@@ -422,8 +440,10 @@ impl WalkContext {{return Classified::Skipped;}- self.progress.count_dir();- return Classified::Subdirectory(path.to_path_buf());+ return Classified::Subdirectory {+ path: path.to_path_buf(),+ name,+ };}let mut facts = Facts::of(&meta, self.options.apparent_size);@@ -439,7 +459,6 @@ impl WalkContext {kind: NodeKind,facts: &Facts,) -> Classified {- self.progress.count_file(facts.size);// A followed link reaches a file a second way, whatever its link// count says.let track = self.options.follow_links || facts.shared;@@ -454,10 +473,14 @@ impl WalkContext {/// would cost an open per file; [`crate::windows`] lists a directory with/// both instead.trait Listed {- fn entry_path(&self) -> PathBuf;+ /// The entry's path inside `dir`, the directory it was listed from.+ fn entry_path(&self, dir: &Path) -> PathBuf;/// Non-UTF-8 names are lossy for display. The scan still measures them/// correctly; only the reported name is approximate.- fn display_name(&self) -> Box<str>;+ fn name(&self) -> &str;+ /// The name, moved out for the tree: after this `name` is empty, and+ /// `entry_path`, which must open the real file, is off limits.+ fn take_name(&mut self) -> Box<str>;fn listing(&self) -> io::Result<Listing>;/// Size, identity and age of a leaf.fn facts(&self, apparent_size: bool) -> io::Result<Facts>;@@ -515,18 +538,29 @@ impl Facts {}}+/// A standard directory entry with its name decoded once, up front.#[cfg(not(windows))]-impl Listed for DirEntry {- fn entry_path(&self) -> PathBuf {- self.path()+struct Named {+ entry: DirEntry,+ name: Box<str>,+}++#[cfg(not(windows))]+impl Listed for Named {+ fn entry_path(&self, _dir: &Path) -> PathBuf {+ self.entry.path()}- fn display_name(&self) -> Box<str> {- self.file_name().to_string_lossy().into_owned().into()+ fn name(&self) -> &str {+ &self.name+ }++ fn take_name(&mut self) -> Box<str> {+ std::mem::take(&mut self.name)}fn listing(&self) -> io::Result<Listing> {- let file_type = self.file_type()?;+ let file_type = self.entry.file_type()?;Ok(if file_type.is_symlink() {Listing::Symlink} else if file_type.is_dir() {@@ -539,11 +573,13 @@ impl Listed for DirEntry {}fn facts(&self, apparent_size: bool) -> io::Result<Facts> {- self.metadata().map(|meta| Facts::of(&meta, apparent_size))+ self.entry+ .metadata()+ .map(|meta| Facts::of(&meta, apparent_size))}fn directory(&self) -> io::Result<Directory> {- self.metadata().map(|meta| Directory {+ self.entry.metadata().map(|meta| Directory {device: device_of(&meta),evicted: is_dataless(&meta),})@@ -552,12 +588,16 @@ impl Listed for DirEntry {#[cfg(windows)]impl Listed for crate::windows::Entry {- fn entry_path(&self) -> PathBuf {- self.path()+ fn entry_path(&self, dir: &Path) -> PathBuf {+ self.path(dir)}- fn display_name(&self) -> Box<str> {- self.file_name().to_string_lossy().into_owned().into()+ fn name(&self) -> &str {+ self.name()+ }++ fn take_name(&mut self) -> Box<str> {+ self.take_name()}fn listing(&self) -> io::Result<Listing> {@@ -600,19 +640,33 @@ impl Listed for crate::windows::Entry {/// List a directory the way [`Listed`] describes.#[cfg(not(windows))]-fn list(path: &Path) -> io::Result<fs::ReadDir> {- fs::read_dir(path)+fn list(+ path: &Path,+ _volume: Option<u64>,+) -> io::Result<impl Iterator<Item = io::Result<Named>>> {+ Ok(fs::read_dir(path)?.map(|entry| {+ entry.map(|entry| Named {+ name: entry.file_name().into_string().map_or_else(+ |raw| raw.to_string_lossy().into(),+ String::into_boxed_str,+ ),+ entry,+ })+ }))}#[cfg(windows)]-fn list(path: &Path) -> io::Result<crate::windows::ReadDir> {- crate::windows::read_dir(path)+fn list(+ path: &Path,+ volume: Option<u64>,+) -> io::Result<crate::windows::ReadDir> {+ crate::windows::read_dir(path, volume)}/// What a directory entry turned out to be.enum Classified {/// Descend into this directory on a new task.- Subdirectory(PathBuf),+ Subdirectory { path: PathBuf, name: Box<str> },/// A leaf that contributes size.Entry(Node),/// Filtered out, unreadable, or a symlink we chose not to follow.@@ -627,15 +681,25 @@ struct PendingDir {/// Starts at 1 for the directory itself; one more per subdirectory task./// When it reaches zero the directory is complete.pending: AtomicUsize,- /// Files found directly here, and finished subdirectories handed back up.- children: Mutex<Vec<Node>>,+ /// The node so far, moved out whole once the directory is complete.+ partial: Mutex<Partial>,read_error: AtomicBool,depth: usize,}+/// What a directory's node is made of before it is built: the name the+/// listing gave it, and the files found directly in it plus the finished+/// subdirectories handed back up.+#[derive(Debug, Default)]+struct Partial {+ name: Box<str>,+ children: Vec<Node>,+}+impl PendingDir {const fn new(path: PathBuf,+ name: Box<str>,parent: Option<Arc<Self>>,depth: usize,) -> Self {@@ -643,21 +707,25 @@ impl PendingDir {path,parent,pending: AtomicUsize::new(1),- children: Mutex::new(Vec::new()),+ partial: Mutex::new(Partial {+ name,+ children: Vec::new(),+ }),read_error: AtomicBool::new(false),depth,}}/// Turn a completed directory into a node. Only called when `pending` has- /// reached zero, so every child is already in `self.children`.+ /// reached zero, so every child is already in `self.partial`.////// `bytes` and `own_bytes` are left at zero on purpose: the walk cannot/// know the aggregate, and [`crate::tree::aggregate`] derives both from the/// children once every child is present.fn build(&self) -> Node {+ let partial = std::mem::take(&mut *lock(&self.partial));Node {- name: file_name(&self.path),+ name: partial.name,kind: NodeKind::Directory,bytes: 0,own_bytes: 0,@@ -669,7 +737,7 @@ impl PendingDir {modified: 0,category: crate::classify::Category::Other,reclaim: None,- children: std::mem::take(&mut *lock(&self.children)),+ children: partial.children,}}}@@ -682,23 +750,64 @@ fn scan_blocking(root: &Path, context: &Arc<WalkContext>) -> io::Result<Node> {format!("{} is not a directory", root.display()),));}- let canonical = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());- let never = crate::space::never_scanned(root, &canonical);+ let resolved = root.canonicalize().ok();+ let canonical = resolved.as_deref().unwrap_or(root);+ let never = crate::space::never_scanned(root, canonical);let _ = context.never_scanned.set(never.into_iter().collect());if context.options.one_filesystem {*lock(&context.root_device) = Some(device_of(&root_meta));- if let Some(foreign) = crate::space::foreign_mounts_for(&canonical) {+ if let Some(foreign) = crate::space::foreign_mounts_for(canonical) {let _ = context.foreign_mounts.set(foreign.into_iter().collect());}}+ // Reading the file table is far faster than any walk, where the+ // volume allows it. On the walk's pool too: see `WALK_POOL`.+ #[cfg(windows)]+ {+ let progress = &context.progress;+ let read = WALK_POOL.install(|| {+ crate::mft::scan(root, canonical, &context.options, progress)+ });+ if let Some(node) = read {+ let node = node?;+ // Cancelled after the reader's last check: the rescan that+ // asked is waiting, and this tree is not wanted.+ if progress.is_cancelled() {+ return Err(crate::mft::cancelled());+ }+ let node = finish_tree(node, &context.options);+ // The reader counted every file on the volume; the tree may+ // hold fewer.+ progress.settle(node.files, node.dirs, node.bytes);+ return Ok(node);+ }+ // The reader gave up, or never started: the walk counts from nothing.+ progress.settle(0, 0, 0);+ // Asking once instead of per directory; see `walk_volume` for+ // when a walk that does not follow links cannot leave the volume.+ // Only on a resolved root: a mapped share's letter kept as typed+ // looks like a local drive.+ if !context.options.follow_links+ && let Some(resolved) = &resolved+ && let Some(serial) = crate::windows::walk_volume(resolved)+ {+ let _ = context.volume.set(serial);+ }+ }if context.options.follow_links&& let Some(key) = identity_of(root, &root_meta){lock(&context.visited_dirs).insert(key);}- let root_dir = Arc::new(PendingDir::new(root.to_path_buf(), None, 0));- rayon::scope(|scope| walk(scope, &root_dir, context));+ let root_dir = Arc::new(PendingDir::new(+ root.to_path_buf(),+ file_name(root),+ None,+ 0,+ ));+ let context: &WalkContext = context;+ WALK_POOL.install(|| rayon::scope(|scope| walk(scope, &root_dir, context)));let node = lock(&context.root).take();let node = node.ok_or_else(|| {@@ -707,32 +816,101 @@ fn scan_blocking(root: &Path, context: &Arc<WalkContext>) -> io::Result<Node> {Ok(finish_tree(node, &context.options))}+/// The walk's workers, and the file table reader's. Listing directories+/// contends in the file system, and past some number of threads more of+/// them only queue: on Windows a home directory of 3.4 million files+/// walked in 6.5 s on 16 threads and 8.1 s on all 24. That cap was+/// measured on one 24-thread NTFS laptop; another disk or CPU may want+/// another. Elsewhere the default count. A pool of its own on every+/// platform all the same: work the UI hands the global pool, such as+/// re-ranking a tree, must not queue behind a scan's blocking reads.+static WALK_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {+ let threads = if cfg!(windows) {+ rayon::current_num_threads().min(16)+ } else {+ rayon::current_num_threads()+ };+ rayon::ThreadPoolBuilder::new()+ .num_threads(threads)+ .thread_name(|index| format!("disktree-walk-{index}"))+ .build()+ .expect("a thread pool")+});++/// Entries counted before the shared progress counters are touched. Every+/// worker adding to the same counters per entry keeps one cache line+/// bouncing between cores; per directory, or per this many entries in a+/// wide one so the meter still moves, is invisible to a reader.+const TALLY_EVERY: u64 = 1024;++/// What one directory's listing has found so far, for the progress meter.+#[derive(Default)]+struct Tally {+ files: u64,+ dirs: u64,+ bytes: u64,+}++impl Tally {+ const fn add(&mut self, node: &Node) {+ if node.is_dir() {+ // A subtree taken whole from an earlier scan.+ self.files += node.files;+ self.dirs += node.dirs;+ } else {+ // A link counts as a file on the meter, as it always has,+ // though the tree does not count it.+ self.files += 1;+ }+ self.bytes += node.bytes;+ }++ fn flush(&mut self, progress: &ScanProgress) {+ progress.add(self.files, self.dirs, self.bytes);+ *self = Self::default();+ }+}+/// Read one directory, spawn a task per subdirectory, then report completion.////// Flat tasks inside the scope: the worker stack never grows with tree depth.-fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {+fn walk<'scope>(+ scope: &Scope<'scope>,+ dir: &Arc<PendingDir>,+ context: &'scope WalkContext,+) {let mut subdirs: Vec<Arc<PendingDir>> = Vec::new();let mut leaves: Vec<Node> = Vec::new();+ let mut tally = Tally::default();- match list(&dir.path) {+ match list(&dir.path, context.volume.get().copied()) {Ok(entries) => {for entry in entries {if context.cancelled() {break;}match entry {- Ok(entry) => match context.classify(&entry) {- Classified::Subdirectory(path) => {- let child = Arc::new(PendingDir::new(- path,- Some(Arc::clone(dir)),- dir.depth + 1,- ));- subdirs.push(child);+ Ok(mut entry) => {+ match context.classify(&dir.path, &mut entry) {+ Classified::Subdirectory { path, name } => {+ tally.dirs += 1;+ subdirs.push(Arc::new(PendingDir::new(+ path,+ name,+ Some(Arc::clone(dir)),+ dir.depth + 1,+ )));+ }+ Classified::Entry(node) => {+ tally.add(&node);+ leaves.push(node);+ }+ Classified::Skipped => {}+ }+ if tally.files + tally.dirs >= TALLY_EVERY {+ tally.flush(&context.progress);}- Classified::Entry(node) => leaves.push(node),- Classified::Skipped => {}- },+ }Err(error) => {context.progress.record_error(&dir.path, &error);}@@ -744,6 +922,7 @@ fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {dir.read_error.store(true, Ordering::Relaxed);}}+ tally.flush(&context.progress);// A depth-limited scan still measures what is directly in the directory,// it just does not descend further.@@ -751,20 +930,30 @@ fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {.options.max_depth.is_none_or(|max_depth| dir.depth < max_depth);+ // Sized once for the leaves and every subdirectory to come, before any+ // subdirectory task can push its node: otherwise that first push+ // doubles a list just sized to the leaves, and the slack stays in the+ // finished tree. The leaves are copied, not moved: the list they grew+ // in carries up to twice their size in slack, and moving it in whole+ // made a home directory walk a second slower (measured, 5 interleaved+ // pairs).+ let spawned = if descend { subdirs.len() } else { 0 };+ {+ let mut partial = lock(&dir.partial);+ partial.children.reserve_exact(leaves.len() + spawned);+ partial.children.extend(leaves);+ }if descend {for subdir in subdirs {if context.cancelled() {break;}dir.pending.fetch_add(1, Ordering::AcqRel);- let subdir = Arc::clone(&subdir);- let context = Arc::clone(context);- scope.spawn(move |scope| walk(scope, &subdir, &context));+ scope.spawn(move |scope| walk(scope, &subdir, context));}}- lock(&dir.children).extend(leaves);- signal_done(&Arc::clone(dir), context);+ signal_done(dir, context);}/// Report that one task for `dir` is done: either its own scan, or one of its@@ -773,44 +962,32 @@ fn walk(scope: &Scope<'_>, dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {/// This is the whole reason for the `+1` sentinel: a directory is only built/// once its own scan *and* every subdirectory task has finished, no matter/// which of them lands last.-fn signal_done(dir: &Arc<PendingDir>, context: &Arc<WalkContext>) {+fn signal_done(dir: &Arc<PendingDir>, context: &WalkContext) {if dir.pending.fetch_sub(1, Ordering::AcqRel) != 1 {return;}let node = dir.build();- let Some(parent) = dir.parent.clone() else {+ let Some(parent) = &dir.parent else {*lock(&context.root) = Some(node);return;};- lock(&parent.children).push(node);- signal_done(&parent, context);+ lock(&parent.partial).children.push(node);+ signal_done(parent, context);}-/// Charge a hardlinked file once, then derive every aggregate from the result.-///-/// Zeroing `own_bytes` rather than `bytes` is deliberate:-/// [`crate::tree::aggregate`] recomputes totals from the direct contents, so a-/// patched `bytes` would be overwritten.+/// Charge a hardlinked file once, derive every aggregate from the result,+/// then classify. On the walk's pool: the UI's own work on the global pool+/// must not queue behind a scan finishing.fn finish_tree(mut node: Node, options: &ScanOptions) -> Node {- if options.dedup_hardlinks {- let mut seen = FxHashSet::default();- mark_duplicate_hardlinks(&mut node, &mut seen);- }- aggregate(&mut node, options.metric);- crate::classify::classify(&mut node);- node-}--fn mark_duplicate_hardlinks(node: &mut Node, seen: &mut FxHashSet<(u64, u64)>) {- if !node.is_dir() {- if node.inode.is_some_and(|key| !seen.insert(key)) {- node.own_bytes = 0;+ WALK_POOL.install(|| {+ if options.dedup_hardlinks {+ aggregate_deduped(&mut node, options.metric, &Seen::new());+ } else {+ aggregate(&mut node, options.metric);}- return;- }- for child in &mut node.children {- mark_duplicate_hardlinks(child, seen);- }+ crate::classify::classify(&mut node);+ });+ node}/// A leaf. Its identity is kept only when `track` says the walk could meet@@ -958,7 +1135,7 @@ fn kind_of(meta: &Metadata, file_type: fs::FileType) -> NodeKind {/// The display name of a scanned root: its final component, or the path itself/// for `/`.-fn file_name(path: &Path) -> Box<str> {+pub(crate) fn file_name(path: &Path) -> Box<str> {path.file_name().map_or_else(|| path.to_string_lossy().into_owned(),@@ -969,7 +1146,7 @@ fn file_name(path: &Path) -> Box<str> {/// A poisoned lock means another task panicked; the data is still a valid tree/// prefix, and refusing to read it would turn one panic into two.-fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {+pub(crate) fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {mutex.lock().unwrap_or_else(PoisonError::into_inner)}@@ -1269,6 +1446,7 @@ mod tests {never_scanned: OnceLock::new(),visited_dirs: Mutex::new(FxHashSet::default()),root: Mutex::new(None),+ volume: OnceLock::new(),});let tree = scan_blocking(root, &context).expect("scan still succeeds");let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o700));diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rsindex 997882f..6f8e93b 100644--- a/crates/disktree-core/src/tree.rs+++ b/crates/disktree-core/src/tree.rs@@ -1,6 +1,11 @@//! The scanned tree.use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{LazyLock, Mutex, OnceLock};++use rayon::prelude::*;+use rustc_hash::FxHashSet;use crate::classify::{Category, Reclaim};@@ -235,7 +240,104 @@ impl Node {/// de-duplication rewrites a leaf's weight, and every total above it —/// including its parent's "direct" figure — follows without a second pass.pub fn aggregate(node: &mut Node, metric: Metric) {+ aggregate_at(node, metric, 0, None);+}++/// [`aggregate`], charging a hardlinked file once: a leaf whose identity+/// `seen` already holds weighs nothing. Which of a file's names is charged+/// is whichever a worker reaches first, so two scans of an unchanged tree+/// can split it differently between folders; the totals are the same.+pub(crate) fn aggregate_deduped(node: &mut Node, metric: Metric, seen: &Seen) {+ aggregate_at(node, metric, 0, Some(seen));+}++/// Identities a finish pass has met.+///+/// A hash set of every file on a disk is millions of random writes into a+/// table too big for any cache. Inode and file record numbers are small+/// integers, though, so a key on the first volume met whose whole number+/// is below the bitmap's size gets one bit instead: an 8 MiB bitmap the+/// workers set without locking. The file table reader's record numbers+/// fit, as do most inodes. Anything else goes into the hash set, whole:+/// a Windows file id keeps the record's reuse count in its top 16 bits,+/// and that is what tells a file from an older one on the same record,+/// which a subtree kept from an earlier scan can still hold. The set is+/// sharded for the walk's millions of those: 4.1 million ids took 150 ms+/// across 64 locks and 1.2 s behind one, in a synthetic run.+pub(crate) struct Seen {+ /// The first volume met, which is the scanned root's in all but a scan+ /// that leaves its volume.+ volume: OnceLock<u64>,+ /// Made on the first key that fits: most Unix trees, where only files+ /// with more than one name carry an identity, never need it.+ bits: LazyLock<Box<[AtomicU64]>>,+ rest: Box<[Shard]>,+}++/// One lock's part of the hash set.+type Shard = Mutex<FxHashSet<(u64, u64)>>;++impl Seen {+ /// Numbers the bitmap covers: sixty-four million, more files than a+ /// desktop volume holds, for 8 MiB written once.+ const BITS: u64 = 1 << 26;++ /// Hash set shards, as a power of two.+ const SHARD_BITS: u32 = 6;++ pub(crate) fn new() -> Self {+ Self {+ volume: OnceLock::new(),+ bits: LazyLock::new(|| {+ std::iter::repeat_with(|| AtomicU64::new(0))+ .take((Self::BITS / 64) as usize)+ .collect()+ }),+ rest: std::iter::repeat_with(Mutex::default)+ .take(1 << Self::SHARD_BITS)+ .collect(),+ }+ }++ /// Whether `key` is new.+ fn insert(&self, key: (u64, u64)) -> bool {+ let (volume, number) = key;+ if *self.volume.get_or_init(|| volume) == volume && number < Self::BITS+ {+ let bit = 1 << (number % 64);+ let word = &self.bits[(number / 64) as usize];+ return word.fetch_or(bit, Ordering::Relaxed) & bit == 0;+ }+ // The top bits of a multiplicative hash, with another constant+ // than the set's own hasher, so a shard's keys still spread over+ // its table.+ let shard = (number.wrapping_mul(0x9E37_79B9_7F4A_7C15)+ >> (u64::BITS - Self::SHARD_BITS)) as usize;+ crate::scan::lock(&self.rest[shard]).insert(key)+ }+}++/// Levels whose subtrees are aggregated in parallel. A top level already+/// splits a disk into enough work for every thread; below it a parallel+/// frame per level would only cut the depth that fits in a worker's stack.+pub(crate) const PARALLEL_LEVELS: usize = 4;++fn aggregate_at(+ node: &mut Node,+ metric: Metric,+ depth: usize,+ seen: Option<&Seen>,+) {if !node.is_dir() {+ // Every name of a file has the file's size, so one that weighs+ // nothing need not be remembered to be charged once.+ if node.own_bytes > 0+ && let Some(seen) = seen+ && let Some(key) = node.inode+ && !seen.insert(key)+ {+ node.own_bytes = 0;+ }node.bytes = node.own_bytes;node.files = node.own_files;node.dirs = 0;@@ -248,14 +350,23 @@ pub fn aggregate(node: &mut Node, metric: Metric) {let mut own_files = 0;let mut dirs: u64 = 1;let mut modified = 0;- for child in &mut node.children {- aggregate(child, metric);+ if depth < PARALLEL_LEVELS {+ node.children+ .par_iter_mut()+ .for_each(|child| aggregate_at(child, metric, depth + 1, seen));+ } else {+ for child in &mut node.children {+ aggregate_at(child, metric, depth + 1, seen);+ }+ }+ for child in &node.children {modified = modified.max(child.modified);- bytes += child.bytes;+ // Saturating: a corrupt volume's file table can claim any size.+ bytes = child.bytes.saturating_add(bytes);files += child.files;dirs += child.dirs;if !child.is_dir() {- own_bytes += child.bytes;+ own_bytes = child.bytes.saturating_add(own_bytes);own_files += child.files;}}@@ -267,8 +378,11 @@ pub fn aggregate(node: &mut Node, metric: Metric) {node.modified = modified;// Largest first: a treemap lays out big tiles best, and the order is what- // makes "descend into the largest child" meaningful.- node.children.sort_by(|left, right| {+ // makes "descend into the largest child" meaningful. Unstable: names in+ // one directory are distinct, and the only ties are names that decoded+ // to the same lossy text, whose order does not matter, so the stable+ // sort's scratch allocation buys nothing.+ node.children.sort_unstable_by(|left, right| {right.value(metric).cmp(&left.value(metric))@@ -300,6 +414,42 @@ mod tests {Node::entry(name, NodeKind::File, bytes)}+ #[test]+ fn seen_charges_an_identity_once_in_either_store() {+ let seen = Seen::new();+ // The first volume: the bitmap.+ assert!(seen.insert((7, 42)));+ assert!(!seen.insert((7, 42)));+ assert!(seen.insert((7, 43)));+ // A number past the bitmap, and another volume: the hash set.+ let far = Seen::BITS + 42;+ assert!(seen.insert((7, far)));+ assert!(!seen.insert((7, far)));+ assert!(seen.insert((8, 42)), "another volume's 42 is another file");+ assert!(!seen.insert((8, 42)));+ }++ #[test]+ fn identities_that_differ_only_in_their_top_bits_are_two_files() {+ // Record 0x2A of one volume under two reuse counts, as a subtree kept+ // from an earlier scan and a later file can hold, and a real second+ // name of the newer one.+ let older = (7, (1 << 48) | 0x2A);+ let newer = (7, (2 << 48) | 0x2A);+ let mut root = Node::directory("root");+ for (name, bytes, key) in [+ ("older", 4096, older),+ ("newer", 8192, newer),+ ("newer again", 8192, newer),+ ] {+ let mut file = leaf(name, bytes);+ file.inode = Some(key);+ root.children.push(file);+ }+ aggregate_deduped(&mut root, Metric::Bytes, &Seen::new());+ assert_eq!(root.bytes, 4096 + 8192);+ }+#[test]fn aggregate_derives_totals_and_orders_children() {let mut root = Node::directory("root");diff --git a/crates/disktree-core/src/windows.rs b/crates/disktree-core/src/windows.rsindex 9e9468d..b036bdb 100644--- a/crates/disktree-core/src/windows.rs+++ b/crates/disktree-core/src/windows.rs@@ -23,11 +23,10 @@ use std::os::windows::ffi::{OsStrExt as _, OsStringExt as _};use std::os::windows::fs::{MetadataExt as _, OpenOptionsExt as _};use std::os::windows::io::AsRawHandle as _;use std::path::{Component, Path, PathBuf, Prefix};-use std::sync::Arc;use windows_sys::Win32::Foundation::{ERROR_INVALID_FUNCTION, ERROR_INVALID_LEVEL, ERROR_INVALID_PARAMETER,- ERROR_NO_MORE_FILES, ERROR_NOT_SUPPORTED, INVALID_HANDLE_VALUE,+ ERROR_NO_MORE_FILES, ERROR_NOT_SUPPORTED, INVALID_HANDLE_VALUE, MAX_PATH,};use windows_sys::Win32::Storage::FileSystem::{FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_HIDDEN,@@ -36,7 +35,8 @@ use windows_sys::Win32::Storage::FileSystem::{FILE_ID_EXTD_DIR_INFO, FILE_LIST_DIRECTORY, FILE_READ_ATTRIBUTES,FileIdExtdDirectoryInfo, FindFirstVolumeW, FindNextVolumeW,FindVolumeClose, GetDiskFreeSpaceExW, GetFileInformationByHandleEx,- GetVolumePathNameW, GetVolumePathNamesForVolumeNameW, SYNCHRONIZE,+ GetVolumeInformationW, GetVolumePathNameW,+ GetVolumePathNamesForVolumeNameW, SYNCHRONIZE,};use crate::space::SpaceInfo;@@ -73,10 +73,15 @@ pub enum Kind {}/// One directory entry, with what a measurement needs.+///+/// The name is decoded to text once, here, since that is what the tree+/// keeps: the listing hands the walk one allocation per entry, not three.#[derive(Debug)]pub struct Entry {- dir: Arc<Path>,- name: OsString,+ name: Box<str>,+ /// The name as Windows spells it, only when `name` could not: an+ /// unpaired surrogate is shown as U+FFFD but must still open.+ exact: Option<Box<OsStr>>,kind: Kind,apparent: u64,allocated: u64,@@ -86,14 +91,29 @@ pub struct Entry {}impl Entry {- pub fn path(&self) -> PathBuf {- self.dir.join(&self.name)+ /// The entry's path inside `dir`, the directory it was listed from.+ pub fn path(&self, dir: &Path) -> PathBuf {+ dir.join(self.file_name())}pub fn file_name(&self) -> &OsStr {+ self.exact+ .as_deref()+ .unwrap_or_else(|| OsStr::new(&*self.name))+ }++ /// The name as text; see [`Self::file_name`] for what it may lose.+ pub fn name(&self) -> &str {&self.name}+ /// The name, moved out: the tree keeps it, so nothing copies it. The+ /// entry's other accessors keep working; `name`, `file_name` and+ /// `path` do not.+ pub fn take_name(&mut self) -> Box<str> {+ std::mem::take(&mut self.name)+ }+pub const fn kind(&self) -> Kind {self.kind}@@ -133,7 +153,7 @@ impl Entry {self.identity}- fn from_std(dir: &Arc<Path>, entry: &fs::DirEntry) -> io::Result<Self> {+ fn from_std(entry: &fs::DirEntry) -> io::Result<Self> {let meta = entry.metadata()?;let file_type = meta.file_type();let kind = if file_type.is_symlink() {@@ -143,9 +163,15 @@ impl Entry {} else {Kind::File};+ let (name, exact) = match entry.file_name().into_string() {+ Ok(name) => (name.into_boxed_str(), None),+ Err(raw) => {+ (raw.to_string_lossy().into(), Some(raw.into_boxed_os_str()))+ }+ };Ok(Self {- dir: Arc::clone(dir),- name: entry.file_name(),+ name,+ exact,kind,apparent: meta.len(),allocated: meta.len(),@@ -162,7 +188,6 @@ impl Entry {/// and `..`.#[derive(Debug)]pub struct ReadDir {- dir: Arc<Path>,source: Source,}@@ -189,7 +214,12 @@ struct Records {}/// List `dir`, with each entry's allocation and file id.-pub fn read_dir(dir: &Path) -> io::Result<ReadDir> {+///+/// `volume` is the serial number of the volume `dir` is on, when the+/// caller knows it for every directory of a walk (see [`walk_volume`]):+/// asking per directory is several requests to the file system on top of+/// the listing. `None` asks.+pub fn read_dir(dir: &Path, volume: Option<u64>) -> io::Result<ReadDir> {let handle = OpenOptions::new().access_mode(FILE_LIST_DIRECTORY | FILE_READ_ATTRIBUTES | SYNCHRONIZE)// Needed to open a directory at all. Without@@ -197,9 +227,11 @@ pub fn read_dir(dir: &Path) -> io::Result<ReadDir> {// a walk that chose to enter one wants..custom_flags(FILE_FLAG_BACKUP_SEMANTICS).open(dir)?;- let volume = winapi_util::file::information(&handle)- .ok()- .map(|info| info.volume_serial_number());+ let volume = volume.or_else(|| {+ winapi_util::file::information(&handle)+ .ok()+ .map(|info| info.volume_serial_number())+ });let mut records = Records {handle,volume,@@ -207,17 +239,16 @@ pub fn read_dir(dir: &Path) -> io::Result<ReadDir> {next: None,done: false,};- let dir: Arc<Path> = Arc::from(dir);// The first fill shows whether the file system implements this listing,// before any entry has been handed out.let source = match records.fill() {Ok(()) => Source::Records(records),Err(error) if unsupported(&error) => {- Source::Std(Box::new(fs::read_dir(&dir)?))+ Source::Std(Box::new(fs::read_dir(dir)?))}Err(error) => return Err(error),};- Ok(ReadDir { dir, source })+ Ok(ReadDir { source })}impl Iterator for ReadDir {@@ -227,7 +258,7 @@ impl Iterator for ReadDir {let records = match &mut self.source {Source::Std(listing) => {return listing.next().map(|entry| {- entry.and_then(|entry| Entry::from_std(&self.dir, &entry))+ entry.and_then(|entry| Entry::from_std(&entry))});}Source::Records(records) => records,@@ -243,7 +274,7 @@ impl Iterator for ReadDir {}continue;};- match records.record(&self.dir, offset) {+ match records.record(offset) {Ok((entry, next)) => {records.next = next;if let Some(entry) = entry {@@ -304,7 +335,6 @@ impl Records {/// the buffer before it is used.fn record(&self,- dir: &Arc<Path>,offset: usize,) -> io::Result<(Option<Entry>, Option<usize>)> {let malformed =@@ -333,11 +363,11 @@ impl Records {_ => return Err(malformed()),};- let name: Vec<u16> = name_bytes+ let units = name_bytes.chunks_exact(2)- .map(|pair| u16::from_ne_bytes([pair[0], pair[1]]))- .collect();- if name == [u16::from(b'.')] || name == [u16::from(b'.'); 2] {+ .map(|pair| u16::from_ne_bytes([pair[0], pair[1]]));+ let (name, exact) = decode_name(units);+ if &*name == "." || &*name == ".." {return Ok((None, next));}@@ -358,8 +388,8 @@ impl Records {header[id_at..id_at + 16].try_into().expect("sixteen"),);let entry = Entry {- dir: Arc::clone(dir),- name: OsString::from_wide(&name),+ name,+ exact,kind,apparent: bytes_from(signed(offset_of!(FILE_ID_EXTD_DIR_INFO,@@ -380,6 +410,27 @@ impl Records {}}+/// A UTF-16 name as text, decoded in one pass; nearly every name is ASCII,+/// so one code unit is one byte and the text is allocated once, at its+/// final size. An unpaired surrogate is replaced, and the exact name kept+/// beside it so the path still opens.+fn decode_name(+ units: impl Iterator<Item = u16> + Clone,+) -> (Box<str>, Option<Box<OsStr>>) {+ let mut name = String::with_capacity(units.size_hint().0);+ let mut lossy = false;+ for unit in char::decode_utf16(units.clone()) {+ name.push(unit.unwrap_or_else(|_| {+ lossy = true;+ char::REPLACEMENT_CHARACTER+ }));+ }+ let exact = lossy.then(|| {+ OsString::from_wide(&units.collect::<Vec<u16>>()).into_boxed_os_str()+ });+ (name.into_boxed_str(), exact)+}+/// The part of a 128-bit file id that names a file by itself.////// NTFS keeps the high half zero ([MS-FSCC] 2.1.10), so the low half is the@@ -403,15 +454,35 @@ fn file_id(id: u128) -> Option<u64> {////// [MS-FSCC]: https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-fscc/2d3333fe-fc98-4a6f-98a2-4bb805aff407pub fn identity(path: &Path) -> Option<(u64, u64)> {+ let info = information(path)?;+ let index = info.file_index();+ (index != 0 && index != u64::MAX)+ .then(|| (info.volume_serial_number(), index))+}++/// Serial number of the volume every directory under `canonical` is on,+/// for a walk that does not follow links. Only a local drive has one:+/// mounted folders there are reparse points the walk takes for links,+/// while a share's DFS links are not, and lead to other servers' volumes.+/// `None` for a share, or when the serial cannot be read.+pub fn walk_volume(canonical: &Path) -> Option<u64> {+ let Some(Component::Prefix(prefix)) = canonical.components().next() else {+ return None;+ };+ if !matches!(prefix.kind(), Prefix::Disk(_) | Prefix::VerbatimDisk(_)) {+ return None;+ }+ information(canonical).map(|info| info.volume_serial_number())+}++/// What the file system says about `path`, following links.+fn information(path: &Path) -> Option<winapi_util::file::Information> {let file = OpenOptions::new().access_mode(FILE_READ_ATTRIBUTES).custom_flags(FILE_FLAG_BACKUP_SEMANTICS).open(path).ok()?;- let info = winapi_util::file::information(&file).ok()?;- let index = info.file_index();- (index != 0 && index != u64::MAX)- .then(|| (info.volume_serial_number(), index))+ winapi_util::file::information(&file).ok()}/// Capacity and free space of the volume holding `path`.@@ -531,6 +602,119 @@ pub fn is_mount_point(path: &Path) -> bool {&& volume_root(path).is_some_and(|root| same(&root, path))}+/// Whether this process runs as an administrator, with the elevated token+/// UAC hands out: what reading a volume directly takes.+pub fn elevated() -> bool {+ // SAFETY: takes no arguments and only reads the process token.+ unsafe { windows_sys::Win32::UI::Shell::IsUserAnAdmin() != 0 }+}++/// Whether `root` is a whole drive an administrator would read from its+/// file table: a drive letter's root, formatted NTFS. Anything else, from+/// a folder or a mounted folder to another file system, is walked either way.+pub fn file_table_readable(root: &Path) -> bool {+ let canonical = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());+ let Some(letter) = drive_letter(&canonical) else {+ return false;+ };+ let volume: Vec<u16> = format!("{letter}:\\\0").encode_utf16().collect();+ let mut name = [0_u16; MAX_PATH as usize + 1];+ let length = u32::try_from(name.len()).unwrap_or(u32::MAX);+ // SAFETY: `volume` is NUL-terminated and outlives the call, `name` is+ // writable for the length passed, and the other outputs may be null.+ let ok = unsafe {+ GetVolumeInformationW(+ volume.as_ptr(),+ std::ptr::null_mut(),+ 0,+ std::ptr::null_mut(),+ std::ptr::null_mut(),+ std::ptr::null_mut(),+ name.as_mut_ptr(),+ length,+ )+ };+ let end = name.iter().position(|&unit| unit == 0).unwrap_or(0);+ ok != 0 && String::from_utf16_lossy(&name[..end]) == "NTFS"+}++/// Start `program` with `args` as an administrator, through the UAC+/// prompt. Returns once the prompt is answered; declining it is an error+/// (`ERROR_CANCELLED`).+pub fn run_elevated(program: &Path, args: &[OsString]) -> io::Result<()> {+ let file = wide(program, false)?;+ let mut line: Vec<u16> = Vec::new();+ for arg in args {+ if !line.is_empty() {+ line.push(u16::from(b' '));+ }+ line.extend(quoted(arg));+ }+ line.push(0);+ let verb: Vec<u16> = "runas\0".encode_utf16().collect();+ // SAFETY: every string is NUL-terminated and outlives the call; a null+ // window and directory are allowed.+ let instance = unsafe {+ windows_sys::Win32::UI::Shell::ShellExecuteW(+ std::ptr::null_mut(),+ verb.as_ptr(),+ file.as_ptr(),+ line.as_ptr(),+ std::ptr::null(),+ windows_sys::Win32::UI::WindowsAndMessaging::SW_SHOWNORMAL,+ )+ };+ // Anything above 32 is success, by the function's own convention.+ if instance as usize > 32 {+ Ok(())+ } else {+ Err(io::Error::last_os_error())+ }+}++/// `C` when `canonical` is the root of drive `C:`; `None` for anything+/// else, a folder or a volume mounted in one.+pub fn drive_letter(canonical: &Path) -> Option<char> {+ let mut components = canonical.components();+ let Some(Component::Prefix(prefix)) = components.next() else {+ return None;+ };+ let (Prefix::Disk(letter) | Prefix::VerbatimDisk(letter)) = prefix.kind()+ else {+ return None;+ };+ (components.next() == Some(Component::RootDir)+ && components.next().is_none())+ .then_some(char::from(letter))+}++/// `arg` quoted so a program's C runtime splits it back out whole: inside+/// quotes, backslashes are literal except before a quote, where they must+/// be doubled, so `C:\` becomes `"C:\\"` and not `"C:\"`.+fn quoted(arg: &OsStr) -> Vec<u16> {+ let quote = u16::from(b'"');+ let backslash = u16::from(b'\\');+ let mut out = vec![quote];+ let mut slashes = 0;+ for unit in arg.encode_wide() {+ if unit == backslash {+ slashes += 1;+ continue;+ }+ let run = if unit == quote {+ 2 * slashes + 1+ } else {+ slashes+ };+ out.extend(std::iter::repeat_n(backslash, run));+ slashes = 0;+ out.push(unit);+ }+ out.extend(std::iter::repeat_n(backslash, 2 * slashes));+ out.push(quote);+ out+}+/// Whether `left` and `right` name the same place, compared the way Windows/// compares names: see [`folded`].pub fn same(left: &Path, right: &Path) -> bool {@@ -615,7 +799,7 @@ fn bytes_from(value: i64) -> u64 {}/// A `FILETIME` as Unix seconds, `0` before 1970.-const fn unix_seconds(ticks: i64) -> i64 {+pub const fn unix_seconds(ticks: i64) -> i64 {if ticks <= EPOCH_TICKS {0} else {@@ -623,6 +807,36 @@ const fn unix_seconds(ticks: i64) -> i64 {}}+/// Bytes aligned to a page, for reads that bypass the file cache: those+/// want memory aligned to the disk's sector, which a page always is.+#[derive(Debug, Default)]+pub struct Aligned {+ pages: Vec<Page>,+}++#[derive(Clone, Copy, Debug)]+#[repr(C, align(4096))]+struct Page([u8; 4096]);++impl Aligned {+ /// `len` bytes, their contents whatever the last use left.+ pub fn bytes(&mut self, len: usize) -> &mut [u8] {+ let pages = len.div_ceil(size_of::<Page>());+ if self.pages.len() < pages {+ self.pages.resize(pages, Page([0; 4096]));+ }+ // SAFETY: `Page` is a `repr(C)` byte array, so the pages are one+ // allocation of initialized bytes with no padding, and `len` fits+ // in it; the borrow of `self` keeps it alive and unaliased.+ unsafe {+ std::slice::from_raw_parts_mut(+ self.pages.as_mut_ptr().cast::<u8>(),+ len,+ )+ }+ }+}+/// Make a directory junction for a test: unlike a symbolic link, one needs/// no privilege to create. `false` when there is no `cmd` to make it with.#[cfg(test)]@@ -643,6 +857,17 @@ mod tests {use super::*;use tempfile::TempDir;+ #[test]+ fn quoted_arguments_survive_the_command_line() {+ let quote =+ |arg: &str| String::from_utf16("ed(OsStr::new(arg))).unwrap();+ // A drive root's trailing backslash would otherwise escape the quote.+ assert_eq!(quote(r"C:\"), r#""C:\\""#);+ assert_eq!(quote(r"C:\Program Files\x"), r#""C:\Program Files\x""#);+ assert_eq!(quote(r#"a\"b"#), r#""a\\\"b""#);+ assert_eq!(quote("--metric"), r#""--metric""#);+ }+#[test]fn the_volume_list_has_the_system_drive() {let drive =@@ -657,7 +882,7 @@ mod tests {}fn listed(dir: &Path) -> Vec<Entry> {- let mut entries: Vec<Entry> = read_dir(dir)+ let mut entries: Vec<Entry> = read_dir(dir, None).expect("list").collect::<io::Result<_>>().expect("entries");@@ -677,11 +902,40 @@ mod tests {assert_eq!(names, ["data.bin", "sub"]);assert_eq!(entries[0].kind(), Kind::File);assert_eq!(entries[0].apparent(), 100_000);- assert_eq!(entries[0].path(), temp.path().join("data.bin"));+ assert_eq!(entries[0].path(temp.path()), temp.path().join("data.bin"));assert_eq!(entries[1].kind(), Kind::Directory);assert!(entries[0].modified() > 0, "written just now");}+ #[test]+ fn a_name_that_is_not_text_is_shown_lossy_and_still_opens() {+ use std::os::windows::ffi::OsStringExt as _;+ let temp = TempDir::new().expect("tempdir");+ // An unpaired surrogate: NTFS takes it, text cannot hold it.+ let raw =+ OsString::from_wide(&[u16::from(b'a'), 0xD800, u16::from(b'b')]);+ fs::write(temp.path().join(&raw), b"x").expect("write");++ let entries = listed(temp.path());+ let [entry] = &entries[..] else {+ panic!("one entry: {}", entries.len());+ };+ assert_eq!(entry.name(), "a\u{FFFD}b");+ assert_eq!(entry.file_name(), raw);+ assert!(fs::metadata(entry.path(temp.path())).is_ok());+ }++ #[test]+ fn only_a_local_drive_gives_a_walk_one_volume() {+ let temp = TempDir::new().expect("tempdir");+ let canonical = temp.path().canonicalize().expect("canonical");+ assert!(walk_volume(&canonical).is_some(), "{}", canonical.display());+ // A share's DFS links lead to other volumes; nothing is asked.+ for share in [r"\\server\share\dir", r"\\?\UNC\server\share\dir"] {+ assert_eq!(walk_volume(Path::new(share)), None, "{share}");+ }+ }+#[test]fn the_allocation_is_whole_clusters_of_the_data() {let temp = TempDir::new().expect("tempdir");@@ -746,11 +1000,15 @@ mod tests {};assert_eq!(kind("junction"), Some(Kind::Link));assert_eq!(kind("target"), Some(Kind::Directory));- let through: Vec<OsString> = listed(&junction)+ let through: Vec<Box<str>> = listed(&junction).into_iter().map(|entry| entry.name).collect();- assert_eq!(through, ["inside.bin"], "a walk that enters one follows");+ assert_eq!(+ &*through[0], "inside.bin",+ "a walk that enters one follows"+ );+ assert_eq!(through.len(), 1);}#[test]@@ -794,6 +1052,7 @@ mod tests {drive.display());assert!(!is_mount_point(temp.path()));+ assert!(!file_table_readable(temp.path()), "a folder is walked");assert!(!is_mount_point(drive), "a drive is a root, not a folder");}