AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): skip dataless macOS placeholders instead of blocking on them (#2223)
xerj-org299bf5f187cb
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rsindex 26bcad3a1..d13b87d83 100644--- a/engine/crates/xerj-autoindex/src/content.rs+++ b/engine/crates/xerj-autoindex/src/content.rs@@ -45,7 +45,7 @@ pub fn resolve_reporting(files: Vec<FileEntry>,hashed: &(dyn Fn(u64) + Sync),) -> Result<Inventory> {- resolve_reporting_carried(files, &|_| None, &|_, _, _| {}, hashed)+ resolve_reporting_carried(files, &|_| None, &|_, _, _| {}, hashed, None)}/// As [`resolve_reporting`], with a caller-supplied digest cache.@@ -67,11 +67,18 @@ pub fn resolve_reporting(/// carries an assignment forward only on a match, so a stale carried digest/// looks like "unchanged" — the failure mode is a missed update, not a bad/// document — and the next event or fingerprint change repairs it.+///+/// `blocking`, when set, names each file on the progress line for the duration+/// of its read. macOS passes the run's progress handle so a read that blocks+/// on a dataless placeholder says which path (#2058). The guard credits+/// nothing; `hashed` still does, once, on completion. Other targets ignore the+/// handle even if it is passed.pub fn resolve_reporting_carried(files: Vec<FileEntry>,carried: &(dyn Fn(&FileEntry) -> Option<String> + Sync),observed: &(dyn Fn(&FileEntry, &str, bool) + Sync),hashed: &(dyn Fn(u64) + Sync),+ blocking: Option<&crate::progress::Progress>,) -> Result<Inventory> {// Phase A belongs to the run's scan pool, not to rayon's global pool:// `--workers` has to bound the CPU-bound phase to mean anything (#240 §2).@@ -102,7 +109,13 @@ pub fn resolve_reporting_carried(// would push the percent past 100 and invent a rate.Some(digest) => (Ok(digest), false),None => {- let digest = full_digest(&file.path, file.size);+ // Held across the read and dropped before the byte+ // credit, so a blocked `read` is visible and a finished+ // one is not counted twice.+ let digest = {+ let _blocking = crate::dataless::watch_blocking_hash(blocking, file);+ full_digest(&file.path, file.size)+ };hashed(file.size);(digest, true)}diff --git a/engine/crates/xerj-autoindex/src/dataless.rs b/engine/crates/xerj-autoindex/src/dataless.rsnew file mode 100644index 000000000..eb95a54d3--- /dev/null+++ b/engine/crates/xerj-autoindex/src/dataless.rs@@ -0,0 +1,397 @@+//! macOS File Provider placeholders (iCloud Drive and other cloud volumes).+//!+//! A dataless file has a directory entry and a logical size, but its bytes are+//! not on this machine. `stat` reports that without downloading anything.+//! `open` / `read` does download, and the call blocks until the transfer+//! finishes — which is how `xerj autoindex` hashing stalled with no path named+//! (#2058, field reports #2042 and #2051).+//!+//! The flag is Apple's `SF_DATALESS` (`0x40000000` in `bsd/sys/stat.h`, "file+//! is dataless object"). It is synthetic and read-only, so a test cannot set+//! it with `chflags`. libc 0.2.191 exposes `stat.st_flags` on macOS and does+//! not export the constant; the value below is the header's. Detection is+//! macOS-only. On every other target [`split`] returns the input unchanged and+//! does not stat.+//!+//! Linux compiles the accounting so the unit tests run in CI. Production on+//! that target calls [`Split::unchanged`] and does not stat; the rest of this+//! module is macOS-only and is allowed to look unused there.++#![cfg_attr(not(any(target_os = "macos", test)), allow(dead_code))]++use crate::state::Plan;+use crate::walk::FileEntry;+use std::collections::HashMap;+use std::path::Path;++/// Apple `SF_DATALESS` (`bsd/sys/stat.h`).+pub(crate) const SF_DATALESS: u32 = 0x4000_0000;++/// Skip reason recorded in the run summary. Stable for anything that counts+/// skips by name.+pub(crate) const SKIP_REASON: &str = "dataless-placeholder";++/// `st_flags` has `SF_DATALESS` set. Other flags do not matter: a placeholder+/// can also be hidden or compressed.+pub(crate) fn flags_indicate_dataless(flags: u32) -> bool {+ flags & SF_DATALESS != 0+}++/// Human line for files left out of this run. `None` when nothing was skipped,+/// so a run with no placeholders prints the lines it always printed.+pub(crate) fn skip_note(count: u64) -> Option<String> {+ if count == 0 {+ return None;+ }+ let files = if count == 1 { "file" } else { "files" };+ Some(format!(+ "skipped {count} {files} — {SKIP_REASON} (macOS File Provider / iCloud placeholder; \+ the bytes are not local, and reading them blocks on a download, so they were not \+ hashed or indexed). Materialize them and re-run to include them"+ ))+}++/// Human line for placeholders this run did not read because a previous pass+/// already hashed them. They stay in the inventory under that digest, so an+/// incremental re-run does not treat an evicted file as deleted.+pub(crate) fn carry_note(count: u64) -> Option<String> {+ if count == 0 {+ return None;+ }+ let files = if count == 1 { "file" } else { "files" };+ Some(format!(+ "left {count} dataless placeholder {files} on their previous content \+ ({SKIP_REASON}); not re-read, so the indexed copy stays"+ ))+}++/// What hashing should do with one walked tree.+pub(crate) struct Split {+ /// Files to resolve. Dataless files with no prior digest are absent.+ pub files: Vec<FileEntry>,+ /// `path_id` → digest for dataless files that must not be opened.+ pub carry: HashMap<String, String>,+ /// Dataless files removed from [`Self::files`].+ pub skipped_files: Vec<FileEntry>,+ /// Files [`Self::files`] that will actually be read. Carried placeholders+ /// are not included. The hash phase recomputes this after the budget+ /// filter, from the files still admitted, so production does not read+ /// these fields — the unit tests do.+ #[cfg_attr(all(target_os = "macos", not(test)), allow(dead_code))]+ pub hash_files: u64,+ #[cfg_attr(all(target_os = "macos", not(test)), allow(dead_code))]+ pub hash_bytes: u64,+}++impl Split {+ /// No stat, no filtering. The non-macOS result.+ ///+ /// macOS goes through `split_with` instead, so this helper is unused there.+ #[cfg_attr(target_os = "macos", allow(dead_code))]+ pub(crate) fn unchanged(files: Vec<FileEntry>) -> Self {+ let hash_files = files.len() as u64;+ let hash_bytes = files.iter().map(|file| file.size).sum();+ Self {+ files,+ carry: HashMap::new(),+ skipped_files: Vec::new(),+ hash_files,+ hash_bytes,+ }+ }+}++/// Split `files` into "hash", "carry the previous digest", and "skip".+///+/// On macOS a file whose `st_flags` include `SF_DATALESS` is not opened. If+/// `prior` has a digest for its `path_id`, that digest is carried and the file+/// stays in the inventory (an evicted file is not a deletion). Otherwise the+/// file is skipped and counted. A failed `stat` is not dataless: the later+/// open reports the real error instead of hiding a readable file.+///+/// On every other target this is [`Split::unchanged`].+pub(crate) fn split(files: Vec<FileEntry>, prior: &HashMap<String, String>) -> Split {+ #[cfg(target_os = "macos")]+ {+ split_with(files, prior, is_dataless_placeholder)+ }+ #[cfg(not(target_os = "macos"))]+ {+ let _ = prior;+ Split::unchanged(files)+ }+}++/// [`split`] with the placeholder test injected. The production macOS path+/// passes [`is_dataless_placeholder`]; tests pass a predicate so the accounting+/// does not need a real iCloud file.+fn split_with(+ files: Vec<FileEntry>,+ prior: &HashMap<String, String>,+ is_dataless: impl Fn(&Path) -> bool,+) -> Split {+ let mut kept = Vec::with_capacity(files.len());+ let mut carry = HashMap::new();+ let mut skipped_files = Vec::new();+ let mut hash_files = 0u64;+ let mut hash_bytes = 0u64;+ for file in files {+ if is_dataless(&file.path) {+ if let Some(digest) = prior.get(&file.rel_id) {+ carry.insert(file.rel_id.clone(), digest.clone());+ kept.push(file);+ } else {+ skipped_files.push(file);+ }+ } else {+ hash_files += 1;+ hash_bytes += file.size;+ kept.push(file);+ }+ }+ Split {+ files: kept,+ carry,+ skipped_files,+ hash_files,+ hash_bytes,+ }+}++/// path_id → full-content digest for every file the previous plan actually+/// hashed. Empty path ids and assignments with no digest are omitted: there+/// is nothing safe to carry, and a placeholder in that state is skipped.+pub(crate) fn prior_digests(plan: Option<&Plan>) -> HashMap<String, String> {+ let Some(plan) = plan else {+ return HashMap::new();+ };+ let mut out = HashMap::new();+ for assignment in plan.files.values() {+ remember_digest(+ &mut out,+ &assignment.path_id,+ assignment.content_digest.as_deref(),+ );+ }+ for duplicate in &plan.duplicate_files {+ let digest = plan+ .files+ .get(&duplicate.file_key)+ .and_then(|assignment| assignment.content_digest.as_deref());+ remember_digest(&mut out, &duplicate.path_id, digest);+ }+ out+}++fn remember_digest(out: &mut HashMap<String, String>, path_id: &str, digest: Option<&str>) {+ if path_id.is_empty() {+ return;+ }+ let Some(digest) = digest.filter(|digest| !digest.is_empty()) else {+ return;+ };+ out.entry(path_id.to_string())+ .or_insert_with(|| digest.to_string());+}++/// Register `file` on the progress in-flight list for the duration of the+/// guard, on macOS only.+///+/// Hashing reads the whole file in one worker. A placeholder the pre-pass+/// missed (it became dataless between `stat` and `open`, or `stat` failed and+/// the open blocks) otherwise stalls with `in_flight` empty, so the progress+/// line cannot name a path. The guard does not credit bytes — the completion+/// callback still does that, once, when the read returns.+///+/// Other targets return `None` and do not touch progress. A hash phase there+/// keeps the line it had before #2058.+pub(crate) fn watch_blocking_hash<'a>(+ progress: Option<&'a crate::progress::Progress>,+ file: &FileEntry,+) -> Option<crate::progress::WatchGuard<'a>> {+ #[cfg(target_os = "macos")]+ {+ progress.map(|progress| progress.watch_file(&file.rel, file.size))+ }+ #[cfg(not(target_os = "macos"))]+ {+ let _ = (progress, file);+ None+ }+}++#[cfg(target_os = "macos")]+fn is_dataless_placeholder(path: &Path) -> bool {+ flags_of(path).is_some_and(flags_indicate_dataless)+}++#[cfg(target_os = "macos")]+fn flags_of(path: &Path) -> Option<u32> {+ use std::os::unix::ffi::OsStrExt;+ let bytes = path.as_os_str().as_bytes();+ // `stat` is a C string. An interior NUL cannot be passed through; treat+ // that as "not proven dataless" and let the open fail on its own.+ if bytes.contains(&0) {+ return None;+ }+ let mut c_path = Vec::with_capacity(bytes.len() + 1);+ c_path.extend_from_slice(bytes);+ c_path.push(0);+ let mut st = std::mem::MaybeUninit::<libc::stat>::uninit();+ // SAFETY: `c_path` is a NUL-terminated path this process may stat, and+ // `st` is written by `stat` on success before it is read.+ let rc = unsafe { libc::stat(c_path.as_ptr().cast(), st.as_mut_ptr()) };+ if rc != 0 {+ return None;+ }+ let st = unsafe { st.assume_init() };+ Some(st.st_flags)+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::state::{DuplicateFile, FileAssignment};+ use std::fs;++ fn assignment(path_id: &str, digest: Option<&str>) -> FileAssignment {+ FileAssignment {+ rel: path_id.to_string(),+ path_id: path_id.to_string(),+ is_symlink: Some(false),+ family: "txt".into(),+ gzip: false,+ content_digest: digest.map(str::to_string),+ assignments: Vec::new(),+ as_document: false,+ }+ }++ fn entry(rel_id: &str, size: u64) -> FileEntry {+ FileEntry {+ path: Path::new(rel_id).to_path_buf(),+ rel: rel_id.to_string(),+ rel_id: rel_id.to_string(),+ is_symlink: false,+ size,+ }+ }++ #[test]+ fn sf_dataless_is_the_apple_bit_and_other_flags_are_not() {+ assert!(flags_indicate_dataless(SF_DATALESS));+ assert!(flags_indicate_dataless(SF_DATALESS | 0x0002_0000));+ assert!(!flags_indicate_dataless(0));+ // UF_HIDDEN, SF_IMMUTABLE, and the other synthetic high bit.+ assert!(!flags_indicate_dataless(0x0000_8000));+ assert!(!flags_indicate_dataless(0x0002_0000));+ assert!(!flags_indicate_dataless(0x8000_0000));+ }++ #[test]+ fn skip_and_carry_notes_name_the_reason_and_the_count() {+ assert!(skip_note(0).is_none());+ let one = skip_note(1).unwrap();+ assert!(+ one.contains("skipped 1 file — dataless-placeholder"),+ "{one}"+ );+ assert!(!one.contains("files"), "{one}");+ let two = skip_note(2).unwrap();+ assert!(+ two.contains("skipped 2 files — dataless-placeholder"),+ "{two}"+ );+ assert!(carry_note(0).is_none());+ let carried = carry_note(1).unwrap();+ assert!(+ carried.contains("1 dataless placeholder file") && carried.contains(SKIP_REASON),+ "{carried}"+ );+ }++ #[test]+ fn a_placeholder_with_no_prior_digest_is_skipped_and_counted() {+ let prior = HashMap::from([("kept-cloud".to_string(), "axf2-prior".to_string())]);+ let split = split_with(+ vec![+ entry("local", 10),+ entry("kept-cloud", 20),+ entry("new-cloud", 30),+ ],+ &prior,+ |path| path.to_str().is_some_and(|path| path.ends_with("cloud")),+ );+ let rels: Vec<&str> = split.files.iter().map(|file| file.rel.as_str()).collect();+ assert_eq!(rels, ["local", "kept-cloud"]);+ assert_eq!(+ split.carry.get("kept-cloud").map(String::as_str),+ Some("axf2-prior")+ );+ assert!(!split.carry.contains_key("local"));+ assert_eq!(split.skipped_files.len(), 1);+ assert_eq!(split.skipped_files[0].rel, "new-cloud");+ // The hash denominator is the file that will be read, not the one+ // whose digest is carried and not the one that was skipped.+ assert_eq!(split.hash_files, 1);+ assert_eq!(split.hash_bytes, 10);+ }++ #[test]+ fn a_non_placeholder_tree_is_unchanged() {+ let files = vec![entry("a", 4), entry("b", 6)];+ let split = split_with(files, &HashMap::new(), |_| false);+ assert!(split.carry.is_empty());+ assert!(split.skipped_files.is_empty());+ assert_eq!(split.hash_files, 2);+ assert_eq!(split.hash_bytes, 10);+ assert_eq!(split.files.len(), 2);+ }++ #[test]+ fn prior_digests_keep_hashed_paths_and_alias_targets() {+ let mut plan = Plan::default();+ plan.files+ .insert("axf2-body".into(), assignment("unix:61", Some("axf2-body")));+ plan.files+ .insert("legacy".into(), assignment("unix:62", None));+ plan.files+ .insert("blank".into(), assignment("", Some("axf2-blank")));+ plan.duplicate_files.push(DuplicateFile {+ file_key: "axf2-body".into(),+ rel: "alias".into(),+ path_id: "unix:63".into(),+ is_symlink: Some(false),+ duplicate_of: "body".into(),+ bytes: 4,+ });+ let prior = prior_digests(Some(&plan));+ assert_eq!(prior.get("unix:61").map(String::as_str), Some("axf2-body"));+ assert_eq!(prior.get("unix:63").map(String::as_str), Some("axf2-body"));+ assert!(!prior.contains_key("unix:62"), "no digest to carry");+ assert!(!prior.contains_key(""), "an empty path id is not a key");+ assert!(prior_digests(None).is_empty());+ }++ /// A real local file is not a placeholder. On macOS this stats it; elsewhere+ /// [`split`] does not stat and still keeps the file.+ #[test]+ fn a_local_file_is_not_a_dataless_placeholder() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("a.txt");+ fs::write(&path, b"hello").unwrap();+ let file = FileEntry {+ path,+ rel: "a.txt".into(),+ rel_id: "unix:61".into(),+ is_symlink: false,+ size: 5,+ };+ let split = split(vec![file], &HashMap::new());+ assert!(split.skipped_files.is_empty(), "{:?}", split.skipped_files);+ assert!(split.carry.is_empty());+ assert_eq!(split.files.len(), 1);+ assert_eq!(split.hash_files, 1);+ assert_eq!(split.hash_bytes, 5);+ }+}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 46b4b1f75..fb041cd9f 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -9,6 +9,7 @@ pub mod cli;pub mod coerce;mod content;pub mod correlate;+mod dataless;pub mod dataset;pub mod def;pub mod detect;@@ -1579,7 +1580,7 @@ pub(crate) fn outcome_headline(/// commit — the one place the two landed changes could contradict each other./// Exit 3 is success ("completed, some input was unusable"), so it is spelled/// out in words rather than left as a bare number.-fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value) {+fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value, skipped_dataless: u64) {let count = |field: &str| summary.get(field).and_then(Value::as_u64).unwrap_or(0);// Read back from the committed generation's own run document, exactly like// every other number on this line — a resumed or no-op run therefore@@ -1617,6 +1618,11 @@ fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value) {if count("bulk_splits") > 0 {extra.push(("bulk_splits", count("bulk_splits")));}+ // #2058: present only when this run left out dataless placeholders, so a+ // corpus with none prints the line it always did.+ if skipped_dataless > 0 {+ extra.push(("skipped_dataless", skipped_dataless));+ }extra.extend(coverage.fields());pr.finish(true,@@ -5531,6 +5537,76 @@ impl UnsupportedInventoryDelta {}}+/// [`UnsupportedInventoryDelta::between`], plus dataless placeholders this run+/// deliberately did not hash (#2058).+///+/// Those files are still on disk. Leaving them out of `files` makes the delta+/// call them exclusions, and the exclusion sweep deletes documents that were+/// indexed the last time the bytes were local. An empty `dataless_still_present`+/// takes the original path and clones nothing.+fn inventory_delta(+ root: &Path,+ files: &[walk::FileEntry],+ keys: &[String],+ plan: &Plan,+ dataless_still_present: &[walk::FileEntry],+) -> UnsupportedInventoryDelta {+ if dataless_still_present.is_empty() {+ return UnsupportedInventoryDelta::between(root, files, keys, plan);+ }+ let mut files = files.to_vec();+ let mut keys = keys.to_vec();+ for file in dataless_still_present {+ files.push(file.clone());+ keys.push(String::new());+ }+ UnsupportedInventoryDelta::between(root, &files, &keys, plan)+}++/// Hash-phase denominator on macOS. Carried placeholders are in `files` but+/// are not read, so they are not part of the work the bar describes.+#[cfg(target_os = "macos")]+fn hash_phase_totals(+ files: &[walk::FileEntry],+ carry: &std::collections::HashMap<String, String>,+ all_bytes: u64,+) -> (u64, u64) {+ if carry.is_empty() {+ return (files.len() as u64, all_bytes);+ }+ let mut count = 0u64;+ let mut bytes = 0u64;+ for file in files {+ if !carry.contains_key(&file.rel_id) {+ count += 1;+ bytes += file.size;+ }+ }+ (count, bytes)+}++/// Watch-pass hash totals, minus placeholders whose previous digest is carried+/// instead of read. An empty carry map returns the watch plan's own totals.+#[cfg(target_os = "macos")]+fn watch_hash_totals(+ plan: &watch::PassPlan,+ files: &[walk::FileEntry],+ carry: &std::collections::HashMap<String, String>,+) -> (u64, u64) {+ if carry.is_empty() {+ return (plan.hash_files, plan.hash_bytes);+ }+ let mut hash_files = plan.hash_files;+ let mut hash_bytes = plan.hash_bytes;+ for file in files {+ if carry.contains_key(&file.rel_id) && plan.carried(file).is_none() {+ hash_files = hash_files.saturating_sub(1);+ hash_bytes = hash_bytes.saturating_sub(file.size);+ }+ }+ (hash_files, hash_bytes)+}+fn alias_keys_to_reindex(previous: &[state::DuplicateFile],current: &[state::DuplicateFile],@@ -7179,7 +7255,7 @@ fn run_index_report_inner(pr.note("autoindex: resumed and committed pending corpus generation from durable source");let summary = finish_generated_run(&es, &mut journal, &cfg)?;let code = generated_exit_code(&summary);- finish_generated_progress(&pr, code, &summary);+ finish_generated_progress(&pr, code, &summary, 0);return Ok((code, Some(summary)));}// #585 case 2: a --no-graph generation's genesis bootstrap (sync_bootstrap@@ -7298,11 +7374,32 @@ fn run_index_report_inner(cfg.exclude_rels.join(", ")));}- let (mut discovered_files, data_files, data_bytes) = if cfg.skip_corpus_data {+ let (discovered_files, data_files, data_bytes) = if cfg.skip_corpus_data {without_corpus_data_files(discovered_files)} else {(discovered_files, 0, 0)};+ // #2058: a dataless File Provider placeholder (iCloud and other cloud+ // volumes) has a directory entry but no local bytes. Hashing it blocks+ // inside `read` until the download finishes, and the progress line used+ // to name nothing. `stat` sees `SF_DATALESS` without downloading. On+ // macOS those files are skipped, or carried from the previous digest when+ // one exists so an evicted file is not treated as deleted. Every other+ // target gets the walked list back unchanged and does not stat.+ #[cfg(target_os = "macos")]+ let dataless_split = dataless::split(+ discovered_files,+ &dataless::prior_digests(preflight.plan.as_ref()),+ );+ #[cfg(not(target_os = "macos"))]+ let dataless_split = dataless::Split::unchanged(discovered_files);+ let mut discovered_files = dataless_split.files;+ let dataless_skipped_files = dataless_split.skipped_files;+ let dataless_skipped = dataless_skipped_files.len() as u64;+ #[cfg(target_os = "macos")]+ let dataless_carry = dataless_split.carry;+ #[cfg(not(target_os = "macos"))]+ let _ = dataless_split.carry;if data_files > 0 {let size = if data_bytes >= 1 << 20 {format!("{:.1} MB", data_bytes as f64 / (1u64 << 20) as f64)@@ -7321,6 +7418,13 @@ fn run_index_report_inner(discovered_bytes / (1 << 20),root_str));+ if let Some(line) = dataless::skip_note(dataless_skipped) {+ pr.note(&format!("autoindex: {line}"));+ }+ #[cfg(target_os = "macos")]+ if let Some(line) = dataless::carry_note(dataless_carry.len() as u64) {+ pr.note(&format!("autoindex: {line}"));+ }// "Where did my files go?" is answered here, on every run — the rules are// named, not just the totals (#276).for line in ignore_report.summary_lines() {@@ -7342,20 +7446,24 @@ fn run_index_report_inner(pr.note("(walk-only — stopped before hashing; file contents were not hashed and nothing was indexed)",);+ let mut walk_done = vec![+ ("files", summary.files),+ ("bytes", summary.bytes),+ ("ignored_files", ignore_report.files_skipped),+ ("ignored_dirs", ignore_report.dirs_pruned),+ (+ "ignored_files_in_pruned_dirs",+ ignore_report.files_inside_pruned_dirs,+ ),+ ];+ if dataless_skipped > 0 {+ walk_done.push(("skipped_dataless", dataless_skipped));+ }pr.finish_with_flags(true,0,"walk-only",- &[- ("files", summary.files),- ("bytes", summary.bytes),- ("ignored_files", ignore_report.files_skipped),- ("ignored_dirs", ignore_report.dirs_pruned),- (- "ignored_files_in_pruned_dirs",- ignore_report.files_inside_pruned_dirs,- ),- ],+ &walk_done,&[("ignored_files_in_pruned_dirs_exact",ignore_report.files_inside_pruned_dirs_is_exact(),@@ -7367,8 +7475,22 @@ fn run_index_report_inner(// state: with a journal present, zero files is a deletion of the whole// corpus and has to be reconciled, not shrugged off.if discovered_files.is_empty() && !preflight.journal_exists {- println!("no files found under {root_str}");- pr.finish(true, 0, "no-files", &[]);+ if dataless_skipped > 0 {+ println!(+ "no indexable files under {root_str} ({dataless_skipped} dataless placeholder{} \+ skipped)",+ if dataless_skipped == 1 { "" } else { "s" }+ );+ pr.finish(+ true,+ 0,+ "no-files",+ &[("skipped_dataless", dataless_skipped)],+ );+ } else {+ println!("no files found under {root_str}");+ pr.finish(true, 0, "no-files", &[]);+ }return Ok((0, None));}// Count, estimate, and — when the default plan does not fit the disk@@ -7508,8 +7630,28 @@ fn run_index_report_inner(// A plain run hashes the whole corpus, every time, on purpose: see// `content::resolve_reporting`.None => {- pr.phase("hash", discovered_files.len() as u64, discovered_bytes);- content::resolve_reporting(discovered_files, &|bytes| pr.item_done(bytes))?+ // macOS names each file for the duration of its read and does not+ // read a dataless placeholder (#2058). Other targets hash the way+ // they always have: every admitted byte, no in-flight registration+ // on this phase.+ #[cfg(target_os = "macos")]+ {+ let (hash_files, hash_bytes) =+ hash_phase_totals(&discovered_files, &dataless_carry, discovered_bytes);+ pr.phase("hash", hash_files, hash_bytes);+ content::resolve_reporting_carried(+ discovered_files,+ &|entry| dataless_carry.get(&entry.rel_id).cloned(),+ &|_, _, _| {},+ &|bytes| pr.item_done(bytes),+ Some(pr.as_ref()),+ )?+ }+ #[cfg(not(target_os = "macos"))]+ {+ pr.phase("hash", discovered_files.len() as u64, discovered_bytes);+ content::resolve_reporting(discovered_files, &|bytes| pr.item_done(bytes))?+ }}// A `--watch` pass hashes what the watcher could not prove unchanged.// The phase's totals are the files this pass will actually read, so the@@ -7517,20 +7659,35 @@ fn run_index_report_inner(// a full run would have done.Some(pass) => {let plan = pass.plan(&discovered_files);+ #[cfg(target_os = "macos")]+ let (hash_files, hash_bytes) =+ watch_hash_totals(&plan, &discovered_files, &dataless_carry);+ #[cfg(not(target_os = "macos"))]+ let (hash_files, hash_bytes) = (plan.hash_files, plan.hash_bytes);let (carried_files, carried_bytes) = (plan.carried_files, plan.carried_bytes);- pr.phase("hash", plan.hash_files, plan.hash_bytes);+ pr.phase("hash", hash_files, hash_bytes);pr.note(&format!("autoindex: --watch: re-hashing {} file(s) ({} MB); {carried_files} file(s) \({} MB) carried from the previous pass",- plan.hash_files,- plan.hash_bytes >> 20,+ hash_files,+ hash_bytes >> 20,carried_bytes >> 20,));let inventory = content::resolve_reporting_carried(discovered_files,- &|entry| plan.carried(entry),+ &|entry| {+ #[cfg(target_os = "macos")]+ if let Some(digest) = dataless_carry.get(&entry.rel_id) {+ return Some(digest.clone());+ }+ plan.carried(entry)+ },&|entry, digest, fresh| plan.observe(entry, digest, fresh),&|bytes| pr.item_done(bytes),+ #[cfg(target_os = "macos")]+ Some(pr.as_ref()),+ #[cfg(not(target_os = "macos"))]+ None,)?;pass.commit(plan);inventory@@ -7572,19 +7729,23 @@ fn run_index_report_inner());// `ignored_files_in_pruned_dirs` is budget-capped, so the flag// saying whether it is a total or a floor travels with it (#279).+ let mut dry_done = vec![+ ("files", inventory.files.len() as u64),+ ("ignored_files", ignore_report.files_skipped),+ ("ignored_dirs", ignore_report.dirs_pruned),+ (+ "ignored_files_in_pruned_dirs",+ ignore_report.files_inside_pruned_dirs,+ ),+ ];+ if dataless_skipped > 0 {+ dry_done.push(("skipped_dataless", dataless_skipped));+ }pr.finish_with_flags(true,0,"dry-run",- &[- ("files", inventory.files.len() as u64),- ("ignored_files", ignore_report.files_skipped),- ("ignored_dirs", ignore_report.dirs_pruned),- (- "ignored_files_in_pruned_dirs",- ignore_report.files_inside_pruned_dirs,- ),- ],+ &dry_done,&[("ignored_files_in_pruned_dirs_exact",ignore_report.files_inside_pruned_dirs_is_exact(),@@ -7665,7 +7826,7 @@ fn run_index_report_inner(}let summary = finish_generated_run(&es, &mut journal, &cfg)?;let code = generated_exit_code(&summary);- finish_generated_progress(&pr, code, &summary);+ finish_generated_progress(&pr, code, &summary, dataless_skipped);return Ok((code, Some(summary)));}begin_non_graph_generation(@@ -7693,7 +7854,7 @@ fn run_index_report_inner()?;let summary = finish_generated_run(&es, &mut journal, &cfg)?;let code = generated_exit_code(&summary);- finish_generated_progress(&pr, code, &summary);+ finish_generated_progress(&pr, code, &summary, dataless_skipped);return Ok((code, Some(summary)));}if cfg.no_graph && preflight.plan.is_some() && !genesis_recovery {@@ -7757,11 +7918,12 @@ fn run_index_report_inner(// `--fresh` is checked here too: discarding the plan does not delete// the documents already published for a file that is now gone, so a// removal is unsafe in place whether or not the plan is kept.- let delta = UnsupportedInventoryDelta::between(+ let delta = inventory_delta(&cfg.root,&inventory.files,&comparison_keys,prior_plan,+ &dataless_skipped_files,);if delta.refuses() {return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, prior_plan)));@@ -7813,8 +7975,22 @@ fn run_index_report_inner(sync_executor::gc_snapshots(&state_dir, &journal)?;let resumed_with_plan = journal.plan.is_some() && !genesis_recovery;if inventory.files.is_empty() && !resumed_with_plan {- println!("no files found under {root_str}");- pr.finish(true, 0, "no-files", &[]);+ if dataless_skipped > 0 {+ println!(+ "no indexable files under {root_str} ({dataless_skipped} dataless placeholder{} \+ skipped)",+ if dataless_skipped == 1 { "" } else { "s" }+ );+ pr.finish(+ true,+ 0,+ "no-files",+ &[("skipped_dataless", dataless_skipped)],+ );+ } else {+ println!("no files found under {root_str}");+ pr.finish(true, 0, "no-files", &[]);+ }return Ok((0, None));}let run_id = journal.run_id.clone();@@ -8069,7 +8245,7 @@ fn run_index_report_inner(// this is set (see `carry_over_unread_mail`).let mut swept_excluded = false;if resumed_with_plan {- let delta = UnsupportedInventoryDelta::between(&cfg.root, &files, &keys, &plan);+ let delta = inventory_delta(&cfg.root, &files, &keys, &plan, &dataless_skipped_files);if delta.refuses() {return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, &plan)));}@@ -8306,19 +8482,23 @@ fn run_index_report_inner(println!("{}", serde_json::to_string_pretty(&plan)?);pr.note("(dry run — nothing indexed)");// As above: the count is capped, so the completeness flag ships with it.+ let mut dry_done = vec![+ ("files", files.len() as u64),+ ("ignored_files", ignore_report.files_skipped),+ ("ignored_dirs", ignore_report.dirs_pruned),+ (+ "ignored_files_in_pruned_dirs",+ ignore_report.files_inside_pruned_dirs,+ ),+ ];+ if dataless_skipped > 0 {+ dry_done.push(("skipped_dataless", dataless_skipped));+ }pr.finish_with_flags(true,0,"dry-run",- &[- ("files", files.len() as u64),- ("ignored_files", ignore_report.files_skipped),- ("ignored_dirs", ignore_report.dirs_pruned),- (- "ignored_files_in_pruned_dirs",- ignore_report.files_inside_pruned_dirs,- ),- ],+ &dry_done,&[("ignored_files_in_pruned_dirs_exact",ignore_report.files_inside_pruned_dirs_is_exact(),@@ -8364,7 +8544,7 @@ fn run_index_report_inner()?;let summary = finish_generated_run(&es, &mut journal, &cfg)?;let code = generated_exit_code(&summary);- finish_generated_progress(&pr, code, &summary);+ finish_generated_progress(&pr, code, &summary, dataless_skipped);return Ok((code, Some(summary)));}@@ -10753,6 +10933,9 @@ fn run_index_report_inner(if skipped_appeared > 0 {done_fields.push(("skipped_appeared", skipped_appeared as u64));}+ if dataless_skipped > 0 {+ done_fields.push(("skipped_dataless", dataless_skipped));+ }pr.finish(true,code,@@ -11704,6 +11887,25 @@ mod inventory_delta_tests {assert!(UnsupportedInventoryDelta::between(nowhere(), &[], &[], &plan).refuses());}+ /// #2058: a dataless placeholder left out of the hash is still on disk.+ /// Omitting it from the inventory would classify the previous group as an+ /// exclusion, and the exclusion sweep would delete documents that were+ /// indexed the last time the bytes were local.+ #[test]+ fn a_dataless_placeholder_kept_out_of_the_hash_is_not_an_exclusion() {+ let mut plan = Plan::default();+ plan.files.insert("cloud".into(), assignment("cloud.txt"));+ let present = [file("cloud.txt")];+ let delta = inventory_delta(nowhere(), &[], &[], &plan, &present);+ assert!(delta.deleted_content_groups.is_empty());+ assert!(delta.excluded_content_groups.is_empty());+ assert!(!delta.refuses());+ assert!(UnsupportedInventoryDelta::between(nowhere(), &[], &[], &plan).refuses());+ let empty = inventory_delta(nowhere(), &[], &[], &plan, &[]);+ assert!(empty.refuses());+ assert_eq!(empty.deleted_content_groups.len(), 1);+ }+#[test]fn classifier_sorts_added_and_vanished_groups() {let mut plan = Plan::default();@@ -13050,6 +13252,7 @@ mod code_coverage_tests {"code_files_indexed": 0,"code_files_junked": 3,}),+ 0,);let text = captured(&buffer);let done = text@@ -13092,15 +13295,47 @@ mod code_coverage_tests {"code_files_indexed": 3,"code_files_junked": 0,}),+ 0,);let text = captured(&buffer);assert!(text.contains("code_files=3 code_files_indexed=3 code_files_junked=0"),"{text}");+ assert!(+ !text.contains("skipped_dataless"),+ "a run that skipped nothing keeps the terminal line it had: {text}"+ );assert!(!text.contains("warning:"), "{text}");}+ #[test]+ fn skipped_dataless_placeholders_are_counted_on_the_terminal_line() {+ let (pr, buffer) = progress::Progress::capture(+ progress::Surface::Plain,+ std::time::Duration::from_secs(3600),+ );+ finish_generated_progress(+ &pr,+ 0,+ &json!({+ "files_indexed": 1,+ "records_total": 1,+ "generation": 2,+ "code_files": 0,+ "code_files_indexed": 0,+ "code_files_junked": 0,+ }),+ 3,+ );+ let text = captured(&buffer);+ let done = text+ .lines()+ .find(|line| line.starts_with("xerj-done "))+ .unwrap_or_else(|| panic!("{text}"));+ assert!(done.contains("skipped_dataless=3"), "{done}");+ }+/// A corpus with no source code in it at all must not start warning, and a/// gzipped source file is still a source file.#[test]diff --git a/engine/crates/xerj-autoindex/src/progress.rs b/engine/crates/xerj-autoindex/src/progress.rsindex 33fb119b0..41b6ebc33 100644--- a/engine/crates/xerj-autoindex/src/progress.rs+++ b/engine/crates/xerj-autoindex/src/progress.rs@@ -97,6 +97,15 @@ const ETA_GOOD_FRACTION: f64 = 0.10;const ETA_MAX_STEP: f64 = 0.20;/// Name at most this many still-running files when the tail goes quiet.const STRAGGLER_MAX: usize = 3;+/// How long one hash read may sit in flight before its path is named, even+/// while the pool still holds more than [`STRAGGLER_MAX`] files (#2058).+///+/// A dataless placeholder blocks inside a single `read`. The tail rule above+/// stays silent until every other worker has finished, so a wide hash reports+/// a stall with no path. A few seconds is long enough that an ordinary file+/// does not flicker onto the line, and short enough that the next progress+/// tick (default 5s) can say which file has not returned.+const HASH_BLOCKING_AFTER: Duration = Duration::from_secs(5);/// Fallback terminal width when `COLUMNS` is unset.const DEFAULT_COLUMNS: usize = 100;/// Cells in the drawn bar on the stream surface.@@ -338,7 +347,9 @@ struct State {last_advance_at: Instant,rate: Option<f64>,shown_eta: Option<f64>,- in_flight: Vec<(u64, String, u64)>,+ /// `(seq, path, bytes, started)`. `started` is when the file was registered,+ /// which is what the hash-phase blocker rule compares against.+ in_flight: Vec<(u64, String, u64, Instant)>,next_seq: u64,tty_width: usize,/// When the last `xerj-bar` display line was written; `None` before the@@ -499,16 +510,7 @@ impl Progress {/// someone else's tree, and it is re-rendered on every tick until the file/// finishes. It is [`sanitize`]d here, once, on the way in.pub fn file(&self, rel: &str, bytes: u64) -> FileGuard<'_> {- let seq = if self.enabled() {- let safe = sanitize(rel, SAFE_PATH_MAX);- let mut state = self.state.lock().unwrap();- let seq = state.next_seq;- state.next_seq += 1;- state.in_flight.push((seq, safe, bytes));- seq- } else {- 0- };+ let seq = self.begin_file(rel, bytes);FileGuard {progress: self,seq,@@ -517,6 +519,30 @@ impl Progress {}}+ /// Register a file as in flight without crediting it on drop.+ ///+ /// Hashing credits bytes from its own completion callback. Using [`Self::file`]+ /// there would count every file twice. This guard only keeps the path on+ /// the in-flight list so a read that does not return can be named.+ pub fn watch_file(&self, rel: &str, bytes: u64) -> WatchGuard<'_> {+ WatchGuard {+ progress: self,+ seq: self.begin_file(rel, bytes),+ }+ }++ fn begin_file(&self, rel: &str, bytes: u64) -> u64 {+ if !self.enabled() {+ return 0;+ }+ let safe = sanitize(rel, SAFE_PATH_MAX);+ let mut state = self.state.lock().unwrap();+ let seq = state.next_seq;+ state.next_seq += 1;+ state.in_flight.push((seq, safe, bytes, Instant::now()));+ seq+ }+/// `bytes` is what is still UNcredited for this file — all of it unless/// the guard reported positions through `FileGuard::advance_to`. The item/// is counted here either way, including for a file whose every byte was@@ -526,7 +552,7 @@ impl Progress {let mut state = self.state.lock().unwrap();state.in_flight- .retain(|(candidate, _, _)| *candidate != seq);+ .retain(|(candidate, _, _, _)| *candidate != seq);}self.item_done(bytes);}@@ -771,13 +797,30 @@ impl Progress {// Name the tail. Once the queue is nearly drained the run looks hung// while one worker chews through the biggest file in the corpus.+ //+ // The hash phase also names a single file that has been in flight for+ // longer than [`HASH_BLOCKING_AFTER`], even when more than+ // [`STRAGGLER_MAX`] files are still open. A dataless placeholder blocks+ // one worker inside `read` while the rest of the pool is also busy, so+ // the tail rule never fires and the stall has no path (#2058). Other+ // phases keep the tail-only rule.let mut waiting_on = Vec::new();if !state.in_flight.is_empty() && state.in_flight.len() <= STRAGGLER_MAX {waiting_on = state.in_flight.iter()- .map(|(_, rel, bytes)| (rel.clone(), *bytes))+ .map(|(_, rel, bytes, _)| (rel.clone(), *bytes)).collect();+ } else if state.phase == "hash" {+ if let Some((_, rel, bytes, started)) = state+ .in_flight+ .iter()+ .min_by_key(|(_, _, _, started)| *started)+ {+ if now.duration_since(*started) >= HASH_BLOCKING_AFTER {+ waiting_on.push((rel.clone(), *bytes));+ }+ }}Snapshot {@@ -997,6 +1040,25 @@ impl Drop for FileGuard<'_> {}}+/// Removes its file from the in-flight list on drop and credits nothing.+///+/// See [`Progress::watch_file`].+pub struct WatchGuard<'a> {+ progress: &'a Progress,+ seq: u64,+}++impl Drop for WatchGuard<'_> {+ fn drop(&mut self) {+ if self.progress.enabled() {+ let mut state = self.progress.state.lock().unwrap();+ state+ .in_flight+ .retain(|(candidate, _, _, _)| *candidate != self.seq);+ }+ }+}+/// Owns the ticker thread. Dropping it stops the thread and — if the run never/// called `finish` — closes the stream with an honest `ok=false` line.pub struct Ticker {@@ -2272,6 +2334,108 @@ mod tests {drop(big);}+ /// #2058: a hash read that has not returned for a few seconds is named even+ /// when the pool still holds more files than the tail rule will list. A+ /// scan or index phase with the same shape stays unnamed — that rule did+ /// not change.+ #[test]+ fn a_hash_read_blocking_for_a_few_seconds_names_its_path() {+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));+ progress.phase("hash", 8, 80);+ let _others: Vec<_> = (0..4)+ .map(|i| progress.watch_file(&format!("local-{i}.txt"), 10))+ .collect();+ let blocked = progress.watch_file("cloud/Notes.txt", 10);+ {+ let mut state = progress.state.lock().unwrap();+ for entry in &mut state.in_flight {+ if entry.1 == "cloud/Notes.txt" {+ entry.3 = Instant::now() - HASH_BLOCKING_AFTER - Duration::from_secs(1);+ }+ }+ }+ progress.tick();+ let line = captured(&buffer)+ .lines()+ .rev()+ .find(|line| line.starts_with("xerj-progress "))+ .unwrap()+ .to_string();+ assert!(+ line.contains("waiting_on=cloud/Notes.txt"),+ "the file that has not returned is the one to name: {line}"+ );+ drop(blocked);++ // The same pile of in-flight files during index is not a hash read.+ // Naming them would change every busy scan.+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));+ progress.phase("index", 8, 80);+ let _busy: Vec<_> = (0..5)+ .map(|i| progress.watch_file(&format!("doc-{i}.txt"), 10))+ .collect();+ {+ let mut state = progress.state.lock().unwrap();+ state.in_flight[0].3 = Instant::now() - Duration::from_secs(30);+ }+ progress.tick();+ let line = captured(&buffer)+ .lines()+ .rev()+ .find(|line| line.starts_with("xerj-progress "))+ .unwrap()+ .to_string();+ assert!(+ !line.contains("waiting_on="),+ "only the hash phase names a blocker before the tail: {line}"+ );+ }++ /// Young hash reads are not blockers. Five files that just started must+ /// not grow a `waiting_on` the tail rule would have omitted.+ #[test]+ fn a_busy_hash_does_not_name_a_file_that_just_started() {+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));+ progress.phase("hash", 8, 80);+ let _guards: Vec<_> = (0..5)+ .map(|i| progress.watch_file(&format!("f{i}.txt"), 10))+ .collect();+ progress.tick();+ let line = captured(&buffer)+ .lines()+ .rev()+ .find(|line| line.starts_with("xerj-progress "))+ .unwrap()+ .to_string();+ assert!(!line.contains("waiting_on="), "{line}");+ }++ /// The hash callback credits the bytes. The watch guard must not credit+ /// them again when the read returns.+ #[test]+ fn watch_file_names_the_path_and_credits_nothing() {+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));+ progress.phase("hash", 1, 10);+ let guard = progress.watch_file("a.txt", 10);+ progress.tick();+ assert!(+ captured(&buffer).contains("waiting_on=a.txt"),+ "{}",+ captured(&buffer)+ );+ drop(guard);+ assert_eq!(progress.items_done.load(Ordering::Relaxed), 0);+ assert_eq!(progress.bytes_done.load(Ordering::Relaxed), 0);+ progress.tick();+ let line = captured(&buffer)+ .lines()+ .rev()+ .find(|line| line.starts_with("xerj-progress "))+ .unwrap()+ .to_string();+ assert!(!line.contains("waiting_on="), "{line}");+ }+/// An empty sample must not drag the measured rate down either — the rate/// is a statement about the work that actually completed.#[test]