Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): skip dataless macOS placeholders instead of blocking on them (#2223)

xerj-org299bf5f187cb
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rs
index 26bcad3a1..d13b87d83 100644
--- a/engine/crates/xerj-autoindex/src/content.rs
+++ b/engine/crates/xerj-autoindex/src/content.rs
@@ -45,7 +45,7 @@ pub fn resolve_reporting(
files: Vec<FileEntry>,
hashed: &(dyn Fn(u64) + Sync),
) -> Result<Inventory> {
- resolve_reporting_carried(files, &|_| None, &|_, _, _| {}, hashed)
+ resolve_reporting_carried(files, &|_| None, &|_, _, _| {}, hashed, None)
}
/// As [`resolve_reporting`], with a caller-supplied digest cache.
@@ -67,11 +67,18 @@ pub fn resolve_reporting(
/// carries an assignment forward only on a match, so a stale carried digest
/// looks like "unchanged" — the failure mode is a missed update, not a bad
/// document — and the next event or fingerprint change repairs it.
+///
+/// `blocking`, when set, names each file on the progress line for the duration
+/// of its read. macOS passes the run's progress handle so a read that blocks
+/// on a dataless placeholder says which path (#2058). The guard credits
+/// nothing; `hashed` still does, once, on completion. Other targets ignore the
+/// handle even if it is passed.
pub fn resolve_reporting_carried(
files: Vec<FileEntry>,
carried: &(dyn Fn(&FileEntry) -> Option<String> + Sync),
observed: &(dyn Fn(&FileEntry, &str, bool) + Sync),
hashed: &(dyn Fn(u64) + Sync),
+ blocking: Option<&crate::progress::Progress>,
) -> Result<Inventory> {
// Phase A belongs to the run's scan pool, not to rayon's global pool:
// `--workers` has to bound the CPU-bound phase to mean anything (#240 §2).
@@ -102,7 +109,13 @@ pub fn resolve_reporting_carried(
// would push the percent past 100 and invent a rate.
Some(digest) => (Ok(digest), false),
None => {
- let digest = full_digest(&file.path, file.size);
+ // Held across the read and dropped before the byte
+ // credit, so a blocked `read` is visible and a finished
+ // one is not counted twice.
+ let digest = {
+ let _blocking = crate::dataless::watch_blocking_hash(blocking, file);
+ full_digest(&file.path, file.size)
+ };
hashed(file.size);
(digest, true)
}
diff --git a/engine/crates/xerj-autoindex/src/dataless.rs b/engine/crates/xerj-autoindex/src/dataless.rs
new file mode 100644
index 000000000..eb95a54d3
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/dataless.rs
@@ -0,0 +1,397 @@
+//! macOS File Provider placeholders (iCloud Drive and other cloud volumes).
+//!
+//! A dataless file has a directory entry and a logical size, but its bytes are
+//! not on this machine. `stat` reports that without downloading anything.
+//! `open` / `read` does download, and the call blocks until the transfer
+//! finishes — which is how `xerj autoindex` hashing stalled with no path named
+//! (#2058, field reports #2042 and #2051).
+//!
+//! The flag is Apple's `SF_DATALESS` (`0x40000000` in `bsd/sys/stat.h`, "file
+//! is dataless object"). It is synthetic and read-only, so a test cannot set
+//! it with `chflags`. libc 0.2.191 exposes `stat.st_flags` on macOS and does
+//! not export the constant; the value below is the header's. Detection is
+//! macOS-only. On every other target [`split`] returns the input unchanged and
+//! does not stat.
+//!
+//! Linux compiles the accounting so the unit tests run in CI. Production on
+//! that target calls [`Split::unchanged`] and does not stat; the rest of this
+//! module is macOS-only and is allowed to look unused there.
+
+#![cfg_attr(not(any(target_os = "macos", test)), allow(dead_code))]
+
+use crate::state::Plan;
+use crate::walk::FileEntry;
+use std::collections::HashMap;
+use std::path::Path;
+
+/// Apple `SF_DATALESS` (`bsd/sys/stat.h`).
+pub(crate) const SF_DATALESS: u32 = 0x4000_0000;
+
+/// Skip reason recorded in the run summary. Stable for anything that counts
+/// skips by name.
+pub(crate) const SKIP_REASON: &str = "dataless-placeholder";
+
+/// `st_flags` has `SF_DATALESS` set. Other flags do not matter: a placeholder
+/// can also be hidden or compressed.
+pub(crate) fn flags_indicate_dataless(flags: u32) -> bool {
+ flags & SF_DATALESS != 0
+}
+
+/// Human line for files left out of this run. `None` when nothing was skipped,
+/// so a run with no placeholders prints the lines it always printed.
+pub(crate) fn skip_note(count: u64) -> Option<String> {
+ if count == 0 {
+ return None;
+ }
+ let files = if count == 1 { "file" } else { "files" };
+ Some(format!(
+ "skipped {count} {files} — {SKIP_REASON} (macOS File Provider / iCloud placeholder; \
+ the bytes are not local, and reading them blocks on a download, so they were not \
+ hashed or indexed). Materialize them and re-run to include them"
+ ))
+}
+
+/// Human line for placeholders this run did not read because a previous pass
+/// already hashed them. They stay in the inventory under that digest, so an
+/// incremental re-run does not treat an evicted file as deleted.
+pub(crate) fn carry_note(count: u64) -> Option<String> {
+ if count == 0 {
+ return None;
+ }
+ let files = if count == 1 { "file" } else { "files" };
+ Some(format!(
+ "left {count} dataless placeholder {files} on their previous content \
+ ({SKIP_REASON}); not re-read, so the indexed copy stays"
+ ))
+}
+
+/// What hashing should do with one walked tree.
+pub(crate) struct Split {
+ /// Files to resolve. Dataless files with no prior digest are absent.
+ pub files: Vec<FileEntry>,
+ /// `path_id` → digest for dataless files that must not be opened.
+ pub carry: HashMap<String, String>,
+ /// Dataless files removed from [`Self::files`].
+ pub skipped_files: Vec<FileEntry>,
+ /// Files [`Self::files`] that will actually be read. Carried placeholders
+ /// are not included. The hash phase recomputes this after the budget
+ /// filter, from the files still admitted, so production does not read
+ /// these fields — the unit tests do.
+ #[cfg_attr(all(target_os = "macos", not(test)), allow(dead_code))]
+ pub hash_files: u64,
+ #[cfg_attr(all(target_os = "macos", not(test)), allow(dead_code))]
+ pub hash_bytes: u64,
+}
+
+impl Split {
+ /// No stat, no filtering. The non-macOS result.
+ ///
+ /// macOS goes through `split_with` instead, so this helper is unused there.
+ #[cfg_attr(target_os = "macos", allow(dead_code))]
+ pub(crate) fn unchanged(files: Vec<FileEntry>) -> Self {
+ let hash_files = files.len() as u64;
+ let hash_bytes = files.iter().map(|file| file.size).sum();
+ Self {
+ files,
+ carry: HashMap::new(),
+ skipped_files: Vec::new(),
+ hash_files,
+ hash_bytes,
+ }
+ }
+}
+
+/// Split `files` into "hash", "carry the previous digest", and "skip".
+///
+/// On macOS a file whose `st_flags` include `SF_DATALESS` is not opened. If
+/// `prior` has a digest for its `path_id`, that digest is carried and the file
+/// stays in the inventory (an evicted file is not a deletion). Otherwise the
+/// file is skipped and counted. A failed `stat` is not dataless: the later
+/// open reports the real error instead of hiding a readable file.
+///
+/// On every other target this is [`Split::unchanged`].
+pub(crate) fn split(files: Vec<FileEntry>, prior: &HashMap<String, String>) -> Split {
+ #[cfg(target_os = "macos")]
+ {
+ split_with(files, prior, is_dataless_placeholder)
+ }
+ #[cfg(not(target_os = "macos"))]
+ {
+ let _ = prior;
+ Split::unchanged(files)
+ }
+}
+
+/// [`split`] with the placeholder test injected. The production macOS path
+/// passes [`is_dataless_placeholder`]; tests pass a predicate so the accounting
+/// does not need a real iCloud file.
+fn split_with(
+ files: Vec<FileEntry>,
+ prior: &HashMap<String, String>,
+ is_dataless: impl Fn(&Path) -> bool,
+) -> Split {
+ let mut kept = Vec::with_capacity(files.len());
+ let mut carry = HashMap::new();
+ let mut skipped_files = Vec::new();
+ let mut hash_files = 0u64;
+ let mut hash_bytes = 0u64;
+ for file in files {
+ if is_dataless(&file.path) {
+ if let Some(digest) = prior.get(&file.rel_id) {
+ carry.insert(file.rel_id.clone(), digest.clone());
+ kept.push(file);
+ } else {
+ skipped_files.push(file);
+ }
+ } else {
+ hash_files += 1;
+ hash_bytes += file.size;
+ kept.push(file);
+ }
+ }
+ Split {
+ files: kept,
+ carry,
+ skipped_files,
+ hash_files,
+ hash_bytes,
+ }
+}
+
+/// path_id → full-content digest for every file the previous plan actually
+/// hashed. Empty path ids and assignments with no digest are omitted: there
+/// is nothing safe to carry, and a placeholder in that state is skipped.
+pub(crate) fn prior_digests(plan: Option<&Plan>) -> HashMap<String, String> {
+ let Some(plan) = plan else {
+ return HashMap::new();
+ };
+ let mut out = HashMap::new();
+ for assignment in plan.files.values() {
+ remember_digest(
+ &mut out,
+ &assignment.path_id,
+ assignment.content_digest.as_deref(),
+ );
+ }
+ for duplicate in &plan.duplicate_files {
+ let digest = plan
+ .files
+ .get(&duplicate.file_key)
+ .and_then(|assignment| assignment.content_digest.as_deref());
+ remember_digest(&mut out, &duplicate.path_id, digest);
+ }
+ out
+}
+
+fn remember_digest(out: &mut HashMap<String, String>, path_id: &str, digest: Option<&str>) {
+ if path_id.is_empty() {
+ return;
+ }
+ let Some(digest) = digest.filter(|digest| !digest.is_empty()) else {
+ return;
+ };
+ out.entry(path_id.to_string())
+ .or_insert_with(|| digest.to_string());
+}
+
+/// Register `file` on the progress in-flight list for the duration of the
+/// guard, on macOS only.
+///
+/// Hashing reads the whole file in one worker. A placeholder the pre-pass
+/// missed (it became dataless between `stat` and `open`, or `stat` failed and
+/// the open blocks) otherwise stalls with `in_flight` empty, so the progress
+/// line cannot name a path. The guard does not credit bytes — the completion
+/// callback still does that, once, when the read returns.
+///
+/// Other targets return `None` and do not touch progress. A hash phase there
+/// keeps the line it had before #2058.
+pub(crate) fn watch_blocking_hash<'a>(
+ progress: Option<&'a crate::progress::Progress>,
+ file: &FileEntry,
+) -> Option<crate::progress::WatchGuard<'a>> {
+ #[cfg(target_os = "macos")]
+ {
+ progress.map(|progress| progress.watch_file(&file.rel, file.size))
+ }
+ #[cfg(not(target_os = "macos"))]
+ {
+ let _ = (progress, file);
+ None
+ }
+}
+
+#[cfg(target_os = "macos")]
+fn is_dataless_placeholder(path: &Path) -> bool {
+ flags_of(path).is_some_and(flags_indicate_dataless)
+}
+
+#[cfg(target_os = "macos")]
+fn flags_of(path: &Path) -> Option<u32> {
+ use std::os::unix::ffi::OsStrExt;
+ let bytes = path.as_os_str().as_bytes();
+ // `stat` is a C string. An interior NUL cannot be passed through; treat
+ // that as "not proven dataless" and let the open fail on its own.
+ if bytes.contains(&0) {
+ return None;
+ }
+ let mut c_path = Vec::with_capacity(bytes.len() + 1);
+ c_path.extend_from_slice(bytes);
+ c_path.push(0);
+ let mut st = std::mem::MaybeUninit::<libc::stat>::uninit();
+ // SAFETY: `c_path` is a NUL-terminated path this process may stat, and
+ // `st` is written by `stat` on success before it is read.
+ let rc = unsafe { libc::stat(c_path.as_ptr().cast(), st.as_mut_ptr()) };
+ if rc != 0 {
+ return None;
+ }
+ let st = unsafe { st.assume_init() };
+ Some(st.st_flags)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::state::{DuplicateFile, FileAssignment};
+ use std::fs;
+
+ fn assignment(path_id: &str, digest: Option<&str>) -> FileAssignment {
+ FileAssignment {
+ rel: path_id.to_string(),
+ path_id: path_id.to_string(),
+ is_symlink: Some(false),
+ family: "txt".into(),
+ gzip: false,
+ content_digest: digest.map(str::to_string),
+ assignments: Vec::new(),
+ as_document: false,
+ }
+ }
+
+ fn entry(rel_id: &str, size: u64) -> FileEntry {
+ FileEntry {
+ path: Path::new(rel_id).to_path_buf(),
+ rel: rel_id.to_string(),
+ rel_id: rel_id.to_string(),
+ is_symlink: false,
+ size,
+ }
+ }
+
+ #[test]
+ fn sf_dataless_is_the_apple_bit_and_other_flags_are_not() {
+ assert!(flags_indicate_dataless(SF_DATALESS));
+ assert!(flags_indicate_dataless(SF_DATALESS | 0x0002_0000));
+ assert!(!flags_indicate_dataless(0));
+ // UF_HIDDEN, SF_IMMUTABLE, and the other synthetic high bit.
+ assert!(!flags_indicate_dataless(0x0000_8000));
+ assert!(!flags_indicate_dataless(0x0002_0000));
+ assert!(!flags_indicate_dataless(0x8000_0000));
+ }
+
+ #[test]
+ fn skip_and_carry_notes_name_the_reason_and_the_count() {
+ assert!(skip_note(0).is_none());
+ let one = skip_note(1).unwrap();
+ assert!(
+ one.contains("skipped 1 file — dataless-placeholder"),
+ "{one}"
+ );
+ assert!(!one.contains("files"), "{one}");
+ let two = skip_note(2).unwrap();
+ assert!(
+ two.contains("skipped 2 files — dataless-placeholder"),
+ "{two}"
+ );
+ assert!(carry_note(0).is_none());
+ let carried = carry_note(1).unwrap();
+ assert!(
+ carried.contains("1 dataless placeholder file") && carried.contains(SKIP_REASON),
+ "{carried}"
+ );
+ }
+
+ #[test]
+ fn a_placeholder_with_no_prior_digest_is_skipped_and_counted() {
+ let prior = HashMap::from([("kept-cloud".to_string(), "axf2-prior".to_string())]);
+ let split = split_with(
+ vec![
+ entry("local", 10),
+ entry("kept-cloud", 20),
+ entry("new-cloud", 30),
+ ],
+ &prior,
+ |path| path.to_str().is_some_and(|path| path.ends_with("cloud")),
+ );
+ let rels: Vec<&str> = split.files.iter().map(|file| file.rel.as_str()).collect();
+ assert_eq!(rels, ["local", "kept-cloud"]);
+ assert_eq!(
+ split.carry.get("kept-cloud").map(String::as_str),
+ Some("axf2-prior")
+ );
+ assert!(!split.carry.contains_key("local"));
+ assert_eq!(split.skipped_files.len(), 1);
+ assert_eq!(split.skipped_files[0].rel, "new-cloud");
+ // The hash denominator is the file that will be read, not the one
+ // whose digest is carried and not the one that was skipped.
+ assert_eq!(split.hash_files, 1);
+ assert_eq!(split.hash_bytes, 10);
+ }
+
+ #[test]
+ fn a_non_placeholder_tree_is_unchanged() {
+ let files = vec![entry("a", 4), entry("b", 6)];
+ let split = split_with(files, &HashMap::new(), |_| false);
+ assert!(split.carry.is_empty());
+ assert!(split.skipped_files.is_empty());
+ assert_eq!(split.hash_files, 2);
+ assert_eq!(split.hash_bytes, 10);
+ assert_eq!(split.files.len(), 2);
+ }
+
+ #[test]
+ fn prior_digests_keep_hashed_paths_and_alias_targets() {
+ let mut plan = Plan::default();
+ plan.files
+ .insert("axf2-body".into(), assignment("unix:61", Some("axf2-body")));
+ plan.files
+ .insert("legacy".into(), assignment("unix:62", None));
+ plan.files
+ .insert("blank".into(), assignment("", Some("axf2-blank")));
+ plan.duplicate_files.push(DuplicateFile {
+ file_key: "axf2-body".into(),
+ rel: "alias".into(),
+ path_id: "unix:63".into(),
+ is_symlink: Some(false),
+ duplicate_of: "body".into(),
+ bytes: 4,
+ });
+ let prior = prior_digests(Some(&plan));
+ assert_eq!(prior.get("unix:61").map(String::as_str), Some("axf2-body"));
+ assert_eq!(prior.get("unix:63").map(String::as_str), Some("axf2-body"));
+ assert!(!prior.contains_key("unix:62"), "no digest to carry");
+ assert!(!prior.contains_key(""), "an empty path id is not a key");
+ assert!(prior_digests(None).is_empty());
+ }
+
+ /// A real local file is not a placeholder. On macOS this stats it; elsewhere
+ /// [`split`] does not stat and still keeps the file.
+ #[test]
+ fn a_local_file_is_not_a_dataless_placeholder() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("a.txt");
+ fs::write(&path, b"hello").unwrap();
+ let file = FileEntry {
+ path,
+ rel: "a.txt".into(),
+ rel_id: "unix:61".into(),
+ is_symlink: false,
+ size: 5,
+ };
+ let split = split(vec![file], &HashMap::new());
+ assert!(split.skipped_files.is_empty(), "{:?}", split.skipped_files);
+ assert!(split.carry.is_empty());
+ assert_eq!(split.files.len(), 1);
+ assert_eq!(split.hash_files, 1);
+ assert_eq!(split.hash_bytes, 5);
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 46b4b1f75..fb041cd9f 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -9,6 +9,7 @@ pub mod cli;
pub mod coerce;
mod content;
pub mod correlate;
+mod dataless;
pub mod dataset;
pub mod def;
pub mod detect;
@@ -1579,7 +1580,7 @@ pub(crate) fn outcome_headline(
/// commit — the one place the two landed changes could contradict each other.
/// Exit 3 is success ("completed, some input was unusable"), so it is spelled
/// out in words rather than left as a bare number.
-fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value) {
+fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value, skipped_dataless: u64) {
let count = |field: &str| summary.get(field).and_then(Value::as_u64).unwrap_or(0);
// Read back from the committed generation's own run document, exactly like
// every other number on this line — a resumed or no-op run therefore
@@ -1617,6 +1618,11 @@ fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value) {
if count("bulk_splits") > 0 {
extra.push(("bulk_splits", count("bulk_splits")));
}
+ // #2058: present only when this run left out dataless placeholders, so a
+ // corpus with none prints the line it always did.
+ if skipped_dataless > 0 {
+ extra.push(("skipped_dataless", skipped_dataless));
+ }
extra.extend(coverage.fields());
pr.finish(
true,
@@ -5531,6 +5537,76 @@ impl UnsupportedInventoryDelta {
}
}
+/// [`UnsupportedInventoryDelta::between`], plus dataless placeholders this run
+/// deliberately did not hash (#2058).
+///
+/// Those files are still on disk. Leaving them out of `files` makes the delta
+/// call them exclusions, and the exclusion sweep deletes documents that were
+/// indexed the last time the bytes were local. An empty `dataless_still_present`
+/// takes the original path and clones nothing.
+fn inventory_delta(
+ root: &Path,
+ files: &[walk::FileEntry],
+ keys: &[String],
+ plan: &Plan,
+ dataless_still_present: &[walk::FileEntry],
+) -> UnsupportedInventoryDelta {
+ if dataless_still_present.is_empty() {
+ return UnsupportedInventoryDelta::between(root, files, keys, plan);
+ }
+ let mut files = files.to_vec();
+ let mut keys = keys.to_vec();
+ for file in dataless_still_present {
+ files.push(file.clone());
+ keys.push(String::new());
+ }
+ UnsupportedInventoryDelta::between(root, &files, &keys, plan)
+}
+
+/// Hash-phase denominator on macOS. Carried placeholders are in `files` but
+/// are not read, so they are not part of the work the bar describes.
+#[cfg(target_os = "macos")]
+fn hash_phase_totals(
+ files: &[walk::FileEntry],
+ carry: &std::collections::HashMap<String, String>,
+ all_bytes: u64,
+) -> (u64, u64) {
+ if carry.is_empty() {
+ return (files.len() as u64, all_bytes);
+ }
+ let mut count = 0u64;
+ let mut bytes = 0u64;
+ for file in files {
+ if !carry.contains_key(&file.rel_id) {
+ count += 1;
+ bytes += file.size;
+ }
+ }
+ (count, bytes)
+}
+
+/// Watch-pass hash totals, minus placeholders whose previous digest is carried
+/// instead of read. An empty carry map returns the watch plan's own totals.
+#[cfg(target_os = "macos")]
+fn watch_hash_totals(
+ plan: &watch::PassPlan,
+ files: &[walk::FileEntry],
+ carry: &std::collections::HashMap<String, String>,
+) -> (u64, u64) {
+ if carry.is_empty() {
+ return (plan.hash_files, plan.hash_bytes);
+ }
+ let mut hash_files = plan.hash_files;
+ let mut hash_bytes = plan.hash_bytes;
+ for file in files {
+ if carry.contains_key(&file.rel_id) && plan.carried(file).is_none() {
+ hash_files = hash_files.saturating_sub(1);
+ hash_bytes = hash_bytes.saturating_sub(file.size);
+ }
+ }
+ (hash_files, hash_bytes)
+}
+
fn alias_keys_to_reindex(
previous: &[state::DuplicateFile],
current: &[state::DuplicateFile],
@@ -7179,7 +7255,7 @@ fn run_index_report_inner(
pr.note("autoindex: resumed and committed pending corpus generation from durable source");
let summary = finish_generated_run(&es, &mut journal, &cfg)?;
let code = generated_exit_code(&summary);
- finish_generated_progress(&pr, code, &summary);
+ finish_generated_progress(&pr, code, &summary, 0);
return Ok((code, Some(summary)));
}
// #585 case 2: a --no-graph generation's genesis bootstrap (sync_bootstrap
@@ -7298,11 +7374,32 @@ fn run_index_report_inner(
cfg.exclude_rels.join(", ")
));
}
- let (mut discovered_files, data_files, data_bytes) = if cfg.skip_corpus_data {
+ let (discovered_files, data_files, data_bytes) = if cfg.skip_corpus_data {
without_corpus_data_files(discovered_files)
} else {
(discovered_files, 0, 0)
};
+ // #2058: a dataless File Provider placeholder (iCloud and other cloud
+ // volumes) has a directory entry but no local bytes. Hashing it blocks
+ // inside `read` until the download finishes, and the progress line used
+ // to name nothing. `stat` sees `SF_DATALESS` without downloading. On
+ // macOS those files are skipped, or carried from the previous digest when
+ // one exists so an evicted file is not treated as deleted. Every other
+ // target gets the walked list back unchanged and does not stat.
+ #[cfg(target_os = "macos")]
+ let dataless_split = dataless::split(
+ discovered_files,
+ &dataless::prior_digests(preflight.plan.as_ref()),
+ );
+ #[cfg(not(target_os = "macos"))]
+ let dataless_split = dataless::Split::unchanged(discovered_files);
+ let mut discovered_files = dataless_split.files;
+ let dataless_skipped_files = dataless_split.skipped_files;
+ let dataless_skipped = dataless_skipped_files.len() as u64;
+ #[cfg(target_os = "macos")]
+ let dataless_carry = dataless_split.carry;
+ #[cfg(not(target_os = "macos"))]
+ let _ = dataless_split.carry;
if data_files > 0 {
let size = if data_bytes >= 1 << 20 {
format!("{:.1} MB", data_bytes as f64 / (1u64 << 20) as f64)
@@ -7321,6 +7418,13 @@ fn run_index_report_inner(
discovered_bytes / (1 << 20),
root_str
));
+ if let Some(line) = dataless::skip_note(dataless_skipped) {
+ pr.note(&format!("autoindex: {line}"));
+ }
+ #[cfg(target_os = "macos")]
+ if let Some(line) = dataless::carry_note(dataless_carry.len() as u64) {
+ pr.note(&format!("autoindex: {line}"));
+ }
// "Where did my files go?" is answered here, on every run — the rules are
// named, not just the totals (#276).
for line in ignore_report.summary_lines() {
@@ -7342,20 +7446,24 @@ fn run_index_report_inner(
pr.note(
"(walk-only — stopped before hashing; file contents were not hashed and nothing was indexed)",
);
+ let mut walk_done = vec![
+ ("files", summary.files),
+ ("bytes", summary.bytes),
+ ("ignored_files", ignore_report.files_skipped),
+ ("ignored_dirs", ignore_report.dirs_pruned),
+ (
+ "ignored_files_in_pruned_dirs",
+ ignore_report.files_inside_pruned_dirs,
+ ),
+ ];
+ if dataless_skipped > 0 {
+ walk_done.push(("skipped_dataless", dataless_skipped));
+ }
pr.finish_with_flags(
true,
0,
"walk-only",
- &[
- ("files", summary.files),
- ("bytes", summary.bytes),
- ("ignored_files", ignore_report.files_skipped),
- ("ignored_dirs", ignore_report.dirs_pruned),
- (
- "ignored_files_in_pruned_dirs",
- ignore_report.files_inside_pruned_dirs,
- ),
- ],
+ &walk_done,
&[(
"ignored_files_in_pruned_dirs_exact",
ignore_report.files_inside_pruned_dirs_is_exact(),
@@ -7367,8 +7475,22 @@ fn run_index_report_inner(
// state: with a journal present, zero files is a deletion of the whole
// corpus and has to be reconciled, not shrugged off.
if discovered_files.is_empty() && !preflight.journal_exists {
- println!("no files found under {root_str}");
- pr.finish(true, 0, "no-files", &[]);
+ if dataless_skipped > 0 {
+ println!(
+ "no indexable files under {root_str} ({dataless_skipped} dataless placeholder{} \
+ skipped)",
+ if dataless_skipped == 1 { "" } else { "s" }
+ );
+ pr.finish(
+ true,
+ 0,
+ "no-files",
+ &[("skipped_dataless", dataless_skipped)],
+ );
+ } else {
+ println!("no files found under {root_str}");
+ pr.finish(true, 0, "no-files", &[]);
+ }
return Ok((0, None));
}
// Count, estimate, and — when the default plan does not fit the disk
@@ -7508,8 +7630,28 @@ fn run_index_report_inner(
// A plain run hashes the whole corpus, every time, on purpose: see
// `content::resolve_reporting`.
None => {
- pr.phase("hash", discovered_files.len() as u64, discovered_bytes);
- content::resolve_reporting(discovered_files, &|bytes| pr.item_done(bytes))?
+ // macOS names each file for the duration of its read and does not
+ // read a dataless placeholder (#2058). Other targets hash the way
+ // they always have: every admitted byte, no in-flight registration
+ // on this phase.
+ #[cfg(target_os = "macos")]
+ {
+ let (hash_files, hash_bytes) =
+ hash_phase_totals(&discovered_files, &dataless_carry, discovered_bytes);
+ pr.phase("hash", hash_files, hash_bytes);
+ content::resolve_reporting_carried(
+ discovered_files,
+ &|entry| dataless_carry.get(&entry.rel_id).cloned(),
+ &|_, _, _| {},
+ &|bytes| pr.item_done(bytes),
+ Some(pr.as_ref()),
+ )?
+ }
+ #[cfg(not(target_os = "macos"))]
+ {
+ pr.phase("hash", discovered_files.len() as u64, discovered_bytes);
+ content::resolve_reporting(discovered_files, &|bytes| pr.item_done(bytes))?
+ }
}
// A `--watch` pass hashes what the watcher could not prove unchanged.
// The phase's totals are the files this pass will actually read, so the
@@ -7517,20 +7659,35 @@ fn run_index_report_inner(
// a full run would have done.
Some(pass) => {
let plan = pass.plan(&discovered_files);
+ #[cfg(target_os = "macos")]
+ let (hash_files, hash_bytes) =
+ watch_hash_totals(&plan, &discovered_files, &dataless_carry);
+ #[cfg(not(target_os = "macos"))]
+ let (hash_files, hash_bytes) = (plan.hash_files, plan.hash_bytes);
let (carried_files, carried_bytes) = (plan.carried_files, plan.carried_bytes);
- pr.phase("hash", plan.hash_files, plan.hash_bytes);
+ pr.phase("hash", hash_files, hash_bytes);
pr.note(&format!(
"autoindex: --watch: re-hashing {} file(s) ({} MB); {carried_files} file(s) \
({} MB) carried from the previous pass",
- plan.hash_files,
- plan.hash_bytes >> 20,
+ hash_files,
+ hash_bytes >> 20,
carried_bytes >> 20,
));
let inventory = content::resolve_reporting_carried(
discovered_files,
- &|entry| plan.carried(entry),
+ &|entry| {
+ #[cfg(target_os = "macos")]
+ if let Some(digest) = dataless_carry.get(&entry.rel_id) {
+ return Some(digest.clone());
+ }
+ plan.carried(entry)
+ },
&|entry, digest, fresh| plan.observe(entry, digest, fresh),
&|bytes| pr.item_done(bytes),
+ #[cfg(target_os = "macos")]
+ Some(pr.as_ref()),
+ #[cfg(not(target_os = "macos"))]
+ None,
)?;
pass.commit(plan);
inventory
@@ -7572,19 +7729,23 @@ fn run_index_report_inner(
));
// `ignored_files_in_pruned_dirs` is budget-capped, so the flag
// saying whether it is a total or a floor travels with it (#279).
+ let mut dry_done = vec![
+ ("files", inventory.files.len() as u64),
+ ("ignored_files", ignore_report.files_skipped),
+ ("ignored_dirs", ignore_report.dirs_pruned),
+ (
+ "ignored_files_in_pruned_dirs",
+ ignore_report.files_inside_pruned_dirs,
+ ),
+ ];
+ if dataless_skipped > 0 {
+ dry_done.push(("skipped_dataless", dataless_skipped));
+ }
pr.finish_with_flags(
true,
0,
"dry-run",
- &[
- ("files", inventory.files.len() as u64),
- ("ignored_files", ignore_report.files_skipped),
- ("ignored_dirs", ignore_report.dirs_pruned),
- (
- "ignored_files_in_pruned_dirs",
- ignore_report.files_inside_pruned_dirs,
- ),
- ],
+ &dry_done,
&[(
"ignored_files_in_pruned_dirs_exact",
ignore_report.files_inside_pruned_dirs_is_exact(),
@@ -7665,7 +7826,7 @@ fn run_index_report_inner(
}
let summary = finish_generated_run(&es, &mut journal, &cfg)?;
let code = generated_exit_code(&summary);
- finish_generated_progress(&pr, code, &summary);
+ finish_generated_progress(&pr, code, &summary, dataless_skipped);
return Ok((code, Some(summary)));
}
begin_non_graph_generation(
@@ -7693,7 +7854,7 @@ fn run_index_report_inner(
)?;
let summary = finish_generated_run(&es, &mut journal, &cfg)?;
let code = generated_exit_code(&summary);
- finish_generated_progress(&pr, code, &summary);
+ finish_generated_progress(&pr, code, &summary, dataless_skipped);
return Ok((code, Some(summary)));
}
if cfg.no_graph && preflight.plan.is_some() && !genesis_recovery {
@@ -7757,11 +7918,12 @@ fn run_index_report_inner(
// `--fresh` is checked here too: discarding the plan does not delete
// the documents already published for a file that is now gone, so a
// removal is unsafe in place whether or not the plan is kept.
- let delta = UnsupportedInventoryDelta::between(
+ let delta = inventory_delta(
&cfg.root,
&inventory.files,
&comparison_keys,
prior_plan,
+ &dataless_skipped_files,
);
if delta.refuses() {
return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, prior_plan)));
@@ -7813,8 +7975,22 @@ fn run_index_report_inner(
sync_executor::gc_snapshots(&state_dir, &journal)?;
let resumed_with_plan = journal.plan.is_some() && !genesis_recovery;
if inventory.files.is_empty() && !resumed_with_plan {
- println!("no files found under {root_str}");
- pr.finish(true, 0, "no-files", &[]);
+ if dataless_skipped > 0 {
+ println!(
+ "no indexable files under {root_str} ({dataless_skipped} dataless placeholder{} \
+ skipped)",
+ if dataless_skipped == 1 { "" } else { "s" }
+ );
+ pr.finish(
+ true,
+ 0,
+ "no-files",
+ &[("skipped_dataless", dataless_skipped)],
+ );
+ } else {
+ println!("no files found under {root_str}");
+ pr.finish(true, 0, "no-files", &[]);
+ }
return Ok((0, None));
}
let run_id = journal.run_id.clone();
@@ -8069,7 +8245,7 @@ fn run_index_report_inner(
// this is set (see `carry_over_unread_mail`).
let mut swept_excluded = false;
if resumed_with_plan {
- let delta = UnsupportedInventoryDelta::between(&cfg.root, &files, &keys, &plan);
+ let delta = inventory_delta(&cfg.root, &files, &keys, &plan, &dataless_skipped_files);
if delta.refuses() {
return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, &plan)));
}
@@ -8306,19 +8482,23 @@ fn run_index_report_inner(
println!("{}", serde_json::to_string_pretty(&plan)?);
pr.note("(dry run — nothing indexed)");
// As above: the count is capped, so the completeness flag ships with it.
+ let mut dry_done = vec![
+ ("files", files.len() as u64),
+ ("ignored_files", ignore_report.files_skipped),
+ ("ignored_dirs", ignore_report.dirs_pruned),
+ (
+ "ignored_files_in_pruned_dirs",
+ ignore_report.files_inside_pruned_dirs,
+ ),
+ ];
+ if dataless_skipped > 0 {
+ dry_done.push(("skipped_dataless", dataless_skipped));
+ }
pr.finish_with_flags(
true,
0,
"dry-run",
- &[
- ("files", files.len() as u64),
- ("ignored_files", ignore_report.files_skipped),
- ("ignored_dirs", ignore_report.dirs_pruned),
- (
- "ignored_files_in_pruned_dirs",
- ignore_report.files_inside_pruned_dirs,
- ),
- ],
+ &dry_done,
&[(
"ignored_files_in_pruned_dirs_exact",
ignore_report.files_inside_pruned_dirs_is_exact(),
@@ -8364,7 +8544,7 @@ fn run_index_report_inner(
)?;
let summary = finish_generated_run(&es, &mut journal, &cfg)?;
let code = generated_exit_code(&summary);
- finish_generated_progress(&pr, code, &summary);
+ finish_generated_progress(&pr, code, &summary, dataless_skipped);
return Ok((code, Some(summary)));
}
@@ -10753,6 +10933,9 @@ fn run_index_report_inner(
if skipped_appeared > 0 {
done_fields.push(("skipped_appeared", skipped_appeared as u64));
}
+ if dataless_skipped > 0 {
+ done_fields.push(("skipped_dataless", dataless_skipped));
+ }
pr.finish(
true,
code,
@@ -11704,6 +11887,25 @@ mod inventory_delta_tests {
assert!(UnsupportedInventoryDelta::between(nowhere(), &[], &[], &plan).refuses());
}
+ /// #2058: a dataless placeholder left out of the hash is still on disk.
+ /// Omitting it from the inventory would classify the previous group as an
+ /// exclusion, and the exclusion sweep would delete documents that were
+ /// indexed the last time the bytes were local.
+ #[test]
+ fn a_dataless_placeholder_kept_out_of_the_hash_is_not_an_exclusion() {
+ let mut plan = Plan::default();
+ plan.files.insert("cloud".into(), assignment("cloud.txt"));
+ let present = [file("cloud.txt")];
+ let delta = inventory_delta(nowhere(), &[], &[], &plan, &present);
+ assert!(delta.deleted_content_groups.is_empty());
+ assert!(delta.excluded_content_groups.is_empty());
+ assert!(!delta.refuses());
+ assert!(UnsupportedInventoryDelta::between(nowhere(), &[], &[], &plan).refuses());
+ let empty = inventory_delta(nowhere(), &[], &[], &plan, &[]);
+ assert!(empty.refuses());
+ assert_eq!(empty.deleted_content_groups.len(), 1);
+ }
+
#[test]
fn classifier_sorts_added_and_vanished_groups() {
let mut plan = Plan::default();
@@ -13050,6 +13252,7 @@ mod code_coverage_tests {
"code_files_indexed": 0,
"code_files_junked": 3,
}),
+ 0,
);
let text = captured(&buffer);
let done = text
@@ -13092,15 +13295,47 @@ mod code_coverage_tests {
"code_files_indexed": 3,
"code_files_junked": 0,
}),
+ 0,
);
let text = captured(&buffer);
assert!(
text.contains("code_files=3 code_files_indexed=3 code_files_junked=0"),
"{text}"
);
+ assert!(
+ !text.contains("skipped_dataless"),
+ "a run that skipped nothing keeps the terminal line it had: {text}"
+ );
assert!(!text.contains("warning:"), "{text}");
}
+ #[test]
+ fn skipped_dataless_placeholders_are_counted_on_the_terminal_line() {
+ let (pr, buffer) = progress::Progress::capture(
+ progress::Surface::Plain,
+ std::time::Duration::from_secs(3600),
+ );
+ finish_generated_progress(
+ &pr,
+ 0,
+ &json!({
+ "files_indexed": 1,
+ "records_total": 1,
+ "generation": 2,
+ "code_files": 0,
+ "code_files_indexed": 0,
+ "code_files_junked": 0,
+ }),
+ 3,
+ );
+ let text = captured(&buffer);
+ let done = text
+ .lines()
+ .find(|line| line.starts_with("xerj-done "))
+ .unwrap_or_else(|| panic!("{text}"));
+ assert!(done.contains("skipped_dataless=3"), "{done}");
+ }
+
/// A corpus with no source code in it at all must not start warning, and a
/// gzipped source file is still a source file.
#[test]
diff --git a/engine/crates/xerj-autoindex/src/progress.rs b/engine/crates/xerj-autoindex/src/progress.rs
index 33fb119b0..41b6ebc33 100644
--- a/engine/crates/xerj-autoindex/src/progress.rs
+++ b/engine/crates/xerj-autoindex/src/progress.rs
@@ -97,6 +97,15 @@ const ETA_GOOD_FRACTION: f64 = 0.10;
const ETA_MAX_STEP: f64 = 0.20;
/// Name at most this many still-running files when the tail goes quiet.
const STRAGGLER_MAX: usize = 3;
+/// How long one hash read may sit in flight before its path is named, even
+/// while the pool still holds more than [`STRAGGLER_MAX`] files (#2058).
+///
+/// A dataless placeholder blocks inside a single `read`. The tail rule above
+/// stays silent until every other worker has finished, so a wide hash reports
+/// a stall with no path. A few seconds is long enough that an ordinary file
+/// does not flicker onto the line, and short enough that the next progress
+/// tick (default 5s) can say which file has not returned.
+const HASH_BLOCKING_AFTER: Duration = Duration::from_secs(5);
/// Fallback terminal width when `COLUMNS` is unset.
const DEFAULT_COLUMNS: usize = 100;
/// Cells in the drawn bar on the stream surface.
@@ -338,7 +347,9 @@ struct State {
last_advance_at: Instant,
rate: Option<f64>,
shown_eta: Option<f64>,
- in_flight: Vec<(u64, String, u64)>,
+ /// `(seq, path, bytes, started)`. `started` is when the file was registered,
+ /// which is what the hash-phase blocker rule compares against.
+ in_flight: Vec<(u64, String, u64, Instant)>,
next_seq: u64,
tty_width: usize,
/// When the last `xerj-bar` display line was written; `None` before the
@@ -499,16 +510,7 @@ impl Progress {
/// someone else's tree, and it is re-rendered on every tick until the file
/// finishes. It is [`sanitize`]d here, once, on the way in.
pub fn file(&self, rel: &str, bytes: u64) -> FileGuard<'_> {
- let seq = if self.enabled() {
- let safe = sanitize(rel, SAFE_PATH_MAX);
- let mut state = self.state.lock().unwrap();
- let seq = state.next_seq;
- state.next_seq += 1;
- state.in_flight.push((seq, safe, bytes));
- seq
- } else {
- 0
- };
+ let seq = self.begin_file(rel, bytes);
FileGuard {
progress: self,
seq,
@@ -517,6 +519,30 @@ impl Progress {
}
}
+ /// Register a file as in flight without crediting it on drop.
+ ///
+ /// Hashing credits bytes from its own completion callback. Using [`Self::file`]
+ /// there would count every file twice. This guard only keeps the path on
+ /// the in-flight list so a read that does not return can be named.
+ pub fn watch_file(&self, rel: &str, bytes: u64) -> WatchGuard<'_> {
+ WatchGuard {
+ progress: self,
+ seq: self.begin_file(rel, bytes),
+ }
+ }
+
+ fn begin_file(&self, rel: &str, bytes: u64) -> u64 {
+ if !self.enabled() {
+ return 0;
+ }
+ let safe = sanitize(rel, SAFE_PATH_MAX);
+ let mut state = self.state.lock().unwrap();
+ let seq = state.next_seq;
+ state.next_seq += 1;
+ state.in_flight.push((seq, safe, bytes, Instant::now()));
+ seq
+ }
+
/// `bytes` is what is still UNcredited for this file — all of it unless
/// the guard reported positions through `FileGuard::advance_to`. The item
/// is counted here either way, including for a file whose every byte was
@@ -526,7 +552,7 @@ impl Progress {
let mut state = self.state.lock().unwrap();
state
.in_flight
- .retain(|(candidate, _, _)| *candidate != seq);
+ .retain(|(candidate, _, _, _)| *candidate != seq);
}
self.item_done(bytes);
}
@@ -771,13 +797,30 @@ impl Progress {
// Name the tail. Once the queue is nearly drained the run looks hung
// while one worker chews through the biggest file in the corpus.
+ //
+ // The hash phase also names a single file that has been in flight for
+ // longer than [`HASH_BLOCKING_AFTER`], even when more than
+ // [`STRAGGLER_MAX`] files are still open. A dataless placeholder blocks
+ // one worker inside `read` while the rest of the pool is also busy, so
+ // the tail rule never fires and the stall has no path (#2058). Other
+ // phases keep the tail-only rule.
let mut waiting_on = Vec::new();
if !state.in_flight.is_empty() && state.in_flight.len() <= STRAGGLER_MAX {
waiting_on = state
.in_flight
.iter()
- .map(|(_, rel, bytes)| (rel.clone(), *bytes))
+ .map(|(_, rel, bytes, _)| (rel.clone(), *bytes))
.collect();
+ } else if state.phase == "hash" {
+ if let Some((_, rel, bytes, started)) = state
+ .in_flight
+ .iter()
+ .min_by_key(|(_, _, _, started)| *started)
+ {
+ if now.duration_since(*started) >= HASH_BLOCKING_AFTER {
+ waiting_on.push((rel.clone(), *bytes));
+ }
+ }
}
Snapshot {
@@ -997,6 +1040,25 @@ impl Drop for FileGuard<'_> {
}
}
+/// Removes its file from the in-flight list on drop and credits nothing.
+///
+/// See [`Progress::watch_file`].
+pub struct WatchGuard<'a> {
+ progress: &'a Progress,
+ seq: u64,
+}
+
+impl Drop for WatchGuard<'_> {
+ fn drop(&mut self) {
+ if self.progress.enabled() {
+ let mut state = self.progress.state.lock().unwrap();
+ state
+ .in_flight
+ .retain(|(candidate, _, _, _)| *candidate != self.seq);
+ }
+ }
+}
+
/// Owns the ticker thread. Dropping it stops the thread and — if the run never
/// called `finish` — closes the stream with an honest `ok=false` line.
pub struct Ticker {
@@ -2272,6 +2334,108 @@ mod tests {
drop(big);
}
+ /// #2058: a hash read that has not returned for a few seconds is named even
+ /// when the pool still holds more files than the tail rule will list. A
+ /// scan or index phase with the same shape stays unnamed — that rule did
+ /// not change.
+ #[test]
+ fn a_hash_read_blocking_for_a_few_seconds_names_its_path() {
+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));
+ progress.phase("hash", 8, 80);
+ let _others: Vec<_> = (0..4)
+ .map(|i| progress.watch_file(&format!("local-{i}.txt"), 10))
+ .collect();
+ let blocked = progress.watch_file("cloud/Notes.txt", 10);
+ {
+ let mut state = progress.state.lock().unwrap();
+ for entry in &mut state.in_flight {
+ if entry.1 == "cloud/Notes.txt" {
+ entry.3 = Instant::now() - HASH_BLOCKING_AFTER - Duration::from_secs(1);
+ }
+ }
+ }
+ progress.tick();
+ let line = captured(&buffer)
+ .lines()
+ .rev()
+ .find(|line| line.starts_with("xerj-progress "))
+ .unwrap()
+ .to_string();
+ assert!(
+ line.contains("waiting_on=cloud/Notes.txt"),
+ "the file that has not returned is the one to name: {line}"
+ );
+ drop(blocked);
+
+ // The same pile of in-flight files during index is not a hash read.
+ // Naming them would change every busy scan.
+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));
+ progress.phase("index", 8, 80);
+ let _busy: Vec<_> = (0..5)
+ .map(|i| progress.watch_file(&format!("doc-{i}.txt"), 10))
+ .collect();
+ {
+ let mut state = progress.state.lock().unwrap();
+ state.in_flight[0].3 = Instant::now() - Duration::from_secs(30);
+ }
+ progress.tick();
+ let line = captured(&buffer)
+ .lines()
+ .rev()
+ .find(|line| line.starts_with("xerj-progress "))
+ .unwrap()
+ .to_string();
+ assert!(
+ !line.contains("waiting_on="),
+ "only the hash phase names a blocker before the tail: {line}"
+ );
+ }
+
+ /// Young hash reads are not blockers. Five files that just started must
+ /// not grow a `waiting_on` the tail rule would have omitted.
+ #[test]
+ fn a_busy_hash_does_not_name_a_file_that_just_started() {
+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));
+ progress.phase("hash", 8, 80);
+ let _guards: Vec<_> = (0..5)
+ .map(|i| progress.watch_file(&format!("f{i}.txt"), 10))
+ .collect();
+ progress.tick();
+ let line = captured(&buffer)
+ .lines()
+ .rev()
+ .find(|line| line.starts_with("xerj-progress "))
+ .unwrap()
+ .to_string();
+ assert!(!line.contains("waiting_on="), "{line}");
+ }
+
+ /// The hash callback credits the bytes. The watch guard must not credit
+ /// them again when the read returns.
+ #[test]
+ fn watch_file_names_the_path_and_credits_nothing() {
+ let (progress, buffer) = Progress::capture(Surface::Plain, Duration::from_secs(3600));
+ progress.phase("hash", 1, 10);
+ let guard = progress.watch_file("a.txt", 10);
+ progress.tick();
+ assert!(
+ captured(&buffer).contains("waiting_on=a.txt"),
+ "{}",
+ captured(&buffer)
+ );
+ drop(guard);
+ assert_eq!(progress.items_done.load(Ordering::Relaxed), 0);
+ assert_eq!(progress.bytes_done.load(Ordering::Relaxed), 0);
+ progress.tick();
+ let line = captured(&buffer)
+ .lines()
+ .rev()
+ .find(|line| line.starts_with("xerj-progress "))
+ .unwrap()
+ .to_string();
+ assert!(!line.contains("waiting_on="), "{line}");
+ }
+
/// An empty sample must not drag the measured rate down either — the rate
/// is a statement about the work that actually completed.
#[test]