Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

merge: upstream/main (#279 ignore rules, #202, #203, rc.14 pipeline rework)

Sergey Gonchar384945158a6a
diff --cc engine/crates/xerj-autoindex/src/cli.rs
index aed7476c9,ce648eff1..84eb4c0da
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@@ -16,14 -31,9 +31,12 @@@ pub struct IndexCfg
pub state_dir: Option<PathBuf>,
pub fresh: bool,
pub follow_symlinks: bool,
- /// Disable the marker-gated generated-dir prune (e.g. Unity `Library/`).
- pub no_default_excludes: bool,
+ /// `--stub <glob>` patterns: matching files are indexed as one
+ /// existence-only name card, contents never opened.
+ pub stub_globs: Vec<String>,
- /// Walk `.gitignore`d files too (by default they are skipped when the
- /// root is inside a git repository).
- pub no_gitignore: bool,
+ /// `.gitignore` / `.xerjignore` / built-in build-output rules (#276).
+ /// `deep_count` is set from `--dry-run`.
+ pub ignore: IgnoreOptions,
pub max_file_gb: u64,
pub sample: usize,
pub no_semantic: bool,
@@@ -82,27 -114,16 +117,22 @@@ pub fn print_help()
valid range 1..=3600)\n\
--prefix <P> index prefix (default ax)\n\
--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\
- --fresh ignore existing journal, restart (ids stay idempotent)\n\
+ --snapshot-max-gb <N> logical payload cap for sealed source+prepared records\n\
+ bytes (default 64); excludes filesystem/manifest overhead\n\
+ --fresh {fresh_help}\n\
--follow-symlinks follow symlinks (loop-safe); off by default\n\
- --no-default-excludes\n\
- walk generated/cache dirs too. By default,\n\
- well-known generated dirs are skipped (and\n\
- recorded) when a sibling marker proves them, at\n\
- any depth: Unity Library/Temp/obj/Logs/\n\
- UserSettings next to ProjectSettings/\n\
- ProjectVersion.txt, node_modules next to\n\
- package.json, target next to Cargo.toml\n\
- --no-gitignore index .gitignore'd files too. By default, when the\n\
- root is inside a git repository, files matched by\n\
- the tree's .gitignore rules are skipped (the\n\
- owner's own declaration of generated noise);\n\
- the user's global gitignore is never consulted\n\
+ --stub <GLOB> index matching files as ONE existence-only name\n\
+ card (title + provenance); contents are never\n\
+ opened. Repeatable. A pattern without '/'\n\
+ matches file names anywhere ('*.bvh'); with '/'\n\
+ it matches the root-relative path\n\
+ ('unity/**/*.csv'). '**' crosses directories\n\
+ --no-ignore index everything: no .gitignore, no .xerjignore, no\n\
+ .git/info/exclude, no built-in defaults. Hidden files\n\
+ (.env, .git/, .ssh) stay skipped either way — that is\n\
+ not an ignore rule, it is what keeps secrets out.\n\
+ --no-default-ignores keep the ignore files, drop only the built-in list\n\
+ ({defaults})\n\
--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\
--sample <N> records sampled per file for inference (default 500)\n\
--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\
@@@ -164,9 -275,8 +284,9 @@@ pub fn parse(args: Vec<String>) -> Resu
let mut state_dir: Option<PathBuf> = None;
let mut fresh = false;
let mut follow_symlinks = false;
- let mut no_default_excludes = false;
+ let mut stub_globs: Vec<String> = Vec::new();
- let mut no_gitignore = false;
+ let mut no_ignore = false;
+ let mut no_default_ignores = false;
let mut max_file_gb = 2u64;
let mut sample = 500usize;
let mut no_semantic = false;
@@@ -225,9 -364,8 +374,9 @@@
"--state-dir" => state_dir = it.next().map(PathBuf::from),
"--fresh" => fresh = true,
"--follow-symlinks" => follow_symlinks = true,
- "--no-default-excludes" => no_default_excludes = true,
+ "--stub" => stub_globs.push(it.next().ok_or("--stub needs a glob pattern")?),
- "--no-gitignore" => no_gitignore = true,
+ "--no-ignore" => no_ignore = true,
+ "--no-default-ignores" => no_default_ignores = true,
"--max-file-gb" => {
max_file_gb = it
.next()
@@@ -290,31 -492,43 +503,44 @@@
prefix,
state_dir,
})),
- (None, Some(root)) => Ok(Cmd::Index(IndexCfg {
- root,
- url,
- api_key,
- workers: workers.max(1),
- pdf_workers,
- pdf_timeout_secs,
- bulk_mb: bulk_mb.clamp(1, 24),
- bulk_timeout_secs,
- prefix,
- state_dir,
- fresh,
- follow_symlinks,
- no_default_excludes,
- stub_globs,
- no_gitignore,
- max_file_gb,
- sample: sample.max(50),
- no_semantic,
- brain,
- no_graph,
- dry_run,
- json,
- quiet,
- })),
+ (None, Some(root)) => {
+ let plan = crate::resources::plan(workers, pdf_workers, bulk_mb);
+ Ok(Cmd::Index(IndexCfg {
+ root,
+ url,
+ api_key,
+ workers: plan.index_workers,
+ scan_workers: plan.scan_threads,
+ pdf_workers: plan.pdf_workers,
+ resource_notes: plan.notes,
+ pdf_timeout_secs,
+ bulk_mb,
+ bulk_timeout_secs,
+ snapshot_max_bytes,
+ prefix,
+ state_dir,
+ fresh,
+ follow_symlinks,
++ stub_globs,
+ ignore: IgnoreOptions {
+ enabled: !no_ignore,
+ defaults: !no_ignore && !no_default_ignores,
+ // Only --dry-run pays for counting what is inside a pruned
+ // directory; a real run's whole point is not touching it.
+ deep_count: dry_run,
+ },
+ max_file_gb,
+ sample: sample.max(50),
+ no_semantic,
+ brain,
+ no_graph,
+ dry_run,
+ json,
+ quiet,
+ progress,
+ progress_interval,
+ }))
+ }
_ => Ok(Cmd::Help),
}
}
diff --cc engine/crates/xerj-autoindex/src/detect/e2e.rs
index 15684ef61,af2848899..d4fd6d667
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@@ -270,9 -268,7 +273,8 @@@ fn cfg(root: &Path, state_dir: &Path, u
state_dir: Some(state_dir.to_owned()),
fresh: false,
follow_symlinks: false,
- no_default_excludes: false,
+ stub_globs: Vec::new(),
- no_gitignore: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
max_file_gb: 1,
sample: 50,
no_semantic: true,
diff --cc engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 84481d716,4a3df54dc..0962ed7c1
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@@ -46,14 -133,13 +133,18 @@@ impl MockEndpoint
let server_state = Arc::clone(&state);
let join = thread::spawn(move || loop {
match listener.accept() {
- Ok((stream, _)) => handle(stream, &server_state),
+ Ok((stream, _)) => {
+ // BSD/macOS: accepted sockets inherit the listener's
+ // O_NONBLOCK; the handler does blocking reads.
+ stream.set_nonblocking(false).unwrap();
+ handle(stream, &server_state)
+ }
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
- if server_state.lock().unwrap().stop {
+ if server_state
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .stop
+ {
break;
}
thread::yield_now();
@@@ -204,9 -515,7 +520,8 @@@ fn cfg(root: &Path, state_dir: &Path, u
state_dir: Some(state_dir.to_owned()),
fresh: false,
follow_symlinks: false,
- no_default_excludes: false,
+ stub_globs: Vec::new(),
- no_gitignore: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
max_file_gb: 1,
sample: 50,
no_semantic: true,
@@@ -623,266 -1750,1262 +1756,1265 @@@ fn file_done_append_and_fsync_failures_
assert_eq!(run_index(config.clone()).unwrap(), 0);
fs::write(&path, "id,value\n0,new\n1,new\n").unwrap();
- state::fail_next_file_done_io(io_boundary);
- let error = run_index(config.clone()).unwrap_err();
- let message = format!("{error:#}");
- assert!(
- message.contains("durably commit completed source"),
- "{message}"
- );
- assert!(
- message.contains(if io_boundary == 1 {
- "append failure"
- } else if io_boundary == 3 {
- "partial file_done write"
- } else {
- "fsync failure"
- }),
- "{message}"
- );
- let replay = state::Journal::open(
- state_dir.path(),
- &config.root.to_string_lossy(),
- &config.url,
- &config.prefix,
- config.bulk_timeout_secs,
- false,
- )
- .unwrap();
- assert!(replay.done.is_empty());
- assert_eq!(replay.pending_replacements.len(), 1);
- drop(replay);
+ state::fail_next_file_done_io(io_boundary, &state_dir.path().join("journal.ndjson"));
+ let error = run_index(config.clone()).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("durably commit completed source"),
+ "{message}"
+ );
+ assert!(
+ message.contains(if io_boundary == 1 {
+ "append failure"
+ } else if io_boundary == 3 {
+ "partial file_done write"
+ } else {
+ "fsync failure"
+ }),
+ "{message}"
+ );
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(replay.done.is_empty());
+ assert_eq!(replay.pending_replacements.len(), 1);
+ drop(replay);
+
+ assert_eq!(run_index(config).unwrap(), 0);
+ let locked = endpoint.state.lock().unwrap();
+ assert_eq!(
+ locked
+ .docs
+ .values()
+ .filter(|doc| doc.get("value").is_some())
+ .count(),
+ 2
+ );
+ assert!(locked
+ .docs
+ .values()
+ .filter(|doc| doc.get("value").is_some())
+ .all(|doc| doc["value"].as_str() == Some("new")));
+ }
+ }
+
+ #[test]
+ fn pending_generation_b_is_superseded_when_source_changes_to_c() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ // run_index reaches Journal::file_done; the file_done IO failpoint is
+ // a global one-shot, so every file_done-reaching test must hold its
+ // lock (armer or not) or it can steal an armed injection. Acquired
+ // after FAILPOINT_TEST_LOCK everywhere — one consistent order.
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let path = corpus.path().join("records.csv");
+ fs::write(&path, "id,value\n0,generation-a\n1,generation-a\n").unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ fs::write(&path, "id,value\n0,generation-b\n1,generation-b\n").unwrap();
+ REPLACEMENT_FAILPOINT.store(2, Ordering::SeqCst);
+ run_index(config.clone()).unwrap_err();
+ let replay_b = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ let key = replay_b.pending_replacements.keys().next().unwrap().clone();
+ let generation_b = replay_b.pending_replacements[&key].clone();
+ drop(replay_b);
+
+ fs::write(&path, "id,value\n0,generation-c\n1,generation-c\n").unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let replay_c = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(replay_c.pending_replacements.is_empty());
+ let committed_generation = replay_c.done[&key].generation.as_deref().unwrap();
+ assert_ne!(committed_generation, generation_b);
+ let events: Vec<Value> = fs::read_to_string(state_dir.path().join("journal.ndjson"))
+ .unwrap()
+ .lines()
+ .map(|line| serde_json::from_str(line).unwrap())
+ .collect();
+ let starts: Vec<&str> = events
+ .iter()
+ .filter(|event| {
+ event["kind"] == "file_replace_start"
+ && event["file_key"].as_str() == Some(key.as_str())
+ })
+ .filter_map(|event| event["generation"].as_str())
+ .collect();
+ assert_eq!(starts[starts.len() - 2], generation_b);
+ assert_eq!(starts.last().copied(), Some(committed_generation));
+ drop(replay_c);
+ let locked = endpoint.state.lock().unwrap();
+ assert_eq!(
+ locked
+ .docs
+ .values()
+ .filter(|doc| doc.get("value").is_some())
+ .count(),
+ 2
+ );
+ assert!(locked
+ .docs
+ .values()
+ .filter(|doc| doc.get("value").is_some())
+ .all(|doc| doc["value"].as_str() == Some("generation-c")));
+ }
+
+ #[test]
+ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ // run_index reaches Journal::file_done; the file_done IO failpoint is
+ // a global one-shot, so every file_done-reaching test must hold its
+ // lock (armer or not) or it can steal an armed injection. Acquired
+ // after FAILPOINT_TEST_LOCK everywhere — one consistent order.
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let original = "id,value\n0,original\n1,original\n";
+ fs::write(corpus.path().join("a.csv"), original).unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ // a.csv keeps its planned key by path while its OLD bytes reappear as
+ // b.csv — whose content key is exactly the planned key a.csv claims. Both
+ // files owning one ax_file key would let each replacement delete the
+ // other's freshly published documents.
+ fs::write(
+ corpus.path().join("a.csv"),
+ "id,value\n0,rewritten\n1,rewritten\n",
+ )
+ .unwrap();
+ fs::write(corpus.path().join("b.csv"), original).unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 3);
+ {
+ let locked = endpoint.state.lock().unwrap();
+ // Graph detection is enabled in this module, so a.csv also carries a
+ // file-level node document (`ax_locator: "file"`, no row fields). The
+ // replacement transaction's subject is the row documents.
+ let rows: Vec<&Value> = data_rows(&locked);
+ assert_eq!(rows.len(), 2);
+ assert!(rows.iter().all(|doc| {
+ doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")
+ }));
+ assert!(locked
+ .docs
+ .values()
+ .all(|doc| doc["ax_path"].as_str() == Some("a.csv")));
+ }
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(replay.pending_replacements.is_empty());
+ assert_eq!(replay.done.len(), 1);
+ let plan = replay.plan.clone().unwrap();
+ assert_eq!(plan.junk_files.len(), 1);
+ assert_eq!(plan.junk_files[0].rel, "b.csv");
+ assert!(plan.junk_files[0]
+ .reason
+ .contains("key ownership exclusive"));
+ drop(replay);
+
+ // The divergence is durable and deterministic: an identical rerun keeps
+ // the same owner, appends no new plan, and changes no documents.
+ let plans_before = event_count(state_dir.path(), "plan");
+ assert_eq!(run_index(config).unwrap(), 3);
+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);
+ let locked = endpoint.state.lock().unwrap();
+ let rows = data_rows(&locked);
+ assert_eq!(rows.len(), 2);
+ assert!(rows
+ .iter()
+ .all(|doc| doc["value"].as_str() == Some("rewritten")));
+ }
+
+ /// #238: a file added after the plan was frozen is skipped and reported in the
+ /// catalog. That report used to be immortal — `new_unplanned` was a per-run
+ /// `Vec`, so once the file left the corpus nothing remembered the document had
+ /// ever been written and no run could remove it. The catalog is the data map
+ /// every `map`/`status`/agent query reads; it must not keep advertising a file
+ /// that is gone.
+
+ #[test]
+ fn an_added_then_deleted_file_leaves_no_immortal_catalog_entry() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("planned.csv"),
+ "id,value\n0,planned\n1,planned\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ // Appears after the plan froze: not indexed, but reported as skipped.
+ fs::write(
+ corpus.path().join("added.csv"),
+ "id,value\n2,added\n3,added\n",
+ )
+ .unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 3);
+ let skipped_id = {
+ let locked = endpoint.state.lock().unwrap();
+ let (id, doc) = locked
+ .catalog_docs
+ .iter()
+ .find(|(_, doc)| doc["path"].as_str() == Some("added.csv"))
+ .expect("a file skipped for being post-freeze is reported in the catalog");
+ assert_eq!(doc["status"].as_str(), Some("skipped"));
+ assert_eq!(doc["doc_kind"].as_str(), Some("file"));
+ id.clone()
+ };
+
+ // The durable plan is the only record that the document exists. Without
+ // it no later run can ever delete it.
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(
+ replay
+ .plan
+ .as_ref()
+ .unwrap()
+ .junk_files
+ .iter()
+ .any(|junk| junk.rel == "added.csv"),
+ "the skipped file must be recorded in the durable plan"
+ );
+ drop(replay);
+
+ // Delete it: the catalog entry goes with it, and the run is clean again.
+ fs::remove_file(corpus.path().join("added.csv")).unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ {
+ let locked = endpoint.state.lock().unwrap();
+ assert!(
+ !locked.catalog_docs.contains_key(&skipped_id),
+ "the catalog entry for a deleted skipped file must not survive"
+ );
+ assert!(
+ locked
+ .catalog_docs
+ .values()
+ .all(|doc| doc["path"].as_str() != Some("added.csv")),
+ "no catalog document may still name the deleted file"
+ );
+ // The file that is still there keeps its entry.
+ assert!(locked
+ .catalog_docs
+ .values()
+ .any(|doc| doc["path"].as_str() == Some("planned.csv")));
+ }
+
+ // Converged: nothing left to add or sweep, so no further plan is appended.
+ let plans_before = event_count(state_dir.path(), "plan");
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);
+ }
+
+ #[test]
+ fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ // run_index reaches Journal::file_done; the file_done IO failpoint is
+ // a global one-shot, so every file_done-reaching test must hold its
+ // lock (armer or not) or it can steal an armed injection. Acquired
+ // after FAILPOINT_TEST_LOCK everywhere — one consistent order.
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("dup-a.csv"), "id,value\n0,dup\n1,dup\n").unwrap();
+ fs::write(corpus.path().join("dup-b.csv"), "id,value\n0,dup\n1,dup\n").unwrap();
+ fs::write(corpus.path().join("keep.csv"), "id,value\n2,keep\n3,keep\n").unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let starts = event_count(state_dir.path(), "file_replace_start");
+
+ // The whole duplicate group disappears: its documents stay live with no
+ // source file, so the rerun is refused. The original invariant still
+ // holds and is still checked — a key with no current file must never get
+ // a replacement intent journaled, which would be re-appended forever
+ // without ever committing.
+ fs::remove_file(corpus.path().join("dup-a.csv")).unwrap();
+ fs::remove_file(corpus.path().join("dup-b.csv")).unwrap();
+ for _ in 0..2 {
+ assert_unsupported_delta_without_remote_mutation(
+ &endpoint,
+ config.clone(),
+ &[],
+ &["dup-a.csv"],
+ );
+ assert_eq!(
+ event_count(state_dir.path(), "file_replace_start"),
+ starts,
+ "an orphaned duplicate-group key must not schedule a replacement"
+ );
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(replay.pending_replacements.is_empty());
+ }
+ }
+
+ #[test]
+ fn partial_file_done_is_rolled_back_before_another_worker_commits() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n0,a-old\n1,a-old\n").unwrap();
+ fs::write(corpus.path().join("b.csv"), "id,value\n2,b-old\n3,b-old\n").unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ config.workers = 2;
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ fs::write(corpus.path().join("a.csv"), "id,value\n0,a-new\n1,a-new\n").unwrap();
+ fs::write(corpus.path().join("b.csv"), "id,value\n2,b-new\n3,b-new\n").unwrap();
+ state::fail_next_file_done_io(3, &state_dir.path().join("journal.ndjson"));
+ let error = run_index(config.clone()).unwrap_err();
+ assert!(format!("{error:#}").contains("partial file_done write"));
+
+ // One worker hit the partial append and rolled back while the other was
+ // allowed to commit. Replay must remain parseable and expose exactly one
+ // pending generation rather than malformed middle corruption.
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert_eq!(replay.pending_replacements.len(), 1);
+ assert_eq!(replay.done.len(), 1);
+ drop(replay);
+
+ assert_eq!(run_index(config).unwrap(), 0);
+ let locked = endpoint.state.lock().unwrap();
+ assert_eq!(
+ locked
+ .docs
+ .values()
+ .filter(|doc| doc.get("value").is_some())
+ .count(),
+ 4
+ );
+ assert!(locked
+ .docs
+ .values()
+ .filter(|doc| doc.get("value").is_some())
+ .all(|doc| {
+ doc["value"]
+ .as_str()
+ .is_some_and(|value| value.ends_with("-new"))
+ }));
+ }
+
+ /// #195: an index write block rejects every bulk item with status 403 (the
+ /// status ES assigns explicit/API blocks — only the flood-stage
+ /// `read_only_allow_delete` block is 429). A status-based classifier filed
+ /// those items under "junk source records", journaled the file COMPLETE, and
+ /// the instructed rerun then resumed past the journal and reported success
+ /// over an empty index. Blocks must be classified by error type: fatal,
+ /// file left pending, and the rerun after the block lifts must really index.
+ #[test]
+ fn write_block_rejections_are_backend_fatal_and_the_file_stays_pending() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("records.csv"),
+ "id,value\n0,blocked\n1,blocked\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+
+ endpoint.state.lock().unwrap().block_writes = true;
+ let error = format!("{:#}", run_index(config.clone()).unwrap_err());
+ assert!(error.contains("blocked"), "{error}");
+ assert_eq!(
+ file_done_count(state_dir.path()),
+ 0,
+ "a write-blocked source file must stay pending in the journal"
+ );
+ assert!(endpoint.state.lock().unwrap().docs.is_empty());
+
+ // Block lifted: the SAME command must resume and actually index.
+ endpoint.state.lock().unwrap().block_writes = false;
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(file_done_count(state_dir.path()), 1);
+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 2);
+ }
+
+ /// Pins WHY the run document carries no backend-rejection count, so nobody
+ /// "restores" one that could only ever read 0.
+ ///
+ /// Making `junk_records_total` durable narrowed it: it no longer folds in the
+ /// per-item rejections that `record_bulk_outcome` counts. That looks like a
+ /// lost signal, and it would be one — except that a single per-item rejection
+ /// also lands in `bulk_errors`, which aborts the run before the run document,
+ /// the map, the summary line and the exit code exist at all. So the narrowed
+ /// number was unreachable in any published map either way.
+ ///
+ /// The rejection count is therefore reported in the one place a rejected run
+ /// ever reaches: the abort message, where it supplies the scale that
+ /// `bulk_errors`' first-five sample cannot.
+ #[test]
+ fn backend_rejected_records_abort_the_run_before_any_map_is_written() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("records.csv"),
+ "id,value\n0,kept\n1,kept\n2,kept\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ endpoint.state.lock().unwrap().reject_first_data_item = true;
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+
+ // A per-item 400 is NOT a server error (not 429, not 5xx, not a block), so
+ // it is the weakest rejection the classifier recognises — and even it is
+ // fatal.
+ let error = format!("{:#}", run_index_report(config).unwrap_err());
+ assert!(
+ error.contains("document_parsing_exception"),
+ "the refusal itself must be quoted: {error}"
+ );
+ assert!(
+ error.contains("The backend refused 1 record(s)."),
+ "the abort must state how many records were refused, not just sample the errors: {error}"
+ );
+
+ // Measured, and deliberately pinned even though it is NOT what the abort
+ // message implies: `record_bulk_outcome` treats a per-item rejection as
+ // non-fatal to the worker (only `server_errors` set `send_err`), so the
+ // file is journaled COMPLETE and the run aborts only at the end. A rerun
+ // therefore resumes past this file and never retries the refused record,
+ // while the message says failed sources "were not journaled complete".
+ //
+ // That divergence predates this branch — it is `main`'s behaviour for the
+ // item-error class — and fixing it means changing when a worker treats a
+ // rejection as fatal, which is a resume-semantics change with nothing to
+ // do with map metadata. Recorded here so the next reader finds the fact
+ // rather than the assumption; tracked in the PR body as a known gap.
+ assert_eq!(
+ file_done_count(state_dir.path()),
+ 1,
+ "pre-existing: a per-item rejection still journals the source complete"
+ );
+ }
+
+ /// #195 last-resort gate: a backend that ACCEPTS every bulk but persists
+ /// nothing (the shape of any rejection path the per-item classifier does
+ /// not recognise) must not yield a success exit. The journal says records
+ /// landed; the live count says zero; the run must fail and say how to
+ /// recover.
+ #[test]
+ fn zero_live_documents_after_journaled_records_fails_the_run() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("records.csv"),
+ "id,value\n0,ghost\n1,ghost\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ endpoint.state.lock().unwrap().swallow_data_bulks = true;
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+
+ let error = format!("{:#}", run_index(config).unwrap_err());
+ assert!(error.contains("verification failed"), "{error}");
+ assert!(error.contains("0 documents are live"), "{error}");
+ assert!(error.contains("--fresh"), "{error}");
+ }
+
+ /// #241 regression: a run must never be silent.
+ ///
+ /// Before this fix `autoindex` printed a handful of phase banners, one line
+ /// per 200 completed files, and then nothing at all through the entire
+ /// `finalize` block — measured at 47-64% of wall in the issue and at 45% (100
+ /// of 222 s) on the corpus used to reproduce it here. An agent watching the
+ /// stream could not tell a running index from a hung one.
+ ///
+ /// This asserts the three properties that make the stream readable, through
+ /// the REAL progress surface a production run uses:
+ /// 1. phase B reports as it goes, with a percent and an ETA field;
+ /// 2. the previously-silent finalize block reports at all;
+ /// 3. every run on a surface that prints at all ends with a terminal line
+ /// stating the outcome in words, because exit 3 is a success an agent
+ /// otherwise reads as failure. (`--progress none` prints nothing —
+ /// asserted by `quiet_runs_emit_no_progress_stream_at_all` below.)
+ #[test]
+ fn a_run_reports_progress_through_every_phase_and_closes_the_stream() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ // Distinct bytes per file: byte-identical files collapse into one
+ // canonical file plus aliases, which would make `files=8` a lie.
+ for n in 0..8 {
+ fs::write(
+ corpus.path().join(format!("rows-{n}.csv")),
+ format!("id,value\n{n}0,alpha\n{n}1,beta\n{n}2,gamma\n"),
+ )
+ .unwrap();
+ }
+
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ config.quiet = false;
+ config.progress = crate::progress::ProgressMode::Plain;
+ config.progress_interval = Some(std::time::Duration::from_secs(1));
+
+ let buffer = Arc::new(Mutex::new(Vec::new()));
+ let (code, _report) = {
+ let _sink = crate::progress::install_test_sink(&buffer);
+ run_index_report(config).unwrap()
+ };
+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
+
+ let progress_lines: Vec<&str> = stream
+ .lines()
+ .filter(|line| line.starts_with("xerj-progress "))
+ .collect();
+ assert!(
+ progress_lines.len() >= 5,
+ "a run must narrate its phases, got {} line(s):\n{stream}",
+ progress_lines.len()
+ );
+ assert!(
+ progress_lines
+ .iter()
+ .any(|line| line.contains("phase=index")),
+ "phase B must report:\n{stream}"
+ );
+ assert!(
+ progress_lines
+ .iter()
+ .any(|line| line.contains("phase=finalize")),
+ "the finalize block was the longest silence in the tool; it must report now:\n{stream}"
+ );
+ assert!(
+ progress_lines
+ .iter()
+ .all(|line| line.contains(" pct=") && line.contains(" eta_s=")),
+ "every progress line carries a percent and an ETA field (possibly `unknown`):\n{stream}"
+ );
+
+ // ...and the same run hands the agent something it can show a person.
+ // Asserted on the REAL pipeline rather than on a hand-built snapshot.
+ //
+ // This corpus indexes in ~0.3 s, so it is also the end-to-end proof of the
+ // burst floor: every one of its phase changes falls inside `BAR_MIN_GAP`,
+ // and a run that short must therefore relay ONE display line, not one per
+ // phase. The machine lines above already proved every phase is on the
+ // stream — the relay is deliberately quieter than the record.
+ let bars: Vec<&str> = stream
+ .lines()
+ .filter(|line| line.starts_with("xerj-bar "))
+ .collect();
+ assert_eq!(
+ bars.len(),
+ 1,
+ "a sub-2s run relays exactly one bar, whatever its phase count:\n{stream}"
+ );
+ assert!(
+ bars[0].starts_with("xerj-bar [") && bars[0].contains(']'),
+ "the display line carries a drawn bar:\n{stream}"
+ );
+ // The display line is derived, never a second source of truth: it may not
+ // claim completion while the machine line still reports work outstanding.
+ assert!(
+ !bars.iter().any(|line| line.contains("[####")
+ && line.contains("----]")
+ && line.contains("100.0%")),
+ "a partly-drawn bar cannot be labelled 100%:\n{stream}"
+ );
+
+ let done = stream
+ .lines()
+ .find(|line| line.starts_with("xerj-done "))
+ .unwrap_or_else(|| panic!("a run must close its own stream:\n{stream}"));
+ assert!(
+ done.contains("ok=true") && done.contains(&format!("exit={code}")),
+ "{done}"
+ );
+ assert!(done.contains("reason=completed"), "{done}");
+ assert!(done.contains("files=8"), "{done}");
+ }
+
+ /// The other half of the contract: `--quiet` still means silence, so nothing
+ /// here can spam a caller that asked for none.
+ #[test]
+ fn quiet_runs_emit_no_progress_stream_at_all() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("rows.csv"), "id,value\n0,alpha\n").unwrap();
+
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(config.progress, crate::progress::ProgressMode::None);
+
+ let buffer = Arc::new(Mutex::new(Vec::new()));
+ {
+ let _sink = crate::progress::install_test_sink(&buffer);
+ run_index_report(config).unwrap();
+ }
+ assert!(
+ buffer.lock().unwrap().is_empty(),
+ "--quiet asked for nothing: {}",
+ String::from_utf8_lossy(&buffer.lock().unwrap())
+ );
+ }
+
+ /// Where #240 (the resource policy) and #241 (the progress stream) meet.
+ ///
+ /// Two rules have to hold at once, and each is easy to break while honouring
+ /// the other:
+ ///
+ /// * The resource decision is **announced on the progress surface**, not on a
+ /// bare `eprintln!` gated on `!quiet`. A raw write would survive
+ /// `--progress none` — which promises silence — and would inject a
+ /// non-JSON line into the middle of `--progress json`, which promises one
+ /// parseable stream. (`quiet_runs_emit_no_progress_stream_at_all` above is
+ /// the other side of this assertion: it fails if any of these lines escape
+ /// the surface.)
+ /// * The numbers announced are the ones the run **got**, not the ones it
+ /// asked for. `pool::configure` is first-call-wins, because rayon pools
+ /// cannot be resized, so a process that already indexed once — `xerj
+ /// brain` does exactly that — keeps its original phase-A width. Reporting
+ /// the request there would make the stream and the run summary a record of
+ /// an intention rather than of a run.
+ ///
+ /// The second rule is made observable by asking for a width this process
+ /// provably cannot grant: the pool is built before the run, then one more
+ /// thread than it has is requested.
+ #[test]
+ fn the_resource_plan_is_announced_on_the_progress_surface_with_the_width_it_got() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("rows.csv"),
+ "id,value\n0,alpha\n1,beta\n",
+ )
+ .unwrap();
+
+ // Build the process-global scan pool first, so the width below is one this
+ // run cannot possibly be granted.
+ let installed = crate::pool::scan_pool().current_num_threads();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ config.quiet = false;
+ config.progress = crate::progress::ProgressMode::Plain;
+ config.progress_interval = Some(std::time::Duration::from_secs(1));
+ config.scan_workers = installed + 1;
+ config.workers = 3;
+ config.resource_notes = vec!["memory safe zone 256 MiB allows 3 index workers".to_string()];
+
+ let buffer = Arc::new(Mutex::new(Vec::new()));
+ let (_code, report) = {
+ let _sink = crate::progress::install_test_sink(&buffer);
+ run_index_report(config).unwrap()
+ };
+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
+
+ assert!(
+ stream.contains(&format!("{installed} scan threads")),
+ "the run must announce the phase-A width it actually got, not {}: {stream}",
+ installed + 1
+ );
+ assert!(
+ !stream.contains(&format!("{} scan threads", installed + 1)),
+ "a width the pool refused must never be reported as fact: {stream}"
+ );
+ assert!(
+ stream.contains("3 index workers"),
+ "phase B must name the count the policy chose: {stream}"
+ );
+ assert!(
+ stream.contains("memory safe zone 256 MiB allows 3 index workers"),
+ "what the machine forced on the run belongs in the same stream: {stream}"
+ );
+ assert!(
+ stream.contains(&format!("with {installed} threads")),
+ "the phase-A banner must carry the real width too: {stream}"
+ );
+
+ // The summary is read after the fact by agents and by `xerj brain`; it has
+ // to agree with the stream rather than repeat the request.
+ let report = report.expect("a completed run produces a summary");
+ assert_eq!(report["scan_workers"], serde_json::json!(installed));
+ assert_eq!(report["workers"], serde_json::json!(3));
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn fresh_pdf_is_parsed_once_and_failed_publication_retry_reparses_once() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ inject_pdf_spool_capacity(state_dir.path());
+ let tools = tempfile::tempdir().unwrap();
+ let pdf = corpus.path().join("quarterly-report.pdf");
+ fs::write(
+ &pdf,
+ b"%PDF-1.4\nfake bytes consumed by the isolated test worker\n",
+ )
+ .unwrap();
+ fs::copy(&pdf, corpus.path().join("quarterly-report-copy.pdf")).unwrap();
+
+ let count = tools.path().join("worker-count");
+ let worker = tools.path().join("pdf-worker");
+ let worker_script = String::from_utf8(
+ br#"#!/bin/sh
+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"
+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"title":"quarterly-report","page":1,"body":"Quarterly revenue increased while operating margin improved.","pdf_pages_total":1,"pdf_pages_with_text":1,"pdf_pages_omitted":0},"locator":"p1-s0","group":null,"origin":"extractor"}]}'
+ "#
+ .to_vec(),
+ )
+ .unwrap()
+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));
+ fs::write(&worker, worker_script).unwrap();
+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();
+
+ let endpoint = MockEndpoint::start(1);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let _env = PdfWorkerEnvGuard::acquire();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);
+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);
+
+ let first = run_index(config.clone()).unwrap_err();
+ assert!(format!("{first:#}").contains("bulk backend failed"));
+ assert_eq!(
+ fs::read_to_string(&count).unwrap().lines().count(),
+ 1,
+ "Phase A's extraction must be replayed in Phase B, not parsed again"
+ );
+ assert_eq!(file_done_count(state_dir.path()), 0);
+
+ // The process-local spool is intentionally not journal state. A retry has
+ // a frozen plan, performs exactly one fresh extraction in Phase B, and
+ // commits only after the backend accepts it.
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(fs::read_to_string(&count).unwrap().lines().count(), 2);
+ assert_eq!(file_done_count(state_dir.path()), 1);
+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 1);
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn refused_pdf_spool_reparses_and_reports_fallback_without_weakening_publication() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let tools = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("quarterly-report.pdf"),
+ b"%PDF-1.4\nfake bytes consumed by the isolated test worker\n",
+ )
+ .unwrap();
+ fs::write(
+ state_dir
+ .path()
+ .join(".autoindex-test-pdf-spool-available-bytes"),
+ ((4_u64 << 30) + (32 << 20) - 1).to_string(),
+ )
+ .unwrap();
+ fs::write(
+ state_dir.path().join(".autoindex-test-pdf-spool-fd-limit"),
+ "4096",
+ )
+ .unwrap();
+ fs::write(
+ state_dir.path().join(".autoindex-test-pdf-spool-fd-open"),
+ "16",
+ )
+ .unwrap();
+
+ let count = tools.path().join("worker-count");
+ let worker = tools.path().join("pdf-worker");
+ let worker_script = String::from_utf8(
+ br#"#!/bin/sh
+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"
+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"title":"quarterly-report","page":1,"body":"Quarterly revenue increased while operating margin improved.","pdf_pages_total":1,"pdf_pages_with_text":1,"pdf_pages_omitted":0},"locator":"p1-s0","group":null,"origin":"extractor"}]}'
+ "#
+ .to_vec(),
+ )
+ .unwrap()
+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));
+ fs::write(&worker, worker_script).unwrap();
+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();
+
+ let endpoint = MockEndpoint::start(0);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let _env = PdfWorkerEnvGuard::acquire();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);
+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);
+
+ let (code, report) = run_index_report(config).unwrap();
+ assert_eq!(code, 0);
+ assert_eq!(fs::read_to_string(&count).unwrap().lines().count(), 2);
+ assert_eq!(file_done_count(state_dir.path()), 1);
+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 1);
+ let reuse = report.unwrap()["pdf_extraction_reuse"].clone();
+ assert_eq!(reuse["reservations_started"], 0);
+ assert_eq!(reuse["capacity_fallbacks"], 1);
+ assert_eq!(reuse["phase_b_pdf_parses"], 1);
+ assert_eq!(reuse["replay_verified"], 0);
+ assert_eq!(reuse["artifacts_not_created"], 1);
+ assert_eq!(reuse["phase_a_pdf_parser_responses"], 1);
+ assert_eq!(reuse["capacity_status"], "disabled");
+ assert_eq!(
+ reuse["fallback_examples"][0]["path"],
+ "quarterly-report.pdf"
+ );
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn refused_spool_does_not_pay_a_second_phase_a_source_hash() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let tools = tempfile::tempdir().unwrap();
+ let pdf = corpus.path().join("quarterly-report.pdf");
+ fs::write(&pdf, b"%PDF-1.4\nsame-size-source-generation\n").unwrap();
+ let size = fs::metadata(&pdf).unwrap().len();
+ let digest = content::resolve_reporting(
+ vec![crate::walk::FileEntry {
+ path: pdf.clone(),
+ rel: "quarterly-report.pdf".into(),
+ rel_id: "quarterly-report.pdf".into(),
+ is_symlink: false,
+ size,
+ }],
+ &|_| {},
+ )
+ .unwrap()
+ .digests
+ .remove(0);
- assert_eq!(run_index(config).unwrap(), 0);
- let locked = endpoint.state.lock().unwrap();
- assert_eq!(locked.docs.len(), 2);
- assert!(locked
- .docs
- .values()
- .all(|doc| doc["value"].as_str() == Some("new")));
+ let worker = tools.path().join("pdf-worker");
+ let worker_script = String::from_utf8(
+ br#"#!/bin/sh
+ printf 'Z' | dd of="$2" bs=1 seek=10 conv=notrunc 2>/dev/null
+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"body":"Quarterly revenue increased while operating margin improved."},"locator":"p1-s0","group":null,"origin":"extractor"}]}'
+ "#
+ .to_vec(),
+ )
+ .unwrap()
+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));
+ fs::write(&worker, worker_script).unwrap();
+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();
+ let _env = PdfWorkerEnvGuard::acquire();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);
+
+ let budget = crate::extract::pdf::ExtractionSpoolBudget::new(0, 0);
+ let progress = Progress::silent();
+ let scan = scan_file(
+ &pdf,
+ size,
+ &digest,
+ &PhaseAContext {
+ state_dir: state_dir.path(),
+ budget: &budget,
+ capacity_warning: None,
+ progress: &progress,
+ },
+ 100,
+ 1,
++ false,
+ );
+ assert!(scan.pdf_spool.is_none());
+ assert_eq!(scan.pdf_spool_fallbacks.len(), 1);
+ for fallback in &scan.pdf_spool_fallbacks {
+ budget.record_fallback_example(
+ "quarterly-report.pdf",
+ fallback.category,
+ &fallback.message,
+ );
}
+ let report = budget.report();
+ assert_eq!(report["artifacts_created"], 0);
+ assert_eq!(report["phase_b_eligible_artifacts"], 0);
+ assert_eq!(report["artifacts_discarded_before_replay"], 0);
+ assert_eq!(report["fallback_categories"]["artifact_count_ceiling"], 1);
+ assert!(report["fallback_categories"]
+ .get("source_generation_changed")
+ .is_none());
}
+ /// The other half of the branch above: with capacity available the artifact
+ /// *is* created, and phase A must then throw it away because the source moved
+ /// underneath the parser. Without this the source-generation arm of
+ /// `scan_file` has no test at all — its sibling above deliberately runs with a
+ /// zero budget, so it never reaches the `content::verify` comparison.
+ #[cfg(unix)]
#[test]
- fn pending_generation_b_is_superseded_when_source_changes_to_c() {
+ fn source_changed_during_phase_a_discards_the_artifact_and_names_the_reason() {
+ use std::os::unix::fs::PermissionsExt;
+
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
- // run_index reaches Journal::file_done; the file_done IO failpoint is
- // a global one-shot, so every file_done-reaching test must hold its
- // lock (armer or not) or it can steal an armed injection. Acquired
- // after FAILPOINT_TEST_LOCK everywhere — one consistent order.
- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
let state_dir = tempfile::tempdir().unwrap();
- let path = corpus.path().join("records.csv");
- fs::write(&path, "id,value\n0,generation-a\n1,generation-a\n").unwrap();
- let endpoint = MockEndpoint::start(usize::MAX);
- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
+ inject_pdf_spool_capacity(state_dir.path());
+ let tools = tempfile::tempdir().unwrap();
+ let pdf = corpus.path().join("quarterly-report.pdf");
+ fs::write(&pdf, b"%PDF-1.4\nsame-size-source-generation\n").unwrap();
+ let size = fs::metadata(&pdf).unwrap().len();
+ let digest = content::resolve_reporting(
+ vec![crate::walk::FileEntry {
+ path: pdf.clone(),
+ rel: "quarterly-report.pdf".into(),
+ rel_id: "quarterly-report.pdf".into(),
+ is_symlink: false,
+ size,
+ }],
+ &|_| {},
+ )
+ .unwrap()
+ .digests
+ .remove(0);
- fs::write(&path, "id,value\n0,generation-b\n1,generation-b\n").unwrap();
- REPLACEMENT_FAILPOINT.store(2, Ordering::SeqCst);
- run_index(config.clone()).unwrap_err();
- let replay_b = state::Journal::open(
- state_dir.path(),
- &config.root.to_string_lossy(),
- &config.url,
- &config.prefix,
- config.bulk_timeout_secs,
- false,
+ // Same trick as the sibling test: the worker rewrites one byte of its own
+ // input, so the file keeps its length and changes its digest.
+ let worker = tools.path().join("pdf-worker");
+ let worker_script = String::from_utf8(
+ br#"#!/bin/sh
+ printf 'Z' | dd of="$2" bs=1 seek=10 conv=notrunc 2>/dev/null
+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"body":"Quarterly revenue increased while operating margin improved."},"locator":"p1-s0","group":null,"origin":"extractor"}]}'
+ "#
+ .to_vec(),
)
- .unwrap();
- let key = replay_b.pending_replacements.keys().next().unwrap().clone();
- let generation_b = replay_b.pending_replacements[&key].clone();
- drop(replay_b);
+ .unwrap()
+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));
+ fs::write(&worker, worker_script).unwrap();
+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();
+ let _env = PdfWorkerEnvGuard::acquire();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);
- fs::write(&path, "id,value\n0,generation-c\n1,generation-c\n").unwrap();
- assert_eq!(run_index(config.clone()).unwrap(), 0);
- let replay_c = state::Journal::open(
- state_dir.path(),
- &config.root.to_string_lossy(),
- &config.url,
- &config.prefix,
- config.bulk_timeout_secs,
+ let (budget, _) =
+ crate::extract::pdf::ExtractionSpoolBudget::for_state_dir(state_dir.path(), 1, 1, 8);
+ let progress = Progress::silent();
+ let scan = scan_file(
+ &pdf,
+ size,
+ &digest,
+ &PhaseAContext {
+ state_dir: state_dir.path(),
+ budget: &budget,
+ capacity_warning: None,
+ progress: &progress,
+ },
+ 100,
+ 1,
+ false,
- )
- .unwrap();
- assert!(replay_c.pending_replacements.is_empty());
- let committed_generation = replay_c.done[&key].generation.as_deref().unwrap();
- assert_ne!(committed_generation, generation_b);
- let events: Vec<Value> = fs::read_to_string(state_dir.path().join("journal.ndjson"))
- .unwrap()
- .lines()
- .map(|line| serde_json::from_str(line).unwrap())
- .collect();
- let starts: Vec<&str> = events
+ );
+
+ assert!(
+ scan.pdf_spool.is_none(),
+ "an artifact bound to a superseded source generation must not reach phase B"
+ );
+ let categories: Vec<&str> = scan
+ .pdf_spool_fallbacks
.iter()
- .filter(|event| {
- event["kind"] == "file_replace_start"
- && event["file_key"].as_str() == Some(key.as_str())
- })
- .filter_map(|event| event["generation"].as_str())
+ .map(|fallback| fallback.category)
.collect();
- assert_eq!(starts[starts.len() - 2], generation_b);
- assert_eq!(starts.last().copied(), Some(committed_generation));
- drop(replay_c);
- let locked = endpoint.state.lock().unwrap();
- assert_eq!(locked.docs.len(), 2);
- assert!(locked
- .docs
- .values()
- .all(|doc| doc["value"].as_str() == Some("generation-c")));
+ assert_eq!(categories, vec!["source_generation_changed"]);
+ let report = budget.report();
+ assert_eq!(report["artifacts_created"], 1);
+ assert_eq!(report["artifacts_discarded_before_replay"], 1);
+ assert_eq!(report["phase_b_eligible_artifacts"], 0);
+ assert_eq!(report["current_live_artifacts"], 0);
+ assert_eq!(report["current_retained_or_reserved_bytes"], 0);
+ assert_eq!(
+ report["fallback_categories"]["source_generation_changed"],
+ 1
+ );
}
+ #[cfg(unix)]
#[test]
- fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {
+ fn verified_but_junk_pdf_spool_is_created_then_discarded_not_eligible() {
+ use std::os::unix::fs::PermissionsExt;
+
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
- // run_index reaches Journal::file_done; the file_done IO failpoint is
- // a global one-shot, so every file_done-reaching test must hold its
- // lock (armer or not) or it can steal an armed injection. Acquired
- // after FAILPOINT_TEST_LOCK everywhere — one consistent order.
- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
let state_dir = tempfile::tempdir().unwrap();
- let original = "id,value\n0,original\n1,original\n";
- fs::write(corpus.path().join("a.csv"), original).unwrap();
- let endpoint = MockEndpoint::start(usize::MAX);
- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
-
- // a.csv keeps its planned key by path while its OLD bytes reappear as
- // b.csv — whose content key is exactly the planned key a.csv claims. Both
- // files owning one ax_file key would let each replacement delete the
- // other's freshly published documents.
- fs::write(
- corpus.path().join("a.csv"),
- "id,value\n0,rewritten\n1,rewritten\n",
- )
- .unwrap();
- fs::write(corpus.path().join("b.csv"), original).unwrap();
- assert_eq!(run_index(config.clone()).unwrap(), 3);
- {
- let locked = endpoint.state.lock().unwrap();
- assert_eq!(locked.docs.len(), 2);
- assert!(locked.docs.values().all(|doc| {
- doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")
- }));
- }
- let replay = state::Journal::open(
- state_dir.path(),
- &config.root.to_string_lossy(),
- &config.url,
- &config.prefix,
- config.bulk_timeout_secs,
- false,
+ inject_pdf_spool_capacity(state_dir.path());
+ let tools = tempfile::tempdir().unwrap();
+ let pdf = corpus.path().join("empty-report.pdf");
+ fs::write(&pdf, b"%PDF-1.4\nempty test input\n").unwrap();
+ let size = fs::metadata(&pdf).unwrap().len();
+ let digest = content::resolve_reporting(
+ vec![crate::walk::FileEntry {
+ path: pdf.clone(),
+ rel: "empty-report.pdf".into(),
+ rel_id: "empty-report.pdf".into(),
+ is_symlink: false,
+ size,
+ }],
+ &|_| {},
)
- .unwrap();
- assert!(replay.pending_replacements.is_empty());
- assert_eq!(replay.done.len(), 1);
- let plan = replay.plan.clone().unwrap();
- assert_eq!(plan.junk_files.len(), 1);
- assert_eq!(plan.junk_files[0].rel, "b.csv");
- assert!(plan.junk_files[0]
- .reason
- .contains("key ownership exclusive"));
- drop(replay);
+ .unwrap()
+ .digests
+ .remove(0);
+ let worker = tools.path().join("pdf-worker");
+ let script = format!(
+ "#!/bin/sh\nprintf '%s' '{{\"schema\":1,\"extractor\":\"xerj-autoindex/{}\",\"parser\":\"pdf_oxide/0.3.75\",\"containment\":\"test worker\",\"records\":[]}}'\n",
+ env!("CARGO_PKG_VERSION")
+ );
+ fs::write(&worker, script).unwrap();
+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();
+ let _env = PdfWorkerEnvGuard::acquire();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);
- // The divergence is durable and deterministic: an identical rerun keeps
- // the same owner, appends no new plan, and changes no documents.
- let plans_before = event_count(state_dir.path(), "plan");
- assert_eq!(run_index(config).unwrap(), 3);
- assert_eq!(event_count(state_dir.path(), "plan"), plans_before);
- let locked = endpoint.state.lock().unwrap();
- assert_eq!(locked.docs.len(), 2);
- assert!(locked
- .docs
- .values()
- .all(|doc| doc["value"].as_str() == Some("rewritten")));
+ let (budget, _) =
+ crate::extract::pdf::ExtractionSpoolBudget::for_state_dir(state_dir.path(), 1, 1, 8);
+ let progress = Progress::silent();
+ let mut scan = scan_file(
+ &pdf,
+ size,
+ &digest,
+ &PhaseAContext {
+ state_dir: state_dir.path(),
+ budget: &budget,
+ capacity_warning: None,
+ progress: &progress,
+ },
+ 100,
+ 1,
++ false,
+ );
+ assert!(scan.junk.is_some());
+ assert!(scan.pdf_spool.is_some());
+ assert!(take_pdf_spool_if_indexable(&mut scan.pdf_spool, true, &budget).is_none());
+ let report = budget.report();
+ assert_eq!(report["artifacts_created"], 1);
+ assert_eq!(report["artifacts_discarded_before_replay"], 1);
+ assert_eq!(report["phase_b_eligible_artifacts"], 0);
}
+ #[cfg(unix)]
#[test]
- fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {
+ fn multi_pdf_run_reuses_every_artifact_and_reports_exact_success_counters() {
+ use std::os::unix::fs::PermissionsExt;
+
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
- // run_index reaches Journal::file_done; the file_done IO failpoint is
- // a global one-shot, so every file_done-reaching test must hold its
- // lock (armer or not) or it can steal an armed injection. Acquired
- // after FAILPOINT_TEST_LOCK everywhere — one consistent order.
- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
let state_dir = tempfile::tempdir().unwrap();
- fs::write(corpus.path().join("dup-a.csv"), "id,value\n0,dup\n1,dup\n").unwrap();
- fs::write(corpus.path().join("dup-b.csv"), "id,value\n0,dup\n1,dup\n").unwrap();
- fs::write(corpus.path().join("keep.csv"), "id,value\n2,keep\n3,keep\n").unwrap();
- let endpoint = MockEndpoint::start(usize::MAX);
- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
- let starts = event_count(state_dir.path(), "file_replace_start");
-
- // The whole duplicate group disappears. There is no current file left to
- // republish its key, so no replacement intent may be journaled — a
- // stranded intent would be re-appended forever without ever committing.
- fs::remove_file(corpus.path().join("dup-a.csv")).unwrap();
- fs::remove_file(corpus.path().join("dup-b.csv")).unwrap();
- for _ in 0..2 {
- assert_eq!(run_index(config.clone()).unwrap(), 0);
- assert_eq!(
- event_count(state_dir.path(), "file_replace_start"),
- starts,
- "an orphaned duplicate-group key must not schedule a replacement"
- );
- let replay = state::Journal::open(
- state_dir.path(),
- &config.root.to_string_lossy(),
- &config.url,
- &config.prefix,
- config.bulk_timeout_secs,
- false,
+ inject_pdf_spool_capacity(state_dir.path());
+ let tools = tempfile::tempdir().unwrap();
+ const PDFS: usize = 6;
+ for index in 0..PDFS {
+ fs::write(
+ corpus.path().join(format!("quarterly-report-{index}.pdf")),
+ format!("%PDF-1.4\nunique isolated worker input {index}\n"),
)
.unwrap();
- assert!(replay.pending_replacements.is_empty());
}
+
+ let count = tools.path().join("worker-count");
+ let worker = tools.path().join("pdf-worker");
+ let worker_script = String::from_utf8(
+ br#"#!/bin/sh
+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"
+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"title":"quarterly-report","page":1,"body":"Quarterly revenue increased while operating margin improved.","pdf_pages_total":1,"pdf_pages_with_text":1,"pdf_pages_omitted":0},"locator":"p1-s0","group":null,"origin":"extractor"}]}'
+ "#
+ .to_vec(),
+ )
+ .unwrap()
+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));
+ fs::write(&worker, worker_script).unwrap();
+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();
+
+ let endpoint = MockEndpoint::start(0);
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ config.workers = 8;
+ config.pdf_workers = 2;
+ let _env = PdfWorkerEnvGuard::acquire();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);
+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);
+ let (code, report) = run_index_report(config).unwrap();
+ assert_eq!(code, 0);
+ assert_eq!(
+ fs::read_to_string(&count).unwrap().lines().count(),
+ PDFS,
+ "each unique PDF must be parsed once in Phase A and replayed in Phase B"
+ );
+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), PDFS);
+ let reuse = report.unwrap()["pdf_extraction_reuse"].clone();
+ assert_eq!(reuse["reservations_started"], PDFS);
+ assert_eq!(
+ reuse["cumulative_reserved_bytes"],
+ PDFS as u64 * (32_u64 << 20)
+ );
+ assert_eq!(reuse["artifacts_created"], PDFS);
+ assert_eq!(reuse["artifacts_not_created"], 0);
+ assert_eq!(reuse["phase_b_eligible_artifacts"], PDFS);
+ assert_eq!(reuse["artifacts_discarded_before_replay"], 0);
+ assert!(reuse["exact_artifact_bytes"].as_u64().unwrap() > 0);
+ assert_eq!(reuse["current_retained_or_reserved_bytes"], 0);
+ assert!(reuse["peak_retained_or_reserved_bytes"].as_u64().unwrap() > 0);
+ assert_eq!(reuse["current_live_artifacts"], 0);
+ assert!(reuse["peak_live_artifacts"].as_u64().unwrap() > 0);
+ assert_eq!(reuse["phase_a_pdf_parser_responses"], PDFS);
+ assert_eq!(reuse["capacity_fallbacks"], 0);
+ assert_eq!(reuse["io_fallbacks"], 0);
+ assert_eq!(reuse["replay_verified"], PDFS);
+ assert_eq!(reuse["replay_integrity_failures"], 0);
+ assert_eq!(reuse["phase_b_pdf_parses"], 0);
+ assert_eq!(reuse["fallback_examples"].as_array().unwrap().len(), 0);
+ assert_eq!(reuse["fallback_examples_truncated"], false);
+ assert_eq!(reuse["fallback_categories"].as_object().unwrap().len(), 0);
}
+ #[cfg(unix)]
#[test]
- fn partial_file_done_is_rolled_back_before_another_worker_commits() {
+ fn corrupted_pdf_replay_reparses_without_blaming_the_source_or_backend() {
+ use std::os::unix::fs::PermissionsExt;
+
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
let state_dir = tempfile::tempdir().unwrap();
- fs::write(corpus.path().join("a.csv"), "id,value\n0,a-old\n1,a-old\n").unwrap();
- fs::write(corpus.path().join("b.csv"), "id,value\n2,b-old\n3,b-old\n").unwrap();
- let endpoint = MockEndpoint::start(usize::MAX);
- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
- config.workers = 2;
- assert_eq!(run_index(config.clone()).unwrap(), 0);
+ inject_pdf_spool_capacity(state_dir.path());
+ let tools = tempfile::tempdir().unwrap();
+ let pdf = corpus.path().join("report.pdf");
+ fs::write(&pdf, b"%PDF-1.4\ntest worker input\n").unwrap();
+ let worker = tools.path().join("pdf-worker");
+ let count = tools.path().join("worker-count");
+ let worker_script = String::from_utf8(
+ br#"#!/bin/sh
+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"
+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"body":"Quarterly revenue increased while operating margin improved."},"locator":"p1-s0","group":null,"origin":"extractor"}]}'
+ "#
+ .to_vec(),
+ )
+ .unwrap()
+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));
+ fs::write(&worker, worker_script).unwrap();
+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();
- fs::write(corpus.path().join("a.csv"), "id,value\n0,a-new\n1,a-new\n").unwrap();
- fs::write(corpus.path().join("b.csv"), "id,value\n2,b-new\n3,b-new\n").unwrap();
- state::fail_next_file_done_io(3);
- let error = run_index(config.clone()).unwrap_err();
- assert!(format!("{error:#}").contains("partial file_done write"));
+ let endpoint = MockEndpoint::start(0);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let _env = PdfWorkerEnvGuard::acquire();
+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);
+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);
+ crate::extract::pdf::corrupt_replay_for_source_size(fs::metadata(&pdf).unwrap().len());
- // One worker hit the partial append and rolled back while the other was
- // allowed to commit. Replay must remain parseable and expose exactly one
- // pending generation rather than malformed middle corruption.
- let replay = state::Journal::open(
- state_dir.path(),
- &config.root.to_string_lossy(),
- &config.url,
- &config.prefix,
- config.bulk_timeout_secs,
- false,
- )
- .unwrap();
- assert_eq!(replay.pending_replacements.len(), 1);
- assert_eq!(replay.done.len(), 1);
- drop(replay);
+ let (code, report) = run_index_report(config).unwrap();
+ assert_eq!(code, 0);
+ assert_eq!(fs::read_to_string(&count).unwrap().lines().count(), 2);
+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 1);
+ assert_eq!(file_done_count(state_dir.path()), 1);
+ assert!(crate::extract::pdf::corrupted_replay_reservation_was_dropped());
+ let reuse = report.unwrap()["pdf_extraction_reuse"].clone();
+ assert_eq!(reuse["replay_verified"], 0);
+ assert_eq!(reuse["replay_integrity_failures"], 1);
+ assert_eq!(
+ reuse["io_fallbacks"], 0,
+ "a corrupted artifact is an integrity failure, not an I/O failure"
+ );
+ assert_eq!(reuse["phase_b_pdf_parses"], 1);
+ assert_eq!(reuse["current_live_artifacts"], 0);
+ assert_eq!(reuse["current_retained_or_reserved_bytes"], 0);
+ assert_eq!(reuse["fallback_categories"]["replay_verification"], 1);
+ assert_eq!(
+ reuse["fallback_examples"][0]["category"],
+ "replay_verification"
+ );
+ }
- assert_eq!(run_index(config).unwrap(), 0);
- let locked = endpoint.state.lock().unwrap();
- assert_eq!(locked.docs.len(), 4);
- assert!(locked.docs.values().all(|doc| {
- doc["value"]
- .as_str()
- .is_some_and(|value| value.ends_with("-new"))
- }));
+ #[test]
+ fn junk_scan_drops_its_spool_before_indexable_spools_are_retained() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+
+ struct DropProbe(Arc<AtomicUsize>);
+ impl Drop for DropProbe {
+ fn drop(&mut self) {
+ self.0.fetch_add(1, Ordering::SeqCst);
+ }
+ }
+
+ let drops = Arc::new(AtomicUsize::new(0));
+ let budget = crate::extract::pdf::ExtractionSpoolBudget::new(1, 1);
+ let mut junk = Some(DropProbe(Arc::clone(&drops)));
+ assert!(take_pdf_spool_if_indexable(&mut junk, true, &budget).is_none());
+ assert!(junk.is_none());
+ assert_eq!(drops.load(Ordering::SeqCst), 1);
+
+ let mut indexable = Some(DropProbe(Arc::clone(&drops)));
+ let retained = take_pdf_spool_if_indexable(&mut indexable, false, &budget);
+ assert!(retained.is_some());
+ assert_eq!(drops.load(Ordering::SeqCst), 1);
+ let report = budget.report();
+ assert_eq!(report["artifacts_discarded_before_replay"], 1);
+ assert_eq!(
+ report["phase_b_eligible_artifacts"], 0,
+ "eligibility is recorded only after the final todo set is known"
+ );
+ drop(retained);
+ assert_eq!(drops.load(Ordering::SeqCst), 2);
}
diff --cc engine/crates/xerj-autoindex/src/ignore_rules.rs
index 000000000,6731af882..dffe9bf8c
mode 000000,100644..100644
--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs
+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs
@@@ -1,0 -1,950 +1,958 @@@
+ //! Ignore rules for the inventory walk (#276).
+ //!
+ //! The cheapest file to index is the one that is never read. A repository's
+ //! `.gitignore` already says which files are build output; honouring it means
+ //! `target/`, `node_modules/` and friends are pruned before the walk descends
+ //! into them, so they cost nothing at all — not a stat, not a hash, not a bulk
+ //! body.
+ //!
+ //! # Where the semantics come from
+ //!
+ //! gitignore matching is easy to get 80% right and very hard to get right:
+ //! nested ignore files, negation (`!keep.log`), directory-only patterns
+ //! (`build/`), anchoring (`/only-here`) and precedence between an ignore file
+ //! in a subdirectory and one at the root. We do not hand-roll any of that. The
+ //! matcher is ripgrep's `ignore` crate (`Unlicense OR MIT`, compatible with
+ //! XERJ's Apache-2.0), used for its **matching** only — traversal stays on
+ //! `walkdir` so the existing symlink, hidden-file and unreadable-entry
+ //! behaviour of [`crate::walk`] is unchanged, and so we can report *which*
+ //! rule discarded *what*, which `ignore`'s own walker does not expose.
+ //!
+ //! Two behaviours are copied deliberately from that reference implementation
+ //! (`ignore-0.4.32/src/dir.rs`, read at `~/.cargo/registry/src/*/ignore-0.4.32`):
+ //!
+ //! * **Deepest ignore file wins, per kind.** `dir.rs:563-594` walks matchers
+ //! from the current directory outward and keeps the first decision each kind
+ //! produces (`if m_gi.is_none() { m_gi = … }`).
+ //! * **Kinds are ranked, and the rank beats depth.** `dir.rs:678-684` combines
+ //! them `m_custom_ignore.or(m_ignore).or(m_gi).or(m_gi_exclude)`. So a
+ //! `.xerjignore` at the root outranks a `.gitignore` in a deep subdirectory.
+ //! That is what makes `.xerjignore` useful as the stated "exclude things git
+ //! tracks" (and re-include things git ignores) override rather than just a
+ //! second `.gitignore`.
+ //!
+ //! Within one file the last matching pattern wins; that is `Gitignore::matched`
+ //! and we do not reimplement it.
+ //!
+ //! # What is deliberately not honoured
+ //!
+ //! The user's *global* gitignore (`core.excludesFile`) is not read: a machine-
+ //! wide preference silently deciding what is in someone's search index is a
+ //! surprise, and it is not visible in the folder being indexed. `.gitignore`,
+ //! `.xerjignore` and `.git/info/exclude` all live in the tree the user pointed
+ //! at, so they are.
+
+ use ignore::gitignore::{Gitignore, GitignoreBuilder, Glob};
+ use ignore::Match;
+ use std::collections::BTreeMap;
+ use std::path::{Path, PathBuf};
+
+ /// Ignore file honoured in every directory, ranked below `.xerjignore`.
+ pub const GITIGNORE: &str = ".gitignore";
+ /// XERJ's own ignore file. Same syntax as `.gitignore`, higher precedence, so
+ /// a user can exclude something git tracks (or re-include something git
+ /// ignores) without editing `.gitignore`.
+ pub const XERJIGNORE: &str = ".xerjignore";
+
+ /// Build output that is junk in practice but is *not* reliably gitignored —
+ /// `vendor/` is committed by Go projects, `dist/` and `build/` are gitignored
+ /// in one repo and checked in the next, and a folder full of downloaded
+ /// packages is very often not a git repository at all.
+ ///
+ /// These are defaults, not law: `--no-default-ignores` turns them off, any
+ /// ignore file in the tree outranks them, and `--dry-run` names them as the
+ /// reason a directory disappeared.
+ pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
+ "node_modules/",
+ "vendor/",
+ "target/",
+ "dist/",
+ "build/",
+ ".venv/",
+ "__pycache__/",
+ ];
+
+ /// Rule label used for the pre-existing dotfile skip. It is not one of the
+ /// ignore rules and `--no-ignore` does not disable it (it is what keeps `.env`
+ /// and `.ssh` out of a queryable brain), but a user whose file vanished still
+ /// needs it named, so it shares the reporting.
+ pub const HIDDEN_RULE: &str = "hidden:dotfile";
+
+ const MAX_WARNINGS: usize = 20;
+ /// Directory entries the `--dry-run` deep count may examine in total. A
+ /// pathological `node_modules` is not allowed to turn "tell me the plan" into
+ /// minutes of stat storm on a laptop; past the budget the count is reported as
+ /// a floor instead of a total.
+ const DEEP_COUNT_BUDGET: u64 = 1_000_000;
+ /// How far above the root we look for an ignore file that covers the root
+ /// itself. Bounded so a root near `/` cannot walk the whole filesystem.
+ const MAX_ANCESTOR_LEVELS: usize = 64;
+
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+ pub struct IgnoreOptions {
+ /// `--no-ignore` clears this: no `.gitignore`, no `.xerjignore`, no
+ /// defaults. The dotfile skip is separate and stays on.
+ pub enabled: bool,
+ /// `--no-default-ignores` clears this: ignore files still apply,
+ /// [`DEFAULT_IGNORE_PATTERNS`] does not.
+ pub defaults: bool,
+ /// Count the files inside a pruned directory instead of only reporting
+ /// that the directory was pruned. Worth its syscalls in `--dry-run`, whose
+ /// whole job is explaining the plan, and not otherwise — the point of
+ /// pruning is to never touch those paths.
+ pub deep_count: bool,
+ }
+
+ impl Default for IgnoreOptions {
+ fn default() -> Self {
+ Self {
+ enabled: true,
+ defaults: true,
+ deep_count: false,
+ }
+ }
+ }
+
+ impl IgnoreOptions {
+ /// `--no-ignore`.
+ pub const fn off() -> Self {
+ Self {
+ enabled: false,
+ defaults: false,
+ deep_count: false,
+ }
+ }
+
+ /// `--dry-run` explains itself; every other run stays cheap.
+ pub const fn with_deep_count(mut self, deep_count: bool) -> Self {
+ self.deep_count = deep_count;
+ self
+ }
+ }
+
+ #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+ pub struct RuleCount {
+ /// Files matched directly by this rule.
+ pub files: u64,
+ /// Directories pruned by this rule (their contents were never walked).
+ pub dirs: u64,
+ /// Files found inside those pruned directories. Only ever non-zero when
+ /// the deep count ran (`--dry-run`).
+ pub files_inside_pruned_dirs: u64,
+ }
+
+ #[derive(Debug, Clone)]
+ pub struct IgnoreReport {
+ /// rule label → what it discarded.
+ pub by_rule: BTreeMap<String, RuleCount>,
+ pub files_skipped: u64,
+ pub dirs_pruned: u64,
+ pub files_inside_pruned_dirs: u64,
+ /// True unless the deep count hit [`DEEP_COUNT_BUDGET`]. When false,
+ /// `files_inside_pruned_dirs` is a floor and every line that prints it
+ /// says "at least".
+ pub deep_count_complete: bool,
+ /// Whether the deep count ran at all. When it did not, no file count
+ /// inside a pruned directory is printed — an unmeasured number is not
+ /// printed as zero.
+ pub deep_count_ran: bool,
+ pub ignore_files_read: u64,
+ /// Set when the folder the user pointed at is itself covered by an ignore
+ /// rule. It is indexed anyway; this is the "and here is why you are seeing
+ /// files you thought were ignored" line.
+ pub root_exemption: Option<String>,
+ /// Unreadable or unparseable ignore files. Never fatal: an ignore file we
+ /// cannot read means we index more, not less.
+ pub warnings: Vec<String>,
+ }
+
+ impl Default for IgnoreReport {
+ fn default() -> Self {
+ Self {
+ by_rule: BTreeMap::new(),
+ files_skipped: 0,
+ dirs_pruned: 0,
+ files_inside_pruned_dirs: 0,
+ deep_count_complete: true,
+ deep_count_ran: false,
+ ignore_files_read: 0,
+ root_exemption: None,
+ warnings: Vec::new(),
+ }
+ }
+ }
+
+ impl IgnoreReport {
+ pub fn nothing_to_report(&self) -> bool {
+ self.by_rule.is_empty() && self.root_exemption.is_none() && self.warnings.is_empty()
+ }
+
+ /// Human-readable lines for the progress surface. Every number here was
+ /// counted by this run; the ones that were deliberately not counted are
+ /// named as such instead of being printed as zero.
+ pub fn summary_lines(&self) -> Vec<String> {
+ let mut out = Vec::new();
+ for warning in &self.warnings {
+ out.push(format!("ignore rules: {warning}"));
+ }
+ if let Some(reason) = &self.root_exemption {
+ out.push(format!("ignore rules: {reason}"));
+ }
+ if self.by_rule.is_empty() {
+ return out;
+ }
+ let mut headline = format!(
+ "ignore rules: skipped {} file{} and pruned {} director{}",
+ self.files_skipped,
+ plural(self.files_skipped),
+ self.dirs_pruned,
+ if self.dirs_pruned == 1 { "y" } else { "ies" },
+ );
+ if self.deep_count_ran && self.dirs_pruned > 0 {
+ headline.push_str(&format!(
+ " ({}{} file{} inside them)",
+ if self.deep_count_complete {
+ ""
+ } else {
+ "at least "
+ },
+ self.files_inside_pruned_dirs,
+ plural(self.files_inside_pruned_dirs),
+ ));
+ } else if self.dirs_pruned > 0 {
+ headline.push_str(" (contents never walked, so not counted)");
+ }
+ if self.ignore_files_read > 0 {
+ headline.push_str(&format!(
+ "; {} ignore file{} read",
+ self.ignore_files_read,
+ plural(self.ignore_files_read)
+ ));
+ }
+ out.push(headline);
+
+ let mut ranked: Vec<(&String, &RuleCount)> = self.by_rule.iter().collect();
+ ranked.sort_by(|a, b| {
+ let weight = |c: &RuleCount| c.files + c.files_inside_pruned_dirs + c.dirs;
+ weight(b.1).cmp(&weight(a.1)).then_with(|| a.0.cmp(b.0))
+ });
+ const SHOWN: usize = 10;
+ for (label, count) in ranked.iter().take(SHOWN) {
+ let mut parts = Vec::new();
+ if count.files > 0 {
+ parts.push(format!("{} file{}", count.files, plural(count.files)));
+ }
+ if count.dirs > 0 {
+ let inside = if self.deep_count_ran {
+ format!(
+ " ({}{} file{} inside)",
+ if self.deep_count_complete {
+ ""
+ } else {
+ "at least "
+ },
+ count.files_inside_pruned_dirs,
+ plural(count.files_inside_pruned_dirs)
+ )
+ } else {
+ String::new()
+ };
+ parts.push(format!(
+ "{} director{} pruned{inside}",
+ count.dirs,
+ if count.dirs == 1 { "y" } else { "ies" }
+ ));
+ }
+ out.push(format!("ignore rules: {label} — {}", parts.join(", ")));
+ }
+ if ranked.len() > SHOWN {
+ out.push(format!(
+ "ignore rules: … and {} more rule{}",
+ ranked.len() - SHOWN,
+ plural((ranked.len() - SHOWN) as u64)
+ ));
+ }
+ out
+ }
+ }
+
+ fn plural(n: u64) -> &'static str {
+ if n == 1 {
+ ""
+ } else {
+ "s"
+ }
+ }
+
+ #[derive(Default)]
+ struct Layer {
+ xerj: Option<Gitignore>,
+ git: Option<Gitignore>,
+ exclude: Option<Gitignore>,
+ }
+
+ /// A stack of ignore matchers, one entry per directory on the current
+ /// root-to-here path, plus the built-in defaults.
+ ///
+ /// The caller drives it: [`IgnoreStack::enter_dir`] when a directory is
+ /// yielded, [`IgnoreStack::skip_dir`] / [`IgnoreStack::skip_file`] to ask
+ /// whether an entry survives.
+ pub struct IgnoreStack {
+ opts: IgnoreOptions,
+ root: PathBuf,
+ layers: Vec<Layer>,
+ defaults: Option<Gitignore>,
+ budget: u64,
+ pub report: IgnoreReport,
+ }
+
+ impl IgnoreStack {
+ /// `root` must be the canonicalised root folder.
+ pub fn new(root: &Path, opts: IgnoreOptions) -> Self {
+ let mut stack = Self {
+ opts,
+ root: root.to_path_buf(),
+ layers: Vec::new(),
+ defaults: None,
+ budget: DEEP_COUNT_BUDGET,
+ report: IgnoreReport {
+ deep_count_ran: opts.deep_count,
+ ..IgnoreReport::default()
+ },
+ };
+ if opts.enabled && opts.defaults {
+ let defaults = stack.build_defaults(root);
+ stack.defaults = defaults;
+ }
+ let exemption = stack.root_exemption(root);
+ stack.report.root_exemption = exemption;
+ stack
+ }
+
+ fn build_defaults(&mut self, root: &Path) -> Option<Gitignore> {
+ let mut builder = GitignoreBuilder::new(root);
+ for pattern in DEFAULT_IGNORE_PATTERNS {
+ if let Err(err) = builder.add_line(None, pattern) {
+ self.warn(format!("built-in rule `{pattern}` is not usable: {err}"));
+ }
+ }
+ match builder.build() {
+ Ok(gi) if gi.is_empty() => None,
+ Ok(gi) => Some(gi),
+ Err(err) => {
+ self.warn(format!("built-in default rules are unusable: {err}"));
+ None
+ }
+ }
+ }
+
+ /// Detect that the folder the user pointed at is itself ignored.
+ ///
+ /// We never act on it — pointing autoindex at `~/proj/target` is an
+ /// explicit instruction and silently indexing nothing would be the worst
+ /// possible answer. We only explain it, because the alternative is a user
+ /// staring at a brain full of build output wondering why their ignore file
+ /// did nothing.
+ fn root_exemption(&self, root: &Path) -> Option<String> {
+ if !self.opts.enabled {
+ return None;
+ }
+ let name = root.file_name()?;
+ let parent = root.parent()?;
+ // The defaults are anchored at the root itself, so they cannot see the
+ // root's own name. Re-anchor a copy one level up to ask the question.
+ if self.opts.defaults {
+ let mut builder = GitignoreBuilder::new(parent);
+ for pattern in DEFAULT_IGNORE_PATTERNS {
+ let _ = builder.add_line(None, pattern);
+ }
+ if let Ok(gi) = builder.build() {
+ if let Match::Ignore(glob) = gi.matched(Path::new(name), true) {
+ return Some(format!(
+ "{} matches the built-in default rule `{}`, but you pointed autoindex at \
+ it, so it is being indexed — pass --no-default-ignores to silence this \
+ note",
+ root.display(),
+ glob.original()
+ ));
+ }
+ }
+ }
+ // Ignore files *above* the root are consulted for this note only, and
+ // never to prune: git itself stops at the repository root, so we do
+ // too (plus a hard level cap).
+ let mut dir = parent.to_path_buf();
+ for _ in 0..MAX_ANCESTOR_LEVELS {
+ for file in [XERJIGNORE, GITIGNORE] {
+ let path = dir.join(file);
+ if !path.is_file() {
+ continue;
+ }
+ let mut builder = GitignoreBuilder::new(&dir);
+ let _ = builder.add(&path);
+ let Ok(gi) = builder.build() else { continue };
+ if gi.is_empty() {
+ continue;
+ }
+ if let Match::Ignore(glob) = gi.matched_path_or_any_parents(root, true) {
+ return Some(format!(
+ "{} is ignored by `{}` in {}, but you pointed autoindex at it, so it is \
+ being indexed",
+ root.display(),
+ glob.original(),
+ path.display()
+ ));
+ }
+ }
+ if dir.join(".git").exists() {
+ break;
+ }
+ match dir.parent() {
+ Some(up) => dir = up.to_path_buf(),
+ None => break,
+ }
+ }
+ None
+ }
+
+ /// Drop the layers of directories the walk has finished with.
+ ///
+ /// Must be called for **every** entry, not only directories. A depth-first
+ /// walk interleaves files and subdirectories in readdir order, so after it
+ /// descends into `a/` and comes back it can hand us `b.txt` at the shallower
+ /// depth while `a/`'s layer is still on the stack. Without this, `a/`'s
+ /// `.gitignore` would silently judge its parent's files.
+ pub fn truncate_to(&mut self, depth: usize) {
+ self.layers.truncate(depth);
+ }
+
+ /// Called for every directory the walk yields, root first, before any of
+ /// its children. `depth` is walkdir's depth, so the stack is indexed by
+ /// depth and a sibling directory replaces its predecessor's layer.
+ pub fn enter_dir(&mut self, dir: &Path, depth: usize) {
+ self.layers.truncate(depth);
+ let layer = if self.opts.enabled {
+ Layer {
+ xerj: self.load(dir, dir.join(XERJIGNORE)),
+ git: self.load(dir, dir.join(GITIGNORE)),
+ // Per-repository excludes, which is where people put "junk
+ // only I have". Loaded per directory so a nested checkout gets
+ // its own.
+ exclude: self.load(dir, dir.join(".git").join("info").join("exclude")),
+ }
+ } else {
+ Layer::default()
+ };
+ self.layers.push(layer);
+ }
+
+ fn load(&mut self, dir: &Path, file: PathBuf) -> Option<Gitignore> {
+ if !file.is_file() {
+ return None;
+ }
+ let mut builder = GitignoreBuilder::new(dir);
+ if let Some(err) = builder.add(&file) {
+ self.warn(format!("{}: {err}", file.display()));
+ }
+ match builder.build() {
+ Ok(gi) => {
+ self.report.ignore_files_read += 1;
+ if gi.is_empty() {
+ None
+ } else {
+ Some(gi)
+ }
+ }
+ Err(err) => {
+ self.warn(format!("{} is not usable: {err}", file.display()));
+ None
+ }
+ }
+ }
+
+ fn warn(&mut self, message: String) {
+ if self.report.warnings.len() < MAX_WARNINGS {
+ self.report.warnings.push(message);
+ }
+ }
+
+ /// ripgrep's precedence, reproduced: the deepest match *per kind*, then
+ /// the kinds ranked `.xerjignore` > `.gitignore` > `.git/info/exclude` >
+ /// built-in defaults (`ignore-0.4.32/src/dir.rs:563-594` and `:678-684`).
+ fn lookup(&self, path: &Path, is_dir: bool) -> Option<(bool, String)> {
+ if !self.opts.enabled {
+ return None;
+ }
+ let mut xerj = None;
+ let mut git = None;
+ let mut exclude = None;
+ for layer in self.layers.iter().rev() {
+ if xerj.is_none() {
+ xerj = self.decide_with(layer.xerj.as_ref(), path, is_dir);
+ }
+ if git.is_none() {
+ git = self.decide_with(layer.git.as_ref(), path, is_dir);
+ }
+ if exclude.is_none() {
+ exclude = self.decide_with(layer.exclude.as_ref(), path, is_dir);
+ }
+ if xerj.is_some() {
+ // Nothing ranked lower can change the answer.
+ break;
+ }
+ }
+ xerj.or(git)
+ .or(exclude)
+ .or_else(|| self.decide_with(self.defaults.as_ref(), path, is_dir))
+ }
+
+ fn decide_with(
+ &self,
+ matcher: Option<&Gitignore>,
+ path: &Path,
+ is_dir: bool,
+ ) -> Option<(bool, String)> {
+ match matcher?.matched(path, is_dir) {
+ Match::None => None,
+ Match::Ignore(glob) => Some((true, self.label(glob))),
+ Match::Whitelist(glob) => Some((false, self.label(glob))),
+ }
+ }
+
+ fn label(&self, glob: &Glob) -> String {
+ let from = glob
+ .from()
+ .map(|p| self.rel(p))
+ .unwrap_or_else(|| "<built-in>".to_string());
+ format!("{from}:{}", glob.original())
+ }
+
+ fn rel(&self, path: &Path) -> String {
+ path.strip_prefix(&self.root)
+ .unwrap_or(path)
+ .to_string_lossy()
+ .replace('\\', "/")
+ }
+
+ /// Ask whether a directory must be pruned. `Some(label)` means do not
+ /// descend; the report has already been updated.
+ pub fn skip_dir(&mut self, path: &Path) -> Option<String> {
+ let (ignored, label) = self.lookup(path, true)?;
+ if !ignored {
+ return None;
+ }
+ self.record_dir(path, &label);
+ Some(label)
+ }
+
+ /// Ask whether a file must be skipped.
+ pub fn skip_file(&mut self, path: &Path) -> Option<String> {
+ let (ignored, label) = self.lookup(path, false)?;
+ if !ignored {
+ return None;
+ }
+ self.record_file(&label);
+ Some(label)
+ }
+
+ /// The dotfile skip is decided by the walker, not here, but it is reported
+ /// through the same accounting so `--dry-run` explains every missing file
+ /// with one mechanism.
+ pub fn record_hidden(&mut self, path: &Path, is_dir: bool) {
+ if is_dir {
+ self.record_dir(path, HIDDEN_RULE);
+ } else {
+ self.record_file(HIDDEN_RULE);
+ }
+ }
+
++ /// The marker-gated generated-dir prune (`walk::MARKER_GENERATED_DIRS`)
++ /// is also decided by the walker — the verdict needs the SIBLING marker
++ /// file, which the per-directory matcher stack cannot see — and reported
++ /// through the same accounting.
++ pub fn record_marker_dir(&mut self, path: &Path, label: &str) {
++ self.record_dir(path, label);
++ }
++
+ fn record_file(&mut self, label: &str) {
+ self.report.files_skipped += 1;
+ self.report
+ .by_rule
+ .entry(label.to_string())
+ .or_default()
+ .files += 1;
+ }
+
+ fn record_dir(&mut self, path: &Path, label: &str) {
+ self.report.dirs_pruned += 1;
+ self.report
+ .by_rule
+ .entry(label.to_string())
+ .or_default()
+ .dirs += 1;
+ let inside = self.count_inside(path);
+ self.report.files_inside_pruned_dirs += inside;
+ self.report
+ .by_rule
+ .entry(label.to_string())
+ .or_default()
+ .files_inside_pruned_dirs += inside;
+ }
+
+ /// Only under `--dry-run`. Symlinks are never followed here whatever the
+ /// run's `--follow-symlinks` setting: this is an accounting pass over a
+ /// directory we have already decided not to index, and it must not be able
+ /// to wander outside it or loop.
+ ///
+ /// The dotfile rule is applied here too, so the number means "files that
+ /// would otherwise have been indexed" and not "inodes under this path".
+ /// Without it a pruned directory containing a nested checkout reports its
+ /// whole `.git`, which was never indexable in the first place — a number
+ /// that does not reconcile with the before/after file count is worse than
+ /// no number.
+ fn count_inside(&mut self, dir: &Path) -> u64 {
+ if !self.opts.deep_count {
+ return 0;
+ }
+ let mut files = 0;
+ let walker = walkdir::WalkDir::new(dir)
+ .follow_links(false)
+ .min_depth(1)
+ .into_iter()
+ .filter_entry(|e| {
+ !e.file_name()
+ .to_str()
+ .is_some_and(|name| name.starts_with('.'))
+ });
+ for entry in walker {
+ if self.budget == 0 {
+ self.report.deep_count_complete = false;
+ break;
+ }
+ self.budget -= 1;
+ if entry.is_ok_and(|e| e.file_type().is_file()) {
+ files += 1;
+ }
+ }
+ files
+ }
+ }
+
+ #[cfg(test)]
+ mod tests {
+ use super::*;
+ use std::fs;
+
+ /// Drive the stack the way `crate::walk` does, over a sorted listing, and
+ /// return the surviving root-relative file paths.
+ fn survivors(root: &Path, opts: IgnoreOptions) -> (Vec<String>, IgnoreReport) {
+ let root = root.canonicalize().unwrap();
+ let mut stack = IgnoreStack::new(&root, opts);
+ let mut kept = Vec::new();
+ let mut it = walkdir::WalkDir::new(&root).into_iter();
+ while let Some(next) = it.next() {
+ let Ok(entry) = next else { continue };
+ let is_dir = entry.file_type().is_dir();
+ if entry.depth() > 0
+ && entry
+ .file_name()
+ .to_str()
+ .is_some_and(|n| n.starts_with('.'))
+ {
+ stack.record_hidden(entry.path(), is_dir);
+ if is_dir {
+ it.skip_current_dir();
+ }
+ continue;
+ }
+ stack.truncate_to(entry.depth());
+ if is_dir {
+ if entry.depth() > 0 && stack.skip_dir(entry.path()).is_some() {
+ it.skip_current_dir();
+ continue;
+ }
+ stack.enter_dir(entry.path(), entry.depth());
+ continue;
+ }
+ if !entry.file_type().is_file() {
+ continue;
+ }
+ if stack.skip_file(entry.path()).is_some() {
+ continue;
+ }
+ kept.push(
+ entry
+ .path()
+ .strip_prefix(&root)
+ .unwrap()
+ .to_string_lossy()
+ .replace('\\', "/"),
+ );
+ }
+ kept.sort();
+ (kept, stack.report)
+ }
+
+ fn write(root: &Path, rel: &str, body: &str) {
+ let path = root.join(rel);
+ fs::create_dir_all(path.parent().unwrap()).unwrap();
+ fs::write(path, body).unwrap();
+ }
+
+ #[test]
+ fn root_gitignore_prunes_and_negation_re_includes() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".gitignore", "*.log\n!keep.log\nbuild/\n");
+ write(root, "a.log", "x");
+ write(root, "keep.log", "x");
+ write(root, "src/main.rs", "x");
+ write(root, "src/nested.log", "x");
+ write(root, "build/out.bin", "x");
+
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["keep.log", "src/main.rs"], "{kept:?}");
+ // `*.log` has no slash, so it matches at any depth — git's rule.
+ assert!(
+ report.by_rule.contains_key(".gitignore:*.log"),
+ "{report:?}"
+ );
+ assert!(
+ report.by_rule.contains_key(".gitignore:build/"),
+ "{report:?}"
+ );
+ assert_eq!(report.dirs_pruned, 1);
+ assert_eq!(
+ report.by_rule[".gitignore:*.log"].files, 2,
+ "a.log and src/nested.log"
+ );
+ // 3, not 2: `.gitignore` is itself a dotfile, so the pre-existing
+ // hidden-file rule accounts for it in the same report.
+ assert_eq!(report.files_skipped, 3, "{report:?}");
+ assert_eq!(report.by_rule[HIDDEN_RULE].files, 1, "{report:?}");
+ }
+
+ #[test]
+ fn a_nested_gitignore_outranks_the_root_one() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".gitignore", "*.log\n");
+ write(root, "logs/.gitignore", "!*.log\n");
+ write(root, "top.log", "x");
+ write(root, "logs/kept.log", "x");
+
+ let (kept, _) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["logs/kept.log"], "{kept:?}");
+ }
+
+ #[test]
+ fn xerjignore_outranks_gitignore_at_any_depth() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ // First line: git tracks it, the user does not want it in the index.
+ // Second: git ignores it, the user *does* want it — and a root
+ // .xerjignore outranks a deep .gitignore, which is ripgrep's rule.
+ write(
+ root,
+ ".xerjignore",
+ "tracked-but-huge.csv\n!deep/wanted.bin\n",
+ );
+ write(root, "deep/.gitignore", "*.bin\n");
+ write(root, "tracked-but-huge.csv", "x");
+ write(root, "deep/wanted.bin", "x");
+ write(root, "deep/other.bin", "x");
+ write(root, "ok.txt", "x");
+
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["deep/wanted.bin", "ok.txt"], "{kept:?}");
+ assert!(
+ report
+ .by_rule
+ .contains_key(".xerjignore:tracked-but-huge.csv"),
+ "{report:?}"
+ );
+ }
+
+ #[test]
+ fn defaults_prune_build_output_that_no_ignore_file_mentions() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, "index.js", "x");
+ write(root, "node_modules/left-pad/index.js", "x");
+ write(root, "target/debug/huge.rlib", "x");
+ write(root, "__pycache__/mod.cpython-311.pyc", "x");
+
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["index.js"], "{kept:?}");
+ assert_eq!(report.dirs_pruned, 3);
+ assert!(
+ report.by_rule.contains_key("<built-in>:node_modules/"),
+ "{report:?}"
+ );
+
+ let (all, _) = survivors(
+ root,
+ IgnoreOptions {
+ defaults: false,
+ ..IgnoreOptions::default()
+ },
+ );
+ assert_eq!(all.len(), 4, "--no-default-ignores must keep them: {all:?}");
+ }
+
+ #[test]
+ fn an_ignore_file_outranks_a_default() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ // A Go project that commits its vendor tree.
+ write(root, ".gitignore", "!vendor/\n");
+ write(root, "vendor/lib/lib.go", "x");
+ let (kept, _) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["vendor/lib/lib.go"], "{kept:?}");
+ }
+
+ #[test]
+ fn no_ignore_disables_every_rule_but_not_the_dotfile_skip() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".gitignore", "*.log\n");
+ write(root, "a.log", "x");
+ write(root, "node_modules/x.js", "x");
+ write(root, ".env", "SECRET=1");
+
+ let (kept, report) = survivors(root, IgnoreOptions::off());
+ assert_eq!(kept, vec!["a.log", "node_modules/x.js"], "{kept:?}");
+ assert!(!kept.iter().any(|k| k == ".env"), "{kept:?}");
+ assert_eq!(report.by_rule.get(HIDDEN_RULE).unwrap().files, 2);
+ }
+
+ #[test]
+ fn git_info_exclude_is_honoured() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".git/info/exclude", "scratch.txt\n");
+ write(root, "scratch.txt", "x");
+ write(root, "real.txt", "x");
+ let (kept, _) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["real.txt"], "{kept:?}");
+ }
+
+ #[test]
+ fn an_ignored_root_is_indexed_anyway_and_says_why() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let repo = dir.path().join("repo");
+ fs::create_dir_all(repo.join(".git")).unwrap();
+ write(&repo, ".gitignore", "generated/\n");
+ write(&repo, "generated/out.o", "x");
+
+ let root = repo.join("generated");
+ let (kept, report) = survivors(&root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["out.o"], "an explicit root must be indexed");
+ let reason = report.root_exemption.expect("must explain itself");
+ assert!(reason.contains("generated/"), "{reason}");
+ assert!(reason.contains(".gitignore"), "{reason}");
+ }
+
+ #[test]
+ fn a_root_named_like_a_default_is_indexed_anyway_and_says_why() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path().join("node_modules");
+ fs::create_dir_all(&root).unwrap();
+ write(&root, "left-pad/index.js", "x");
+ let (kept, report) = survivors(&root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["left-pad/index.js"], "{kept:?}");
+ let reason = report.root_exemption.expect("must explain itself");
+ assert!(reason.contains("node_modules/"), "{reason}");
+ }
+
+ #[test]
+ fn deep_count_is_off_by_default_and_counts_under_dry_run() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, "keep.txt", "x");
+ for i in 0..7 {
+ write(root, &format!("node_modules/pkg/f{i}.js"), "x");
+ }
+
+ let (_, cheap) = survivors(root, IgnoreOptions::default());
+ assert_eq!(cheap.files_inside_pruned_dirs, 0);
+ assert!(!cheap.deep_count_ran);
+ assert!(
+ cheap
+ .summary_lines()
+ .iter()
+ .any(|l| l.contains("contents never walked")),
+ "{:?}",
+ cheap.summary_lines()
+ );
+
+ let (_, deep) = survivors(root, IgnoreOptions::default().with_deep_count(true));
+ assert_eq!(deep.files_inside_pruned_dirs, 7);
+ assert!(deep.deep_count_complete);
+ assert!(
+ deep.summary_lines().iter().any(|l| l.contains("7 files")),
+ "{:?}",
+ deep.summary_lines()
+ );
+ }
+
+ /// Regression, order-independent: a depth-first walk interleaves files and
+ /// subdirectories, so a nested `.gitignore` must be off the stack again by
+ /// the time the walk hands back a file from the parent directory. Driven
+ /// through the exact call sequence `crate::walk` makes, because readdir
+ /// order is not something a test can pin.
+ #[test]
+ fn a_nested_ignore_file_never_judges_its_parents_files() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path().canonicalize().unwrap();
+ write(&root, "a/.gitignore", "*.txt\n");
+ write(&root, "a/inside.txt", "x");
+ write(&root, "b.txt", "x");
+
+ let mut stack = IgnoreStack::new(&root, IgnoreOptions::default());
+ stack.truncate_to(0);
+ stack.enter_dir(&root, 0);
+ // …descend into a/ …
+ stack.truncate_to(1);
+ stack.enter_dir(&root.join("a"), 1);
+ stack.truncate_to(2);
+ assert!(
+ stack
+ .skip_file(&root.join("a").join("inside.txt"))
+ .is_some(),
+ "a/.gitignore must apply inside a/"
+ );
+ // …and come back out to a sibling file of a/.
+ stack.truncate_to(1);
+ assert!(
+ stack.skip_file(&root.join("b.txt")).is_none(),
+ "a/.gitignore leaked onto the parent's files"
+ );
+ }
+
+ /// A broken ignore file must never be able to make files disappear
+ /// silently: it is reported, and its only possible effect is that we index
+ /// MORE than the user asked for, never less.
+ #[test]
+ fn an_unreadable_ignore_file_warns_and_indexes_more_not_less() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::write(root.join(".gitignore"), b"\xff\xfe not text\n*.txt\n").unwrap();
+ fs::write(root.join("a.txt"), "x").unwrap();
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(
+ kept,
+ vec!["a.txt"],
+ "an ignore file we cannot read must not hide anything"
+ );
+ assert!(!report.warnings.is_empty(), "must warn: {report:?}");
+ assert!(
+ report
+ .summary_lines()
+ .iter()
+ .any(|l| l.contains(".gitignore")),
+ "the warning must reach the user: {:?}",
+ report.summary_lines()
+ );
+ }
+ }
diff --cc engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 000000000,ae70c788d..ab7aa2474
mode 000000,100644..100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@@ -1,0 -1,1277 +1,1283 @@@
+ //! Real-HTTP end-to-end coverage for incremental corpus generations.
+ //!
+ //! The endpoint below is deliberately stateful. It applies bulk index/update
+ //! actions, delete-by-query, and exact count searches so `run_index` traverses
+ //! the same HTTP client and validation barriers as a real server.
+
+ use super::*;
+ use std::collections::HashMap;
+ use std::fs;
+ use std::io::{BufRead, BufReader, Read, Write};
+ use std::net::{TcpListener, TcpStream};
+ use std::path::Path;
+ use std::sync::{Arc, Mutex};
+ use std::thread;
+
+ static HTTP_E2E_LOCK: Mutex<()> = Mutex::new(());
+
+ #[derive(Default)]
+ struct HttpState {
+ docs: HashMap<(String, String), Value>,
+ requests: Vec<(String, String)>,
+ data_bulk_requests: usize,
+ fail_next_data_bulk: bool,
+ /// Apply the first half of the next data bulk's items and *then* report
+ /// `errors: true`. This is the dangerous backend shape the legacy
+ /// replacement transaction was hardened against (visibility changed, but
+ /// the durable record must not be): `no_graph` corpora take the generated
+ /// executor now, so the shape has to be proven here too.
+ partially_apply_next_data_bulk: bool,
+ fail_embedding_identity: bool,
+ stop: bool,
+ embedding_identity_sha256: String,
+ saw_dataset_mapping_update: bool,
+ saw_catalog_mapping_update: bool,
+ }
+
+ struct HttpEndpoint {
+ url: String,
+ state: Arc<Mutex<HttpState>>,
+ join: Option<thread::JoinHandle<()>>,
+ }
+
+ impl HttpEndpoint {
+ fn start() -> Self {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ listener.set_nonblocking(true).unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let state = Arc::new(Mutex::new(HttpState {
+ embedding_identity_sha256: "a".repeat(64),
+ ..HttpState::default()
+ }));
+ let server_state = Arc::clone(&state);
+ let join = thread::spawn(move || loop {
+ match listener.accept() {
- Ok((stream, _)) => handle_http(stream, &server_state),
++ Ok((stream, _)) => {
++ // BSD/macOS: accepted sockets inherit the listener's
++ // O_NONBLOCK; the handler does blocking reads.
++ stream.set_nonblocking(false).unwrap();
++ handle_http(stream, &server_state)
++ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+ if server_state.lock().unwrap().stop {
+ break;
+ }
+ thread::yield_now();
+ }
+ Err(error) => panic!("HTTP test endpoint accept failed: {error}"),
+ }
+ });
+ Self {
+ url,
+ state,
+ join: Some(join),
+ }
+ }
+
+ fn data_docs(&self) -> Vec<Value> {
+ self.state
+ .lock()
+ .unwrap()
+ .docs
+ .iter()
+ .filter(|((index, _), _)| index != catalog::CATALOG_INDEX)
+ .map(|(_, doc)| doc.clone())
+ .collect()
+ }
+
+ fn data_bulk_requests(&self) -> usize {
+ self.state.lock().unwrap().data_bulk_requests
+ }
+ }
+
+ impl Drop for HttpEndpoint {
+ fn drop(&mut self) {
+ self.state.lock().unwrap().stop = true;
+ let _ = TcpStream::connect(self.url.trim_start_matches("http://"));
+ self.join.take().unwrap().join().unwrap();
+ }
+ }
+
+ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {
+ let mut reader = BufReader::new(stream.try_clone().unwrap());
+ let mut request_line = String::new();
+ if reader.read_line(&mut request_line).unwrap() == 0 {
+ return;
+ }
+ let mut content_length = 0usize;
+ loop {
+ let mut line = String::new();
+ reader.read_line(&mut line).unwrap();
+ if line == "\r\n" || line.is_empty() {
+ break;
+ }
+ if let Some(value) = line
+ .to_ascii_lowercase()
+ .strip_prefix("content-length:")
+ .map(str::trim)
+ {
+ content_length = value.parse().unwrap();
+ }
+ }
+ let mut body = vec![0; content_length];
+ reader.read_exact(&mut body).unwrap();
+ let mut parts = request_line.split_whitespace();
+ let method = parts.next().unwrap_or("");
+ let path = parts.next().unwrap_or("");
+ state
+ .lock()
+ .unwrap()
+ .requests
+ .push((method.to_owned(), path.to_owned()));
+
+ let (status, response) = if method == "GET" && path == "/v1/embedding/identity" {
+ let locked = state.lock().unwrap();
+ if locked.fail_embedding_identity {
+ (
+ 500,
+ json!({"error": {
+ "type": "injected_identity_failure",
+ "reason": "genesis recovery boundary"
+ }}),
+ )
+ } else {
+ (
+ 200,
+ json!({"data": {
+ "version": 1,
+ "backend": "lexical",
+ "identity_sha256": locked.embedding_identity_sha256.clone(),
+ "dimensions": 384,
+ "semantic_contract": "semantic_text-derived-vector.v1",
+ "resumable": true
+ }, "took_ms": 0, "request_id": "incremental-http-test"}),
+ )
+ }
+ } else if method == "POST" && path == "/_bulk" {
+ let locked = state.lock().unwrap();
+ assert!(
+ locked.saw_dataset_mapping_update && locked.saw_catalog_mapping_update,
+ "bulk publication preceded exact generation mapping updates"
+ );
+ drop(locked);
+ bulk_http(&body, state)
+ } else if method == "POST" && path.contains("/_delete_by_query") {
+ let deleted = delete_http(path, &body, state);
+ (200, json!({"deleted": deleted, "failures": []}))
+ } else if method == "POST" && path.ends_with("/_search") {
+ (200, search_http(path, &body, state))
+ } else if method == "GET" && path.ends_with("/_count") {
+ let index = path.trim_start_matches('/').trim_end_matches("/_count");
+ let count = state
+ .lock()
+ .unwrap()
+ .docs
+ .keys()
+ .filter(|(candidate, _)| candidate == index)
+ .count();
+ (200, json!({"count": count}))
+ } else if method == "PUT" {
+ let value: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
+ let mut locked = state.lock().unwrap();
+ if path.ends_with("/_mapping") {
+ assert!(value.get("properties").is_some());
+ assert!(value.get("mappings").is_none());
+ if path.starts_with(&format!("/{}/", catalog::CATALOG_INDEX)) {
+ locked.saw_catalog_mapping_update = true;
+ } else {
+ locked.saw_dataset_mapping_update = true;
+ }
+ } else {
+ assert!(value.pointer("/mappings/properties").is_some());
+ assert!(value.get("properties").is_none());
+ }
+ (200, json!({"acknowledged": true}))
+ } else {
+ // Readiness, index creation/mapping, and refresh.
+ (200, json!({"acknowledged": true}))
+ };
+ let bytes = response.to_string();
+ write!(
+ stream,
+ "HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
+ if status == 200 {
+ "OK"
+ } else {
+ "Internal Server Error"
+ },
+ bytes.len(),
+ bytes
+ )
+ .unwrap();
+ }
+
+ fn bulk_http(body: &[u8], state: &Arc<Mutex<HttpState>>) -> (u16, Value) {
+ let lines: Vec<&[u8]> = body
+ .split(|byte| *byte == b'\n')
+ .filter(|line| !line.is_empty())
+ .collect();
+ let is_data = lines.iter().step_by(2).any(|line| {
+ serde_json::from_slice::<Value>(line)
+ .ok()
+ .and_then(|action| {
+ action
+ .pointer("/index/_index")
+ .or_else(|| action.pointer("/update/_index"))
+ .and_then(Value::as_str)
+ .map(|index| index != catalog::CATALOG_INDEX)
+ })
+ .unwrap_or(false)
+ });
+ let mut locked = state.lock().unwrap();
+ if is_data {
+ locked.data_bulk_requests += 1;
+ if std::mem::take(&mut locked.fail_next_data_bulk) {
+ return (
+ 200,
+ json!({
+ "errors": true,
+ "items": [{"index": {
+ "status": 500,
+ "error": {
+ "type": "injected_failure",
+ "reason": "leave durable generation pending"
+ }
+ }}]
+ }),
+ );
+ }
+ }
+ // Half-applied bulk: apply the leading actions, then report failure.
+ let applied_limit = if is_data && std::mem::take(&mut locked.partially_apply_next_data_bulk) {
+ Some(lines.len() / 2)
+ } else {
+ None
+ };
+ let mut cursor = 0;
+ while cursor < lines.len() {
+ if applied_limit.is_some_and(|limit| cursor >= limit) {
+ break;
+ }
+ let action: Value = serde_json::from_slice(lines[cursor]).unwrap();
+ cursor += 1;
+ if let Some(meta) = action.get("delete") {
+ let index = meta["_index"].as_str().unwrap().to_owned();
+ let id = meta["_id"].as_str().unwrap().to_owned();
+ locked.docs.remove(&(index, id));
+ continue;
+ }
+ let payload: Value = serde_json::from_slice(lines[cursor]).unwrap();
+ cursor += 1;
+ if let Some(meta) = action.get("index") {
+ let index = meta["_index"].as_str().unwrap().to_owned();
+ let id = meta["_id"].as_str().unwrap().to_owned();
+ locked.docs.insert((index, id), payload);
+ } else if let Some(meta) = action.get("update") {
+ let index = meta["_index"].as_str().unwrap().to_owned();
+ let id = meta["_id"].as_str().unwrap().to_owned();
+ let patch = payload["doc"].as_object().unwrap();
+ let target = locked.docs.get_mut(&(index, id)).unwrap();
+ let target = target.as_object_mut().unwrap();
+ target.extend(patch.clone());
+ }
+ }
+ if applied_limit.is_some() {
+ return (
+ 200,
+ json!({
+ "errors": true,
+ "items": [{"index": {
+ "status": 500,
+ "error": {
+ "type": "injected_failure",
+ "reason": "partial bulk applied before failing"
+ }
+ }}]
+ }),
+ );
+ }
+ (200, json!({"errors": false, "items": []}))
+ }
+
+ fn delete_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> usize {
+ let index = path
+ .trim_start_matches('/')
+ .split("/_delete_by_query")
+ .next()
+ .unwrap();
+ let query: Value = serde_json::from_slice(body).unwrap();
+ let term = query
+ .pointer("/query/term")
+ .and_then(Value::as_object)
+ .and_then(|term| term.iter().next())
+ .map(|(field, value)| (field.clone(), value.clone()));
+ let mut locked = state.lock().unwrap();
+ let before = locked.docs.len();
+ locked.docs.retain(|(candidate, _), doc| {
+ if candidate != index {
+ return true;
+ }
+ let Some((field, expected)) = &term else {
+ return false;
+ };
+ doc.get(field) != Some(expected)
+ });
+ before - locked.docs.len()
+ }
+
+ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value {
+ let indices = path
+ .trim_start_matches('/')
+ .trim_end_matches("/_search")
+ .split(',')
+ .collect::<Vec<_>>();
+ let query: Value = serde_json::from_slice(body).unwrap();
+ let term = query
+ .pointer("/query/term")
+ .or_else(|| query.pointer("/query/bool/must/0/term"))
+ .and_then(Value::as_object)
+ .and_then(|term| term.iter().next());
+ let terms = query
+ .pointer("/query/terms")
+ .and_then(Value::as_object)
+ .and_then(|terms| terms.iter().next());
+ let filters = query
+ .pointer("/query/bool/filter")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let exists = query
+ .pointer("/query/bool/must/1/exists/field")
+ .and_then(Value::as_str);
+ let locked = state.lock().unwrap();
+ let matching: Vec<_> = locked
+ .docs
+ .iter()
+ .filter(|((index, _), doc)| {
+ indices.contains(&index.as_str())
+ && term
+ .map(|(field, value)| doc.get(field) == Some(value))
+ .unwrap_or(true)
+ && terms
+ .map(|(field, values)| {
+ values.as_array().is_some_and(|values| {
+ values.iter().any(|value| doc.get(field) == Some(value))
+ })
+ })
+ .unwrap_or(true)
+ && filters.iter().all(|filter| {
+ filter
+ .get("term")
+ .and_then(Value::as_object)
+ .and_then(|term| term.iter().next())
+ .is_none_or(|(field, value)| doc.get(field) == Some(value))
+ })
+ && exists.map(|field| doc.get(field).is_some()).unwrap_or(true)
+ })
+ .collect();
+ let hits = matching
+ .iter()
+ .map(|((_, id), source)| json!({"_id": id, "_source": source}))
+ .collect::<Vec<_>>();
+ json!({
+ "hits": {
+ "total": {"value": matching.len(), "relation": "eq"},
+ "hits": hits
+ },
+ "aggregations": {}
+ })
+ }
+
+ fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {
+ IndexCfg {
+ root: root.to_owned(),
++ stub_globs: Vec::new(),
+ url: url.to_owned(),
+ api_key: None,
+ workers: 1,
+ scan_workers: 1,
+ pdf_workers: 1,
+ resource_notes: Vec::new(),
+ pdf_timeout_secs: 30,
+ bulk_mb: 1,
+ bulk_timeout_secs: 30,
+ snapshot_max_bytes: 64 << 30,
+ prefix: "incremental-http".into(),
+ state_dir: Some(state_dir.to_owned()),
+ fresh: false,
+ follow_symlinks: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
+ max_file_gb: 1,
+ sample: 50,
+ no_semantic: !semantic,
+ brain: None,
+ no_graph: true,
+ dry_run: false,
+ json: false,
+ quiet: true,
+ progress: crate::progress::ProgressMode::None,
+ progress_interval: None,
+ }
+ }
+
+ fn journal_events(state_dir: &Path, kind: &str) -> usize {
+ fs::read_to_string(state_dir.join("journal.ndjson"))
+ .unwrap()
+ .lines()
+ .filter_map(|line| serde_json::from_str::<Value>(line).ok())
+ .filter(|event| event.get("kind").and_then(Value::as_str) == Some(kind))
+ .count()
+ }
+
+ fn paths(docs: &[Value]) -> Vec<String> {
+ let mut paths: Vec<_> = docs
+ .iter()
+ .filter_map(|doc| doc.get("ax_path").and_then(Value::as_str))
+ .map(str::to_owned)
+ .collect();
+ paths.sort();
+ paths.dedup();
+ paths
+ }
+
+ fn snapshot_names(state_dir: &Path) -> Vec<String> {
+ let mut names: Vec<String> = fs::read_dir(state_dir.join("sync-snapshots"))
+ .map(|entries| {
+ entries
+ .filter_map(Result::ok)
+ .filter(|entry| {
+ entry.file_type().is_ok_and(|kind| kind.is_dir())
+ && !entry.file_name().to_string_lossy().starts_with('.')
+ })
+ .map(|entry| entry.file_name().to_string_lossy().into_owned())
+ .collect()
+ })
+ .unwrap_or_default();
+ names.sort();
+ names
+ }
+
+ fn final_snapshot_count(state_dir: &Path) -> usize {
+ fs::read_dir(state_dir.join("sync-snapshots"))
+ .map(|entries| {
+ entries
+ .filter_map(Result::ok)
+ .filter(|entry| {
+ entry.file_type().is_ok_and(|kind| kind.is_dir())
+ && !entry.file_name().to_string_lossy().starts_with('.')
+ })
+ .count()
+ })
+ .unwrap_or(0)
+ }
+
+ #[test]
+ fn genesis_recovers_after_snapshot_budget_failure_before_sync_begin() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ config.snapshot_max_bytes = 1;
+
+ let error = run_index(config.clone()).unwrap_err();
+ assert!(format!("{error:#}").contains("snapshot source footprint"));
+ assert_eq!(journal_events(state_dir.path(), "sync_bootstrap"), 1);
+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 0);
+ assert_eq!(endpoint.data_bulk_requests(), 0);
+
+ config.snapshot_max_bytes = 64 << 30;
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_bootstrap"), 1);
+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 1);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+ assert_eq!(endpoint.data_docs().len(), 1);
+ }
+
+ #[test]
+ fn no_semantic_generation_does_not_require_embedding_identity_endpoint() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ endpoint.state.lock().unwrap().fail_embedding_identity = true;
+
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_bootstrap"), 1);
+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 1);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+ assert_eq!(journal_events(state_dir.path(), "finish"), 1);
+ assert_eq!(final_snapshot_count(state_dir.path()), 1);
+ assert_eq!(endpoint.data_docs().len(), 1);
+ }
+
+ #[test]
+ fn initial_generation_and_noop_are_end_to_end_idempotent() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n2,beta\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+
+ let (code, summary) = run_index_report(config.clone()).unwrap();
+ assert_eq!(code, 0);
+ let summary = summary.expect("generated run must return its committed run projection");
+ assert_eq!(summary["generation"], 1);
+ assert_eq!(summary["records_total"], 2);
+ let first_docs = endpoint.data_docs();
+ assert_eq!(first_docs.len(), 2);
+ assert_eq!(paths(&first_docs), ["a.csv"]);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+ let root = corpus.path().canonicalize().unwrap();
+ let preflight = state::Journal::preflight(
+ state_dir.path(),
+ &root.to_string_lossy(),
+ &endpoint.url,
+ "incremental-http",
+ false,
+ )
+ .unwrap();
+ assert!(
+ preflight.committed_manifest.is_some(),
+ "sync_commit must be visible to the next preflight"
+ );
+ drop(preflight);
+
+ let bulks = endpoint.data_bulk_requests();
+ let (_, noop_summary) = run_index_report(config).unwrap();
+ assert_eq!(noop_summary.unwrap()["generation"], 1);
+ assert_eq!(endpoint.data_docs(), first_docs);
+ assert_eq!(endpoint.data_bulk_requests(), bulks);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+ assert_eq!(journal_events(state_dir.path(), "finish"), 2);
+ }
+
+ #[test]
+ fn add_change_delete_and_rename_converge_over_real_http() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ fs::write(corpus.path().join("b.csv"), "id,value\n2,bravo\n").unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv", "b.csv"]);
+
+ fs::write(
+ corpus.path().join("a.csv"),
+ "id,value\n1,alpha-new\n3,charlie\n",
+ )
+ .unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ assert_eq!(endpoint.data_docs().len(), 3);
+
+ fs::remove_file(corpus.path().join("b.csv")).unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);
+
+ fs::rename(
+ corpus.path().join("a.csv"),
+ corpus.path().join("renamed.csv"),
+ )
+ .unwrap();
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(paths(&endpoint.data_docs()), ["renamed.csv"]);
+ assert_eq!(endpoint.data_docs().len(), 2);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 5);
+ }
+
+ #[test]
+ fn pending_generation_replays_sealed_source_after_live_source_mutates() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let source = corpus.path().join("a.csv");
+ fs::write(&source, "id,value\n1,committed\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ fs::write(&source, "id,value\n1,sealed-pending\n2,sealed-second\n").unwrap();
+ endpoint.state.lock().unwrap().fail_next_data_bulk = true;
+ assert!(run_index(config.clone()).is_err());
+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 2);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ fs::write(&source, "id,value\n9,mutated-after-sync-begin\n").unwrap();
+ assert_eq!(run_index(config).unwrap(), 0);
+ let docs = endpoint.data_docs();
+ assert_eq!(docs.len(), 2);
+ let rendered = serde_json::to_string(&docs).unwrap();
+ assert!(rendered.contains("sealed-pending"), "{rendered}");
+ assert!(rendered.contains("sealed-second"), "{rendered}");
+ assert!(!rendered.contains("mutated-after-sync-begin"), "{rendered}");
+ }
+
+ /// Crash/resume with a *partially applied* data bulk, on `--no-graph`.
+ ///
+ /// `failure_resume_http_tests` used to run its whole injected-failure matrix
+ /// with `no_graph: true`; since `--no-graph` now takes the generated executor,
+ /// that module keeps the legacy graph-enabled transaction and this test carries
+ /// the no-graph half of the coverage. The endpoint applies half of a bulk and
+ /// *then* reports failure — visibility has changed, so the generation must stay
+ /// uncommitted, and the retry must converge on exactly the new content with no
+ /// duplicate or stale document surviving the half-applied attempt.
+ #[test]
+ fn partially_applied_data_bulk_leaves_the_generation_pending_and_the_retry_converges() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let source = corpus.path().join("rows.csv");
+ fs::write(&source, "id,value\n1,first\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ fs::write(
+ &source,
+ "id,value\n1,replaced\n2,second\n3,third\n4,fourth\n",
+ )
+ .unwrap();
+ endpoint
+ .state
+ .lock()
+ .unwrap()
+ .partially_apply_next_data_bulk = true;
+ assert!(
+ run_index(config.clone()).is_err(),
+ "a partially applied bulk must fail the run, not be absorbed"
+ );
+ assert_eq!(
+ journal_events(state_dir.path(), "sync_begin"),
+ 2,
+ "the generation was begun"
+ );
+ assert_eq!(
+ journal_events(state_dir.path(), "sync_commit"),
+ 1,
+ "a partially applied bulk must never commit the generation"
+ );
+
+ // Retry: the sealed source replays, and the destination converges exactly.
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+ let docs = endpoint.data_docs();
+ assert_eq!(paths(&docs), ["rows.csv"]);
+ assert_eq!(
+ docs.len(),
+ 4,
+ "exactly the four new records survive — no half-applied duplicate, no stale first record"
+ );
+ let rendered = serde_json::to_string(&docs).unwrap();
+ assert!(rendered.contains("replaced"), "{rendered}");
+ assert!(rendered.contains("fourth"), "{rendered}");
+ assert!(
+ !rendered.contains("\"value\":\"first\""),
+ "the replaced record must not survive: {rendered}"
+ );
+ }
+
+ #[test]
+ fn pending_semantic_identity_drift_rejects_before_any_further_bulk() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let source = corpus.path().join("report.txt");
+ fs::write(
+ &source,
+ "Initial quarterly report discusses durable subscription revenue and operating income.",
+ )
+ .unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, true);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ fs::write(
+ &source,
+ "Updated quarterly report discusses materially stronger subscription revenue and margin.",
+ )
+ .unwrap();
+ endpoint.state.lock().unwrap().fail_next_data_bulk = true;
+ assert!(run_index(config.clone()).is_err());
+ let bulks_before = endpoint.data_bulk_requests();
+ endpoint.state.lock().unwrap().embedding_identity_sha256 = "b".repeat(64);
+
+ let error = run_index(config).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("different embedding execution identity"),
+ "{message}"
+ );
+ assert!(
+ message.contains("no remote mutation was attempted"),
+ "{message}"
+ );
+ assert_eq!(
+ endpoint.data_bulk_requests(),
+ bulks_before,
+ "identity drift must be rejected before another data bulk"
+ );
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+ }
+
+ /// `--fresh` must be refused before it can destroy generation authority.
+ ///
+ /// `Journal::open_after_preflight` deletes `journal.ndjson` whenever `fresh` is
+ /// set, and `gc_snapshots` then sees an empty protected set and removes every
+ /// sealed snapshot. Both run inside the generated branches, ahead of the legacy
+ /// plan gate, so without the preflight refusal a single `--fresh` erases the
+ /// committed manifest and every sealed source the next run needs.
+ #[test]
+ fn fresh_cannot_destroy_a_committed_generation() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ fs::write(corpus.path().join("b.csv"), "id,value\n2,bravo\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let committed_docs = endpoint.data_docs();
+ assert_eq!(paths(&committed_docs), ["a.csv", "b.csv"]);
+
+ let journal_path = state_dir.path().join("journal.ndjson");
+ let journal_before = fs::read(&journal_path).unwrap();
+ let snapshots_before = snapshot_names(state_dir.path());
+ assert_eq!(snapshots_before.len(), 1, "{snapshots_before:?}");
+ let requests_before = endpoint.state.lock().unwrap().requests.len();
+
+ // The folder also changed, which is exactly when a user reaches for
+ // --fresh.
+ fs::remove_file(corpus.path().join("b.csv")).unwrap();
+ config.fresh = true;
+ let error = run_index(config.clone()).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("`--fresh` cannot discard committed corpus generation 1"),
+ "the refusal must name the durable state that blocks it: {message}"
+ );
+ assert!(
+ message.contains("without `--fresh`"),
+ "the refusal must point at the incremental path that does work: {message}"
+ );
+
+ assert_eq!(
+ fs::read(&journal_path).unwrap(),
+ journal_before,
+ "a refused --fresh must not rewrite the journal"
+ );
+ assert_eq!(
+ snapshot_names(state_dir.path()),
+ snapshots_before,
+ "a refused --fresh must not garbage-collect sealed snapshots"
+ );
+ let attempted = endpoint.state.lock().unwrap().requests[requests_before..].to_vec();
+ assert_eq!(
+ attempted,
+ [("GET".to_owned(), "/".to_owned())],
+ "a refused --fresh may perform only the endpoint-readiness GET"
+ );
+
+ // The surviving authority still reconciles the real folder change.
+ config.fresh = false;
+ let (code, summary) = run_index_report(config).unwrap();
+ assert_eq!(code, 0);
+ assert_eq!(summary.unwrap()["generation"], 2);
+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+ }
+
+ /// Every refusal on the generated path now sends the operator to an isolated
+ /// rebuild, so that recipe has to work. `--fresh` is no longer a rebuild-in-
+ /// place escape hatch, and a new `--prefix` alone is not enough when the state
+ /// directory was named explicitly: `preflight` refuses a journal recorded for a
+ /// different prefix before the run reaches anything else.
+ #[test]
+ fn the_isolated_rebuild_the_refusals_recommend_is_followable() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let committed_state = tempfile::tempdir().unwrap();
+ let rebuild_state = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), committed_state.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ // A new --prefix on its own cannot rebuild: the recorded journal owns it.
+ let mut prefix_only = config.clone();
+ prefix_only.prefix = "incremental-http-rebuild".into();
+ let message = format!("{:#}", run_index(prefix_only).unwrap_err());
+ assert!(message.contains("was created for root="), "{message}");
+
+ // A new --state-dir and a new --prefix together do rebuild, and leave the
+ // original generation and its sealed snapshot untouched.
+ let journal_before = fs::read(committed_state.path().join("journal.ndjson")).unwrap();
+ let snapshots_before = snapshot_names(committed_state.path());
+ let mut rebuild = cfg(corpus.path(), rebuild_state.path(), &endpoint.url, false);
+ rebuild.prefix = "incremental-http-rebuild".into();
+ let (code, summary) = run_index_report(rebuild).unwrap();
+ assert_eq!(code, 0);
+ assert_eq!(summary.unwrap()["generation"], 1);
+ assert_eq!(journal_events(rebuild_state.path(), "sync_commit"), 1);
+ assert_eq!(
+ fs::read(committed_state.path().join("journal.ndjson")).unwrap(),
+ journal_before,
+ "an isolated rebuild must not touch the original journal"
+ );
+ assert_eq!(
+ snapshot_names(committed_state.path()),
+ snapshots_before,
+ "an isolated rebuild must not touch the original sealed snapshots"
+ );
+ }
+
+ /// The same refusal on a crashed generation, where the journal additionally
+ /// holds the only record of the pending transaction and its sealed source.
+ #[test]
+ fn fresh_cannot_destroy_a_pending_generation() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let source = corpus.path().join("a.csv");
+ fs::write(&source, "id,value\n1,committed\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ fs::write(&source, "id,value\n1,sealed-pending\n2,sealed-second\n").unwrap();
+ endpoint.state.lock().unwrap().fail_next_data_bulk = true;
+ assert!(run_index(config.clone()).is_err());
+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 2);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ let journal_path = state_dir.path().join("journal.ndjson");
+ let journal_before = fs::read(&journal_path).unwrap();
+ let snapshots_before = snapshot_names(state_dir.path());
+ assert_eq!(snapshots_before.len(), 2, "{snapshots_before:?}");
+ let requests_before = endpoint.state.lock().unwrap().requests.len();
+
+ config.fresh = true;
+ let error = run_index(config.clone()).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("`--fresh` cannot discard an uncommitted pending corpus generation"),
+ "the refusal must name the durable state that blocks it: {message}"
+ );
+ assert_eq!(
+ fs::read(&journal_path).unwrap(),
+ journal_before,
+ "a refused --fresh must not rewrite a pending journal"
+ );
+ assert_eq!(
+ snapshot_names(state_dir.path()),
+ snapshots_before,
+ "a refused --fresh must not delete the pending sealed source"
+ );
+ let attempted = endpoint.state.lock().unwrap().requests[requests_before..].to_vec();
+ assert_eq!(
+ attempted,
+ [("GET".to_owned(), "/".to_owned())],
+ "a refused --fresh may perform only the endpoint-readiness GET"
+ );
+
+ // The pending transaction is still resumable from its sealed source.
+ config.fresh = false;
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+ let rendered = serde_json::to_string(&endpoint.data_docs()).unwrap();
+ assert!(rendered.contains("sealed-pending"), "{rendered}");
+ assert!(rendered.contains("sealed-second"), "{rendered}");
+ }
+
+ /// Two runs started in the same process and the same UTC second must not share
+ /// a `run_id`.
+ ///
+ /// The generation catalog keys its managed documents on `run_id`, so a
+ /// collision makes the second run read the first run's documents back as its
+ /// own generation. `xerj brain` calls `run_index_report` more than once per
+ /// process, so this is reachable without any concurrency.
+ #[test]
+ fn same_second_runs_in_one_process_do_not_share_a_generation_identity() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let first_state = tempfile::tempdir().unwrap();
+ let second_state = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+
+ assert_eq!(
+ run_index(cfg(corpus.path(), first_state.path(), &endpoint.url, false)).unwrap(),
+ 0
+ );
+ let second = run_index_report(cfg(
+ corpus.path(),
+ second_state.path(),
+ &endpoint.url,
+ false,
+ ));
+ let (code, summary) = second.expect("a second run in the same second must commit");
+ assert_eq!(code, 0);
+ assert_eq!(summary.unwrap()["generation"], 1);
+
+ let run_id = |state_dir: &Path| -> String {
+ fs::read_to_string(state_dir.join("journal.ndjson"))
+ .unwrap()
+ .lines()
+ .filter_map(|line| serde_json::from_str::<Value>(line).ok())
+ .find_map(|event| {
+ event
+ .get("run_id")
+ .and_then(Value::as_str)
+ .map(str::to_owned)
+ })
+ .expect("journal records a run_id")
+ };
+ assert_ne!(run_id(first_state.path()), run_id(second_state.path()));
+ }
+
+ /// An upsert must clear the desired content identity as well as the committed
+ /// one before it replays.
+ ///
+ /// A byte-identical retry of the same prepared stream is idempotent on its own,
+ /// because `ids::doc_id` is derived from dataset slug, content id and locator.
+ /// The guard earns its keep when a prior aborted attempt left documents under
+ /// the desired `ax_file` whose ids the current stream does not overwrite — an
+ /// earlier generation of the same content under a different dataset
+ /// assignment, or a partially rolled-back attempt. This test models that
+ /// survivor directly, because the exact record validation at the commit
+ /// barrier is what turns such a leftover into a hard, unrecoverable failure.
+ #[test]
+ fn upsert_clears_a_survivor_under_the_desired_content_identity() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let source = corpus.path().join("a.csv");
+ let original = "id,value\n1,alpha\n";
+ fs::write(&source, original).unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ let (index, content_id) = {
+ let locked = endpoint.state.lock().unwrap();
+ let ((index, _), doc) = locked
+ .docs
+ .iter()
+ .find(|((index, _), _)| index != catalog::CATALOG_INDEX)
+ .expect("the first generation published a data document");
+ (
+ index.clone(),
+ doc.get("ax_file")
+ .and_then(Value::as_str)
+ .expect("data documents carry ax_file")
+ .to_owned(),
+ )
+ };
+
+ // Replace the content, then restore it, so the third generation upserts the
+ // original content identity again.
+ fs::write(&source, "id,value\n2,bravo\n").unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ fs::write(&source, original).unwrap();
+
+ // A document from an aborted earlier attempt at exactly this content, under
+ // an id the prepared stream will not overwrite.
+ endpoint.state.lock().unwrap().docs.insert(
+ (index.clone(), "aborted-attempt-survivor".to_owned()),
+ json!({
+ "id": 1,
+ "value": "alpha",
+ "ax_file": content_id,
+ "ax_path": "a.csv",
+ "ax_locator": "aborted-attempt-survivor"
+ }),
+ );
+
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert!(
+ !endpoint
+ .state
+ .lock()
+ .unwrap()
+ .docs
+ .contains_key(&(index, "aborted-attempt-survivor".to_owned())),
+ "the upsert must delete the desired content identity before replaying it"
+ );
+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 3);
+ }
+
+ /// `--dry-run` used to be consulted only *after* the generated branches had
+ /// already published and committed, so on any already-generated state
+ /// directory the flag was accepted, ignored, and the destination mutated.
+ #[test]
+ fn dry_run_on_a_generated_state_directory_publishes_nothing() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let committed_docs = endpoint.data_docs();
+ let bulks = endpoint.data_bulk_requests();
+ let journal_before = fs::read(state_dir.path().join("journal.ndjson")).unwrap();
+
+ // A real change, so the dry run has something it would publish.
+ fs::write(
+ corpus.path().join("a.csv"),
+ "id,value\n1,CHANGED\n2,added\n",
+ )
+ .unwrap();
+ let mut dry = config.clone();
+ dry.dry_run = true;
+ let (code, summary) = run_index_report(dry.clone()).unwrap();
+ assert_eq!(code, 0);
+ assert!(
+ summary.is_none(),
+ "--dry-run must not report a committed run summary"
+ );
+ assert_eq!(
+ endpoint.data_bulk_requests(),
+ bulks,
+ "--dry-run issued a data bulk"
+ );
+ assert_eq!(
+ endpoint.data_docs(),
+ committed_docs,
+ "--dry-run mutated the destination"
+ );
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+ assert_eq!(
+ fs::read(state_dir.path().join("journal.ndjson")).unwrap(),
+ journal_before,
+ "--dry-run appended to the durable journal"
+ );
+
+ // An unchanged folder takes the no-op arm of the same branch: still inert.
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ assert_eq!(run_index(dry).unwrap(), 0);
+ assert_eq!(endpoint.data_bulk_requests(), bulks);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ // And the real run that follows still converges on the change.
+ fs::write(
+ corpus.path().join("a.csv"),
+ "id,value\n1,CHANGED\n2,added\n",
+ )
+ .unwrap();
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(endpoint.data_docs().len(), 2);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+ }
+
+ /// A junk/skipped file has no `plan.files` entry by construction, so demanding
+ /// a prepared artifact for every inventory entry failed the whole run on any
+ /// folder containing one unreadable, empty or unrecognised file.
+ #[test]
+ fn a_junk_file_is_recorded_and_never_fatal() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n2,beta\n").unwrap();
+ fs::write(corpus.path().join("empty.csv"), "").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+
+ let (code, summary) = run_index_report(config.clone()).unwrap();
+ let summary = summary.expect("a corpus with one junk file still commits a generation");
+ assert_eq!(
+ code, 3,
+ "a completed run that skipped a file exits 3, not 0 (cli.rs EXIT CODES)"
+ );
+ assert_eq!(summary["generation"], 1);
+ assert_eq!(summary["files_junk"], 1);
+ assert_eq!(summary["records_total"], 2);
+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+ let junk_docs: Vec<Value> = endpoint
+ .state
+ .lock()
+ .unwrap()
+ .docs
+ .iter()
+ .filter(|((index, _), doc)| {
+ index == catalog::CATALOG_INDEX
+ && doc.get("doc_kind").and_then(Value::as_str) == Some("file")
+ && doc.get("path").and_then(Value::as_str) == Some("empty.csv")
+ })
+ .map(|(_, doc)| doc.clone())
+ .collect();
+ assert_eq!(
+ junk_docs.len(),
+ 1,
+ "the skipped file must still get exactly one catalog entry"
+ );
+
+ // The generation reconciles normally afterwards, junk file and all.
+ fs::write(corpus.path().join("b.csv"), "id,value\n3,gamma\n").unwrap();
+ assert_eq!(run_index(config).unwrap(), 3);
+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv", "b.csv"]);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+ }
+
+ /// Junk *records* were fatal on the generated path
+ /// (`durable preparation of X produced N junk records`) while the legacy path
+ /// merely accumulated them — directly contradicting the documented contract
+ /// "3 completed-with-junk (junk recorded, never fatal)".
+ #[test]
+ fn a_junk_record_is_counted_into_the_generation_and_never_fatal() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let mut log = String::from("!!! rotated by logrotate, not a log line\n");
+ for n in 1..=40 {
+ log.push_str(&format!(
+ "10.0.0.{n} - - [09/Aug/2026:10:00:{n:02} +0000] \"GET /p/{n} HTTP/1.1\" 200 {n}\n"
+ ));
+ }
+ fs::write(corpus.path().join("access.log"), log).unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+
+ let (code, summary) = run_index_report(config).unwrap();
+ let summary = summary.expect("a corpus with one junk record still commits a generation");
+ assert_eq!(code, 3, "completed-with-junk exits 3");
+ assert_eq!(summary["generation"], 1);
+ assert_eq!(summary["files_junk"], 0);
+ assert_eq!(
+ summary["junk_records_total"], 1,
+ "the unparseable line must be counted, not hardcoded to zero"
+ );
+ assert_eq!(summary["records_total"], 40);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ // The same count reaches the per-file catalog document.
+ let file_doc = endpoint
+ .state
+ .lock()
+ .unwrap()
+ .docs
+ .iter()
+ .find(|((index, _), doc)| {
+ index == catalog::CATALOG_INDEX
+ && doc.get("doc_kind").and_then(Value::as_str) == Some("file")
+ && doc.get("path").and_then(Value::as_str) == Some("access.log")
+ })
+ .map(|(_, doc)| doc.clone())
+ .expect("the indexed file has a catalog document");
+ assert_eq!(file_doc["junk"], 1);
+ assert_eq!(file_doc["records"], 40);
+ }
+
+ /// Where this branch and #241 meet.
+ ///
+ /// The generated `--no-graph` route returns from inside `run_index_report`,
+ /// several hundred lines before the legacy `pr.finish` at the bottom of the
+ /// function. `Ticker::drop` closes an unfinished stream with `ok=false`, so a
+ /// generated run that merely returned would print an *aborted* terminal line
+ /// immediately after committing a generation successfully — a silent
+ /// contradiction between two changes that each looked right on its own.
+ ///
+ /// This pins all three properties on the generated path: the phase-A scan is
+ /// narrated (it is a per-file, minutes-long phase here exactly as on the legacy
+ /// path), the stream closes with a truthful terminal line, and `--progress
+ /// none` on the same path stays completely silent.
+ #[test]
+ fn a_generated_run_narrates_its_scan_and_closes_its_own_stream() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ for n in 0..4 {
+ fs::write(
+ corpus.path().join(format!("rows-{n}.csv")),
+ format!("id,value\n{n}0,alpha\n{n}1,beta\n"),
+ )
+ .unwrap();
+ }
+ let endpoint = HttpEndpoint::start();
+
+ // Generation 1: the initial commit, through the real progress surface.
+ let mut loud = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ loud.quiet = false;
+ loud.progress = crate::progress::ProgressMode::Plain;
+ loud.progress_interval = Some(std::time::Duration::from_secs(1));
+ let buffer = Arc::new(Mutex::new(Vec::new()));
+ let code = {
+ let _sink = crate::progress::install_test_sink(&buffer);
+ run_index(loud.clone()).unwrap()
+ };
+ assert_eq!(code, 0);
+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
+ let done = stream
+ .lines()
+ .find(|line| line.starts_with("xerj-done "))
+ .unwrap_or_else(|| panic!("a generated run must close its own stream:\n{stream}"));
+ assert!(
+ done.contains("ok=true") && done.contains("exit=0"),
+ "a committed generation is not an aborted run: {done}"
+ );
+ assert!(done.contains("reason=completed"), "{done}");
+
+ // Generation 2: the reconcile branch, which runs its own phase-A scan.
+ fs::write(corpus.path().join("rows-0.csv"), "id,value\n00,CHANGED\n").unwrap();
+ let buffer = Arc::new(Mutex::new(Vec::new()));
+ let code = {
+ let _sink = crate::progress::install_test_sink(&buffer);
+ run_index(loud).unwrap()
+ };
+ assert_eq!(code, 0);
+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
+ assert!(
+ stream
+ .lines()
+ .any(|line| line.starts_with("xerj-progress ") && line.contains("phase=scan")),
+ "the reconcile projection re-scans every file and must say so:\n{stream}"
+ );
+ let done = stream
+ .lines()
+ .find(|line| line.starts_with("xerj-done "))
+ .unwrap_or_else(|| panic!("a reconciled generation must close its own stream:\n{stream}"));
+ assert!(done.contains("ok=true"), "{done}");
+
+ // The other half of the contract on the same path: silence means silence.
+ fs::write(corpus.path().join("rows-1.csv"), "id,value\n10,CHANGED\n").unwrap();
+ let quiet = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(quiet.progress, crate::progress::ProgressMode::None);
+ let buffer = Arc::new(Mutex::new(Vec::new()));
+ {
+ let _sink = crate::progress::install_test_sink(&buffer);
+ assert_eq!(run_index(quiet).unwrap(), 0);
+ }
+ assert!(
+ buffer.lock().unwrap().is_empty(),
+ "--progress none asked for nothing: {}",
+ String::from_utf8_lossy(&buffer.lock().unwrap())
+ );
+ }
diff --cc engine/crates/xerj-autoindex/src/lib.rs
index 1a3c29e54,5fe5ce843..0819b2172
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@@ -32,8 -43,411 +43,413 @@@ use cli::{Cmd, IndexCfg, MapCfg, Status
// `Box<dyn EdgeDetector>` like the registry entries).
use detect::EdgeDetector as _;
use esclient::Es;
+ use progress::Progress;
use sniff::{Family, Sniffed};
- use state::{FileAssignment, FileDone, JunkFile, Plan, PlanDataset};
+ use state::{DuplicateFile, FileAssignment, FileDone, JunkFile, Plan, PlanDataset};
+
+ const PREPARED_RECORDS_IDENTITY: &str = "prepared-records-v1";
+ const DOCUMENT_IDS_IDENTITY: &str = "document-ids-v1";
+ const DETECTOR_DISABLED_IDENTITY: &str = "disabled";
+
+ fn prepared_records_identity(cfg: &IndexCfg) -> Result<String> {
+ let value = json!({
+ "contract": PREPARED_RECORDS_IDENTITY,
+ "sample": cfg.sample,
+ "max_file_gb": cfg.max_file_gb,
+ "no_semantic": cfg.no_semantic,
+ // Worker counts are operational only. The timeout can change whether
+ // a PDF yields records, so it is part of the semantic contract.
+ "pdf_timeout_secs": cfg.pdf_timeout_secs,
+ });
+ Ok(format!(
+ "{}-{:032x}",
+ PREPARED_RECORDS_IDENTITY,
+ xxhash_rust::xxh3::xxh3_128(&serde_json::to_vec(&value)?)
+ ))
+ }
+
+ fn preparation_contract_digest(cfg: &IndexCfg, plan: &Plan) -> Result<String> {
+ let (schema_identity, index_identity) = generation_contract_identities(plan)?;
+ let encoded = serde_json::to_vec(&json!({
+ "prepared_records": prepared_records_identity(cfg)?,
+ "document_ids": DOCUMENT_IDS_IDENTITY,
+ "schema_identity": schema_identity,
+ "index_identity": index_identity,
+ "plan": plan,
+ }))?;
+ Ok(format!(
+ "axpc1-{:032x}",
+ xxhash_rust::xxh3::xxh3_128(&encoded)
+ ))
+ }
+
+ pub(crate) fn generation_contract_identities(plan: &Plan) -> Result<(String, String)> {
+ let mut datasets = plan.datasets.iter().collect::<Vec<_>>();
+ datasets.sort_by(|left, right| {
+ left.slug
+ .cmp(&right.slug)
+ .then_with(|| left.index.cmp(&right.index))
+ });
+ let schema = datasets
+ .iter()
+ .map(|dataset| {
+ json!({
+ "slug": dataset.slug,
+ "family": dataset.family,
+ "group": dataset.group,
+ "specs": dataset.specs,
+ "time_field": dataset.time_field,
+ "semantic_field": dataset.semantic_field,
+ })
+ })
+ .collect::<Vec<_>>();
+ let indices = datasets
+ .iter()
+ .map(|dataset| {
+ json!({
+ "index": dataset.index,
+ "mapping": build_mapping(&dataset.specs),
+ })
+ })
+ .collect::<Vec<_>>();
+ let digest = |label: &str, value: Value| -> Result<String> {
+ let bytes = serde_json::to_vec(&json!({"contract": label, "value": value}))?;
+ Ok(format!("{:032x}", xxhash_rust::xxh3::xxh3_128(&bytes)))
+ };
+ Ok((
+ digest(PREPARED_RECORDS_IDENTITY, Value::Array(schema))?,
+ digest(
+ DOCUMENT_IDS_IDENTITY,
+ json!({
+ "datasets": indices,
+ "catalog_index": catalog::CATALOG_INDEX,
+ "catalog_mapping": catalog::catalog_mapping(),
+ "document_ids": DOCUMENT_IDS_IDENTITY,
+ }),
+ )?,
+ ))
+ }
+
+ pub(crate) fn ensure_generation_mappings(es: &Es, plan: &Plan) -> Result<()> {
+ for dataset in &plan.datasets {
+ let mut create_body = build_mapping(&dataset.specs);
+ create_body["mappings"]["properties"]["ax_paths"] = json!({"type": "keyword"});
+ let update_body = json!({
+ "properties": create_body["mappings"]["properties"].clone()
+ });
+ es.ensure_index(&dataset.index, &create_body)
+ .with_context(|| format!("create generation index {}", dataset.index))?;
+ es.update_mapping(&dataset.index, &update_body)
+ .with_context(|| format!("install generation mapping for {}", dataset.index))?;
+ }
+ let mut catalog_create_body = catalog::catalog_mapping();
+ catalog_create_body["mappings"]["properties"]["duplicate_of"] = json!({"type": "keyword"});
+ let catalog_update_body = json!({
+ "properties": catalog_create_body["mappings"]["properties"].clone()
+ });
+ es.ensure_index(catalog::CATALOG_INDEX, &catalog_create_body)?;
+ es.update_mapping(catalog::CATALOG_INDEX, &catalog_update_body)
+ .context("install generation catalog mapping")
+ }
+
+ /// Project the current inventory onto a committed plan.
+ ///
+ /// Pure by design and deliberately shared: `--dry-run` prints exactly the plan
+ /// the real reconcile would act on, because it is produced by this same call
+ /// rather than by a parallel implementation that could drift away from it.
+ ///
+ /// The scan itself obeys the same two policies the legacy phase A does. It runs
+ /// inside the run's own pool, because `--workers` has to bound the CPU-bound
+ /// phase for the knob to mean anything (#240 §2), and it opens a per-file
+ /// progress guard, because a file this route touches must not drop out of the
+ /// progress denominator (#241). Run-local PDF artifact reuse is deliberately
+ /// disabled here: the generated route publishes from its own sealed snapshot
+ /// and never reaches the legacy phase B, so a retained artifact could never be
+ /// replayed (#248).
+ fn project_reconcile_plan(
+ inventory: &content::Inventory,
+ base_plan: &Plan,
+ cfg: &IndexCfg,
+ state_dir: &Path,
+ pr: &Progress,
+ ) -> Result<Plan> {
+ let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);
+ let ctx = PhaseAContext {
+ state_dir,
+ budget: &budget,
+ capacity_warning: None,
+ progress: pr,
+ };
+ pr.phase(
+ "scan",
+ inventory.files.len() as u64,
+ inventory.files.iter().map(|file| file.size).sum(),
+ );
++ let stub_matcher = StubMatcher::compile(&cfg.stub_globs).expect("validated at startup");
+ let scans: Vec<FileScan> = crate::pool::install(|| {
+ use rayon::prelude::*;
+ inventory
+ .files
+ .par_iter()
+ .zip(inventory.digests.par_iter())
+ .map(|(file, digest)| {
+ let _in_flight = pr.file(&file.rel, file.size);
+ scan_file(
+ &file.path,
+ file.size,
+ digest,
+ &ctx,
+ cfg.sample,
+ cfg.max_file_gb,
++ stub_matcher.matches(&file.rel),
+ )
+ })
+ .collect()
+ });
+ reconcile_plan::reconcile_plan(inventory, base_plan, scans, cfg.sample)
+ }
+
+ /// Exit code for a run that committed (or confirmed) a corpus generation.
+ ///
+ /// `3` is "completed with junk — recorded, never fatal", the same contract the
+ /// legacy path publishes (`cli.rs` EXIT CODES, and CLAUDE.md's own indexing
+ /// workflow treats 3 as success). The generated path returned a flat `0`, which
+ /// silently downgraded that signal for `--no-graph`. Both inputs come from the
+ /// committed generation's own run document, so a no-op re-run over a corpus
+ /// that still contains junk reports 3 again instead of flapping to 0.
+ fn generated_exit_code(summary: &Value) -> i32 {
+ let count = |field: &str| summary.get(field).and_then(Value::as_u64).unwrap_or(0);
+ if count("junk_records_total") > 0 || count("files_junk") > 0 {
+ 3
+ } else {
+ 0
+ }
+ }
+
+ /// Terminal progress line for a run that ends on the generated `--no-graph`
+ /// path.
+ ///
+ /// Every exit closes the stream itself (#241). `Ticker::drop` reports
+ /// `ok=false` for a run that never called `finish`, so a generated run that
+ /// merely returned would print an aborted line straight after a successful
+ /// commit — the one place the two landed changes could contradict each other.
+ /// Exit 3 is success ("completed, some input was unusable"), so it is spelled
+ /// out in words rather than left as a bare number.
+ fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value) {
+ let count = |field: &str| summary.get(field).and_then(Value::as_u64).unwrap_or(0);
+ pr.finish(
+ true,
+ code,
+ if code == 3 {
+ "completed-with-junk"
+ } else {
+ "completed"
+ },
+ &[
+ ("files", count("files_indexed")),
+ ("records", count("records_total")),
+ ("generation", count("generation")),
+ ],
+ );
+ }
+
+ fn begin_non_graph_generation(
+ es: &Es,
+ journal: &mut state::Journal,
+ state_dir: &Path,
+ cfg: &IndexCfg,
+ root_identity: &str,
+ inventory: &content::Inventory,
+ plan: Plan,
+ ) -> Result<()> {
+ anyhow::ensure!(
+ cfg.no_graph,
+ "non-graph generation cutover requires --no-graph"
+ );
+ anyhow::ensure!(
+ journal.pending_sync.is_none(),
+ "cannot prepare over an existing pending generation"
+ );
+ if journal.committed_manifest.is_none() {
+ journal.sync_bootstrap_genesis()?;
+ }
+ let base = journal
+ .committed_manifest
+ .as_ref()
+ .context("generation cutover has no committed base")?
+ .clone();
+ let tx_id = format!("{}-g{}", journal.run_id, base.generation + 1);
+ let preparation_contract = preparation_contract_digest(cfg, &plan)?;
+ let snapshot = sync_executor::create_prepared_snapshot(
+ state_dir,
+ &tx_id,
+ inventory,
+ &plan,
+ &preparation_contract,
+ cfg.snapshot_max_bytes,
+ )?;
+ let chunker_identity = prepared_records_identity(cfg)?;
+ let semantic = plan
+ .datasets
+ .iter()
+ .any(|dataset| dataset.semantic_field.is_some());
+ let identity = if semantic {
+ let identity = es
+ .embedding_execution_identity()
+ .context("generation cutover could not pin the server embedding execution identity")?;
+ anyhow::ensure!(
+ identity.resumable,
+ "generation cutover requires a resumable embedding execution identity: {}",
+ identity
+ .non_resumable_reason
+ .as_deref()
+ .unwrap_or("the server did not provide an immutable identity")
+ );
+ journal.pin_embedding_identity(
+ &identity.identity_sha256,
+ identity.resumable,
+ identity.non_resumable_reason.as_deref(),
+ )?;
+ identity
+ } else {
+ crate::esclient::EmbeddingExecutionIdentity {
+ version: 1,
+ backend: "disabled".into(),
+ identity_sha256: "0".repeat(64),
+ // A disabled embedding execution has no vector width at all. `None`
+ // says exactly that; any number here would be a fiction the
+ // generation contract would then hold future runs to.
+ dimensions: None,
+ semantic_contract: "disabled-no-semantic-fields-v1".into(),
+ resumable: true,
+ non_resumable_reason: None,
+ }
+ };
+
+ let aliases_by_content = plan.duplicate_files.iter().fold(
+ HashMap::<&str, Vec<sync::ManifestPath>>::new(),
+ |mut aliases, alias| {
+ aliases
+ .entry(alias.file_key.as_str())
+ .or_default()
+ .push(sync::ManifestPath {
+ path_id: alias.path_id.clone(),
+ rel: alias.rel.clone(),
+ is_symlink: alias.is_symlink.unwrap_or(false),
+ });
+ aliases
+ },
+ );
+ let file_by_content = inventory
+ .keys
+ .iter()
+ .zip(&inventory.files)
+ .zip(&inventory.digests)
+ .map(|((key, file), digest)| (key.as_str(), (file, digest)))
+ .collect::<HashMap<_, _>>();
+ let mut candidates = Vec::with_capacity(plan.files.len());
+ for (content_id, assignment) in &plan.files {
+ let (file, digest) = file_by_content
+ .get(content_id.as_str())
+ .with_context(|| format!("planned content {content_id} is absent from inventory"))?;
+ let mut paths = vec![sync::ManifestPath {
+ path_id: assignment.path_id.clone(),
+ rel: assignment.rel.clone(),
+ is_symlink: assignment.is_symlink.unwrap_or(file.is_symlink),
+ }];
+ paths.extend(
+ aliases_by_content
+ .get(content_id.as_str())
+ .cloned()
+ .unwrap_or_default(),
+ );
+ candidates.push(sync::DesiredContentGroup {
+ content_id: content_id.clone(),
+ content_digest: (*digest).clone(),
+ content_size: file.size,
+ dataset_slugs: assignment
+ .assignments
+ .iter()
+ .map(|(_, slug)| slug.clone())
+ .collect(),
+ paths,
+ expected_records: 0,
+ expected_passages: 0,
+ expected_vectors: 0,
+ expected_junk_records: 0,
+ });
+ }
+ let mut groups = sync::reconcile_groups(&base.groups, candidates)?;
+ sync_executor::bind_prepared_counts(&mut groups, &snapshot, &inventory.keys)?;
+ let (schema_identity, index_identity) = generation_contract_identities(&plan)?;
+ let desired = sync::GenerationManifest {
+ generation: base.generation + 1,
+ execution: Some(sync::ExecutionIdentity {
+ version: sync::EXECUTION_IDENTITY_VERSION,
+ root_identity: root_identity.to_owned(),
+ url: cfg.url.clone(),
+ prefix: cfg.prefix.clone(),
+ follow_symlinks: cfg.follow_symlinks,
+ chunker_identity,
+ embedding_identity_sha256: identity.identity_sha256,
+ embedding_backend: identity.backend,
+ embedding_dimension: identity.dimensions,
+ embedding_semantic_contract: identity.semantic_contract,
+ embedding_resumable: identity.resumable,
+ graph_enabled: false,
+ brain: "disabled".into(),
+ detector_identity: DETECTOR_DISABLED_IDENTITY.into(),
+ schema_identity,
+ index_identity,
+ source_policy: sync::SourceExecutionPolicy::DurableSnapshot {
+ reference: format!("sync-snapshots/{tx_id}"),
+ snapshot_digest: snapshot.snapshot_digest,
+ },
+ }),
+ plan,
+ groups,
+ };
+ let pending = sync::PendingSync::new(tx_id, &base, desired)?;
+ journal.sync_begin(&pending)
+ }
+
+ #[derive(Debug, PartialEq, Eq)]
+ struct CliErrorRoute {
+ exit_code: i32,
+ stdout: Option<String>,
+ stderr: Option<String>,
+ }
+
+ fn route_cli_error(error: &anyhow::Error, json_output: bool) -> CliErrorRoute {
+ if json_output {
+ // Two distinct refusals reach the JSON route, and both must keep it.
+ // `UnsafeFreshGenerationError` guards durable *generation* state on the
+ // `--no-graph` path; `UnsupportedInventoryDeltaError` (#254) guards the
+ // legacy graph-enabled path against a rerun that would strand live
+ // documents. Neither supersedes the other — they protect different
+ // destinations — so routing only one would silently drop the other's
+ // machine-readable rendering back to a prose stderr line.
+ if let Some(unsafe_fresh) = error.downcast_ref::<UnsafeFreshGenerationError>() {
+ return CliErrorRoute {
+ exit_code: 1,
+ stdout: Some(unsafe_fresh.to_json().to_string()),
+ stderr: None,
+ };
+ }
+ if let Some(delta) = error.downcast_ref::<UnsupportedInventoryDeltaError>() {
+ return CliErrorRoute {
+ exit_code: 1,
+ stdout: Some(delta.to_json().to_string()),
+ stderr: None,
+ };
+ }
+ }
+ CliErrorRoute {
+ exit_code: 1,
+ stdout: None,
+ stderr: Some(format!("error: {error:#}")),
+ }
+ }
/// Entry point for the `xerj autoindex` subcommand (blocking; the server
/// binary calls this via spawn_blocking). Returns the process exit code.
@@@ -322,21 -635,59 +768,64 @@@ struct GroupSketch
records: u64,
}
- fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64, stub: bool) -> FileScan {
+ fn take_pdf_spool_if_indexable<T>(
+ spool: &mut Option<T>,
+ is_junk: bool,
+ budget: &extract::pdf::ExtractionSpoolBudget,
+ ) -> Option<T> {
+ if is_junk {
+ if spool.is_some() {
+ budget.record_discarded_before_replay();
+ }
+ spool.take();
+ None
+ } else {
+ spool.take()
+ }
+ }
+
+ /// Everything phase A needs beyond the file list and the run config: where a
+ /// run-local PDF artifact may live, the shared admission budget, the one-line
+ /// capacity explanation reported when files fall back — and the progress
+ /// surface those lines go out through, because phase A now reports every file
+ /// it touches (#241) as well as every artifact it could not keep.
+ ///
+ /// Grouped so `scan_file` and `build_phase_a` each take one parameter instead
+ /// of four.
+ struct PhaseAContext<'a> {
+ state_dir: &'a Path,
+ budget: &'a std::sync::Arc<extract::pdf::ExtractionSpoolBudget>,
+ capacity_warning: Option<&'a str>,
+ progress: &'a Progress,
+ }
+
+ fn scan_file(
+ path: &Path,
+ size: u64,
+ digest: &str,
+ ctx: &PhaseAContext<'_>,
+ sample: usize,
+ max_file_gb: u64,
++ stub: bool,
+ ) -> FileScan {
+ let state_dir = ctx.state_dir;
+ let pdf_spool_budget = ctx.budget;
let mut out = FileScan {
sniffed: None,
sketches: Vec::new(),
junk: None,
+ pdf_spool: None,
+ pdf_spool_fallbacks: Vec::new(),
};
- let sn = match sniff::sniff(path) {
- Ok(s) => s,
- Err(e) => {
- out.junk = Some(("junk".into(), format!("unreadable: {e}")));
- return out;
+ let sn = if stub {
+ stub_sniffed()
+ } else {
+ match sniff::sniff(path) {
+ Ok(s) => s,
+ Err(e) => {
+ out.junk = Some(("junk".into(), format!("unreadable: {e}")));
+ return out;
+ }
}
};
if sn.family == Family::Binary {
@@@ -450,7 -838,17 +985,17 @@@ mod clustering_key_tests
let path = dir.join(name);
std::fs::write(&path, body).unwrap();
let size = std::fs::metadata(&path).unwrap().len();
- scan_file(&path, size, 500, 2, false)
+ // Clustering keys are decided by extraction, not by PDF artifact
+ // reuse: a zero budget keeps these cases on the plain parse path.
+ let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);
+ let progress = Progress::silent();
+ let ctx = PhaseAContext {
+ state_dir: dir,
+ budget: &budget,
+ capacity_warning: None,
+ progress: &progress,
+ };
- scan_file(&path, size, "d0", &ctx, 500, 2)
++ scan_file(&path, size, "d0", &ctx, 500, 2, false)
}
/// The #178 mechanism, from the extractor to the clustering key: a source
@@@ -524,6 -923,471 +1070,493 @@@
}
}
+ #[cfg(test)]
+ mod phase_a_grouping_tests {
+ use super::*;
+
+ fn cfg_for(root: &Path) -> IndexCfg {
+ IndexCfg {
+ root: root.to_path_buf(),
++ stub_globs: Vec::new(),
+ url: "http://unused.invalid".into(),
+ api_key: None,
+ workers: 1,
+ scan_workers: 1,
+ pdf_workers: 1,
+ resource_notes: Vec::new(),
+ pdf_timeout_secs: 10,
+ bulk_mb: 1,
+ bulk_timeout_secs: 10,
+ snapshot_max_bytes: 64 << 30,
+ prefix: "t".into(),
+ state_dir: None,
+ fresh: true,
+ follow_symlinks: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
+ max_file_gb: 2,
+ sample: 500,
+ no_semantic: false,
+ brain: None,
+ no_graph: true,
+ dry_run: true,
+ json: false,
+ quiet: true,
+ progress: crate::progress::ProgressMode::None,
+ progress_interval: None,
+ }
+ }
+
+ fn plan_for(root: &Path) -> Plan {
+ let files = walk::walk(root, false).unwrap();
+ let keys: Vec<String> = files
+ .iter()
+ .map(|f| ids::file_key(&f.path, f.size).unwrap())
+ .collect();
+ let digests: Vec<String> = (0..files.len()).map(|i| format!("d{i}")).collect();
+ // Planning is what these cases assert on; a zero budget keeps every
+ // file on the plain parse path so no artifact is ever retained.
+ let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);
+ let progress = Progress::silent();
+ let ctx = PhaseAContext {
+ state_dir: root,
+ budget: &budget,
+ capacity_warning: None,
+ progress: &progress,
+ };
+ build_phase_a(
+ root,
+ &files,
+ &keys,
+ &digests,
+ Vec::new(),
+ &ctx,
+ &cfg_for(root),
+ )
+ .plan
+ }
+
+ const CODE: &str = "// The event loop dispatches every ready connection to a worker.\n\
+ // Each worker drains its queue before polling for more sockets.\n\
+ static void dispatch_ready_connections(struct event_loop *loop) {\n\
+ for (int index = 0; index < loop->ready_count; index++) {\n\
+ worker_submit(loop->workers, loop->ready[index]);\n\
+ }\n\
+ }\n";
+
+ const PROSE: &str = "# Overview\n\nThis server accepts client connections and stores \
+ keys in memory. Every command travels through the same parser before the \
+ dispatcher routes it to the matching handler function.\n";
+
+ /// #173 end to end through the real planner: a tree holding two
+ /// repositories of source, prose and one-off config JSON yields one
+ /// document dataset per repository — with `body` elected `semantic_text`
+ /// — plus the genuine data dataset, instead of one dataset per incidental
+ /// config schema with no vector arm.
+ #[test]
+ fn a_two_repo_tree_plans_one_document_dataset_per_repo() {
+ let dir = tempfile::tempdir().unwrap();
+ let root = dir.path();
+ for repo in ["valkey", "memcached"] {
+ std::fs::create_dir_all(root.join(repo).join(".git")).unwrap();
+ std::fs::write(root.join(repo).join(".git").join("HEAD"), "ref: x").unwrap();
+ }
+ std::fs::create_dir_all(root.join("valkey/src")).unwrap();
+ std::fs::create_dir_all(root.join("valkey/commands")).unwrap();
+ std::fs::create_dir_all(root.join("memcached/data")).unwrap();
+ std::fs::write(root.join("valkey/src/server.c"), CODE).unwrap();
+ std::fs::write(root.join("valkey/README.md"), PROSE).unwrap();
+ // one-off configs: single-record JSON, each with its own key set
+ std::fs::write(
+ root.join("valkey/commands/get.json"),
+ r#"{"GET": {"summary": "Return the string value stored at the given key.", "arity": 2}}"#,
+ )
+ .unwrap();
+ std::fs::write(
+ root.join("valkey/commands/set.json"),
+ r#"{"SET": {"summary": "Store the given string value under the given key.", "arity": 3}}"#,
+ )
+ .unwrap();
+ // distinct bytes — byte-identical files alias into one canonical copy
+ std::fs::write(root.join("memcached/proto.c"), format!("{CODE}\n// v2\n")).unwrap();
+ // a genuine data file: recurring rows, real schema
+ std::fs::write(
+ root.join("memcached/data/events.csv"),
+ "id,email,level\n1,[email protected],info\n2,[email protected],warn\n3,[email protected],info\n",
+ )
+ .unwrap();
+
+ let plan = plan_for(root);
+ let mut slugs: Vec<&str> = plan.datasets.iter().map(|d| d.slug.as_str()).collect();
+ slugs.sort();
+ assert_eq!(
+ slugs,
+ ["memcached-data", "memcached-docs", "valkey-docs"],
+ "{:#?}",
+ plan.datasets
+ );
+
+ // every docs dataset carries the vector arm on body
+ for d in plan.datasets.iter().filter(|d| d.family == "docs") {
+ let body = d.specs.iter().find(|s| s.name == "body").unwrap();
+ assert_eq!(body.es_type, "semantic_text", "{}: {:#?}", d.slug, d.specs);
+ assert_eq!(d.semantic_field.as_deref(), Some("body"), "{}", d.slug);
+ }
+
+ // the one-off configs were demoted: document-rendered, and their
+ // config keys never reached the docs mapping
+ let by_rel: HashMap<&str, &FileAssignment> =
+ plan.files.values().map(|f| (f.rel.as_str(), f)).collect();
+ assert!(by_rel["valkey/commands/get.json"].as_document);
+ assert!(by_rel["valkey/commands/set.json"].as_document);
+ assert!(!by_rel["valkey/src/server.c"].as_document);
+ assert!(!by_rel["memcached/data/events.csv"].as_document);
+ let vdocs = plan
+ .datasets
+ .iter()
+ .find(|d| d.slug == "valkey-docs")
+ .unwrap();
+ assert!(
+ !vdocs.specs.iter().any(|s| s.name.contains("summary")),
+ "config keys leaked into the docs mapping: {:#?}",
+ vdocs.specs
+ );
+ assert_eq!(vdocs.file_count, 4, "code + prose + 2 demoted configs");
+
+ // the CSV kept its real schema
+ let data = plan
+ .datasets
+ .iter()
+ .find(|d| d.slug == "memcached-data")
+ .unwrap();
+ assert!(data.specs.iter().any(|s| s.name == "email"), "{data:#?}");
+ }
+
+ /// #196: the same tree WITHOUT nested `.git` markers (or with one at the
+ /// root — a workspace) is ONE scope: a single document corpus.
+ #[test]
+ fn a_single_workspace_plans_one_document_dataset() {
+ let dir = tempfile::tempdir().unwrap();
+ let root = dir.path();
+ std::fs::create_dir_all(root.join(".git")).unwrap();
+ std::fs::create_dir_all(root.join("src")).unwrap();
+ std::fs::create_dir_all(root.join("wasm/examples")).unwrap();
+ std::fs::write(root.join("src/a.c"), CODE).unwrap();
+ std::fs::write(root.join("src/b.c"), CODE).unwrap();
+ std::fs::write(root.join("README.md"), PROSE).unwrap();
+ std::fs::write(root.join("wasm/examples/demo.md"), PROSE).unwrap();
+
+ let plan = plan_for(root);
+ assert_eq!(
+ plan.datasets.len(),
+ 1,
+ "one workspace, one corpus: {:#?}",
+ plan.datasets
+ );
+ assert_eq!(plan.datasets[0].slug, "docs");
+ assert_eq!(plan.datasets[0].file_count, 4);
+ }
+ }
+
+ // ─── Phase A plan building (pure: no server contact) ─────────────────────
+
+ /// Repository scope per file: the deepest ancestor directory (root-relative,
+ /// "/"-separated) containing a `.git` entry, or "" when the file is under
+ /// none. The walk never descends into `.git` itself, but the marker is still
+ /// on disk. A `.git` at the autoindex root yields "" for every file — the
+ /// whole tree is one scope — and so does a tree with no `.git` at all, which
+ /// is exactly the #173/#196 property: one autoindex of one folder yields a
+ /// corpus searchable as one corpus, split only at nested repository roots.
+ fn compute_scopes(root: &Path, rels: &[String]) -> Vec<String> {
+ let mut cache: HashMap<String, bool> = HashMap::new();
+ rels.iter()
+ .map(|rel| {
+ let mut scope = String::new();
+ let mut prefix = String::new();
+ let mut segs = rel.split('/').peekable();
+ while let Some(seg) = segs.next() {
+ if segs.peek().is_none() {
+ break; // final segment is the file name
+ }
+ if !prefix.is_empty() {
+ prefix.push('/');
+ }
+ prefix.push_str(seg);
+ let repo = *cache
+ .entry(prefix.clone())
+ .or_insert_with(|| root.join(&prefix).join(".git").exists());
+ if repo {
+ scope = prefix.clone();
+ }
+ }
+ scope
+ })
+ .collect()
+ }
+
+ /// What phase A produces: the frozen plan, the clusters phase B needs, and
+ /// the run-local PDF artifacts (index-aligned with `files`) that phase B may
+ /// replay instead of parsing again. A spool is always optional — every entry
+ /// may legitimately be `None`.
+ struct PhaseA {
+ plan: Plan,
+ clusters: Vec<dataset::Cluster>,
+ pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>>,
+ }
+
+ /// Record and report why individual PDFs could not retain a run-local
+ /// artifact. Reuse is an accelerator, so this is purely informational: every
+ /// listed file is parsed again by the normal phase B path. Recording is kept
+ /// even when the budget is globally disabled — the stored examples are the
+ /// only place the `--json` report explains *why* nothing was reused, and they
+ /// are already capped at three.
+ ///
+ /// Reporting goes out through the progress surface, never a bare `eprintln!`:
+ /// stderr belongs to that surface, so `--progress none` stays silent and
+ /// `--progress json` stays a single parseable stream (#241).
+ fn report_pdf_spool_fallbacks(
+ files: &[walk::FileEntry],
+ scans: &[FileScan],
+ ctx: &PhaseAContext<'_>,
+ ) {
+ let pdf_spool_budget = ctx.budget;
+ let pr = ctx.progress;
+ let reasons: Vec<(&str, &extract::pdf::SpoolFallback)> = scans
+ .iter()
+ .enumerate()
+ .flat_map(|(index, scan)| {
+ scan.pdf_spool_fallbacks
+ .iter()
+ .map(move |fallback| (files[index].rel.as_str(), fallback))
+ })
+ .collect();
+ for (path, fallback) in &reasons {
+ pdf_spool_budget.record_fallback_example(path, fallback.category, &fallback.message);
+ }
+ if reasons.is_empty() {
+ return;
+ }
+ if pdf_spool_budget.platform_reuse_is_unavailable() {
+ pr.note(
+ "phase A: run-local PDF extraction reuse is unavailable on this platform; \
+ phase B will use the normal parser",
+ );
+ return;
+ }
+ pr.note(&format!(
+ "phase A: {} PDF extraction(s) could not retain a bounded run-local artifact; \
+ phase B will parse them again safely",
+ reasons.len()
+ ));
+ if let Some(warning) = ctx.capacity_warning {
+ pr.note(&format!(" PDF reuse capacity: {warning}"));
+ }
+ for (path, fallback) in reasons.iter().take(3) {
+ pr.note(&format!(
+ " PDF reuse fallback for {path}: {}",
+ fallback.message
+ ));
+ }
+ if reasons.len() > 3 {
+ pr.note(&format!(
+ " … and {} more PDF reuse fallback(s)",
+ reasons.len() - 3
+ ));
+ }
+ }
+
+ /// Sniff + sample every file, cluster into datasets, and assemble the plan.
+ /// Pure planning: reads the tree, never the server — which is what makes the
+ /// #173/#196 grouping behaviour testable end-to-end without a cluster.
+ fn build_phase_a(
+ root: &Path,
+ files: &[walk::FileEntry],
+ keys: &[String],
+ digests: &[String],
+ duplicate_files: Vec<DuplicateFile>,
+ ctx: &PhaseAContext<'_>,
+ cfg: &IndexCfg,
+ ) -> PhaseA {
+ use rayon::prelude::*;
+ let pdf_spool_budget = ctx.budget;
+ let pr = ctx.progress;
+ // Same pool as the digest phase: sniffing and sampling are the other half
+ // of the CPU-bound phase `--workers` has to bound (#240 §2). Progress is
+ // reported from inside that pool, so the straggler the ticker names is the
+ // file a scan-pool thread is genuinely sitting on (#241). Retaining a PDF
+ // artifact happens inside that same guard, so a file whose extraction is
+ // spooled is counted exactly like a plainly parsed one.
++ // Patterns were validated (loudly) at run start; a failure here would be
++ // a programming error, not user input.
++ let stub_matcher = StubMatcher::compile(&cfg.stub_globs).expect("validated at startup");
+ let scans: Vec<FileScan> = crate::pool::install(|| {
+ files
+ .par_iter()
+ .zip(digests.par_iter())
+ .map(|(f, digest)| {
+ let _in_flight = pr.file(&f.rel, f.size);
- scan_file(&f.path, f.size, digest, ctx, cfg.sample, cfg.max_file_gb)
++ scan_file(
++ &f.path,
++ f.size,
++ digest,
++ ctx,
++ cfg.sample,
++ cfg.max_file_gb,
++ stub_matcher.matches(&f.rel),
++ )
+ })
+ .collect()
+ });
+
+ report_pdf_spool_fallbacks(files, &scans, ctx);
+
+ let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();
+ let scopes = compute_scopes(root, &rels);
+ // (family, gzip) per file, from the scan's sniff — reused for assignments
+ // and demoted-file re-sampling instead of re-sniffing every file.
+ let file_meta: Vec<Option<(Family, bool)>> = scans
+ .iter()
+ .map(|sc| sc.sniffed.as_ref().map(|s| (s.family, s.gzip)))
+ .collect();
+ let mut sketches = Vec::new();
+ let mut junk_files = Vec::new();
+ let mut pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>> =
+ (0..files.len()).map(|_| None).collect();
+ for (i, mut sc) in scans.into_iter().enumerate() {
+ let family = sc
+ .sniffed
+ .as_ref()
+ .map(|s| s.family)
+ .unwrap_or(Family::Binary);
+ // A file that phase A junks is never indexed, so its artifact is
+ // refunded here rather than held to the phase A→B boundary.
+ pdf_spools[i] =
+ take_pdf_spool_if_indexable(&mut sc.pdf_spool, sc.junk.is_some(), pdf_spool_budget);
+ if let Some((status, reason)) = sc.junk {
+ junk_files.push(JunkFile {
+ file_key: keys[i].clone(),
+ rel: files[i].rel.clone(),
+ format: format_str(sc.sniffed.as_ref()),
+ status,
+ reason,
+ bytes: files[i].size,
+ });
+ continue;
+ }
+ for gs in sc.sketches {
+ sketches.push(dataset::Sketch {
+ file_idx: i,
+ group: gs.group,
+ family,
+ fields: gs.fields,
+ key_fields: gs.key_fields,
+ records: gs.records,
+ });
+ }
+ }
+ let mut clusters = dataset::cluster(sketches, &rels, &scopes);
+
+ // Demoted one-off config files (#173) were sampled as flattened records;
+ // phase B will index them as documents. Re-sample them through the same
+ // document renderer so the docs dataset's mapping and stats describe what
+ // actually gets indexed.
+ for c in clusters.iter_mut().filter(|c| !c.demoted.is_empty()) {
+ let demoted = c.demoted.clone();
+ let fields = &mut c.fields;
+ let mut sampled_total = 0u64;
+ for m in demoted {
+ let gzip = file_meta[m].map(|(_, g)| g).unwrap_or(false);
+ let mut sampled = 0u64;
+ let mut sink = |rec: extract::RawRecord| -> bool {
+ if (sampled as usize) < cfg.sample {
+ for (k, v) in &rec.fields {
+ fields.entry(k.clone()).or_default().add(v);
+ }
+ }
+ sampled += 1;
+ (sampled as usize) < cfg.sample
+ };
+ // An unreadable file junks at phase B like any other; the plan
+ // keeps its membership either way.
+ let _ = extract::extract_as_document(&files[m].path, gzip, &mut sink);
+ sampled_total += sampled;
+ }
+ c.records += sampled_total;
+ }
+
+ // per-file assignments
+ let mut file_assignments: HashMap<String, FileAssignment> = HashMap::new();
+ for (ci, c) in clusters.iter().enumerate() {
+ let demoted: std::collections::HashSet<usize> = c.demoted.iter().copied().collect();
+ for &m in &c.members {
+ let key = &keys[m];
+ let (family, gzip) = file_meta[m].unwrap_or((c.family, false));
+ let fa = file_assignments
+ .entry(key.clone())
+ .or_insert_with(|| FileAssignment {
+ rel: files[m].rel.clone(),
+ path_id: files[m].rel_id.clone(),
+ is_symlink: Some(files[m].is_symlink),
+ family: family.as_str().to_string(),
+ gzip,
+ content_digest: Some(digests[m].clone()),
+ assignments: Vec::new(),
+ as_document: false,
+ });
+ fa.as_document |= demoted.contains(&m);
+ fa.assignments
+ .push((c.group.clone(), clusters[ci].slug.clone()));
+ }
+ }
+
+ let mut datasets = Vec::new();
+ for c in &clusters {
- let specs =
++ let mut specs =
+ infer::infer_fields_with_policy(&c.fields, c.records, cfg.no_semantic, c.is_docs);
++ // Unity script-link enrichment fields are stamped by the phase-B
++ // pipeline (not the extractor), so inference never sees them —
++ // register them here or they would be dynamic-mapped coarsely.
++ if c.family == Family::UnityYaml && c.fields.contains_key("script_guid") {
++ specs.push(pipeline_keyword_spec("script_path"));
++ specs.push(pipeline_keyword_spec("script_class"));
++ }
++ if c.family == Family::UnityMeta {
++ specs.push(pipeline_keyword_spec("asset_path"));
++ }
+ let time_field = infer::elect_time_field(&specs);
+ let semantic_field = specs
+ .iter()
+ .find(|s| s.es_type == "semantic_text")
+ .map(|s| s.name.clone());
+ datasets.push(PlanDataset {
+ slug: c.slug.clone(),
+ index: format!("{}-{}", cfg.prefix, c.slug),
+ family: if c.is_docs {
+ "docs".to_string()
+ } else {
+ c.family.as_str().to_string()
+ },
+ group: c.group.clone(),
+ specs,
+ time_field,
+ semantic_field,
+ sampled_records: c.records,
+ file_count: c.members.len(),
+ });
+ }
+ let plan = Plan {
+ datasets,
+ files: file_assignments,
+ junk_files,
+ duplicate_files,
+ alias_paths_indexed: true,
+ };
+ PhaseA {
+ plan,
+ clusters,
+ pdf_spools,
+ }
+ }
+
// ─── mapping builder ─────────────────────────────────────────────────────
pub const PROVENANCE_FIELDS: &[&str] = &[
@@@ -797,41 -2092,61 +2352,62 @@@ fn run_index(cfg: IndexCfg) -> Result<i
/// `None` when the run ended before a plan produced one (empty folder,
/// `--dry-run`).
pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
+ // The very first statement of the function, deliberately: `started` must
+ // be when this invocation began, not when its summary was built.
+ let invocation_started = chrono::Utc::now();
+ // Fix the phase-A pool width BEFORE anything parallel starts: hashing and
+ // sniffing are the CPU-bound phase, and they used to take every core no
+ // matter what the caller asked for (#240 §2).
+ pool::configure(cfg.scan_workers);
+ // What phase A is *actually* running on. `pool::configure` is first-call-wins
+ // (rayon pools cannot be resized), so in a process that already indexed once
+ // — `xerj-server`'s brain endpoint does exactly that — the installed width
+ // can differ from what this run's plan asked for. Progress must state the
+ // number the policy got, not the one it requested (#240 + #241).
+ let scan_threads = pool::scan_pool().current_num_threads();
extract::pdf::configure_workers(cfg.pdf_workers);
extract::pdf::configure_timeout(cfg.pdf_timeout_secs);
- use rayon::prelude::*;
let t0 = Instant::now();
- if !cfg.quiet {
- eprintln!(
- "autoindex: bulk HTTP request timeout: {}s",
- cfg.bulk_timeout_secs
- );
+ // The progress surface and its ticker are the FIRST things built: every
+ // later phase reports through them, and the ticker guarantees the stream
+ // closes with a terminal line even if this function bails (#241). The
+ // resource plan is announced through the same surface rather than through a
+ // bare `eprintln!`, so `--quiet` / `--progress none` still means silent and
+ // `--progress json` still means one machine-readable stream (#240 + #241).
+ let surface = progress::detect(cfg.progress);
+ let pr = Progress::new(
+ surface,
+ cfg.progress_interval
+ .unwrap_or_else(|| progress::default_interval(surface)),
+ );
+ let ticker = pr.spawn_ticker();
+ // What this run decided to take from the machine, before it takes it.
+ pr.note(&format!(
+ "autoindex: {} scan threads, {} index workers, {} pdf workers, --bulk-mb {} [{}]",
+ scan_threads,
+ cfg.workers,
+ cfg.pdf_workers,
+ cfg.bulk_mb,
+ xerj_common::resource::describe(),
+ ));
+ for note in &cfg.resource_notes {
+ pr.note(&format!("autoindex: {note}"));
}
- let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;
+ pr.note(&format!(
+ "autoindex: bulk HTTP request timeout: {}s",
+ cfg.bulk_timeout_secs
+ ));
+ // The run's bulk load is admitted through one window `--workers` wide, so
+ // a 429 can shrink what the run offers instead of only delaying it
+ // (#240 §8). Enabled here and nowhere else: probes have nothing to
+ // throttle. Its shrink/recover announcements go to stderr, which the
+ // progress surface owns — so they are emitted exactly when that surface is
+ // enabled, not merely when `--quiet` is absent.
+ let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?
+ .with_bulk_concurrency(cfg.workers, pr.enabled());
es.ping()?;
+ let stub_matcher = StubMatcher::compile(&cfg.stub_globs)?;
- let (discovered_files, skipped_dirs) =
- walk::walk(
- &cfg.root,
- cfg.follow_symlinks,
- !cfg.no_default_excludes,
- !cfg.no_gitignore,
- )?;
- if !skipped_dirs.is_empty() && !cfg.quiet {
- let names: Vec<&str> = skipped_dirs.iter().map(|s| s.rel.as_str()).collect();
- eprintln!(
- "skipping {} generated director{} ({}): {} — pass --no-default-excludes to include",
- skipped_dirs.len(),
- if skipped_dirs.len() == 1 { "y" } else { "ies" },
- skipped_dirs[0].reason,
- names.join(", ")
- );
- }
- if discovered_files.is_empty() {
- println!("no files found under {}", cfg.root.display());
- return Ok((0, None));
- }
let root_str = cfg
.root
.canonicalize()
@@@ -1624,9 -3357,10 +3625,11 @@@
let mut fields = rec.fields;
let dropped = coerce::coerce_record(&mut fields, &rt.plan);
if dropped > 0 {
- rt.dropped.fetch_add(dropped as u64, Ordering::Relaxed);
+ let durable =
+ file_dropped_by_dataset.entry(slug.clone()).or_default();
+ *durable = durable.saturating_add(dropped as u64);
}
+ enrich_unity_fields(sn.family, &mut fields, &unity_guid_map, &f.rel);
fields.insert("ax_path".into(), Value::String(f.rel.clone()));
fields.insert(
"ax_paths".into(),
diff --cc engine/crates/xerj-autoindex/src/sniff.rs
index 49f31bdfb,4802cd260..1a7161093
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@@ -605,115 -518,6 +620,115 @@@ fn sniff_csv_dialect(nonblank: &[&str]
})
}
+#[cfg(test)]
+mod unity_sniff_tests {
+ use super::*;
+ use std::path::Path;
+
+ fn sniff_str(s: &str, name: &str) -> Family {
- sniff_bytes(s.as_bytes(), Path::new(name), false)
++ sniff_bytes(s.as_bytes(), Path::new(name), Path::new(name), false)
+ .unwrap()
+ .family
+ }
+
+ #[test]
+ fn unity_tagged_yaml_is_detected_by_header_not_extension() {
+ let scene = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &123\nGameObject:\n m_Name: X\n";
+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);
+ assert_eq!(
+ sniff_str(scene, "renamed.txt"),
+ Family::UnityYaml,
+ "content decides, never the extension"
+ );
+ }
+
+ #[test]
+ fn a_bom_before_the_yaml_directive_is_tolerated() {
+ let scene =
+ "\u{feff}%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1\nGameObject:\n m_Name: X\n";
+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);
+ }
+
+ #[test]
+ fn meta_needs_the_first_line_rule_and_a_guid() {
+ let meta = "fileFormatVersion: 2\nguid: 9f1c4d0ab2e34f6\nMonoImporter:\n serializedVersion: 2\n";
+ assert_eq!(sniff_str(meta, "Player.cs.meta"), Family::UnityMeta);
+ let stray = "config: true\nfileFormatVersion: 2\nguid: abc\n";
+ assert_ne!(
+ sniff_str(stray, "some.yaml"),
+ Family::UnityMeta,
+ "guid keys inside ordinary YAML must not reclassify it"
+ );
+ let no_guid = "fileFormatVersion: 2\nsettings:\n a: 1\n";
+ assert_ne!(sniff_str(no_guid, "x.meta"), Family::UnityMeta);
+ }
+
+ #[test]
+ fn bvh_is_detected_by_hierarchy_root_header() {
+ let bvh = "HIERARCHY\nROOT Hips\n{\n OFFSET 0 90 0\n CHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation\n}\nMOTION\nFrames: 2\nFrame Time: 0.033\n1 2 3\n4 5 6\n";
+ assert_eq!(sniff_str(bvh, "clip.bvh"), Family::Bvh);
+ assert_eq!(
+ sniff_str(bvh, "clip.txt"),
+ Family::Bvh,
+ "content decides, never the extension"
+ );
+ let not_bvh = "HIERARCHY\nof needs (Maslow):\n- physiological\n- safety\n";
+ assert_ne!(sniff_str(not_bvh, "notes.txt"), Family::Bvh);
+ }
+
+ #[test]
+ fn plain_yaml_without_the_unity_tag_stays_yaml() {
+ let plain = "%YAML 1.2\n---\nkey: value\nother: 1\nnested:\n a: 2\n";
+ assert_ne!(sniff_str(plain, "doc.yaml"), Family::UnityYaml);
+ }
+
+ /// Regression: PSD image data decoded via lossy windows-1252 passed the
+ /// NUL/control-char heuristics and classified as txt-prose, turning a
+ /// multi-MB texture into ~100x its size of prose sections. Media magic
+ /// bytes must win before any text heuristic runs.
+ #[test]
+ fn media_containers_are_binary_by_magic_not_heuristics() {
+ for (name, head) in [
+ ("t.psd", &b"8BPS\x00\x01"[..]),
+ ("t.tif", &b"II*\x00\x08\x00"[..]),
+ ("t.tif2", &b"MM\x00*\x00\x08"[..]),
+ ("t.wav", &b"RIFF\x24\x08\x00\x00WAVE"[..]),
+ ("t.ogg", &b"OggS\x00\x02"[..]),
+ ("t.fbx", &b"Kaydara FBX Binary \x00"[..]),
+ ] {
+ let mut bytes = head.to_vec();
+ // A printable tail that WOULD pass the prose heuristics.
+ bytes.extend(b"lorem ipsum dolor sit amet, consectetur adipiscing elit. ".repeat(20));
- let sn = sniff_bytes(&bytes, Path::new(name), false).unwrap();
++ let sn = sniff_bytes(&bytes, Path::new(name), Path::new(name), false).unwrap();
+ assert_eq!(sn.family, Family::Binary, "{name} must be binary");
+ }
+ }
+
+ /// Regression: an uncompressed TGA (no magic bytes exist for TGA) decoded
+ /// via lossy windows-1252 into printable soup with near-zero whitespace
+ /// and classified txt-prose. Whitespace ratio is the discriminator.
+ #[test]
+ fn whitespace_free_printable_soup_is_binary_not_prose() {
+ let soup: String = (0..8192u32)
+ .map(|i| char::from_u32(0x21 + (i * 7) % 0x5d).unwrap())
+ .collect();
- let sn = sniff_bytes(soup.as_bytes(), Path::new("texture.tga"), false).unwrap();
++ let sn = sniff_bytes(soup.as_bytes(), Path::new("texture.tga"), Path::new("texture.tga"), false).unwrap();
+ assert_eq!(sn.family, Family::Binary);
+ // Real prose of the same size keeps its family.
+ let prose = "The quick brown fox jumps over the lazy dog. ".repeat(200);
- let sn = sniff_bytes(prose.as_bytes(), Path::new("note.txt"), false).unwrap();
++ let sn = sniff_bytes(prose.as_bytes(), Path::new("note.txt"), Path::new("note.txt"), false).unwrap();
+ assert_ne!(sn.family, Family::Binary, "real prose must stay text");
+ }
+
+ #[test]
+ fn binary_serialized_unity_assets_stay_binary() {
+ let mut bytes = b"UnityFS\x00\x00\x00\x00\x08".to_vec();
+ bytes.extend(std::iter::repeat(0u8).take(64));
- let sn = sniff_bytes(&bytes, Path::new("scene.unity"), false).unwrap();
++ let sn = sniff_bytes(&bytes, Path::new("scene.unity"), Path::new("scene.unity"), false).unwrap();
+ assert_eq!(sn.family, Family::Binary);
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --cc engine/crates/xerj-autoindex/src/walk.rs
index bdfd7fd55,c230cac04..10eb29266
--- a/engine/crates/xerj-autoindex/src/walk.rs
+++ b/engine/crates/xerj-autoindex/src/walk.rs
@@@ -26,136 -23,87 +23,148 @@@ pub struct FileEntry
pub size: u64,
}
- /// A directory pruned whole by the generated-dir hygiene rule, recorded so
- /// the run can surface it (junk report / map) instead of skipping silently.
- #[derive(Debug, Clone)]
- pub struct SkippedDir {
- pub rel: String,
- pub reason: String,
- }
-
- /// Well-known generated/cache directories, pruned at ANY depth but ONLY when
- /// a sibling marker file proves the parent directory is that kind of project
- /// (a monorepo may nest e.g. `apps/game/Library`, `apps/web/node_modules`).
- /// Marker-gated like the dotfile rule (hygiene, evidence-based) — never
- /// extension guessing, never a per-corpus code path.
- /// `--no-default-excludes` disables the rule.
++/// Generated/cache directories whose NAMES are too common for the blunt
++/// [`crate::ignore_rules::DEFAULT_IGNORE_PATTERNS`] (a folder named `Logs/`
++/// or `Library/` is often real data), pruned at any depth ONLY when a sibling
++/// marker file proves the parent directory is that kind of project. Part of
++/// the built-in defaults: `--no-ignore` / `--no-default-ignores` disable it,
++/// and any ignore file in the tree can re-include (`!Library/`) as usual —
++/// these run after the ignore files have had their say.
+///
- /// (dir name, marker path relative to the dir's PARENT, reason)
- const GENERATED_DIR_RULES: &[(&str, &str, &str)] = &[
++/// (dir name, marker path relative to the dir's PARENT, report label)
++const MARKER_GENERATED_DIRS: &[(&str, &str, &str)] = &[
+ (
+ "Library",
+ "ProjectSettings/ProjectVersion.txt",
- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
++ "default:unity-generated",
+ ),
+ (
+ "Temp",
+ "ProjectSettings/ProjectVersion.txt",
- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
++ "default:unity-generated",
+ ),
+ (
+ "obj",
+ "ProjectSettings/ProjectVersion.txt",
- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
++ "default:unity-generated",
+ ),
+ (
+ "Logs",
+ "ProjectSettings/ProjectVersion.txt",
- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
++ "default:unity-generated",
+ ),
+ (
+ "UserSettings",
+ "ProjectSettings/ProjectVersion.txt",
- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
- ),
- (
- "node_modules",
- "package.json",
- "installed npm/yarn dependency tree (marker: sibling package.json)",
- ),
- (
- "target",
- "Cargo.toml",
- "Cargo build directory (marker: sibling Cargo.toml)",
++ "default:unity-generated",
+ ),
+];
+
- pub fn walk(
++/// The marker-gated verdict for one directory entry.
++fn marker_generated_dir(path: &Path) -> Option<&'static str> {
++ let name = path.file_name()?;
++ for (dir, marker, label) in MARKER_GENERATED_DIRS {
++ if name == *dir && path.parent().is_some_and(|p| p.join(marker).is_file()) {
++ return Some(label);
++ }
++ }
++ None
++}
++
+ /// Walk with the default ignore rules on and the report discarded.
+ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
+ walk_reporting(root, follow_symlinks, IgnoreOptions::default()).map(|(files, _)| files)
+ }
+
+ /// Walk, and say what was left out and why.
+ ///
+ /// The returned [`IgnoreReport`] is the answer to "my files vanished" — it
+ /// names every rule that discarded something and how much. It is reported on
+ /// every run and, under `--dry-run`, also counts what was inside each pruned
+ /// directory.
+ pub fn walk_reporting(
root: &Path,
follow_symlinks: bool,
- default_excludes: bool,
- gitignore: bool,
- ) -> Result<(Vec<FileEntry>, Vec<SkippedDir>)> {
+ ignore: IgnoreOptions,
+ ) -> Result<(Vec<FileEntry>, IgnoreReport)> {
let root_canon = root
.canonicalize()
.with_context(|| format!("resolve root folder {}", root.display()))?;
if !root_canon.is_dir() {
anyhow::bail!("{} is not a directory", root_canon.display());
}
- // Arc<Mutex>: the `ignore` walker's filter must be Send+Sync even for the
- // sequential build() we use.
- let skipped: std::sync::Arc<std::sync::Mutex<Vec<SkippedDir>>> = Default::default();
let mut out = Vec::new();
- // SECURITY / hygiene: never index hidden files or descend into hidden
- // directories (`hidden(true)`). Without this the walker happily indexed
- // `.env` (secrets, API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles
- // into a queryable brain with no per-brain authorization — a real exposure
- // for the "point it at my project folder" use case. The root itself is
- // exempt (the walk origin is always yielded) so a brain over a dot-named
- // folder still works.
- //
- // `.gitignore` handling stays deterministic and corpus-local: tree
- // `.gitignore`s + `.git/info/exclude` only, never the user's global
- // gitignore config; `require_git` keeps the semantics evidence-based (a
- // stray .gitignore outside a repo is not an owner declaration).
- let sk = std::sync::Arc::clone(&skipped);
- let filter_root = root_canon.clone();
- let mut builder = ignore::WalkBuilder::new(&root_canon);
- builder
++ let marker_defaults = ignore.enabled && ignore.defaults;
+ let mut stack = IgnoreStack::new(&root_canon, ignore);
+ // Manual iteration rather than `filter_entry`, because the ignore stack has
+ // to load a directory's ignore files *after* that directory is admitted and
+ // *before* its children are judged. `skip_current_dir` on a just-yielded
+ // directory prunes its contents — walkdir's own documented pattern, and the
+ // same effect `filter_entry` had.
+ let mut it = walkdir::WalkDir::new(&root_canon)
.follow_links(follow_symlinks)
- .hidden(true)
- .ignore(false)
- .parents(gitignore)
- .git_ignore(gitignore)
- .git_exclude(gitignore)
- .git_global(false)
- .require_git(true)
- .filter_entry(move |e| {
- if e.depth() == 0 {
- return true;
- }
- if default_excludes && e.file_type().is_some_and(|t| t.is_dir()) {
- for (name, marker, reason) in GENERATED_DIR_RULES {
- if e.file_name() == *name
- && e.path()
- .parent()
- .is_some_and(|p| p.join(marker).is_file())
- {
- let rel = e
- .path()
- .strip_prefix(&filter_root)
- .unwrap_or(e.path())
- .to_string_lossy()
- .replace('\\', "/");
- sk.lock().expect("skip list lock").push(SkippedDir {
- rel,
- reason: (*reason).to_string(),
- });
- return false;
- }
- }
- }
- true
- });
- for entry in builder.build() {
- let entry = match entry {
+ .into_iter();
+ while let Some(next) = it.next() {
+ let entry = match next {
Ok(e) => e,
Err(e) => {
eprintln!("walk: skipping unreadable entry: {e}");
continue;
}
};
- if !entry.file_type().is_some_and(|t| t.is_file()) {
+ let is_dir = entry.file_type().is_dir();
+ // Retire the layers of directories this entry is no longer inside; see
+ // `IgnoreStack::truncate_to` for why every entry needs it, not just
+ // directories.
+ stack.truncate_to(entry.depth());
+ // SECURITY / hygiene: never index hidden files or descend into hidden
+ // directories. Without this the walker happily indexed `.env` (secrets,
+ // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a
+ // queryable brain with no per-brain authorization — a real exposure for
+ // the "point it at my project folder" use case. A hidden directory is
+ // pruned before descending, so `.git/` is skipped whole. The root
+ // itself (depth 0) is exempt so a brain over a dot-named folder still
+ // works. `--no-ignore` does NOT turn this off: it is not one of the
+ // ignore rules, it is the reason secrets stay out.
+ if entry.depth() > 0
+ && entry
+ .file_name()
+ .to_str()
+ .is_some_and(|n| n.starts_with('.'))
+ {
+ stack.record_hidden(entry.path(), is_dir);
+ if is_dir {
+ it.skip_current_dir();
+ }
+ continue;
+ }
+ if is_dir {
+ // The root is never judged by the rules below it: pointing
+ // autoindex at a folder is an explicit instruction, and the note
+ // explaining that it *would* have been ignored is on the report.
+ if entry.depth() > 0 && stack.skip_dir(entry.path()).is_some() {
+ it.skip_current_dir();
+ continue;
+ }
++ // Marker-gated generated dirs (part of the built-in defaults, so
++ // `--no-ignore` / `--no-default-ignores` disable it): names like
++ // Unity's `Library/` or `Logs/` are too common for the blunt
++ // DEFAULT_IGNORE_PATTERNS, so they are pruned only when a sibling
++ // marker file proves the parent is that kind of project.
++ if entry.depth() > 0 && marker_defaults {
++ if let Some(label) = marker_generated_dir(entry.path()) {
++ stack.record_marker_dir(entry.path(), label);
++ it.skip_current_dir();
++ continue;
++ }
++ }
+ stack.enter_dir(entry.path(), entry.depth());
+ continue;
+ }
+ if !entry.file_type().is_file() {
+ continue;
+ }
+ if stack.skip_file(entry.path()).is_some() {
continue;
}
let md = match entry.metadata() {
@@@ -268,209 -257,3 +318,96 @@@ mod hidden_skip_tests
);
}
}
+
+#[cfg(test)]
- mod generated_dir_tests {
- use super::walk;
++mod marker_generated_dir_tests {
++ use super::{walk, walk_reporting};
++ use crate::ignore_rules::IgnoreOptions;
+ use std::fs;
+ use std::path::Path;
+
- fn make_unity_root(root: &Path, with_marker: bool) {
++ fn make_unity_project(root: &Path, with_marker: bool) {
+ fs::create_dir_all(root.join("Assets")).unwrap();
+ fs::write(root.join("Assets/Player.cs"), "class Player {}").unwrap();
+ fs::create_dir_all(root.join("Library/Artifacts")).unwrap();
+ fs::write(root.join("Library/Artifacts/blob.bin"), b"\x00\x01").unwrap();
- fs::create_dir_all(root.join("Temp")).unwrap();
- fs::write(root.join("Temp/scratch"), "x").unwrap();
++ fs::create_dir_all(root.join("Logs")).unwrap();
++ fs::write(root.join("Logs/editor.log"), "x").unwrap();
+ if with_marker {
+ fs::create_dir_all(root.join("ProjectSettings")).unwrap();
+ fs::write(
+ root.join("ProjectSettings/ProjectVersion.txt"),
+ "m_EditorVersion: 2022.3.10f1\n",
+ )
+ .unwrap();
+ }
+ }
+
++ /// Monorepo case: the Unity project is a SUBFOLDER of the walk root; the
++ /// sibling marker decides at any depth.
+ #[test]
- fn unity_marker_prunes_generated_dirs_and_records_them() {
++ fn a_nested_unity_project_is_pruned_and_reported() {
+ let dir = tempfile::TempDir::new().unwrap();
- make_unity_root(dir.path(), true);
- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
++ let unity = dir.path().join("platform/unity");
++ fs::create_dir_all(&unity).unwrap();
++ make_unity_project(&unity, true);
++ let (files, report) =
++ walk_reporting(dir.path(), false, IgnoreOptions::default()).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(rels.contains(&"Assets/Player.cs"));
++ assert!(rels.contains(&"platform/unity/Assets/Player.cs"));
+ assert!(
- !rels.iter().any(|r| r.starts_with("Library/")),
++ !rels.iter().any(|r| r.contains("/Library/")),
+ "Library/ must be pruned whole: {rels:?}"
+ );
- assert!(!rels.iter().any(|r| r.starts_with("Temp/")));
- let mut skipped_rels: Vec<&str> = skipped.iter().map(|s| s.rel.as_str()).collect();
- skipped_rels.sort();
- assert_eq!(skipped_rels, ["Library", "Temp"], "each pruned dir is recorded");
++ assert!(
++ report
++ .by_rule
++ .keys()
++ .any(|k| k == "default:unity-generated"),
++ "the prune must be named on the report: {report:?}"
++ );
+ }
+
++ /// Without the marker, a folder merely NAMED Library or Logs is data.
+ #[test]
+ fn without_the_marker_nothing_is_pruned() {
+ let dir = tempfile::TempDir::new().unwrap();
- make_unity_root(dir.path(), false);
- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
++ make_unity_project(dir.path(), false);
++ let files = walk(dir.path(), false).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(
- rels.iter().any(|r| r.starts_with("Library/")),
- "a folder merely NAMED Library is not evidence: {rels:?}"
- );
- assert!(skipped.is_empty());
++ assert!(rels.contains(&"Library/Artifacts/blob.bin"), "{rels:?}");
++ assert!(rels.contains(&"Logs/editor.log"), "{rels:?}");
+ }
+
++ /// The marker rule is one of the built-in defaults, so both switches
++ /// disable it.
+ #[test]
- fn no_default_excludes_walks_everything() {
++ fn no_ignore_and_no_default_ignores_both_disable_the_marker_rule() {
+ let dir = tempfile::TempDir::new().unwrap();
- make_unity_root(dir.path(), true);
- let (files, skipped) = walk(dir.path(), false, false, true).unwrap();
- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(rels.iter().any(|r| r.starts_with("Library/")));
- assert!(skipped.is_empty());
++ make_unity_project(dir.path(), true);
++ let no_defaults = IgnoreOptions {
++ defaults: false,
++ ..IgnoreOptions::default()
++ };
++ for opts in [IgnoreOptions::off(), no_defaults] {
++ let (files, _) = walk_reporting(dir.path(), false, opts).unwrap();
++ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
++ assert!(
++ rels.contains(&"Library/Artifacts/blob.bin"),
++ "opts {opts:?} must walk generated dirs: {rels:?}"
++ );
++ }
+ }
+
+ #[test]
+ fn a_file_named_like_a_generated_dir_is_not_pruned() {
+ let dir = tempfile::TempDir::new().unwrap();
- make_unity_root(dir.path(), true);
++ make_unity_project(dir.path(), true);
+ fs::write(dir.path().join("obj"), "a FILE named obj").unwrap();
- let (files, _) = walk(dir.path(), false, true, true).unwrap();
++ let files = walk(dir.path(), false).unwrap();
+ assert!(
+ files.iter().any(|e| e.rel == "obj"),
+ "the prune applies to directories only"
+ );
+ }
-
- /// Monorepo case: the Unity project is a SUBFOLDER of the walk root. The
- /// marker is checked against each candidate's parent, so nesting depth is
- /// irrelevant.
- #[test]
- fn a_nested_unity_project_is_pruned_by_its_sibling_marker() {
- let dir = tempfile::TempDir::new().unwrap();
- let unity = dir.path().join("apps/game");
- fs::create_dir_all(&unity).unwrap();
- make_unity_root(&unity, true);
- fs::write(dir.path().join("README.md"), "monorepo").unwrap();
- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(rels.contains(&"apps/game/Assets/Player.cs"));
- assert!(!rels.iter().any(|r| r.contains("game/Library/")));
- assert!(skipped.iter().any(|s| s.rel == "apps/game/Library"));
- }
-
- #[test]
- fn node_modules_is_pruned_only_next_to_a_package_json() {
- let dir = tempfile::TempDir::new().unwrap();
- let app = dir.path().join("web");
- fs::create_dir_all(app.join("node_modules/lodash")).unwrap();
- fs::write(app.join("node_modules/lodash/index.js"), "x").unwrap();
- fs::write(app.join("package.json"), "{}").unwrap();
- fs::write(app.join("index.js"), "x").unwrap();
- let bare = dir.path().join("data/node_modules");
- fs::create_dir_all(&bare).unwrap();
- fs::write(bare.join("notes.txt"), "not an npm tree").unwrap();
- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(rels.contains(&"web/index.js"));
- assert!(!rels.iter().any(|r| r.starts_with("web/node_modules")));
- assert!(
- rels.contains(&"data/node_modules/notes.txt"),
- "without the package.json marker there is no evidence: {rels:?}"
- );
- assert_eq!(skipped.len(), 1);
- assert_eq!(skipped[0].rel, "web/node_modules");
- }
-
- #[test]
- fn gitignored_files_are_skipped_only_inside_a_git_repo() {
- let dir = tempfile::TempDir::new().unwrap();
- let root = dir.path();
- fs::create_dir(root.join(".git")).unwrap();
- fs::write(root.join(".gitignore"), "build/\n*.log\n").unwrap();
- fs::create_dir_all(root.join("build/intermediates")).unwrap();
- fs::write(root.join("build/intermediates/R.txt"), "generated").unwrap();
- fs::write(root.join("debug.log"), "noise").unwrap();
- fs::write(root.join("main.rs"), "fn main() {}").unwrap();
- fs::create_dir(root.join("src")).unwrap();
- fs::write(root.join("src/app.log"), "noise").unwrap();
- fs::write(root.join("src/lib.rs"), "pub fn f() {}").unwrap();
-
- let (files, _) = walk(root, false, true, true).unwrap();
- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(rels.contains(&"main.rs"));
- assert!(rels.contains(&"src/lib.rs"));
- assert!(
- !rels.iter().any(|r| r.starts_with("build/")),
- "gitignored dir must be pruned: {rels:?}"
- );
- assert!(
- !rels.iter().any(|r| r.ends_with(".log")),
- "gitignored glob applies at every depth: {rels:?}"
- );
-
- // --no-gitignore restores the full walk.
- let (files, _) = walk(root, false, true, false).unwrap();
- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(rels.contains(&"build/intermediates/R.txt"));
- assert!(rels.contains(&"debug.log"));
- }
-
- /// A `.gitignore` OUTSIDE a git repository is not an owner declaration —
- /// require_git keeps the rule evidence-based.
- #[test]
- fn a_gitignore_without_a_git_repo_is_inert() {
- let dir = tempfile::TempDir::new().unwrap();
- let root = dir.path();
- fs::write(root.join(".gitignore"), "*.log\n").unwrap();
- fs::write(root.join("debug.log"), "kept").unwrap();
- let (files, _) = walk(root, false, true, true).unwrap();
- assert!(
- files.iter().any(|e| e.rel == "debug.log"),
- "no .git dir, so gitignore semantics must not apply"
- );
- }
-
- /// Nested .gitignore files scope to their own subtree, like git itself.
- #[test]
- fn nested_gitignore_files_scope_to_their_subtree() {
- let dir = tempfile::TempDir::new().unwrap();
- let root = dir.path();
- fs::create_dir(root.join(".git")).unwrap();
- fs::create_dir(root.join("web")).unwrap();
- fs::write(root.join("web/.gitignore"), "dist/\n").unwrap();
- fs::create_dir_all(root.join("web/dist")).unwrap();
- fs::write(root.join("web/dist/bundle.js"), "x").unwrap();
- fs::create_dir_all(root.join("docs/dist")).unwrap();
- fs::write(root.join("docs/dist/manual.md"), "kept").unwrap();
- let (files, _) = walk(root, false, true, true).unwrap();
- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(!rels.iter().any(|r| r.starts_with("web/dist/")));
- assert!(rels.contains(&"docs/dist/manual.md"));
- }
-
- #[test]
- fn cargo_target_is_pruned_only_next_to_a_cargo_toml() {
- let dir = tempfile::TempDir::new().unwrap();
- let crate_dir = dir.path().join("mycrate");
- fs::create_dir_all(crate_dir.join("target/debug")).unwrap();
- fs::write(crate_dir.join("target/debug/build.log"), "x").unwrap();
- fs::write(crate_dir.join("Cargo.toml"), "[package]").unwrap();
- fs::write(crate_dir.join("main.rs"), "fn main() {}").unwrap();
- let plain = dir.path().join("shooting/target");
- fs::create_dir_all(&plain).unwrap();
- fs::write(plain.join("scores.csv"), "a,b\n1,2\n").unwrap();
- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
- assert!(rels.contains(&"mycrate/main.rs"));
- assert!(!rels.iter().any(|r| r.starts_with("mycrate/target")));
- assert!(
- rels.contains(&"shooting/target/scores.csv"),
- "a folder merely NAMED target is data, not a build dir: {rels:?}"
- );
- assert_eq!(skipped.len(), 1);
- assert_eq!(skipped[0].rel, "mycrate/target");
- }
+}
diff --cc engine/crates/xerj-server/src/brain.rs
index 4951aa6ba,cddd4c61e..59d0aa779
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@@ -432,9 -515,10 +515,11 @@@ fn index_cfg(cfg: &BrainCfg, brain: &st
state_dir: None,
fresh: cfg.fresh,
follow_symlinks: false,
- no_default_excludes: false,
+ stub_globs: Vec::new(),
- no_gitignore: false,
+ // `xerj brain` composes autoindex, so it inherits its ignore rules
+ // rather than inventing its own: build output does not belong in a
+ // second brain either (#276).
+ ignore: xerj_autoindex::ignore_rules::IgnoreOptions::default(),
max_file_gb: 2,
sample: 500,
no_semantic: false,