AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
merge: upstream/main (#279 ignore rules, #202, #203, rc.14 pipeline rework)
Sergey Gonchar384945158a6a
diff --cc engine/crates/xerj-autoindex/src/cli.rsindex aed7476c9,ce648eff1..84eb4c0da--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@@ -16,14 -31,9 +31,12 @@@ pub struct IndexCfgpub state_dir: Option<PathBuf>,pub fresh: bool,pub follow_symlinks: bool,- /// Disable the marker-gated generated-dir prune (e.g. Unity `Library/`).- pub no_default_excludes: bool,+ /// `--stub <glob>` patterns: matching files are indexed as one+ /// existence-only name card, contents never opened.+ pub stub_globs: Vec<String>,- /// Walk `.gitignore`d files too (by default they are skipped when the- /// root is inside a git repository).- pub no_gitignore: bool,+ /// `.gitignore` / `.xerjignore` / built-in build-output rules (#276).+ /// `deep_count` is set from `--dry-run`.+ pub ignore: IgnoreOptions,pub max_file_gb: u64,pub sample: usize,pub no_semantic: bool,@@@ -82,27 -114,16 +117,22 @@@ pub fn print_help()valid range 1..=3600)\n\--prefix <P> index prefix (default ax)\n\--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\- --fresh ignore existing journal, restart (ids stay idempotent)\n\+ --snapshot-max-gb <N> logical payload cap for sealed source+prepared records\n\+ bytes (default 64); excludes filesystem/manifest overhead\n\+ --fresh {fresh_help}\n\--follow-symlinks follow symlinks (loop-safe); off by default\n\- --no-default-excludes\n\- walk generated/cache dirs too. By default,\n\- well-known generated dirs are skipped (and\n\- recorded) when a sibling marker proves them, at\n\- any depth: Unity Library/Temp/obj/Logs/\n\- UserSettings next to ProjectSettings/\n\- ProjectVersion.txt, node_modules next to\n\- package.json, target next to Cargo.toml\n\- --no-gitignore index .gitignore'd files too. By default, when the\n\- root is inside a git repository, files matched by\n\- the tree's .gitignore rules are skipped (the\n\- owner's own declaration of generated noise);\n\- the user's global gitignore is never consulted\n\+ --stub <GLOB> index matching files as ONE existence-only name\n\+ card (title + provenance); contents are never\n\+ opened. Repeatable. A pattern without '/'\n\+ matches file names anywhere ('*.bvh'); with '/'\n\+ it matches the root-relative path\n\+ ('unity/**/*.csv'). '**' crosses directories\n\+ --no-ignore index everything: no .gitignore, no .xerjignore, no\n\+ .git/info/exclude, no built-in defaults. Hidden files\n\+ (.env, .git/, .ssh) stay skipped either way — that is\n\+ not an ignore rule, it is what keeps secrets out.\n\+ --no-default-ignores keep the ignore files, drop only the built-in list\n\+ ({defaults})\n\--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\--sample <N> records sampled per file for inference (default 500)\n\--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\@@@ -164,9 -275,8 +284,9 @@@ pub fn parse(args: Vec<String>) -> Resulet mut state_dir: Option<PathBuf> = None;let mut fresh = false;let mut follow_symlinks = false;- let mut no_default_excludes = false;+ let mut stub_globs: Vec<String> = Vec::new();- let mut no_gitignore = false;+ let mut no_ignore = false;+ let mut no_default_ignores = false;let mut max_file_gb = 2u64;let mut sample = 500usize;let mut no_semantic = false;@@@ -225,9 -364,8 +374,9 @@@"--state-dir" => state_dir = it.next().map(PathBuf::from),"--fresh" => fresh = true,"--follow-symlinks" => follow_symlinks = true,- "--no-default-excludes" => no_default_excludes = true,+ "--stub" => stub_globs.push(it.next().ok_or("--stub needs a glob pattern")?),- "--no-gitignore" => no_gitignore = true,+ "--no-ignore" => no_ignore = true,+ "--no-default-ignores" => no_default_ignores = true,"--max-file-gb" => {max_file_gb = it.next()@@@ -290,31 -492,43 +503,44 @@@prefix,state_dir,})),- (None, Some(root)) => Ok(Cmd::Index(IndexCfg {- root,- url,- api_key,- workers: workers.max(1),- pdf_workers,- pdf_timeout_secs,- bulk_mb: bulk_mb.clamp(1, 24),- bulk_timeout_secs,- prefix,- state_dir,- fresh,- follow_symlinks,- no_default_excludes,- stub_globs,- no_gitignore,- max_file_gb,- sample: sample.max(50),- no_semantic,- brain,- no_graph,- dry_run,- json,- quiet,- })),+ (None, Some(root)) => {+ let plan = crate::resources::plan(workers, pdf_workers, bulk_mb);+ Ok(Cmd::Index(IndexCfg {+ root,+ url,+ api_key,+ workers: plan.index_workers,+ scan_workers: plan.scan_threads,+ pdf_workers: plan.pdf_workers,+ resource_notes: plan.notes,+ pdf_timeout_secs,+ bulk_mb,+ bulk_timeout_secs,+ snapshot_max_bytes,+ prefix,+ state_dir,+ fresh,+ follow_symlinks,++ stub_globs,+ ignore: IgnoreOptions {+ enabled: !no_ignore,+ defaults: !no_ignore && !no_default_ignores,+ // Only --dry-run pays for counting what is inside a pruned+ // directory; a real run's whole point is not touching it.+ deep_count: dry_run,+ },+ max_file_gb,+ sample: sample.max(50),+ no_semantic,+ brain,+ no_graph,+ dry_run,+ json,+ quiet,+ progress,+ progress_interval,+ }))+ }_ => Ok(Cmd::Help),}}diff --cc engine/crates/xerj-autoindex/src/detect/e2e.rsindex 15684ef61,af2848899..d4fd6d667--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs@@@ -270,9 -268,7 +273,8 @@@ fn cfg(root: &Path, state_dir: &Path, ustate_dir: Some(state_dir.to_owned()),fresh: false,follow_symlinks: false,- no_default_excludes: false,+ stub_globs: Vec::new(),- no_gitignore: false,+ ignore: crate::ignore_rules::IgnoreOptions::default(),max_file_gb: 1,sample: 50,no_semantic: true,diff --cc engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 84481d716,4a3df54dc..0962ed7c1--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@@ -46,14 -133,13 +133,18 @@@ impl MockEndpointlet server_state = Arc::clone(&state);let join = thread::spawn(move || loop {match listener.accept() {- Ok((stream, _)) => handle(stream, &server_state),+ Ok((stream, _)) => {+ // BSD/macOS: accepted sockets inherit the listener's+ // O_NONBLOCK; the handler does blocking reads.+ stream.set_nonblocking(false).unwrap();+ handle(stream, &server_state)+ }Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {- if server_state.lock().unwrap().stop {+ if server_state+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .stop+ {break;}thread::yield_now();@@@ -204,9 -515,7 +520,8 @@@ fn cfg(root: &Path, state_dir: &Path, ustate_dir: Some(state_dir.to_owned()),fresh: false,follow_symlinks: false,- no_default_excludes: false,+ stub_globs: Vec::new(),- no_gitignore: false,+ ignore: crate::ignore_rules::IgnoreOptions::default(),max_file_gb: 1,sample: 50,no_semantic: true,@@@ -623,266 -1750,1262 +1756,1265 @@@ fn file_done_append_and_fsync_failures_assert_eq!(run_index(config.clone()).unwrap(), 0);fs::write(&path, "id,value\n0,new\n1,new\n").unwrap();- state::fail_next_file_done_io(io_boundary);- let error = run_index(config.clone()).unwrap_err();- let message = format!("{error:#}");- assert!(- message.contains("durably commit completed source"),- "{message}"- );- assert!(- message.contains(if io_boundary == 1 {- "append failure"- } else if io_boundary == 3 {- "partial file_done write"- } else {- "fsync failure"- }),- "{message}"- );- let replay = state::Journal::open(- state_dir.path(),- &config.root.to_string_lossy(),- &config.url,- &config.prefix,- config.bulk_timeout_secs,- false,- )- .unwrap();- assert!(replay.done.is_empty());- assert_eq!(replay.pending_replacements.len(), 1);- drop(replay);+ state::fail_next_file_done_io(io_boundary, &state_dir.path().join("journal.ndjson"));+ let error = run_index(config.clone()).unwrap_err();+ let message = format!("{error:#}");+ assert!(+ message.contains("durably commit completed source"),+ "{message}"+ );+ assert!(+ message.contains(if io_boundary == 1 {+ "append failure"+ } else if io_boundary == 3 {+ "partial file_done write"+ } else {+ "fsync failure"+ }),+ "{message}"+ );+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(replay.done.is_empty());+ assert_eq!(replay.pending_replacements.len(), 1);+ drop(replay);++ assert_eq!(run_index(config).unwrap(), 0);+ let locked = endpoint.state.lock().unwrap();+ assert_eq!(+ locked+ .docs+ .values()+ .filter(|doc| doc.get("value").is_some())+ .count(),+ 2+ );+ assert!(locked+ .docs+ .values()+ .filter(|doc| doc.get("value").is_some())+ .all(|doc| doc["value"].as_str() == Some("new")));+ }+ }++ #[test]+ fn pending_generation_b_is_superseded_when_source_changes_to_c() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ // run_index reaches Journal::file_done; the file_done IO failpoint is+ // a global one-shot, so every file_done-reaching test must hold its+ // lock (armer or not) or it can steal an armed injection. Acquired+ // after FAILPOINT_TEST_LOCK everywhere — one consistent order.+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let path = corpus.path().join("records.csv");+ fs::write(&path, "id,value\n0,generation-a\n1,generation-a\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(&path, "id,value\n0,generation-b\n1,generation-b\n").unwrap();+ REPLACEMENT_FAILPOINT.store(2, Ordering::SeqCst);+ run_index(config.clone()).unwrap_err();+ let replay_b = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ let key = replay_b.pending_replacements.keys().next().unwrap().clone();+ let generation_b = replay_b.pending_replacements[&key].clone();+ drop(replay_b);++ fs::write(&path, "id,value\n0,generation-c\n1,generation-c\n").unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ let replay_c = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(replay_c.pending_replacements.is_empty());+ let committed_generation = replay_c.done[&key].generation.as_deref().unwrap();+ assert_ne!(committed_generation, generation_b);+ let events: Vec<Value> = fs::read_to_string(state_dir.path().join("journal.ndjson"))+ .unwrap()+ .lines()+ .map(|line| serde_json::from_str(line).unwrap())+ .collect();+ let starts: Vec<&str> = events+ .iter()+ .filter(|event| {+ event["kind"] == "file_replace_start"+ && event["file_key"].as_str() == Some(key.as_str())+ })+ .filter_map(|event| event["generation"].as_str())+ .collect();+ assert_eq!(starts[starts.len() - 2], generation_b);+ assert_eq!(starts.last().copied(), Some(committed_generation));+ drop(replay_c);+ let locked = endpoint.state.lock().unwrap();+ assert_eq!(+ locked+ .docs+ .values()+ .filter(|doc| doc.get("value").is_some())+ .count(),+ 2+ );+ assert!(locked+ .docs+ .values()+ .filter(|doc| doc.get("value").is_some())+ .all(|doc| doc["value"].as_str() == Some("generation-c")));+ }++ #[test]+ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ // run_index reaches Journal::file_done; the file_done IO failpoint is+ // a global one-shot, so every file_done-reaching test must hold its+ // lock (armer or not) or it can steal an armed injection. Acquired+ // after FAILPOINT_TEST_LOCK everywhere — one consistent order.+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let original = "id,value\n0,original\n1,original\n";+ fs::write(corpus.path().join("a.csv"), original).unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ // a.csv keeps its planned key by path while its OLD bytes reappear as+ // b.csv — whose content key is exactly the planned key a.csv claims. Both+ // files owning one ax_file key would let each replacement delete the+ // other's freshly published documents.+ fs::write(+ corpus.path().join("a.csv"),+ "id,value\n0,rewritten\n1,rewritten\n",+ )+ .unwrap();+ fs::write(corpus.path().join("b.csv"), original).unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 3);+ {+ let locked = endpoint.state.lock().unwrap();+ // Graph detection is enabled in this module, so a.csv also carries a+ // file-level node document (`ax_locator: "file"`, no row fields). The+ // replacement transaction's subject is the row documents.+ let rows: Vec<&Value> = data_rows(&locked);+ assert_eq!(rows.len(), 2);+ assert!(rows.iter().all(|doc| {+ doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")+ }));+ assert!(locked+ .docs+ .values()+ .all(|doc| doc["ax_path"].as_str() == Some("a.csv")));+ }+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(replay.pending_replacements.is_empty());+ assert_eq!(replay.done.len(), 1);+ let plan = replay.plan.clone().unwrap();+ assert_eq!(plan.junk_files.len(), 1);+ assert_eq!(plan.junk_files[0].rel, "b.csv");+ assert!(plan.junk_files[0]+ .reason+ .contains("key ownership exclusive"));+ drop(replay);++ // The divergence is durable and deterministic: an identical rerun keeps+ // the same owner, appends no new plan, and changes no documents.+ let plans_before = event_count(state_dir.path(), "plan");+ assert_eq!(run_index(config).unwrap(), 3);+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);+ let locked = endpoint.state.lock().unwrap();+ let rows = data_rows(&locked);+ assert_eq!(rows.len(), 2);+ assert!(rows+ .iter()+ .all(|doc| doc["value"].as_str() == Some("rewritten")));+ }++ /// #238: a file added after the plan was frozen is skipped and reported in the+ /// catalog. That report used to be immortal — `new_unplanned` was a per-run+ /// `Vec`, so once the file left the corpus nothing remembered the document had+ /// ever been written and no run could remove it. The catalog is the data map+ /// every `map`/`status`/agent query reads; it must not keep advertising a file+ /// that is gone.++ #[test]+ fn an_added_then_deleted_file_leaves_no_immortal_catalog_entry() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("planned.csv"),+ "id,value\n0,planned\n1,planned\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ // Appears after the plan froze: not indexed, but reported as skipped.+ fs::write(+ corpus.path().join("added.csv"),+ "id,value\n2,added\n3,added\n",+ )+ .unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 3);+ let skipped_id = {+ let locked = endpoint.state.lock().unwrap();+ let (id, doc) = locked+ .catalog_docs+ .iter()+ .find(|(_, doc)| doc["path"].as_str() == Some("added.csv"))+ .expect("a file skipped for being post-freeze is reported in the catalog");+ assert_eq!(doc["status"].as_str(), Some("skipped"));+ assert_eq!(doc["doc_kind"].as_str(), Some("file"));+ id.clone()+ };++ // The durable plan is the only record that the document exists. Without+ // it no later run can ever delete it.+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(+ replay+ .plan+ .as_ref()+ .unwrap()+ .junk_files+ .iter()+ .any(|junk| junk.rel == "added.csv"),+ "the skipped file must be recorded in the durable plan"+ );+ drop(replay);++ // Delete it: the catalog entry goes with it, and the run is clean again.+ fs::remove_file(corpus.path().join("added.csv")).unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ {+ let locked = endpoint.state.lock().unwrap();+ assert!(+ !locked.catalog_docs.contains_key(&skipped_id),+ "the catalog entry for a deleted skipped file must not survive"+ );+ assert!(+ locked+ .catalog_docs+ .values()+ .all(|doc| doc["path"].as_str() != Some("added.csv")),+ "no catalog document may still name the deleted file"+ );+ // The file that is still there keeps its entry.+ assert!(locked+ .catalog_docs+ .values()+ .any(|doc| doc["path"].as_str() == Some("planned.csv")));+ }++ // Converged: nothing left to add or sweep, so no further plan is appended.+ let plans_before = event_count(state_dir.path(), "plan");+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);+ }++ #[test]+ fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ // run_index reaches Journal::file_done; the file_done IO failpoint is+ // a global one-shot, so every file_done-reaching test must hold its+ // lock (armer or not) or it can steal an armed injection. Acquired+ // after FAILPOINT_TEST_LOCK everywhere — one consistent order.+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("dup-a.csv"), "id,value\n0,dup\n1,dup\n").unwrap();+ fs::write(corpus.path().join("dup-b.csv"), "id,value\n0,dup\n1,dup\n").unwrap();+ fs::write(corpus.path().join("keep.csv"), "id,value\n2,keep\n3,keep\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ let starts = event_count(state_dir.path(), "file_replace_start");++ // The whole duplicate group disappears: its documents stay live with no+ // source file, so the rerun is refused. The original invariant still+ // holds and is still checked — a key with no current file must never get+ // a replacement intent journaled, which would be re-appended forever+ // without ever committing.+ fs::remove_file(corpus.path().join("dup-a.csv")).unwrap();+ fs::remove_file(corpus.path().join("dup-b.csv")).unwrap();+ for _ in 0..2 {+ assert_unsupported_delta_without_remote_mutation(+ &endpoint,+ config.clone(),+ &[],+ &["dup-a.csv"],+ );+ assert_eq!(+ event_count(state_dir.path(), "file_replace_start"),+ starts,+ "an orphaned duplicate-group key must not schedule a replacement"+ );+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(replay.pending_replacements.is_empty());+ }+ }++ #[test]+ fn partial_file_done_is_rolled_back_before_another_worker_commits() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n0,a-old\n1,a-old\n").unwrap();+ fs::write(corpus.path().join("b.csv"), "id,value\n2,b-old\n3,b-old\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ config.workers = 2;+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(corpus.path().join("a.csv"), "id,value\n0,a-new\n1,a-new\n").unwrap();+ fs::write(corpus.path().join("b.csv"), "id,value\n2,b-new\n3,b-new\n").unwrap();+ state::fail_next_file_done_io(3, &state_dir.path().join("journal.ndjson"));+ let error = run_index(config.clone()).unwrap_err();+ assert!(format!("{error:#}").contains("partial file_done write"));++ // One worker hit the partial append and rolled back while the other was+ // allowed to commit. Replay must remain parseable and expose exactly one+ // pending generation rather than malformed middle corruption.+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert_eq!(replay.pending_replacements.len(), 1);+ assert_eq!(replay.done.len(), 1);+ drop(replay);++ assert_eq!(run_index(config).unwrap(), 0);+ let locked = endpoint.state.lock().unwrap();+ assert_eq!(+ locked+ .docs+ .values()+ .filter(|doc| doc.get("value").is_some())+ .count(),+ 4+ );+ assert!(locked+ .docs+ .values()+ .filter(|doc| doc.get("value").is_some())+ .all(|doc| {+ doc["value"]+ .as_str()+ .is_some_and(|value| value.ends_with("-new"))+ }));+ }++ /// #195: an index write block rejects every bulk item with status 403 (the+ /// status ES assigns explicit/API blocks — only the flood-stage+ /// `read_only_allow_delete` block is 429). A status-based classifier filed+ /// those items under "junk source records", journaled the file COMPLETE, and+ /// the instructed rerun then resumed past the journal and reported success+ /// over an empty index. Blocks must be classified by error type: fatal,+ /// file left pending, and the rerun after the block lifts must really index.+ #[test]+ fn write_block_rejections_are_backend_fatal_and_the_file_stays_pending() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("records.csv"),+ "id,value\n0,blocked\n1,blocked\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);++ endpoint.state.lock().unwrap().block_writes = true;+ let error = format!("{:#}", run_index(config.clone()).unwrap_err());+ assert!(error.contains("blocked"), "{error}");+ assert_eq!(+ file_done_count(state_dir.path()),+ 0,+ "a write-blocked source file must stay pending in the journal"+ );+ assert!(endpoint.state.lock().unwrap().docs.is_empty());++ // Block lifted: the SAME command must resume and actually index.+ endpoint.state.lock().unwrap().block_writes = false;+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(file_done_count(state_dir.path()), 1);+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 2);+ }++ /// Pins WHY the run document carries no backend-rejection count, so nobody+ /// "restores" one that could only ever read 0.+ ///+ /// Making `junk_records_total` durable narrowed it: it no longer folds in the+ /// per-item rejections that `record_bulk_outcome` counts. That looks like a+ /// lost signal, and it would be one — except that a single per-item rejection+ /// also lands in `bulk_errors`, which aborts the run before the run document,+ /// the map, the summary line and the exit code exist at all. So the narrowed+ /// number was unreachable in any published map either way.+ ///+ /// The rejection count is therefore reported in the one place a rejected run+ /// ever reaches: the abort message, where it supplies the scale that+ /// `bulk_errors`' first-five sample cannot.+ #[test]+ fn backend_rejected_records_abort_the_run_before_any_map_is_written() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("records.csv"),+ "id,value\n0,kept\n1,kept\n2,kept\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ endpoint.state.lock().unwrap().reject_first_data_item = true;+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);++ // A per-item 400 is NOT a server error (not 429, not 5xx, not a block), so+ // it is the weakest rejection the classifier recognises — and even it is+ // fatal.+ let error = format!("{:#}", run_index_report(config).unwrap_err());+ assert!(+ error.contains("document_parsing_exception"),+ "the refusal itself must be quoted: {error}"+ );+ assert!(+ error.contains("The backend refused 1 record(s)."),+ "the abort must state how many records were refused, not just sample the errors: {error}"+ );++ // Measured, and deliberately pinned even though it is NOT what the abort+ // message implies: `record_bulk_outcome` treats a per-item rejection as+ // non-fatal to the worker (only `server_errors` set `send_err`), so the+ // file is journaled COMPLETE and the run aborts only at the end. A rerun+ // therefore resumes past this file and never retries the refused record,+ // while the message says failed sources "were not journaled complete".+ //+ // That divergence predates this branch — it is `main`'s behaviour for the+ // item-error class — and fixing it means changing when a worker treats a+ // rejection as fatal, which is a resume-semantics change with nothing to+ // do with map metadata. Recorded here so the next reader finds the fact+ // rather than the assumption; tracked in the PR body as a known gap.+ assert_eq!(+ file_done_count(state_dir.path()),+ 1,+ "pre-existing: a per-item rejection still journals the source complete"+ );+ }++ /// #195 last-resort gate: a backend that ACCEPTS every bulk but persists+ /// nothing (the shape of any rejection path the per-item classifier does+ /// not recognise) must not yield a success exit. The journal says records+ /// landed; the live count says zero; the run must fail and say how to+ /// recover.+ #[test]+ fn zero_live_documents_after_journaled_records_fails_the_run() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("records.csv"),+ "id,value\n0,ghost\n1,ghost\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ endpoint.state.lock().unwrap().swallow_data_bulks = true;+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);++ let error = format!("{:#}", run_index(config).unwrap_err());+ assert!(error.contains("verification failed"), "{error}");+ assert!(error.contains("0 documents are live"), "{error}");+ assert!(error.contains("--fresh"), "{error}");+ }++ /// #241 regression: a run must never be silent.+ ///+ /// Before this fix `autoindex` printed a handful of phase banners, one line+ /// per 200 completed files, and then nothing at all through the entire+ /// `finalize` block — measured at 47-64% of wall in the issue and at 45% (100+ /// of 222 s) on the corpus used to reproduce it here. An agent watching the+ /// stream could not tell a running index from a hung one.+ ///+ /// This asserts the three properties that make the stream readable, through+ /// the REAL progress surface a production run uses:+ /// 1. phase B reports as it goes, with a percent and an ETA field;+ /// 2. the previously-silent finalize block reports at all;+ /// 3. every run on a surface that prints at all ends with a terminal line+ /// stating the outcome in words, because exit 3 is a success an agent+ /// otherwise reads as failure. (`--progress none` prints nothing —+ /// asserted by `quiet_runs_emit_no_progress_stream_at_all` below.)+ #[test]+ fn a_run_reports_progress_through_every_phase_and_closes_the_stream() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ // Distinct bytes per file: byte-identical files collapse into one+ // canonical file plus aliases, which would make `files=8` a lie.+ for n in 0..8 {+ fs::write(+ corpus.path().join(format!("rows-{n}.csv")),+ format!("id,value\n{n}0,alpha\n{n}1,beta\n{n}2,gamma\n"),+ )+ .unwrap();+ }++ let endpoint = MockEndpoint::start(usize::MAX);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ config.quiet = false;+ config.progress = crate::progress::ProgressMode::Plain;+ config.progress_interval = Some(std::time::Duration::from_secs(1));++ let buffer = Arc::new(Mutex::new(Vec::new()));+ let (code, _report) = {+ let _sink = crate::progress::install_test_sink(&buffer);+ run_index_report(config).unwrap()+ };+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();++ let progress_lines: Vec<&str> = stream+ .lines()+ .filter(|line| line.starts_with("xerj-progress "))+ .collect();+ assert!(+ progress_lines.len() >= 5,+ "a run must narrate its phases, got {} line(s):\n{stream}",+ progress_lines.len()+ );+ assert!(+ progress_lines+ .iter()+ .any(|line| line.contains("phase=index")),+ "phase B must report:\n{stream}"+ );+ assert!(+ progress_lines+ .iter()+ .any(|line| line.contains("phase=finalize")),+ "the finalize block was the longest silence in the tool; it must report now:\n{stream}"+ );+ assert!(+ progress_lines+ .iter()+ .all(|line| line.contains(" pct=") && line.contains(" eta_s=")),+ "every progress line carries a percent and an ETA field (possibly `unknown`):\n{stream}"+ );++ // ...and the same run hands the agent something it can show a person.+ // Asserted on the REAL pipeline rather than on a hand-built snapshot.+ //+ // This corpus indexes in ~0.3 s, so it is also the end-to-end proof of the+ // burst floor: every one of its phase changes falls inside `BAR_MIN_GAP`,+ // and a run that short must therefore relay ONE display line, not one per+ // phase. The machine lines above already proved every phase is on the+ // stream — the relay is deliberately quieter than the record.+ let bars: Vec<&str> = stream+ .lines()+ .filter(|line| line.starts_with("xerj-bar "))+ .collect();+ assert_eq!(+ bars.len(),+ 1,+ "a sub-2s run relays exactly one bar, whatever its phase count:\n{stream}"+ );+ assert!(+ bars[0].starts_with("xerj-bar [") && bars[0].contains(']'),+ "the display line carries a drawn bar:\n{stream}"+ );+ // The display line is derived, never a second source of truth: it may not+ // claim completion while the machine line still reports work outstanding.+ assert!(+ !bars.iter().any(|line| line.contains("[####")+ && line.contains("----]")+ && line.contains("100.0%")),+ "a partly-drawn bar cannot be labelled 100%:\n{stream}"+ );++ let done = stream+ .lines()+ .find(|line| line.starts_with("xerj-done "))+ .unwrap_or_else(|| panic!("a run must close its own stream:\n{stream}"));+ assert!(+ done.contains("ok=true") && done.contains(&format!("exit={code}")),+ "{done}"+ );+ assert!(done.contains("reason=completed"), "{done}");+ assert!(done.contains("files=8"), "{done}");+ }++ /// The other half of the contract: `--quiet` still means silence, so nothing+ /// here can spam a caller that asked for none.+ #[test]+ fn quiet_runs_emit_no_progress_stream_at_all() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("rows.csv"), "id,value\n0,alpha\n").unwrap();++ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(config.progress, crate::progress::ProgressMode::None);++ let buffer = Arc::new(Mutex::new(Vec::new()));+ {+ let _sink = crate::progress::install_test_sink(&buffer);+ run_index_report(config).unwrap();+ }+ assert!(+ buffer.lock().unwrap().is_empty(),+ "--quiet asked for nothing: {}",+ String::from_utf8_lossy(&buffer.lock().unwrap())+ );+ }++ /// Where #240 (the resource policy) and #241 (the progress stream) meet.+ ///+ /// Two rules have to hold at once, and each is easy to break while honouring+ /// the other:+ ///+ /// * The resource decision is **announced on the progress surface**, not on a+ /// bare `eprintln!` gated on `!quiet`. A raw write would survive+ /// `--progress none` — which promises silence — and would inject a+ /// non-JSON line into the middle of `--progress json`, which promises one+ /// parseable stream. (`quiet_runs_emit_no_progress_stream_at_all` above is+ /// the other side of this assertion: it fails if any of these lines escape+ /// the surface.)+ /// * The numbers announced are the ones the run **got**, not the ones it+ /// asked for. `pool::configure` is first-call-wins, because rayon pools+ /// cannot be resized, so a process that already indexed once — `xerj+ /// brain` does exactly that — keeps its original phase-A width. Reporting+ /// the request there would make the stream and the run summary a record of+ /// an intention rather than of a run.+ ///+ /// The second rule is made observable by asking for a width this process+ /// provably cannot grant: the pool is built before the run, then one more+ /// thread than it has is requested.+ #[test]+ fn the_resource_plan_is_announced_on_the_progress_surface_with_the_width_it_got() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("rows.csv"),+ "id,value\n0,alpha\n1,beta\n",+ )+ .unwrap();++ // Build the process-global scan pool first, so the width below is one this+ // run cannot possibly be granted.+ let installed = crate::pool::scan_pool().current_num_threads();+ let endpoint = MockEndpoint::start(usize::MAX);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ config.quiet = false;+ config.progress = crate::progress::ProgressMode::Plain;+ config.progress_interval = Some(std::time::Duration::from_secs(1));+ config.scan_workers = installed + 1;+ config.workers = 3;+ config.resource_notes = vec!["memory safe zone 256 MiB allows 3 index workers".to_string()];++ let buffer = Arc::new(Mutex::new(Vec::new()));+ let (_code, report) = {+ let _sink = crate::progress::install_test_sink(&buffer);+ run_index_report(config).unwrap()+ };+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();++ assert!(+ stream.contains(&format!("{installed} scan threads")),+ "the run must announce the phase-A width it actually got, not {}: {stream}",+ installed + 1+ );+ assert!(+ !stream.contains(&format!("{} scan threads", installed + 1)),+ "a width the pool refused must never be reported as fact: {stream}"+ );+ assert!(+ stream.contains("3 index workers"),+ "phase B must name the count the policy chose: {stream}"+ );+ assert!(+ stream.contains("memory safe zone 256 MiB allows 3 index workers"),+ "what the machine forced on the run belongs in the same stream: {stream}"+ );+ assert!(+ stream.contains(&format!("with {installed} threads")),+ "the phase-A banner must carry the real width too: {stream}"+ );++ // The summary is read after the fact by agents and by `xerj brain`; it has+ // to agree with the stream rather than repeat the request.+ let report = report.expect("a completed run produces a summary");+ assert_eq!(report["scan_workers"], serde_json::json!(installed));+ assert_eq!(report["workers"], serde_json::json!(3));+ }++ #[cfg(unix)]+ #[test]+ fn fresh_pdf_is_parsed_once_and_failed_publication_retry_reparses_once() {+ use std::os::unix::fs::PermissionsExt;++ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ inject_pdf_spool_capacity(state_dir.path());+ let tools = tempfile::tempdir().unwrap();+ let pdf = corpus.path().join("quarterly-report.pdf");+ fs::write(+ &pdf,+ b"%PDF-1.4\nfake bytes consumed by the isolated test worker\n",+ )+ .unwrap();+ fs::copy(&pdf, corpus.path().join("quarterly-report-copy.pdf")).unwrap();++ let count = tools.path().join("worker-count");+ let worker = tools.path().join("pdf-worker");+ let worker_script = String::from_utf8(+ br#"#!/bin/sh+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"title":"quarterly-report","page":1,"body":"Quarterly revenue increased while operating margin improved.","pdf_pages_total":1,"pdf_pages_with_text":1,"pdf_pages_omitted":0},"locator":"p1-s0","group":null,"origin":"extractor"}]}'+ "#+ .to_vec(),+ )+ .unwrap()+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));+ fs::write(&worker, worker_script).unwrap();+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();++ let endpoint = MockEndpoint::start(1);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let _env = PdfWorkerEnvGuard::acquire();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);++ let first = run_index(config.clone()).unwrap_err();+ assert!(format!("{first:#}").contains("bulk backend failed"));+ assert_eq!(+ fs::read_to_string(&count).unwrap().lines().count(),+ 1,+ "Phase A's extraction must be replayed in Phase B, not parsed again"+ );+ assert_eq!(file_done_count(state_dir.path()), 0);++ // The process-local spool is intentionally not journal state. A retry has+ // a frozen plan, performs exactly one fresh extraction in Phase B, and+ // commits only after the backend accepts it.+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(fs::read_to_string(&count).unwrap().lines().count(), 2);+ assert_eq!(file_done_count(state_dir.path()), 1);+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 1);+ }++ #[cfg(unix)]+ #[test]+ fn refused_pdf_spool_reparses_and_reports_fallback_without_weakening_publication() {+ use std::os::unix::fs::PermissionsExt;++ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let tools = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("quarterly-report.pdf"),+ b"%PDF-1.4\nfake bytes consumed by the isolated test worker\n",+ )+ .unwrap();+ fs::write(+ state_dir+ .path()+ .join(".autoindex-test-pdf-spool-available-bytes"),+ ((4_u64 << 30) + (32 << 20) - 1).to_string(),+ )+ .unwrap();+ fs::write(+ state_dir.path().join(".autoindex-test-pdf-spool-fd-limit"),+ "4096",+ )+ .unwrap();+ fs::write(+ state_dir.path().join(".autoindex-test-pdf-spool-fd-open"),+ "16",+ )+ .unwrap();++ let count = tools.path().join("worker-count");+ let worker = tools.path().join("pdf-worker");+ let worker_script = String::from_utf8(+ br#"#!/bin/sh+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"title":"quarterly-report","page":1,"body":"Quarterly revenue increased while operating margin improved.","pdf_pages_total":1,"pdf_pages_with_text":1,"pdf_pages_omitted":0},"locator":"p1-s0","group":null,"origin":"extractor"}]}'+ "#+ .to_vec(),+ )+ .unwrap()+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));+ fs::write(&worker, worker_script).unwrap();+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();++ let endpoint = MockEndpoint::start(0);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let _env = PdfWorkerEnvGuard::acquire();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);++ let (code, report) = run_index_report(config).unwrap();+ assert_eq!(code, 0);+ assert_eq!(fs::read_to_string(&count).unwrap().lines().count(), 2);+ assert_eq!(file_done_count(state_dir.path()), 1);+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 1);+ let reuse = report.unwrap()["pdf_extraction_reuse"].clone();+ assert_eq!(reuse["reservations_started"], 0);+ assert_eq!(reuse["capacity_fallbacks"], 1);+ assert_eq!(reuse["phase_b_pdf_parses"], 1);+ assert_eq!(reuse["replay_verified"], 0);+ assert_eq!(reuse["artifacts_not_created"], 1);+ assert_eq!(reuse["phase_a_pdf_parser_responses"], 1);+ assert_eq!(reuse["capacity_status"], "disabled");+ assert_eq!(+ reuse["fallback_examples"][0]["path"],+ "quarterly-report.pdf"+ );+ }++ #[cfg(unix)]+ #[test]+ fn refused_spool_does_not_pay_a_second_phase_a_source_hash() {+ use std::os::unix::fs::PermissionsExt;++ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let tools = tempfile::tempdir().unwrap();+ let pdf = corpus.path().join("quarterly-report.pdf");+ fs::write(&pdf, b"%PDF-1.4\nsame-size-source-generation\n").unwrap();+ let size = fs::metadata(&pdf).unwrap().len();+ let digest = content::resolve_reporting(+ vec![crate::walk::FileEntry {+ path: pdf.clone(),+ rel: "quarterly-report.pdf".into(),+ rel_id: "quarterly-report.pdf".into(),+ is_symlink: false,+ size,+ }],+ &|_| {},+ )+ .unwrap()+ .digests+ .remove(0);- assert_eq!(run_index(config).unwrap(), 0);- let locked = endpoint.state.lock().unwrap();- assert_eq!(locked.docs.len(), 2);- assert!(locked- .docs- .values()- .all(|doc| doc["value"].as_str() == Some("new")));+ let worker = tools.path().join("pdf-worker");+ let worker_script = String::from_utf8(+ br#"#!/bin/sh+ printf 'Z' | dd of="$2" bs=1 seek=10 conv=notrunc 2>/dev/null+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"body":"Quarterly revenue increased while operating margin improved."},"locator":"p1-s0","group":null,"origin":"extractor"}]}'+ "#+ .to_vec(),+ )+ .unwrap()+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));+ fs::write(&worker, worker_script).unwrap();+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();+ let _env = PdfWorkerEnvGuard::acquire();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);++ let budget = crate::extract::pdf::ExtractionSpoolBudget::new(0, 0);+ let progress = Progress::silent();+ let scan = scan_file(+ &pdf,+ size,+ &digest,+ &PhaseAContext {+ state_dir: state_dir.path(),+ budget: &budget,+ capacity_warning: None,+ progress: &progress,+ },+ 100,+ 1,++ false,+ );+ assert!(scan.pdf_spool.is_none());+ assert_eq!(scan.pdf_spool_fallbacks.len(), 1);+ for fallback in &scan.pdf_spool_fallbacks {+ budget.record_fallback_example(+ "quarterly-report.pdf",+ fallback.category,+ &fallback.message,+ );}+ let report = budget.report();+ assert_eq!(report["artifacts_created"], 0);+ assert_eq!(report["phase_b_eligible_artifacts"], 0);+ assert_eq!(report["artifacts_discarded_before_replay"], 0);+ assert_eq!(report["fallback_categories"]["artifact_count_ceiling"], 1);+ assert!(report["fallback_categories"]+ .get("source_generation_changed")+ .is_none());}+ /// The other half of the branch above: with capacity available the artifact+ /// *is* created, and phase A must then throw it away because the source moved+ /// underneath the parser. Without this the source-generation arm of+ /// `scan_file` has no test at all — its sibling above deliberately runs with a+ /// zero budget, so it never reaches the `content::verify` comparison.+ #[cfg(unix)]#[test]- fn pending_generation_b_is_superseded_when_source_changes_to_c() {+ fn source_changed_during_phase_a_discards_the_artifact_and_names_the_reason() {+ use std::os::unix::fs::PermissionsExt;+let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();- // run_index reaches Journal::file_done; the file_done IO failpoint is- // a global one-shot, so every file_done-reaching test must hold its- // lock (armer or not) or it can steal an armed injection. Acquired- // after FAILPOINT_TEST_LOCK everywhere — one consistent order.- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();let state_dir = tempfile::tempdir().unwrap();- let path = corpus.path().join("records.csv");- fs::write(&path, "id,value\n0,generation-a\n1,generation-a\n").unwrap();- let endpoint = MockEndpoint::start(usize::MAX);- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);- assert_eq!(run_index(config.clone()).unwrap(), 0);+ inject_pdf_spool_capacity(state_dir.path());+ let tools = tempfile::tempdir().unwrap();+ let pdf = corpus.path().join("quarterly-report.pdf");+ fs::write(&pdf, b"%PDF-1.4\nsame-size-source-generation\n").unwrap();+ let size = fs::metadata(&pdf).unwrap().len();+ let digest = content::resolve_reporting(+ vec![crate::walk::FileEntry {+ path: pdf.clone(),+ rel: "quarterly-report.pdf".into(),+ rel_id: "quarterly-report.pdf".into(),+ is_symlink: false,+ size,+ }],+ &|_| {},+ )+ .unwrap()+ .digests+ .remove(0);- fs::write(&path, "id,value\n0,generation-b\n1,generation-b\n").unwrap();- REPLACEMENT_FAILPOINT.store(2, Ordering::SeqCst);- run_index(config.clone()).unwrap_err();- let replay_b = state::Journal::open(- state_dir.path(),- &config.root.to_string_lossy(),- &config.url,- &config.prefix,- config.bulk_timeout_secs,- false,+ // Same trick as the sibling test: the worker rewrites one byte of its own+ // input, so the file keeps its length and changes its digest.+ let worker = tools.path().join("pdf-worker");+ let worker_script = String::from_utf8(+ br#"#!/bin/sh+ printf 'Z' | dd of="$2" bs=1 seek=10 conv=notrunc 2>/dev/null+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"body":"Quarterly revenue increased while operating margin improved."},"locator":"p1-s0","group":null,"origin":"extractor"}]}'+ "#+ .to_vec(),)- .unwrap();- let key = replay_b.pending_replacements.keys().next().unwrap().clone();- let generation_b = replay_b.pending_replacements[&key].clone();- drop(replay_b);+ .unwrap()+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));+ fs::write(&worker, worker_script).unwrap();+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();+ let _env = PdfWorkerEnvGuard::acquire();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);- fs::write(&path, "id,value\n0,generation-c\n1,generation-c\n").unwrap();- assert_eq!(run_index(config.clone()).unwrap(), 0);- let replay_c = state::Journal::open(- state_dir.path(),- &config.root.to_string_lossy(),- &config.url,- &config.prefix,- config.bulk_timeout_secs,+ let (budget, _) =+ crate::extract::pdf::ExtractionSpoolBudget::for_state_dir(state_dir.path(), 1, 1, 8);+ let progress = Progress::silent();+ let scan = scan_file(+ &pdf,+ size,+ &digest,+ &PhaseAContext {+ state_dir: state_dir.path(),+ budget: &budget,+ capacity_warning: None,+ progress: &progress,+ },+ 100,+ 1,+ false,- )- .unwrap();- assert!(replay_c.pending_replacements.is_empty());- let committed_generation = replay_c.done[&key].generation.as_deref().unwrap();- assert_ne!(committed_generation, generation_b);- let events: Vec<Value> = fs::read_to_string(state_dir.path().join("journal.ndjson"))- .unwrap()- .lines()- .map(|line| serde_json::from_str(line).unwrap())- .collect();- let starts: Vec<&str> = events+ );++ assert!(+ scan.pdf_spool.is_none(),+ "an artifact bound to a superseded source generation must not reach phase B"+ );+ let categories: Vec<&str> = scan+ .pdf_spool_fallbacks.iter()- .filter(|event| {- event["kind"] == "file_replace_start"- && event["file_key"].as_str() == Some(key.as_str())- })- .filter_map(|event| event["generation"].as_str())+ .map(|fallback| fallback.category).collect();- assert_eq!(starts[starts.len() - 2], generation_b);- assert_eq!(starts.last().copied(), Some(committed_generation));- drop(replay_c);- let locked = endpoint.state.lock().unwrap();- assert_eq!(locked.docs.len(), 2);- assert!(locked- .docs- .values()- .all(|doc| doc["value"].as_str() == Some("generation-c")));+ assert_eq!(categories, vec!["source_generation_changed"]);+ let report = budget.report();+ assert_eq!(report["artifacts_created"], 1);+ assert_eq!(report["artifacts_discarded_before_replay"], 1);+ assert_eq!(report["phase_b_eligible_artifacts"], 0);+ assert_eq!(report["current_live_artifacts"], 0);+ assert_eq!(report["current_retained_or_reserved_bytes"], 0);+ assert_eq!(+ report["fallback_categories"]["source_generation_changed"],+ 1+ );}+ #[cfg(unix)]#[test]- fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {+ fn verified_but_junk_pdf_spool_is_created_then_discarded_not_eligible() {+ use std::os::unix::fs::PermissionsExt;+let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();- // run_index reaches Journal::file_done; the file_done IO failpoint is- // a global one-shot, so every file_done-reaching test must hold its- // lock (armer or not) or it can steal an armed injection. Acquired- // after FAILPOINT_TEST_LOCK everywhere — one consistent order.- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();let state_dir = tempfile::tempdir().unwrap();- let original = "id,value\n0,original\n1,original\n";- fs::write(corpus.path().join("a.csv"), original).unwrap();- let endpoint = MockEndpoint::start(usize::MAX);- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);- assert_eq!(run_index(config.clone()).unwrap(), 0);-- // a.csv keeps its planned key by path while its OLD bytes reappear as- // b.csv — whose content key is exactly the planned key a.csv claims. Both- // files owning one ax_file key would let each replacement delete the- // other's freshly published documents.- fs::write(- corpus.path().join("a.csv"),- "id,value\n0,rewritten\n1,rewritten\n",- )- .unwrap();- fs::write(corpus.path().join("b.csv"), original).unwrap();- assert_eq!(run_index(config.clone()).unwrap(), 3);- {- let locked = endpoint.state.lock().unwrap();- assert_eq!(locked.docs.len(), 2);- assert!(locked.docs.values().all(|doc| {- doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")- }));- }- let replay = state::Journal::open(- state_dir.path(),- &config.root.to_string_lossy(),- &config.url,- &config.prefix,- config.bulk_timeout_secs,- false,+ inject_pdf_spool_capacity(state_dir.path());+ let tools = tempfile::tempdir().unwrap();+ let pdf = corpus.path().join("empty-report.pdf");+ fs::write(&pdf, b"%PDF-1.4\nempty test input\n").unwrap();+ let size = fs::metadata(&pdf).unwrap().len();+ let digest = content::resolve_reporting(+ vec![crate::walk::FileEntry {+ path: pdf.clone(),+ rel: "empty-report.pdf".into(),+ rel_id: "empty-report.pdf".into(),+ is_symlink: false,+ size,+ }],+ &|_| {},)- .unwrap();- assert!(replay.pending_replacements.is_empty());- assert_eq!(replay.done.len(), 1);- let plan = replay.plan.clone().unwrap();- assert_eq!(plan.junk_files.len(), 1);- assert_eq!(plan.junk_files[0].rel, "b.csv");- assert!(plan.junk_files[0]- .reason- .contains("key ownership exclusive"));- drop(replay);+ .unwrap()+ .digests+ .remove(0);+ let worker = tools.path().join("pdf-worker");+ let script = format!(+ "#!/bin/sh\nprintf '%s' '{{\"schema\":1,\"extractor\":\"xerj-autoindex/{}\",\"parser\":\"pdf_oxide/0.3.75\",\"containment\":\"test worker\",\"records\":[]}}'\n",+ env!("CARGO_PKG_VERSION")+ );+ fs::write(&worker, script).unwrap();+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();+ let _env = PdfWorkerEnvGuard::acquire();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);- // The divergence is durable and deterministic: an identical rerun keeps- // the same owner, appends no new plan, and changes no documents.- let plans_before = event_count(state_dir.path(), "plan");- assert_eq!(run_index(config).unwrap(), 3);- assert_eq!(event_count(state_dir.path(), "plan"), plans_before);- let locked = endpoint.state.lock().unwrap();- assert_eq!(locked.docs.len(), 2);- assert!(locked- .docs- .values()- .all(|doc| doc["value"].as_str() == Some("rewritten")));+ let (budget, _) =+ crate::extract::pdf::ExtractionSpoolBudget::for_state_dir(state_dir.path(), 1, 1, 8);+ let progress = Progress::silent();+ let mut scan = scan_file(+ &pdf,+ size,+ &digest,+ &PhaseAContext {+ state_dir: state_dir.path(),+ budget: &budget,+ capacity_warning: None,+ progress: &progress,+ },+ 100,+ 1,++ false,+ );+ assert!(scan.junk.is_some());+ assert!(scan.pdf_spool.is_some());+ assert!(take_pdf_spool_if_indexable(&mut scan.pdf_spool, true, &budget).is_none());+ let report = budget.report();+ assert_eq!(report["artifacts_created"], 1);+ assert_eq!(report["artifacts_discarded_before_replay"], 1);+ assert_eq!(report["phase_b_eligible_artifacts"], 0);}+ #[cfg(unix)]#[test]- fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {+ fn multi_pdf_run_reuses_every_artifact_and_reports_exact_success_counters() {+ use std::os::unix::fs::PermissionsExt;+let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();- // run_index reaches Journal::file_done; the file_done IO failpoint is- // a global one-shot, so every file_done-reaching test must hold its- // lock (armer or not) or it can steal an armed injection. Acquired- // after FAILPOINT_TEST_LOCK everywhere — one consistent order.- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();let state_dir = tempfile::tempdir().unwrap();- fs::write(corpus.path().join("dup-a.csv"), "id,value\n0,dup\n1,dup\n").unwrap();- fs::write(corpus.path().join("dup-b.csv"), "id,value\n0,dup\n1,dup\n").unwrap();- fs::write(corpus.path().join("keep.csv"), "id,value\n2,keep\n3,keep\n").unwrap();- let endpoint = MockEndpoint::start(usize::MAX);- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);- assert_eq!(run_index(config.clone()).unwrap(), 0);- let starts = event_count(state_dir.path(), "file_replace_start");-- // The whole duplicate group disappears. There is no current file left to- // republish its key, so no replacement intent may be journaled — a- // stranded intent would be re-appended forever without ever committing.- fs::remove_file(corpus.path().join("dup-a.csv")).unwrap();- fs::remove_file(corpus.path().join("dup-b.csv")).unwrap();- for _ in 0..2 {- assert_eq!(run_index(config.clone()).unwrap(), 0);- assert_eq!(- event_count(state_dir.path(), "file_replace_start"),- starts,- "an orphaned duplicate-group key must not schedule a replacement"- );- let replay = state::Journal::open(- state_dir.path(),- &config.root.to_string_lossy(),- &config.url,- &config.prefix,- config.bulk_timeout_secs,- false,+ inject_pdf_spool_capacity(state_dir.path());+ let tools = tempfile::tempdir().unwrap();+ const PDFS: usize = 6;+ for index in 0..PDFS {+ fs::write(+ corpus.path().join(format!("quarterly-report-{index}.pdf")),+ format!("%PDF-1.4\nunique isolated worker input {index}\n"),).unwrap();- assert!(replay.pending_replacements.is_empty());}++ let count = tools.path().join("worker-count");+ let worker = tools.path().join("pdf-worker");+ let worker_script = String::from_utf8(+ br#"#!/bin/sh+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"title":"quarterly-report","page":1,"body":"Quarterly revenue increased while operating margin improved.","pdf_pages_total":1,"pdf_pages_with_text":1,"pdf_pages_omitted":0},"locator":"p1-s0","group":null,"origin":"extractor"}]}'+ "#+ .to_vec(),+ )+ .unwrap()+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));+ fs::write(&worker, worker_script).unwrap();+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();++ let endpoint = MockEndpoint::start(0);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ config.workers = 8;+ config.pdf_workers = 2;+ let _env = PdfWorkerEnvGuard::acquire();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);+ let (code, report) = run_index_report(config).unwrap();+ assert_eq!(code, 0);+ assert_eq!(+ fs::read_to_string(&count).unwrap().lines().count(),+ PDFS,+ "each unique PDF must be parsed once in Phase A and replayed in Phase B"+ );+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), PDFS);+ let reuse = report.unwrap()["pdf_extraction_reuse"].clone();+ assert_eq!(reuse["reservations_started"], PDFS);+ assert_eq!(+ reuse["cumulative_reserved_bytes"],+ PDFS as u64 * (32_u64 << 20)+ );+ assert_eq!(reuse["artifacts_created"], PDFS);+ assert_eq!(reuse["artifacts_not_created"], 0);+ assert_eq!(reuse["phase_b_eligible_artifacts"], PDFS);+ assert_eq!(reuse["artifacts_discarded_before_replay"], 0);+ assert!(reuse["exact_artifact_bytes"].as_u64().unwrap() > 0);+ assert_eq!(reuse["current_retained_or_reserved_bytes"], 0);+ assert!(reuse["peak_retained_or_reserved_bytes"].as_u64().unwrap() > 0);+ assert_eq!(reuse["current_live_artifacts"], 0);+ assert!(reuse["peak_live_artifacts"].as_u64().unwrap() > 0);+ assert_eq!(reuse["phase_a_pdf_parser_responses"], PDFS);+ assert_eq!(reuse["capacity_fallbacks"], 0);+ assert_eq!(reuse["io_fallbacks"], 0);+ assert_eq!(reuse["replay_verified"], PDFS);+ assert_eq!(reuse["replay_integrity_failures"], 0);+ assert_eq!(reuse["phase_b_pdf_parses"], 0);+ assert_eq!(reuse["fallback_examples"].as_array().unwrap().len(), 0);+ assert_eq!(reuse["fallback_examples_truncated"], false);+ assert_eq!(reuse["fallback_categories"].as_object().unwrap().len(), 0);}+ #[cfg(unix)]#[test]- fn partial_file_done_is_rolled_back_before_another_worker_commits() {+ fn corrupted_pdf_replay_reparses_without_blaming_the_source_or_backend() {+ use std::os::unix::fs::PermissionsExt;+let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();let state_dir = tempfile::tempdir().unwrap();- fs::write(corpus.path().join("a.csv"), "id,value\n0,a-old\n1,a-old\n").unwrap();- fs::write(corpus.path().join("b.csv"), "id,value\n2,b-old\n3,b-old\n").unwrap();- let endpoint = MockEndpoint::start(usize::MAX);- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);- config.workers = 2;- assert_eq!(run_index(config.clone()).unwrap(), 0);+ inject_pdf_spool_capacity(state_dir.path());+ let tools = tempfile::tempdir().unwrap();+ let pdf = corpus.path().join("report.pdf");+ fs::write(&pdf, b"%PDF-1.4\ntest worker input\n").unwrap();+ let worker = tools.path().join("pdf-worker");+ let count = tools.path().join("worker-count");+ let worker_script = String::from_utf8(+ br#"#!/bin/sh+ printf 'parse\n' >> "$XERJ_TEST_PDF_COUNT"+ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":"pdf_oxide/0.3.75","containment":"test worker","records":[{"fields":{"body":"Quarterly revenue increased while operating margin improved."},"locator":"p1-s0","group":null,"origin":"extractor"}]}'+ "#+ .to_vec(),+ )+ .unwrap()+ .replace("__XERJ_VERSION__", env!("CARGO_PKG_VERSION"));+ fs::write(&worker, worker_script).unwrap();+ fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap();- fs::write(corpus.path().join("a.csv"), "id,value\n0,a-new\n1,a-new\n").unwrap();- fs::write(corpus.path().join("b.csv"), "id,value\n2,b-new\n3,b-new\n").unwrap();- state::fail_next_file_done_io(3);- let error = run_index(config.clone()).unwrap_err();- assert!(format!("{error:#}").contains("partial file_done write"));+ let endpoint = MockEndpoint::start(0);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let _env = PdfWorkerEnvGuard::acquire();+ std::env::set_var("XERJ_PDF_WORKER_BIN", &worker);+ std::env::set_var("XERJ_TEST_PDF_COUNT", &count);+ crate::extract::pdf::corrupt_replay_for_source_size(fs::metadata(&pdf).unwrap().len());- // One worker hit the partial append and rolled back while the other was- // allowed to commit. Replay must remain parseable and expose exactly one- // pending generation rather than malformed middle corruption.- let replay = state::Journal::open(- state_dir.path(),- &config.root.to_string_lossy(),- &config.url,- &config.prefix,- config.bulk_timeout_secs,- false,- )- .unwrap();- assert_eq!(replay.pending_replacements.len(), 1);- assert_eq!(replay.done.len(), 1);- drop(replay);+ let (code, report) = run_index_report(config).unwrap();+ assert_eq!(code, 0);+ assert_eq!(fs::read_to_string(&count).unwrap().lines().count(), 2);+ assert_eq!(data_rows(&endpoint.state.lock().unwrap()).len(), 1);+ assert_eq!(file_done_count(state_dir.path()), 1);+ assert!(crate::extract::pdf::corrupted_replay_reservation_was_dropped());+ let reuse = report.unwrap()["pdf_extraction_reuse"].clone();+ assert_eq!(reuse["replay_verified"], 0);+ assert_eq!(reuse["replay_integrity_failures"], 1);+ assert_eq!(+ reuse["io_fallbacks"], 0,+ "a corrupted artifact is an integrity failure, not an I/O failure"+ );+ assert_eq!(reuse["phase_b_pdf_parses"], 1);+ assert_eq!(reuse["current_live_artifacts"], 0);+ assert_eq!(reuse["current_retained_or_reserved_bytes"], 0);+ assert_eq!(reuse["fallback_categories"]["replay_verification"], 1);+ assert_eq!(+ reuse["fallback_examples"][0]["category"],+ "replay_verification"+ );+ }- assert_eq!(run_index(config).unwrap(), 0);- let locked = endpoint.state.lock().unwrap();- assert_eq!(locked.docs.len(), 4);- assert!(locked.docs.values().all(|doc| {- doc["value"]- .as_str()- .is_some_and(|value| value.ends_with("-new"))- }));+ #[test]+ fn junk_scan_drops_its_spool_before_indexable_spools_are_retained() {+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::sync::Arc;++ struct DropProbe(Arc<AtomicUsize>);+ impl Drop for DropProbe {+ fn drop(&mut self) {+ self.0.fetch_add(1, Ordering::SeqCst);+ }+ }++ let drops = Arc::new(AtomicUsize::new(0));+ let budget = crate::extract::pdf::ExtractionSpoolBudget::new(1, 1);+ let mut junk = Some(DropProbe(Arc::clone(&drops)));+ assert!(take_pdf_spool_if_indexable(&mut junk, true, &budget).is_none());+ assert!(junk.is_none());+ assert_eq!(drops.load(Ordering::SeqCst), 1);++ let mut indexable = Some(DropProbe(Arc::clone(&drops)));+ let retained = take_pdf_spool_if_indexable(&mut indexable, false, &budget);+ assert!(retained.is_some());+ assert_eq!(drops.load(Ordering::SeqCst), 1);+ let report = budget.report();+ assert_eq!(report["artifacts_discarded_before_replay"], 1);+ assert_eq!(+ report["phase_b_eligible_artifacts"], 0,+ "eligibility is recorded only after the final todo set is known"+ );+ drop(retained);+ assert_eq!(drops.load(Ordering::SeqCst), 2);}diff --cc engine/crates/xerj-autoindex/src/ignore_rules.rsindex 000000000,6731af882..dffe9bf8cmode 000000,100644..100644--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs@@@ -1,0 -1,950 +1,958 @@@+ //! Ignore rules for the inventory walk (#276).+ //!+ //! The cheapest file to index is the one that is never read. A repository's+ //! `.gitignore` already says which files are build output; honouring it means+ //! `target/`, `node_modules/` and friends are pruned before the walk descends+ //! into them, so they cost nothing at all — not a stat, not a hash, not a bulk+ //! body.+ //!+ //! # Where the semantics come from+ //!+ //! gitignore matching is easy to get 80% right and very hard to get right:+ //! nested ignore files, negation (`!keep.log`), directory-only patterns+ //! (`build/`), anchoring (`/only-here`) and precedence between an ignore file+ //! in a subdirectory and one at the root. We do not hand-roll any of that. The+ //! matcher is ripgrep's `ignore` crate (`Unlicense OR MIT`, compatible with+ //! XERJ's Apache-2.0), used for its **matching** only — traversal stays on+ //! `walkdir` so the existing symlink, hidden-file and unreadable-entry+ //! behaviour of [`crate::walk`] is unchanged, and so we can report *which*+ //! rule discarded *what*, which `ignore`'s own walker does not expose.+ //!+ //! Two behaviours are copied deliberately from that reference implementation+ //! (`ignore-0.4.32/src/dir.rs`, read at `~/.cargo/registry/src/*/ignore-0.4.32`):+ //!+ //! * **Deepest ignore file wins, per kind.** `dir.rs:563-594` walks matchers+ //! from the current directory outward and keeps the first decision each kind+ //! produces (`if m_gi.is_none() { m_gi = … }`).+ //! * **Kinds are ranked, and the rank beats depth.** `dir.rs:678-684` combines+ //! them `m_custom_ignore.or(m_ignore).or(m_gi).or(m_gi_exclude)`. So a+ //! `.xerjignore` at the root outranks a `.gitignore` in a deep subdirectory.+ //! That is what makes `.xerjignore` useful as the stated "exclude things git+ //! tracks" (and re-include things git ignores) override rather than just a+ //! second `.gitignore`.+ //!+ //! Within one file the last matching pattern wins; that is `Gitignore::matched`+ //! and we do not reimplement it.+ //!+ //! # What is deliberately not honoured+ //!+ //! The user's *global* gitignore (`core.excludesFile`) is not read: a machine-+ //! wide preference silently deciding what is in someone's search index is a+ //! surprise, and it is not visible in the folder being indexed. `.gitignore`,+ //! `.xerjignore` and `.git/info/exclude` all live in the tree the user pointed+ //! at, so they are.++ use ignore::gitignore::{Gitignore, GitignoreBuilder, Glob};+ use ignore::Match;+ use std::collections::BTreeMap;+ use std::path::{Path, PathBuf};++ /// Ignore file honoured in every directory, ranked below `.xerjignore`.+ pub const GITIGNORE: &str = ".gitignore";+ /// XERJ's own ignore file. Same syntax as `.gitignore`, higher precedence, so+ /// a user can exclude something git tracks (or re-include something git+ /// ignores) without editing `.gitignore`.+ pub const XERJIGNORE: &str = ".xerjignore";++ /// Build output that is junk in practice but is *not* reliably gitignored —+ /// `vendor/` is committed by Go projects, `dist/` and `build/` are gitignored+ /// in one repo and checked in the next, and a folder full of downloaded+ /// packages is very often not a git repository at all.+ ///+ /// These are defaults, not law: `--no-default-ignores` turns them off, any+ /// ignore file in the tree outranks them, and `--dry-run` names them as the+ /// reason a directory disappeared.+ pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[+ "node_modules/",+ "vendor/",+ "target/",+ "dist/",+ "build/",+ ".venv/",+ "__pycache__/",+ ];++ /// Rule label used for the pre-existing dotfile skip. It is not one of the+ /// ignore rules and `--no-ignore` does not disable it (it is what keeps `.env`+ /// and `.ssh` out of a queryable brain), but a user whose file vanished still+ /// needs it named, so it shares the reporting.+ pub const HIDDEN_RULE: &str = "hidden:dotfile";++ const MAX_WARNINGS: usize = 20;+ /// Directory entries the `--dry-run` deep count may examine in total. A+ /// pathological `node_modules` is not allowed to turn "tell me the plan" into+ /// minutes of stat storm on a laptop; past the budget the count is reported as+ /// a floor instead of a total.+ const DEEP_COUNT_BUDGET: u64 = 1_000_000;+ /// How far above the root we look for an ignore file that covers the root+ /// itself. Bounded so a root near `/` cannot walk the whole filesystem.+ const MAX_ANCESTOR_LEVELS: usize = 64;++ #[derive(Debug, Clone, Copy, PartialEq, Eq)]+ pub struct IgnoreOptions {+ /// `--no-ignore` clears this: no `.gitignore`, no `.xerjignore`, no+ /// defaults. The dotfile skip is separate and stays on.+ pub enabled: bool,+ /// `--no-default-ignores` clears this: ignore files still apply,+ /// [`DEFAULT_IGNORE_PATTERNS`] does not.+ pub defaults: bool,+ /// Count the files inside a pruned directory instead of only reporting+ /// that the directory was pruned. Worth its syscalls in `--dry-run`, whose+ /// whole job is explaining the plan, and not otherwise — the point of+ /// pruning is to never touch those paths.+ pub deep_count: bool,+ }++ impl Default for IgnoreOptions {+ fn default() -> Self {+ Self {+ enabled: true,+ defaults: true,+ deep_count: false,+ }+ }+ }++ impl IgnoreOptions {+ /// `--no-ignore`.+ pub const fn off() -> Self {+ Self {+ enabled: false,+ defaults: false,+ deep_count: false,+ }+ }++ /// `--dry-run` explains itself; every other run stays cheap.+ pub const fn with_deep_count(mut self, deep_count: bool) -> Self {+ self.deep_count = deep_count;+ self+ }+ }++ #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+ pub struct RuleCount {+ /// Files matched directly by this rule.+ pub files: u64,+ /// Directories pruned by this rule (their contents were never walked).+ pub dirs: u64,+ /// Files found inside those pruned directories. Only ever non-zero when+ /// the deep count ran (`--dry-run`).+ pub files_inside_pruned_dirs: u64,+ }++ #[derive(Debug, Clone)]+ pub struct IgnoreReport {+ /// rule label → what it discarded.+ pub by_rule: BTreeMap<String, RuleCount>,+ pub files_skipped: u64,+ pub dirs_pruned: u64,+ pub files_inside_pruned_dirs: u64,+ /// True unless the deep count hit [`DEEP_COUNT_BUDGET`]. When false,+ /// `files_inside_pruned_dirs` is a floor and every line that prints it+ /// says "at least".+ pub deep_count_complete: bool,+ /// Whether the deep count ran at all. When it did not, no file count+ /// inside a pruned directory is printed — an unmeasured number is not+ /// printed as zero.+ pub deep_count_ran: bool,+ pub ignore_files_read: u64,+ /// Set when the folder the user pointed at is itself covered by an ignore+ /// rule. It is indexed anyway; this is the "and here is why you are seeing+ /// files you thought were ignored" line.+ pub root_exemption: Option<String>,+ /// Unreadable or unparseable ignore files. Never fatal: an ignore file we+ /// cannot read means we index more, not less.+ pub warnings: Vec<String>,+ }++ impl Default for IgnoreReport {+ fn default() -> Self {+ Self {+ by_rule: BTreeMap::new(),+ files_skipped: 0,+ dirs_pruned: 0,+ files_inside_pruned_dirs: 0,+ deep_count_complete: true,+ deep_count_ran: false,+ ignore_files_read: 0,+ root_exemption: None,+ warnings: Vec::new(),+ }+ }+ }++ impl IgnoreReport {+ pub fn nothing_to_report(&self) -> bool {+ self.by_rule.is_empty() && self.root_exemption.is_none() && self.warnings.is_empty()+ }++ /// Human-readable lines for the progress surface. Every number here was+ /// counted by this run; the ones that were deliberately not counted are+ /// named as such instead of being printed as zero.+ pub fn summary_lines(&self) -> Vec<String> {+ let mut out = Vec::new();+ for warning in &self.warnings {+ out.push(format!("ignore rules: {warning}"));+ }+ if let Some(reason) = &self.root_exemption {+ out.push(format!("ignore rules: {reason}"));+ }+ if self.by_rule.is_empty() {+ return out;+ }+ let mut headline = format!(+ "ignore rules: skipped {} file{} and pruned {} director{}",+ self.files_skipped,+ plural(self.files_skipped),+ self.dirs_pruned,+ if self.dirs_pruned == 1 { "y" } else { "ies" },+ );+ if self.deep_count_ran && self.dirs_pruned > 0 {+ headline.push_str(&format!(+ " ({}{} file{} inside them)",+ if self.deep_count_complete {+ ""+ } else {+ "at least "+ },+ self.files_inside_pruned_dirs,+ plural(self.files_inside_pruned_dirs),+ ));+ } else if self.dirs_pruned > 0 {+ headline.push_str(" (contents never walked, so not counted)");+ }+ if self.ignore_files_read > 0 {+ headline.push_str(&format!(+ "; {} ignore file{} read",+ self.ignore_files_read,+ plural(self.ignore_files_read)+ ));+ }+ out.push(headline);++ let mut ranked: Vec<(&String, &RuleCount)> = self.by_rule.iter().collect();+ ranked.sort_by(|a, b| {+ let weight = |c: &RuleCount| c.files + c.files_inside_pruned_dirs + c.dirs;+ weight(b.1).cmp(&weight(a.1)).then_with(|| a.0.cmp(b.0))+ });+ const SHOWN: usize = 10;+ for (label, count) in ranked.iter().take(SHOWN) {+ let mut parts = Vec::new();+ if count.files > 0 {+ parts.push(format!("{} file{}", count.files, plural(count.files)));+ }+ if count.dirs > 0 {+ let inside = if self.deep_count_ran {+ format!(+ " ({}{} file{} inside)",+ if self.deep_count_complete {+ ""+ } else {+ "at least "+ },+ count.files_inside_pruned_dirs,+ plural(count.files_inside_pruned_dirs)+ )+ } else {+ String::new()+ };+ parts.push(format!(+ "{} director{} pruned{inside}",+ count.dirs,+ if count.dirs == 1 { "y" } else { "ies" }+ ));+ }+ out.push(format!("ignore rules: {label} — {}", parts.join(", ")));+ }+ if ranked.len() > SHOWN {+ out.push(format!(+ "ignore rules: … and {} more rule{}",+ ranked.len() - SHOWN,+ plural((ranked.len() - SHOWN) as u64)+ ));+ }+ out+ }+ }++ fn plural(n: u64) -> &'static str {+ if n == 1 {+ ""+ } else {+ "s"+ }+ }++ #[derive(Default)]+ struct Layer {+ xerj: Option<Gitignore>,+ git: Option<Gitignore>,+ exclude: Option<Gitignore>,+ }++ /// A stack of ignore matchers, one entry per directory on the current+ /// root-to-here path, plus the built-in defaults.+ ///+ /// The caller drives it: [`IgnoreStack::enter_dir`] when a directory is+ /// yielded, [`IgnoreStack::skip_dir`] / [`IgnoreStack::skip_file`] to ask+ /// whether an entry survives.+ pub struct IgnoreStack {+ opts: IgnoreOptions,+ root: PathBuf,+ layers: Vec<Layer>,+ defaults: Option<Gitignore>,+ budget: u64,+ pub report: IgnoreReport,+ }++ impl IgnoreStack {+ /// `root` must be the canonicalised root folder.+ pub fn new(root: &Path, opts: IgnoreOptions) -> Self {+ let mut stack = Self {+ opts,+ root: root.to_path_buf(),+ layers: Vec::new(),+ defaults: None,+ budget: DEEP_COUNT_BUDGET,+ report: IgnoreReport {+ deep_count_ran: opts.deep_count,+ ..IgnoreReport::default()+ },+ };+ if opts.enabled && opts.defaults {+ let defaults = stack.build_defaults(root);+ stack.defaults = defaults;+ }+ let exemption = stack.root_exemption(root);+ stack.report.root_exemption = exemption;+ stack+ }++ fn build_defaults(&mut self, root: &Path) -> Option<Gitignore> {+ let mut builder = GitignoreBuilder::new(root);+ for pattern in DEFAULT_IGNORE_PATTERNS {+ if let Err(err) = builder.add_line(None, pattern) {+ self.warn(format!("built-in rule `{pattern}` is not usable: {err}"));+ }+ }+ match builder.build() {+ Ok(gi) if gi.is_empty() => None,+ Ok(gi) => Some(gi),+ Err(err) => {+ self.warn(format!("built-in default rules are unusable: {err}"));+ None+ }+ }+ }++ /// Detect that the folder the user pointed at is itself ignored.+ ///+ /// We never act on it — pointing autoindex at `~/proj/target` is an+ /// explicit instruction and silently indexing nothing would be the worst+ /// possible answer. We only explain it, because the alternative is a user+ /// staring at a brain full of build output wondering why their ignore file+ /// did nothing.+ fn root_exemption(&self, root: &Path) -> Option<String> {+ if !self.opts.enabled {+ return None;+ }+ let name = root.file_name()?;+ let parent = root.parent()?;+ // The defaults are anchored at the root itself, so they cannot see the+ // root's own name. Re-anchor a copy one level up to ask the question.+ if self.opts.defaults {+ let mut builder = GitignoreBuilder::new(parent);+ for pattern in DEFAULT_IGNORE_PATTERNS {+ let _ = builder.add_line(None, pattern);+ }+ if let Ok(gi) = builder.build() {+ if let Match::Ignore(glob) = gi.matched(Path::new(name), true) {+ return Some(format!(+ "{} matches the built-in default rule `{}`, but you pointed autoindex at \+ it, so it is being indexed — pass --no-default-ignores to silence this \+ note",+ root.display(),+ glob.original()+ ));+ }+ }+ }+ // Ignore files *above* the root are consulted for this note only, and+ // never to prune: git itself stops at the repository root, so we do+ // too (plus a hard level cap).+ let mut dir = parent.to_path_buf();+ for _ in 0..MAX_ANCESTOR_LEVELS {+ for file in [XERJIGNORE, GITIGNORE] {+ let path = dir.join(file);+ if !path.is_file() {+ continue;+ }+ let mut builder = GitignoreBuilder::new(&dir);+ let _ = builder.add(&path);+ let Ok(gi) = builder.build() else { continue };+ if gi.is_empty() {+ continue;+ }+ if let Match::Ignore(glob) = gi.matched_path_or_any_parents(root, true) {+ return Some(format!(+ "{} is ignored by `{}` in {}, but you pointed autoindex at it, so it is \+ being indexed",+ root.display(),+ glob.original(),+ path.display()+ ));+ }+ }+ if dir.join(".git").exists() {+ break;+ }+ match dir.parent() {+ Some(up) => dir = up.to_path_buf(),+ None => break,+ }+ }+ None+ }++ /// Drop the layers of directories the walk has finished with.+ ///+ /// Must be called for **every** entry, not only directories. A depth-first+ /// walk interleaves files and subdirectories in readdir order, so after it+ /// descends into `a/` and comes back it can hand us `b.txt` at the shallower+ /// depth while `a/`'s layer is still on the stack. Without this, `a/`'s+ /// `.gitignore` would silently judge its parent's files.+ pub fn truncate_to(&mut self, depth: usize) {+ self.layers.truncate(depth);+ }++ /// Called for every directory the walk yields, root first, before any of+ /// its children. `depth` is walkdir's depth, so the stack is indexed by+ /// depth and a sibling directory replaces its predecessor's layer.+ pub fn enter_dir(&mut self, dir: &Path, depth: usize) {+ self.layers.truncate(depth);+ let layer = if self.opts.enabled {+ Layer {+ xerj: self.load(dir, dir.join(XERJIGNORE)),+ git: self.load(dir, dir.join(GITIGNORE)),+ // Per-repository excludes, which is where people put "junk+ // only I have". Loaded per directory so a nested checkout gets+ // its own.+ exclude: self.load(dir, dir.join(".git").join("info").join("exclude")),+ }+ } else {+ Layer::default()+ };+ self.layers.push(layer);+ }++ fn load(&mut self, dir: &Path, file: PathBuf) -> Option<Gitignore> {+ if !file.is_file() {+ return None;+ }+ let mut builder = GitignoreBuilder::new(dir);+ if let Some(err) = builder.add(&file) {+ self.warn(format!("{}: {err}", file.display()));+ }+ match builder.build() {+ Ok(gi) => {+ self.report.ignore_files_read += 1;+ if gi.is_empty() {+ None+ } else {+ Some(gi)+ }+ }+ Err(err) => {+ self.warn(format!("{} is not usable: {err}", file.display()));+ None+ }+ }+ }++ fn warn(&mut self, message: String) {+ if self.report.warnings.len() < MAX_WARNINGS {+ self.report.warnings.push(message);+ }+ }++ /// ripgrep's precedence, reproduced: the deepest match *per kind*, then+ /// the kinds ranked `.xerjignore` > `.gitignore` > `.git/info/exclude` >+ /// built-in defaults (`ignore-0.4.32/src/dir.rs:563-594` and `:678-684`).+ fn lookup(&self, path: &Path, is_dir: bool) -> Option<(bool, String)> {+ if !self.opts.enabled {+ return None;+ }+ let mut xerj = None;+ let mut git = None;+ let mut exclude = None;+ for layer in self.layers.iter().rev() {+ if xerj.is_none() {+ xerj = self.decide_with(layer.xerj.as_ref(), path, is_dir);+ }+ if git.is_none() {+ git = self.decide_with(layer.git.as_ref(), path, is_dir);+ }+ if exclude.is_none() {+ exclude = self.decide_with(layer.exclude.as_ref(), path, is_dir);+ }+ if xerj.is_some() {+ // Nothing ranked lower can change the answer.+ break;+ }+ }+ xerj.or(git)+ .or(exclude)+ .or_else(|| self.decide_with(self.defaults.as_ref(), path, is_dir))+ }++ fn decide_with(+ &self,+ matcher: Option<&Gitignore>,+ path: &Path,+ is_dir: bool,+ ) -> Option<(bool, String)> {+ match matcher?.matched(path, is_dir) {+ Match::None => None,+ Match::Ignore(glob) => Some((true, self.label(glob))),+ Match::Whitelist(glob) => Some((false, self.label(glob))),+ }+ }++ fn label(&self, glob: &Glob) -> String {+ let from = glob+ .from()+ .map(|p| self.rel(p))+ .unwrap_or_else(|| "<built-in>".to_string());+ format!("{from}:{}", glob.original())+ }++ fn rel(&self, path: &Path) -> String {+ path.strip_prefix(&self.root)+ .unwrap_or(path)+ .to_string_lossy()+ .replace('\\', "/")+ }++ /// Ask whether a directory must be pruned. `Some(label)` means do not+ /// descend; the report has already been updated.+ pub fn skip_dir(&mut self, path: &Path) -> Option<String> {+ let (ignored, label) = self.lookup(path, true)?;+ if !ignored {+ return None;+ }+ self.record_dir(path, &label);+ Some(label)+ }++ /// Ask whether a file must be skipped.+ pub fn skip_file(&mut self, path: &Path) -> Option<String> {+ let (ignored, label) = self.lookup(path, false)?;+ if !ignored {+ return None;+ }+ self.record_file(&label);+ Some(label)+ }++ /// The dotfile skip is decided by the walker, not here, but it is reported+ /// through the same accounting so `--dry-run` explains every missing file+ /// with one mechanism.+ pub fn record_hidden(&mut self, path: &Path, is_dir: bool) {+ if is_dir {+ self.record_dir(path, HIDDEN_RULE);+ } else {+ self.record_file(HIDDEN_RULE);+ }+ }+++ /// The marker-gated generated-dir prune (`walk::MARKER_GENERATED_DIRS`)++ /// is also decided by the walker — the verdict needs the SIBLING marker++ /// file, which the per-directory matcher stack cannot see — and reported++ /// through the same accounting.++ pub fn record_marker_dir(&mut self, path: &Path, label: &str) {++ self.record_dir(path, label);++ }+++ fn record_file(&mut self, label: &str) {+ self.report.files_skipped += 1;+ self.report+ .by_rule+ .entry(label.to_string())+ .or_default()+ .files += 1;+ }++ fn record_dir(&mut self, path: &Path, label: &str) {+ self.report.dirs_pruned += 1;+ self.report+ .by_rule+ .entry(label.to_string())+ .or_default()+ .dirs += 1;+ let inside = self.count_inside(path);+ self.report.files_inside_pruned_dirs += inside;+ self.report+ .by_rule+ .entry(label.to_string())+ .or_default()+ .files_inside_pruned_dirs += inside;+ }++ /// Only under `--dry-run`. Symlinks are never followed here whatever the+ /// run's `--follow-symlinks` setting: this is an accounting pass over a+ /// directory we have already decided not to index, and it must not be able+ /// to wander outside it or loop.+ ///+ /// The dotfile rule is applied here too, so the number means "files that+ /// would otherwise have been indexed" and not "inodes under this path".+ /// Without it a pruned directory containing a nested checkout reports its+ /// whole `.git`, which was never indexable in the first place — a number+ /// that does not reconcile with the before/after file count is worse than+ /// no number.+ fn count_inside(&mut self, dir: &Path) -> u64 {+ if !self.opts.deep_count {+ return 0;+ }+ let mut files = 0;+ let walker = walkdir::WalkDir::new(dir)+ .follow_links(false)+ .min_depth(1)+ .into_iter()+ .filter_entry(|e| {+ !e.file_name()+ .to_str()+ .is_some_and(|name| name.starts_with('.'))+ });+ for entry in walker {+ if self.budget == 0 {+ self.report.deep_count_complete = false;+ break;+ }+ self.budget -= 1;+ if entry.is_ok_and(|e| e.file_type().is_file()) {+ files += 1;+ }+ }+ files+ }+ }++ #[cfg(test)]+ mod tests {+ use super::*;+ use std::fs;++ /// Drive the stack the way `crate::walk` does, over a sorted listing, and+ /// return the surviving root-relative file paths.+ fn survivors(root: &Path, opts: IgnoreOptions) -> (Vec<String>, IgnoreReport) {+ let root = root.canonicalize().unwrap();+ let mut stack = IgnoreStack::new(&root, opts);+ let mut kept = Vec::new();+ let mut it = walkdir::WalkDir::new(&root).into_iter();+ while let Some(next) = it.next() {+ let Ok(entry) = next else { continue };+ let is_dir = entry.file_type().is_dir();+ if entry.depth() > 0+ && entry+ .file_name()+ .to_str()+ .is_some_and(|n| n.starts_with('.'))+ {+ stack.record_hidden(entry.path(), is_dir);+ if is_dir {+ it.skip_current_dir();+ }+ continue;+ }+ stack.truncate_to(entry.depth());+ if is_dir {+ if entry.depth() > 0 && stack.skip_dir(entry.path()).is_some() {+ it.skip_current_dir();+ continue;+ }+ stack.enter_dir(entry.path(), entry.depth());+ continue;+ }+ if !entry.file_type().is_file() {+ continue;+ }+ if stack.skip_file(entry.path()).is_some() {+ continue;+ }+ kept.push(+ entry+ .path()+ .strip_prefix(&root)+ .unwrap()+ .to_string_lossy()+ .replace('\\', "/"),+ );+ }+ kept.sort();+ (kept, stack.report)+ }++ fn write(root: &Path, rel: &str, body: &str) {+ let path = root.join(rel);+ fs::create_dir_all(path.parent().unwrap()).unwrap();+ fs::write(path, body).unwrap();+ }++ #[test]+ fn root_gitignore_prunes_and_negation_re_includes() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ write(root, ".gitignore", "*.log\n!keep.log\nbuild/\n");+ write(root, "a.log", "x");+ write(root, "keep.log", "x");+ write(root, "src/main.rs", "x");+ write(root, "src/nested.log", "x");+ write(root, "build/out.bin", "x");++ let (kept, report) = survivors(root, IgnoreOptions::default());+ assert_eq!(kept, vec!["keep.log", "src/main.rs"], "{kept:?}");+ // `*.log` has no slash, so it matches at any depth — git's rule.+ assert!(+ report.by_rule.contains_key(".gitignore:*.log"),+ "{report:?}"+ );+ assert!(+ report.by_rule.contains_key(".gitignore:build/"),+ "{report:?}"+ );+ assert_eq!(report.dirs_pruned, 1);+ assert_eq!(+ report.by_rule[".gitignore:*.log"].files, 2,+ "a.log and src/nested.log"+ );+ // 3, not 2: `.gitignore` is itself a dotfile, so the pre-existing+ // hidden-file rule accounts for it in the same report.+ assert_eq!(report.files_skipped, 3, "{report:?}");+ assert_eq!(report.by_rule[HIDDEN_RULE].files, 1, "{report:?}");+ }++ #[test]+ fn a_nested_gitignore_outranks_the_root_one() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ write(root, ".gitignore", "*.log\n");+ write(root, "logs/.gitignore", "!*.log\n");+ write(root, "top.log", "x");+ write(root, "logs/kept.log", "x");++ let (kept, _) = survivors(root, IgnoreOptions::default());+ assert_eq!(kept, vec!["logs/kept.log"], "{kept:?}");+ }++ #[test]+ fn xerjignore_outranks_gitignore_at_any_depth() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ // First line: git tracks it, the user does not want it in the index.+ // Second: git ignores it, the user *does* want it — and a root+ // .xerjignore outranks a deep .gitignore, which is ripgrep's rule.+ write(+ root,+ ".xerjignore",+ "tracked-but-huge.csv\n!deep/wanted.bin\n",+ );+ write(root, "deep/.gitignore", "*.bin\n");+ write(root, "tracked-but-huge.csv", "x");+ write(root, "deep/wanted.bin", "x");+ write(root, "deep/other.bin", "x");+ write(root, "ok.txt", "x");++ let (kept, report) = survivors(root, IgnoreOptions::default());+ assert_eq!(kept, vec!["deep/wanted.bin", "ok.txt"], "{kept:?}");+ assert!(+ report+ .by_rule+ .contains_key(".xerjignore:tracked-but-huge.csv"),+ "{report:?}"+ );+ }++ #[test]+ fn defaults_prune_build_output_that_no_ignore_file_mentions() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ write(root, "index.js", "x");+ write(root, "node_modules/left-pad/index.js", "x");+ write(root, "target/debug/huge.rlib", "x");+ write(root, "__pycache__/mod.cpython-311.pyc", "x");++ let (kept, report) = survivors(root, IgnoreOptions::default());+ assert_eq!(kept, vec!["index.js"], "{kept:?}");+ assert_eq!(report.dirs_pruned, 3);+ assert!(+ report.by_rule.contains_key("<built-in>:node_modules/"),+ "{report:?}"+ );++ let (all, _) = survivors(+ root,+ IgnoreOptions {+ defaults: false,+ ..IgnoreOptions::default()+ },+ );+ assert_eq!(all.len(), 4, "--no-default-ignores must keep them: {all:?}");+ }++ #[test]+ fn an_ignore_file_outranks_a_default() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ // A Go project that commits its vendor tree.+ write(root, ".gitignore", "!vendor/\n");+ write(root, "vendor/lib/lib.go", "x");+ let (kept, _) = survivors(root, IgnoreOptions::default());+ assert_eq!(kept, vec!["vendor/lib/lib.go"], "{kept:?}");+ }++ #[test]+ fn no_ignore_disables_every_rule_but_not_the_dotfile_skip() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ write(root, ".gitignore", "*.log\n");+ write(root, "a.log", "x");+ write(root, "node_modules/x.js", "x");+ write(root, ".env", "SECRET=1");++ let (kept, report) = survivors(root, IgnoreOptions::off());+ assert_eq!(kept, vec!["a.log", "node_modules/x.js"], "{kept:?}");+ assert!(!kept.iter().any(|k| k == ".env"), "{kept:?}");+ assert_eq!(report.by_rule.get(HIDDEN_RULE).unwrap().files, 2);+ }++ #[test]+ fn git_info_exclude_is_honoured() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ write(root, ".git/info/exclude", "scratch.txt\n");+ write(root, "scratch.txt", "x");+ write(root, "real.txt", "x");+ let (kept, _) = survivors(root, IgnoreOptions::default());+ assert_eq!(kept, vec!["real.txt"], "{kept:?}");+ }++ #[test]+ fn an_ignored_root_is_indexed_anyway_and_says_why() {+ let dir = tempfile::TempDir::new().unwrap();+ let repo = dir.path().join("repo");+ fs::create_dir_all(repo.join(".git")).unwrap();+ write(&repo, ".gitignore", "generated/\n");+ write(&repo, "generated/out.o", "x");++ let root = repo.join("generated");+ let (kept, report) = survivors(&root, IgnoreOptions::default());+ assert_eq!(kept, vec!["out.o"], "an explicit root must be indexed");+ let reason = report.root_exemption.expect("must explain itself");+ assert!(reason.contains("generated/"), "{reason}");+ assert!(reason.contains(".gitignore"), "{reason}");+ }++ #[test]+ fn a_root_named_like_a_default_is_indexed_anyway_and_says_why() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path().join("node_modules");+ fs::create_dir_all(&root).unwrap();+ write(&root, "left-pad/index.js", "x");+ let (kept, report) = survivors(&root, IgnoreOptions::default());+ assert_eq!(kept, vec!["left-pad/index.js"], "{kept:?}");+ let reason = report.root_exemption.expect("must explain itself");+ assert!(reason.contains("node_modules/"), "{reason}");+ }++ #[test]+ fn deep_count_is_off_by_default_and_counts_under_dry_run() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ write(root, "keep.txt", "x");+ for i in 0..7 {+ write(root, &format!("node_modules/pkg/f{i}.js"), "x");+ }++ let (_, cheap) = survivors(root, IgnoreOptions::default());+ assert_eq!(cheap.files_inside_pruned_dirs, 0);+ assert!(!cheap.deep_count_ran);+ assert!(+ cheap+ .summary_lines()+ .iter()+ .any(|l| l.contains("contents never walked")),+ "{:?}",+ cheap.summary_lines()+ );++ let (_, deep) = survivors(root, IgnoreOptions::default().with_deep_count(true));+ assert_eq!(deep.files_inside_pruned_dirs, 7);+ assert!(deep.deep_count_complete);+ assert!(+ deep.summary_lines().iter().any(|l| l.contains("7 files")),+ "{:?}",+ deep.summary_lines()+ );+ }++ /// Regression, order-independent: a depth-first walk interleaves files and+ /// subdirectories, so a nested `.gitignore` must be off the stack again by+ /// the time the walk hands back a file from the parent directory. Driven+ /// through the exact call sequence `crate::walk` makes, because readdir+ /// order is not something a test can pin.+ #[test]+ fn a_nested_ignore_file_never_judges_its_parents_files() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path().canonicalize().unwrap();+ write(&root, "a/.gitignore", "*.txt\n");+ write(&root, "a/inside.txt", "x");+ write(&root, "b.txt", "x");++ let mut stack = IgnoreStack::new(&root, IgnoreOptions::default());+ stack.truncate_to(0);+ stack.enter_dir(&root, 0);+ // …descend into a/ …+ stack.truncate_to(1);+ stack.enter_dir(&root.join("a"), 1);+ stack.truncate_to(2);+ assert!(+ stack+ .skip_file(&root.join("a").join("inside.txt"))+ .is_some(),+ "a/.gitignore must apply inside a/"+ );+ // …and come back out to a sibling file of a/.+ stack.truncate_to(1);+ assert!(+ stack.skip_file(&root.join("b.txt")).is_none(),+ "a/.gitignore leaked onto the parent's files"+ );+ }++ /// A broken ignore file must never be able to make files disappear+ /// silently: it is reported, and its only possible effect is that we index+ /// MORE than the user asked for, never less.+ #[test]+ fn an_unreadable_ignore_file_warns_and_indexes_more_not_less() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ fs::write(root.join(".gitignore"), b"\xff\xfe not text\n*.txt\n").unwrap();+ fs::write(root.join("a.txt"), "x").unwrap();+ let (kept, report) = survivors(root, IgnoreOptions::default());+ assert_eq!(+ kept,+ vec!["a.txt"],+ "an ignore file we cannot read must not hide anything"+ );+ assert!(!report.warnings.is_empty(), "must warn: {report:?}");+ assert!(+ report+ .summary_lines()+ .iter()+ .any(|l| l.contains(".gitignore")),+ "the warning must reach the user: {:?}",+ report.summary_lines()+ );+ }+ }diff --cc engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 000000000,ae70c788d..ab7aa2474mode 000000,100644..100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@@ -1,0 -1,1277 +1,1283 @@@+ //! Real-HTTP end-to-end coverage for incremental corpus generations.+ //!+ //! The endpoint below is deliberately stateful. It applies bulk index/update+ //! actions, delete-by-query, and exact count searches so `run_index` traverses+ //! the same HTTP client and validation barriers as a real server.++ use super::*;+ use std::collections::HashMap;+ use std::fs;+ use std::io::{BufRead, BufReader, Read, Write};+ use std::net::{TcpListener, TcpStream};+ use std::path::Path;+ use std::sync::{Arc, Mutex};+ use std::thread;++ static HTTP_E2E_LOCK: Mutex<()> = Mutex::new(());++ #[derive(Default)]+ struct HttpState {+ docs: HashMap<(String, String), Value>,+ requests: Vec<(String, String)>,+ data_bulk_requests: usize,+ fail_next_data_bulk: bool,+ /// Apply the first half of the next data bulk's items and *then* report+ /// `errors: true`. This is the dangerous backend shape the legacy+ /// replacement transaction was hardened against (visibility changed, but+ /// the durable record must not be): `no_graph` corpora take the generated+ /// executor now, so the shape has to be proven here too.+ partially_apply_next_data_bulk: bool,+ fail_embedding_identity: bool,+ stop: bool,+ embedding_identity_sha256: String,+ saw_dataset_mapping_update: bool,+ saw_catalog_mapping_update: bool,+ }++ struct HttpEndpoint {+ url: String,+ state: Arc<Mutex<HttpState>>,+ join: Option<thread::JoinHandle<()>>,+ }++ impl HttpEndpoint {+ fn start() -> Self {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ listener.set_nonblocking(true).unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let state = Arc::new(Mutex::new(HttpState {+ embedding_identity_sha256: "a".repeat(64),+ ..HttpState::default()+ }));+ let server_state = Arc::clone(&state);+ let join = thread::spawn(move || loop {+ match listener.accept() {- Ok((stream, _)) => handle_http(stream, &server_state),++ Ok((stream, _)) => {++ // BSD/macOS: accepted sockets inherit the listener's++ // O_NONBLOCK; the handler does blocking reads.++ stream.set_nonblocking(false).unwrap();++ handle_http(stream, &server_state)++ }+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {+ if server_state.lock().unwrap().stop {+ break;+ }+ thread::yield_now();+ }+ Err(error) => panic!("HTTP test endpoint accept failed: {error}"),+ }+ });+ Self {+ url,+ state,+ join: Some(join),+ }+ }++ fn data_docs(&self) -> Vec<Value> {+ self.state+ .lock()+ .unwrap()+ .docs+ .iter()+ .filter(|((index, _), _)| index != catalog::CATALOG_INDEX)+ .map(|(_, doc)| doc.clone())+ .collect()+ }++ fn data_bulk_requests(&self) -> usize {+ self.state.lock().unwrap().data_bulk_requests+ }+ }++ impl Drop for HttpEndpoint {+ fn drop(&mut self) {+ self.state.lock().unwrap().stop = true;+ let _ = TcpStream::connect(self.url.trim_start_matches("http://"));+ self.join.take().unwrap().join().unwrap();+ }+ }++ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {+ let mut reader = BufReader::new(stream.try_clone().unwrap());+ let mut request_line = String::new();+ if reader.read_line(&mut request_line).unwrap() == 0 {+ return;+ }+ let mut content_length = 0usize;+ loop {+ let mut line = String::new();+ reader.read_line(&mut line).unwrap();+ if line == "\r\n" || line.is_empty() {+ break;+ }+ if let Some(value) = line+ .to_ascii_lowercase()+ .strip_prefix("content-length:")+ .map(str::trim)+ {+ content_length = value.parse().unwrap();+ }+ }+ let mut body = vec![0; content_length];+ reader.read_exact(&mut body).unwrap();+ let mut parts = request_line.split_whitespace();+ let method = parts.next().unwrap_or("");+ let path = parts.next().unwrap_or("");+ state+ .lock()+ .unwrap()+ .requests+ .push((method.to_owned(), path.to_owned()));++ let (status, response) = if method == "GET" && path == "/v1/embedding/identity" {+ let locked = state.lock().unwrap();+ if locked.fail_embedding_identity {+ (+ 500,+ json!({"error": {+ "type": "injected_identity_failure",+ "reason": "genesis recovery boundary"+ }}),+ )+ } else {+ (+ 200,+ json!({"data": {+ "version": 1,+ "backend": "lexical",+ "identity_sha256": locked.embedding_identity_sha256.clone(),+ "dimensions": 384,+ "semantic_contract": "semantic_text-derived-vector.v1",+ "resumable": true+ }, "took_ms": 0, "request_id": "incremental-http-test"}),+ )+ }+ } else if method == "POST" && path == "/_bulk" {+ let locked = state.lock().unwrap();+ assert!(+ locked.saw_dataset_mapping_update && locked.saw_catalog_mapping_update,+ "bulk publication preceded exact generation mapping updates"+ );+ drop(locked);+ bulk_http(&body, state)+ } else if method == "POST" && path.contains("/_delete_by_query") {+ let deleted = delete_http(path, &body, state);+ (200, json!({"deleted": deleted, "failures": []}))+ } else if method == "POST" && path.ends_with("/_search") {+ (200, search_http(path, &body, state))+ } else if method == "GET" && path.ends_with("/_count") {+ let index = path.trim_start_matches('/').trim_end_matches("/_count");+ let count = state+ .lock()+ .unwrap()+ .docs+ .keys()+ .filter(|(candidate, _)| candidate == index)+ .count();+ (200, json!({"count": count}))+ } else if method == "PUT" {+ let value: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);+ let mut locked = state.lock().unwrap();+ if path.ends_with("/_mapping") {+ assert!(value.get("properties").is_some());+ assert!(value.get("mappings").is_none());+ if path.starts_with(&format!("/{}/", catalog::CATALOG_INDEX)) {+ locked.saw_catalog_mapping_update = true;+ } else {+ locked.saw_dataset_mapping_update = true;+ }+ } else {+ assert!(value.pointer("/mappings/properties").is_some());+ assert!(value.get("properties").is_none());+ }+ (200, json!({"acknowledged": true}))+ } else {+ // Readiness, index creation/mapping, and refresh.+ (200, json!({"acknowledged": true}))+ };+ let bytes = response.to_string();+ write!(+ stream,+ "HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",+ if status == 200 {+ "OK"+ } else {+ "Internal Server Error"+ },+ bytes.len(),+ bytes+ )+ .unwrap();+ }++ fn bulk_http(body: &[u8], state: &Arc<Mutex<HttpState>>) -> (u16, Value) {+ let lines: Vec<&[u8]> = body+ .split(|byte| *byte == b'\n')+ .filter(|line| !line.is_empty())+ .collect();+ let is_data = lines.iter().step_by(2).any(|line| {+ serde_json::from_slice::<Value>(line)+ .ok()+ .and_then(|action| {+ action+ .pointer("/index/_index")+ .or_else(|| action.pointer("/update/_index"))+ .and_then(Value::as_str)+ .map(|index| index != catalog::CATALOG_INDEX)+ })+ .unwrap_or(false)+ });+ let mut locked = state.lock().unwrap();+ if is_data {+ locked.data_bulk_requests += 1;+ if std::mem::take(&mut locked.fail_next_data_bulk) {+ return (+ 200,+ json!({+ "errors": true,+ "items": [{"index": {+ "status": 500,+ "error": {+ "type": "injected_failure",+ "reason": "leave durable generation pending"+ }+ }}]+ }),+ );+ }+ }+ // Half-applied bulk: apply the leading actions, then report failure.+ let applied_limit = if is_data && std::mem::take(&mut locked.partially_apply_next_data_bulk) {+ Some(lines.len() / 2)+ } else {+ None+ };+ let mut cursor = 0;+ while cursor < lines.len() {+ if applied_limit.is_some_and(|limit| cursor >= limit) {+ break;+ }+ let action: Value = serde_json::from_slice(lines[cursor]).unwrap();+ cursor += 1;+ if let Some(meta) = action.get("delete") {+ let index = meta["_index"].as_str().unwrap().to_owned();+ let id = meta["_id"].as_str().unwrap().to_owned();+ locked.docs.remove(&(index, id));+ continue;+ }+ let payload: Value = serde_json::from_slice(lines[cursor]).unwrap();+ cursor += 1;+ if let Some(meta) = action.get("index") {+ let index = meta["_index"].as_str().unwrap().to_owned();+ let id = meta["_id"].as_str().unwrap().to_owned();+ locked.docs.insert((index, id), payload);+ } else if let Some(meta) = action.get("update") {+ let index = meta["_index"].as_str().unwrap().to_owned();+ let id = meta["_id"].as_str().unwrap().to_owned();+ let patch = payload["doc"].as_object().unwrap();+ let target = locked.docs.get_mut(&(index, id)).unwrap();+ let target = target.as_object_mut().unwrap();+ target.extend(patch.clone());+ }+ }+ if applied_limit.is_some() {+ return (+ 200,+ json!({+ "errors": true,+ "items": [{"index": {+ "status": 500,+ "error": {+ "type": "injected_failure",+ "reason": "partial bulk applied before failing"+ }+ }}]+ }),+ );+ }+ (200, json!({"errors": false, "items": []}))+ }++ fn delete_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> usize {+ let index = path+ .trim_start_matches('/')+ .split("/_delete_by_query")+ .next()+ .unwrap();+ let query: Value = serde_json::from_slice(body).unwrap();+ let term = query+ .pointer("/query/term")+ .and_then(Value::as_object)+ .and_then(|term| term.iter().next())+ .map(|(field, value)| (field.clone(), value.clone()));+ let mut locked = state.lock().unwrap();+ let before = locked.docs.len();+ locked.docs.retain(|(candidate, _), doc| {+ if candidate != index {+ return true;+ }+ let Some((field, expected)) = &term else {+ return false;+ };+ doc.get(field) != Some(expected)+ });+ before - locked.docs.len()+ }++ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value {+ let indices = path+ .trim_start_matches('/')+ .trim_end_matches("/_search")+ .split(',')+ .collect::<Vec<_>>();+ let query: Value = serde_json::from_slice(body).unwrap();+ let term = query+ .pointer("/query/term")+ .or_else(|| query.pointer("/query/bool/must/0/term"))+ .and_then(Value::as_object)+ .and_then(|term| term.iter().next());+ let terms = query+ .pointer("/query/terms")+ .and_then(Value::as_object)+ .and_then(|terms| terms.iter().next());+ let filters = query+ .pointer("/query/bool/filter")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default();+ let exists = query+ .pointer("/query/bool/must/1/exists/field")+ .and_then(Value::as_str);+ let locked = state.lock().unwrap();+ let matching: Vec<_> = locked+ .docs+ .iter()+ .filter(|((index, _), doc)| {+ indices.contains(&index.as_str())+ && term+ .map(|(field, value)| doc.get(field) == Some(value))+ .unwrap_or(true)+ && terms+ .map(|(field, values)| {+ values.as_array().is_some_and(|values| {+ values.iter().any(|value| doc.get(field) == Some(value))+ })+ })+ .unwrap_or(true)+ && filters.iter().all(|filter| {+ filter+ .get("term")+ .and_then(Value::as_object)+ .and_then(|term| term.iter().next())+ .is_none_or(|(field, value)| doc.get(field) == Some(value))+ })+ && exists.map(|field| doc.get(field).is_some()).unwrap_or(true)+ })+ .collect();+ let hits = matching+ .iter()+ .map(|((_, id), source)| json!({"_id": id, "_source": source}))+ .collect::<Vec<_>>();+ json!({+ "hits": {+ "total": {"value": matching.len(), "relation": "eq"},+ "hits": hits+ },+ "aggregations": {}+ })+ }++ fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {+ IndexCfg {+ root: root.to_owned(),++ stub_globs: Vec::new(),+ url: url.to_owned(),+ api_key: None,+ workers: 1,+ scan_workers: 1,+ pdf_workers: 1,+ resource_notes: Vec::new(),+ pdf_timeout_secs: 30,+ bulk_mb: 1,+ bulk_timeout_secs: 30,+ snapshot_max_bytes: 64 << 30,+ prefix: "incremental-http".into(),+ state_dir: Some(state_dir.to_owned()),+ fresh: false,+ follow_symlinks: false,+ ignore: crate::ignore_rules::IgnoreOptions::default(),+ max_file_gb: 1,+ sample: 50,+ no_semantic: !semantic,+ brain: None,+ no_graph: true,+ dry_run: false,+ json: false,+ quiet: true,+ progress: crate::progress::ProgressMode::None,+ progress_interval: None,+ }+ }++ fn journal_events(state_dir: &Path, kind: &str) -> usize {+ fs::read_to_string(state_dir.join("journal.ndjson"))+ .unwrap()+ .lines()+ .filter_map(|line| serde_json::from_str::<Value>(line).ok())+ .filter(|event| event.get("kind").and_then(Value::as_str) == Some(kind))+ .count()+ }++ fn paths(docs: &[Value]) -> Vec<String> {+ let mut paths: Vec<_> = docs+ .iter()+ .filter_map(|doc| doc.get("ax_path").and_then(Value::as_str))+ .map(str::to_owned)+ .collect();+ paths.sort();+ paths.dedup();+ paths+ }++ fn snapshot_names(state_dir: &Path) -> Vec<String> {+ let mut names: Vec<String> = fs::read_dir(state_dir.join("sync-snapshots"))+ .map(|entries| {+ entries+ .filter_map(Result::ok)+ .filter(|entry| {+ entry.file_type().is_ok_and(|kind| kind.is_dir())+ && !entry.file_name().to_string_lossy().starts_with('.')+ })+ .map(|entry| entry.file_name().to_string_lossy().into_owned())+ .collect()+ })+ .unwrap_or_default();+ names.sort();+ names+ }++ fn final_snapshot_count(state_dir: &Path) -> usize {+ fs::read_dir(state_dir.join("sync-snapshots"))+ .map(|entries| {+ entries+ .filter_map(Result::ok)+ .filter(|entry| {+ entry.file_type().is_ok_and(|kind| kind.is_dir())+ && !entry.file_name().to_string_lossy().starts_with('.')+ })+ .count()+ })+ .unwrap_or(0)+ }++ #[test]+ fn genesis_recovers_after_snapshot_budget_failure_before_sync_begin() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ config.snapshot_max_bytes = 1;++ let error = run_index(config.clone()).unwrap_err();+ assert!(format!("{error:#}").contains("snapshot source footprint"));+ assert_eq!(journal_events(state_dir.path(), "sync_bootstrap"), 1);+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 0);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 0);+ assert_eq!(endpoint.data_bulk_requests(), 0);++ config.snapshot_max_bytes = 64 << 30;+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(journal_events(state_dir.path(), "sync_bootstrap"), 1);+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 1);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);+ assert_eq!(endpoint.data_docs().len(), 1);+ }++ #[test]+ fn no_semantic_generation_does_not_require_embedding_identity_endpoint() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ endpoint.state.lock().unwrap().fail_embedding_identity = true;++ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(journal_events(state_dir.path(), "sync_bootstrap"), 1);+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 1);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);+ assert_eq!(journal_events(state_dir.path(), "finish"), 1);+ assert_eq!(final_snapshot_count(state_dir.path()), 1);+ assert_eq!(endpoint.data_docs().len(), 1);+ }++ #[test]+ fn initial_generation_and_noop_are_end_to_end_idempotent() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n2,beta\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);++ let (code, summary) = run_index_report(config.clone()).unwrap();+ assert_eq!(code, 0);+ let summary = summary.expect("generated run must return its committed run projection");+ assert_eq!(summary["generation"], 1);+ assert_eq!(summary["records_total"], 2);+ let first_docs = endpoint.data_docs();+ assert_eq!(first_docs.len(), 2);+ assert_eq!(paths(&first_docs), ["a.csv"]);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);+ let root = corpus.path().canonicalize().unwrap();+ let preflight = state::Journal::preflight(+ state_dir.path(),+ &root.to_string_lossy(),+ &endpoint.url,+ "incremental-http",+ false,+ )+ .unwrap();+ assert!(+ preflight.committed_manifest.is_some(),+ "sync_commit must be visible to the next preflight"+ );+ drop(preflight);++ let bulks = endpoint.data_bulk_requests();+ let (_, noop_summary) = run_index_report(config).unwrap();+ assert_eq!(noop_summary.unwrap()["generation"], 1);+ assert_eq!(endpoint.data_docs(), first_docs);+ assert_eq!(endpoint.data_bulk_requests(), bulks);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);+ assert_eq!(journal_events(state_dir.path(), "finish"), 2);+ }++ #[test]+ fn add_change_delete_and_rename_converge_over_real_http() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(corpus.path().join("b.csv"), "id,value\n2,bravo\n").unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv", "b.csv"]);++ fs::write(+ corpus.path().join("a.csv"),+ "id,value\n1,alpha-new\n3,charlie\n",+ )+ .unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ assert_eq!(endpoint.data_docs().len(), 3);++ fs::remove_file(corpus.path().join("b.csv")).unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);++ fs::rename(+ corpus.path().join("a.csv"),+ corpus.path().join("renamed.csv"),+ )+ .unwrap();+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(paths(&endpoint.data_docs()), ["renamed.csv"]);+ assert_eq!(endpoint.data_docs().len(), 2);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 5);+ }++ #[test]+ fn pending_generation_replays_sealed_source_after_live_source_mutates() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let source = corpus.path().join("a.csv");+ fs::write(&source, "id,value\n1,committed\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(&source, "id,value\n1,sealed-pending\n2,sealed-second\n").unwrap();+ endpoint.state.lock().unwrap().fail_next_data_bulk = true;+ assert!(run_index(config.clone()).is_err());+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 2);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);++ fs::write(&source, "id,value\n9,mutated-after-sync-begin\n").unwrap();+ assert_eq!(run_index(config).unwrap(), 0);+ let docs = endpoint.data_docs();+ assert_eq!(docs.len(), 2);+ let rendered = serde_json::to_string(&docs).unwrap();+ assert!(rendered.contains("sealed-pending"), "{rendered}");+ assert!(rendered.contains("sealed-second"), "{rendered}");+ assert!(!rendered.contains("mutated-after-sync-begin"), "{rendered}");+ }++ /// Crash/resume with a *partially applied* data bulk, on `--no-graph`.+ ///+ /// `failure_resume_http_tests` used to run its whole injected-failure matrix+ /// with `no_graph: true`; since `--no-graph` now takes the generated executor,+ /// that module keeps the legacy graph-enabled transaction and this test carries+ /// the no-graph half of the coverage. The endpoint applies half of a bulk and+ /// *then* reports failure — visibility has changed, so the generation must stay+ /// uncommitted, and the retry must converge on exactly the new content with no+ /// duplicate or stale document surviving the half-applied attempt.+ #[test]+ fn partially_applied_data_bulk_leaves_the_generation_pending_and_the_retry_converges() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let source = corpus.path().join("rows.csv");+ fs::write(&source, "id,value\n1,first\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);++ fs::write(+ &source,+ "id,value\n1,replaced\n2,second\n3,third\n4,fourth\n",+ )+ .unwrap();+ endpoint+ .state+ .lock()+ .unwrap()+ .partially_apply_next_data_bulk = true;+ assert!(+ run_index(config.clone()).is_err(),+ "a partially applied bulk must fail the run, not be absorbed"+ );+ assert_eq!(+ journal_events(state_dir.path(), "sync_begin"),+ 2,+ "the generation was begun"+ );+ assert_eq!(+ journal_events(state_dir.path(), "sync_commit"),+ 1,+ "a partially applied bulk must never commit the generation"+ );++ // Retry: the sealed source replays, and the destination converges exactly.+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);+ let docs = endpoint.data_docs();+ assert_eq!(paths(&docs), ["rows.csv"]);+ assert_eq!(+ docs.len(),+ 4,+ "exactly the four new records survive — no half-applied duplicate, no stale first record"+ );+ let rendered = serde_json::to_string(&docs).unwrap();+ assert!(rendered.contains("replaced"), "{rendered}");+ assert!(rendered.contains("fourth"), "{rendered}");+ assert!(+ !rendered.contains("\"value\":\"first\""),+ "the replaced record must not survive: {rendered}"+ );+ }++ #[test]+ fn pending_semantic_identity_drift_rejects_before_any_further_bulk() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let source = corpus.path().join("report.txt");+ fs::write(+ &source,+ "Initial quarterly report discusses durable subscription revenue and operating income.",+ )+ .unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, true);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(+ &source,+ "Updated quarterly report discusses materially stronger subscription revenue and margin.",+ )+ .unwrap();+ endpoint.state.lock().unwrap().fail_next_data_bulk = true;+ assert!(run_index(config.clone()).is_err());+ let bulks_before = endpoint.data_bulk_requests();+ endpoint.state.lock().unwrap().embedding_identity_sha256 = "b".repeat(64);++ let error = run_index(config).unwrap_err();+ let message = format!("{error:#}");+ assert!(+ message.contains("different embedding execution identity"),+ "{message}"+ );+ assert!(+ message.contains("no remote mutation was attempted"),+ "{message}"+ );+ assert_eq!(+ endpoint.data_bulk_requests(),+ bulks_before,+ "identity drift must be rejected before another data bulk"+ );+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);+ }++ /// `--fresh` must be refused before it can destroy generation authority.+ ///+ /// `Journal::open_after_preflight` deletes `journal.ndjson` whenever `fresh` is+ /// set, and `gc_snapshots` then sees an empty protected set and removes every+ /// sealed snapshot. Both run inside the generated branches, ahead of the legacy+ /// plan gate, so without the preflight refusal a single `--fresh` erases the+ /// committed manifest and every sealed source the next run needs.+ #[test]+ fn fresh_cannot_destroy_a_committed_generation() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ fs::write(corpus.path().join("b.csv"), "id,value\n2,bravo\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ let committed_docs = endpoint.data_docs();+ assert_eq!(paths(&committed_docs), ["a.csv", "b.csv"]);++ let journal_path = state_dir.path().join("journal.ndjson");+ let journal_before = fs::read(&journal_path).unwrap();+ let snapshots_before = snapshot_names(state_dir.path());+ assert_eq!(snapshots_before.len(), 1, "{snapshots_before:?}");+ let requests_before = endpoint.state.lock().unwrap().requests.len();++ // The folder also changed, which is exactly when a user reaches for+ // --fresh.+ fs::remove_file(corpus.path().join("b.csv")).unwrap();+ config.fresh = true;+ let error = run_index(config.clone()).unwrap_err();+ let message = format!("{error:#}");+ assert!(+ message.contains("`--fresh` cannot discard committed corpus generation 1"),+ "the refusal must name the durable state that blocks it: {message}"+ );+ assert!(+ message.contains("without `--fresh`"),+ "the refusal must point at the incremental path that does work: {message}"+ );++ assert_eq!(+ fs::read(&journal_path).unwrap(),+ journal_before,+ "a refused --fresh must not rewrite the journal"+ );+ assert_eq!(+ snapshot_names(state_dir.path()),+ snapshots_before,+ "a refused --fresh must not garbage-collect sealed snapshots"+ );+ let attempted = endpoint.state.lock().unwrap().requests[requests_before..].to_vec();+ assert_eq!(+ attempted,+ [("GET".to_owned(), "/".to_owned())],+ "a refused --fresh may perform only the endpoint-readiness GET"+ );++ // The surviving authority still reconciles the real folder change.+ config.fresh = false;+ let (code, summary) = run_index_report(config).unwrap();+ assert_eq!(code, 0);+ assert_eq!(summary.unwrap()["generation"], 2);+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);+ }++ /// Every refusal on the generated path now sends the operator to an isolated+ /// rebuild, so that recipe has to work. `--fresh` is no longer a rebuild-in-+ /// place escape hatch, and a new `--prefix` alone is not enough when the state+ /// directory was named explicitly: `preflight` refuses a journal recorded for a+ /// different prefix before the run reaches anything else.+ #[test]+ fn the_isolated_rebuild_the_refusals_recommend_is_followable() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let committed_state = tempfile::tempdir().unwrap();+ let rebuild_state = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), committed_state.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ // A new --prefix on its own cannot rebuild: the recorded journal owns it.+ let mut prefix_only = config.clone();+ prefix_only.prefix = "incremental-http-rebuild".into();+ let message = format!("{:#}", run_index(prefix_only).unwrap_err());+ assert!(message.contains("was created for root="), "{message}");++ // A new --state-dir and a new --prefix together do rebuild, and leave the+ // original generation and its sealed snapshot untouched.+ let journal_before = fs::read(committed_state.path().join("journal.ndjson")).unwrap();+ let snapshots_before = snapshot_names(committed_state.path());+ let mut rebuild = cfg(corpus.path(), rebuild_state.path(), &endpoint.url, false);+ rebuild.prefix = "incremental-http-rebuild".into();+ let (code, summary) = run_index_report(rebuild).unwrap();+ assert_eq!(code, 0);+ assert_eq!(summary.unwrap()["generation"], 1);+ assert_eq!(journal_events(rebuild_state.path(), "sync_commit"), 1);+ assert_eq!(+ fs::read(committed_state.path().join("journal.ndjson")).unwrap(),+ journal_before,+ "an isolated rebuild must not touch the original journal"+ );+ assert_eq!(+ snapshot_names(committed_state.path()),+ snapshots_before,+ "an isolated rebuild must not touch the original sealed snapshots"+ );+ }++ /// The same refusal on a crashed generation, where the journal additionally+ /// holds the only record of the pending transaction and its sealed source.+ #[test]+ fn fresh_cannot_destroy_a_pending_generation() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let source = corpus.path().join("a.csv");+ fs::write(&source, "id,value\n1,committed\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(&source, "id,value\n1,sealed-pending\n2,sealed-second\n").unwrap();+ endpoint.state.lock().unwrap().fail_next_data_bulk = true;+ assert!(run_index(config.clone()).is_err());+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 2);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);++ let journal_path = state_dir.path().join("journal.ndjson");+ let journal_before = fs::read(&journal_path).unwrap();+ let snapshots_before = snapshot_names(state_dir.path());+ assert_eq!(snapshots_before.len(), 2, "{snapshots_before:?}");+ let requests_before = endpoint.state.lock().unwrap().requests.len();++ config.fresh = true;+ let error = run_index(config.clone()).unwrap_err();+ let message = format!("{error:#}");+ assert!(+ message.contains("`--fresh` cannot discard an uncommitted pending corpus generation"),+ "the refusal must name the durable state that blocks it: {message}"+ );+ assert_eq!(+ fs::read(&journal_path).unwrap(),+ journal_before,+ "a refused --fresh must not rewrite a pending journal"+ );+ assert_eq!(+ snapshot_names(state_dir.path()),+ snapshots_before,+ "a refused --fresh must not delete the pending sealed source"+ );+ let attempted = endpoint.state.lock().unwrap().requests[requests_before..].to_vec();+ assert_eq!(+ attempted,+ [("GET".to_owned(), "/".to_owned())],+ "a refused --fresh may perform only the endpoint-readiness GET"+ );++ // The pending transaction is still resumable from its sealed source.+ config.fresh = false;+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);+ let rendered = serde_json::to_string(&endpoint.data_docs()).unwrap();+ assert!(rendered.contains("sealed-pending"), "{rendered}");+ assert!(rendered.contains("sealed-second"), "{rendered}");+ }++ /// Two runs started in the same process and the same UTC second must not share+ /// a `run_id`.+ ///+ /// The generation catalog keys its managed documents on `run_id`, so a+ /// collision makes the second run read the first run's documents back as its+ /// own generation. `xerj brain` calls `run_index_report` more than once per+ /// process, so this is reachable without any concurrency.+ #[test]+ fn same_second_runs_in_one_process_do_not_share_a_generation_identity() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let first_state = tempfile::tempdir().unwrap();+ let second_state = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();++ assert_eq!(+ run_index(cfg(corpus.path(), first_state.path(), &endpoint.url, false)).unwrap(),+ 0+ );+ let second = run_index_report(cfg(+ corpus.path(),+ second_state.path(),+ &endpoint.url,+ false,+ ));+ let (code, summary) = second.expect("a second run in the same second must commit");+ assert_eq!(code, 0);+ assert_eq!(summary.unwrap()["generation"], 1);++ let run_id = |state_dir: &Path| -> String {+ fs::read_to_string(state_dir.join("journal.ndjson"))+ .unwrap()+ .lines()+ .filter_map(|line| serde_json::from_str::<Value>(line).ok())+ .find_map(|event| {+ event+ .get("run_id")+ .and_then(Value::as_str)+ .map(str::to_owned)+ })+ .expect("journal records a run_id")+ };+ assert_ne!(run_id(first_state.path()), run_id(second_state.path()));+ }++ /// An upsert must clear the desired content identity as well as the committed+ /// one before it replays.+ ///+ /// A byte-identical retry of the same prepared stream is idempotent on its own,+ /// because `ids::doc_id` is derived from dataset slug, content id and locator.+ /// The guard earns its keep when a prior aborted attempt left documents under+ /// the desired `ax_file` whose ids the current stream does not overwrite — an+ /// earlier generation of the same content under a different dataset+ /// assignment, or a partially rolled-back attempt. This test models that+ /// survivor directly, because the exact record validation at the commit+ /// barrier is what turns such a leftover into a hard, unrecoverable failure.+ #[test]+ fn upsert_clears_a_survivor_under_the_desired_content_identity() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let source = corpus.path().join("a.csv");+ let original = "id,value\n1,alpha\n";+ fs::write(&source, original).unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ let (index, content_id) = {+ let locked = endpoint.state.lock().unwrap();+ let ((index, _), doc) = locked+ .docs+ .iter()+ .find(|((index, _), _)| index != catalog::CATALOG_INDEX)+ .expect("the first generation published a data document");+ (+ index.clone(),+ doc.get("ax_file")+ .and_then(Value::as_str)+ .expect("data documents carry ax_file")+ .to_owned(),+ )+ };++ // Replace the content, then restore it, so the third generation upserts the+ // original content identity again.+ fs::write(&source, "id,value\n2,bravo\n").unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ fs::write(&source, original).unwrap();++ // A document from an aborted earlier attempt at exactly this content, under+ // an id the prepared stream will not overwrite.+ endpoint.state.lock().unwrap().docs.insert(+ (index.clone(), "aborted-attempt-survivor".to_owned()),+ json!({+ "id": 1,+ "value": "alpha",+ "ax_file": content_id,+ "ax_path": "a.csv",+ "ax_locator": "aborted-attempt-survivor"+ }),+ );++ assert_eq!(run_index(config).unwrap(), 0);+ assert!(+ !endpoint+ .state+ .lock()+ .unwrap()+ .docs+ .contains_key(&(index, "aborted-attempt-survivor".to_owned())),+ "the upsert must delete the desired content identity before replaying it"+ );+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 3);+ }++ /// `--dry-run` used to be consulted only *after* the generated branches had+ /// already published and committed, so on any already-generated state+ /// directory the flag was accepted, ignored, and the destination mutated.+ #[test]+ fn dry_run_on_a_generated_state_directory_publishes_nothing() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ let committed_docs = endpoint.data_docs();+ let bulks = endpoint.data_bulk_requests();+ let journal_before = fs::read(state_dir.path().join("journal.ndjson")).unwrap();++ // A real change, so the dry run has something it would publish.+ fs::write(+ corpus.path().join("a.csv"),+ "id,value\n1,CHANGED\n2,added\n",+ )+ .unwrap();+ let mut dry = config.clone();+ dry.dry_run = true;+ let (code, summary) = run_index_report(dry.clone()).unwrap();+ assert_eq!(code, 0);+ assert!(+ summary.is_none(),+ "--dry-run must not report a committed run summary"+ );+ assert_eq!(+ endpoint.data_bulk_requests(),+ bulks,+ "--dry-run issued a data bulk"+ );+ assert_eq!(+ endpoint.data_docs(),+ committed_docs,+ "--dry-run mutated the destination"+ );+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);+ assert_eq!(+ fs::read(state_dir.path().join("journal.ndjson")).unwrap(),+ journal_before,+ "--dry-run appended to the durable journal"+ );++ // An unchanged folder takes the no-op arm of the same branch: still inert.+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ assert_eq!(run_index(dry).unwrap(), 0);+ assert_eq!(endpoint.data_bulk_requests(), bulks);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);++ // And the real run that follows still converges on the change.+ fs::write(+ corpus.path().join("a.csv"),+ "id,value\n1,CHANGED\n2,added\n",+ )+ .unwrap();+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(endpoint.data_docs().len(), 2);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);+ }++ /// A junk/skipped file has no `plan.files` entry by construction, so demanding+ /// a prepared artifact for every inventory entry failed the whole run on any+ /// folder containing one unreadable, empty or unrecognised file.+ #[test]+ fn a_junk_file_is_recorded_and_never_fatal() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n2,beta\n").unwrap();+ fs::write(corpus.path().join("empty.csv"), "").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);++ let (code, summary) = run_index_report(config.clone()).unwrap();+ let summary = summary.expect("a corpus with one junk file still commits a generation");+ assert_eq!(+ code, 3,+ "a completed run that skipped a file exits 3, not 0 (cli.rs EXIT CODES)"+ );+ assert_eq!(summary["generation"], 1);+ assert_eq!(summary["files_junk"], 1);+ assert_eq!(summary["records_total"], 2);+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);+ let junk_docs: Vec<Value> = endpoint+ .state+ .lock()+ .unwrap()+ .docs+ .iter()+ .filter(|((index, _), doc)| {+ index == catalog::CATALOG_INDEX+ && doc.get("doc_kind").and_then(Value::as_str) == Some("file")+ && doc.get("path").and_then(Value::as_str) == Some("empty.csv")+ })+ .map(|(_, doc)| doc.clone())+ .collect();+ assert_eq!(+ junk_docs.len(),+ 1,+ "the skipped file must still get exactly one catalog entry"+ );++ // The generation reconciles normally afterwards, junk file and all.+ fs::write(corpus.path().join("b.csv"), "id,value\n3,gamma\n").unwrap();+ assert_eq!(run_index(config).unwrap(), 3);+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv", "b.csv"]);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);+ }++ /// Junk *records* were fatal on the generated path+ /// (`durable preparation of X produced N junk records`) while the legacy path+ /// merely accumulated them — directly contradicting the documented contract+ /// "3 completed-with-junk (junk recorded, never fatal)".+ #[test]+ fn a_junk_record_is_counted_into_the_generation_and_never_fatal() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let mut log = String::from("!!! rotated by logrotate, not a log line\n");+ for n in 1..=40 {+ log.push_str(&format!(+ "10.0.0.{n} - - [09/Aug/2026:10:00:{n:02} +0000] \"GET /p/{n} HTTP/1.1\" 200 {n}\n"+ ));+ }+ fs::write(corpus.path().join("access.log"), log).unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);++ let (code, summary) = run_index_report(config).unwrap();+ let summary = summary.expect("a corpus with one junk record still commits a generation");+ assert_eq!(code, 3, "completed-with-junk exits 3");+ assert_eq!(summary["generation"], 1);+ assert_eq!(summary["files_junk"], 0);+ assert_eq!(+ summary["junk_records_total"], 1,+ "the unparseable line must be counted, not hardcoded to zero"+ );+ assert_eq!(summary["records_total"], 40);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);++ // The same count reaches the per-file catalog document.+ let file_doc = endpoint+ .state+ .lock()+ .unwrap()+ .docs+ .iter()+ .find(|((index, _), doc)| {+ index == catalog::CATALOG_INDEX+ && doc.get("doc_kind").and_then(Value::as_str) == Some("file")+ && doc.get("path").and_then(Value::as_str) == Some("access.log")+ })+ .map(|(_, doc)| doc.clone())+ .expect("the indexed file has a catalog document");+ assert_eq!(file_doc["junk"], 1);+ assert_eq!(file_doc["records"], 40);+ }++ /// Where this branch and #241 meet.+ ///+ /// The generated `--no-graph` route returns from inside `run_index_report`,+ /// several hundred lines before the legacy `pr.finish` at the bottom of the+ /// function. `Ticker::drop` closes an unfinished stream with `ok=false`, so a+ /// generated run that merely returned would print an *aborted* terminal line+ /// immediately after committing a generation successfully — a silent+ /// contradiction between two changes that each looked right on its own.+ ///+ /// This pins all three properties on the generated path: the phase-A scan is+ /// narrated (it is a per-file, minutes-long phase here exactly as on the legacy+ /// path), the stream closes with a truthful terminal line, and `--progress+ /// none` on the same path stays completely silent.+ #[test]+ fn a_generated_run_narrates_its_scan_and_closes_its_own_stream() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ for n in 0..4 {+ fs::write(+ corpus.path().join(format!("rows-{n}.csv")),+ format!("id,value\n{n}0,alpha\n{n}1,beta\n"),+ )+ .unwrap();+ }+ let endpoint = HttpEndpoint::start();++ // Generation 1: the initial commit, through the real progress surface.+ let mut loud = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ loud.quiet = false;+ loud.progress = crate::progress::ProgressMode::Plain;+ loud.progress_interval = Some(std::time::Duration::from_secs(1));+ let buffer = Arc::new(Mutex::new(Vec::new()));+ let code = {+ let _sink = crate::progress::install_test_sink(&buffer);+ run_index(loud.clone()).unwrap()+ };+ assert_eq!(code, 0);+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();+ let done = stream+ .lines()+ .find(|line| line.starts_with("xerj-done "))+ .unwrap_or_else(|| panic!("a generated run must close its own stream:\n{stream}"));+ assert!(+ done.contains("ok=true") && done.contains("exit=0"),+ "a committed generation is not an aborted run: {done}"+ );+ assert!(done.contains("reason=completed"), "{done}");++ // Generation 2: the reconcile branch, which runs its own phase-A scan.+ fs::write(corpus.path().join("rows-0.csv"), "id,value\n00,CHANGED\n").unwrap();+ let buffer = Arc::new(Mutex::new(Vec::new()));+ let code = {+ let _sink = crate::progress::install_test_sink(&buffer);+ run_index(loud).unwrap()+ };+ assert_eq!(code, 0);+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();+ assert!(+ stream+ .lines()+ .any(|line| line.starts_with("xerj-progress ") && line.contains("phase=scan")),+ "the reconcile projection re-scans every file and must say so:\n{stream}"+ );+ let done = stream+ .lines()+ .find(|line| line.starts_with("xerj-done "))+ .unwrap_or_else(|| panic!("a reconciled generation must close its own stream:\n{stream}"));+ assert!(done.contains("ok=true"), "{done}");++ // The other half of the contract on the same path: silence means silence.+ fs::write(corpus.path().join("rows-1.csv"), "id,value\n10,CHANGED\n").unwrap();+ let quiet = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(quiet.progress, crate::progress::ProgressMode::None);+ let buffer = Arc::new(Mutex::new(Vec::new()));+ {+ let _sink = crate::progress::install_test_sink(&buffer);+ assert_eq!(run_index(quiet).unwrap(), 0);+ }+ assert!(+ buffer.lock().unwrap().is_empty(),+ "--progress none asked for nothing: {}",+ String::from_utf8_lossy(&buffer.lock().unwrap())+ );+ }diff --cc engine/crates/xerj-autoindex/src/lib.rsindex 1a3c29e54,5fe5ce843..0819b2172--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@@ -32,8 -43,411 +43,413 @@@ use cli::{Cmd, IndexCfg, MapCfg, Status// `Box<dyn EdgeDetector>` like the registry entries).use detect::EdgeDetector as _;use esclient::Es;+ use progress::Progress;use sniff::{Family, Sniffed};- use state::{FileAssignment, FileDone, JunkFile, Plan, PlanDataset};+ use state::{DuplicateFile, FileAssignment, FileDone, JunkFile, Plan, PlanDataset};++ const PREPARED_RECORDS_IDENTITY: &str = "prepared-records-v1";+ const DOCUMENT_IDS_IDENTITY: &str = "document-ids-v1";+ const DETECTOR_DISABLED_IDENTITY: &str = "disabled";++ fn prepared_records_identity(cfg: &IndexCfg) -> Result<String> {+ let value = json!({+ "contract": PREPARED_RECORDS_IDENTITY,+ "sample": cfg.sample,+ "max_file_gb": cfg.max_file_gb,+ "no_semantic": cfg.no_semantic,+ // Worker counts are operational only. The timeout can change whether+ // a PDF yields records, so it is part of the semantic contract.+ "pdf_timeout_secs": cfg.pdf_timeout_secs,+ });+ Ok(format!(+ "{}-{:032x}",+ PREPARED_RECORDS_IDENTITY,+ xxhash_rust::xxh3::xxh3_128(&serde_json::to_vec(&value)?)+ ))+ }++ fn preparation_contract_digest(cfg: &IndexCfg, plan: &Plan) -> Result<String> {+ let (schema_identity, index_identity) = generation_contract_identities(plan)?;+ let encoded = serde_json::to_vec(&json!({+ "prepared_records": prepared_records_identity(cfg)?,+ "document_ids": DOCUMENT_IDS_IDENTITY,+ "schema_identity": schema_identity,+ "index_identity": index_identity,+ "plan": plan,+ }))?;+ Ok(format!(+ "axpc1-{:032x}",+ xxhash_rust::xxh3::xxh3_128(&encoded)+ ))+ }++ pub(crate) fn generation_contract_identities(plan: &Plan) -> Result<(String, String)> {+ let mut datasets = plan.datasets.iter().collect::<Vec<_>>();+ datasets.sort_by(|left, right| {+ left.slug+ .cmp(&right.slug)+ .then_with(|| left.index.cmp(&right.index))+ });+ let schema = datasets+ .iter()+ .map(|dataset| {+ json!({+ "slug": dataset.slug,+ "family": dataset.family,+ "group": dataset.group,+ "specs": dataset.specs,+ "time_field": dataset.time_field,+ "semantic_field": dataset.semantic_field,+ })+ })+ .collect::<Vec<_>>();+ let indices = datasets+ .iter()+ .map(|dataset| {+ json!({+ "index": dataset.index,+ "mapping": build_mapping(&dataset.specs),+ })+ })+ .collect::<Vec<_>>();+ let digest = |label: &str, value: Value| -> Result<String> {+ let bytes = serde_json::to_vec(&json!({"contract": label, "value": value}))?;+ Ok(format!("{:032x}", xxhash_rust::xxh3::xxh3_128(&bytes)))+ };+ Ok((+ digest(PREPARED_RECORDS_IDENTITY, Value::Array(schema))?,+ digest(+ DOCUMENT_IDS_IDENTITY,+ json!({+ "datasets": indices,+ "catalog_index": catalog::CATALOG_INDEX,+ "catalog_mapping": catalog::catalog_mapping(),+ "document_ids": DOCUMENT_IDS_IDENTITY,+ }),+ )?,+ ))+ }++ pub(crate) fn ensure_generation_mappings(es: &Es, plan: &Plan) -> Result<()> {+ for dataset in &plan.datasets {+ let mut create_body = build_mapping(&dataset.specs);+ create_body["mappings"]["properties"]["ax_paths"] = json!({"type": "keyword"});+ let update_body = json!({+ "properties": create_body["mappings"]["properties"].clone()+ });+ es.ensure_index(&dataset.index, &create_body)+ .with_context(|| format!("create generation index {}", dataset.index))?;+ es.update_mapping(&dataset.index, &update_body)+ .with_context(|| format!("install generation mapping for {}", dataset.index))?;+ }+ let mut catalog_create_body = catalog::catalog_mapping();+ catalog_create_body["mappings"]["properties"]["duplicate_of"] = json!({"type": "keyword"});+ let catalog_update_body = json!({+ "properties": catalog_create_body["mappings"]["properties"].clone()+ });+ es.ensure_index(catalog::CATALOG_INDEX, &catalog_create_body)?;+ es.update_mapping(catalog::CATALOG_INDEX, &catalog_update_body)+ .context("install generation catalog mapping")+ }++ /// Project the current inventory onto a committed plan.+ ///+ /// Pure by design and deliberately shared: `--dry-run` prints exactly the plan+ /// the real reconcile would act on, because it is produced by this same call+ /// rather than by a parallel implementation that could drift away from it.+ ///+ /// The scan itself obeys the same two policies the legacy phase A does. It runs+ /// inside the run's own pool, because `--workers` has to bound the CPU-bound+ /// phase for the knob to mean anything (#240 §2), and it opens a per-file+ /// progress guard, because a file this route touches must not drop out of the+ /// progress denominator (#241). Run-local PDF artifact reuse is deliberately+ /// disabled here: the generated route publishes from its own sealed snapshot+ /// and never reaches the legacy phase B, so a retained artifact could never be+ /// replayed (#248).+ fn project_reconcile_plan(+ inventory: &content::Inventory,+ base_plan: &Plan,+ cfg: &IndexCfg,+ state_dir: &Path,+ pr: &Progress,+ ) -> Result<Plan> {+ let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);+ let ctx = PhaseAContext {+ state_dir,+ budget: &budget,+ capacity_warning: None,+ progress: pr,+ };+ pr.phase(+ "scan",+ inventory.files.len() as u64,+ inventory.files.iter().map(|file| file.size).sum(),+ );++ let stub_matcher = StubMatcher::compile(&cfg.stub_globs).expect("validated at startup");+ let scans: Vec<FileScan> = crate::pool::install(|| {+ use rayon::prelude::*;+ inventory+ .files+ .par_iter()+ .zip(inventory.digests.par_iter())+ .map(|(file, digest)| {+ let _in_flight = pr.file(&file.rel, file.size);+ scan_file(+ &file.path,+ file.size,+ digest,+ &ctx,+ cfg.sample,+ cfg.max_file_gb,++ stub_matcher.matches(&file.rel),+ )+ })+ .collect()+ });+ reconcile_plan::reconcile_plan(inventory, base_plan, scans, cfg.sample)+ }++ /// Exit code for a run that committed (or confirmed) a corpus generation.+ ///+ /// `3` is "completed with junk — recorded, never fatal", the same contract the+ /// legacy path publishes (`cli.rs` EXIT CODES, and CLAUDE.md's own indexing+ /// workflow treats 3 as success). The generated path returned a flat `0`, which+ /// silently downgraded that signal for `--no-graph`. Both inputs come from the+ /// committed generation's own run document, so a no-op re-run over a corpus+ /// that still contains junk reports 3 again instead of flapping to 0.+ fn generated_exit_code(summary: &Value) -> i32 {+ let count = |field: &str| summary.get(field).and_then(Value::as_u64).unwrap_or(0);+ if count("junk_records_total") > 0 || count("files_junk") > 0 {+ 3+ } else {+ 0+ }+ }++ /// Terminal progress line for a run that ends on the generated `--no-graph`+ /// path.+ ///+ /// Every exit closes the stream itself (#241). `Ticker::drop` reports+ /// `ok=false` for a run that never called `finish`, so a generated run that+ /// merely returned would print an aborted line straight after a successful+ /// commit — the one place the two landed changes could contradict each other.+ /// Exit 3 is success ("completed, some input was unusable"), so it is spelled+ /// out in words rather than left as a bare number.+ fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value) {+ let count = |field: &str| summary.get(field).and_then(Value::as_u64).unwrap_or(0);+ pr.finish(+ true,+ code,+ if code == 3 {+ "completed-with-junk"+ } else {+ "completed"+ },+ &[+ ("files", count("files_indexed")),+ ("records", count("records_total")),+ ("generation", count("generation")),+ ],+ );+ }++ fn begin_non_graph_generation(+ es: &Es,+ journal: &mut state::Journal,+ state_dir: &Path,+ cfg: &IndexCfg,+ root_identity: &str,+ inventory: &content::Inventory,+ plan: Plan,+ ) -> Result<()> {+ anyhow::ensure!(+ cfg.no_graph,+ "non-graph generation cutover requires --no-graph"+ );+ anyhow::ensure!(+ journal.pending_sync.is_none(),+ "cannot prepare over an existing pending generation"+ );+ if journal.committed_manifest.is_none() {+ journal.sync_bootstrap_genesis()?;+ }+ let base = journal+ .committed_manifest+ .as_ref()+ .context("generation cutover has no committed base")?+ .clone();+ let tx_id = format!("{}-g{}", journal.run_id, base.generation + 1);+ let preparation_contract = preparation_contract_digest(cfg, &plan)?;+ let snapshot = sync_executor::create_prepared_snapshot(+ state_dir,+ &tx_id,+ inventory,+ &plan,+ &preparation_contract,+ cfg.snapshot_max_bytes,+ )?;+ let chunker_identity = prepared_records_identity(cfg)?;+ let semantic = plan+ .datasets+ .iter()+ .any(|dataset| dataset.semantic_field.is_some());+ let identity = if semantic {+ let identity = es+ .embedding_execution_identity()+ .context("generation cutover could not pin the server embedding execution identity")?;+ anyhow::ensure!(+ identity.resumable,+ "generation cutover requires a resumable embedding execution identity: {}",+ identity+ .non_resumable_reason+ .as_deref()+ .unwrap_or("the server did not provide an immutable identity")+ );+ journal.pin_embedding_identity(+ &identity.identity_sha256,+ identity.resumable,+ identity.non_resumable_reason.as_deref(),+ )?;+ identity+ } else {+ crate::esclient::EmbeddingExecutionIdentity {+ version: 1,+ backend: "disabled".into(),+ identity_sha256: "0".repeat(64),+ // A disabled embedding execution has no vector width at all. `None`+ // says exactly that; any number here would be a fiction the+ // generation contract would then hold future runs to.+ dimensions: None,+ semantic_contract: "disabled-no-semantic-fields-v1".into(),+ resumable: true,+ non_resumable_reason: None,+ }+ };++ let aliases_by_content = plan.duplicate_files.iter().fold(+ HashMap::<&str, Vec<sync::ManifestPath>>::new(),+ |mut aliases, alias| {+ aliases+ .entry(alias.file_key.as_str())+ .or_default()+ .push(sync::ManifestPath {+ path_id: alias.path_id.clone(),+ rel: alias.rel.clone(),+ is_symlink: alias.is_symlink.unwrap_or(false),+ });+ aliases+ },+ );+ let file_by_content = inventory+ .keys+ .iter()+ .zip(&inventory.files)+ .zip(&inventory.digests)+ .map(|((key, file), digest)| (key.as_str(), (file, digest)))+ .collect::<HashMap<_, _>>();+ let mut candidates = Vec::with_capacity(plan.files.len());+ for (content_id, assignment) in &plan.files {+ let (file, digest) = file_by_content+ .get(content_id.as_str())+ .with_context(|| format!("planned content {content_id} is absent from inventory"))?;+ let mut paths = vec![sync::ManifestPath {+ path_id: assignment.path_id.clone(),+ rel: assignment.rel.clone(),+ is_symlink: assignment.is_symlink.unwrap_or(file.is_symlink),+ }];+ paths.extend(+ aliases_by_content+ .get(content_id.as_str())+ .cloned()+ .unwrap_or_default(),+ );+ candidates.push(sync::DesiredContentGroup {+ content_id: content_id.clone(),+ content_digest: (*digest).clone(),+ content_size: file.size,+ dataset_slugs: assignment+ .assignments+ .iter()+ .map(|(_, slug)| slug.clone())+ .collect(),+ paths,+ expected_records: 0,+ expected_passages: 0,+ expected_vectors: 0,+ expected_junk_records: 0,+ });+ }+ let mut groups = sync::reconcile_groups(&base.groups, candidates)?;+ sync_executor::bind_prepared_counts(&mut groups, &snapshot, &inventory.keys)?;+ let (schema_identity, index_identity) = generation_contract_identities(&plan)?;+ let desired = sync::GenerationManifest {+ generation: base.generation + 1,+ execution: Some(sync::ExecutionIdentity {+ version: sync::EXECUTION_IDENTITY_VERSION,+ root_identity: root_identity.to_owned(),+ url: cfg.url.clone(),+ prefix: cfg.prefix.clone(),+ follow_symlinks: cfg.follow_symlinks,+ chunker_identity,+ embedding_identity_sha256: identity.identity_sha256,+ embedding_backend: identity.backend,+ embedding_dimension: identity.dimensions,+ embedding_semantic_contract: identity.semantic_contract,+ embedding_resumable: identity.resumable,+ graph_enabled: false,+ brain: "disabled".into(),+ detector_identity: DETECTOR_DISABLED_IDENTITY.into(),+ schema_identity,+ index_identity,+ source_policy: sync::SourceExecutionPolicy::DurableSnapshot {+ reference: format!("sync-snapshots/{tx_id}"),+ snapshot_digest: snapshot.snapshot_digest,+ },+ }),+ plan,+ groups,+ };+ let pending = sync::PendingSync::new(tx_id, &base, desired)?;+ journal.sync_begin(&pending)+ }++ #[derive(Debug, PartialEq, Eq)]+ struct CliErrorRoute {+ exit_code: i32,+ stdout: Option<String>,+ stderr: Option<String>,+ }++ fn route_cli_error(error: &anyhow::Error, json_output: bool) -> CliErrorRoute {+ if json_output {+ // Two distinct refusals reach the JSON route, and both must keep it.+ // `UnsafeFreshGenerationError` guards durable *generation* state on the+ // `--no-graph` path; `UnsupportedInventoryDeltaError` (#254) guards the+ // legacy graph-enabled path against a rerun that would strand live+ // documents. Neither supersedes the other — they protect different+ // destinations — so routing only one would silently drop the other's+ // machine-readable rendering back to a prose stderr line.+ if let Some(unsafe_fresh) = error.downcast_ref::<UnsafeFreshGenerationError>() {+ return CliErrorRoute {+ exit_code: 1,+ stdout: Some(unsafe_fresh.to_json().to_string()),+ stderr: None,+ };+ }+ if let Some(delta) = error.downcast_ref::<UnsupportedInventoryDeltaError>() {+ return CliErrorRoute {+ exit_code: 1,+ stdout: Some(delta.to_json().to_string()),+ stderr: None,+ };+ }+ }+ CliErrorRoute {+ exit_code: 1,+ stdout: None,+ stderr: Some(format!("error: {error:#}")),+ }+ }/// Entry point for the `xerj autoindex` subcommand (blocking; the server/// binary calls this via spawn_blocking). Returns the process exit code.@@@ -322,21 -635,59 +768,64 @@@ struct GroupSketchrecords: u64,}- fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64, stub: bool) -> FileScan {+ fn take_pdf_spool_if_indexable<T>(+ spool: &mut Option<T>,+ is_junk: bool,+ budget: &extract::pdf::ExtractionSpoolBudget,+ ) -> Option<T> {+ if is_junk {+ if spool.is_some() {+ budget.record_discarded_before_replay();+ }+ spool.take();+ None+ } else {+ spool.take()+ }+ }++ /// Everything phase A needs beyond the file list and the run config: where a+ /// run-local PDF artifact may live, the shared admission budget, the one-line+ /// capacity explanation reported when files fall back — and the progress+ /// surface those lines go out through, because phase A now reports every file+ /// it touches (#241) as well as every artifact it could not keep.+ ///+ /// Grouped so `scan_file` and `build_phase_a` each take one parameter instead+ /// of four.+ struct PhaseAContext<'a> {+ state_dir: &'a Path,+ budget: &'a std::sync::Arc<extract::pdf::ExtractionSpoolBudget>,+ capacity_warning: Option<&'a str>,+ progress: &'a Progress,+ }++ fn scan_file(+ path: &Path,+ size: u64,+ digest: &str,+ ctx: &PhaseAContext<'_>,+ sample: usize,+ max_file_gb: u64,++ stub: bool,+ ) -> FileScan {+ let state_dir = ctx.state_dir;+ let pdf_spool_budget = ctx.budget;let mut out = FileScan {sniffed: None,sketches: Vec::new(),junk: None,+ pdf_spool: None,+ pdf_spool_fallbacks: Vec::new(),};- let sn = match sniff::sniff(path) {- Ok(s) => s,- Err(e) => {- out.junk = Some(("junk".into(), format!("unreadable: {e}")));- return out;+ let sn = if stub {+ stub_sniffed()+ } else {+ match sniff::sniff(path) {+ Ok(s) => s,+ Err(e) => {+ out.junk = Some(("junk".into(), format!("unreadable: {e}")));+ return out;+ }}};if sn.family == Family::Binary {@@@ -450,7 -838,17 +985,17 @@@ mod clustering_key_testslet path = dir.join(name);std::fs::write(&path, body).unwrap();let size = std::fs::metadata(&path).unwrap().len();- scan_file(&path, size, 500, 2, false)+ // Clustering keys are decided by extraction, not by PDF artifact+ // reuse: a zero budget keeps these cases on the plain parse path.+ let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);+ let progress = Progress::silent();+ let ctx = PhaseAContext {+ state_dir: dir,+ budget: &budget,+ capacity_warning: None,+ progress: &progress,+ };- scan_file(&path, size, "d0", &ctx, 500, 2)++ scan_file(&path, size, "d0", &ctx, 500, 2, false)}/// The #178 mechanism, from the extractor to the clustering key: a source@@@ -524,6 -923,471 +1070,493 @@@}}+ #[cfg(test)]+ mod phase_a_grouping_tests {+ use super::*;++ fn cfg_for(root: &Path) -> IndexCfg {+ IndexCfg {+ root: root.to_path_buf(),++ stub_globs: Vec::new(),+ url: "http://unused.invalid".into(),+ api_key: None,+ workers: 1,+ scan_workers: 1,+ pdf_workers: 1,+ resource_notes: Vec::new(),+ pdf_timeout_secs: 10,+ bulk_mb: 1,+ bulk_timeout_secs: 10,+ snapshot_max_bytes: 64 << 30,+ prefix: "t".into(),+ state_dir: None,+ fresh: true,+ follow_symlinks: false,+ ignore: crate::ignore_rules::IgnoreOptions::default(),+ max_file_gb: 2,+ sample: 500,+ no_semantic: false,+ brain: None,+ no_graph: true,+ dry_run: true,+ json: false,+ quiet: true,+ progress: crate::progress::ProgressMode::None,+ progress_interval: None,+ }+ }++ fn plan_for(root: &Path) -> Plan {+ let files = walk::walk(root, false).unwrap();+ let keys: Vec<String> = files+ .iter()+ .map(|f| ids::file_key(&f.path, f.size).unwrap())+ .collect();+ let digests: Vec<String> = (0..files.len()).map(|i| format!("d{i}")).collect();+ // Planning is what these cases assert on; a zero budget keeps every+ // file on the plain parse path so no artifact is ever retained.+ let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);+ let progress = Progress::silent();+ let ctx = PhaseAContext {+ state_dir: root,+ budget: &budget,+ capacity_warning: None,+ progress: &progress,+ };+ build_phase_a(+ root,+ &files,+ &keys,+ &digests,+ Vec::new(),+ &ctx,+ &cfg_for(root),+ )+ .plan+ }++ const CODE: &str = "// The event loop dispatches every ready connection to a worker.\n\+ // Each worker drains its queue before polling for more sockets.\n\+ static void dispatch_ready_connections(struct event_loop *loop) {\n\+ for (int index = 0; index < loop->ready_count; index++) {\n\+ worker_submit(loop->workers, loop->ready[index]);\n\+ }\n\+ }\n";++ const PROSE: &str = "# Overview\n\nThis server accepts client connections and stores \+ keys in memory. Every command travels through the same parser before the \+ dispatcher routes it to the matching handler function.\n";++ /// #173 end to end through the real planner: a tree holding two+ /// repositories of source, prose and one-off config JSON yields one+ /// document dataset per repository — with `body` elected `semantic_text`+ /// — plus the genuine data dataset, instead of one dataset per incidental+ /// config schema with no vector arm.+ #[test]+ fn a_two_repo_tree_plans_one_document_dataset_per_repo() {+ let dir = tempfile::tempdir().unwrap();+ let root = dir.path();+ for repo in ["valkey", "memcached"] {+ std::fs::create_dir_all(root.join(repo).join(".git")).unwrap();+ std::fs::write(root.join(repo).join(".git").join("HEAD"), "ref: x").unwrap();+ }+ std::fs::create_dir_all(root.join("valkey/src")).unwrap();+ std::fs::create_dir_all(root.join("valkey/commands")).unwrap();+ std::fs::create_dir_all(root.join("memcached/data")).unwrap();+ std::fs::write(root.join("valkey/src/server.c"), CODE).unwrap();+ std::fs::write(root.join("valkey/README.md"), PROSE).unwrap();+ // one-off configs: single-record JSON, each with its own key set+ std::fs::write(+ root.join("valkey/commands/get.json"),+ r#"{"GET": {"summary": "Return the string value stored at the given key.", "arity": 2}}"#,+ )+ .unwrap();+ std::fs::write(+ root.join("valkey/commands/set.json"),+ r#"{"SET": {"summary": "Store the given string value under the given key.", "arity": 3}}"#,+ )+ .unwrap();+ // distinct bytes — byte-identical files alias into one canonical copy+ std::fs::write(root.join("memcached/proto.c"), format!("{CODE}\n// v2\n")).unwrap();+ // a genuine data file: recurring rows, real schema+ std::fs::write(+ root.join("memcached/data/events.csv"),+ )+ .unwrap();++ let plan = plan_for(root);+ let mut slugs: Vec<&str> = plan.datasets.iter().map(|d| d.slug.as_str()).collect();+ slugs.sort();+ assert_eq!(+ slugs,+ ["memcached-data", "memcached-docs", "valkey-docs"],+ "{:#?}",+ plan.datasets+ );++ // every docs dataset carries the vector arm on body+ for d in plan.datasets.iter().filter(|d| d.family == "docs") {+ let body = d.specs.iter().find(|s| s.name == "body").unwrap();+ assert_eq!(body.es_type, "semantic_text", "{}: {:#?}", d.slug, d.specs);+ assert_eq!(d.semantic_field.as_deref(), Some("body"), "{}", d.slug);+ }++ // the one-off configs were demoted: document-rendered, and their+ // config keys never reached the docs mapping+ let by_rel: HashMap<&str, &FileAssignment> =+ plan.files.values().map(|f| (f.rel.as_str(), f)).collect();+ assert!(by_rel["valkey/commands/get.json"].as_document);+ assert!(by_rel["valkey/commands/set.json"].as_document);+ assert!(!by_rel["valkey/src/server.c"].as_document);+ assert!(!by_rel["memcached/data/events.csv"].as_document);+ let vdocs = plan+ .datasets+ .iter()+ .find(|d| d.slug == "valkey-docs")+ .unwrap();+ assert!(+ !vdocs.specs.iter().any(|s| s.name.contains("summary")),+ "config keys leaked into the docs mapping: {:#?}",+ vdocs.specs+ );+ assert_eq!(vdocs.file_count, 4, "code + prose + 2 demoted configs");++ // the CSV kept its real schema+ let data = plan+ .datasets+ .iter()+ .find(|d| d.slug == "memcached-data")+ .unwrap();+ assert!(data.specs.iter().any(|s| s.name == "email"), "{data:#?}");+ }++ /// #196: the same tree WITHOUT nested `.git` markers (or with one at the+ /// root — a workspace) is ONE scope: a single document corpus.+ #[test]+ fn a_single_workspace_plans_one_document_dataset() {+ let dir = tempfile::tempdir().unwrap();+ let root = dir.path();+ std::fs::create_dir_all(root.join(".git")).unwrap();+ std::fs::create_dir_all(root.join("src")).unwrap();+ std::fs::create_dir_all(root.join("wasm/examples")).unwrap();+ std::fs::write(root.join("src/a.c"), CODE).unwrap();+ std::fs::write(root.join("src/b.c"), CODE).unwrap();+ std::fs::write(root.join("README.md"), PROSE).unwrap();+ std::fs::write(root.join("wasm/examples/demo.md"), PROSE).unwrap();++ let plan = plan_for(root);+ assert_eq!(+ plan.datasets.len(),+ 1,+ "one workspace, one corpus: {:#?}",+ plan.datasets+ );+ assert_eq!(plan.datasets[0].slug, "docs");+ assert_eq!(plan.datasets[0].file_count, 4);+ }+ }++ // ─── Phase A plan building (pure: no server contact) ─────────────────────++ /// Repository scope per file: the deepest ancestor directory (root-relative,+ /// "/"-separated) containing a `.git` entry, or "" when the file is under+ /// none. The walk never descends into `.git` itself, but the marker is still+ /// on disk. A `.git` at the autoindex root yields "" for every file — the+ /// whole tree is one scope — and so does a tree with no `.git` at all, which+ /// is exactly the #173/#196 property: one autoindex of one folder yields a+ /// corpus searchable as one corpus, split only at nested repository roots.+ fn compute_scopes(root: &Path, rels: &[String]) -> Vec<String> {+ let mut cache: HashMap<String, bool> = HashMap::new();+ rels.iter()+ .map(|rel| {+ let mut scope = String::new();+ let mut prefix = String::new();+ let mut segs = rel.split('/').peekable();+ while let Some(seg) = segs.next() {+ if segs.peek().is_none() {+ break; // final segment is the file name+ }+ if !prefix.is_empty() {+ prefix.push('/');+ }+ prefix.push_str(seg);+ let repo = *cache+ .entry(prefix.clone())+ .or_insert_with(|| root.join(&prefix).join(".git").exists());+ if repo {+ scope = prefix.clone();+ }+ }+ scope+ })+ .collect()+ }++ /// What phase A produces: the frozen plan, the clusters phase B needs, and+ /// the run-local PDF artifacts (index-aligned with `files`) that phase B may+ /// replay instead of parsing again. A spool is always optional — every entry+ /// may legitimately be `None`.+ struct PhaseA {+ plan: Plan,+ clusters: Vec<dataset::Cluster>,+ pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>>,+ }++ /// Record and report why individual PDFs could not retain a run-local+ /// artifact. Reuse is an accelerator, so this is purely informational: every+ /// listed file is parsed again by the normal phase B path. Recording is kept+ /// even when the budget is globally disabled — the stored examples are the+ /// only place the `--json` report explains *why* nothing was reused, and they+ /// are already capped at three.+ ///+ /// Reporting goes out through the progress surface, never a bare `eprintln!`:+ /// stderr belongs to that surface, so `--progress none` stays silent and+ /// `--progress json` stays a single parseable stream (#241).+ fn report_pdf_spool_fallbacks(+ files: &[walk::FileEntry],+ scans: &[FileScan],+ ctx: &PhaseAContext<'_>,+ ) {+ let pdf_spool_budget = ctx.budget;+ let pr = ctx.progress;+ let reasons: Vec<(&str, &extract::pdf::SpoolFallback)> = scans+ .iter()+ .enumerate()+ .flat_map(|(index, scan)| {+ scan.pdf_spool_fallbacks+ .iter()+ .map(move |fallback| (files[index].rel.as_str(), fallback))+ })+ .collect();+ for (path, fallback) in &reasons {+ pdf_spool_budget.record_fallback_example(path, fallback.category, &fallback.message);+ }+ if reasons.is_empty() {+ return;+ }+ if pdf_spool_budget.platform_reuse_is_unavailable() {+ pr.note(+ "phase A: run-local PDF extraction reuse is unavailable on this platform; \+ phase B will use the normal parser",+ );+ return;+ }+ pr.note(&format!(+ "phase A: {} PDF extraction(s) could not retain a bounded run-local artifact; \+ phase B will parse them again safely",+ reasons.len()+ ));+ if let Some(warning) = ctx.capacity_warning {+ pr.note(&format!(" PDF reuse capacity: {warning}"));+ }+ for (path, fallback) in reasons.iter().take(3) {+ pr.note(&format!(+ " PDF reuse fallback for {path}: {}",+ fallback.message+ ));+ }+ if reasons.len() > 3 {+ pr.note(&format!(+ " … and {} more PDF reuse fallback(s)",+ reasons.len() - 3+ ));+ }+ }++ /// Sniff + sample every file, cluster into datasets, and assemble the plan.+ /// Pure planning: reads the tree, never the server — which is what makes the+ /// #173/#196 grouping behaviour testable end-to-end without a cluster.+ fn build_phase_a(+ root: &Path,+ files: &[walk::FileEntry],+ keys: &[String],+ digests: &[String],+ duplicate_files: Vec<DuplicateFile>,+ ctx: &PhaseAContext<'_>,+ cfg: &IndexCfg,+ ) -> PhaseA {+ use rayon::prelude::*;+ let pdf_spool_budget = ctx.budget;+ let pr = ctx.progress;+ // Same pool as the digest phase: sniffing and sampling are the other half+ // of the CPU-bound phase `--workers` has to bound (#240 §2). Progress is+ // reported from inside that pool, so the straggler the ticker names is the+ // file a scan-pool thread is genuinely sitting on (#241). Retaining a PDF+ // artifact happens inside that same guard, so a file whose extraction is+ // spooled is counted exactly like a plainly parsed one.++ // Patterns were validated (loudly) at run start; a failure here would be++ // a programming error, not user input.++ let stub_matcher = StubMatcher::compile(&cfg.stub_globs).expect("validated at startup");+ let scans: Vec<FileScan> = crate::pool::install(|| {+ files+ .par_iter()+ .zip(digests.par_iter())+ .map(|(f, digest)| {+ let _in_flight = pr.file(&f.rel, f.size);- scan_file(&f.path, f.size, digest, ctx, cfg.sample, cfg.max_file_gb)++ scan_file(++ &f.path,++ f.size,++ digest,++ ctx,++ cfg.sample,++ cfg.max_file_gb,++ stub_matcher.matches(&f.rel),++ )+ })+ .collect()+ });++ report_pdf_spool_fallbacks(files, &scans, ctx);++ let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();+ let scopes = compute_scopes(root, &rels);+ // (family, gzip) per file, from the scan's sniff — reused for assignments+ // and demoted-file re-sampling instead of re-sniffing every file.+ let file_meta: Vec<Option<(Family, bool)>> = scans+ .iter()+ .map(|sc| sc.sniffed.as_ref().map(|s| (s.family, s.gzip)))+ .collect();+ let mut sketches = Vec::new();+ let mut junk_files = Vec::new();+ let mut pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>> =+ (0..files.len()).map(|_| None).collect();+ for (i, mut sc) in scans.into_iter().enumerate() {+ let family = sc+ .sniffed+ .as_ref()+ .map(|s| s.family)+ .unwrap_or(Family::Binary);+ // A file that phase A junks is never indexed, so its artifact is+ // refunded here rather than held to the phase A→B boundary.+ pdf_spools[i] =+ take_pdf_spool_if_indexable(&mut sc.pdf_spool, sc.junk.is_some(), pdf_spool_budget);+ if let Some((status, reason)) = sc.junk {+ junk_files.push(JunkFile {+ file_key: keys[i].clone(),+ rel: files[i].rel.clone(),+ format: format_str(sc.sniffed.as_ref()),+ status,+ reason,+ bytes: files[i].size,+ });+ continue;+ }+ for gs in sc.sketches {+ sketches.push(dataset::Sketch {+ file_idx: i,+ group: gs.group,+ family,+ fields: gs.fields,+ key_fields: gs.key_fields,+ records: gs.records,+ });+ }+ }+ let mut clusters = dataset::cluster(sketches, &rels, &scopes);++ // Demoted one-off config files (#173) were sampled as flattened records;+ // phase B will index them as documents. Re-sample them through the same+ // document renderer so the docs dataset's mapping and stats describe what+ // actually gets indexed.+ for c in clusters.iter_mut().filter(|c| !c.demoted.is_empty()) {+ let demoted = c.demoted.clone();+ let fields = &mut c.fields;+ let mut sampled_total = 0u64;+ for m in demoted {+ let gzip = file_meta[m].map(|(_, g)| g).unwrap_or(false);+ let mut sampled = 0u64;+ let mut sink = |rec: extract::RawRecord| -> bool {+ if (sampled as usize) < cfg.sample {+ for (k, v) in &rec.fields {+ fields.entry(k.clone()).or_default().add(v);+ }+ }+ sampled += 1;+ (sampled as usize) < cfg.sample+ };+ // An unreadable file junks at phase B like any other; the plan+ // keeps its membership either way.+ let _ = extract::extract_as_document(&files[m].path, gzip, &mut sink);+ sampled_total += sampled;+ }+ c.records += sampled_total;+ }++ // per-file assignments+ let mut file_assignments: HashMap<String, FileAssignment> = HashMap::new();+ for (ci, c) in clusters.iter().enumerate() {+ let demoted: std::collections::HashSet<usize> = c.demoted.iter().copied().collect();+ for &m in &c.members {+ let key = &keys[m];+ let (family, gzip) = file_meta[m].unwrap_or((c.family, false));+ let fa = file_assignments+ .entry(key.clone())+ .or_insert_with(|| FileAssignment {+ rel: files[m].rel.clone(),+ path_id: files[m].rel_id.clone(),+ is_symlink: Some(files[m].is_symlink),+ family: family.as_str().to_string(),+ gzip,+ content_digest: Some(digests[m].clone()),+ assignments: Vec::new(),+ as_document: false,+ });+ fa.as_document |= demoted.contains(&m);+ fa.assignments+ .push((c.group.clone(), clusters[ci].slug.clone()));+ }+ }++ let mut datasets = Vec::new();+ for c in &clusters {- let specs =++ let mut specs =+ infer::infer_fields_with_policy(&c.fields, c.records, cfg.no_semantic, c.is_docs);++ // Unity script-link enrichment fields are stamped by the phase-B++ // pipeline (not the extractor), so inference never sees them —++ // register them here or they would be dynamic-mapped coarsely.++ if c.family == Family::UnityYaml && c.fields.contains_key("script_guid") {++ specs.push(pipeline_keyword_spec("script_path"));++ specs.push(pipeline_keyword_spec("script_class"));++ }++ if c.family == Family::UnityMeta {++ specs.push(pipeline_keyword_spec("asset_path"));++ }+ let time_field = infer::elect_time_field(&specs);+ let semantic_field = specs+ .iter()+ .find(|s| s.es_type == "semantic_text")+ .map(|s| s.name.clone());+ datasets.push(PlanDataset {+ slug: c.slug.clone(),+ index: format!("{}-{}", cfg.prefix, c.slug),+ family: if c.is_docs {+ "docs".to_string()+ } else {+ c.family.as_str().to_string()+ },+ group: c.group.clone(),+ specs,+ time_field,+ semantic_field,+ sampled_records: c.records,+ file_count: c.members.len(),+ });+ }+ let plan = Plan {+ datasets,+ files: file_assignments,+ junk_files,+ duplicate_files,+ alias_paths_indexed: true,+ };+ PhaseA {+ plan,+ clusters,+ pdf_spools,+ }+ }+// ─── mapping builder ─────────────────────────────────────────────────────pub const PROVENANCE_FIELDS: &[&str] = &[@@@ -797,41 -2092,61 +2352,62 @@@ fn run_index(cfg: IndexCfg) -> Result<i/// `None` when the run ended before a plan produced one (empty folder,/// `--dry-run`).pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {+ // The very first statement of the function, deliberately: `started` must+ // be when this invocation began, not when its summary was built.+ let invocation_started = chrono::Utc::now();+ // Fix the phase-A pool width BEFORE anything parallel starts: hashing and+ // sniffing are the CPU-bound phase, and they used to take every core no+ // matter what the caller asked for (#240 §2).+ pool::configure(cfg.scan_workers);+ // What phase A is *actually* running on. `pool::configure` is first-call-wins+ // (rayon pools cannot be resized), so in a process that already indexed once+ // — `xerj-server`'s brain endpoint does exactly that — the installed width+ // can differ from what this run's plan asked for. Progress must state the+ // number the policy got, not the one it requested (#240 + #241).+ let scan_threads = pool::scan_pool().current_num_threads();extract::pdf::configure_workers(cfg.pdf_workers);extract::pdf::configure_timeout(cfg.pdf_timeout_secs);- use rayon::prelude::*;let t0 = Instant::now();- if !cfg.quiet {- eprintln!(- "autoindex: bulk HTTP request timeout: {}s",- cfg.bulk_timeout_secs- );+ // The progress surface and its ticker are the FIRST things built: every+ // later phase reports through them, and the ticker guarantees the stream+ // closes with a terminal line even if this function bails (#241). The+ // resource plan is announced through the same surface rather than through a+ // bare `eprintln!`, so `--quiet` / `--progress none` still means silent and+ // `--progress json` still means one machine-readable stream (#240 + #241).+ let surface = progress::detect(cfg.progress);+ let pr = Progress::new(+ surface,+ cfg.progress_interval+ .unwrap_or_else(|| progress::default_interval(surface)),+ );+ let ticker = pr.spawn_ticker();+ // What this run decided to take from the machine, before it takes it.+ pr.note(&format!(+ "autoindex: {} scan threads, {} index workers, {} pdf workers, --bulk-mb {} [{}]",+ scan_threads,+ cfg.workers,+ cfg.pdf_workers,+ cfg.bulk_mb,+ xerj_common::resource::describe(),+ ));+ for note in &cfg.resource_notes {+ pr.note(&format!("autoindex: {note}"));}- let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;+ pr.note(&format!(+ "autoindex: bulk HTTP request timeout: {}s",+ cfg.bulk_timeout_secs+ ));+ // The run's bulk load is admitted through one window `--workers` wide, so+ // a 429 can shrink what the run offers instead of only delaying it+ // (#240 §8). Enabled here and nowhere else: probes have nothing to+ // throttle. Its shrink/recover announcements go to stderr, which the+ // progress surface owns — so they are emitted exactly when that surface is+ // enabled, not merely when `--quiet` is absent.+ let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?+ .with_bulk_concurrency(cfg.workers, pr.enabled());es.ping()?;+ let stub_matcher = StubMatcher::compile(&cfg.stub_globs)?;- let (discovered_files, skipped_dirs) =- walk::walk(- &cfg.root,- cfg.follow_symlinks,- !cfg.no_default_excludes,- !cfg.no_gitignore,- )?;- if !skipped_dirs.is_empty() && !cfg.quiet {- let names: Vec<&str> = skipped_dirs.iter().map(|s| s.rel.as_str()).collect();- eprintln!(- "skipping {} generated director{} ({}): {} — pass --no-default-excludes to include",- skipped_dirs.len(),- if skipped_dirs.len() == 1 { "y" } else { "ies" },- skipped_dirs[0].reason,- names.join(", ")- );- }- if discovered_files.is_empty() {- println!("no files found under {}", cfg.root.display());- return Ok((0, None));- }let root_str = cfg.root.canonicalize()@@@ -1624,9 -3357,10 +3625,11 @@@let mut fields = rec.fields;let dropped = coerce::coerce_record(&mut fields, &rt.plan);if dropped > 0 {- rt.dropped.fetch_add(dropped as u64, Ordering::Relaxed);+ let durable =+ file_dropped_by_dataset.entry(slug.clone()).or_default();+ *durable = durable.saturating_add(dropped as u64);}+ enrich_unity_fields(sn.family, &mut fields, &unity_guid_map, &f.rel);fields.insert("ax_path".into(), Value::String(f.rel.clone()));fields.insert("ax_paths".into(),diff --cc engine/crates/xerj-autoindex/src/sniff.rsindex 49f31bdfb,4802cd260..1a7161093--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@@ -605,115 -518,6 +620,115 @@@ fn sniff_csv_dialect(nonblank: &[&str]})}+#[cfg(test)]+mod unity_sniff_tests {+ use super::*;+ use std::path::Path;++ fn sniff_str(s: &str, name: &str) -> Family {- sniff_bytes(s.as_bytes(), Path::new(name), false)++ sniff_bytes(s.as_bytes(), Path::new(name), Path::new(name), false)+ .unwrap()+ .family+ }++ #[test]+ fn unity_tagged_yaml_is_detected_by_header_not_extension() {+ let scene = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &123\nGameObject:\n m_Name: X\n";+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);+ assert_eq!(+ sniff_str(scene, "renamed.txt"),+ Family::UnityYaml,+ "content decides, never the extension"+ );+ }++ #[test]+ fn a_bom_before_the_yaml_directive_is_tolerated() {+ let scene =+ "\u{feff}%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1\nGameObject:\n m_Name: X\n";+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);+ }++ #[test]+ fn meta_needs_the_first_line_rule_and_a_guid() {+ let meta = "fileFormatVersion: 2\nguid: 9f1c4d0ab2e34f6\nMonoImporter:\n serializedVersion: 2\n";+ assert_eq!(sniff_str(meta, "Player.cs.meta"), Family::UnityMeta);+ let stray = "config: true\nfileFormatVersion: 2\nguid: abc\n";+ assert_ne!(+ sniff_str(stray, "some.yaml"),+ Family::UnityMeta,+ "guid keys inside ordinary YAML must not reclassify it"+ );+ let no_guid = "fileFormatVersion: 2\nsettings:\n a: 1\n";+ assert_ne!(sniff_str(no_guid, "x.meta"), Family::UnityMeta);+ }++ #[test]+ fn bvh_is_detected_by_hierarchy_root_header() {+ let bvh = "HIERARCHY\nROOT Hips\n{\n OFFSET 0 90 0\n CHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation\n}\nMOTION\nFrames: 2\nFrame Time: 0.033\n1 2 3\n4 5 6\n";+ assert_eq!(sniff_str(bvh, "clip.bvh"), Family::Bvh);+ assert_eq!(+ sniff_str(bvh, "clip.txt"),+ Family::Bvh,+ "content decides, never the extension"+ );+ let not_bvh = "HIERARCHY\nof needs (Maslow):\n- physiological\n- safety\n";+ assert_ne!(sniff_str(not_bvh, "notes.txt"), Family::Bvh);+ }++ #[test]+ fn plain_yaml_without_the_unity_tag_stays_yaml() {+ let plain = "%YAML 1.2\n---\nkey: value\nother: 1\nnested:\n a: 2\n";+ assert_ne!(sniff_str(plain, "doc.yaml"), Family::UnityYaml);+ }++ /// Regression: PSD image data decoded via lossy windows-1252 passed the+ /// NUL/control-char heuristics and classified as txt-prose, turning a+ /// multi-MB texture into ~100x its size of prose sections. Media magic+ /// bytes must win before any text heuristic runs.+ #[test]+ fn media_containers_are_binary_by_magic_not_heuristics() {+ for (name, head) in [+ ("t.psd", &b"8BPS\x00\x01"[..]),+ ("t.tif", &b"II*\x00\x08\x00"[..]),+ ("t.tif2", &b"MM\x00*\x00\x08"[..]),+ ("t.wav", &b"RIFF\x24\x08\x00\x00WAVE"[..]),+ ("t.ogg", &b"OggS\x00\x02"[..]),+ ("t.fbx", &b"Kaydara FBX Binary \x00"[..]),+ ] {+ let mut bytes = head.to_vec();+ // A printable tail that WOULD pass the prose heuristics.+ bytes.extend(b"lorem ipsum dolor sit amet, consectetur adipiscing elit. ".repeat(20));- let sn = sniff_bytes(&bytes, Path::new(name), false).unwrap();++ let sn = sniff_bytes(&bytes, Path::new(name), Path::new(name), false).unwrap();+ assert_eq!(sn.family, Family::Binary, "{name} must be binary");+ }+ }++ /// Regression: an uncompressed TGA (no magic bytes exist for TGA) decoded+ /// via lossy windows-1252 into printable soup with near-zero whitespace+ /// and classified txt-prose. Whitespace ratio is the discriminator.+ #[test]+ fn whitespace_free_printable_soup_is_binary_not_prose() {+ let soup: String = (0..8192u32)+ .map(|i| char::from_u32(0x21 + (i * 7) % 0x5d).unwrap())+ .collect();- let sn = sniff_bytes(soup.as_bytes(), Path::new("texture.tga"), false).unwrap();++ let sn = sniff_bytes(soup.as_bytes(), Path::new("texture.tga"), Path::new("texture.tga"), false).unwrap();+ assert_eq!(sn.family, Family::Binary);+ // Real prose of the same size keeps its family.+ let prose = "The quick brown fox jumps over the lazy dog. ".repeat(200);- let sn = sniff_bytes(prose.as_bytes(), Path::new("note.txt"), false).unwrap();++ let sn = sniff_bytes(prose.as_bytes(), Path::new("note.txt"), Path::new("note.txt"), false).unwrap();+ assert_ne!(sn.family, Family::Binary, "real prose must stay text");+ }++ #[test]+ fn binary_serialized_unity_assets_stay_binary() {+ let mut bytes = b"UnityFS\x00\x00\x00\x00\x08".to_vec();+ bytes.extend(std::iter::repeat(0u8).take(64));- let sn = sniff_bytes(&bytes, Path::new("scene.unity"), false).unwrap();++ let sn = sniff_bytes(&bytes, Path::new("scene.unity"), Path::new("scene.unity"), false).unwrap();+ assert_eq!(sn.family, Family::Binary);+ }+}+#[cfg(test)]mod tests {use super::*;diff --cc engine/crates/xerj-autoindex/src/walk.rsindex bdfd7fd55,c230cac04..10eb29266--- a/engine/crates/xerj-autoindex/src/walk.rs+++ b/engine/crates/xerj-autoindex/src/walk.rs@@@ -26,136 -23,87 +23,148 @@@ pub struct FileEntrypub size: u64,}- /// A directory pruned whole by the generated-dir hygiene rule, recorded so- /// the run can surface it (junk report / map) instead of skipping silently.- #[derive(Debug, Clone)]- pub struct SkippedDir {- pub rel: String,- pub reason: String,- }-- /// Well-known generated/cache directories, pruned at ANY depth but ONLY when- /// a sibling marker file proves the parent directory is that kind of project- /// (a monorepo may nest e.g. `apps/game/Library`, `apps/web/node_modules`).- /// Marker-gated like the dotfile rule (hygiene, evidence-based) — never- /// extension guessing, never a per-corpus code path.- /// `--no-default-excludes` disables the rule.++/// Generated/cache directories whose NAMES are too common for the blunt++/// [`crate::ignore_rules::DEFAULT_IGNORE_PATTERNS`] (a folder named `Logs/`++/// or `Library/` is often real data), pruned at any depth ONLY when a sibling++/// marker file proves the parent directory is that kind of project. Part of++/// the built-in defaults: `--no-ignore` / `--no-default-ignores` disable it,++/// and any ignore file in the tree can re-include (`!Library/`) as usual —++/// these run after the ignore files have had their say.+///- /// (dir name, marker path relative to the dir's PARENT, reason)- const GENERATED_DIR_RULES: &[(&str, &str, &str)] = &[++/// (dir name, marker path relative to the dir's PARENT, report label)++const MARKER_GENERATED_DIRS: &[(&str, &str, &str)] = &[+ (+ "Library",+ "ProjectSettings/ProjectVersion.txt",- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",++ "default:unity-generated",+ ),+ (+ "Temp",+ "ProjectSettings/ProjectVersion.txt",- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",++ "default:unity-generated",+ ),+ (+ "obj",+ "ProjectSettings/ProjectVersion.txt",- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",++ "default:unity-generated",+ ),+ (+ "Logs",+ "ProjectSettings/ProjectVersion.txt",- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",++ "default:unity-generated",+ ),+ (+ "UserSettings",+ "ProjectSettings/ProjectVersion.txt",- "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",- ),- (- "node_modules",- "package.json",- "installed npm/yarn dependency tree (marker: sibling package.json)",- ),- (- "target",- "Cargo.toml",- "Cargo build directory (marker: sibling Cargo.toml)",++ "default:unity-generated",+ ),+];+- pub fn walk(++/// The marker-gated verdict for one directory entry.++fn marker_generated_dir(path: &Path) -> Option<&'static str> {++ let name = path.file_name()?;++ for (dir, marker, label) in MARKER_GENERATED_DIRS {++ if name == *dir && path.parent().is_some_and(|p| p.join(marker).is_file()) {++ return Some(label);++ }++ }++ None++}+++ /// Walk with the default ignore rules on and the report discarded.+ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {+ walk_reporting(root, follow_symlinks, IgnoreOptions::default()).map(|(files, _)| files)+ }++ /// Walk, and say what was left out and why.+ ///+ /// The returned [`IgnoreReport`] is the answer to "my files vanished" — it+ /// names every rule that discarded something and how much. It is reported on+ /// every run and, under `--dry-run`, also counts what was inside each pruned+ /// directory.+ pub fn walk_reporting(root: &Path,follow_symlinks: bool,- default_excludes: bool,- gitignore: bool,- ) -> Result<(Vec<FileEntry>, Vec<SkippedDir>)> {+ ignore: IgnoreOptions,+ ) -> Result<(Vec<FileEntry>, IgnoreReport)> {let root_canon = root.canonicalize().with_context(|| format!("resolve root folder {}", root.display()))?;if !root_canon.is_dir() {anyhow::bail!("{} is not a directory", root_canon.display());}- // Arc<Mutex>: the `ignore` walker's filter must be Send+Sync even for the- // sequential build() we use.- let skipped: std::sync::Arc<std::sync::Mutex<Vec<SkippedDir>>> = Default::default();let mut out = Vec::new();- // SECURITY / hygiene: never index hidden files or descend into hidden- // directories (`hidden(true)`). Without this the walker happily indexed- // `.env` (secrets, API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles- // into a queryable brain with no per-brain authorization — a real exposure- // for the "point it at my project folder" use case. The root itself is- // exempt (the walk origin is always yielded) so a brain over a dot-named- // folder still works.- //- // `.gitignore` handling stays deterministic and corpus-local: tree- // `.gitignore`s + `.git/info/exclude` only, never the user's global- // gitignore config; `require_git` keeps the semantics evidence-based (a- // stray .gitignore outside a repo is not an owner declaration).- let sk = std::sync::Arc::clone(&skipped);- let filter_root = root_canon.clone();- let mut builder = ignore::WalkBuilder::new(&root_canon);- builder++ let marker_defaults = ignore.enabled && ignore.defaults;+ let mut stack = IgnoreStack::new(&root_canon, ignore);+ // Manual iteration rather than `filter_entry`, because the ignore stack has+ // to load a directory's ignore files *after* that directory is admitted and+ // *before* its children are judged. `skip_current_dir` on a just-yielded+ // directory prunes its contents — walkdir's own documented pattern, and the+ // same effect `filter_entry` had.+ let mut it = walkdir::WalkDir::new(&root_canon).follow_links(follow_symlinks)- .hidden(true)- .ignore(false)- .parents(gitignore)- .git_ignore(gitignore)- .git_exclude(gitignore)- .git_global(false)- .require_git(true)- .filter_entry(move |e| {- if e.depth() == 0 {- return true;- }- if default_excludes && e.file_type().is_some_and(|t| t.is_dir()) {- for (name, marker, reason) in GENERATED_DIR_RULES {- if e.file_name() == *name- && e.path()- .parent()- .is_some_and(|p| p.join(marker).is_file())- {- let rel = e- .path()- .strip_prefix(&filter_root)- .unwrap_or(e.path())- .to_string_lossy()- .replace('\\', "/");- sk.lock().expect("skip list lock").push(SkippedDir {- rel,- reason: (*reason).to_string(),- });- return false;- }- }- }- true- });- for entry in builder.build() {- let entry = match entry {+ .into_iter();+ while let Some(next) = it.next() {+ let entry = match next {Ok(e) => e,Err(e) => {eprintln!("walk: skipping unreadable entry: {e}");continue;}};- if !entry.file_type().is_some_and(|t| t.is_file()) {+ let is_dir = entry.file_type().is_dir();+ // Retire the layers of directories this entry is no longer inside; see+ // `IgnoreStack::truncate_to` for why every entry needs it, not just+ // directories.+ stack.truncate_to(entry.depth());+ // SECURITY / hygiene: never index hidden files or descend into hidden+ // directories. Without this the walker happily indexed `.env` (secrets,+ // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a+ // queryable brain with no per-brain authorization — a real exposure for+ // the "point it at my project folder" use case. A hidden directory is+ // pruned before descending, so `.git/` is skipped whole. The root+ // itself (depth 0) is exempt so a brain over a dot-named folder still+ // works. `--no-ignore` does NOT turn this off: it is not one of the+ // ignore rules, it is the reason secrets stay out.+ if entry.depth() > 0+ && entry+ .file_name()+ .to_str()+ .is_some_and(|n| n.starts_with('.'))+ {+ stack.record_hidden(entry.path(), is_dir);+ if is_dir {+ it.skip_current_dir();+ }+ continue;+ }+ if is_dir {+ // The root is never judged by the rules below it: pointing+ // autoindex at a folder is an explicit instruction, and the note+ // explaining that it *would* have been ignored is on the report.+ if entry.depth() > 0 && stack.skip_dir(entry.path()).is_some() {+ it.skip_current_dir();+ continue;+ }++ // Marker-gated generated dirs (part of the built-in defaults, so++ // `--no-ignore` / `--no-default-ignores` disable it): names like++ // Unity's `Library/` or `Logs/` are too common for the blunt++ // DEFAULT_IGNORE_PATTERNS, so they are pruned only when a sibling++ // marker file proves the parent is that kind of project.++ if entry.depth() > 0 && marker_defaults {++ if let Some(label) = marker_generated_dir(entry.path()) {++ stack.record_marker_dir(entry.path(), label);++ it.skip_current_dir();++ continue;++ }++ }+ stack.enter_dir(entry.path(), entry.depth());+ continue;+ }+ if !entry.file_type().is_file() {+ continue;+ }+ if stack.skip_file(entry.path()).is_some() {continue;}let md = match entry.metadata() {@@@ -268,209 -257,3 +318,96 @@@ mod hidden_skip_tests);}}++#[cfg(test)]- mod generated_dir_tests {- use super::walk;++mod marker_generated_dir_tests {++ use super::{walk, walk_reporting};++ use crate::ignore_rules::IgnoreOptions;+ use std::fs;+ use std::path::Path;+- fn make_unity_root(root: &Path, with_marker: bool) {++ fn make_unity_project(root: &Path, with_marker: bool) {+ fs::create_dir_all(root.join("Assets")).unwrap();+ fs::write(root.join("Assets/Player.cs"), "class Player {}").unwrap();+ fs::create_dir_all(root.join("Library/Artifacts")).unwrap();+ fs::write(root.join("Library/Artifacts/blob.bin"), b"\x00\x01").unwrap();- fs::create_dir_all(root.join("Temp")).unwrap();- fs::write(root.join("Temp/scratch"), "x").unwrap();++ fs::create_dir_all(root.join("Logs")).unwrap();++ fs::write(root.join("Logs/editor.log"), "x").unwrap();+ if with_marker {+ fs::create_dir_all(root.join("ProjectSettings")).unwrap();+ fs::write(+ root.join("ProjectSettings/ProjectVersion.txt"),+ "m_EditorVersion: 2022.3.10f1\n",+ )+ .unwrap();+ }+ }+++ /// Monorepo case: the Unity project is a SUBFOLDER of the walk root; the++ /// sibling marker decides at any depth.+ #[test]- fn unity_marker_prunes_generated_dirs_and_records_them() {++ fn a_nested_unity_project_is_pruned_and_reported() {+ let dir = tempfile::TempDir::new().unwrap();- make_unity_root(dir.path(), true);- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();++ let unity = dir.path().join("platform/unity");++ fs::create_dir_all(&unity).unwrap();++ make_unity_project(&unity, true);++ let (files, report) =++ walk_reporting(dir.path(), false, IgnoreOptions::default()).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(rels.contains(&"Assets/Player.cs"));++ assert!(rels.contains(&"platform/unity/Assets/Player.cs"));+ assert!(- !rels.iter().any(|r| r.starts_with("Library/")),++ !rels.iter().any(|r| r.contains("/Library/")),+ "Library/ must be pruned whole: {rels:?}"+ );- assert!(!rels.iter().any(|r| r.starts_with("Temp/")));- let mut skipped_rels: Vec<&str> = skipped.iter().map(|s| s.rel.as_str()).collect();- skipped_rels.sort();- assert_eq!(skipped_rels, ["Library", "Temp"], "each pruned dir is recorded");++ assert!(++ report++ .by_rule++ .keys()++ .any(|k| k == "default:unity-generated"),++ "the prune must be named on the report: {report:?}"++ );+ }+++ /// Without the marker, a folder merely NAMED Library or Logs is data.+ #[test]+ fn without_the_marker_nothing_is_pruned() {+ let dir = tempfile::TempDir::new().unwrap();- make_unity_root(dir.path(), false);- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();++ make_unity_project(dir.path(), false);++ let files = walk(dir.path(), false).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(- rels.iter().any(|r| r.starts_with("Library/")),- "a folder merely NAMED Library is not evidence: {rels:?}"- );- assert!(skipped.is_empty());++ assert!(rels.contains(&"Library/Artifacts/blob.bin"), "{rels:?}");++ assert!(rels.contains(&"Logs/editor.log"), "{rels:?}");+ }+++ /// The marker rule is one of the built-in defaults, so both switches++ /// disable it.+ #[test]- fn no_default_excludes_walks_everything() {++ fn no_ignore_and_no_default_ignores_both_disable_the_marker_rule() {+ let dir = tempfile::TempDir::new().unwrap();- make_unity_root(dir.path(), true);- let (files, skipped) = walk(dir.path(), false, false, true).unwrap();- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(rels.iter().any(|r| r.starts_with("Library/")));- assert!(skipped.is_empty());++ make_unity_project(dir.path(), true);++ let no_defaults = IgnoreOptions {++ defaults: false,++ ..IgnoreOptions::default()++ };++ for opts in [IgnoreOptions::off(), no_defaults] {++ let (files, _) = walk_reporting(dir.path(), false, opts).unwrap();++ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();++ assert!(++ rels.contains(&"Library/Artifacts/blob.bin"),++ "opts {opts:?} must walk generated dirs: {rels:?}"++ );++ }+ }++ #[test]+ fn a_file_named_like_a_generated_dir_is_not_pruned() {+ let dir = tempfile::TempDir::new().unwrap();- make_unity_root(dir.path(), true);++ make_unity_project(dir.path(), true);+ fs::write(dir.path().join("obj"), "a FILE named obj").unwrap();- let (files, _) = walk(dir.path(), false, true, true).unwrap();++ let files = walk(dir.path(), false).unwrap();+ assert!(+ files.iter().any(|e| e.rel == "obj"),+ "the prune applies to directories only"+ );+ }-- /// Monorepo case: the Unity project is a SUBFOLDER of the walk root. The- /// marker is checked against each candidate's parent, so nesting depth is- /// irrelevant.- #[test]- fn a_nested_unity_project_is_pruned_by_its_sibling_marker() {- let dir = tempfile::TempDir::new().unwrap();- let unity = dir.path().join("apps/game");- fs::create_dir_all(&unity).unwrap();- make_unity_root(&unity, true);- fs::write(dir.path().join("README.md"), "monorepo").unwrap();- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(rels.contains(&"apps/game/Assets/Player.cs"));- assert!(!rels.iter().any(|r| r.contains("game/Library/")));- assert!(skipped.iter().any(|s| s.rel == "apps/game/Library"));- }-- #[test]- fn node_modules_is_pruned_only_next_to_a_package_json() {- let dir = tempfile::TempDir::new().unwrap();- let app = dir.path().join("web");- fs::create_dir_all(app.join("node_modules/lodash")).unwrap();- fs::write(app.join("node_modules/lodash/index.js"), "x").unwrap();- fs::write(app.join("package.json"), "{}").unwrap();- fs::write(app.join("index.js"), "x").unwrap();- let bare = dir.path().join("data/node_modules");- fs::create_dir_all(&bare).unwrap();- fs::write(bare.join("notes.txt"), "not an npm tree").unwrap();- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(rels.contains(&"web/index.js"));- assert!(!rels.iter().any(|r| r.starts_with("web/node_modules")));- assert!(- rels.contains(&"data/node_modules/notes.txt"),- "without the package.json marker there is no evidence: {rels:?}"- );- assert_eq!(skipped.len(), 1);- assert_eq!(skipped[0].rel, "web/node_modules");- }-- #[test]- fn gitignored_files_are_skipped_only_inside_a_git_repo() {- let dir = tempfile::TempDir::new().unwrap();- let root = dir.path();- fs::create_dir(root.join(".git")).unwrap();- fs::write(root.join(".gitignore"), "build/\n*.log\n").unwrap();- fs::create_dir_all(root.join("build/intermediates")).unwrap();- fs::write(root.join("build/intermediates/R.txt"), "generated").unwrap();- fs::write(root.join("debug.log"), "noise").unwrap();- fs::write(root.join("main.rs"), "fn main() {}").unwrap();- fs::create_dir(root.join("src")).unwrap();- fs::write(root.join("src/app.log"), "noise").unwrap();- fs::write(root.join("src/lib.rs"), "pub fn f() {}").unwrap();-- let (files, _) = walk(root, false, true, true).unwrap();- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(rels.contains(&"main.rs"));- assert!(rels.contains(&"src/lib.rs"));- assert!(- !rels.iter().any(|r| r.starts_with("build/")),- "gitignored dir must be pruned: {rels:?}"- );- assert!(- !rels.iter().any(|r| r.ends_with(".log")),- "gitignored glob applies at every depth: {rels:?}"- );-- // --no-gitignore restores the full walk.- let (files, _) = walk(root, false, true, false).unwrap();- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(rels.contains(&"build/intermediates/R.txt"));- assert!(rels.contains(&"debug.log"));- }-- /// A `.gitignore` OUTSIDE a git repository is not an owner declaration —- /// require_git keeps the rule evidence-based.- #[test]- fn a_gitignore_without_a_git_repo_is_inert() {- let dir = tempfile::TempDir::new().unwrap();- let root = dir.path();- fs::write(root.join(".gitignore"), "*.log\n").unwrap();- fs::write(root.join("debug.log"), "kept").unwrap();- let (files, _) = walk(root, false, true, true).unwrap();- assert!(- files.iter().any(|e| e.rel == "debug.log"),- "no .git dir, so gitignore semantics must not apply"- );- }-- /// Nested .gitignore files scope to their own subtree, like git itself.- #[test]- fn nested_gitignore_files_scope_to_their_subtree() {- let dir = tempfile::TempDir::new().unwrap();- let root = dir.path();- fs::create_dir(root.join(".git")).unwrap();- fs::create_dir(root.join("web")).unwrap();- fs::write(root.join("web/.gitignore"), "dist/\n").unwrap();- fs::create_dir_all(root.join("web/dist")).unwrap();- fs::write(root.join("web/dist/bundle.js"), "x").unwrap();- fs::create_dir_all(root.join("docs/dist")).unwrap();- fs::write(root.join("docs/dist/manual.md"), "kept").unwrap();- let (files, _) = walk(root, false, true, true).unwrap();- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(!rels.iter().any(|r| r.starts_with("web/dist/")));- assert!(rels.contains(&"docs/dist/manual.md"));- }-- #[test]- fn cargo_target_is_pruned_only_next_to_a_cargo_toml() {- let dir = tempfile::TempDir::new().unwrap();- let crate_dir = dir.path().join("mycrate");- fs::create_dir_all(crate_dir.join("target/debug")).unwrap();- fs::write(crate_dir.join("target/debug/build.log"), "x").unwrap();- fs::write(crate_dir.join("Cargo.toml"), "[package]").unwrap();- fs::write(crate_dir.join("main.rs"), "fn main() {}").unwrap();- let plain = dir.path().join("shooting/target");- fs::create_dir_all(&plain).unwrap();- fs::write(plain.join("scores.csv"), "a,b\n1,2\n").unwrap();- let (files, skipped) = walk(dir.path(), false, true, true).unwrap();- let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();- assert!(rels.contains(&"mycrate/main.rs"));- assert!(!rels.iter().any(|r| r.starts_with("mycrate/target")));- assert!(- rels.contains(&"shooting/target/scores.csv"),- "a folder merely NAMED target is data, not a build dir: {rels:?}"- );- assert_eq!(skipped.len(), 1);- assert_eq!(skipped[0].rel, "mycrate/target");- }+}diff --cc engine/crates/xerj-server/src/brain.rsindex 4951aa6ba,cddd4c61e..59d0aa779--- a/engine/crates/xerj-server/src/brain.rs+++ b/engine/crates/xerj-server/src/brain.rs@@@ -432,9 -515,10 +515,11 @@@ fn index_cfg(cfg: &BrainCfg, brain: &ststate_dir: None,fresh: cfg.fresh,follow_symlinks: false,- no_default_excludes: false,+ stub_globs: Vec::new(),- no_gitignore: false,+ // `xerj brain` composes autoindex, so it inherits its ignore rules+ // rather than inventing its own: build output does not belong in a+ // second brain either (#276).+ ignore: xerj_autoindex::ignore_rules::IgnoreOptions::default(),max_file_gb: 2,sample: 500,no_semantic: false,