AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
test(autoindex): scope the phase-A parse count to the run that made it (#891) (#894)
xerj-org389164c323a8
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 5125290b0..0fdf40d12 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -3113,6 +3113,7 @@ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":capacity_warning: None,progress: &progress,meter: &crate::estimate::Meter::new(),+ tally: &crate::ScanTally::default(),},100,1,@@ -3201,6 +3202,7 @@ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":capacity_warning: None,progress: &progress,meter: &crate::estimate::Meter::new(),+ tally: &crate::ScanTally::default(),},100,1,@@ -3280,6 +3282,7 @@ fn verified_but_junk_pdf_spool_is_created_then_discarded_not_eligible() {capacity_warning: None,progress: &progress,meter: &crate::estimate::Meter::new(),+ tally: &crate::ScanTally::default(),},100,1,diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex f5f106130..8c0978f9c 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -999,12 +999,16 @@ fn initial_generation_and_noop_are_end_to_end_idempotent() {/// Incremental re-index must NOT re-run the Phase-A scan/parse for files whose/// content is byte-identical to the committed generation — that is the ~100x/// win for the edit-and-rerun / CI workflow, since the tree-sitter parse-/// dominates per-file cost. Proven directly with the `SCAN_FILE_PARSED` counter+/// dominates per-file cost. Proven directly with the run's own `ScanTally`/// (0 on an unchanged re-run, exactly the changed count when one file changes),/// and the committed index stays byte-identical.+///+/// #891: each measurement uses a tally created for THAT run. The counter used+/// to be a process-global `static`, which meant this test read whatever every+/// other test in the binary happened to parse in the same window — see+/// `another_runs_parses_never_land_in_this_runs_tally` below.#[test]fn unchanged_reindex_skips_the_parse_and_stays_byte_identical() {- use std::sync::atomic::Ordering;let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock()@@ -1022,15 +1026,21 @@ fn unchanged_reindex_skips_the_parse_and_stays_byte_identical() {let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);// Genesis: everything is parsed.- assert_eq!(run_index(config.clone()).unwrap(), 0);+ let genesis = crate::ScanTally::default();+ assert_eq!(run_index_tallied(config.clone(), &genesis).unwrap(), 0);+ assert_eq!(+ genesis.files_parsed(),+ 2,+ "genesis has no committed generation to skip against: both files are parsed"+ );let first_docs = endpoint.data_docs();assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);// Re-index with NOTHING changed → zero files parsed, index byte-identical.- crate::SCAN_FILE_PARSED.store(0, Ordering::Relaxed);- assert_eq!(run_index(config.clone()).unwrap(), 0);+ let unchanged = crate::ScanTally::default();+ assert_eq!(run_index_tallied(config.clone(), &unchanged).unwrap(), 0);assert_eq!(- crate::SCAN_FILE_PARSED.load(Ordering::Relaxed),+ unchanged.files_parsed(),0,"unchanged re-index must parse zero files");@@ -1048,15 +1058,111 @@ fn unchanged_reindex_skips_the_parse_and_stays_byte_identical() {"def add(a, b):\n return a + b + 1\n",).unwrap();- crate::SCAN_FILE_PARSED.store(0, Ordering::Relaxed);- assert_eq!(run_index(config.clone()).unwrap(), 0);+ let one_changed = crate::ScanTally::default();+ assert_eq!(run_index_tallied(config.clone(), &one_changed).unwrap(), 0);assert_eq!(- crate::SCAN_FILE_PARSED.load(Ordering::Relaxed),+ one_changed.files_parsed(),1,"only the one changed file must be parsed; the unchanged file stays skipped");}+/// #891: a parse measurement must describe the run that made it, and nothing+/// else the process is doing.+///+/// The counter above used to be one process-global `static`. `cargo test` runs+/// this lib test binary multi-threaded, the measuring test only holds+/// `HTTP_E2E_LOCK`, and every other test that parses files — those under+/// `FAILPOINT_TEST_LOCK`, `generation_catalog_http_tests`, plain unit tests —+/// added to the same number. Both assertions above could therefore read a total+/// that was not theirs: 2 of 6 and then 1 of 6 local suite runs on the branch+/// that found it (#886) went red here with no code change between them. It is a+/// scheduling race, so plain `main` can also sit clean for twenty consecutive+/// runs — which is exactly why the proof below does not rely on luck.+///+/// This reproduces that interference deterministically instead of waiting for+/// the scheduler to reproduce it: a SECOND, unrelated corpus is indexed inside+/// the first run's measurement window — after its run finishes, before its+/// number is read. That is the event the flake needed, sequenced rather than+/// raced; a genuinely concurrent second run would trade this flake for a new+/// one, since these fixtures are serialised by `HTTP_E2E_LOCK` for reasons of+/// their own. Against the old global counter the unchanged re-index reads the+/// interfering run's 2 parses instead of its own 0; against a per-run tally+/// each run reads only its own work.+#[test]+fn another_runs_parses_never_land_in_this_runs_tally() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());++ // The measured corpus, indexed once so that its re-index has something to+ // skip.+ let measured_corpus = tempfile::tempdir().unwrap();+ let measured_state = tempfile::tempdir().unwrap();+ fs::write(+ measured_corpus.path().join("main.py"),+ "def add(a, b):\n return a + b\n",+ )+ .unwrap();+ fs::write(+ measured_corpus.path().join("a.csv"),+ "id,value\n1,alpha\n2,beta\n",+ )+ .unwrap();+ let measured_endpoint = HttpEndpoint::start();+ let measured_config = cfg(+ measured_corpus.path(),+ measured_state.path(),+ &measured_endpoint.url,+ false,+ );+ assert_eq!(+ run_index_tallied(measured_config.clone(), &crate::ScanTally::default()).unwrap(),+ 0+ );++ // An unrelated corpus, standing in for whatever else the test binary is+ // running. Its own genesis parses two files.+ let other_corpus = tempfile::tempdir().unwrap();+ let other_state = tempfile::tempdir().unwrap();+ fs::write(+ other_corpus.path().join("other.py"),+ "def mul(a, b):\n return a * b\n",+ )+ .unwrap();+ fs::write(+ other_corpus.path().join("other.csv"),+ "id,value\n7,gamma\n8,delta\n",+ )+ .unwrap();+ let other_endpoint = HttpEndpoint::start();+ let other_config = cfg(+ other_corpus.path(),+ other_state.path(),+ &other_endpoint.url,+ false,+ );++ // The measurement window: measure, let the other run happen, then read.+ let measured = crate::ScanTally::default();+ assert_eq!(run_index_tallied(measured_config, &measured).unwrap(), 0);+ let other = crate::ScanTally::default();+ assert_eq!(run_index_tallied(other_config, &other).unwrap(), 0);++ assert_eq!(+ other.files_parsed(),+ 2,+ "the interfering run really did parse files — otherwise this proves nothing"+ );+ assert_eq!(+ measured.files_parsed(),+ 0,+ "an unchanged re-index parses nothing; another run's parses must not be \+ added to this run's tally (#891)"+ );+}+#[test]fn add_change_delete_and_rename_converge_over_real_http() {let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex f40d36b8e..b5f8597e9 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -532,6 +532,7 @@ fn project_reconcile_plan(state_dir: &Path,pr: &Progress,meter: &estimate::Meter,+ tally: &ScanTally,) -> Result<Plan> {let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);let ctx = PhaseAContext {@@ -540,6 +541,7 @@ fn project_reconcile_plan(capacity_warning: None,progress: pr,meter,+ tally,};pr.phase("scan",@@ -1566,14 +1568,45 @@ struct PhaseAContext<'a> {/// every file; timing that parse is the only way to price the run on the/// machine it will actually run on rather than on ours.meter: &'a estimate::Meter,+ /// How much phase-A work THIS run did. Owned by the run, not by the+ /// process — see [`ScanTally`].+ tally: &'a ScanTally,}-/// Count of files that actually ran the Phase-A scan/parse. The incremental-/// re-index fast path skips this for byte-identical files, so a test can assert-/// that an unchanged re-run parses zero files. `Relaxed` — a single counter per-/// scanned file, negligible against the parse it guards.-pub(crate) static SCAN_FILE_PARSED: std::sync::atomic::AtomicUsize =- std::sync::atomic::AtomicUsize::new(0);+/// Files that actually ran the Phase-A scan/parse, counted **per run**.+///+/// The incremental re-index fast path skips the parse for byte-identical+/// files, so this is how a test proves an unchanged re-run parses zero files.+///+/// It used to be a process-global `static`, which made that proof unsound+/// (#891): `cargo test` runs one lib test binary multi-threaded, every+/// concurrent run added to the same number, and a test reading it after its own+/// run got a total that included whatever the other tests parsed in the+/// meantime — a false red measured at 2 of 6 and 1 of 6 local suite runs on the+/// branch that found it (#886). One tally is created per `run_index_report`+/// invocation and reaches phase A through [`PhaseAContext`], so cross-run+/// interference is now unrepresentable rather than merely unlikely.+///+/// `Relaxed`: one increment per scanned file, ordered against nothing — the+/// cost has to stay negligible against the parse it counts, and readers only+/// look after their run has joined every phase-A thread.+#[derive(Debug, Default)]+pub(crate) struct ScanTally {+ parsed: std::sync::atomic::AtomicUsize,+}++impl ScanTally {+ fn record_parse(&self) {+ self.parsed+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ }++ /// Files this run ran the Phase-A scan/parse for.+ #[cfg(test)]+ pub(crate) fn files_parsed(&self) -> usize {+ self.parsed.load(std::sync::atomic::Ordering::Relaxed)+ }+}fn scan_file(path: &Path,@@ -1584,7 +1617,7 @@ fn scan_file(max_file_gb: u64,stub: bool,) -> FileScan {- SCAN_FILE_PARSED.fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ ctx.tally.record_parse();let state_dir = ctx.state_dir;let pdf_spool_budget = ctx.budget;let mut out = FileScan {@@ -1791,6 +1824,7 @@ mod clustering_key_tests {capacity_warning: None,progress: &progress,meter: &meter,+ tally: &ScanTally::default(),};scan_file(&path, size, "d0", &ctx, 500, 2, false)}@@ -1930,6 +1964,7 @@ mod phase_a_grouping_tests {capacity_warning: None,progress: &progress,meter: &meter,+ tally: &ScanTally::default(),};build_phase_a(root,@@ -3772,6 +3807,14 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {run_index_report(cfg).map(|(code, _)| code)}+/// `run_index` with the caller's own phase-A tally, so a test can measure the+/// work of *its* run and nothing else's (#891). Production has no reason to+/// look at the number, which is why the public entry points own a throwaway.+#[cfg(test)]+fn run_index_tallied(cfg: IndexCfg, tally: &ScanTally) -> Result<i32> {+ run_index_report_tallied(cfg, tally).map(|(code, _)| code)+}+/// #381/#759: the per-file record cap dropped these files' tails. Name them (up/// to a bound) so a truncated document is identifiable on a multi-file corpus,/// not merely counted — the "never silent" guarantee is only useful if the user@@ -3848,6 +3891,18 @@ mod truncation_note_tests {/// `None` when the run ended before a plan produced one (empty folder,/// `--dry-run`).pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {+ run_index_report_tallied(cfg, &ScanTally::default())+}++/// The body of [`run_index_report`], with the run's phase-A tally supplied by+/// the caller instead of created here.+///+/// The tally is per-invocation on purpose: it replaced a process-global counter+/// that every concurrent run in the process wrote to (#891). Every phase-A+/// route this function can take — the legacy `build_phase_a` scan and the+/// generated route's `project_reconcile_plan` — is handed this one tally, so+/// "files parsed by this run" is exactly what it holds.+fn run_index_report_tallied(cfg: IndexCfg, tally: &ScanTally) -> Result<(i32, Option<Value>)> {// The very first statement of the function, deliberately: `started` must// be when this invocation began, not when its summary was built.let invocation_started = chrono::Utc::now();@@ -4156,8 +4211,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {.committed_manifest.as_ref().context("branch guard proved a committed manifest")?;- let plan =- project_reconcile_plan(&inventory, &base.plan, &cfg, &state_dir, &pr, &scan_meter)?;+ let plan = project_reconcile_plan(+ &inventory,+ &base.plan,+ &cfg,+ &state_dir,+ &pr,+ &scan_meter,+ tally,+ )?;let unchanged = serde_json::to_value(&plan)? == serde_json::to_value(&base.plan)?;println!("{}", serde_json::to_string_pretty(&plan)?);// stdout is the RESULT, stderr is PROGRESS: the projection above is@@ -4213,8 +4275,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {.as_ref().context("generated journal lost its committed manifest")?.clone();- let plan =- project_reconcile_plan(&inventory, &base.plan, &cfg, &state_dir, &pr, &scan_meter)?;+ let plan = project_reconcile_plan(+ &inventory,+ &base.plan,+ &cfg,+ &state_dir,+ &pr,+ &scan_meter,+ tally,+ )?;// Say out loud that the decision gate is not on this route. It is an// incremental reconcile of an already-committed generation, so the work// is the *changed* set and is published from a sealed snapshot rather@@ -4625,6 +4694,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {capacity_warning: pdf_spool_capacity_warning.as_deref(),progress: &pr,meter: &scan_meter,+ tally,};let mut plan: Plan = if let Some(p) = journal.plan.clone().filter(|_| !genesis_recovery) {p@@ -8618,6 +8688,7 @@ mod unity_pipeline_tests {capacity_warning: None,progress: &progress,meter: &meter,+ tally: &ScanTally::default(),};let mut cfg = super::phase_a_grouping_tests::cfg_for(root);cfg.sample = sample;