Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

test(autoindex): scope the phase-A parse count to the run that made it (#891) (#894)

xerj-org389164c323a8
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 5125290b0..0fdf40d12 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -3113,6 +3113,7 @@ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":
capacity_warning: None,
progress: &progress,
meter: &crate::estimate::Meter::new(),
+ tally: &crate::ScanTally::default(),
},
100,
1,
@@ -3201,6 +3202,7 @@ printf '%s' '{"schema":1,"extractor":"xerj-autoindex/__XERJ_VERSION__","parser":
capacity_warning: None,
progress: &progress,
meter: &crate::estimate::Meter::new(),
+ tally: &crate::ScanTally::default(),
},
100,
1,
@@ -3280,6 +3282,7 @@ fn verified_but_junk_pdf_spool_is_created_then_discarded_not_eligible() {
capacity_warning: None,
progress: &progress,
meter: &crate::estimate::Meter::new(),
+ tally: &crate::ScanTally::default(),
},
100,
1,
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index f5f106130..8c0978f9c 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -999,12 +999,16 @@ fn initial_generation_and_noop_are_end_to_end_idempotent() {
/// Incremental re-index must NOT re-run the Phase-A scan/parse for files whose
/// content is byte-identical to the committed generation — that is the ~100x
/// win for the edit-and-rerun / CI workflow, since the tree-sitter parse
-/// dominates per-file cost. Proven directly with the `SCAN_FILE_PARSED` counter
+/// dominates per-file cost. Proven directly with the run's own `ScanTally`
/// (0 on an unchanged re-run, exactly the changed count when one file changes),
/// and the committed index stays byte-identical.
+///
+/// #891: each measurement uses a tally created for THAT run. The counter used
+/// to be a process-global `static`, which meant this test read whatever every
+/// other test in the binary happened to parse in the same window — see
+/// `another_runs_parses_never_land_in_this_runs_tally` below.
#[test]
fn unchanged_reindex_skips_the_parse_and_stays_byte_identical() {
- use std::sync::atomic::Ordering;
let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK
.lock()
@@ -1022,15 +1026,21 @@ fn unchanged_reindex_skips_the_parse_and_stays_byte_identical() {
let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
// Genesis: everything is parsed.
- assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let genesis = crate::ScanTally::default();
+ assert_eq!(run_index_tallied(config.clone(), &genesis).unwrap(), 0);
+ assert_eq!(
+ genesis.files_parsed(),
+ 2,
+ "genesis has no committed generation to skip against: both files are parsed"
+ );
let first_docs = endpoint.data_docs();
assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
// Re-index with NOTHING changed → zero files parsed, index byte-identical.
- crate::SCAN_FILE_PARSED.store(0, Ordering::Relaxed);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let unchanged = crate::ScanTally::default();
+ assert_eq!(run_index_tallied(config.clone(), &unchanged).unwrap(), 0);
assert_eq!(
- crate::SCAN_FILE_PARSED.load(Ordering::Relaxed),
+ unchanged.files_parsed(),
0,
"unchanged re-index must parse zero files"
);
@@ -1048,15 +1058,111 @@ fn unchanged_reindex_skips_the_parse_and_stays_byte_identical() {
"def add(a, b):\n return a + b + 1\n",
)
.unwrap();
- crate::SCAN_FILE_PARSED.store(0, Ordering::Relaxed);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let one_changed = crate::ScanTally::default();
+ assert_eq!(run_index_tallied(config.clone(), &one_changed).unwrap(), 0);
assert_eq!(
- crate::SCAN_FILE_PARSED.load(Ordering::Relaxed),
+ one_changed.files_parsed(),
1,
"only the one changed file must be parsed; the unchanged file stays skipped"
);
}
+/// #891: a parse measurement must describe the run that made it, and nothing
+/// else the process is doing.
+///
+/// The counter above used to be one process-global `static`. `cargo test` runs
+/// this lib test binary multi-threaded, the measuring test only holds
+/// `HTTP_E2E_LOCK`, and every other test that parses files — those under
+/// `FAILPOINT_TEST_LOCK`, `generation_catalog_http_tests`, plain unit tests —
+/// added to the same number. Both assertions above could therefore read a total
+/// that was not theirs: 2 of 6 and then 1 of 6 local suite runs on the branch
+/// that found it (#886) went red here with no code change between them. It is a
+/// scheduling race, so plain `main` can also sit clean for twenty consecutive
+/// runs — which is exactly why the proof below does not rely on luck.
+///
+/// This reproduces that interference deterministically instead of waiting for
+/// the scheduler to reproduce it: a SECOND, unrelated corpus is indexed inside
+/// the first run's measurement window — after its run finishes, before its
+/// number is read. That is the event the flake needed, sequenced rather than
+/// raced; a genuinely concurrent second run would trade this flake for a new
+/// one, since these fixtures are serialised by `HTTP_E2E_LOCK` for reasons of
+/// their own. Against the old global counter the unchanged re-index reads the
+/// interfering run's 2 parses instead of its own 0; against a per-run tally
+/// each run reads only its own work.
+#[test]
+fn another_runs_parses_never_land_in_this_runs_tally() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+
+ // The measured corpus, indexed once so that its re-index has something to
+ // skip.
+ let measured_corpus = tempfile::tempdir().unwrap();
+ let measured_state = tempfile::tempdir().unwrap();
+ fs::write(
+ measured_corpus.path().join("main.py"),
+ "def add(a, b):\n return a + b\n",
+ )
+ .unwrap();
+ fs::write(
+ measured_corpus.path().join("a.csv"),
+ "id,value\n1,alpha\n2,beta\n",
+ )
+ .unwrap();
+ let measured_endpoint = HttpEndpoint::start();
+ let measured_config = cfg(
+ measured_corpus.path(),
+ measured_state.path(),
+ &measured_endpoint.url,
+ false,
+ );
+ assert_eq!(
+ run_index_tallied(measured_config.clone(), &crate::ScanTally::default()).unwrap(),
+ 0
+ );
+
+ // An unrelated corpus, standing in for whatever else the test binary is
+ // running. Its own genesis parses two files.
+ let other_corpus = tempfile::tempdir().unwrap();
+ let other_state = tempfile::tempdir().unwrap();
+ fs::write(
+ other_corpus.path().join("other.py"),
+ "def mul(a, b):\n return a * b\n",
+ )
+ .unwrap();
+ fs::write(
+ other_corpus.path().join("other.csv"),
+ "id,value\n7,gamma\n8,delta\n",
+ )
+ .unwrap();
+ let other_endpoint = HttpEndpoint::start();
+ let other_config = cfg(
+ other_corpus.path(),
+ other_state.path(),
+ &other_endpoint.url,
+ false,
+ );
+
+ // The measurement window: measure, let the other run happen, then read.
+ let measured = crate::ScanTally::default();
+ assert_eq!(run_index_tallied(measured_config, &measured).unwrap(), 0);
+ let other = crate::ScanTally::default();
+ assert_eq!(run_index_tallied(other_config, &other).unwrap(), 0);
+
+ assert_eq!(
+ other.files_parsed(),
+ 2,
+ "the interfering run really did parse files — otherwise this proves nothing"
+ );
+ assert_eq!(
+ measured.files_parsed(),
+ 0,
+ "an unchanged re-index parses nothing; another run's parses must not be \
+ added to this run's tally (#891)"
+ );
+}
+
#[test]
fn add_change_delete_and_rename_converge_over_real_http() {
let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index f40d36b8e..b5f8597e9 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -532,6 +532,7 @@ fn project_reconcile_plan(
state_dir: &Path,
pr: &Progress,
meter: &estimate::Meter,
+ tally: &ScanTally,
) -> Result<Plan> {
let budget = extract::pdf::ExtractionSpoolBudget::new(0, 0);
let ctx = PhaseAContext {
@@ -540,6 +541,7 @@ fn project_reconcile_plan(
capacity_warning: None,
progress: pr,
meter,
+ tally,
};
pr.phase(
"scan",
@@ -1566,14 +1568,45 @@ struct PhaseAContext<'a> {
/// every file; timing that parse is the only way to price the run on the
/// machine it will actually run on rather than on ours.
meter: &'a estimate::Meter,
+ /// How much phase-A work THIS run did. Owned by the run, not by the
+ /// process — see [`ScanTally`].
+ tally: &'a ScanTally,
}
-/// Count of files that actually ran the Phase-A scan/parse. The incremental
-/// re-index fast path skips this for byte-identical files, so a test can assert
-/// that an unchanged re-run parses zero files. `Relaxed` — a single counter per
-/// scanned file, negligible against the parse it guards.
-pub(crate) static SCAN_FILE_PARSED: std::sync::atomic::AtomicUsize =
- std::sync::atomic::AtomicUsize::new(0);
+/// Files that actually ran the Phase-A scan/parse, counted **per run**.
+///
+/// The incremental re-index fast path skips the parse for byte-identical
+/// files, so this is how a test proves an unchanged re-run parses zero files.
+///
+/// It used to be a process-global `static`, which made that proof unsound
+/// (#891): `cargo test` runs one lib test binary multi-threaded, every
+/// concurrent run added to the same number, and a test reading it after its own
+/// run got a total that included whatever the other tests parsed in the
+/// meantime — a false red measured at 2 of 6 and 1 of 6 local suite runs on the
+/// branch that found it (#886). One tally is created per `run_index_report`
+/// invocation and reaches phase A through [`PhaseAContext`], so cross-run
+/// interference is now unrepresentable rather than merely unlikely.
+///
+/// `Relaxed`: one increment per scanned file, ordered against nothing — the
+/// cost has to stay negligible against the parse it counts, and readers only
+/// look after their run has joined every phase-A thread.
+#[derive(Debug, Default)]
+pub(crate) struct ScanTally {
+ parsed: std::sync::atomic::AtomicUsize,
+}
+
+impl ScanTally {
+ fn record_parse(&self) {
+ self.parsed
+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ }
+
+ /// Files this run ran the Phase-A scan/parse for.
+ #[cfg(test)]
+ pub(crate) fn files_parsed(&self) -> usize {
+ self.parsed.load(std::sync::atomic::Ordering::Relaxed)
+ }
+}
fn scan_file(
path: &Path,
@@ -1584,7 +1617,7 @@ fn scan_file(
max_file_gb: u64,
stub: bool,
) -> FileScan {
- SCAN_FILE_PARSED.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ ctx.tally.record_parse();
let state_dir = ctx.state_dir;
let pdf_spool_budget = ctx.budget;
let mut out = FileScan {
@@ -1791,6 +1824,7 @@ mod clustering_key_tests {
capacity_warning: None,
progress: &progress,
meter: &meter,
+ tally: &ScanTally::default(),
};
scan_file(&path, size, "d0", &ctx, 500, 2, false)
}
@@ -1930,6 +1964,7 @@ mod phase_a_grouping_tests {
capacity_warning: None,
progress: &progress,
meter: &meter,
+ tally: &ScanTally::default(),
};
build_phase_a(
root,
@@ -3772,6 +3807,14 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
run_index_report(cfg).map(|(code, _)| code)
}
+/// `run_index` with the caller's own phase-A tally, so a test can measure the
+/// work of *its* run and nothing else's (#891). Production has no reason to
+/// look at the number, which is why the public entry points own a throwaway.
+#[cfg(test)]
+fn run_index_tallied(cfg: IndexCfg, tally: &ScanTally) -> Result<i32> {
+ run_index_report_tallied(cfg, tally).map(|(code, _)| code)
+}
+
/// #381/#759: the per-file record cap dropped these files' tails. Name them (up
/// to a bound) so a truncated document is identifiable on a multi-file corpus,
/// not merely counted — the "never silent" guarantee is only useful if the user
@@ -3848,6 +3891,18 @@ mod truncation_note_tests {
/// `None` when the run ended before a plan produced one (empty folder,
/// `--dry-run`).
pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
+ run_index_report_tallied(cfg, &ScanTally::default())
+}
+
+/// The body of [`run_index_report`], with the run's phase-A tally supplied by
+/// the caller instead of created here.
+///
+/// The tally is per-invocation on purpose: it replaced a process-global counter
+/// that every concurrent run in the process wrote to (#891). Every phase-A
+/// route this function can take — the legacy `build_phase_a` scan and the
+/// generated route's `project_reconcile_plan` — is handed this one tally, so
+/// "files parsed by this run" is exactly what it holds.
+fn run_index_report_tallied(cfg: IndexCfg, tally: &ScanTally) -> Result<(i32, Option<Value>)> {
// The very first statement of the function, deliberately: `started` must
// be when this invocation began, not when its summary was built.
let invocation_started = chrono::Utc::now();
@@ -4156,8 +4211,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
.committed_manifest
.as_ref()
.context("branch guard proved a committed manifest")?;
- let plan =
- project_reconcile_plan(&inventory, &base.plan, &cfg, &state_dir, &pr, &scan_meter)?;
+ let plan = project_reconcile_plan(
+ &inventory,
+ &base.plan,
+ &cfg,
+ &state_dir,
+ &pr,
+ &scan_meter,
+ tally,
+ )?;
let unchanged = serde_json::to_value(&plan)? == serde_json::to_value(&base.plan)?;
println!("{}", serde_json::to_string_pretty(&plan)?);
// stdout is the RESULT, stderr is PROGRESS: the projection above is
@@ -4213,8 +4275,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
.as_ref()
.context("generated journal lost its committed manifest")?
.clone();
- let plan =
- project_reconcile_plan(&inventory, &base.plan, &cfg, &state_dir, &pr, &scan_meter)?;
+ let plan = project_reconcile_plan(
+ &inventory,
+ &base.plan,
+ &cfg,
+ &state_dir,
+ &pr,
+ &scan_meter,
+ tally,
+ )?;
// Say out loud that the decision gate is not on this route. It is an
// incremental reconcile of an already-committed generation, so the work
// is the *changed* set and is published from a sealed snapshot rather
@@ -4625,6 +4694,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
capacity_warning: pdf_spool_capacity_warning.as_deref(),
progress: &pr,
meter: &scan_meter,
+ tally,
};
let mut plan: Plan = if let Some(p) = journal.plan.clone().filter(|_| !genesis_recovery) {
p
@@ -8618,6 +8688,7 @@ mod unity_pipeline_tests {
capacity_warning: None,
progress: &progress,
meter: &meter,
+ tally: &ScanTally::default(),
};
let mut cfg = super::phase_a_grouping_tests::cfg_for(root);
cfg.sample = sample;