AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
test(engine): remove the process-wide state and the residual races from the engine suite (#118)
xerj-orgf70ed338465c
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 933f63bf4..ea8ccf391 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -104,24 +104,110 @@ pub fn run_aggs(aggs_def: &Value, docs: &[Value]) -> Value {// Without this guard, a `terms` agg over a high-cardinality field (e.g. 50M// unique user IDs) allocates one HashMap entry per unique value before any// `size`/`shard_size` cap can drop them — easy OOM. Per-bucket-allocator-// loops in this module poll `max_buckets()` on each insert; when exceeded-// the accumulator stops adding new keys (existing keys still increment).+// loops in this module read `max_buckets()` ONCE, hoisted above the loop,+// and compare against it on each insert; when exceeded the accumulator stops+// adding new keys (existing keys still increment). The read used to sit+// inside the loop; hoisting it is what lets the cap be a per-call value+// without paying for the lookup per document.//-// Set once at Index construction via `set_max_buckets`; defaults to the same-// 65 536 ES uses if never set. Atomic + Relaxed: read in hot agg loops, write-// once at startup, no ordering needed.+// The node-wide default is set once at Engine construction via+// `set_max_buckets`; it defaults to the same 65 536 ES uses if never set.+// Atomic + Relaxed: read at the head of agg loops, written once at startup,+// no ordering needed.+//+// A single aggregation run can *override* that default for the duration of+// one call via `run_aggs_with_limits`. The override lives in a thread-local+// (see `BUCKET_CAP_OVERRIDE`) rather than in the static, so one caller's+// tighter cap is never observable by an aggregation running concurrently on+// another thread. Mutating the static to express a per-call limit would be a+// data race in the semantic sense: every other in-flight aggregation in the+// process would silently adopt the foreign cap.static MAX_BUCKETS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(65_536);-/// Override the per-aggregation bucket cap. Call once at Index startup with-/// `config.limits.max_buckets`. Subsequent calls overwrite the previous value.+thread_local! {+ /// Per-call bucket-cap override installed by `BucketCapScope`, i.e. by+ /// `run_aggs_with_limits`. `None` (the common case) means "use the+ /// node-wide default". Scoped to the calling thread: the whole agg+ /// evaluator is synchronous and runs on the caller's thread, so the+ /// override covers exactly the call it was installed for.+ static BUCKET_CAP_OVERRIDE: std::cell::Cell<Option<usize>> =+ const { std::cell::Cell::new(None) };+}++/// Set the node-wide default per-aggregation bucket cap. Called once at+/// Engine startup with `config.limits.max_buckets`. Subsequent calls+/// overwrite the previous value.+///+/// This is deliberately *not* the way to express a limit for a single+/// aggregation — use [`run_aggs_with_limits`] for that.pub fn set_max_buckets(n: usize) {MAX_BUCKETS.store(n.max(1), std::sync::atomic::Ordering::Relaxed);}-/// Current per-aggregation bucket cap.+/// The bucket cap in force for the aggregation running on this thread: the+/// per-call override if one is installed, otherwise the node-wide default.+///+/// Callers hoist this out of their per-document loops; it is a cap for the+/// whole aggregation, not something that may change mid-run.#[inline]pub fn max_buckets() -> usize {- MAX_BUCKETS.load(std::sync::atomic::Ordering::Relaxed)+ match BUCKET_CAP_OVERRIDE.with(std::cell::Cell::get) {+ Some(n) => n,+ None => MAX_BUCKETS.load(std::sync::atomic::Ordering::Relaxed),+ }+}++/// Per-call aggregation limits. Passed to [`run_aggs_with_limits`] by callers+/// that need a limit other than the node-wide default (a per-request+/// `search.max_buckets` override, or a test pinning the cap for one call).+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct AggLimits {+ /// Maximum number of buckets a single aggregation may materialise.+ pub max_buckets: usize,+}++impl Default for AggLimits {+ fn default() -> Self {+ Self {+ max_buckets: max_buckets(),+ }+ }+}++/// RAII installer for a per-call bucket cap. Restores the previous value on+/// drop — including on unwind — so a panicking aggregation cannot leave a+/// foreign cap behind for the next aggregation on this thread. Nesting is+/// supported (the inner scope wins, the outer value is restored).+struct BucketCapScope {+ previous: Option<usize>,+}++impl BucketCapScope {+ fn new(max_buckets: usize) -> Self {+ let previous = BUCKET_CAP_OVERRIDE.with(|cell| cell.replace(Some(max_buckets.max(1))));+ Self { previous }+ }+}++impl Drop for BucketCapScope {+ fn drop(&mut self) {+ BUCKET_CAP_OVERRIDE.with(|cell| cell.set(self.previous));+ }+}++/// Run an aggregation under explicit limits instead of the node-wide+/// defaults. Identical to [`run_aggs_with_all`] in every other respect.+///+/// The limits apply to this call only and are invisible to aggregations+/// running concurrently on other threads.+pub fn run_aggs_with_limits(+ aggs_def: &Value,+ docs: &[Value],+ all_docs: &[Value],+ limits: AggLimits,+) -> Value {+ let _scope = BucketCapScope::new(limits.max_buckets);+ run_aggs_with_all(aggs_def, docs, all_docs)}thread_local! {@@ -9130,6 +9216,7 @@ fn run_geotile_grid(};let mut bucket_map: HashMap<String, Vec<usize>> = HashMap::new();+ let bucket_cap = max_buckets();for (i, doc) in docs.iter().enumerate() {let pts: Vec<(f64, f64)> = collect_geo_points(doc, field);for (lat, lon) in pts {@@ -9143,7 +9230,7 @@ fn run_geotile_grid(let max = n - 1;let key = format!("{}/{}/{}", precision, x.clamp(0, max), y.clamp(0, max));// Bucket cap: skip new keys past the limit; existing keys still grow.- if bucket_map.contains_key(&key) || bucket_map.len() < max_buckets() {+ if bucket_map.contains_key(&key) || bucket_map.len() < bucket_cap {bucket_map.entry(key).or_default().push(i);}}@@ -12521,12 +12608,6 @@ mod tests {// ── RC4-W2 item 5: sum_other_doc_count / ES size pipeline ───────────- /// Serializes the RC4-W2 agg tests: `multi_terms_past_bucket_cap_...`- /// temporarily shrinks the process-global `max_buckets()` to 3, and the- /// skewed-corpus tests build 5 distinct buckets — running them- /// concurrently would flake. Every test in this block takes the lock.- static AGG_CAP_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());-/// Skewed corpus: a=5 b=4 c=3 d=2 e=1 (15 docs). All expectations in/// the pipeline tests below were verified against live ES 8.13.4.fn skewed_docs() -> Vec<Value> {@@ -12541,7 +12622,6 @@ mod tests {#[test]fn terms_sum_other_doc_count_on_truncation() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());let agg = json!({ "t": { "terms": { "field": "tag", "size": 2 } } });let result = run_aggs(&agg, &skewed_docs());let buckets = result["t"]["buckets"].as_array().unwrap();@@ -12554,7 +12634,6 @@ mod tests {#[test]fn terms_min_doc_count_drops_silently_within_shard_size() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());// size=3 → default shard_size 14 covers all 5 buckets; b/c/d/e fail// min_doc_count=5 at final reduce and are dropped WITHOUT counting// toward other (live-ES-verified).@@ -12568,7 +12647,6 @@ mod tests {#[test]fn terms_shard_size_cut_feeds_sum_other() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());// size=2, shard_size=3, min_doc_count=3: shard set [a,b,c]; d,e cut// at the shard boundary (+3); c passes min but overflows size (+3).let agg = json!({ "t": { "terms": {@@ -12581,7 +12659,6 @@ mod tests {#[test]fn multi_terms_sum_other_doc_count_on_truncation() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());let mut docs = skewed_docs();for d in &mut docs {d["s"] = json!("x");@@ -12598,7 +12675,6 @@ mod tests {#[test]fn composite_keyword_keys_stay_strings_and_sort_lexicographically() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());let docs = vec![json!({"agent": "007"}),json!({"agent": "9"}),@@ -12628,7 +12704,6 @@ mod tests {#[test]fn composite_boolean_keys_are_json_bools() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());let docs = vec![json!({"active": true}),json!({"active": false}),@@ -12650,7 +12725,6 @@ mod tests {#[test]fn composite_unmapped_field_keeps_legacy_heuristic() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());// No sentinel → numeric-looking strings keep the historical// number-typed behavior (unmapped/dynamic fields).let docs = vec![json!({"n": 7}), json!({"n": 10})];@@ -12668,13 +12742,12 @@ mod tests {#[test]fn multi_terms_past_bucket_cap_errors_instead_of_silent_drop() {- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());- let old_cap = max_buckets();- set_max_buckets(3);+ // The cap is an argument of the call under test — nothing+ // process-wide is mutated, so this test is safe to run beside any+ // other aggregation test.let docs: Vec<Value> = (0..5).map(|i| json!({ "k": format!("k{i}") })).collect();let agg = json!({ "t": { "multi_terms": { "terms": [ {"field": "k"} ], "size": 2 } } });- let result = run_aggs(&agg, &docs);- set_max_buckets(old_cap);+ let result = run_aggs_with_limits(&agg, &docs, &docs, AggLimits { max_buckets: 3 });let err = result["t"]["error"].as_str().unwrap_or_default();assert!(err.contains("Trying to create too many buckets"),@@ -12682,6 +12755,73 @@ mod tests {);assert_eq!(result["t"]["__error_status__"], json!(400));}++ /// Issue #100: the per-call cap must not be visible to aggregations on+ /// other threads. Deterministic — the two threads rendezvous on a+ /// barrier, so the reader observes the writer *while* its scope is+ /// open, not "usually" or "if it wins the race".+ #[test]+ fn per_call_bucket_cap_is_invisible_to_other_threads() {+ use std::sync::{Arc, Barrier};++ let default_cap = max_buckets();+ assert!(+ default_cap > 3,+ "test assumes the node default is looser than the scoped cap"+ );+ let entered = Arc::new(Barrier::new(2));+ let observed = Arc::new(Barrier::new(2));++ let writer = {+ let (entered, observed) = (Arc::clone(&entered), Arc::clone(&observed));+ std::thread::spawn(move || {+ let scope = BucketCapScope::new(3);+ assert_eq!(max_buckets(), 3, "the scope owner sees its own cap");+ entered.wait();+ // Held open across the reader's assertion below.+ observed.wait();+ drop(scope);+ assert_eq!(max_buckets(), default_cap, "scope must restore on drop");+ })+ };++ entered.wait();+ assert_eq!(+ max_buckets(),+ default_cap,+ "another thread's scoped bucket cap leaked into this thread"+ );+ // And a real aggregation on this thread must still be able to build+ // more buckets than the foreign scope allows.+ let docs: Vec<Value> = (0..5).map(|i| json!({ "k": format!("k{i}") })).collect();+ let agg = json!({ "t": { "multi_terms": { "terms": [ {"field": "k"} ], "size": 5 } } });+ let result = run_aggs(&agg, &docs);+ assert_eq!(+ result["t"]["buckets"].as_array().map(Vec::len),+ Some(5),+ "expected 5 buckets, got {result}"+ );+ observed.wait();+ writer.join().unwrap();+ }++ /// A per-call cap is confined to that call: the next aggregation on the+ /// same thread is back on the node-wide default, even though the capped+ /// call bailed out early with an error.+ #[test]+ fn per_call_bucket_cap_does_not_outlive_the_call() {+ let docs: Vec<Value> = (0..5).map(|i| json!({ "k": format!("k{i}") })).collect();+ let agg = json!({ "t": { "multi_terms": { "terms": [ {"field": "k"} ], "size": 5 } } });+ let capped = run_aggs_with_limits(&agg, &docs, &docs, AggLimits { max_buckets: 3 });+ assert_eq!(capped["t"]["__error_status__"], json!(400));++ let after = run_aggs(&agg, &docs);+ assert_eq!(+ after["t"]["buckets"].as_array().map(Vec::len),+ Some(5),+ "cap outlived its call, got {after}"+ );+ }}#[cfg(test)]diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 0a7070a4f..43f55f4fe 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2116,10 +2116,19 @@ mod semantic_deadline_regression_tests {boost: None,similarity: None,};+ // Issue #100: the 5 ms budget below used to be measured against the+ // real clock, so on a loaded box it was already gone when the scan+ // took its *first* checkpoint — the request timed out having done no+ // partial work and the count assertion saw 1 instead of 2. The budget+ // is unchanged; what changed is that this index's scan clock is+ // scripted, so checkpoint zero reads `base` (inside the budget) and+ // the next checkpoint reads `base + 6 ms` (past it) no matter how+ // long the machine takes to get between them.+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(6));let result = idx.run_multi_knn_brute_force(&match_all(250),- std::time::Instant::now() + std::time::Duration::from_millis(5),+ base + std::time::Duration::from_millis(5),vec![clause],).await@@ -2140,6 +2149,14 @@ mod semantic_deadline_regression_tests {engine.create_index("multi-stop", Schema::empty()).unwrap();let idx = engine.get_index("multi-stop").unwrap();seed_vectors(&idx).await;+ // `seed_vectors` leaves a 20 ms per-checkpoint sleep behind. Left in+ // place it makes the 5 ms budget expire against the REAL clock inside+ // clause 1, so the test passes even if the cooperative checkpoint it+ // is meant to pin is gone — the parent's own clause-entry check stops+ // clause 2 either way. Zeroing it puts the scripted clock back in+ // charge of when the budget is crossed.+ idx.test_scan_checkpoint_delay_ms+ .store(0, Ordering::Relaxed);let clause = PeeledKnn {field: "embedding".into(),vector: vec![1.0, 0.0],@@ -2149,10 +2166,14 @@ mod semantic_deadline_regression_tests {boost: None,similarity: None,};+ // See `expired_multi_knn_preserves_partial_timeout_state`: same 5 ms+ // budget, now crossed at the first clause's second checkpoint by+ // arithmetic rather than by whatever the machine was doing.+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(6));let result = idx.run_multi_knn_brute_force(&match_all(250),- std::time::Instant::now() + std::time::Duration::from_millis(5),+ base + std::time::Duration::from_millis(5),vec![clause.clone(), clause],).await@@ -2194,15 +2215,28 @@ mod semantic_deadline_regression_tests {.await.unwrap();- for timeout_ms in [1, 10, 250] {- idx.test_scan_checkpoint_delay_ms- .store(timeout_ms + 10, Ordering::Relaxed);+ // Issue #100: this used to sleep `budget + 10 ms` inside the+ // checkpoint and bet that the machine would still reach checkpoint+ // zero within the budget. On a loaded box it does not — the scan+ // stops at position 0 and the count below is 1 instead of 2.+ //+ // The deadlines stay genuinely in the future when the scan starts,+ // which is the property this test is named for (as opposed to the+ // pre-expired deadlines the `expired_*` tests cover), and the small+ // budgets stay small. What changed is that the clock the scan reads+ // is scripted: `base` at checkpoint zero, `base + budget + 1 ms`+ // from position 1 on. The sleep is gone — it existed only to make+ // real time pass, and the scripted clock does not care about real+ // time.+ idx.test_scan_checkpoint_delay_ms+ .store(0, Ordering::Relaxed);+ for budget_ms in [1u64, 10, 250] {idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);- let started = std::time::Instant::now();+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(budget_ms + 1));let result = idx.run_knn_brute_force_with_deadline(&request,- started + std::time::Duration::from_millis(timeout_ms),+ base + std::time::Duration::from_millis(budget_ms),"embedding",&[1.0, 0.0],10,@@ -2220,7 +2254,6 @@ mod semantic_deadline_regression_tests {2,"scan must pass checkpoint zero and stop at the later checkpoint");- assert!(started.elapsed() >= std::time::Duration::from_millis(timeout_ms));}}@@ -2242,10 +2275,14 @@ mod semantic_deadline_regression_tests {boost: None,similarity: None,};+ // See `expired_multi_knn_preserves_partial_timeout_state` — same 5 ms+ // race, same fix: the budget is crossed at the child scan's second+ // checkpoint because the scripted clock says so.+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(6));let result = idx.run_hybrid_with_deadline(&match_all(250),- std::time::Instant::now() + std::time::Duration::from_millis(5),+ base + std::time::Duration::from_millis(5),vec![xerj_query::ast::WeightedQuery {query: knn,weight: 1.0,@@ -2271,18 +2308,21 @@ mod semantic_deadline_regression_tests {.create_index("nested-timeout", Schema::empty()).unwrap();let idx = engine.get_index("nested-timeout").unwrap();- for n in 0..256 {- idx.index_document(- Some(format!("n{n}")),- serde_json::json!({"items": [{"vector": [1.0, 0.0]}]}),- )- .await- .unwrap();+ // 8 parents × 32 nested elements. The shape matters: the parent loop+ // checkpoints every 128 *parents* and so never fires past position 0+ // here, which leaves the nested-array checkpoint (every 128+ // *elements*, first hit inside parent 3) as the only cooperative+ // stop in the scan. Deleting it is therefore visible to this test —+ // with one element per parent the two checkpoints alias at 128 and+ // the parent loop silently covers for the missing one.+ for n in 0..8 {+ let items: Vec<serde_json::Value> = (0..32)+ .map(|_| serde_json::json!({"vector": [1.0, 0.0]}))+ .collect();+ idx.index_document(Some(format!("n{n}")), serde_json::json!({ "items": items }))+ .await+ .unwrap();}- idx.test_scan_checkpoint_delay_ms- .store(20, Ordering::Relaxed);- idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);-let request = SearchRequest {query: QueryNode::Nested {path: "items".into(),@@ -2297,16 +2337,73 @@ mod semantic_deadline_regression_tests {}),score_mode: None,},- timeout_ms: Some(5),+ // Large on purpose. The gate parks the SCAN, but it does not stop+ // the request's own budget, which keeps burning real time while+ // the test holds the scan there. A 100 ms budget therefore raced:+ // on a loaded box the budget could expire against the real clock+ // before the test observed the park, and the request then timed+ // out through a different path than the nested-array checkpoint+ // this test exists to pin. The budget has to outlast any+ // plausible parking time while still sitting well under the+ // scripted clock's ten-minute jump, which is what must trip it.+ timeout_ms: Some(60_000),..SearchRequest::default()};- let result = idx.search(&request).await.unwrap();++ // Issue #100: this used to assert that a 20 ms sleep inside the scan+ // overran a 5 ms request budget — true on an idle box, false the+ // moment the machine is loaded enough that the scan needs more than+ // 5 ms just to reach its first checkpoint.+ //+ // Two facts replace that bet, and neither depends on how busy the+ // machine is.+ //+ // The gate replaces the sleep. While the test holds it, the scan is+ // parked *inside* its nested-array loop, before it has looked at the+ // clock, and it stays there for as long as we like — so "the scan is+ // still mid-flight" is established rather than hoped for. Failing to+ // park at all is the regression this test exists to catch: it means+ // the nested-array loop has no cooperative checkpoint and ran to+ // completion, and it is asserted on explicitly below.+ //+ // The scripted clock replaces the budget race. Checkpoint zero reads+ // an instant taken before the request was even built, so it is+ // always inside the request budget; the nested-array checkpoint+ // reads ten minutes later, so it is always past it.+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ idx.test_scan_checkpoint_parked.store(0, Ordering::SeqCst);+ idx.script_scan_clock(1, std::time::Duration::from_secs(600));+ let gate = Arc::clone(&idx.test_scan_checkpoint_gate)+ .lock_owned()+ .await;++ let scan = {+ let idx = Arc::clone(&idx);+ tokio::spawn(async move { idx.search(&request).await })+ };+ tokio::time::timeout(std::time::Duration::from_secs(60), async {+ while idx.test_scan_checkpoint_parked.load(Ordering::SeqCst) == 0 && !scan.is_finished()+ {+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;+ }+ })+ .await+ .expect("nested kNN scan neither parked nor finished within 60 s");+ assert!(+ idx.test_scan_checkpoint_parked.load(Ordering::SeqCst) > 0,+ "nested kNN ran to completion without a cooperative checkpoint inside \+ its nested-array loop (checkpoints reached: {})",+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed)+ );+ drop(gate);++ let result = scan.await.unwrap().unwrap();assert!(result.timed_out);assert_eq!(result.total.relation, TotalHitsRelation::Gte);assert_eq!(idx.test_scan_checkpoint_count.load(Ordering::Relaxed),2,- "nested scan must stop at its first non-zero checkpoint"+ "nested scan must stop at its first nested-array checkpoint");}@@ -2334,11 +2431,30 @@ mod semantic_deadline_regression_tests {.await.unwrap();}+ // Issue #100: two wall-clock bets used to live here. The checkpoint+ // count could come back below 16 because a loaded machine burnt the+ // 100 ms budget before a request reached its *first* checkpoint, and+ // "these ran concurrently" was inferred from total elapsed time+ // being under 500 ms — a threshold that says as much about the box+ // as about the engine.+ //+ // Both are now facts rather than bets. The scripted clock puts every+ // request's checkpoint zero inside its budget and every later+ // checkpoint past it, so each request reaches exactly two. And+ // concurrency is established by the gate: all eight scans are parked+ // on it at the same time, with none of the eight tasks finished,+ // before any of them is allowed to observe its deadline. Eight scans+ // simultaneously mid-flight is what "share bounded deadlines" means,+ // and it is now observed instead of timed.idx.test_scan_checkpoint_delay_ms- .store(200, Ordering::Relaxed);+ .store(0, Ordering::Relaxed);idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ idx.test_scan_checkpoint_parked.store(0, Ordering::SeqCst);+ idx.script_scan_clock(1, std::time::Duration::from_secs(600));+ let gate = Arc::clone(&idx.test_scan_checkpoint_gate)+ .lock_owned()+ .await;- let started = std::time::Instant::now();let mut tasks = Vec::new();for n in 0..8 {let idx = Arc::clone(&idx);@@ -2351,12 +2467,37 @@ mod semantic_deadline_regression_tests {filter: None,boost: None,},- timeout_ms: Some(100),+ // Same reason as `nested_knn_uses_the_shared_request_deadline`:+ // the gate parks the scan but the request's budget keeps+ // running against the real clock. This test only needs the+ // eight requests to be in flight together, so the budget+ // must simply outlast the parking rather than expire during+ // it. At 100 ms a loaded box could retire a request before+ // all eight reached the gate, which is what made this flaky.+ timeout_ms: Some(60_000),..SearchRequest::default()};idx.search(&request).await}));}++ tokio::time::timeout(std::time::Duration::from_secs(60), async {+ while idx.test_scan_checkpoint_parked.load(Ordering::SeqCst) < 8 {+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;+ }+ })+ .await+ .expect("the eight semantic scans never all reached a cooperative checkpoint");+ // Every scan that reached the gate is still holding there, so none of+ // the eight requests can have completed: they are in flight together.+ for (n, task) in tasks.iter().enumerate() {+ assert!(+ !task.is_finished(),+ "request {n} finished while the other scans were still parked"+ );+ }+ drop(gate);+for task in tasks {let result = task.await.unwrap().unwrap();assert!(result.timed_out);@@ -2366,10 +2507,6 @@ mod semantic_deadline_regression_tests {idx.test_scan_checkpoint_count.load(Ordering::Relaxed) >= 16,"all semantic requests must reach multiple cooperative scan checkpoints");- assert!(- started.elapsed() < std::time::Duration::from_millis(500),- "concurrent semantic scans exceeded their own deadlines"- );}}@@ -2756,6 +2893,33 @@ enum PublicationTestPoint {#[cfg(test)]type PublicationTestHook = Arc<dyn Fn(&str, PublicationTestPoint) + Send + Sync + 'static>;+/// A scripted reading of the clock that cooperative scan checkpoints compare+/// against their deadline, installed per index by+/// [`Index::script_scan_clock`].+///+/// Checkpoints at scan positions below `advance_from` read exactly `base`;+/// `advance_from` and later read `base + skew`. Both readings are fixed at+/// install time, so a deadline derived from `base` is crossed at a position+/// the test chose, on an idle box and a thrashing one alike.+#[cfg(test)]+#[derive(Debug, Clone, Copy)]+struct ScanClockScript {+ base: std::time::Instant,+ advance_from: usize,+ skew: std::time::Duration,+}++#[cfg(test)]+impl ScanClockScript {+ fn reading_at(&self, position: usize) -> std::time::Instant {+ if position < self.advance_from {+ self.base+ } else {+ self.base.checked_add(self.skew).unwrap_or(self.base)+ }+ }+}+/// Per-index coordinator.////// Owns the storage layer (`IndexStore`), the FTS memtable, and the schema.@@ -2925,6 +3089,32 @@ pub struct Index {test_scan_checkpoint_delay_ms: Arc<AtomicU64>,#[cfg(test)]test_scan_checkpoint_count: Arc<AtomicU64>,+ /// Deterministic stand-in for the wall-clock delay above: while a test+ /// holds this mutex, the first cooperative checkpoint of a scan that is+ /// past position 0 parks on it and cannot observe the deadline until the+ /// test lets go. That turns "the scan is still inside its loop when the+ /// request budget runs out" into a fact established by the test instead+ /// of a race against the machine's load.+ #[cfg(test)]+ test_scan_checkpoint_gate: Arc<tokio::sync::Mutex<()>>,+ /// Number of checkpoints that have reached the gate. Lets a test wait+ /// for the scan to be provably parked before it does anything else.+ #[cfg(test)]+ test_scan_checkpoint_parked: Arc<AtomicU64>,+ /// Injected clock for the cooperative scan deadline check, owned by this+ /// index — i.e. by the object under test, never process-wide. `None`+ /// (the default) means every checkpoint reads the real clock.+ ///+ /// This is how a test states "the request's budget is spent by the time+ /// the scan reaches position p" as a fact. The alternative — hand the+ /// scan a budget of a few milliseconds and hope the machine reaches the+ /// first checkpoint inside it — is a race the test loses whenever the+ /// box is busy (issue #100). The deadline comparison itself is+ /// untouched: the scan still compares a clock reading against the real+ /// deadline it was handed, so a scan that ignores its deadline still+ /// fails these tests.+ #[cfg(test)]+ test_scan_clock: Arc<std::sync::Mutex<Option<ScanClockScript>>>,#[cfg(test)]test_ann_passage_guard_pause: Arc<AtomicBool>,#[cfg(test)]@@ -3419,6 +3609,12 @@ impl Index {#[cfg(test)]test_scan_checkpoint_count: Arc::new(AtomicU64::new(0)),#[cfg(test)]+ test_scan_checkpoint_gate: Arc::new(tokio::sync::Mutex::new(())),+ #[cfg(test)]+ test_scan_checkpoint_parked: Arc::new(AtomicU64::new(0)),+ #[cfg(test)]+ test_scan_clock: Arc::new(std::sync::Mutex::new(None)),+ #[cfg(test)]test_ann_passage_guard_pause: Arc::new(AtomicBool::new(false)),#[cfg(test)]test_ann_passage_guard_ready: Arc::new(AtomicBool::new(false)),@@ -3730,6 +3926,12 @@ impl Index {#[cfg(test)]test_scan_checkpoint_count: Arc::new(AtomicU64::new(0)),#[cfg(test)]+ test_scan_checkpoint_gate: Arc::new(tokio::sync::Mutex::new(())),+ #[cfg(test)]+ test_scan_checkpoint_parked: Arc::new(AtomicU64::new(0)),+ #[cfg(test)]+ test_scan_clock: Arc::new(std::sync::Mutex::new(None)),+ #[cfg(test)]test_ann_passage_guard_pause: Arc::new(AtomicBool::new(false)),#[cfg(test)]test_ann_passage_guard_ready: Arc::new(AtomicBool::new(false)),@@ -7741,8 +7943,75 @@ impl Index {if _position > 0 && delay_ms > 0 {tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;}+ // Deterministic gate (see `test_scan_checkpoint_gate`). Free when+ // no test holds the mutex, so this is a no-op for every other+ // test; when one does hold it, the scan waits here — inside its+ // loop, before it looks at the clock — until released.+ if _position > 0 {+ self.test_scan_checkpoint_parked+ .fetch_add(1, Ordering::SeqCst);+ let _gate = self.test_scan_checkpoint_gate.lock().await;+ }}- std::time::Instant::now() >= deadline+ self.scan_clock_now(_position) >= deadline+ }++ /// The clock a cooperative scan checkpoint compares against its deadline.+ /// Always the real clock in a non-test build.+ #[cfg(not(test))]+ #[inline]+ fn scan_clock_now(&self, _position: usize) -> std::time::Instant {+ std::time::Instant::now()+ }++ /// The clock a cooperative scan checkpoint compares against its deadline.+ ///+ /// Real, unless a test has scripted this index's clock (see+ /// [`Index::script_scan_clock`]), in which case it returns whatever that+ /// script says the given scan position sees. Scripted readings do not+ /// move with the machine, which is the whole point: it turns "the budget+ /// runs out between checkpoint zero and the next one" from a race the+ /// test can lose into an arithmetic fact.+ #[cfg(test)]+ #[inline]+ fn scan_clock_now(&self, position: usize) -> std::time::Instant {+ let script = *self+ .test_scan_clock+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ match script {+ Some(script) => script.reading_at(position),+ None => std::time::Instant::now(),+ }+ }++ /// Replace this index's scan clock with a scripted one and return the+ /// instant every checkpoint before `advance_from` will read. Checkpoints+ /// at `advance_from` and later read `base + skew` instead.+ ///+ /// A test derives its deadline from the returned `base`, so the position+ /// at which the deadline is crossed is fixed by arithmetic rather than+ /// by how quickly the machine gets from one checkpoint to the next.+ ///+ /// Scoped to this `Index` — the object under test — so a test that+ /// scripts a clock is invisible to every other test in the process,+ /// however they are scheduled.+ #[cfg(test)]+ fn script_scan_clock(+ &self,+ advance_from: usize,+ skew: std::time::Duration,+ ) -> std::time::Instant {+ let base = std::time::Instant::now();+ *self+ .test_scan_clock+ .lock()+ .unwrap_or_else(|p| p.into_inner()) = Some(ScanClockScript {+ base,+ advance_from,+ skew,+ });+ base}#[allow(clippy::too_many_arguments)]