Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

test(engine): remove the process-wide state and the residual races from the engine suite (#118)

xerj-orgf70ed338465c
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index 933f63bf4..ea8ccf391 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -104,24 +104,110 @@ pub fn run_aggs(aggs_def: &Value, docs: &[Value]) -> Value {
// Without this guard, a `terms` agg over a high-cardinality field (e.g. 50M
// unique user IDs) allocates one HashMap entry per unique value before any
// `size`/`shard_size` cap can drop them — easy OOM. Per-bucket-allocator
-// loops in this module poll `max_buckets()` on each insert; when exceeded
-// the accumulator stops adding new keys (existing keys still increment).
+// loops in this module read `max_buckets()` ONCE, hoisted above the loop,
+// and compare against it on each insert; when exceeded the accumulator stops
+// adding new keys (existing keys still increment). The read used to sit
+// inside the loop; hoisting it is what lets the cap be a per-call value
+// without paying for the lookup per document.
//
-// Set once at Index construction via `set_max_buckets`; defaults to the same
-// 65 536 ES uses if never set. Atomic + Relaxed: read in hot agg loops, write
-// once at startup, no ordering needed.
+// The node-wide default is set once at Engine construction via
+// `set_max_buckets`; it defaults to the same 65 536 ES uses if never set.
+// Atomic + Relaxed: read at the head of agg loops, written once at startup,
+// no ordering needed.
+//
+// A single aggregation run can *override* that default for the duration of
+// one call via `run_aggs_with_limits`. The override lives in a thread-local
+// (see `BUCKET_CAP_OVERRIDE`) rather than in the static, so one caller's
+// tighter cap is never observable by an aggregation running concurrently on
+// another thread. Mutating the static to express a per-call limit would be a
+// data race in the semantic sense: every other in-flight aggregation in the
+// process would silently adopt the foreign cap.
static MAX_BUCKETS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(65_536);
-/// Override the per-aggregation bucket cap. Call once at Index startup with
-/// `config.limits.max_buckets`. Subsequent calls overwrite the previous value.
+thread_local! {
+ /// Per-call bucket-cap override installed by `BucketCapScope`, i.e. by
+ /// `run_aggs_with_limits`. `None` (the common case) means "use the
+ /// node-wide default". Scoped to the calling thread: the whole agg
+ /// evaluator is synchronous and runs on the caller's thread, so the
+ /// override covers exactly the call it was installed for.
+ static BUCKET_CAP_OVERRIDE: std::cell::Cell<Option<usize>> =
+ const { std::cell::Cell::new(None) };
+}
+
+/// Set the node-wide default per-aggregation bucket cap. Called once at
+/// Engine startup with `config.limits.max_buckets`. Subsequent calls
+/// overwrite the previous value.
+///
+/// This is deliberately *not* the way to express a limit for a single
+/// aggregation — use [`run_aggs_with_limits`] for that.
pub fn set_max_buckets(n: usize) {
MAX_BUCKETS.store(n.max(1), std::sync::atomic::Ordering::Relaxed);
}
-/// Current per-aggregation bucket cap.
+/// The bucket cap in force for the aggregation running on this thread: the
+/// per-call override if one is installed, otherwise the node-wide default.
+///
+/// Callers hoist this out of their per-document loops; it is a cap for the
+/// whole aggregation, not something that may change mid-run.
#[inline]
pub fn max_buckets() -> usize {
- MAX_BUCKETS.load(std::sync::atomic::Ordering::Relaxed)
+ match BUCKET_CAP_OVERRIDE.with(std::cell::Cell::get) {
+ Some(n) => n,
+ None => MAX_BUCKETS.load(std::sync::atomic::Ordering::Relaxed),
+ }
+}
+
+/// Per-call aggregation limits. Passed to [`run_aggs_with_limits`] by callers
+/// that need a limit other than the node-wide default (a per-request
+/// `search.max_buckets` override, or a test pinning the cap for one call).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct AggLimits {
+ /// Maximum number of buckets a single aggregation may materialise.
+ pub max_buckets: usize,
+}
+
+impl Default for AggLimits {
+ fn default() -> Self {
+ Self {
+ max_buckets: max_buckets(),
+ }
+ }
+}
+
+/// RAII installer for a per-call bucket cap. Restores the previous value on
+/// drop — including on unwind — so a panicking aggregation cannot leave a
+/// foreign cap behind for the next aggregation on this thread. Nesting is
+/// supported (the inner scope wins, the outer value is restored).
+struct BucketCapScope {
+ previous: Option<usize>,
+}
+
+impl BucketCapScope {
+ fn new(max_buckets: usize) -> Self {
+ let previous = BUCKET_CAP_OVERRIDE.with(|cell| cell.replace(Some(max_buckets.max(1))));
+ Self { previous }
+ }
+}
+
+impl Drop for BucketCapScope {
+ fn drop(&mut self) {
+ BUCKET_CAP_OVERRIDE.with(|cell| cell.set(self.previous));
+ }
+}
+
+/// Run an aggregation under explicit limits instead of the node-wide
+/// defaults. Identical to [`run_aggs_with_all`] in every other respect.
+///
+/// The limits apply to this call only and are invisible to aggregations
+/// running concurrently on other threads.
+pub fn run_aggs_with_limits(
+ aggs_def: &Value,
+ docs: &[Value],
+ all_docs: &[Value],
+ limits: AggLimits,
+) -> Value {
+ let _scope = BucketCapScope::new(limits.max_buckets);
+ run_aggs_with_all(aggs_def, docs, all_docs)
}
thread_local! {
@@ -9130,6 +9216,7 @@ fn run_geotile_grid(
};
let mut bucket_map: HashMap<String, Vec<usize>> = HashMap::new();
+ let bucket_cap = max_buckets();
for (i, doc) in docs.iter().enumerate() {
let pts: Vec<(f64, f64)> = collect_geo_points(doc, field);
for (lat, lon) in pts {
@@ -9143,7 +9230,7 @@ fn run_geotile_grid(
let max = n - 1;
let key = format!("{}/{}/{}", precision, x.clamp(0, max), y.clamp(0, max));
// Bucket cap: skip new keys past the limit; existing keys still grow.
- if bucket_map.contains_key(&key) || bucket_map.len() < max_buckets() {
+ if bucket_map.contains_key(&key) || bucket_map.len() < bucket_cap {
bucket_map.entry(key).or_default().push(i);
}
}
@@ -12521,12 +12608,6 @@ mod tests {
// ── RC4-W2 item 5: sum_other_doc_count / ES size pipeline ───────────
- /// Serializes the RC4-W2 agg tests: `multi_terms_past_bucket_cap_...`
- /// temporarily shrinks the process-global `max_buckets()` to 3, and the
- /// skewed-corpus tests build 5 distinct buckets — running them
- /// concurrently would flake. Every test in this block takes the lock.
- static AGG_CAP_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
-
/// Skewed corpus: a=5 b=4 c=3 d=2 e=1 (15 docs). All expectations in
/// the pipeline tests below were verified against live ES 8.13.4.
fn skewed_docs() -> Vec<Value> {
@@ -12541,7 +12622,6 @@ mod tests {
#[test]
fn terms_sum_other_doc_count_on_truncation() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
let agg = json!({ "t": { "terms": { "field": "tag", "size": 2 } } });
let result = run_aggs(&agg, &skewed_docs());
let buckets = result["t"]["buckets"].as_array().unwrap();
@@ -12554,7 +12634,6 @@ mod tests {
#[test]
fn terms_min_doc_count_drops_silently_within_shard_size() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
// size=3 → default shard_size 14 covers all 5 buckets; b/c/d/e fail
// min_doc_count=5 at final reduce and are dropped WITHOUT counting
// toward other (live-ES-verified).
@@ -12568,7 +12647,6 @@ mod tests {
#[test]
fn terms_shard_size_cut_feeds_sum_other() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
// size=2, shard_size=3, min_doc_count=3: shard set [a,b,c]; d,e cut
// at the shard boundary (+3); c passes min but overflows size (+3).
let agg = json!({ "t": { "terms": {
@@ -12581,7 +12659,6 @@ mod tests {
#[test]
fn multi_terms_sum_other_doc_count_on_truncation() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
let mut docs = skewed_docs();
for d in &mut docs {
d["s"] = json!("x");
@@ -12598,7 +12675,6 @@ mod tests {
#[test]
fn composite_keyword_keys_stay_strings_and_sort_lexicographically() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
let docs = vec![
json!({"agent": "007"}),
json!({"agent": "9"}),
@@ -12628,7 +12704,6 @@ mod tests {
#[test]
fn composite_boolean_keys_are_json_bools() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
let docs = vec![
json!({"active": true}),
json!({"active": false}),
@@ -12650,7 +12725,6 @@ mod tests {
#[test]
fn composite_unmapped_field_keeps_legacy_heuristic() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
// No sentinel → numeric-looking strings keep the historical
// number-typed behavior (unmapped/dynamic fields).
let docs = vec![json!({"n": 7}), json!({"n": 10})];
@@ -12668,13 +12742,12 @@ mod tests {
#[test]
fn multi_terms_past_bucket_cap_errors_instead_of_silent_drop() {
- let _g = AGG_CAP_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner());
- let old_cap = max_buckets();
- set_max_buckets(3);
+ // The cap is an argument of the call under test — nothing
+ // process-wide is mutated, so this test is safe to run beside any
+ // other aggregation test.
let docs: Vec<Value> = (0..5).map(|i| json!({ "k": format!("k{i}") })).collect();
let agg = json!({ "t": { "multi_terms": { "terms": [ {"field": "k"} ], "size": 2 } } });
- let result = run_aggs(&agg, &docs);
- set_max_buckets(old_cap);
+ let result = run_aggs_with_limits(&agg, &docs, &docs, AggLimits { max_buckets: 3 });
let err = result["t"]["error"].as_str().unwrap_or_default();
assert!(
err.contains("Trying to create too many buckets"),
@@ -12682,6 +12755,73 @@ mod tests {
);
assert_eq!(result["t"]["__error_status__"], json!(400));
}
+
+ /// Issue #100: the per-call cap must not be visible to aggregations on
+ /// other threads. Deterministic — the two threads rendezvous on a
+ /// barrier, so the reader observes the writer *while* its scope is
+ /// open, not "usually" or "if it wins the race".
+ #[test]
+ fn per_call_bucket_cap_is_invisible_to_other_threads() {
+ use std::sync::{Arc, Barrier};
+
+ let default_cap = max_buckets();
+ assert!(
+ default_cap > 3,
+ "test assumes the node default is looser than the scoped cap"
+ );
+ let entered = Arc::new(Barrier::new(2));
+ let observed = Arc::new(Barrier::new(2));
+
+ let writer = {
+ let (entered, observed) = (Arc::clone(&entered), Arc::clone(&observed));
+ std::thread::spawn(move || {
+ let scope = BucketCapScope::new(3);
+ assert_eq!(max_buckets(), 3, "the scope owner sees its own cap");
+ entered.wait();
+ // Held open across the reader's assertion below.
+ observed.wait();
+ drop(scope);
+ assert_eq!(max_buckets(), default_cap, "scope must restore on drop");
+ })
+ };
+
+ entered.wait();
+ assert_eq!(
+ max_buckets(),
+ default_cap,
+ "another thread's scoped bucket cap leaked into this thread"
+ );
+ // And a real aggregation on this thread must still be able to build
+ // more buckets than the foreign scope allows.
+ let docs: Vec<Value> = (0..5).map(|i| json!({ "k": format!("k{i}") })).collect();
+ let agg = json!({ "t": { "multi_terms": { "terms": [ {"field": "k"} ], "size": 5 } } });
+ let result = run_aggs(&agg, &docs);
+ assert_eq!(
+ result["t"]["buckets"].as_array().map(Vec::len),
+ Some(5),
+ "expected 5 buckets, got {result}"
+ );
+ observed.wait();
+ writer.join().unwrap();
+ }
+
+ /// A per-call cap is confined to that call: the next aggregation on the
+ /// same thread is back on the node-wide default, even though the capped
+ /// call bailed out early with an error.
+ #[test]
+ fn per_call_bucket_cap_does_not_outlive_the_call() {
+ let docs: Vec<Value> = (0..5).map(|i| json!({ "k": format!("k{i}") })).collect();
+ let agg = json!({ "t": { "multi_terms": { "terms": [ {"field": "k"} ], "size": 5 } } });
+ let capped = run_aggs_with_limits(&agg, &docs, &docs, AggLimits { max_buckets: 3 });
+ assert_eq!(capped["t"]["__error_status__"], json!(400));
+
+ let after = run_aggs(&agg, &docs);
+ assert_eq!(
+ after["t"]["buckets"].as_array().map(Vec::len),
+ Some(5),
+ "cap outlived its call, got {after}"
+ );
+ }
}
#[cfg(test)]
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0a7070a4f..43f55f4fe 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2116,10 +2116,19 @@ mod semantic_deadline_regression_tests {
boost: None,
similarity: None,
};
+ // Issue #100: the 5 ms budget below used to be measured against the
+ // real clock, so on a loaded box it was already gone when the scan
+ // took its *first* checkpoint — the request timed out having done no
+ // partial work and the count assertion saw 1 instead of 2. The budget
+ // is unchanged; what changed is that this index's scan clock is
+ // scripted, so checkpoint zero reads `base` (inside the budget) and
+ // the next checkpoint reads `base + 6 ms` (past it) no matter how
+ // long the machine takes to get between them.
+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(6));
let result = idx
.run_multi_knn_brute_force(
&match_all(250),
- std::time::Instant::now() + std::time::Duration::from_millis(5),
+ base + std::time::Duration::from_millis(5),
vec![clause],
)
.await
@@ -2140,6 +2149,14 @@ mod semantic_deadline_regression_tests {
engine.create_index("multi-stop", Schema::empty()).unwrap();
let idx = engine.get_index("multi-stop").unwrap();
seed_vectors(&idx).await;
+ // `seed_vectors` leaves a 20 ms per-checkpoint sleep behind. Left in
+ // place it makes the 5 ms budget expire against the REAL clock inside
+ // clause 1, so the test passes even if the cooperative checkpoint it
+ // is meant to pin is gone — the parent's own clause-entry check stops
+ // clause 2 either way. Zeroing it puts the scripted clock back in
+ // charge of when the budget is crossed.
+ idx.test_scan_checkpoint_delay_ms
+ .store(0, Ordering::Relaxed);
let clause = PeeledKnn {
field: "embedding".into(),
vector: vec![1.0, 0.0],
@@ -2149,10 +2166,14 @@ mod semantic_deadline_regression_tests {
boost: None,
similarity: None,
};
+ // See `expired_multi_knn_preserves_partial_timeout_state`: same 5 ms
+ // budget, now crossed at the first clause's second checkpoint by
+ // arithmetic rather than by whatever the machine was doing.
+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(6));
let result = idx
.run_multi_knn_brute_force(
&match_all(250),
- std::time::Instant::now() + std::time::Duration::from_millis(5),
+ base + std::time::Duration::from_millis(5),
vec![clause.clone(), clause],
)
.await
@@ -2194,15 +2215,28 @@ mod semantic_deadline_regression_tests {
.await
.unwrap();
- for timeout_ms in [1, 10, 250] {
- idx.test_scan_checkpoint_delay_ms
- .store(timeout_ms + 10, Ordering::Relaxed);
+ // Issue #100: this used to sleep `budget + 10 ms` inside the
+ // checkpoint and bet that the machine would still reach checkpoint
+ // zero within the budget. On a loaded box it does not — the scan
+ // stops at position 0 and the count below is 1 instead of 2.
+ //
+ // The deadlines stay genuinely in the future when the scan starts,
+ // which is the property this test is named for (as opposed to the
+ // pre-expired deadlines the `expired_*` tests cover), and the small
+ // budgets stay small. What changed is that the clock the scan reads
+ // is scripted: `base` at checkpoint zero, `base + budget + 1 ms`
+ // from position 1 on. The sleep is gone — it existed only to make
+ // real time pass, and the scripted clock does not care about real
+ // time.
+ idx.test_scan_checkpoint_delay_ms
+ .store(0, Ordering::Relaxed);
+ for budget_ms in [1u64, 10, 250] {
idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
- let started = std::time::Instant::now();
+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(budget_ms + 1));
let result = idx
.run_knn_brute_force_with_deadline(
&request,
- started + std::time::Duration::from_millis(timeout_ms),
+ base + std::time::Duration::from_millis(budget_ms),
"embedding",
&[1.0, 0.0],
10,
@@ -2220,7 +2254,6 @@ mod semantic_deadline_regression_tests {
2,
"scan must pass checkpoint zero and stop at the later checkpoint"
);
- assert!(started.elapsed() >= std::time::Duration::from_millis(timeout_ms));
}
}
@@ -2242,10 +2275,14 @@ mod semantic_deadline_regression_tests {
boost: None,
similarity: None,
};
+ // See `expired_multi_knn_preserves_partial_timeout_state` — same 5 ms
+ // race, same fix: the budget is crossed at the child scan's second
+ // checkpoint because the scripted clock says so.
+ let base = idx.script_scan_clock(1, std::time::Duration::from_millis(6));
let result = idx
.run_hybrid_with_deadline(
&match_all(250),
- std::time::Instant::now() + std::time::Duration::from_millis(5),
+ base + std::time::Duration::from_millis(5),
vec![xerj_query::ast::WeightedQuery {
query: knn,
weight: 1.0,
@@ -2271,18 +2308,21 @@ mod semantic_deadline_regression_tests {
.create_index("nested-timeout", Schema::empty())
.unwrap();
let idx = engine.get_index("nested-timeout").unwrap();
- for n in 0..256 {
- idx.index_document(
- Some(format!("n{n}")),
- serde_json::json!({"items": [{"vector": [1.0, 0.0]}]}),
- )
- .await
- .unwrap();
+ // 8 parents × 32 nested elements. The shape matters: the parent loop
+ // checkpoints every 128 *parents* and so never fires past position 0
+ // here, which leaves the nested-array checkpoint (every 128
+ // *elements*, first hit inside parent 3) as the only cooperative
+ // stop in the scan. Deleting it is therefore visible to this test —
+ // with one element per parent the two checkpoints alias at 128 and
+ // the parent loop silently covers for the missing one.
+ for n in 0..8 {
+ let items: Vec<serde_json::Value> = (0..32)
+ .map(|_| serde_json::json!({"vector": [1.0, 0.0]}))
+ .collect();
+ idx.index_document(Some(format!("n{n}")), serde_json::json!({ "items": items }))
+ .await
+ .unwrap();
}
- idx.test_scan_checkpoint_delay_ms
- .store(20, Ordering::Relaxed);
- idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
-
let request = SearchRequest {
query: QueryNode::Nested {
path: "items".into(),
@@ -2297,16 +2337,73 @@ mod semantic_deadline_regression_tests {
}),
score_mode: None,
},
- timeout_ms: Some(5),
+ // Large on purpose. The gate parks the SCAN, but it does not stop
+ // the request's own budget, which keeps burning real time while
+ // the test holds the scan there. A 100 ms budget therefore raced:
+ // on a loaded box the budget could expire against the real clock
+ // before the test observed the park, and the request then timed
+ // out through a different path than the nested-array checkpoint
+ // this test exists to pin. The budget has to outlast any
+ // plausible parking time while still sitting well under the
+ // scripted clock's ten-minute jump, which is what must trip it.
+ timeout_ms: Some(60_000),
..SearchRequest::default()
};
- let result = idx.search(&request).await.unwrap();
+
+ // Issue #100: this used to assert that a 20 ms sleep inside the scan
+ // overran a 5 ms request budget — true on an idle box, false the
+ // moment the machine is loaded enough that the scan needs more than
+ // 5 ms just to reach its first checkpoint.
+ //
+ // Two facts replace that bet, and neither depends on how busy the
+ // machine is.
+ //
+ // The gate replaces the sleep. While the test holds it, the scan is
+ // parked *inside* its nested-array loop, before it has looked at the
+ // clock, and it stays there for as long as we like — so "the scan is
+ // still mid-flight" is established rather than hoped for. Failing to
+ // park at all is the regression this test exists to catch: it means
+ // the nested-array loop has no cooperative checkpoint and ran to
+ // completion, and it is asserted on explicitly below.
+ //
+ // The scripted clock replaces the budget race. Checkpoint zero reads
+ // an instant taken before the request was even built, so it is
+ // always inside the request budget; the nested-array checkpoint
+ // reads ten minutes later, so it is always past it.
+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ idx.test_scan_checkpoint_parked.store(0, Ordering::SeqCst);
+ idx.script_scan_clock(1, std::time::Duration::from_secs(600));
+ let gate = Arc::clone(&idx.test_scan_checkpoint_gate)
+ .lock_owned()
+ .await;
+
+ let scan = {
+ let idx = Arc::clone(&idx);
+ tokio::spawn(async move { idx.search(&request).await })
+ };
+ tokio::time::timeout(std::time::Duration::from_secs(60), async {
+ while idx.test_scan_checkpoint_parked.load(Ordering::SeqCst) == 0 && !scan.is_finished()
+ {
+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;
+ }
+ })
+ .await
+ .expect("nested kNN scan neither parked nor finished within 60 s");
+ assert!(
+ idx.test_scan_checkpoint_parked.load(Ordering::SeqCst) > 0,
+ "nested kNN ran to completion without a cooperative checkpoint inside \
+ its nested-array loop (checkpoints reached: {})",
+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed)
+ );
+ drop(gate);
+
+ let result = scan.await.unwrap().unwrap();
assert!(result.timed_out);
assert_eq!(result.total.relation, TotalHitsRelation::Gte);
assert_eq!(
idx.test_scan_checkpoint_count.load(Ordering::Relaxed),
2,
- "nested scan must stop at its first non-zero checkpoint"
+ "nested scan must stop at its first nested-array checkpoint"
);
}
@@ -2334,11 +2431,30 @@ mod semantic_deadline_regression_tests {
.await
.unwrap();
}
+ // Issue #100: two wall-clock bets used to live here. The checkpoint
+ // count could come back below 16 because a loaded machine burnt the
+ // 100 ms budget before a request reached its *first* checkpoint, and
+ // "these ran concurrently" was inferred from total elapsed time
+ // being under 500 ms — a threshold that says as much about the box
+ // as about the engine.
+ //
+ // Both are now facts rather than bets. The scripted clock puts every
+ // request's checkpoint zero inside its budget and every later
+ // checkpoint past it, so each request reaches exactly two. And
+ // concurrency is established by the gate: all eight scans are parked
+ // on it at the same time, with none of the eight tasks finished,
+ // before any of them is allowed to observe its deadline. Eight scans
+ // simultaneously mid-flight is what "share bounded deadlines" means,
+ // and it is now observed instead of timed.
idx.test_scan_checkpoint_delay_ms
- .store(200, Ordering::Relaxed);
+ .store(0, Ordering::Relaxed);
idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ idx.test_scan_checkpoint_parked.store(0, Ordering::SeqCst);
+ idx.script_scan_clock(1, std::time::Duration::from_secs(600));
+ let gate = Arc::clone(&idx.test_scan_checkpoint_gate)
+ .lock_owned()
+ .await;
- let started = std::time::Instant::now();
let mut tasks = Vec::new();
for n in 0..8 {
let idx = Arc::clone(&idx);
@@ -2351,12 +2467,37 @@ mod semantic_deadline_regression_tests {
filter: None,
boost: None,
},
- timeout_ms: Some(100),
+ // Same reason as `nested_knn_uses_the_shared_request_deadline`:
+ // the gate parks the scan but the request's budget keeps
+ // running against the real clock. This test only needs the
+ // eight requests to be in flight together, so the budget
+ // must simply outlast the parking rather than expire during
+ // it. At 100 ms a loaded box could retire a request before
+ // all eight reached the gate, which is what made this flaky.
+ timeout_ms: Some(60_000),
..SearchRequest::default()
};
idx.search(&request).await
}));
}
+
+ tokio::time::timeout(std::time::Duration::from_secs(60), async {
+ while idx.test_scan_checkpoint_parked.load(Ordering::SeqCst) < 8 {
+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;
+ }
+ })
+ .await
+ .expect("the eight semantic scans never all reached a cooperative checkpoint");
+ // Every scan that reached the gate is still holding there, so none of
+ // the eight requests can have completed: they are in flight together.
+ for (n, task) in tasks.iter().enumerate() {
+ assert!(
+ !task.is_finished(),
+ "request {n} finished while the other scans were still parked"
+ );
+ }
+ drop(gate);
+
for task in tasks {
let result = task.await.unwrap().unwrap();
assert!(result.timed_out);
@@ -2366,10 +2507,6 @@ mod semantic_deadline_regression_tests {
idx.test_scan_checkpoint_count.load(Ordering::Relaxed) >= 16,
"all semantic requests must reach multiple cooperative scan checkpoints"
);
- assert!(
- started.elapsed() < std::time::Duration::from_millis(500),
- "concurrent semantic scans exceeded their own deadlines"
- );
}
}
@@ -2756,6 +2893,33 @@ enum PublicationTestPoint {
#[cfg(test)]
type PublicationTestHook = Arc<dyn Fn(&str, PublicationTestPoint) + Send + Sync + 'static>;
+/// A scripted reading of the clock that cooperative scan checkpoints compare
+/// against their deadline, installed per index by
+/// [`Index::script_scan_clock`].
+///
+/// Checkpoints at scan positions below `advance_from` read exactly `base`;
+/// `advance_from` and later read `base + skew`. Both readings are fixed at
+/// install time, so a deadline derived from `base` is crossed at a position
+/// the test chose, on an idle box and a thrashing one alike.
+#[cfg(test)]
+#[derive(Debug, Clone, Copy)]
+struct ScanClockScript {
+ base: std::time::Instant,
+ advance_from: usize,
+ skew: std::time::Duration,
+}
+
+#[cfg(test)]
+impl ScanClockScript {
+ fn reading_at(&self, position: usize) -> std::time::Instant {
+ if position < self.advance_from {
+ self.base
+ } else {
+ self.base.checked_add(self.skew).unwrap_or(self.base)
+ }
+ }
+}
+
/// Per-index coordinator.
///
/// Owns the storage layer (`IndexStore`), the FTS memtable, and the schema.
@@ -2925,6 +3089,32 @@ pub struct Index {
test_scan_checkpoint_delay_ms: Arc<AtomicU64>,
#[cfg(test)]
test_scan_checkpoint_count: Arc<AtomicU64>,
+ /// Deterministic stand-in for the wall-clock delay above: while a test
+ /// holds this mutex, the first cooperative checkpoint of a scan that is
+ /// past position 0 parks on it and cannot observe the deadline until the
+ /// test lets go. That turns "the scan is still inside its loop when the
+ /// request budget runs out" into a fact established by the test instead
+ /// of a race against the machine's load.
+ #[cfg(test)]
+ test_scan_checkpoint_gate: Arc<tokio::sync::Mutex<()>>,
+ /// Number of checkpoints that have reached the gate. Lets a test wait
+ /// for the scan to be provably parked before it does anything else.
+ #[cfg(test)]
+ test_scan_checkpoint_parked: Arc<AtomicU64>,
+ /// Injected clock for the cooperative scan deadline check, owned by this
+ /// index — i.e. by the object under test, never process-wide. `None`
+ /// (the default) means every checkpoint reads the real clock.
+ ///
+ /// This is how a test states "the request's budget is spent by the time
+ /// the scan reaches position p" as a fact. The alternative — hand the
+ /// scan a budget of a few milliseconds and hope the machine reaches the
+ /// first checkpoint inside it — is a race the test loses whenever the
+ /// box is busy (issue #100). The deadline comparison itself is
+ /// untouched: the scan still compares a clock reading against the real
+ /// deadline it was handed, so a scan that ignores its deadline still
+ /// fails these tests.
+ #[cfg(test)]
+ test_scan_clock: Arc<std::sync::Mutex<Option<ScanClockScript>>>,
#[cfg(test)]
test_ann_passage_guard_pause: Arc<AtomicBool>,
#[cfg(test)]
@@ -3419,6 +3609,12 @@ impl Index {
#[cfg(test)]
test_scan_checkpoint_count: Arc::new(AtomicU64::new(0)),
#[cfg(test)]
+ test_scan_checkpoint_gate: Arc::new(tokio::sync::Mutex::new(())),
+ #[cfg(test)]
+ test_scan_checkpoint_parked: Arc::new(AtomicU64::new(0)),
+ #[cfg(test)]
+ test_scan_clock: Arc::new(std::sync::Mutex::new(None)),
+ #[cfg(test)]
test_ann_passage_guard_pause: Arc::new(AtomicBool::new(false)),
#[cfg(test)]
test_ann_passage_guard_ready: Arc::new(AtomicBool::new(false)),
@@ -3730,6 +3926,12 @@ impl Index {
#[cfg(test)]
test_scan_checkpoint_count: Arc::new(AtomicU64::new(0)),
#[cfg(test)]
+ test_scan_checkpoint_gate: Arc::new(tokio::sync::Mutex::new(())),
+ #[cfg(test)]
+ test_scan_checkpoint_parked: Arc::new(AtomicU64::new(0)),
+ #[cfg(test)]
+ test_scan_clock: Arc::new(std::sync::Mutex::new(None)),
+ #[cfg(test)]
test_ann_passage_guard_pause: Arc::new(AtomicBool::new(false)),
#[cfg(test)]
test_ann_passage_guard_ready: Arc::new(AtomicBool::new(false)),
@@ -7741,8 +7943,75 @@ impl Index {
if _position > 0 && delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
}
+ // Deterministic gate (see `test_scan_checkpoint_gate`). Free when
+ // no test holds the mutex, so this is a no-op for every other
+ // test; when one does hold it, the scan waits here — inside its
+ // loop, before it looks at the clock — until released.
+ if _position > 0 {
+ self.test_scan_checkpoint_parked
+ .fetch_add(1, Ordering::SeqCst);
+ let _gate = self.test_scan_checkpoint_gate.lock().await;
+ }
}
- std::time::Instant::now() >= deadline
+ self.scan_clock_now(_position) >= deadline
+ }
+
+ /// The clock a cooperative scan checkpoint compares against its deadline.
+ /// Always the real clock in a non-test build.
+ #[cfg(not(test))]
+ #[inline]
+ fn scan_clock_now(&self, _position: usize) -> std::time::Instant {
+ std::time::Instant::now()
+ }
+
+ /// The clock a cooperative scan checkpoint compares against its deadline.
+ ///
+ /// Real, unless a test has scripted this index's clock (see
+ /// [`Index::script_scan_clock`]), in which case it returns whatever that
+ /// script says the given scan position sees. Scripted readings do not
+ /// move with the machine, which is the whole point: it turns "the budget
+ /// runs out between checkpoint zero and the next one" from a race the
+ /// test can lose into an arithmetic fact.
+ #[cfg(test)]
+ #[inline]
+ fn scan_clock_now(&self, position: usize) -> std::time::Instant {
+ let script = *self
+ .test_scan_clock
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ match script {
+ Some(script) => script.reading_at(position),
+ None => std::time::Instant::now(),
+ }
+ }
+
+ /// Replace this index's scan clock with a scripted one and return the
+ /// instant every checkpoint before `advance_from` will read. Checkpoints
+ /// at `advance_from` and later read `base + skew` instead.
+ ///
+ /// A test derives its deadline from the returned `base`, so the position
+ /// at which the deadline is crossed is fixed by arithmetic rather than
+ /// by how quickly the machine gets from one checkpoint to the next.
+ ///
+ /// Scoped to this `Index` — the object under test — so a test that
+ /// scripts a clock is invisible to every other test in the process,
+ /// however they are scheduled.
+ #[cfg(test)]
+ fn script_scan_clock(
+ &self,
+ advance_from: usize,
+ skew: std::time::Duration,
+ ) -> std::time::Instant {
+ let base = std::time::Instant::now();
+ *self
+ .test_scan_clock
+ .lock()
+ .unwrap_or_else(|p| p.into_inner()) = Some(ScanClockScript {
+ base,
+ advance_from,
+ skew,
+ });
+ base
}
#[allow(clippy::too_many_arguments)]