AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(#1267): row-exact fast aggs under resident ghosts — the 803s verify window
xerj-orge0a34aa146ff
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex 22c57c5fe..cb9e78797 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -37,10 +37,14 @@//! segment's ghost-position bitmap is empty — per-segment admission via//! `ghost_positions_for`, with the memtable views liveness-gated, the//! delete-blind memtable columnar summaries switched off, and the-//! version-map epoch re-checked before the result is returned. A single-//! superseded row in ANY segment still bails; what changed is that a-//! MERGED index (ghosts compacted away) re-qualifies instead of staying-//! disabled forever on the monotonic `ghost_events` counter.+//! version-map epoch re-checked before the result is returned. A MERGED+//! index (ghosts compacted away) re-qualifies instead of staying disabled+//! forever on the monotonic `ghost_events` counter. Since #1267 a segment+//! with resident ghosts is admitted for ONE provably row-exact plan — a+//! top-level filter plus a single plain `terms` agg — where every number+//! comes from the fused row pass (ghosted rows skipped) and the+//! live-filtered memtable walks, never from a delete-blind O(1) stat;+//! every other plan still bails on the first dirty bitmap.//! * Every non-empty segment must have a decodable `.dv` sidecar.//!//! Known accepted divergences vs brute (documented, benign for the fast-path@@ -178,6 +182,23 @@ struct SegEntry {id: String,cols: super::Resident<super::DocValueMap>,docs: u32,+ /// [#1267] This segment's ghost-position bitmap, carried only when it+ /// is NON-empty and the admission gate let the segment through anyway+ /// (the row-exact plan — see `try_fast_aggs`). Bit set ⇔ the row at+ /// that position is deleted or superseded; every row-fold in the+ /// row-exact plan must skip set bits. `None` = provably ghost-free,+ /// the shape every executor was originally built under.+ ghosts: Option<std::sync::Arc<Vec<u64>>>,+}++/// Bit `row` of a ghost-position bitmap (bit pos set ⇔ row is deleted or+/// superseded). Out-of-range rows are live: the bitmap is sized for the+/// segment's row space, and a short one simply has no ghosts past its end.+#[inline]+fn ghost_bit(ghosts: &[u64], row: u32) -> bool {+ ghosts+ .get((row >> 6) as usize)+ .is_some_and(|w| (w >> (row & 63)) & 1 != 0)}/// Look up a doc-value column for `field`, falling back to the `.keyword`@@ -708,6 +729,24 @@ fn params_only(params: &Value, allowed: &[&str]) -> bool {}}+/// [#1267] The one plan shape admitted with DIRTY ghost bitmaps: the aggs+/// tree is EXACTLY one plain `terms` aggregation — a single named entry+/// whose body is `{"terms": {...}}` and nothing else. No sub-aggs (no+/// `aggs` key), no metrics, no `meta`, no siblings: anything beyond the+/// single terms body makes the shape's cost depend on an executor that may+/// read a delete-blind whole-segment stat, so it keeps today's bail. The+/// caller additionally requires a top-level filter (checked before calling)+/// so `exec_terms` routes through the fused row pass.+fn single_plain_terms_agg(aggs_obj: &serde_json::Map<String, Value>) -> bool {+ if aggs_obj.len() != 1 {+ return false;+ }+ let Some(body) = aggs_obj.values().next().and_then(|v| v.as_object()) else {+ return false;+ };+ body.len() == 1 && body.contains_key("terms")+}+impl Index {/// Entry point — see module docs. Returns `Some(agg_result)` (shaped/// exactly like `run_aggs_with_all`'s return) or `None` to fall back.@@ -774,14 +813,33 @@ impl Index {// stored doc at that row is deleted or superseded; the merge// survivor filter guarantees at most one live copy per id, so the// id-level predicate is exact and its row space IS the `.dv`- // column row space). A segment with a NON-EMPTY bitmap still bails- // (its stats would double-count ghosts); a segment with an EMPTY- // bitmap is provably ghost-free — live == physical — which is- // exactly the invariant every executor was built under, so the- // row-level arithmetic runs unchanged. Steady state after merges- // is clean bitmaps, so the fast path comes BACK; active rewrite- // windows keep falling to brute, as before. `None` (stored- // section unreadable) keeps today's bail.+ // column row space). A segment with an EMPTY bitmap is provably+ // ghost-free — live == physical — which is exactly the invariant+ // every executor was built under, so the row-level arithmetic runs+ // unchanged. Steady state after merges is clean bitmaps, so the+ // fast path comes BACK. `None` (stored section unreadable) keeps+ // today's bail.+ //+ // [#1267] A segment with a NON-EMPTY bitmap no longer bails every+ // request: that was the 803,735 ms verify window. A resumed crawl+ // shard carries tombstones from the resume's overwrites, so EVERY+ // one of its ~800 finalize-verify windows fell to the brute agg+ // corpus, which materialises the whole index per window — measured+ // live on the 53,153-doc `cves-28` shard (issue #1267). Dirty+ // bitmaps are now admitted for exactly one plan shape that is+ // provably row-exact under ghosts: a top-level filter present (so+ // `exec_terms` takes the fused row pass, never the whole-segment+ // `per_ord_count` shortcut) plus a single plain `terms` agg — no+ // sub-aggs, no metrics, no siblings. Every number that shape+ // produces comes from per-row accounting (`fused_seg_pass` skips+ // ghosted rows; the filtered total row-scans dirty segments) and+ // the live-filtered memtable walks. Any other plan still bails on+ // the first dirty bitmap, as before — the O(1) stats it would read+ // (`per_ord_count`, `live_count`, `range_count`) count physical+ // rows and would double-count ghosts.+ let row_exact = top_filter.is_some() && single_plain_terms_agg(aggs_obj);+ let mut dirty_ghosts: std::collections::HashMap<String, std::sync::Arc<Vec<u64>>> =+ std::collections::HashMap::new();let mem_live_gate: Option<std::collections::HashSet<String>> = if deletes_present {for meta in &snap.segments {if meta.doc_count == 0 {@@ -789,7 +847,10 @@ impl Index {}let ghosts = self.ghost_positions_for(&meta.id, meta.doc_count)?;if ghosts.iter().any(|w| *w != 0) {- return None;+ if !row_exact {+ return None;+ }+ dirty_ghosts.insert(meta.id.clone(), ghosts);}}// The memtable leg of the same gate: a buffered doc whose live@@ -833,6 +894,7 @@ impl Index {id: meta.id.clone(),cols,docs: meta.doc_count as u32,+ ghosts: dirty_ghosts.get(&meta.id).cloned(),});}@@ -867,17 +929,25 @@ impl Index {// Filtered `hits.total`: the number of live docs matching the query.// For match_all the caller derives it from segment + memtable counts// (and later overwrites with the delete-aware live count), so we only- // compute a total when a filter narrowed the corpus. No deletes are- // present on the fast path (gated above), so physical row counts are- // exact, and each memtable doc is one hit (weights affect agg- // `doc_count`, not `hits.total`).+ // compute a total when a filter narrowed the corpus. Clean segments+ // have live == physical (gated above), so the O(1) stats are exact;+ // [#1267] a dirty-but-admitted segment instead counts live matching+ // rows directly — `seg_pred_count`'s whole-segment arms+ // (`per_ord_count`, `range_count`, `live_count`) are delete-blind+ // and would count ghosted rows. Each memtable doc is one hit+ // (weights affect agg `doc_count`, not `hits.total`).let filtered_total: Option<u64> = match &ctx.top_filter {None => None,Some(pred) => {let mut total: u64 = 0;for seg in &ctx.segs {let sp = resolve_pred(&seg.cols, pred, &ctx.segs, ctx.mapped_fields)?;- total += seg_pred_count(&sp, seg.docs);+ total += match seg.ghosts.as_deref() {+ None => seg_pred_count(&sp, seg.docs),+ Some(g) => (0..seg.docs)+ .filter(|&row| !ghost_bit(g, row) && seg_pred_matches(&sp, row))+ .count() as u64,+ };}if let Some(q) = &ctx.top_filter_query {// O(matching) columnar fold when the filter columnarises;@@ -2410,7 +2480,19 @@ impl<'a> FastCtx<'a> {.collect();let th_dense = th_col.is_some_and(|n| n.null_bitmap.is_empty());+ // [#1267] Ghosted rows (deleted/superseded but still resident) are+ // skipped before ANY accounting — present only on dirty-but-admitted+ // segments (the row-exact plan), where counting them would+ // double-count docs the brute path no longer sees. Hoisted beside+ // the null probes: clean segments pay one `Option` check.+ let seg_ghosts = seg.ghosts.as_deref();+for row in 0..seg.docs {+ if let Some(g) = seg_ghosts {+ if ghost_bit(g, row) {+ continue;+ }+ }if let Some(sp) = &top_sp {if !seg_pred_matches(sp, row) {continue;@@ -5992,6 +6074,7 @@ mod range_kw_tests {id: "seg1".to_string(),cols: crate::segment_cache_budget::CacheResident::uncached(m),docs: 2,+ ghosts: None,};assert!(seg.col("extension").is_some(),@@ -6169,6 +6252,7 @@ mod range_kw_tests {id: "seg1".to_string(),cols: crate::segment_cache_budget::CacheResident::uncached(cols),docs: 1,+ ghosts: None,}}diff --git a/engine/crates/xerj-engine/tests/verify_window_terms_perf.rs b/engine/crates/xerj-engine/tests/verify_window_terms_perf.rsnew file mode 100644index 000000000..2304ccc23--- /dev/null+++ b/engine/crates/xerj-engine/tests/verify_window_terms_perf.rs@@ -0,0 +1,364 @@+//! #1267 — the batched finalize-verify windows against a data index whose+//! every record shares one of the window's digests (the cve-records resume+//! shape: ~52 records per source file, so a 1,024-digest `terms` window+//! covers an entire shard). Two lanes, two questions:+//!+//! 1. COUNT lane — `bool.filter [terms ax_file ×1024]` (+ the semantic+//! leg's `exists`) answering a `terms` agg at `size: window.len()`+//! (1,024 buckets). On the live crawl node one such window against+//! the 53,153-doc `cves-28` shard ran 803,735 ms in `segment_loop`+//! with `shortcut=None` — the brute agg corpus materialising the whole+//! index per window. The #1260 test (`fast_aggs_exists_and_deletes`)+//! pins the same filter shape at TWO values; this file pins it at the+//! client's real `VERIFY_WINDOW` (1,024) and at full-shard coverage,+//! both on clean segments AND — the #1267 residual — on a shard whose+//! flushed segment carries RESIDENT GHOSTS (a resume's overwrites and+//! deletes), which is what had every window bailed to the brute corpus.+//! 2. DOC lane — the catalog read-back shape (`size: 8192`,+//! `_source: ["file_key"]`, `terms file_key ×1024` + `term run_id`),+//! measured for wall time only: this is the #1267 "verify windows+//! with size > 0" residual, pinned clean and under tombstones.+//!+//! Both lanes assert fast-path/brute-path agreement, not just timing.++use serde_json::json;+use std::time::Instant;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::Schema;+use xerj_engine::aggs::fast_path_aggs_served;+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++/// The client's window size (`SyncExecutor::VERIFY_WINDOW`).+const WINDOW: usize = 1024;+/// Groups (distinct digests); at `DOCS_PER_GROUP` records each the index+/// clears `FAST_AGG_MIN_DOCS` (10,000) and a full window covers the whole+/// index — the cve-records resume shape.+const GROUPS: usize = 1024;+const DOCS_PER_GROUP: usize = 60;++static ONE_AT_A_TIME: std::sync::LazyLock<tokio::sync::Mutex<()>> =+ std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++async fn run(idx: &std::sync::Arc<Index>, body: serde_json::Value) -> serde_json::Value {+ let req = parse_request(&body).unwrap();+ let res = idx.search(&req).await.unwrap();+ json!({+ "total": res.total.value,+ "hits": res.hits.len(),+ "aggs": res.aggs.clone().unwrap_or(json!(null)),+ })+}++/// The record body of group `g`, record `r` — shared by the seed and the+/// byte-identical rewrite (whose ghost must be counted out while its+/// memtable replacement is counted in, live corpus unchanged).+fn doc_body(g: usize, r: usize) -> serde_json::Value {+ json!({+ "ax_file": format!("axf2-{:032x}-{:04x}", g * 7, g),+ "ax_dataset": "cvelistv5-cves-28",+ "cve_id": format!("CVE-2026-{g:04}"),+ "text": format!(+ "A heap-based buffer overflow was found in record {r} of group {g}. {}",+ "The parser reads a length prefix without validating it against the allocation. ".repeat(14),+ ),+ })+}++/// `GROUPS × DOCS_PER_GROUP` flushed docs shaped like the crawl's data+/// records: every doc of a group carries the group's `ax_file` content+/// digest, ~1 KB of prose (a real CVE record's description block), and the+/// dataset keyword the exact-dataset read-back filters on.+async fn seed(idx: &std::sync::Arc<Index>) -> Vec<String> {+ let mut digests = Vec::with_capacity(GROUPS);+ for g in 0..GROUPS {+ digests.push(format!("axf2-{:032x}-{:04x}", g * 7, g));+ for r in 0..DOCS_PER_GROUP {+ idx.index_document(Some(format!("doc-{}", g * DOCS_PER_GROUP + r)), doc_body(g, r))+ .await+ .unwrap();+ }+ }+ idx.flush().await.unwrap();+ digests+}++fn count_lane(digests: &[String], exists: bool) -> serde_json::Value {+ let mut filter = vec![json!({ "terms": { "ax_file": digests } })];+ if exists {+ filter.push(json!({ "exists": { "field": "text" } }));+ }+ json!({+ "size": 0,+ "query": { "bool": { "filter": filter } },+ "aggs": { "values": { "terms": { "field": "ax_file", "size": digests.len() } } }+ })+}++/// `key → doc_count` for a terms bucket array (order varies with counts).+fn bucket_counts(buckets: &serde_json::Value) -> std::collections::HashMap<String, u64> {+ buckets+ .as_array()+ .unwrap()+ .iter()+ .map(|b| {+ (+ b["key"].as_str().unwrap().to_string(),+ b["doc_count"].as_u64().unwrap(),+ )+ })+ .collect()+}++/// The count-lane window must be served by the columnar fast path even at+/// the client's full window size with whole-index coverage, and its buckets+/// must agree with the brute path (same request under+/// `XERJ_DISABLE_FAST_AGGS`-equivalent… which cannot be flipped per-test in+/// one process — so agreement is pinned against the per-digest expectation:+/// every bucket holds exactly `DOCS_PER_GROUP`).+#[tokio::test]+async fn count_lane_full_window_is_served_columnar() {+ let _sequential = ONE_AT_A_TIME.lock().await;+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine.create_index("cves28", Schema::empty()).unwrap();+ let idx = engine.get_index("cves28").unwrap();+ let digests = seed(&idx).await;++ for exists in [false, true] {+ let before = fast_path_aggs_served();+ let t0 = Instant::now();+ let res = run(&idx, count_lane(&digests, exists)).await;+ let elapsed = t0.elapsed();+ let served = fast_path_aggs_served() - before;++ assert_eq!(+ res["total"].as_u64().unwrap(),+ (GROUPS * DOCS_PER_GROUP) as u64,+ "exists={exists} total"+ );+ let buckets = res["aggs"]["values"]["buckets"].as_array().unwrap();+ assert_eq!(buckets.len(), WINDOW, "exists={exists} bucket count");+ for b in buckets {+ assert_eq!(b["doc_count"].as_u64().unwrap(), DOCS_PER_GROUP as u64);+ }+ // THE regression signal: the live crawl node answered this shape from+ // the brute agg corpus (803,735 ms segment_loop). If the fast path+ // refuses it again, fail loudly rather than ship the slow answer.+ assert!(+ served >= 1,+ "exists={exists}: columnar fast path did not serve the full-window verify shape ({} ms)",+ elapsed.as_millis()+ );+ }+}++/// THE #1267 repro: a resumed crawl shard carries tombstones — the resume+/// overwrites/deletes rows whose superseded copies are still resident in a+/// flushed segment — and until this fix ANY dirty ghost bitmap bailed the+/// whole fast-agg request, so every verify window paid the brute agg corpus+/// (803,735 ms measured on the live 53,153-doc `cves-28`). The admission+/// gate now admits the verify shape (filter + single plain terms agg) with+/// dirty bitmaps: the row pass skips ghosted rows, the filtered total+/// row-scans dirty segments, and the memtable replacement copies are+/// counted live. Exactness is pinned by arithmetic (known deletes, known+/// byte-identical rewrites) AND by agreement with the brute reference —+/// the same terms agg plus a sibling `value_count`, which makes the aggs+/// tree non-row-exact and so falls to the brute executor.+#[tokio::test]+async fn count_lane_with_resident_ghosts_is_served_and_exact() {+ let _sequential = ONE_AT_A_TIME.lock().await;+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine.create_index("cves28g", Schema::empty()).unwrap();+ let idx = engine.get_index("cves28g").unwrap();+ let digests = seed(&idx).await;++ // Group 0 loses its first DELETED rows (ghosts that MATCH the window+ // filter — their `ax_file` is `digests[0]`, in-window).+ const DELETED: usize = 10;+ // Group 1's first REWRITTEN rows are overwritten byte-identically:+ // ghosts in the segment, fresh live copies in the memtable, live+ // corpus and every bucket count unchanged.+ const REWRITTEN: usize = 25;+ for i in 0..DELETED {+ idx.delete_document(&format!("doc-{i}")).await.unwrap();+ }+ for r in 0..REWRITTEN {+ idx.index_document(Some(format!("doc-{}", DOCS_PER_GROUP + r)), doc_body(1, r))+ .await+ .unwrap();+ }++ let before = fast_path_aggs_served();+ let t0 = Instant::now();+ let fast = run(&idx, count_lane(&digests, false)).await;+ let elapsed = t0.elapsed();+ assert_eq!(+ fast_path_aggs_served() - before,+ 1,+ "a dirty-but-admitted segment must serve the verify shape columnarly ({} ms)",+ elapsed.as_millis()+ );++ assert_eq!(+ fast["total"].as_u64().unwrap(),+ (GROUPS * DOCS_PER_GROUP - DELETED) as u64,+ "ghosted rows must leave the filtered total"+ );+ let counts = bucket_counts(&fast["aggs"]["values"]["buckets"]);+ assert_eq!(counts.len(), WINDOW, "every digest still has live docs");+ assert_eq!(+ counts[&digests[0]],+ (DOCS_PER_GROUP - DELETED) as u64,+ "deleted rows must leave their bucket"+ );+ assert_eq!(+ counts[&digests[1]],+ DOCS_PER_GROUP as u64,+ "byte-identical rewrites: ghost out, memtable copy in — net zero"+ );+ for (d, c) in &counts {+ if *d != digests[0] {+ assert_eq!(*c, DOCS_PER_GROUP as u64, "untouched group {d}");+ }+ }++ // Brute reference: the sibling agg makes the tree non-row-exact, so+ // the dirty segments bail admission and the brute executor answers.+ // Its terms buckets and total are the reference the fast path must+ // reproduce.+ let mut reference = count_lane(&digests, false);+ reference["aggs"]["n"] = json!({ "value_count": { "field": "cve_id" } });+ let ref_before = fast_path_aggs_served();+ let brute = run(&idx, reference).await;+ assert_eq!(+ fast_path_aggs_served() - ref_before,+ 0,+ "the sibling agg must force the brute executor"+ );+ assert_eq!(fast["total"], brute["total"], "total disagrees with brute");+ assert_eq!(+ fast["aggs"]["values"]["buckets"], brute["aggs"]["values"]["buckets"],+ "columnar buckets disagree with the brute reference under ghosts"+ );++ // Wall-time guard: the row pass over one dirty segment + 1,024-bucket+ // build is milliseconds-scale work; the brute corpus this replaced+ // materialised every matched doc.+ assert!(+ elapsed.as_millis() < 5_000,+ "dirty-segment verify window took {} ms",+ elapsed.as_millis()+ );+}++/// The catalog read-back shape: a fetching search, no aggs. Wall-time+/// regression guard for the #1267 doc lane — a 1,024-digest window over a+/// fully-matching index must answer in seconds, not minutes, and return+/// exactly the window's documents — clean AND with tombstones resident+/// (the resume state of the live catalog).+#[tokio::test]+async fn doc_lane_window_fetch_is_bounded() {+ let _sequential = ONE_AT_A_TIME.lock().await;+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine.create_index("catalog", Schema::empty()).unwrap();+ let idx = engine.get_index("catalog").unwrap();+ // The catalog holds ONE doc per (digest, alias) — seed the same digests+ // plus a run_id that matches everything (the resume-rewrote-everything+ // case) and a second run's docs that the run_id term must exclude.+ let mut digests = Vec::with_capacity(GROUPS);+ let mut run_this_ids = Vec::with_capacity(GROUPS);+ let mut run_other_ids = Vec::new();+ let mut docs = Vec::new();+ for g in 0..GROUPS {+ let digest = format!("axf2-{:032x}-{:04x}", g * 7, g);+ digests.push(digest.clone());+ docs.push(json!({+ "file_key": digest,+ "run_id": "run-this",+ "ax_path": format!("CVE-2026-{g:04}.json"),+ }));+ if g % 3 == 0 {+ docs.push(json!({+ "file_key": digest,+ "run_id": "run-other",+ "ax_path": format!("CVE-2026-{g:04}.json"),+ }));+ }+ }+ for (i, doc) in docs.into_iter().enumerate() {+ if doc["run_id"] == json!("run-this") {+ run_this_ids.push(format!("cat-{i}"));+ } else {+ run_other_ids.push(format!("cat-{i}"));+ }+ idx.index_document(Some(format!("cat-{i}")), doc).await.unwrap();+ }+ idx.flush().await.unwrap();++ let body = json!({+ "size": 8192,+ "from": 0,+ "track_total_hits": true,+ "_source": ["file_key"],+ "query": { "bool": { "filter": [+ { "terms": { "file_key": digests } },+ { "term": { "run_id": "run-this" } }+ ]}}+ });+ let t0 = Instant::now();+ let res = run(&idx, body.clone()).await;+ let elapsed = t0.elapsed();++ assert_eq!(res["total"].as_u64().unwrap(), GROUPS as u64);+ assert_eq!(res["hits"].as_u64().unwrap(), GROUPS as u64);+ // The live catalog leg measured 31 ms cold / 14 ms warm per window on+ // 402,814 docs; a full second here means the fetch path regressed to+ // parsing the whole index per window.+ assert!(+ elapsed.as_millis() < 1_000,+ "catalog window fetch took {} ms",+ elapsed.as_millis()+ );++ // Dirty phase (#1267): the resume state — deletes on run-this rows+ // (ghosts that MATCH the window) and byte-identical rewrites of+ // run-other rows (ghosts the run_id term excludes anyway). The fetch+ // must count exactly the live run-this docs.+ let mut deleted = 0usize;+ for (g, id) in run_this_ids.iter().enumerate() {+ if g % 10 == 0 {+ idx.delete_document(id).await.unwrap();+ deleted += 1;+ }+ }+ for id in run_other_ids.iter().take(40) {+ let doc = idx.get_document(id).await.unwrap().unwrap();+ idx.index_document(Some(id.clone()), doc).await.unwrap();+ }++ let t0 = Instant::now();+ let res = run(&idx, body.clone()).await;+ let elapsed = t0.elapsed();++ assert_eq!(+ res["total"].as_u64().unwrap(),+ (GROUPS - deleted) as u64,+ "tombstoned catalog rows must leave the fetch total"+ );+ assert_eq!(res["hits"].as_u64().unwrap(), (GROUPS - deleted) as u64);+ assert!(+ elapsed.as_millis() < 1_000,+ "catalog window fetch under tombstones took {} ms",+ elapsed.as_millis()+ );+}