Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): #421 item 1 — meta-aware memtable pre-clone rejection (restore O(1) for _seq_no/_version sorts)

xerj-org12754abb17aa
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 8971f4dcf..194d6fb01 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -282,6 +282,63 @@ mod collection_publication_fail_closed_tests {
);
}
+ /// #421: the memtable pre-clone rejection must be META-AWARE. A `_seq_no`
+ /// sort resolves its key from the version map (`meta_sort_value`), the same
+ /// resolver the heap ranks on, so `memtable_primary_key_rejects` can and
+ /// must reject a doc whose `_seq_no` is strictly worse than the bounded
+ /// frontier — skipping its `_source` clone. Before the fix the gate
+ /// declined every meta-sort field and returned `false` for this exact call,
+ /// forcing the full O(memtable) clone-and-offer walk (#401's `search_after`
+ /// latency cliff).
+ #[tokio::test]
+ async fn memtable_preclone_rejection_is_meta_aware_for_seq_no() {
+ use std::sync::Arc;
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine.create_index("seq-reject", text_schema()).unwrap();
+ let idx = engine.get_index("seq-reject").unwrap();
+ idx.abort_background_tasks();
+ for i in 0..6u64 {
+ idx.index_document(Some(format!("d{i}")), json!({ "body": format!("doc {i}") }))
+ .await
+ .unwrap();
+ }
+
+ // Hits sorted by `_seq_no` asc: d0..d5 with strictly increasing seq_no.
+ let req = xerj_query::parse_request(&json!({
+ "query": {"match_all": {}},
+ "size": 6,
+ "sort": [{"_seq_no": "asc"}],
+ }))
+ .unwrap();
+ let all = idx.search(&req).await.unwrap().hits;
+ assert_eq!(all.len(), 6, "corpus size");
+
+ // Fill a size-5 frontier with the five lowest-`_seq_no` docs (d0..d4).
+ let mut topk = super::SortTopK::new(Arc::new(req.sort.clone()), 5, None);
+ for (seq, hit) in all[..5].iter().enumerate() {
+ topk.offer(hit.clone(), seq as u64, &idx);
+ }
+
+ // d5 (highest `_seq_no`) is strictly worse than the frontier for an ASC
+ // sort → the meta-aware pre-clone gate MUST reject it (before the fix:
+ // `false`, the whole point of #421).
+ let worse = &all[5];
+ assert!(
+ super::memtable_primary_key_rejects(&topk, &worse.source, &worse.id, &idx),
+ "#421: a `_seq_no` sort must reject a worse-than-frontier doc via the \
+ pre-clone gate"
+ );
+ // Sanity: the best-`_seq_no` doc belongs on the page and must NOT be rejected.
+ let best = &all[0];
+ assert!(
+ !super::memtable_primary_key_rejects(&topk, &best.source, &best.id, &idx),
+ "the best `_seq_no` doc belongs on the page and must not be rejected"
+ );
+ }
+
async fn assert_flush_publication_blocks_reader_then_finishes(inject_error: bool) {
let dir = tempfile::tempdir().unwrap();
let mut config = Config::default();
@@ -15018,7 +15075,7 @@ impl Index {
let src_arc = self.memtable.get_doc_source_arc(&id);
let rejected = src_arc
.as_deref()
- .is_some_and(|s| memtable_primary_key_rejects(topk, s));
+ .is_some_and(|s| memtable_primary_key_rejects(topk, s, &id, self));
if !rejected {
let source = src_arc.map(|a| (*a).clone()).unwrap_or(Value::Null);
seen_ids.insert(id.clone());
@@ -15674,7 +15731,7 @@ impl Index {
// like `admit_hit!` would have counted it).
if let Some(topk) = sort_topk.as_ref() {
if !seen_ids.contains(&doc_id)
- && memtable_primary_key_rejects(topk, &source)
+ && memtable_primary_key_rejects(topk, &source, &doc_id, self)
{
total_count += 1;
continue;
@@ -39559,7 +39616,7 @@ fn sort_date_normalize(s: &str) -> Option<Value> {
/// ties conservatively admitted, cursor never consulted). Any shape we
/// can't derive → `false` → the doc takes the identical admit path it
/// always took.
-fn memtable_primary_key_rejects(topk: &SortTopK, source: &Value) -> bool {
+fn memtable_primary_key_rejects(topk: &SortTopK, source: &Value, id: &str, idx: &Index) -> bool {
let Some(sf) = topk.fields.first() else {
return false;
};
@@ -39584,7 +39641,20 @@ fn memtable_primary_key_rejects(topk: &SortTopK, source: &Value) -> bool {
// size 5 and size 3000. Give `_id` a numeric sort value and it becomes
// the same defect; add it here if that ever changes.
if is_meta_sort_field(&sf.field) {
- return false;
+ // #421: derive the meta key from the SAME resolver the heap ranks on
+ // (`meta_sort_value`, the version map — NOT `_source`), so a literal
+ // `_seq_no`/`_version` inside a document cannot shadow the real
+ // metadata (the shadowing hazard the old unconditional decline
+ // guarded against), while restoring the O(1) pre-clone rejection:
+ // declining forced the full O(memtable) walk this accelerator exists
+ // to kill (~1.1 µs/buffered doc, ~1.1 s at 1M docs on the
+ // `search_after`/reindex workload #401 was filed for). Unresolvable
+ // (`Null`), `_index` (string) and `_primary_term` (constant) all
+ // yield `None` from `as_f64()` and are conservatively admitted.
+ return match idx.meta_sort_value(&sf.field, id).and_then(|m| m.as_f64()) {
+ Some(v) => topk.primary_f64_rejects(v),
+ None => false,
+ };
}
// Root-level literal key only — mirrors `get_field_value`'s fast path.
// A dotted/nested primary field misses here and is admitted (correct,