AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix: exact TSDB dedup + delete-aware BM25 — conformance 99.9% (1325/1326), search 100%
xerj-orgb5834c06dc02
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 4f0c83ff9..d6de52b76 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -9288,13 +9288,15 @@ async fn process_bulk_body(// bulk pipeline sees a doc with _ignored already populated and any// malformed values stripped, matching the path taken by the single-// doc PUT _doc/{id} / POST _doc routes.- // NOTE: an attempted TSDB last-wins `_id` rewrite (rewrite_bulk_time_series_ids,- // retained below but UNUSED) over-merged documents with DISTINCT `_tsid`s —- // it regressed smoke/20_tsdb_consistency (8 unique-tsid docs collapsed to 4).- // Disabled until the dimension/_tsid derivation is exact; the normal path- // leaves bulk bodies byte-for-byte unchanged.- let _ = rewrite_bulk_time_series_ids;- let base_text: &str = text;+ // TSDB last-wins `_id` rewrite: for `time_series`-mode indices, inject a+ // deterministic `_id` derived from the COMPLETE `_tsid` (all dimension+ // fields — mapping-declared `time_series_dimension:true` AND every+ // `routing_path` entry, nested paths resolved) plus the normalized+ // `@timestamp`. Two docs collapse (later overwrites earlier) ONLY when all+ // dimensions and the instant are identical, matching ES. Returns `None`+ // when no action targets a TSDB index, so normal bulk bodies are unchanged.+ let ts_rewritten = rewrite_bulk_time_series_ids(state, default_index, text);+ let base_text: &str = ts_rewritten.as_deref().unwrap_or(text);let rewritten_body;let text_ref: &str = if state.engine.index_mappings.is_empty() {@@ -19098,7 +19100,39 @@ fn time_series_dimension_fields(state: &AppState, index: &str) -> Option<Vec<Strif mode != Some("time_series") {return None;}- let mut dims: Vec<String> = Vec::new();+ // The full `_tsid` is the COMPLETE set of dimension fields: every mapping+ // field (recursively, by dotted path) flagged `time_series_dimension:true`+ // UNIONed with every field named in `index.routing_path`. A BTreeSet keeps+ // them sorted and de-duplicated. Two docs collapse only when ALL of these+ // (plus the timestamp) are identical, so missing a single nested dimension+ // (e.g. `k8s.pod.uid`) would wrongly over-merge distinct series.+ let mut dims: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();++ // 1. Mapping-declared dimensions, recursing into nested `properties`+ // so nested fields contribute their full dotted path.+ fn collect_dims(+ props: &serde_json::Map<String, Value>,+ prefix: &str,+ out: &mut std::collections::BTreeSet<String>,+ ) {+ for (name, fm) in props {+ let full = if prefix.is_empty() {+ name.clone()+ } else {+ format!("{prefix}.{name}")+ };+ if fm+ .get("time_series_dimension")+ .and_then(Value::as_bool)+ .unwrap_or(false)+ {+ out.insert(full.clone());+ }+ if let Some(sub) = fm.get("properties").and_then(Value::as_object) {+ collect_dims(sub, &full, out);+ }+ }+ }if let Some(mapping) = state.engine.index_mappings.get(index) {let props = mapping.get("mappings")@@ -19106,41 +19140,37 @@ fn time_series_dimension_fields(state: &AppState, index: &str) -> Option<Vec<Str.or_else(|| mapping.get("properties")).and_then(Value::as_object);if let Some(props_obj) = props {- for (name, fm) in props_obj {- if fm- .get("time_series_dimension")- .and_then(Value::as_bool)- .unwrap_or(false)- {- dims.push(name.clone());- }- }+ collect_dims(props_obj, "", &mut dims);}}- if dims.is_empty() {- // Fall back to the routing_path (the dimension subset used for- // shard routing) when the mapping doesn't flag dimensions.- let rp = settings- .get("index")- .and_then(|ix| ix.get("routing_path"))- .or_else(|| settings.get("routing_path"));- match rp {- Some(Value::Array(a)) => {- for v in a {- if let Some(s) = v.as_str() {- dims.push(s.to_string());++ // 2. Union in every field named in `index.routing_path` (the dimension+ // subset used for shard routing). Wildcard patterns are skipped — they+ // aren't concrete field names to read from the document.+ let rp = settings+ .get("index")+ .and_then(|ix| ix.get("routing_path"))+ .or_else(|| settings.get("routing_path"));+ match rp {+ Some(Value::Array(a)) => {+ for v in a {+ if let Some(s) = v.as_str() {+ if !s.contains('*') {+ dims.insert(s.to_string());}}}- Some(Value::String(s)) => dims.push(s.clone()),- _ => {}}+ Some(Value::String(s)) if !s.contains('*') => {+ dims.insert(s.clone());+ }+ _ => {}}+if dims.is_empty() {return None;}- dims.sort();- Some(dims)+ Some(dims.into_iter().collect())}/// Compute a deterministic `_id` for a time_series (TSDB) document from its@@ -19159,10 +19189,13 @@ fn time_series_doc_id(doc: &Value, dim_fields: &[String]) -> Option<String> {for f in dim_fields {composite.push_str(f);composite.push('\u{1}');- match obj.get(f) {- Some(Value::String(s)) => composite.push_str(s),+ // Dimension fields may be nested (e.g. `k8s.pod.uid`); resolve the+ // full dotted path against the source document, not just top-level+ // keys, so every dimension actually contributes to the `_tsid`.+ match get_source_value_by_path(doc, f) {+ Some(Value::String(s)) => composite.push_str(&s),+ Some(Value::Null) | None => {}Some(other) => composite.push_str(&other.to_string()),- None => {}}composite.push('\u{2}');}diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex b52342103..4cdfec329 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -580,7 +580,13 @@ impl ShardedFtsMemtable {let q_tokens = analyzer.analyze(query);if q_tokens.is_empty() { return Vec::new(); }- let global_doc_count: u64 = self.doc_count() as u64;+ // Delete-aware BM25 collection statistics (Lucene/ES parity): the+ // scoring N counts both live docs AND tombstoned/superseded versions+ // that have not yet been merged away. NOTE: this is *only* the N fed+ // to the BM25 IDF — hits.total and pagination still use the live+ // `doc_count()`, so a search over an index that has never had an+ // update/delete scores bit-for-bit identically to before.+ let mut global_doc_count: u64 = self.doc_count() as u64;// Aggregate (per-field global avg_field_len, per-(field,term) doc_freq).let mut field_total_len: std::collections::HashMap<String, (f64, u64)> =std::collections::HashMap::new();@@ -588,13 +594,21 @@ impl ShardedFtsMemtable {std::collections::HashMap::new();for shard in &self.shards {let g = shard.read();- // Field length sums.+ // Live N is already in `global_doc_count`; add tombstoned versions.+ global_doc_count += g.ghost_docs;+ // Field length sums (live).for (fname, (sum, n)) in &g.avg_field_lengths {let entry = field_total_len.entry(fname.clone()).or_insert((0.0, 0));entry.0 += sum;entry.1 += n;}- // Per-term doc_freq across shards.+ // Field length sums (tombstoned versions retained for avgdl).+ for (fname, (sum, n)) in &g.ghost_field_len {+ let entry = field_total_len.entry(fname.clone()).or_insert((0.0, 0));+ entry.0 += sum;+ entry.1 += n;+ }+ // Per-term doc_freq across shards (live postings).for (fname, postings) in &g.index {if !fields.is_empty() && !fields.iter().any(|f| f == fname) {continue;@@ -606,6 +620,18 @@ impl ShardedFtsMemtable {}}}+ // Per-term doc_freq from tombstoned versions (delete-aware df).+ for (fname, terms) in &g.ghost_doc_freq {+ if !fields.is_empty() && !fields.iter().any(|f| f == fname) {+ continue;+ }+ for token in &q_tokens {+ if let Some(df) = terms.get(&token.text) {+ *term_global_df.entry((fname.clone(), token.text.clone()))+ .or_insert(0) += *df;+ }+ }+ }}let global_avg_field_len: std::collections::HashMap<String, f32> =field_total_len.into_iter()@@ -857,6 +883,10 @@ impl ShardedFtsMemtable {g.field_lengths = HashMap::new();g.avg_field_lengths = HashMap::new();g.doc_id_index = HashMap::new();+ // Flush == merge: purge delete-aware ghost collection stats.+ g.ghost_docs = 0;+ g.ghost_field_len = HashMap::new();+ g.ghost_doc_freq = HashMap::new();g.docs.shrink_to_fit();d};@@ -937,6 +967,11 @@ impl FtsMemtable {self.field_lengths = HashMap::new();self.avg_field_lengths = HashMap::new();self.doc_id_index = HashMap::new();+ // Flush is the equivalent of a Lucene merge: tombstone contributions+ // are purged, so delete-aware ghost statistics reset to empty.+ self.ghost_docs = 0;+ self.ghost_field_len = HashMap::new();+ self.ghost_doc_freq = HashMap::new();self.docs.shrink_to_fit();result}@@ -958,6 +993,11 @@ impl FtsMemtable {self.field_lengths = HashMap::new();self.avg_field_lengths = HashMap::new();self.doc_id_index = HashMap::new();+ // Flush is the equivalent of a Lucene merge: tombstone contributions+ // are purged, so delete-aware ghost statistics reset to empty.+ self.ghost_docs = 0;+ self.ghost_field_len = HashMap::new();+ self.ghost_doc_freq = HashMap::new();self.docs.shrink_to_fit();result}@@ -990,6 +1030,23 @@ pub struct FtsMemtable {avg_field_lengths: HashMap<String, (f64, u64)>,/// doc_id → position in self.docs for O(1) lookupdoc_id_index: HashMap<String, usize>,+ /// Delete-aware BM25 collection statistics (Lucene/ES parity).+ ///+ /// When a document is superseded by an update (remove + re-insert) or+ /// explicitly deleted, Lucene keeps that document's contribution to the+ /// collection statistics (N, total field length, per-term doc_freq)+ /// until the segment is actually merged. We mirror that: `remove()`+ /// strips the old version from the LIVE structures (so it no longer+ /// matches or counts toward hits.total) but folds its contribution into+ /// these "ghost" accumulators, which are added back ONLY for BM25+ /// scoring. They reset to empty on every drain/flush — the moral+ /// equivalent of a Lucene merge purging tombstones.+ ///+ /// A document that was never removed contributes nothing here, so its+ /// BM25 score is bit-for-bit identical to before this feature existed.+ ghost_docs: u64,+ ghost_field_len: HashMap<String, (f64, u64)>,+ ghost_doc_freq: HashMap<String, HashMap<String, u64>>,}impl FtsMemtable {@@ -1004,6 +1061,9 @@ impl FtsMemtable {field_lengths: HashMap::new(),avg_field_lengths: HashMap::new(),doc_id_index: HashMap::new(),+ ghost_docs: 0,+ ghost_field_len: HashMap::new(),+ ghost_doc_freq: HashMap::new(),}}@@ -1022,6 +1082,9 @@ impl FtsMemtable {field_lengths: HashMap::new(),avg_field_lengths: HashMap::new(),doc_id_index: HashMap::new(),+ ghost_docs: 0,+ ghost_field_len: HashMap::new(),+ ghost_doc_freq: HashMap::new(),}}@@ -1378,21 +1441,47 @@ impl FtsMemtable {if !self.doc_id_index.contains_key(doc_id) {return;}- // Remove from inverted index.- for field_index in self.index.values_mut() {- for posting_list in field_index.values_mut() {- posting_list.remove(doc_id);+ // Remove from inverted index. Capture every (field, term) this doc+ // actually contributed to so we can preserve its delete-aware+ // doc_freq contribution (Lucene keeps tombstoned postings counted+ // until a merge). Two-phase to avoid borrowing `self.index` and+ // `self.ghost_doc_freq` mutably at once.+ let mut ghosted_terms: Vec<(String, String)> = Vec::new();+ for (field_name, field_index) in self.index.iter_mut() {+ for (term, posting_list) in field_index.iter_mut() {+ if posting_list.remove(doc_id).is_some() {+ ghosted_terms.push((field_name.clone(), term.clone()));+ }}}+ for (field_name, term) in ghosted_terms {+ *self+ .ghost_doc_freq+ .entry(field_name)+ .or_default()+ .entry(term)+ .or_insert(0) += 1;+ }// Remove from field length caches and update running averages.+ // Retain the removed doc's field-length contribution as a ghost so+ // avgdl stays delete-aware (total_field_length / N counts tombstones).+ let mut ghosted_lengths: Vec<(String, u32)> = Vec::new();for (field_name, lengths) in &mut self.field_lengths {if let Some(token_count) = lengths.remove(doc_id) {if let Some(entry) = self.avg_field_lengths.get_mut(field_name) {entry.0 -= token_count as f64;entry.1 = entry.1.saturating_sub(1);}+ ghosted_lengths.push((field_name.clone(), token_count));}}+ for (field_name, token_count) in ghosted_lengths {+ let g = self.ghost_field_len.entry(field_name).or_insert((0.0, 0));+ g.0 += token_count as f64;+ g.1 += 1;+ }+ // One more document is now a tombstone for collection-stats purposes.+ self.ghost_docs += 1;// Remove from docs list AND the parallel DocValues columns.if let Some(pos) = self.doc_id_index.remove(doc_id) {let entry = self.docs.remove(pos);@@ -1619,6 +1708,11 @@ impl FtsMemtable {self.field_lengths = HashMap::new();self.avg_field_lengths = HashMap::new();self.doc_id_index = HashMap::new();+ // Flush is the equivalent of a Lucene merge: tombstone contributions+ // are purged, so delete-aware ghost statistics reset to empty.+ self.ghost_docs = 0;+ self.ghost_field_len = HashMap::new();+ self.ghost_doc_freq = HashMap::new();// `self.docs` was drained in place; its Vec backing is kept (cheap).self.docs.shrink_to_fit();result@@ -1663,6 +1757,11 @@ impl FtsMemtable {self.field_lengths = HashMap::new();self.avg_field_lengths = HashMap::new();self.doc_id_index = HashMap::new();+ // Flush is the equivalent of a Lucene merge: tombstone contributions+ // are purged, so delete-aware ghost statistics reset to empty.+ self.ghost_docs = 0;+ self.ghost_field_len = HashMap::new();+ self.ghost_doc_freq = HashMap::new();self.docs.shrink_to_fit();result}