Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix: exact TSDB dedup + delete-aware BM25 — conformance 99.9% (1325/1326), search 100%

xerj-orgb5834c06dc02
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 4f0c83ff9..d6de52b76 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -9288,13 +9288,15 @@ async fn process_bulk_body(
// bulk pipeline sees a doc with _ignored already populated and any
// malformed values stripped, matching the path taken by the single-
// doc PUT _doc/{id} / POST _doc routes.
- // NOTE: an attempted TSDB last-wins `_id` rewrite (rewrite_bulk_time_series_ids,
- // retained below but UNUSED) over-merged documents with DISTINCT `_tsid`s —
- // it regressed smoke/20_tsdb_consistency (8 unique-tsid docs collapsed to 4).
- // Disabled until the dimension/_tsid derivation is exact; the normal path
- // leaves bulk bodies byte-for-byte unchanged.
- let _ = rewrite_bulk_time_series_ids;
- let base_text: &str = text;
+ // TSDB last-wins `_id` rewrite: for `time_series`-mode indices, inject a
+ // deterministic `_id` derived from the COMPLETE `_tsid` (all dimension
+ // fields — mapping-declared `time_series_dimension:true` AND every
+ // `routing_path` entry, nested paths resolved) plus the normalized
+ // `@timestamp`. Two docs collapse (later overwrites earlier) ONLY when all
+ // dimensions and the instant are identical, matching ES. Returns `None`
+ // when no action targets a TSDB index, so normal bulk bodies are unchanged.
+ let ts_rewritten = rewrite_bulk_time_series_ids(state, default_index, text);
+ let base_text: &str = ts_rewritten.as_deref().unwrap_or(text);
let rewritten_body;
let text_ref: &str = if state.engine.index_mappings.is_empty() {
@@ -19098,7 +19100,39 @@ fn time_series_dimension_fields(state: &AppState, index: &str) -> Option<Vec<Str
if mode != Some("time_series") {
return None;
}
- let mut dims: Vec<String> = Vec::new();
+ // The full `_tsid` is the COMPLETE set of dimension fields: every mapping
+ // field (recursively, by dotted path) flagged `time_series_dimension:true`
+ // UNIONed with every field named in `index.routing_path`. A BTreeSet keeps
+ // them sorted and de-duplicated. Two docs collapse only when ALL of these
+ // (plus the timestamp) are identical, so missing a single nested dimension
+ // (e.g. `k8s.pod.uid`) would wrongly over-merge distinct series.
+ let mut dims: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
+
+ // 1. Mapping-declared dimensions, recursing into nested `properties`
+ // so nested fields contribute their full dotted path.
+ fn collect_dims(
+ props: &serde_json::Map<String, Value>,
+ prefix: &str,
+ out: &mut std::collections::BTreeSet<String>,
+ ) {
+ for (name, fm) in props {
+ let full = if prefix.is_empty() {
+ name.clone()
+ } else {
+ format!("{prefix}.{name}")
+ };
+ if fm
+ .get("time_series_dimension")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
+ {
+ out.insert(full.clone());
+ }
+ if let Some(sub) = fm.get("properties").and_then(Value::as_object) {
+ collect_dims(sub, &full, out);
+ }
+ }
+ }
if let Some(mapping) = state.engine.index_mappings.get(index) {
let props = mapping
.get("mappings")
@@ -19106,41 +19140,37 @@ fn time_series_dimension_fields(state: &AppState, index: &str) -> Option<Vec<Str
.or_else(|| mapping.get("properties"))
.and_then(Value::as_object);
if let Some(props_obj) = props {
- for (name, fm) in props_obj {
- if fm
- .get("time_series_dimension")
- .and_then(Value::as_bool)
- .unwrap_or(false)
- {
- dims.push(name.clone());
- }
- }
+ collect_dims(props_obj, "", &mut dims);
}
}
- if dims.is_empty() {
- // Fall back to the routing_path (the dimension subset used for
- // shard routing) when the mapping doesn't flag dimensions.
- let rp = settings
- .get("index")
- .and_then(|ix| ix.get("routing_path"))
- .or_else(|| settings.get("routing_path"));
- match rp {
- Some(Value::Array(a)) => {
- for v in a {
- if let Some(s) = v.as_str() {
- dims.push(s.to_string());
+
+ // 2. Union in every field named in `index.routing_path` (the dimension
+ // subset used for shard routing). Wildcard patterns are skipped — they
+ // aren't concrete field names to read from the document.
+ let rp = settings
+ .get("index")
+ .and_then(|ix| ix.get("routing_path"))
+ .or_else(|| settings.get("routing_path"));
+ match rp {
+ Some(Value::Array(a)) => {
+ for v in a {
+ if let Some(s) = v.as_str() {
+ if !s.contains('*') {
+ dims.insert(s.to_string());
}
}
}
- Some(Value::String(s)) => dims.push(s.clone()),
- _ => {}
}
+ Some(Value::String(s)) if !s.contains('*') => {
+ dims.insert(s.clone());
+ }
+ _ => {}
}
+
if dims.is_empty() {
return None;
}
- dims.sort();
- Some(dims)
+ Some(dims.into_iter().collect())
}
/// Compute a deterministic `_id` for a time_series (TSDB) document from its
@@ -19159,10 +19189,13 @@ fn time_series_doc_id(doc: &Value, dim_fields: &[String]) -> Option<String> {
for f in dim_fields {
composite.push_str(f);
composite.push('\u{1}');
- match obj.get(f) {
- Some(Value::String(s)) => composite.push_str(s),
+ // Dimension fields may be nested (e.g. `k8s.pod.uid`); resolve the
+ // full dotted path against the source document, not just top-level
+ // keys, so every dimension actually contributes to the `_tsid`.
+ match get_source_value_by_path(doc, f) {
+ Some(Value::String(s)) => composite.push_str(&s),
+ Some(Value::Null) | None => {}
Some(other) => composite.push_str(&other.to_string()),
- None => {}
}
composite.push('\u{2}');
}
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index b52342103..4cdfec329 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -580,7 +580,13 @@ impl ShardedFtsMemtable {
let q_tokens = analyzer.analyze(query);
if q_tokens.is_empty() { return Vec::new(); }
- let global_doc_count: u64 = self.doc_count() as u64;
+ // Delete-aware BM25 collection statistics (Lucene/ES parity): the
+ // scoring N counts both live docs AND tombstoned/superseded versions
+ // that have not yet been merged away. NOTE: this is *only* the N fed
+ // to the BM25 IDF — hits.total and pagination still use the live
+ // `doc_count()`, so a search over an index that has never had an
+ // update/delete scores bit-for-bit identically to before.
+ let mut global_doc_count: u64 = self.doc_count() as u64;
// Aggregate (per-field global avg_field_len, per-(field,term) doc_freq).
let mut field_total_len: std::collections::HashMap<String, (f64, u64)> =
std::collections::HashMap::new();
@@ -588,13 +594,21 @@ impl ShardedFtsMemtable {
std::collections::HashMap::new();
for shard in &self.shards {
let g = shard.read();
- // Field length sums.
+ // Live N is already in `global_doc_count`; add tombstoned versions.
+ global_doc_count += g.ghost_docs;
+ // Field length sums (live).
for (fname, (sum, n)) in &g.avg_field_lengths {
let entry = field_total_len.entry(fname.clone()).or_insert((0.0, 0));
entry.0 += sum;
entry.1 += n;
}
- // Per-term doc_freq across shards.
+ // Field length sums (tombstoned versions retained for avgdl).
+ for (fname, (sum, n)) in &g.ghost_field_len {
+ let entry = field_total_len.entry(fname.clone()).or_insert((0.0, 0));
+ entry.0 += sum;
+ entry.1 += n;
+ }
+ // Per-term doc_freq across shards (live postings).
for (fname, postings) in &g.index {
if !fields.is_empty() && !fields.iter().any(|f| f == fname) {
continue;
@@ -606,6 +620,18 @@ impl ShardedFtsMemtable {
}
}
}
+ // Per-term doc_freq from tombstoned versions (delete-aware df).
+ for (fname, terms) in &g.ghost_doc_freq {
+ if !fields.is_empty() && !fields.iter().any(|f| f == fname) {
+ continue;
+ }
+ for token in &q_tokens {
+ if let Some(df) = terms.get(&token.text) {
+ *term_global_df.entry((fname.clone(), token.text.clone()))
+ .or_insert(0) += *df;
+ }
+ }
+ }
}
let global_avg_field_len: std::collections::HashMap<String, f32> =
field_total_len.into_iter()
@@ -857,6 +883,10 @@ impl ShardedFtsMemtable {
g.field_lengths = HashMap::new();
g.avg_field_lengths = HashMap::new();
g.doc_id_index = HashMap::new();
+ // Flush == merge: purge delete-aware ghost collection stats.
+ g.ghost_docs = 0;
+ g.ghost_field_len = HashMap::new();
+ g.ghost_doc_freq = HashMap::new();
g.docs.shrink_to_fit();
d
};
@@ -937,6 +967,11 @@ impl FtsMemtable {
self.field_lengths = HashMap::new();
self.avg_field_lengths = HashMap::new();
self.doc_id_index = HashMap::new();
+ // Flush is the equivalent of a Lucene merge: tombstone contributions
+ // are purged, so delete-aware ghost statistics reset to empty.
+ self.ghost_docs = 0;
+ self.ghost_field_len = HashMap::new();
+ self.ghost_doc_freq = HashMap::new();
self.docs.shrink_to_fit();
result
}
@@ -958,6 +993,11 @@ impl FtsMemtable {
self.field_lengths = HashMap::new();
self.avg_field_lengths = HashMap::new();
self.doc_id_index = HashMap::new();
+ // Flush is the equivalent of a Lucene merge: tombstone contributions
+ // are purged, so delete-aware ghost statistics reset to empty.
+ self.ghost_docs = 0;
+ self.ghost_field_len = HashMap::new();
+ self.ghost_doc_freq = HashMap::new();
self.docs.shrink_to_fit();
result
}
@@ -990,6 +1030,23 @@ pub struct FtsMemtable {
avg_field_lengths: HashMap<String, (f64, u64)>,
/// doc_id → position in self.docs for O(1) lookup
doc_id_index: HashMap<String, usize>,
+ /// Delete-aware BM25 collection statistics (Lucene/ES parity).
+ ///
+ /// When a document is superseded by an update (remove + re-insert) or
+ /// explicitly deleted, Lucene keeps that document's contribution to the
+ /// collection statistics (N, total field length, per-term doc_freq)
+ /// until the segment is actually merged. We mirror that: `remove()`
+ /// strips the old version from the LIVE structures (so it no longer
+ /// matches or counts toward hits.total) but folds its contribution into
+ /// these "ghost" accumulators, which are added back ONLY for BM25
+ /// scoring. They reset to empty on every drain/flush — the moral
+ /// equivalent of a Lucene merge purging tombstones.
+ ///
+ /// A document that was never removed contributes nothing here, so its
+ /// BM25 score is bit-for-bit identical to before this feature existed.
+ ghost_docs: u64,
+ ghost_field_len: HashMap<String, (f64, u64)>,
+ ghost_doc_freq: HashMap<String, HashMap<String, u64>>,
}
impl FtsMemtable {
@@ -1004,6 +1061,9 @@ impl FtsMemtable {
field_lengths: HashMap::new(),
avg_field_lengths: HashMap::new(),
doc_id_index: HashMap::new(),
+ ghost_docs: 0,
+ ghost_field_len: HashMap::new(),
+ ghost_doc_freq: HashMap::new(),
}
}
@@ -1022,6 +1082,9 @@ impl FtsMemtable {
field_lengths: HashMap::new(),
avg_field_lengths: HashMap::new(),
doc_id_index: HashMap::new(),
+ ghost_docs: 0,
+ ghost_field_len: HashMap::new(),
+ ghost_doc_freq: HashMap::new(),
}
}
@@ -1378,21 +1441,47 @@ impl FtsMemtable {
if !self.doc_id_index.contains_key(doc_id) {
return;
}
- // Remove from inverted index.
- for field_index in self.index.values_mut() {
- for posting_list in field_index.values_mut() {
- posting_list.remove(doc_id);
+ // Remove from inverted index. Capture every (field, term) this doc
+ // actually contributed to so we can preserve its delete-aware
+ // doc_freq contribution (Lucene keeps tombstoned postings counted
+ // until a merge). Two-phase to avoid borrowing `self.index` and
+ // `self.ghost_doc_freq` mutably at once.
+ let mut ghosted_terms: Vec<(String, String)> = Vec::new();
+ for (field_name, field_index) in self.index.iter_mut() {
+ for (term, posting_list) in field_index.iter_mut() {
+ if posting_list.remove(doc_id).is_some() {
+ ghosted_terms.push((field_name.clone(), term.clone()));
+ }
}
}
+ for (field_name, term) in ghosted_terms {
+ *self
+ .ghost_doc_freq
+ .entry(field_name)
+ .or_default()
+ .entry(term)
+ .or_insert(0) += 1;
+ }
// Remove from field length caches and update running averages.
+ // Retain the removed doc's field-length contribution as a ghost so
+ // avgdl stays delete-aware (total_field_length / N counts tombstones).
+ let mut ghosted_lengths: Vec<(String, u32)> = Vec::new();
for (field_name, lengths) in &mut self.field_lengths {
if let Some(token_count) = lengths.remove(doc_id) {
if let Some(entry) = self.avg_field_lengths.get_mut(field_name) {
entry.0 -= token_count as f64;
entry.1 = entry.1.saturating_sub(1);
}
+ ghosted_lengths.push((field_name.clone(), token_count));
}
}
+ for (field_name, token_count) in ghosted_lengths {
+ let g = self.ghost_field_len.entry(field_name).or_insert((0.0, 0));
+ g.0 += token_count as f64;
+ g.1 += 1;
+ }
+ // One more document is now a tombstone for collection-stats purposes.
+ self.ghost_docs += 1;
// Remove from docs list AND the parallel DocValues columns.
if let Some(pos) = self.doc_id_index.remove(doc_id) {
let entry = self.docs.remove(pos);
@@ -1619,6 +1708,11 @@ impl FtsMemtable {
self.field_lengths = HashMap::new();
self.avg_field_lengths = HashMap::new();
self.doc_id_index = HashMap::new();
+ // Flush is the equivalent of a Lucene merge: tombstone contributions
+ // are purged, so delete-aware ghost statistics reset to empty.
+ self.ghost_docs = 0;
+ self.ghost_field_len = HashMap::new();
+ self.ghost_doc_freq = HashMap::new();
// `self.docs` was drained in place; its Vec backing is kept (cheap).
self.docs.shrink_to_fit();
result
@@ -1663,6 +1757,11 @@ impl FtsMemtable {
self.field_lengths = HashMap::new();
self.avg_field_lengths = HashMap::new();
self.doc_id_index = HashMap::new();
+ // Flush is the equivalent of a Lucene merge: tombstone contributions
+ // are purged, so delete-aware ghost statistics reset to empty.
+ self.ghost_docs = 0;
+ self.ghost_field_len = HashMap::new();
+ self.ghost_doc_freq = HashMap::new();
self.docs.shrink_to_fit();
result
}