Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(dv): a field that is multi-valued anywhere in a segment ships NO doc-values column

xerj-orge5112fbd47eb
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index cde29a562..12e6ccb0c 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1108,9 +1108,8 @@ impl Index {
// (spawned at the end of open — RC4 W2 item 17)
// re-derives the WAL-tail divergence and clears the
// flag. (The loader gate guarantees `field` is Some.)
- let stale = l.stale
- || l.seq_no
- .map_or(true, |stamp| stamp != store.current_seq_no());
+ let stale =
+ l.stale || l.seq_no.is_none_or(|stamp| stamp != store.current_seq_no());
(Some(l.graph), l.id_map, l.id_rev, l.next_id, l.field, stale)
}
None => (None, HashMap::new(), HashMap::new(), 1, None, false),
@@ -3317,16 +3316,13 @@ impl Index {
// across restarts.
let tomb_carry: Vec<(u64, String)> = tomb_union
.into_iter()
- .filter(|(id, seq)| match store_for_task.version_map.get(id) {
- Some(e)
- if !e.deleted
- && e.seq_no > *seq
- && &*e.segment_id
- != xerj_storage::version_map::IN_MEMORY_SEGMENT_ID =>
- {
- false
- }
- _ => true,
+ .filter(|(id, seq)| {
+ !matches!(store_for_task.version_map.get(id),
+ Some(e)
+ if !e.deleted
+ && e.seq_no > *seq
+ && &*e.segment_id
+ != xerj_storage::version_map::IN_MEMORY_SEGMENT_ID)
})
.map(|(id, seq)| (seq, id))
.collect();
@@ -4157,10 +4153,10 @@ impl Index {
seen.insert(doc_id.clone());
if let Some(vector) = extract_numeric_vector(src, &field) {
compared += 1;
- if !self.hnsw_vector_current(doc_id, &vector).await {
- if self.hnsw_insert_vector(doc_id, vector).await {
- reinserted += 1;
- }
+ if !self.hnsw_vector_current(doc_id, &vector).await
+ && self.hnsw_insert_vector(doc_id, vector).await
+ {
+ reinserted += 1;
}
}
walked += 1;
@@ -4224,10 +4220,10 @@ impl Index {
};
if let Some(vector) = extract_numeric_vector(src, &field) {
compared += 1;
- if !self.hnsw_vector_current(id, &vector).await {
- if self.hnsw_insert_vector(id, vector).await {
- reinserted += 1;
- }
+ if !self.hnsw_vector_current(id, &vector).await
+ && self.hnsw_insert_vector(id, vector).await
+ {
+ reinserted += 1;
}
}
walked += 1;
@@ -5513,7 +5509,7 @@ impl Index {
let stale = loaded.stale
|| loaded
.seq_no
- .map_or(true, |stamp| stamp != self.store.current_seq_no());
+ .is_none_or(|stamp| stamp != self.store.current_seq_no());
*self.hnsw_field.write().unwrap() = loaded.field;
self.hnsw_stale
.store(stale, std::sync::atomic::Ordering::Release);
@@ -10378,6 +10374,19 @@ fn build_doc_value_columns<'a>(
// O(docs × fields) memory but much smaller than the segment itself.
let mut numeric: BTreeMap<String, Vec<Option<i64>>> = BTreeMap::new();
let mut keyword: BTreeMap<String, Vec<Option<String>>> = BTreeMap::new();
+ // Fields that carry an ARRAY value in ANY doc of this segment. The
+ // one-slot-per-row column cannot represent a multi-valued doc, and
+ // filing it as null makes the column LIE: every consumer (count
+ // shortcut, prefilters, fast aggs) treats a PRESENT column as exact,
+ // so a range that should match `[5, 105]` silently drops the doc —
+ // live-verified: bool(term + range-on-multi-valued) at size:0 counted
+ // 27 of 37 whenever the flush landed all docs in one segment (the
+ // 2-core CI shape; ≥32-core boxes scatter docs across shards, the
+ // array-only segments then have no column and the shortcut abandons
+ // to the correct brute scan). A poisoned field therefore ships NO
+ // column at all — consumers take the same abandon-to-scan path they
+ // already take for a field that is absent from the segment.
+ let mut multi_valued: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut total_docs: usize = 0;
for src in sources {
@@ -10430,11 +10439,21 @@ fn build_doc_value_columns<'a>(
let v = (if *b { 1.0_f64 } else { 0.0_f64 }).to_bits() as i64;
col.push(Some(v));
}
+ Value::Array(_) if !multi_valued.contains(field.as_str()) => {
+ multi_valued.insert(field.clone());
+ }
_ => {}
}
}
}
+ // Drop every column whose field was multi-valued anywhere in the
+ // segment (see `multi_valued` above) — no column beats a lying one.
+ for f in &multi_valued {
+ numeric.remove(f);
+ keyword.remove(f);
+ }
+
// Pad every column to total_docs so position == doc_id.
for col in numeric.values_mut() {
pad_with_none(col, total_docs);
@@ -11843,6 +11862,7 @@ impl Index {
Some(bm25_keyword_term_score(df, total_docs, boost))
}
+ #[allow(clippy::too_many_arguments)]
fn scored_columnar(
&self,
plan: &ScoredPlan,
@@ -13792,6 +13812,7 @@ impl Index {
///
/// Same output contract as `narrow_matches_to_sort_candidates`: the
/// caller must source `hits.total` independently (shortcut count).
+ #[allow(clippy::too_many_arguments)]
fn narrow_term_values_to_sort_candidates(
&self,
segments_dir: &std::path::Path,
@@ -14833,6 +14854,7 @@ impl Index {
/// 3. Build the FTS inverted index files for the new segment using ordinal
/// positions (0, 1, 2, …) so that the segment search path can look up
/// stored docs by their ordinal.
+///
/// RAII timer that records one `flush_duration` (seconds) observation when it
/// drops — i.e. when `do_flush_shard` returns on any path. It holds an owned
/// `Arc<Metrics>` cloned from the process-global handle so it survives across
@@ -15779,9 +15801,7 @@ fn lower_span_to_orig(
.partition_point(|e| e.0 <= lo_start)
.saturating_sub(1);
let oa = entries[si].1;
- let ei = entries
- .partition_point(|e| e.0 <= lo_end - 1)
- .saturating_sub(1);
+ let ei = entries.partition_point(|e| e.0 < lo_end).saturating_sub(1);
let ob = entries[ei].2.max(oa);
(oa.min(orig_len), ob.min(orig_len))
}
@@ -15845,6 +15865,7 @@ fn highlight_full_text(
out
}
+#[allow(clippy::too_many_arguments)]
fn build_highlight_fragments(
text: &str,
text_lower: &str,