Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(memtable): bound term/terms/range id materialisation + cache whitespace flag

xerj-org908c3431995f
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 6796ee033..51880cfd0 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -4914,10 +4914,20 @@ impl Index {
})() {
let uncollected = total.saturating_sub(hits.len() as u64);
MemSnapshot::DocValuesHits(hits, uncollected)
- } else if let Some(hits) = try_doc_values_query(query, &mem) {
+ } else if let Some((hits, total)) = try_doc_values_query(
+ query,
+ &mem,
+ // Field-sorted requests must offer EVERY matching doc to
+ // the top-N heap (sort correctness); unsorted requests
+ // only materialise the id for the first
+ // `materialisation_limit` matches and carry the rest as
+ // an uncollected count (mirrors the fused-bool path).
+ if sort_topk.is_some() { usize::MAX } else { materialisation_limit },
+ ) {
// DocValues fast path for queries that actually need
// the hit list (size > 0 or sort-on-field).
- MemSnapshot::DocValuesHits(hits, 0)
+ let uncollected = total.saturating_sub(hits.len() as u64);
+ MemSnapshot::DocValuesHits(hits, uncollected)
} else {
// Composite queries (e.g. bool) that the memtable DV
// path can't resolve: share the buffered sources via
@@ -4955,8 +4965,13 @@ impl Index {
} else {
MemSnapshot::Empty
}
- } else if let Some(hits) = try_doc_values_query(query, &mem) {
- MemSnapshot::DocValuesHits(hits, 0)
+ } else if let Some((hits, total)) = try_doc_values_query(
+ query,
+ &mem,
+ if sort_topk.is_some() { usize::MAX } else { materialisation_limit },
+ ) {
+ let uncollected = total.saturating_sub(hits.len() as u64);
+ MemSnapshot::DocValuesHits(hits, uncollected)
} else {
MemSnapshot::Empty
}
@@ -11115,7 +11130,11 @@ fn mem_bool_preds(q: &QueryNode) -> Option<Vec<crate::memtable::MemBoolPred>> {
Some(preds)
}
-fn try_doc_values_query(q: &QueryNode, mem: &crate::memtable::ShardedFtsMemtable) -> Option<Vec<(String, usize)>> {
+fn try_doc_values_query(
+ q: &QueryNode,
+ mem: &crate::memtable::ShardedFtsMemtable,
+ limit: usize,
+) -> Option<(Vec<(String, usize)>, u64)> {
match q {
QueryNode::Term { field, value, .. } => {
// Skip if the value is complex or the field uses CIDR notation.
@@ -11128,7 +11147,7 @@ fn try_doc_values_query(q: &QueryNode, mem: &crate::memtable::ShardedFtsMemtable
Value::Bool(b) => b.to_string(),
_ => return None,
};
- mem.doc_values_term_query(field, &val_str)
+ mem.doc_values_term_query(field, &val_str, limit)
}
QueryNode::Terms { field, values, .. } => {
@@ -11145,7 +11164,7 @@ fn try_doc_values_query(q: &QueryNode, mem: &crate::memtable::ShardedFtsMemtable
if strs.len() != values.len() {
return None; // complex values — fall back
}
- mem.doc_values_terms_query(field, &strs)
+ mem.doc_values_terms_query(field, &strs, limit)
}
QueryNode::Range { field, gte, gt, lte, lt, .. } => {
@@ -11172,12 +11191,12 @@ fn try_doc_values_query(q: &QueryNode, mem: &crate::memtable::ShardedFtsMemtable
if lte_f.is_none() && lte.is_some() { return None; }
if lt_f.is_none() && lt.is_some() { return None; }
- mem.doc_values_range_query(field, gte_f, gt_f, lte_f, lt_f)
+ mem.doc_values_range_query(field, gte_f, gt_f, lte_f, lt_f, limit)
}
// Wrapper queries: delegate to inner query.
QueryNode::Constant { query, .. }
- | QueryNode::Boosted { query, .. } => try_doc_values_query(query, mem),
+ | QueryNode::Boosted { query, .. } => try_doc_values_query(query, mem, limit),
_ => None,
}
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 50069c4f1..fa48a4467 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -180,6 +180,17 @@ pub struct DocValues {
/// removed on delete — conservative (a stale flag only costs the
/// fast path, not correctness).
pub array_fields: FxHashSet<String>,
+ /// Fields whose keyword column has EVER contained a whitespace-bearing
+ /// value. Such a column stores an analyzed-text field's full source
+ /// string verbatim; the `term` / `terms` / fused-`bool` doc-values fast
+ /// paths compare against a single token, which can never equal a
+ /// multi-token value, so they MUST bail to the scan path for these
+ /// fields. Computed ONCE at insert time (see `push_field`) instead of
+ /// the old O(N) `col.iter().any(is_whitespace)` prescan every query ran.
+ /// Never cleared on delete — conservative (a stale flag only sends the
+ /// query down the always-correct scan path, it can't change results),
+ /// mirroring `array_fields`.
+ pub keyword_has_whitespace: FxHashSet<String>,
/// Incremental per-(field, order) sorted-candidate cache for the
/// field-sorted match_all memtable arm — see
/// `FtsMemtable::sort_candidates_cached`. Keyed
@@ -305,6 +316,12 @@ impl DocValues {
kcol.push(Some(n.to_string()));
}
Value::String(s) => {
+ // Step 2: cache the analyzed-text eligibility flag at insert
+ // time so the term/terms/bool fast paths skip the per-query
+ // whole-column whitespace prescan.
+ if s.contains(char::is_whitespace) && !self.keyword_has_whitespace.contains(field) {
+ self.keyword_has_whitespace.insert(field.to_string());
+ }
let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
pad_to(kcol, doc_index);
kcol.push(Some(s.clone()));
@@ -340,6 +357,9 @@ impl DocValues {
ncol.push(first_num);
if let Some(ref s) = first_str {
+ if s.contains(char::is_whitespace) && !self.keyword_has_whitespace.contains(field) {
+ self.keyword_has_whitespace.insert(field.to_string());
+ }
entry_no_clone(&mut self.keyword_set, field, Default::default).insert(s.clone());
}
let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
@@ -1103,18 +1123,29 @@ impl ShardedFtsMemtable {
/// Returns `Some(Vec<(doc_id, local_idx)>)` if any shard matched.
/// The `local_idx` is shard-local; callers use the doc_id to
/// resolve the source via `get_doc_source_*`.
- pub fn doc_values_term_query(&self, field: &str, value: &str) -> Option<Vec<(String, usize)>> {
+ pub fn doc_values_term_query(
+ &self,
+ field: &str,
+ value: &str,
+ limit: usize,
+ ) -> Option<(Vec<(String, usize)>, u64)> {
let mut out: Vec<(String, usize)> = Vec::new();
+ let mut total: u64 = 0;
let mut any_hit = false;
for s in &self.shards {
let g = s.read();
- if let Some(mut hits) = g.doc_values_term_query(field, value) {
+ // Step 1: bound the id clone to the remaining global window;
+ // the total is still exact per shard. Mirrors
+ // `doc_values_bool_query`.
+ let room = limit.saturating_sub(out.len());
+ if let Some((mut hits, t)) = g.doc_values_term_query(field, value, room) {
any_hit = true;
out.append(&mut hits);
+ total += t;
}
}
if any_hit {
- Some(out)
+ Some((out, total))
} else {
None
}
@@ -1124,18 +1155,22 @@ impl ShardedFtsMemtable {
&self,
field: &str,
values: &[String],
- ) -> Option<Vec<(String, usize)>> {
+ limit: usize,
+ ) -> Option<(Vec<(String, usize)>, u64)> {
let mut out: Vec<(String, usize)> = Vec::new();
+ let mut total: u64 = 0;
let mut any_hit = false;
for s in &self.shards {
let g = s.read();
- if let Some(mut hits) = g.doc_values_terms_query(field, values) {
+ let room = limit.saturating_sub(out.len());
+ if let Some((mut hits, t)) = g.doc_values_terms_query(field, values, room) {
any_hit = true;
out.append(&mut hits);
+ total += t;
}
}
if any_hit {
- Some(out)
+ Some((out, total))
} else {
None
}
@@ -1160,18 +1195,22 @@ impl ShardedFtsMemtable {
gt: Option<f64>,
lte: Option<f64>,
lt: Option<f64>,
- ) -> Option<Vec<(String, usize)>> {
+ limit: usize,
+ ) -> Option<(Vec<(String, usize)>, u64)> {
let mut out: Vec<(String, usize)> = Vec::new();
+ let mut total: u64 = 0;
let mut any_hit = false;
for s in &self.shards {
let g = s.read();
- if let Some(mut hits) = g.doc_values_range_query(field, gte, gt, lte, lt) {
+ let room = limit.saturating_sub(out.len());
+ if let Some((mut hits, t)) = g.doc_values_range_query(field, gte, gt, lte, lt, room) {
any_hit = true;
out.append(&mut hits);
+ total += t;
}
}
if any_hit {
- Some(out)
+ Some((out, total))
} else {
None
}
@@ -2387,10 +2426,9 @@ impl FtsMemtable {
match p {
MemBoolPred::Term { field, value } => {
let col = self.doc_values.keyword.get(field.as_str())?;
- // Same analyzed-text bailout as doc_values_term_query.
- if col.iter().any(|v| {
- v.as_deref().map(|s| s.contains(char::is_whitespace)).unwrap_or(false)
- }) {
+ // Step 2: analyzed-text bailout via the insert-time
+ // cached flag instead of an O(N) per-query column prescan.
+ if self.doc_values.keyword_has_whitespace.contains(field.as_str()) {
return None;
}
cols.push(Col::Kw(col, value.as_str()));
@@ -2447,27 +2485,36 @@ impl FtsMemtable {
/// column for `field` equals `value` (case-sensitive exact match).
///
/// Returns `None` when the field has no keyword column (fall back to JSON scan).
- pub fn doc_values_term_query(&self, field: &str, value: &str) -> Option<Vec<(String, usize)>> {
+ pub fn doc_values_term_query(
+ &self,
+ field: &str,
+ value: &str,
+ limit: usize,
+ ) -> Option<(Vec<(String, usize)>, u64)> {
let col = self.doc_values.keyword.get(field)?;
- // Same text-field bailout as doc_values_terms_query: a whitespace-
- // containing keyword value means the column stores the full
- // analyzed-text source; a `term` query expects a token match which
- // only the FTS / scan path can serve.
- if col.iter().any(|v| v.as_deref().map(|s| s.contains(char::is_whitespace)).unwrap_or(false)) {
+ // Step 2: text-field bailout via the insert-time cached flag — a
+ // whitespace-containing keyword value means the column stores the
+ // full analyzed-text source; a `term` query expects a token match
+ // which only the FTS / scan path can serve. Was an O(N) per-query
+ // column prescan.
+ if self.doc_values.keyword_has_whitespace.contains(field) {
return None;
}
- let results = col
- .iter()
- .enumerate()
- .filter_map(|(idx, opt)| {
- if opt.as_deref() == Some(value) {
- Some((self.docs[idx].doc_id.clone(), idx))
- } else {
- None
+ // Step 1: walk the whole column for an exact total but only clone
+ // the doc_id String for the first `limit` matches — mirrors
+ // `doc_values_bool_hits`. The unbounded clone was ~all-matching-doc
+ // String allocations per query at a drain-lagged memtable.
+ let mut out: Vec<(String, usize)> = Vec::new();
+ let mut total: u64 = 0;
+ for (idx, opt) in col.iter().enumerate() {
+ if opt.as_deref() == Some(value) {
+ total += 1;
+ if out.len() < limit {
+ out.push((self.docs[idx].doc_id.clone(), idx));
}
- })
- .collect();
- Some(results)
+ }
+ }
+ Some((out, total))
}
/// Index-only variant of `doc_values_term_query` that returns only
@@ -2531,29 +2578,32 @@ impl FtsMemtable {
&self,
field: &str,
values: &[String],
- ) -> Option<Vec<(String, usize)>> {
+ limit: usize,
+ ) -> Option<(Vec<(String, usize)>, u64)> {
let col = self.doc_values.keyword.get(field)?;
+ // Step 2: analyzed-text bailout via the insert-time cached flag.
// If any stored keyword value in this column contains whitespace
// it's likely an analyzed text field whose doc-values were built
// from the full source string (not the token stream). A `terms`
// query compares against tokens in that case, which doc-values
// can't serve — bail so callers fall through to the scan path.
- if col.iter().any(|v| v.as_deref().map(|s| s.contains(char::is_whitespace)).unwrap_or(false)) {
+ if self.doc_values.keyword_has_whitespace.contains(field) {
return None;
}
- let results: Vec<(String, usize)> = col
- .iter()
- .enumerate()
- .filter_map(|(idx, opt)| {
- if let Some(v) = opt.as_deref() {
- if values.iter().any(|qv| qv == v) {
- return Some((self.docs[idx].doc_id.clone(), idx));
+ // Step 1: exact total, bounded doc_id materialisation.
+ let mut out: Vec<(String, usize)> = Vec::new();
+ let mut total: u64 = 0;
+ for (idx, opt) in col.iter().enumerate() {
+ if let Some(v) = opt.as_deref() {
+ if values.iter().any(|qv| qv == v) {
+ total += 1;
+ if out.len() < limit {
+ out.push((self.docs[idx].doc_id.clone(), idx));
}
}
- None
- })
- .collect();
- Some(results)
+ }
+ }
+ Some((out, total))
}
/// Fast range query using the numeric column — O(N * f64_compare).
@@ -2568,31 +2618,37 @@ impl FtsMemtable {
gt: Option<f64>,
lte: Option<f64>,
lt: Option<f64>,
- ) -> Option<Vec<(String, usize)>> {
+ limit: usize,
+ ) -> Option<(Vec<(String, usize)>, u64)> {
let col = self.doc_values.numeric.get(field)?;
- let results = col
- .iter()
- .enumerate()
- .filter_map(|(idx, opt)| {
- let v = (*opt)?;
- let passes_lower = match (gte, gt) {
- (Some(b), _) => v >= b,
- (None, Some(b)) => v > b,
- (None, None) => true,
- };
- let passes_upper = match (lte, lt) {
- (Some(b), _) => v <= b,
- (None, Some(b)) => v < b,
- (None, None) => true,
- };
- if passes_lower && passes_upper {
- Some((self.docs[idx].doc_id.clone(), idx))
- } else {
- None
+ // Step 1: exact total, bounded doc_id materialisation — the
+ // unbounded id clone was the biggest read-under-write term for
+ // broad range filters (~all-of-memtable String allocations/query).
+ let mut out: Vec<(String, usize)> = Vec::new();
+ let mut total: u64 = 0;
+ for (idx, opt) in col.iter().enumerate() {
+ let v = match *opt {
+ Some(v) => v,
+ None => continue,
+ };
+ let passes_lower = match (gte, gt) {
+ (Some(b), _) => v >= b,
+ (None, Some(b)) => v > b,
+ (None, None) => true,
+ };
+ let passes_upper = match (lte, lt) {
+ (Some(b), _) => v <= b,
+ (None, Some(b)) => v < b,
+ (None, None) => true,
+ };
+ if passes_lower && passes_upper {
+ total += 1;
+ if out.len() < limit {
+ out.push((self.docs[idx].doc_id.clone(), idx));
}
- })
- .collect();
- Some(results)
+ }
+ }
+ Some((out, total))
}
/// Return all (term, frequency) pairs for a given field from the inverted index.