Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(aggs): columnar-filtered memtable fold for size:0 filtered aggregations

xerj-org1d057d017010
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 6a7411391..7ffd2fa5f 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -50,10 +50,106 @@ use crate::aggs::{
parse_date_ms, render_date_format, render_iso_date, resolve_sibling_pipelines,
run_pipeline_agg, run_sampler, run_top_hits_with_total, typed_term_key,
};
+use crate::memtable::MemBoolPred;
use serde_json::{json, Map, Value};
use std::collections::HashMap;
use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn};
+/// Compile the top-level agg filter JSON (the restricted shape
+/// `query_node_to_agg_filter` emits: `term`/`range`/`bool{must,filter}`) into a
+/// pure conjunction of memtable columnar predicates, for the O(matching)
+/// filtered-memtable fold (`ShardedFtsMemtable::filtered_docs_arc`).
+///
+/// Only Term (keyword string) and numeric Range leaves — and a `bool` with just
+/// `must`/`filter` clauses of those — are expressible as a `[MemBoolPred]`
+/// conjunction. `terms` (a disjunction), `match_all`, and anything richer yield
+/// `None`, so the caller keeps the exact full-memtable walk. This is the
+/// columnar-predicate sibling of `compile_top_pred`; the fast-agg memtable fold
+/// always re-applies `doc_matches_filter` on top, so this only needs to be a
+/// no-false-negative candidate generator.
+fn mem_preds_from_agg_filter(filter: &Value) -> Option<Vec<MemBoolPred>> {
+ fn push_leaf(filter: &Value, out: &mut Vec<MemBoolPred>) -> Option<()> {
+ let obj = filter.as_object()?;
+ if obj.len() != 1 {
+ return None;
+ }
+ let (qtype, body) = obj.iter().next()?;
+ match qtype.as_str() {
+ "term" => {
+ let fm = body.as_object()?;
+ if fm.len() != 1 {
+ return None;
+ }
+ let (field, expected) = fm.iter().next()?;
+ let value = match expected {
+ Value::String(s) => {
+ if s.contains('/') {
+ return None; // CIDR — mirror mem_bool_preds
+ }
+ s.clone()
+ }
+ _ => return None,
+ };
+ out.push(MemBoolPred::Term {
+ field: field.clone(),
+ value,
+ });
+ Some(())
+ }
+ "range" => {
+ let fm = body.as_object()?;
+ if fm.len() != 1 {
+ return None;
+ }
+ let (field, bounds) = fm.iter().next()?;
+ let b = bounds.as_object()?;
+ let get = |k: &str| -> Option<Option<f64>> {
+ match b.get(k) {
+ None => Some(None),
+ Some(Value::Number(n)) => Some(n.as_f64()),
+ _ => None, // date-string / non-numeric bound → bail
+ }
+ };
+ // Reject any present-but-non-numeric bound.
+ for k in ["gte", "gt", "lte", "lt"] {
+ if b.contains_key(k) && get(k)?.is_none() {
+ return None;
+ }
+ }
+ out.push(MemBoolPred::Range {
+ field: field.clone(),
+ gte: get("gte")?,
+ gt: get("gt")?,
+ lte: get("lte")?,
+ lt: get("lt")?,
+ });
+ Some(())
+ }
+ "bool" => {
+ let bo = body.as_object()?;
+ if bo.keys().any(|k| k != "must" && k != "filter") {
+ return None;
+ }
+ for key in ["must", "filter"] {
+ if let Some(clauses) = bo.get(key) {
+ for c in clauses.as_array()? {
+ push_leaf(c, out)?;
+ }
+ }
+ }
+ Some(())
+ }
+ _ => None, // terms / match_all / … → not a pure Term/Range conjunction
+ }
+ }
+ let mut out = Vec::new();
+ push_leaf(filter, &mut out)?;
+ if out.is_empty() {
+ return None;
+ }
+ Some(out)
+}
+
/// Minimum index size (live docs) before the columnar path activates.
/// Below this the brute JSON path answers in single-digit milliseconds and
/// remains the reference implementation for conformance corpora.
@@ -163,6 +259,18 @@ pub(super) struct FastCtx<'a> {
/// memtable per-doc match (the trusted sibling of `compile_pred`, exactly
/// as `exec_filter` pairs them). `None` == match_all.
top_filter_query: Option<Value>,
+ /// The top-level filter compiled to memtable columnar predicates (a pure
+ /// Term/Range conjunction), when it is one. Drives the O(matching)
+ /// filtered-memtable fold (`filtered_mem`); `None` when the filter isn't a
+ /// pure conjunction (e.g. `terms`) or there is no filter — the executors
+ /// then walk the full `mem()` gated by `doc_matches_filter`, exactly as
+ /// before.
+ top_filter_mem_preds: Option<Vec<MemBoolPred>>,
+ /// Lazily-materialised columnar-filtered memtable docs (see `filtered_mem`).
+ /// The outer `Option` is the OnceLock cell; the inner `Option` is the
+ /// memtable's own "columnar path applicable?" answer (`None` == a bailout,
+ /// fall back to full `mem()`).
+ mem_filtered: std::sync::OnceLock<Option<Vec<(String, std::sync::Arc<Value>)>>>,
}
impl<'a> FastCtx<'a> {
@@ -206,6 +314,29 @@ impl<'a> FastCtx<'a> {
}
})
}
+
+ /// Columnar-filtered memtable docs for the top-level filter, or `None` when
+ /// the columnar fold doesn't apply — the caller then walks the full `mem()`
+ /// gated by `doc_matches_filter`, exactly as before.
+ ///
+ /// Applies ONLY when the top-level filter compiled to a pure Term/Range
+ /// conjunction AND the agg tree can't observe meta fields (`needs_owned_mem`
+ /// == false — the columnar sources carry no `_id`/`_index`/`_seq_no`, and
+ /// under that condition `plan.top_hits` is always `None`, so the positional
+ /// `DocRef::Mem`/`mem().len()` paths never run) AND every predicate field is
+ /// a scalar doc-valued column in every shard (the memtable returns `None`
+ /// otherwise). The returned set is a no-false-negative superset of the
+ /// `doc_matches_filter` set — callers re-apply the JSON matcher for
+ /// byte-identical results.
+ fn filtered_mem(&self) -> Option<&[(String, std::sync::Arc<Value>)]> {
+ if self.needs_owned_mem {
+ return None;
+ }
+ let preds = self.top_filter_mem_preds.as_ref()?;
+ self.mem_filtered
+ .get_or_init(|| self.idx.memtable.filtered_docs_arc(preds))
+ .as_deref()
+ }
}
#[derive(Clone, Copy, PartialEq, Eq)]
@@ -371,6 +502,11 @@ impl Index {
Some(q) => Some(compile_top_pred(q)?),
};
let top_filter_query: Option<Value> = filter.cloned();
+ // Columnar-predicate form of the filter for the O(matching) memtable
+ // fold. `None` for match_all or a non-conjunction filter (`terms`, …)
+ // — those keep the full-memtable walk.
+ let top_filter_mem_preds: Option<Vec<MemBoolPred>> =
+ filter.and_then(mem_preds_from_agg_filter);
// Size + delete gates.
let seg_physical: u64 = snap.segments.iter().map(|m| m.doc_count).sum();
@@ -441,6 +577,8 @@ impl Index {
bool_fields,
top_filter,
top_filter_query,
+ top_filter_mem_preds,
+ mem_filtered: std::sync::OnceLock::new(),
};
// Filtered `hits.total`: the number of live docs matching the query.
@@ -459,9 +597,20 @@ impl Index {
total += seg_pred_count(&sp, seg.docs);
}
if let Some(q) = &ctx.top_filter_query {
- for doc in ctx.mem().iter() {
- if doc_matches_filter(doc, q) {
- total += 1;
+ // O(matching) columnar fold when the filter columnarises;
+ // else the full-memtable walk. Both re-check
+ // `doc_matches_filter` so the count is exact either way.
+ if let Some(fdocs) = ctx.filtered_mem() {
+ for (_id, src) in fdocs {
+ if doc_matches_filter(src, q) {
+ total += 1;
+ }
+ }
+ } else {
+ for doc in ctx.mem().iter() {
+ if doc_matches_filter(doc, q) {
+ total += 1;
+ }
}
}
}
@@ -600,9 +749,21 @@ impl<'a> FastCtx<'a> {
}
fn mem_field_numeric_safe(&self, field: &str) -> bool {
- self.mem()
- .iter()
- .all(|d| matches!(get_nested_field(d, field), Value::Number(_) | Value::Null))
+ // Under a columnarisable top filter, only the MATCHING docs are folded
+ // (segments + the columnar memtable candidate set, `doc_matches_filter`-
+ // rechecked), so the metric fold never touches a non-matching doc — the
+ // numeric-safety check need only cover the candidate set. That set is a
+ // superset of the true matches, so this stays conservative (it may bail
+ // to brute on a non-matching candidate, never the reverse) while
+ // avoiding the full-memtable hydration. When the columnar path doesn't
+ // apply it walks the full `mem()` exactly as before.
+ let is_num =
+ |d: &Value| matches!(get_nested_field(d, field), Value::Number(_) | Value::Null);
+ if let Some(fdocs) = self.filtered_mem() {
+ fdocs.iter().all(|(_id, src)| is_num(src))
+ } else {
+ self.mem().iter().all(is_num)
+ }
}
fn plan_top_hits(
@@ -826,6 +987,8 @@ impl<'a> FastCtx<'a> {
bool_fields: self.bool_fields,
top_filter: None,
top_filter_query: None,
+ top_filter_mem_preds: None,
+ mem_filtered: std::sync::OnceLock::new(),
};
let mut bucket = Map::new();
// Whole-corpus doc_count (no deletes on the fast path; segment rows are
@@ -930,9 +1093,20 @@ impl<'a> FastCtx<'a> {
}
let mut acc = std::mem::take(&mut accs[0][0]);
if let Some(q) = &self.top_filter_query {
- for d in self.mem().iter() {
- if doc_matches_filter(d, q) {
- Self::fold_mem_metric(d, &spec, &mut acc);
+ // O(matching) columnar fold when the filter columnarises; else
+ // the full-memtable walk. Both gate on `doc_matches_filter`, so
+ // the folded set is byte-identical.
+ if let Some(fdocs) = self.filtered_mem() {
+ for (_id, src) in fdocs {
+ if doc_matches_filter(src, q) {
+ Self::fold_mem_metric(src, &spec, &mut acc);
+ }
+ }
+ } else {
+ for d in self.mem().iter() {
+ if doc_matches_filter(d, q) {
+ Self::fold_mem_metric(d, &spec, &mut acc);
+ }
}
}
}
@@ -1496,34 +1670,62 @@ impl<'a> FastCtx<'a> {
}
}
None => {
- // Brute extractor semantics (multi-value, weights).
- for (di, doc) in self.mem().iter().enumerate() {
- // Top-level query filter: skip non-matching memtable docs
- // (segment rows are already gated inside `fused_seg_pass`).
- if let Some(q) = &self.top_filter_query {
+ // O(matching) columnar fold: when the top filter columnarises
+ // AND no meta-observing sub-agg is present (`filtered_mem` is
+ // Some ⇒ `needs_owned_mem` false ⇒ `plan.top_hits` is None, so
+ // the positional `DocRef::Mem` path is unreachable here), fold
+ // only the columnar-matched candidates — re-checked against the
+ // JSON matcher for byte-identical results. Otherwise the full
+ // `mem()` walk (brute extractor semantics; multi-value, weights,
+ // top_hits) exactly as before.
+ let columnar = match (self.filtered_mem(), self.top_filter_query.as_ref()) {
+ (Some(fdocs), Some(q)) if plan.top_hits.is_none() => Some((fdocs, q)),
+ _ => None,
+ };
+ if let Some((fdocs, q)) = columnar {
+ for (_id, doc) in fdocs {
if !doc_matches_filter(doc, q) {
continue;
}
+ let weight = doc_count_weight(doc);
+ let vals = extract_field_values(doc, field);
+ for term in vals {
+ let st = terms_map.entry(term).or_insert_with(|| new_state(&plan));
+ st.count += weight;
+ for (mi, spec) in plan.metrics.iter().enumerate() {
+ Self::fold_mem_metric(doc, spec, &mut st.accs[mi]);
+ }
+ }
}
- let weight = doc_count_weight(doc);
- let vals = extract_field_values(doc, field);
- for term in vals {
- let st = terms_map.entry(term).or_insert_with(|| new_state(&plan));
- st.count += weight;
- for (mi, spec) in plan.metrics.iter().enumerate() {
- Self::fold_mem_metric(doc, spec, &mut st.accs[mi]);
+ } else {
+ for (di, doc) in self.mem().iter().enumerate() {
+ // Top-level query filter: skip non-matching memtable docs
+ // (segment rows are already gated inside `fused_seg_pass`).
+ if let Some(q) = &self.top_filter_query {
+ if !doc_matches_filter(doc, q) {
+ continue;
+ }
}
- if let Some(spec) = &plan.top_hits {
- match extract_numeric(doc, &spec.sort_field) {
- Some(v) => push_top(
- &mut st.top,
- spec.k,
- spec.desc,
- v,
- di as u64,
- DocRef::Mem(di),
- ),
- None => missing_sort_seen = true,
+ let weight = doc_count_weight(doc);
+ let vals = extract_field_values(doc, field);
+ for term in vals {
+ let st = terms_map.entry(term).or_insert_with(|| new_state(&plan));
+ st.count += weight;
+ for (mi, spec) in plan.metrics.iter().enumerate() {
+ Self::fold_mem_metric(doc, spec, &mut st.accs[mi]);
+ }
+ if let Some(spec) = &plan.top_hits {
+ match extract_numeric(doc, &spec.sort_field) {
+ Some(v) => push_top(
+ &mut st.top,
+ spec.k,
+ spec.desc,
+ v,
+ di as u64,
+ DocRef::Mem(di),
+ ),
+ None => missing_sort_seen = true,
+ }
}
}
}
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 1e860a7e9..486821377 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -756,6 +756,33 @@ impl ShardedFtsMemtable {
out
}
+ /// Columnar-filtered twin of `all_docs_with_sources_arc`: materialise ONLY
+ /// the `(doc_id, source_arc)` pairs matching `preds` (a pure conjunction of
+ /// Term/Range predicates), so a filtered `size:0` aggregation folds
+ /// O(matching) docs instead of hydrating the whole memtable under the shard
+ /// read lock. Returns `None` (caller must fall back to
+ /// `all_docs_with_sources_arc`) when any shard can't prove column↔source
+ /// equivalence for a predicate field — see `filtered_docs_arc_into`.
+ ///
+ /// Each shard's enumeration + source materialisation happens under a single
+ /// `s.read()` held only for that shard's walk (the same brief per-shard hold
+ /// `all_docs_with_sources_arc` takes), so this both shrinks the work and the
+ /// lock hold that the mixed read-under-write contention is bound on.
+ pub fn filtered_docs_arc(&self, preds: &[MemBoolPred]) -> Option<Vec<(String, Arc<Value>)>> {
+ if preds.is_empty() {
+ return None;
+ }
+ let mut out: Vec<(String, Arc<Value>)> = Vec::new();
+ for s in &self.shards {
+ let g = s.read();
+ if g.doc_count() == 0 {
+ continue;
+ }
+ g.filtered_docs_arc_into(preds, &mut out)?;
+ }
+ Some(out)
+ }
+
/// Drain every shard, merge-sort by seq_no, and return the
/// combined (doc_id, text_fields, source) stream. This is the
/// single entry point used by the flush path — the sort
@@ -2685,6 +2712,118 @@ impl FtsMemtable {
Some((out, total))
}
+ /// Push `(doc_id, source_arc)` for every buffered position whose
+ /// doc-values columns satisfy EVERY predicate (a pure conjunction of
+ /// Term/Range). Column resolution + per-predicate match semantics are the
+ /// exact ones `doc_values_bool_hits` uses, so this enumerates the same doc
+ /// set the fused-bool search walk would — but UNBOUNDED (all matches, the
+ /// aggregation contract) and materialising the matching sources instead of
+ /// just ids.
+ ///
+ /// Returns `None` (caller must fall back to the full-corpus walk) whenever
+ /// column↔source equivalence with the JSON matcher isn't provable for a
+ /// predicate field:
+ /// * the field has EVER carried an ARRAY value in this shard — the column
+ /// keeps only the first element, so a later element that matches under
+ /// `doc_matches_filter` would be silently dropped (a false negative the
+ /// aggregation could NOT recover);
+ /// * a keyword predicate field holds analyzed-text (whitespace) values;
+ /// * a predicate field has no column at all.
+ ///
+ /// For the shapes it DOES accept (scalar numeric/keyword columns) the
+ /// columnar match set equals the `doc_matches_filter` set — the fast-agg
+ /// caller still re-applies the JSON matcher as a safety net, so any residual
+ /// false POSITIVE is removed and only the provable no-false-negative
+ /// guarantee is load-bearing.
+ pub fn filtered_docs_arc_into(
+ &self,
+ preds: &[MemBoolPred],
+ out: &mut Vec<(String, Arc<Value>)>,
+ ) -> Option<()> {
+ enum Col<'a> {
+ Kw(&'a Vec<Option<String>>, &'a str),
+ NumEq(&'a Vec<Option<f64>>, f64),
+ Num(
+ &'a Vec<Option<f64>>,
+ Option<f64>,
+ Option<f64>,
+ Option<f64>,
+ Option<f64>,
+ ),
+ }
+ let mut cols: Vec<Col<'_>> = Vec::with_capacity(preds.len());
+ for p in preds {
+ match p {
+ MemBoolPred::Term { field, value } => {
+ // Array-valued field: the column is lossy (first element
+ // only) → bail so a later matching element can't be dropped.
+ if self.doc_values.array_fields.contains(field.as_str()) {
+ return None;
+ }
+ if let Some(col) = self.doc_values.keyword.get(field.as_str()) {
+ if self
+ .doc_values
+ .keyword_has_whitespace
+ .contains(field.as_str())
+ {
+ return None;
+ }
+ cols.push(Col::Kw(col, value.as_str()));
+ } else if let Some(col) = self.doc_values.numeric.get(field.as_str()) {
+ let needle = value.parse::<f64>().ok()?;
+ cols.push(Col::NumEq(col, needle));
+ } else {
+ return None;
+ }
+ }
+ MemBoolPred::Range {
+ field,
+ gte,
+ gt,
+ lte,
+ lt,
+ } => {
+ if self.doc_values.array_fields.contains(field.as_str()) {
+ return None;
+ }
+ let col = self.doc_values.numeric.get(field.as_str())?;
+ cols.push(Col::Num(col, *gte, *gt, *lte, *lt));
+ }
+ }
+ }
+ let n = self.docs.len();
+ 'doc: for idx in 0..n {
+ for c in &cols {
+ let ok = match c {
+ Col::Kw(col, want) => col.get(idx).and_then(|o| o.as_deref()) == Some(*want),
+ Col::NumEq(col, want) => col.get(idx).copied().flatten() == Some(*want),
+ Col::Num(col, gte, gt, lte, lt) => match col.get(idx).copied().flatten() {
+ None => false,
+ Some(v) => {
+ let pl = match (gte, gt) {
+ (Some(b), _) => v >= *b,
+ (None, Some(b)) => v > *b,
+ (None, None) => true,
+ };
+ let pu = match (lte, lt) {
+ (Some(b), _) => v <= *b,
+ (None, Some(b)) => v < *b,
+ (None, None) => true,
+ };
+ pl && pu
+ }
+ },
+ };
+ if !ok {
+ continue 'doc;
+ }
+ }
+ let e = &self.docs[idx];
+ out.push((e.doc_id.clone(), Self::resolve_source_arc(e)));
+ }
+ Some(())
+ }
+
/// Fast term query using the keyword column — O(N * string_compare).
///
/// Returns `(doc_id, doc_index)` pairs for documents where the keyword
@@ -3165,3 +3304,154 @@ mod bounded_delta_counts_tests {
assert_maintained_eq_recount(&dv);
}
}
+
+#[cfg(test)]
+mod filtered_docs_arc_tests {
+ use super::*;
+ use crate::aggs::doc_matches_filter;
+ use serde_json::json;
+
+ /// Build a sharded memtable of `n_docs` docs with scalar fields:
+ /// n (long = i), status (keyword, 5-card = i%5), lat (double).
+ fn build_mem(n_docs: usize) -> ShardedFtsMemtable {
+ let mem = ShardedFtsMemtable::new();
+ let schema = Schema::default();
+ for i in 0..n_docs {
+ let doc = json!({
+ "n": i as i64,
+ "status": format!("s{}", i % 5),
+ "lat": (i % 1000) as f64 * 0.01,
+ });
+ mem.insert(format!("d{i}"), &doc, &schema, i as u64);
+ }
+ mem
+ }
+
+ /// Brute reference: every buffered doc whose source matches `filter_json`
+ /// under the exact JSON matcher, as a sorted `doc_id` vec.
+ fn brute_ids(mem: &ShardedFtsMemtable, filter_json: &Value) -> Vec<String> {
+ let mut ids: Vec<String> = mem
+ .all_docs_with_sources_arc()
+ .into_iter()
+ .filter(|(_id, src)| doc_matches_filter(src, filter_json))
+ .map(|(id, _)| id)
+ .collect();
+ ids.sort();
+ ids
+ }
+
+ fn columnar_ids(mem: &ShardedFtsMemtable, preds: &[MemBoolPred]) -> Vec<String> {
+ let mut ids: Vec<String> = mem
+ .filtered_docs_arc(preds)
+ .expect("columnar path should apply for scalar fields")
+ .into_iter()
+ .map(|(id, _)| id)
+ .collect();
+ ids.sort();
+ ids
+ }
+
+ /// The columnar candidate set must EQUAL the `doc_matches_filter` set —
+ /// including on a filter that matches FAR more than the historical
+ /// `materialisation_limit` (256) so a bounded/truncated fold would be
+ /// caught (the all-matches aggregation contract).
+ #[test]
+ fn columnar_range_matches_brute_over_256() {
+ let mem = build_mem(4000);
+ // Range n ∈ [500, 3500) → 3000 matches, an order of magnitude past 256.
+ let preds = vec![MemBoolPred::Range {
+ field: "n".to_string(),
+ gte: Some(500.0),
+ gt: None,
+ lte: None,
+ lt: Some(3500.0),
+ }];
+ let filter = json!({ "range": { "n": { "gte": 500, "lt": 3500 } } });
+ let brute = brute_ids(&mem, &filter);
+ let columnar = columnar_ids(&mem, &preds);
+ assert_eq!(brute.len(), 3000, "sanity: 3000 docs in [500,3500)");
+ assert_eq!(columnar, brute, "columnar range set diverged from brute");
+ // Value fold parity: sum over the columnar candidate set (metric-agg
+ // fold input) equals the brute sum over the matcher set.
+ let sum_of = |ids: &[String]| -> f64 {
+ ids.iter()
+ .map(|id| id.trim_start_matches('d').parse::<f64>().unwrap())
+ .sum()
+ };
+ assert_eq!(sum_of(&columnar), sum_of(&brute));
+ }
+
+ /// Term + Range conjunction (a `bool{filter}`), also > 256 matches.
+ #[test]
+ fn columnar_bool_conjunction_matches_brute() {
+ let mem = build_mem(4000);
+ // status == "s2" (800 docs) AND n ∈ [0,4000) → the 800 status-s2 docs.
+ let preds = vec![
+ MemBoolPred::Term {
+ field: "status".to_string(),
+ value: "s2".to_string(),
+ },
+ MemBoolPred::Range {
+ field: "n".to_string(),
+ gte: Some(0.0),
+ gt: None,
+ lte: None,
+ lt: Some(4000.0),
+ },
+ ];
+ let filter = json!({
+ "bool": { "filter": [
+ { "term": { "status": "s2" } },
+ { "range": { "n": { "gte": 0, "lt": 4000 } } },
+ ]}
+ });
+ let brute = brute_ids(&mem, &filter);
+ assert_eq!(brute.len(), 800);
+ assert_eq!(columnar_ids(&mem, &preds), brute);
+ }
+
+ /// Bare keyword term.
+ #[test]
+ fn columnar_term_matches_brute() {
+ let mem = build_mem(2000);
+ let preds = vec![MemBoolPred::Term {
+ field: "status".to_string(),
+ value: "s3".to_string(),
+ }];
+ let filter = json!({ "term": { "status": "s3" } });
+ let brute = brute_ids(&mem, &filter);
+ assert_eq!(brute.len(), 400);
+ assert_eq!(columnar_ids(&mem, &preds), brute);
+ }
+
+ /// A predicate field that ever carried an ARRAY value makes the column
+ /// lossy (first element only), so the columnar path MUST bail (`None`) so
+ /// the caller keeps the exact full-corpus walk — otherwise a later matching
+ /// array element would be a silent false negative.
+ #[test]
+ fn columnar_bails_on_array_predicate_field() {
+ let mem = ShardedFtsMemtable::new();
+ let schema = Schema::default();
+ for i in 0..2000usize {
+ // Every 500th doc stores `n` as an array — poisons the shard's
+ // `array_fields` for whichever shard owns it.
+ let n_val = if i % 500 == 0 {
+ json!([i as i64, (i as i64) + 1])
+ } else {
+ json!(i as i64)
+ };
+ mem.insert(format!("d{i}"), &json!({ "n": n_val }), &schema, i as u64);
+ }
+ let preds = vec![MemBoolPred::Range {
+ field: "n".to_string(),
+ gte: Some(0.0),
+ gt: None,
+ lte: None,
+ lt: Some(2000.0),
+ }];
+ assert!(
+ mem.filtered_docs_arc(&preds).is_none(),
+ "array-valued predicate field must force the full-corpus fallback"
+ );
+ }
+}