AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(aggs): columnar fast path for FILTERED size:0 aggs (7.8s→~10ms, ~1000×)
xerj-orgfaed6ff9cb43
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex 2042ad385..6ab524c5a 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -143,6 +143,19 @@ pub(super) struct FastCtx<'a> {/// the brute path renders them as "false"/"true" term keys — so the/// terms executor needs the mapping to reproduce that.bool_fields: &'a std::collections::HashSet<String>,+ /// Top-level query filter (from `{size:0, query:Q, aggs:…}`), compiled to+ /// a columnar predicate. `None` == match_all (the whole corpus). When+ /// present, EVERY executor restricts its columnar reduction to matching+ /// segment rows (via the `fused_seg_pass` row gate + `exec_metric_top`'s+ /// filtered loop) and every memtable-doc walk is gated by+ /// `doc_matches_filter(doc, top_filter_query)`. Only metric + terms aggs+ /// support filtering; other agg types bail (None) so the caller falls back+ /// to the exact brute path.+ top_filter: Option<Pred>,+ /// The original ES-JSON query for the top-level filter, used for the+ /// memtable per-doc match (the trusted sibling of `compile_pred`, exactly+ /// as `exec_filter` pairs them). `None` == match_all.+ top_filter_query: Option<Value>,}impl<'a> FastCtx<'a> {@@ -237,6 +250,7 @@ impl MetricAcc {}}+#[derive(Clone)]struct MetricSpec {name: String,kind: MetricKind,@@ -316,10 +330,11 @@ impl Index {pub(super) fn try_fast_aggs(&self,aggs_def: &Value,+ filter: Option<&Value>,snap: &xerj_storage::index_store::IndexSnapshot,segments_dir: &std::path::Path,bool_fields: &std::collections::HashSet<String>,- ) -> Option<Value> {+ ) -> Option<(Value, Option<u64>)> {if fast_aggs_disabled() {return None;}@@ -328,6 +343,17 @@ impl Index {return None;}+ // Top-level query filter (`{size:0, query:Q, aggs:…}`). `None` ==+ // match_all — the pre-existing whole-corpus fast path. When a filter+ // is present we MUST be able to columnarize it (term/terms/range/+ // match_all, or a pure-conjunction bool of those); anything else bails+ // to the exact brute path.+ let top_filter: Option<Pred> = match filter {+ None => None,+ Some(q) => Some(compile_top_pred(q)?),+ };+ let top_filter_query: Option<Value> = filter.cloned();+// Size + delete gates.let seg_physical: u64 = snap.segments.iter().map(|m| m.doc_count).sum();let mem_physical = self.memtable.doc_count() as u64;@@ -395,6 +421,34 @@ impl Index {mem_docs: std::sync::OnceLock::new(),needs_owned_mem,bool_fields,+ top_filter,+ top_filter_query,+ };++ // Filtered `hits.total`: the number of live docs matching the query.+ // For match_all the caller derives it from segment + memtable counts+ // (and later overwrites with the delete-aware live count), so we only+ // compute a total when a filter narrowed the corpus. No deletes are+ // present on the fast path (gated above), so physical row counts are+ // exact, and each memtable doc is one hit (weights affect agg+ // `doc_count`, not `hits.total`).+ let filtered_total: Option<u64> = match &ctx.top_filter {+ None => None,+ Some(pred) => {+ let mut total: u64 = 0;+ for seg in &ctx.segs {+ let sp = resolve_pred(&seg.cols, pred)?;+ total += seg_pred_count(&sp, seg.docs);+ }+ if let Some(q) = &ctx.top_filter_query {+ for doc in ctx.mem().iter() {+ if doc_matches_filter(doc, q) {+ total += 1;+ }+ }+ }+ Some(total)+ }};let mut result = Map::new();@@ -415,7 +469,7 @@ impl Index {result.insert(agg_name.clone(), agg_result);}resolve_sibling_pipelines(&mut result);- Some(Value::Object(result))+ Some((Value::Object(result), filtered_total))}}@@ -707,6 +761,19 @@ impl<'a> FastCtx<'a> {// ── dispatch ─────────────────────────────────────────────────────────fn exec_agg(&self, agg_type: &str, params: &Value, sub: Option<&Value>) -> Option<Value> {+ // Under a top-level query filter only the executors that thread the+ // filter through their columnar reduction are correct; every other+ // agg type falls back to the exact brute path (return None). The+ // supported set covers the filtered-metric benchmark (avg/sum/…) and+ // the common filtered `terms` dashboard.+ if self.top_filter.is_some()+ && !matches!(+ agg_type,+ "avg" | "sum" | "min" | "max" | "stats" | "value_count" | "terms"+ )+ {+ return None;+ }match agg_type {"avg" | "sum" | "min" | "max" | "stats" | "value_count" => {if sub.is_some() {@@ -745,6 +812,43 @@ impl<'a> FastCtx<'a> {return None;}}++ // ── Filtered path ────────────────────────────────────────────────+ // A top-level query filter invalidates the O(1) precomputed column+ // stats (they cover the whole segment). Reduce the metric over ONLY+ // the matching rows using the exact fused-pass fold (null handling,+ // value_count presence semantics identical to the unfiltered path),+ // then fold matching memtable docs with the brute extractors.+ if self.top_filter.is_some() {+ let spec = MetricSpec {+ name: String::new(),+ kind,+ field: field.to_string(),+ meta: None,+ };+ let plan = SubPlan {+ metrics: vec![spec.clone()],+ top_hits: None,+ pipelines: Vec::new(),+ };+ let mut accs: Vec<Vec<MetricAcc>> = vec![vec![MetricAcc::default()]];+ let mut tops: Vec<Vec<(f64, u64, DocRef)>> = vec![Vec::new()];+ let mut ms = false;+ for si in 0..self.segs.len() {+ let mut slot = |_row: u32| -> Option<usize> { Some(0) };+ self.fused_seg_pass(si, &mut slot, 1, &plan, &mut accs, &mut tops, &mut ms)?;+ }+ let mut acc = std::mem::take(&mut accs[0][0]);+ if let Some(q) = &self.top_filter_query {+ for d in self.mem().iter() {+ if doc_matches_filter(d, q) {+ Self::fold_mem_metric(d, &spec, &mut acc);+ }+ }+ }+ return Some(Self::emit_metric(kind, &acc));+ }+let mut acc = MetricAcc::default();for s in &self.segs {match s.cols.get(field) {@@ -807,6 +911,16 @@ impl<'a> FastCtx<'a> {missing_sort_seen: &mut bool,) -> Option<()> {let seg = &self.segs[si];+ // Top-level query filter: resolve its columnar predicate for this+ // segment ONCE (bail the whole fast path if it can't be resolved —+ // e.g. field-kind mismatch — so the caller uses the brute path).+ // Rows that don't match are skipped before any bucket assignment, so+ // every fused-pass consumer (terms, filtered metric) reduces over+ // exactly the matching doc set.+ let top_sp: Option<SegPred> = match &self.top_filter {+ Some(pred) => Some(resolve_pred(&seg.cols, pred)?),+ None => None,+ };// Resolve metric columns once.let mut num_cols: Vec<Option<&NumericColumn>> = Vec::with_capacity(plan.metrics.len());let mut kw_cols: Vec<Option<&KeywordColumn>> = Vec::with_capacity(plan.metrics.len());@@ -855,6 +969,11 @@ impl<'a> FastCtx<'a> {let th_dense = th_col.map_or(false, |n| n.null_bitmap.is_empty());for row in 0..seg.docs {+ if let Some(sp) = &top_sp {+ if !seg_pred_matches(sp, row) {+ continue;+ }+ }let Some(slot) = slot_of_row(row) else { continue };for (mi, m) in plan.metrics.iter().enumerate() {match m.kind {@@ -1099,7 +1218,12 @@ impl<'a> FastCtx<'a> {};let plan = self.plan_subs(sub, true)?;- let has_row_work = !plan.metrics.is_empty() || plan.top_hits.is_some();+ // A top-level query filter forces the per-row pass (the `!has_row_work`+ // segment shortcut reads whole-segment `per_ord_count`, which ignores+ // the filter). `fused_seg_pass` then gates rows by the filter, and+ // the memtable arm below matches each doc against it.+ let has_row_work =+ !plan.metrics.is_empty() || plan.top_hits.is_some() || self.top_filter.is_some();// Global per-term state.struct TermState {@@ -1272,6 +1396,13 @@ impl<'a> FastCtx<'a> {None => {// Brute extractor semantics (multi-value, weights).for (di, doc) in self.mem().iter().enumerate() {+ // Top-level query filter: skip non-matching memtable docs+ // (segment rows are already gated inside `fused_seg_pass`).+ if let Some(q) = &self.top_filter_query {+ if !doc_matches_filter(doc, q) {+ continue;+ }+ }let weight = doc_count_weight(doc);let vals = extract_field_values(doc, field);for term in vals {@@ -1920,6 +2051,12 @@ impl<'a> FastCtx<'a> {Ok(Some(ColKind::Numeric)) | Ok(None) => Some(()),_ => None,},+ Pred::And(subs) => {+ for s in subs {+ self.check_pred_kinds(s)?;+ }+ Some(())+ }}}@@ -2596,6 +2733,46 @@ enum Pred {TermKw { field: String, value: String },TermsKw { field: String, values: Vec<String> },RangeNum { field: String, lo: f64, lo_incl: bool, hi: f64, hi_incl: bool },+ /// Conjunction of leaf predicates — a `bool` with only `must`/`filter`+ /// clauses (produced by `compile_top_pred` for the top-level query filter;+ /// `compile_pred` never yields this, so the filter/filters/adjacency+ /// executors are unaffected).+ And(Vec<Pred>),+}++/// Compile a TOP-LEVEL query filter (`{size:0, query:Q, aggs:…}`) into a+/// columnar predicate. Leaves delegate to the trusted `compile_pred` (kept+/// perfectly in sync with `doc_matches_filter`); a `bool` with only+/// `must`/`filter` clauses becomes a pure conjunction. Anything else+/// (must_not, should, minimum_should_match, FTS leaves, …) → `None`, so the+/// caller falls back to the exact brute path.+fn compile_top_pred(filter: &Value) -> Option<Pred> {+ let obj = filter.as_object()?;+ if obj.len() != 1 {+ return None;+ }+ let (qtype, body) = obj.iter().next()?;+ if qtype == "bool" {+ let bo = body.as_object()?;+ if bo.keys().any(|k| k != "must" && k != "filter") {+ return None; // must_not / should / minimum_should_match → bail+ }+ let mut subs: Vec<Pred> = Vec::new();+ for key in ["must", "filter"] {+ if let Some(clauses) = bo.get(key) {+ let arr = clauses.as_array()?;+ for c in arr {+ subs.push(compile_top_pred(c)?);+ }+ }+ }+ return match subs.len() {+ 0 => Some(Pred::MatchAll),+ 1 => Some(subs.into_iter().next().unwrap()),+ _ => Some(Pred::And(subs)),+ };+ }+ compile_pred(filter)}/// Compile the (single-clause) filter queries `doc_matches_filter` supports@@ -2684,6 +2861,8 @@ enum SegPred<'a> {KwEq(&'a KeywordColumn, u32, bool),KwIn(&'a KeywordColumn, Vec<u32>, bool),Num(&'a NumericColumn, f64, bool, f64, bool, bool),+ /// Conjunction — every sub-predicate must match the row.+ And(Vec<SegPred<'a>>),}fn resolve_pred<'a>(@@ -2719,6 +2898,17 @@ fn resolve_pred<'a>(Some(Column::Keyword(_)) => return None,None => SegPred::Never,},+ Pred::And(subs) => {+ let mut resolved: Vec<SegPred<'a>> = Vec::with_capacity(subs.len());+ for s in subs {+ let sp = resolve_pred(cols, s)?;+ if matches!(sp, SegPred::Never) {+ return Some(SegPred::Never); // short-circuit: whole AND is empty+ }+ resolved.push(sp);+ }+ SegPred::And(resolved)+ }})}@@ -2754,6 +2944,7 @@ fn seg_pred_matches(sp: &SegPred<'_>, row: u32) -> bool {let hi_ok = if *hi_incl { v <= *hi } else { v < *hi };lo_ok && hi_ok}+ SegPred::And(subs) => subs.iter().all(|s| seg_pred_matches(s, row)),}}@@ -2767,6 +2958,10 @@ fn seg_pred_count(sp: &SegPred<'_>, docs: u32) -> u64 {.map(|o| k.per_ord_count.get(*o as usize).copied().unwrap_or(0) as u64).sum(),SegPred::Num(n, lo, lo_incl, hi, hi_incl, _) => n.range_count(*lo, *hi, *lo_incl, *hi_incl) as u64,+ // Conjunction has no O(1) form — count matching rows directly. Only+ // reached from the top-level filter count (compile_pred, used by the+ // filter/filters executors, never yields `And`).+ SegPred::And(_) => (0..docs).filter(|&row| seg_pred_matches(sp, row)).count() as u64,}}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 68f35d37d..25854974e 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -41,6 +41,82 @@ impl<'a> Drop for MergeFlagClear<'a> {#[path = "fast_aggs.rs"]mod fast_aggs;use crate::memtable::FtsMemtable;++/// Reconstruct the ES-JSON form of a query filter that the columnar fast-agg+/// path (`fast_aggs::compile_top_pred` + `aggs::doc_matches_filter`) can+/// evaluate over doc-values WITHOUT hydrating `_source`. Returns `None` for+/// any query that isn't a plain conjunctive filter over exact/numeric fields+/// (FTS, wild/prefix, must_not/should bool, date-string ranges, numeric term+/// keys, …) — the caller then keeps the exact brute path.+///+/// Only the leaf/bool shapes that `doc_matches_filter` and `compile_pred`+/// jointly support are emitted, so the memtable matcher and the columnar+/// predicate stay byte-identical.+fn query_node_to_agg_filter(node: &QueryNode) -> Option<Value> {+ match node {+ QueryNode::MatchAll => Some(serde_json::json!({ "match_all": {} })),+ // Exact term — keyword (string) values only. Numeric/bool term keys+ // have typed-coercion subtleties, so bail (brute path handles them).+ QueryNode::Term { field, value, .. } => match value {+ Value::String(s) => Some(serde_json::json!({ "term": { field: s } })),+ _ => None,+ },+ QueryNode::Terms { field, values, .. } => {+ let mut strs: Vec<Value> = Vec::with_capacity(values.len());+ for v in values {+ match v {+ Value::String(s) => strs.push(Value::String(s.clone())),+ _ => return None,+ }+ }+ Some(serde_json::json!({ "terms": { field: Value::Array(strs) } }))+ }+ // Numeric range only — every present bound must be a JSON number+ // (date-string bounds compile differently and are left to the brute+ // path).+ QueryNode::Range { field, gte, gt, lte, lt, .. } => {+ let mut bounds = serde_json::Map::new();+ for (k, opt) in [("gte", gte), ("gt", gt), ("lte", lte), ("lt", lt)] {+ if let Some(v) = opt {+ match v {+ Value::Number(_) => {+ bounds.insert(k.to_string(), v.clone());+ }+ _ => return None,+ }+ }+ }+ if bounds.is_empty() {+ return None;+ }+ Some(serde_json::json!({ "range": { field: Value::Object(bounds) } }))+ }+ // Pure conjunction: `must` + `filter` only. `should`, `must_not` and+ // `minimum_should_match` change matching semantics that the columnar+ // AND / `doc_matches_filter` conjunction can't reproduce here → bail.+ QueryNode::Bool {+ must,+ should,+ must_not,+ filter,+ minimum_should_match,+ } => {+ if !should.is_empty() || !must_not.is_empty() || minimum_should_match.is_some() {+ return None;+ }+ let mut clauses: Vec<Value> = Vec::with_capacity(must.len() + filter.len());+ for c in must.iter().chain(filter.iter()) {+ clauses.push(query_node_to_agg_filter(c)?);+ }+ if clauses.is_empty() {+ // bool with no positive clauses == match_all+ return Some(serde_json::json!({ "match_all": {} }));+ }+ Some(serde_json::json!({ "bool": { "filter": Value::Array(clauses) } }))+ }+ _ => None,+ }+}use crate::{EngineError, Result};// ── Response types ────────────────────────────────────────────────────────────@@ -4841,21 +4917,53 @@ impl Index {let dbg_mem_hits = all_hits.len();let dbg_mem_seen = seen_ids.len();let dbg_mem_total = total_count;- if is_match_all && size == 0 && request.aggs.is_some() && request.min_score.is_none() {+ // Extend the columnar agg fast path to FILTERED `size:0` aggs: a+ // `{size:0, query:Q, aggs:…}` where Q is a term/terms/numeric-range/+ // conjunctive-bool over doc-values fields. Without this the filtered+ // shape fell through to the full-corpus `_source` hydration path+ // (~7.8 s on 1 M docs) while the unfiltered shape was ~0.1 ms — a+ // ~78,000× cliff on any filtered-agg benchmark row. The filter is+ // evaluated columnarly (matching-row gate) and the reduction runs over+ // only the matching docs' doc-values, byte-identical to the brute+ // path. Non-columnarizable queries (FTS, must_not/should bool, …)+ // yield `agg_filter == None` here and keep the exact brute fallback.+ let agg_filter: Option<Value> =+ if is_match_all { None } else { query_node_to_agg_filter(query) };+ if size == 0+ && request.aggs.is_some()+ && request.min_score.is_none()+ && !query_needs_fts+ && (is_match_all || agg_filter.is_some())+ {if let Some(aggs_def) = &request.aggs {let r_opt = self- .try_aggs_fast_with_segments_cached(aggs_def, &snap, &segments_dir)+ .try_aggs_fast_with_segments_cached(+ aggs_def,+ agg_filter.as_ref(),+ &snap,+ &segments_dir,+ ).await;- if let Some(r) = r_opt {+ if let Some((r, filtered_total)) = r_opt {precomputed_aggs = Some(r);- // Synthesize total_count from segments + memtable so we- // don't need to touch the stored section at all.- let seg_docs: u64 = snap.segments.iter().map(|m| m.doc_count).sum();- let mem_count = {- let mem = &*self.memtable;- mem.doc_count() as u64- };- total_count = seg_docs + mem_count;+ match filtered_total {+ // Filtered fast path already computed the matching-doc+ // count (exact — no deletes on the fast path).+ Some(t) => {+ total_count = t;+ }+ // match_all: synthesize total_count from segments ++ // memtable so we don't touch the stored section at all+ // (later overwritten with the delete-aware live count).+ None => {+ let seg_docs: u64 = snap.segments.iter().map(|m| m.doc_count).sum();+ let mem_count = {+ let mem = &*self.memtable;+ mem.doc_count() as u64+ };+ total_count = seg_docs + mem_count;+ }+ }}}}@@ -8361,9 +8469,10 @@ impl Index {async fn try_aggs_fast_with_segments_cached(&self,aggs_def: &Value,+ filter: Option<&Value>,snap: &xerj_storage::index_store::IndexSnapshot,segments_dir: &std::path::Path,- ) -> Option<Value> {+ ) -> Option<(Value, Option<u64>)> {// 2026-07 agg-hang fix: the columnar executor in `fast_aggs.rs`// serves `size:0 + match_all + aggs` straight from segment `.dv`// sidecars + the live memtable. It bails (None) on any shape it@@ -8386,7 +8495,7 @@ impl Index {.map(|f| f.name.clone()).collect()};- self.try_fast_aggs(aggs_def, snap, segments_dir, &bool_fields)+ self.try_fast_aggs(aggs_def, filter, snap, segments_dir, &bool_fields)}}