AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(aggs): columnar filtered `cardinality` — kill the mixed-p99 brute stall (~20x)
xerj-orgef15b0b9b5d4
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex 7bcc0a622..2fef3dc0a 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -1022,7 +1022,15 @@ impl<'a> FastCtx<'a> {if self.top_filter.is_some()&& !matches!(agg_type,- "avg" | "sum" | "min" | "max" | "stats" | "value_count" | "terms" | "global"+ "avg"+ | "sum"+ | "min"+ | "max"+ | "stats"+ | "value_count"+ | "terms"+ | "cardinality"+ | "global"){return None;@@ -1354,87 +1362,273 @@ impl<'a> FastCtx<'a> {let mut distinct: std::collections::HashSet<String> = std::collections::HashSet::new();const BOOL_TERMS: [&str; 2] = ["false", "true"];- for seg in &self.segs {+ // Under a top-level query filter the distinct set must cover ONLY the+ // matching docs — the whole-segment `per_ord_count` shortcut (which+ // sees every doc) is valid only for the unfiltered path. The empty+ // plan drives `fused_seg_pass` purely for its columnar filter gate;+ // `slot_of_row` records which term-ords appear among matching rows.+ // This is what stops a `{size:0, query:range, cardinality}` from+ // bailing the whole request to the O(N) brute `_source` hydrate+ // (~1.2 s on a 600 k-doc unflushed memtable) — the mixed-read tail.+ let filtered = self.top_filter.is_some();+ let empty_plan = if filtered {+ Some(self.plan_subs(None, false)?)+ } else {+ None+ };+ for si in 0..self.segs.len() {+ let seg = &self.segs[si];+ if !filtered {+ // ── Unfiltered: whole-segment term-dictionary scan (fast) ──+ match seg.cols.get(field) {+ Some(Column::Keyword(k)) => {+ for (ord, &cnt) in k.per_ord_count.iter().enumerate() {+ if cnt > 0 {+ distinct.insert(k.terms[ord].clone());+ }+ }+ if let Some(ph) = &missing_placeholder {+ if !k.null_bitmap.is_empty() {+ distinct.insert(ph.clone());+ }+ }+ }+ Some(Column::Numeric(n)) if is_bool => {+ // Same 0/1 purity check as exec_terms: stray numbers+ // would render as "0"/"2.5" on the brute path → bail.+ let zeros = n.range_count(0.0, 0.0, true, true);+ let ones = n.range_count(1.0, 1.0, true, true);+ if zeros + ones != n.live_count {+ return None;+ }+ if zeros > 0 {+ distinct.insert(BOOL_TERMS[0].to_string());+ }+ if ones > 0 {+ distinct.insert(BOOL_TERMS[1].to_string());+ }+ if let Some(ph) = &missing_placeholder {+ if !n.null_bitmap.is_empty() {+ distinct.insert(ph.clone());+ }+ }+ }+ Some(_) => return None,+ None => {+ // Field absent from the whole segment: every doc counts+ // as missing (brute: empty vals → placeholder).+ if let Some(ph) = &missing_placeholder {+ if seg.docs > 0 {+ distinct.insert(ph.clone());+ }+ }+ }+ }+ continue;+ }+ // ── Filtered: per-row pass, distinct over matching rows only ──+ let plan = empty_plan.as_ref().unwrap();+ let mut mss = false;match seg.cols.get(field) {Some(Column::Keyword(k)) => {- for (ord, &cnt) in k.per_ord_count.iter().enumerate() {- if cnt > 0 {+ let mut seen = vec![false; k.terms.len()];+ let mut saw_missing = false;+ {+ let dense = k.null_bitmap.is_empty();+ let mut slot_of_row = |row: u32| -> Option<usize> {+ let ord = if dense {+ *k.ords.get(row as usize)? as usize+ } else {+ match k.ord_for(row) {+ Some(o) => o as usize,+ None => {+ saw_missing = true;+ return None;+ }+ }+ };+ seen[ord] = true;+ Some(ord)+ };+ self.fused_seg_pass(+ si,+ &mut slot_of_row,+ k.terms.len(),+ plan,+ &mut [],+ &mut [],+ &mut mss,+ )?;+ }+ for (ord, &s) in seen.iter().enumerate() {+ if s {distinct.insert(k.terms[ord].clone());}}- if let Some(ph) = &missing_placeholder {- if !k.null_bitmap.is_empty() {+ if saw_missing {+ if let Some(ph) = &missing_placeholder {distinct.insert(ph.clone());}}}Some(Column::Numeric(n)) if is_bool => {- // Same 0/1 purity check as exec_terms: stray numbers- // would render as "0"/"2.5" on the brute path → bail.let zeros = n.range_count(0.0, 0.0, true, true);let ones = n.range_count(1.0, 1.0, true, true);if zeros + ones != n.live_count {return None;}- if zeros > 0 {+ let mut seen_bool = [false, false];+ let mut saw_missing = false;+ {+ let dense = n.null_bitmap.is_empty();+ let mut slot_of_row = |row: u32| -> Option<usize> {+ let bits = if dense {+ *n.data.get(row as usize)?+ } else {+ match n.get(row) {+ Some(b) => b,+ None => {+ saw_missing = true;+ return None;+ }+ }+ };+ let idx = usize::from(f64::from_bits(bits as u64) != 0.0);+ seen_bool[idx] = true;+ Some(idx)+ };+ self.fused_seg_pass(+ si,+ &mut slot_of_row,+ 2,+ plan,+ &mut [],+ &mut [],+ &mut mss,+ )?;+ }+ if seen_bool[0] {distinct.insert(BOOL_TERMS[0].to_string());}- if ones > 0 {+ if seen_bool[1] {distinct.insert(BOOL_TERMS[1].to_string());}- if let Some(ph) = &missing_placeholder {- if !n.null_bitmap.is_empty() {+ if saw_missing {+ if let Some(ph) = &missing_placeholder {distinct.insert(ph.clone());}}}Some(_) => return None,None => {- // Field absent from the whole segment: every doc counts- // as missing (brute: empty vals → placeholder).+ // Field absent from this segment → every matching row is+ // "missing"; only need whether ≥1 row matches the filter.if let Some(ph) = &missing_placeholder {- if seg.docs > 0 {+ let mut any = false;+ {+ let mut slot_of_row = |_row: u32| -> Option<usize> {+ any = true;+ None+ };+ self.fused_seg_pass(+ si,+ &mut slot_of_row,+ 0,+ plan,+ &mut [],+ &mut [],+ &mut mss,+ )?;+ }+ if any {distinct.insert(ph.clone());}}}}}- // Memtable docs. Columnar fast arm first (same equivalence gates- // as exec_terms — see `terms_counts_columnar`); the per-doc- // extraction walk below was the cardinality read-under-write tail- // at a drain-lagged memtable.- let mem_columnar: Option<(std::collections::HashMap<String, u64>, u64)> =- if !field.contains('.') {- self.idx.memtable.terms_counts_columnar(field)- } else {- None+ // ── Memtable docs ─────────────────────────────────────────────────+ if filtered {+ // Distinct over MATCHING memtable docs only. O(matching) columnar+ // candidates when the filter columnarises (re-checked against the+ // JSON matcher for byte-identical results); else the full walk+ // gated by `doc_matches_filter`.+ let cols = match (self.filtered_mem(), self.top_filter_query.as_ref()) {+ (Some(fdocs), Some(q)) => Some((fdocs, q)),+ _ => None,};- match mem_columnar {- Some((counts, missing)) => {- for (term, cnt) in counts {- if cnt > 0 {- distinct.insert(term);+ if let Some((fdocs, q)) = cols {+ for (_id, doc) in fdocs {+ if !doc_matches_filter(doc, q) {+ continue;+ }+ let vals = extract_field_values(doc, field);+ if vals.is_empty() {+ if let Some(ph) = &missing_placeholder {+ distinct.insert(ph.clone());+ }+ } else {+ for v in vals {+ distinct.insert(v);+ }}}- if missing > 0 {- if let Some(ph) = &missing_placeholder {- distinct.insert(ph.clone());+ } else {+ for doc in self.mem().iter() {+ if let Some(q) = &self.top_filter_query {+ if !doc_matches_filter(doc, q) {+ continue;+ }+ }+ let vals = extract_field_values(doc, field);+ if vals.is_empty() {+ if let Some(ph) = &missing_placeholder {+ distinct.insert(ph.clone());+ }+ } else {+ for v in vals {+ distinct.insert(v);+ }}}}- None => {- // Exact brute extractor so string rendering is identical.- for d in self.mem().iter() {- let vals = crate::aggs::extract_field_values(d, field);- if vals.is_empty() {+ } else {+ // Unfiltered memtable. Columnar fast arm first (same equivalence+ // gates as exec_terms — see `terms_counts_columnar`); the per-doc+ // extraction walk below was the cardinality read-under-write tail+ // at a drain-lagged memtable.+ let mem_columnar: Option<(std::collections::HashMap<String, u64>, u64)> =+ if !field.contains('.') {+ self.idx.memtable.terms_counts_columnar(field)+ } else {+ None+ };+ match mem_columnar {+ Some((counts, missing)) => {+ for (term, cnt) in counts {+ if cnt > 0 {+ distinct.insert(term);+ }+ }+ if missing > 0 {if let Some(ph) = &missing_placeholder {distinct.insert(ph.clone());}- continue;}- for v in vals {- distinct.insert(v);+ }+ None => {+ // Exact brute extractor so string rendering is identical.+ for d in self.mem().iter() {+ let vals = crate::aggs::extract_field_values(d, field);+ if vals.is_empty() {+ if let Some(ph) = &missing_placeholder {+ distinct.insert(ph.clone());+ }+ continue;+ }+ for v in vals {+ distinct.insert(v);+ }}}}