AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Merge regexp term-dictionary fix (acb2df6) — union resolution with sorted-DV prefilter
xerj-org3aebfde99cb1
diff --cc engine/crates/xerj-engine/src/index.rsindex 8a39bb26a,69ef43db1..99f1a8993--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@@ -343,22 -343,19 +343,35 @@@ pub struct Index/// invalidation. Same `Left unbounded for now` caveat as `dv_cache`.stored_value_cache: Arc<dashmap::DashMap<String, Arc<Vec<Value>>>>,+ /// Per-segment cache of the DECOMPRESSED stored section plus per-doc+ /// `(start, end)` byte offsets, used by the sorted-DV candidate path+ /// (field-sorted match_all / search_after). With it, hydrating the+ /// O(from+size) candidate positions of a segment is two vector+ /// lookups + a per-candidate `simd_json` parse — no per-query+ /// zstd/LZ4 decompress and no O(segment-bytes) brace re-scan.+ ///+ /// Unlike `stored_value_cache` (unbounded parsed `Value`s, ~3-6× raw+ /// bytes) this holds the raw bytes once per segment and is BUDGETED:+ /// `stored_slices_cache_bytes` tracks retained size and inserts stop+ /// at `STORED_SLICES_CACHE_BUDGET` (miss → per-query decompress path,+ /// exactly the pre-cache behaviour). Segments are immutable, so+ /// entries stay valid until a merge drops the segment id (evicted at+ /// the merge-completion site alongside dv_cache/stored_value_cache).+ stored_slices_cache: Arc<dashmap::DashMap<String, Arc<StoredSlices>>>,+ stored_slices_cache_bytes: Arc<AtomicU64>,+ /// Regexp term-dictionary expansion cache — keyed by+ /// `(segment_id, field, pattern)`, holds the exact union doc count+ /// plus the first `REGEXP_EXPANSION_POS_CAP` matching doc positions+ /// for that segment (ES keeps the equivalent in its per-segment+ /// query cache). Segments are immutable, so entries stay valid for+ /// the segment's lifetime — same no-invalidation contract as+ /// `dv_cache` / `stored_value_cache` above. Bounded: cleared+ /// wholesale when it exceeds `REGEXP_EXPANSION_CACHE_MAX` entries+ /// because patterns are query-supplied. This is what makes a+ /// cache-busted (size-varied) regexp count O(1) after the first+ /// touch instead of re-opening FTS side-cars and re-merging postings+ /// on every request.+ regexp_expand_cache: Arc<dashmap::DashMap<(String, String, String), Arc<RegexpExpansion>>>,/// M3 framework — response cache for identical queries. Keyed on/// `(query_body_hash, dataset_version)` and holds the pre-computed@@@ -498,8 -495,7 +511,9 @@@ impl Indexembedding_proxy: Arc::new(make_embedding_proxy(&config.embedding)),dv_cache: Arc::new(dashmap::DashMap::new()),stored_value_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),+ regexp_expand_cache: Arc::new(dashmap::DashMap::new()),query_cache: Arc::new(dashmap::DashMap::new()),dataset_version: Arc::new(AtomicU64::new(0)),flush_signal: Arc::new(SyncFlushCoord::new()),@@@ -651,8 -647,7 +665,9 @@@embedding_proxy: Arc::new(make_embedding_proxy(&config.embedding)),dv_cache: Arc::new(dashmap::DashMap::new()),stored_value_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),+ regexp_expand_cache: Arc::new(dashmap::DashMap::new()),query_cache: Arc::new(dashmap::DashMap::new()),dataset_version: Arc::new(AtomicU64::new(0)),flush_signal: Arc::new(SyncFlushCoord::new()),@@@ -4575,26 -4378,27 +4606,47 @@@lte.as_ref(),lt.as_ref(),)+ } else if is_match_all && !deletes_present {+ // Sorted-DV candidate pruning for field-sorted+ // match_all: the segment's sorted numeric+ // doc-values index yields the only positions that+ // can reach the global top-(from+size) page —+ // including the `search_after` cursor bound — so+ // the stored scan parses O(from+size) docs per+ // segment instead of every doc. `None` when the+ // shape isn't provably safe (non-numeric field,+ // nulls/arrays present, dv/stored misalignment,+ // deletes) → full scan, still correct.+ sort_topk.as_ref().and_then(|topk| {+ self.build_sort_candidates_prefilter(+ &segments_dir,+ &seg_id,+ meta.doc_count,+ topk,+ materialisation_limit,+ )+ })+ } else if let QueryNode::Regexp { field, pattern } = query {+ // Dictionary-expansion pre-filter (mirrors the+ // `try_shortcut_count` Regexp arm). When the+ // exact total comes from the shortcut+ // (count_authoritative) the scan early-breaks at+ // `materialisation_limit` hits, so cap the+ // position set there; otherwise the scan tallies+ // the total itself and needs every position.+ let cap = if count_authoritative {+ Some(materialisation_limit)+ } else {+ None+ };+ self.build_regexp_prefilter_cached(+ &segments_dir,+ &seg_id,+ field,+ pattern,+ cap,+ regexp_field_is_keyword,+ )} else {None};@@@ -6621,175 -6638,171 +6757,340 @@@ impl IndexSome(matching.into_iter().collect())}+ /// Sorted-DV candidate pruning for field-sorted `match_all` queries+ /// (the search_after hot-path fix).+ ///+ /// For a segment with a numeric doc-values column on the PRIMARY sort+ /// field, the column's sorted `(value, position)` index tells us exactly+ /// which positions can possibly reach the global top-`cap` page:+ /// - `asc`: the first `cap` positions at/after the cursor's primary+ /// value (cursor ties INCLUDED — the secondary sort key may still+ /// qualify them), plus all positions tied with the boundary value so+ /// the secondary key can arbitrate;+ /// - `desc`: symmetric from the top end.+ ///+ /// Every other position in the segment provably sorts outside the page+ /// (worse primary key than `cap` closer candidates) or at/before the+ /// cursor, so the stored scan can skip parsing it entirely. This is+ /// what turns a 1M-doc `search_after` page from an O(N) parse-everything+ /// scan into O(from+size) per segment.+ ///+ /// Returns `None` (→ caller falls back to the full scan, still correct)+ /// unless the shape is provably safe:+ /// - primary sort is a plain field (not `_score`/`_doc`) with a+ /// `Column::Numeric` sidecar in this segment;+ /// - the column has NO nulls (a null means a missing field, an array,+ /// or a non-scalar — `missing`/`mode` semantics would need the full+ /// scan) and covers exactly `seg_doc_count` docs (position alignment);+ /// - the `search_after` primary cursor value (when present) is numeric.+ ///+ /// The caller additionally gates on `is_match_all && !deletes_present`:+ /// deleted/superseded docs occupy positions in the sorted index, so with+ /// deletes present the candidate slice could under-fill the page.+ fn build_sort_candidates_prefilter(+ &self,+ segments_dir: &std::path::Path,+ segment_id: &str,+ seg_doc_count: u64,+ topk: &SortTopK,+ cap: usize,+ ) -> Option<HashSet<u32>> {+ use xerj_query::sort::SortOrder;+ use xerj_storage::doc_values::Column;++ let sf = topk.fields.first()?;+ if sf.is_score() || sf.is_doc_order() {+ return None;+ }+ let cols = self.dv_columns_for(segments_dir, segment_id)?;+ let Some(Column::Numeric(n)) = cols.get(&sf.field) else {+ return None;+ };+ if !n.null_bitmap.is_empty() || n.doc_count as u64 != seg_doc_count {+ return None;+ }+ let sorted = &n.sorted;+ if sorted.is_empty() {+ return None;+ }++ // Primary cursor bound (search_after). `None` cursor → page 1.+ let cursor_v: Option<f64> = match topk.after.as_deref() {+ Some(vals) => match vals.first() {+ Some(Value::Number(x)) => Some(x.as_f64()?),+ // Non-numeric cursor against a numeric column — don't+ // guess, take the full scan.+ Some(_) => return None,+ None => None,+ },+ None => None,+ };++ let len = sorted.len();+ let cap = cap.max(1);+ let (start, end) = if sf.order == SortOrder::Asc {+ let lo = match cursor_v {+ // Include cursor ties: `< v0` keeps every position whose+ // value equals the cursor's primary value; the collector's+ // full-key comparison decides via the secondary fields.+ Some(v0) => sorted.partition_point(|(b, _)| f64::from_bits(*b as u64) < v0),+ None => 0,+ };+ let mut end = (lo + cap).min(len);+ // Extend across boundary ties so the secondary sort key can+ // arbitrate between equal primary values.+ if end > lo && end < len {+ let boundary = sorted[end - 1].0;+ while end < len && sorted[end].0 == boundary {+ end += 1;+ }+ }+ (lo, end)+ } else {+ let hi = match cursor_v {+ // Include cursor ties from the other side: `<= v0` keeps+ // positions with value == v0 inside the slice.+ Some(v0) => sorted.partition_point(|(b, _)| f64::from_bits(*b as u64) <= v0),+ None => len,+ };+ let mut start = hi.saturating_sub(cap);+ if start < hi && start > 0 {+ let boundary = sorted[start].0;+ while start > 0 && sorted[start - 1].0 == boundary {+ start -= 1;+ }+ }+ (start, hi)+ };++ Some(sorted[start..end].iter().map(|(_, d)| *d).collect())+ }++ /// Hydrate the sorted-DV candidate positions of one segment from the+ /// warm `stored_slices_cache` — the fast twin of the+ /// `scan_stored_section_into` sort-topk branch (same version-map delete+ /// check, same dedup, same pre-heap cursor/worst rejection, same+ /// `total_count` tally for parsed live candidates), minus the per-query+ /// decompress and the O(segment-bytes) brace scan.+ ///+ /// Only called on the `is_match_all` sorted-candidates path, so hits+ /// score 1.0 and every live candidate matches by construction.+ fn hydrate_sorted_candidates(+ &self,+ slices: &StoredSlices,+ candidates: &HashSet<u32>,+ topk: &mut SortTopK,+ seen_ids: &mut HashSet<String>,+ total_count: &mut u64,+ ) {+ // Ascending positions for forward locality over `slices.bytes`.+ let mut positions: Vec<u32> = candidates.iter().copied().collect();+ positions.sort_unstable();+ for pos in positions {+ let Some(&(start, end)) = slices.offsets.get(pos as usize) else {+ continue;+ };+ let Some(slice) = slices.bytes.get(start as usize..end as usize) else {+ continue;+ };+ let mut doc_buf = slice.to_vec();+ let doc: Value = match simd_json::serde::from_slice(&mut doc_buf) {+ Ok(v) => v,+ Err(_) => continue,+ };+ let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");+ if let Some(ver) = self.store.version_map.get(id_ref) {+ if ver.deleted {+ continue;+ }+ }+ *total_count += 1;+ if seen_ids.contains(id_ref) {+ continue;+ }+ let source_ref = doc.get("_source").unwrap_or(&doc);+ let key = compute_sort_values(source_ref, 1.0, id_ref, topk.fields.as_slice());+ if !topk.would_admit(&key) {+ continue;+ }+ let id = id_ref.to_string();+ seen_ids.insert(id.clone());+ topk.offer_keyed(Hit {+ id,+ score: 1.0,+ source: source_ref.clone(),+ sort: key,+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ });+ }+ }+ /// Regexp pre-filter — the set of internal doc positions whose keyword+ /// term matches the (compile-once) anchored pattern, used to skip+ /// non-matching docs in the stored-section scan the same way the+ /// Range pre-filter does.+ ///+ /// The regex runs over the segment's DISTINCT terms (doc-values+ /// keyword dictionary when present, else the field's FTS `.fst`+ /// dictionary), not per doc: O(dictionary) matcher work + one ords+ /// walk / bounded postings merge. Exactness mirrors the+ /// `try_shortcut_count` Regexp arm: the doc-scan Regexp arm matches+ /// only `Value::String` sources; the keyword column stores exactly+ /// those strings (non-strings are null), and the FST route is only+ /// taken when `fst_ok` (schema says keyword-typed → term == raw+ /// string, and FTS doc ids == stored positions, see the FTS hit+ /// materialisation which indexes stored docs by `doc_id`). Fields+ /// with neither source return `None` → full scan, never a wrong skip.+ ///+ /// `cap`: when the caller's `hits.total` is authoritative (F1) the+ /// scan early-breaks after materialising `cap` hits anyway, so we stop+ /// collecting positions there instead of building an ~850 k-entry set+ /// for a broad pattern. Pass `None` when the scan must tally the+ /// exact total itself — then the set must be complete. The capped+ /// set holds the FIRST `cap` matching positions in doc order (ords+ /// walk / ascending postings merge), so the scan materialises the+ /// same leading hits the uncapped scan would.+ fn build_regexp_prefilter_cached(+ &self,+ segments_dir: &std::path::Path,+ segment_id: &str,+ field: &str,+ pattern: &str,+ cap: Option<usize>,+ fst_ok: bool,+ ) -> Option<HashSet<u32>> {+ let exp =+ self.regexp_segment_expansion(segments_dir, segment_id, field, pattern, fst_ok)?;+ if exp.count == 0 {+ // No dictionary term matches — whole segment skipped without+ // decompressing / parsing a single stored doc.+ return Some(HashSet::new());+ }+ match cap {+ // Bounded (F1 count-authoritative) mode: the scan stops after+ // materialising `c` hits, so the FIRST `c` matching positions+ // (ascending == scan order) reproduce exactly the hits the+ // unfiltered scan would emit. The expansion retains+ // `REGEXP_EXPANSION_POS_CAP` ≥ any default page, so this only+ // falls back to a full scan for very deep pagination.+ Some(c) => {+ if exp.complete || exp.positions.len() >= c {+ Some(exp.positions.iter().take(c).copied().collect())+ } else {+ None+ }+ }+ // Unbounded mode (scan must tally the exact total itself):+ // only a COMPLETE position set is a valid filter.+ None => {+ if exp.complete {+ Some(exp.positions.iter().copied().collect())+ } else {+ None+ }+ }+ }+ }++ /// Expand a regexp against one segment's term dictionary, cached per+ /// `(segment_id, field, pattern)` — see `regexp_expand_cache`.+ ///+ /// Sources, in order:+ /// (a) the `.dv` keyword column (sorted distinct terms + ords) — exact+ /// for any field type because it stores raw strings only;+ /// (b) the field's FTS `.fst` term dictionary + postings union —+ /// taken only when `fst_ok` (schema says the field is+ /// keyword-typed, so FST terms are the raw un-analyzed strings and+ /// the expansion agrees with the stored-doc scan it stands in+ /// for). Flushed segments currently carry no `.dv` sidecar, so+ /// (b) is the path that serves live traffic; merged segments have+ /// both.+ ///+ /// Returns `None` when neither source can answer exactly (missing+ /// side-cars, non-keyword field, or a pathological pattern matching+ /// > 1024 dictionary terms) — callers fall back to the stored scan.+ fn regexp_segment_expansion(+ &self,+ segments_dir: &std::path::Path,+ segment_id: &str,+ field: &str,+ pattern: &str,+ fst_ok: bool,+ ) -> Option<Arc<RegexpExpansion>> {+ let key = (+ segment_id.to_string(),+ field.to_string(),+ pattern.to_string(),+ );+ if let Some(hit) = self.regexp_expand_cache.get(&key) {+ return Some(Arc::clone(hit.value()));+ }++ // An invalid pattern matches nothing (the doc-scan arm fails the+ // per-doc compile and returns false) — expansion is empty+complete.+ let re = compiled_anchored_regex(pattern);++ let mut exp: Option<RegexpExpansion> = None;+ if let Some(cols) = self.dv_columns_for(segments_dir, segment_id) {+ if let Some(xerj_storage::doc_values::Column::Keyword(k)) = cols.get(field) {+ let mut count: u64 = 0;+ let mut positions: Vec<u32> = Vec::new();+ if let Some(re) = re.as_ref() {+ let matched_ords: Vec<bool> =+ k.terms.iter().map(|t| re.is_match(t)).collect();+ if matched_ords.iter().any(|&m| m) {+ for (i, &ord) in k.ords.iter().enumerate() {+ let pos = i as u32;+ if k.null_bitmap.contains(pos) {+ continue;+ }+ if matched_ords.get(ord as usize).copied().unwrap_or(false) {+ count += 1;+ if positions.len() < REGEXP_EXPANSION_POS_CAP {+ positions.push(pos);+ }+ }+ }+ }+ }+ let complete = count == positions.len() as u64;+ exp = Some(RegexpExpansion { count, positions, complete });+ }+ }++ let exp = match exp {+ Some(e) => e,+ None => {+ if !fst_ok {+ return None;+ }+ let reader =+ FtsIndexReader::open(segments_dir, segment_id, &[field]).ok()?;+ reader.field_stats(field)?;+ let matched = regexp_matched_fst_terms(&reader, field, re.as_ref())?;+ if matched.is_empty() {+ RegexpExpansion { count: 0, positions: Vec::new(), complete: true }+ } else {+ let (count, positions) = postings_union_expand(+ &reader,+ field,+ &matched,+ REGEXP_EXPANSION_POS_CAP,+ );+ let complete = count == positions.len() as u64;+ RegexpExpansion { count, positions, complete }+ }+ }+ };++ if self.regexp_expand_cache.len() >= REGEXP_EXPANSION_CACHE_MAX {+ self.regexp_expand_cache.clear();+ }+ let arc = Arc::new(exp);+ self.regexp_expand_cache.insert(key, Arc::clone(&arc));+ Some(arc)+ }/// Version of `try_aggs_fast_with_segments` that caches decoded columns/// per segment on `self.dv_cache`. Segments are immutable once flushed,diff --cc engine/crates/xerj-storage/src/version_map.rsindex ee1e03e7d,f4e6a180e..a0447aba2--- a/engine/crates/xerj-storage/src/version_map.rs+++ b/engine/crates/xerj-storage/src/version_map.rs@@@ -103,45 -125,12 +125,48 @@@ impl VersionMap) {let doc_id = doc_id.into();debug!(?doc_id, seq_no, "version_map::set");- self.inner.insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });+ let old = self+ .inner+ .insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });+ self.live_delta(old.map_or(false, |e| !e.deleted), !deleted);}+ /// Record `doc_id → (seq_no, segment_id)` only if `seq_no` is >= the+ /// currently recorded seq_no (or the doc is unknown).+ ///+ /// Used by the merge task to repoint surviving docs at the merged+ /// segment with their REAL per-doc seq_nos:+ /// - `>=` (not `>`): the surviving doc's entry currently points at a+ /// merged-away INPUT segment with the SAME seq_no — equality must+ /// win or `apply_merge → remove_segment` would drop the entry (and+ /// the doc) entirely.+ /// - guarded (not unconditional `set`): a doc updated concurrently+ /// while the merge ran has a NEWER entry pointing at the memtable /+ /// another segment — clobbering it with the merged copy would+ /// resurrect the stale version.+ ///+ /// Atomic per key via the dashmap entry API.+ pub fn set_if_latest(+ &self,+ doc_id: impl Into<String>,+ seq_no: SeqNo,+ segment_id: impl Into<Arc<str>>,+ deleted: bool,+ ) {+ let doc_id = doc_id.into();+ use dashmap::mapref::entry::Entry;+ match self.inner.entry(doc_id) {+ Entry::Occupied(mut occ) => {+ if seq_no >= occ.get().seq_no {+ occ.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });+ }+ }+ Entry::Vacant(vac) => {+ vac.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });+ }+ }+ }+/// Attempt an atomic compare-and-set.////// - If `expected_seq_no` is `None`, succeeds only if the document does NOT