Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Merge regexp term-dictionary fix (acb2df6) — union resolution with sorted-DV prefilter

xerj-org3aebfde99cb1
diff --cc engine/crates/xerj-engine/src/index.rs
index 8a39bb26a,69ef43db1..99f1a8993
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@@ -343,22 -343,19 +343,35 @@@ pub struct Index
/// invalidation. Same `Left unbounded for now` caveat as `dv_cache`.
stored_value_cache: Arc<dashmap::DashMap<String, Arc<Vec<Value>>>>,
+ /// Per-segment cache of the DECOMPRESSED stored section plus per-doc
+ /// `(start, end)` byte offsets, used by the sorted-DV candidate path
+ /// (field-sorted match_all / search_after). With it, hydrating the
+ /// O(from+size) candidate positions of a segment is two vector
+ /// lookups + a per-candidate `simd_json` parse — no per-query
+ /// zstd/LZ4 decompress and no O(segment-bytes) brace re-scan.
+ ///
+ /// Unlike `stored_value_cache` (unbounded parsed `Value`s, ~3-6× raw
+ /// bytes) this holds the raw bytes once per segment and is BUDGETED:
+ /// `stored_slices_cache_bytes` tracks retained size and inserts stop
+ /// at `STORED_SLICES_CACHE_BUDGET` (miss → per-query decompress path,
+ /// exactly the pre-cache behaviour). Segments are immutable, so
+ /// entries stay valid until a merge drops the segment id (evicted at
+ /// the merge-completion site alongside dv_cache/stored_value_cache).
+ stored_slices_cache: Arc<dashmap::DashMap<String, Arc<StoredSlices>>>,
+ stored_slices_cache_bytes: Arc<AtomicU64>,
+ /// Regexp term-dictionary expansion cache — keyed by
+ /// `(segment_id, field, pattern)`, holds the exact union doc count
+ /// plus the first `REGEXP_EXPANSION_POS_CAP` matching doc positions
+ /// for that segment (ES keeps the equivalent in its per-segment
+ /// query cache). Segments are immutable, so entries stay valid for
+ /// the segment's lifetime — same no-invalidation contract as
+ /// `dv_cache` / `stored_value_cache` above. Bounded: cleared
+ /// wholesale when it exceeds `REGEXP_EXPANSION_CACHE_MAX` entries
+ /// because patterns are query-supplied. This is what makes a
+ /// cache-busted (size-varied) regexp count O(1) after the first
+ /// touch instead of re-opening FTS side-cars and re-merging postings
+ /// on every request.
+ regexp_expand_cache: Arc<dashmap::DashMap<(String, String, String), Arc<RegexpExpansion>>>,
/// M3 framework — response cache for identical queries. Keyed on
/// `(query_body_hash, dataset_version)` and holds the pre-computed
@@@ -498,8 -495,7 +511,9 @@@ impl Index
embedding_proxy: Arc::new(make_embedding_proxy(&config.embedding)),
dv_cache: Arc::new(dashmap::DashMap::new()),
stored_value_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),
+ regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
query_cache: Arc::new(dashmap::DashMap::new()),
dataset_version: Arc::new(AtomicU64::new(0)),
flush_signal: Arc::new(SyncFlushCoord::new()),
@@@ -651,8 -647,7 +665,9 @@@
embedding_proxy: Arc::new(make_embedding_proxy(&config.embedding)),
dv_cache: Arc::new(dashmap::DashMap::new()),
stored_value_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),
+ regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
query_cache: Arc::new(dashmap::DashMap::new()),
dataset_version: Arc::new(AtomicU64::new(0)),
flush_signal: Arc::new(SyncFlushCoord::new()),
@@@ -4575,26 -4378,27 +4606,47 @@@
lte.as_ref(),
lt.as_ref(),
)
+ } else if is_match_all && !deletes_present {
+ // Sorted-DV candidate pruning for field-sorted
+ // match_all: the segment's sorted numeric
+ // doc-values index yields the only positions that
+ // can reach the global top-(from+size) page —
+ // including the `search_after` cursor bound — so
+ // the stored scan parses O(from+size) docs per
+ // segment instead of every doc. `None` when the
+ // shape isn't provably safe (non-numeric field,
+ // nulls/arrays present, dv/stored misalignment,
+ // deletes) → full scan, still correct.
+ sort_topk.as_ref().and_then(|topk| {
+ self.build_sort_candidates_prefilter(
+ &segments_dir,
+ &seg_id,
+ meta.doc_count,
+ topk,
+ materialisation_limit,
+ )
+ })
+ } else if let QueryNode::Regexp { field, pattern } = query {
+ // Dictionary-expansion pre-filter (mirrors the
+ // `try_shortcut_count` Regexp arm). When the
+ // exact total comes from the shortcut
+ // (count_authoritative) the scan early-breaks at
+ // `materialisation_limit` hits, so cap the
+ // position set there; otherwise the scan tallies
+ // the total itself and needs every position.
+ let cap = if count_authoritative {
+ Some(materialisation_limit)
+ } else {
+ None
+ };
+ self.build_regexp_prefilter_cached(
+ &segments_dir,
+ &seg_id,
+ field,
+ pattern,
+ cap,
+ regexp_field_is_keyword,
+ )
} else {
None
};
@@@ -6621,175 -6638,171 +6757,340 @@@ impl Index
Some(matching.into_iter().collect())
}
+ /// Sorted-DV candidate pruning for field-sorted `match_all` queries
+ /// (the search_after hot-path fix).
+ ///
+ /// For a segment with a numeric doc-values column on the PRIMARY sort
+ /// field, the column's sorted `(value, position)` index tells us exactly
+ /// which positions can possibly reach the global top-`cap` page:
+ /// - `asc`: the first `cap` positions at/after the cursor's primary
+ /// value (cursor ties INCLUDED — the secondary sort key may still
+ /// qualify them), plus all positions tied with the boundary value so
+ /// the secondary key can arbitrate;
+ /// - `desc`: symmetric from the top end.
+ ///
+ /// Every other position in the segment provably sorts outside the page
+ /// (worse primary key than `cap` closer candidates) or at/before the
+ /// cursor, so the stored scan can skip parsing it entirely. This is
+ /// what turns a 1M-doc `search_after` page from an O(N) parse-everything
+ /// scan into O(from+size) per segment.
+ ///
+ /// Returns `None` (→ caller falls back to the full scan, still correct)
+ /// unless the shape is provably safe:
+ /// - primary sort is a plain field (not `_score`/`_doc`) with a
+ /// `Column::Numeric` sidecar in this segment;
+ /// - the column has NO nulls (a null means a missing field, an array,
+ /// or a non-scalar — `missing`/`mode` semantics would need the full
+ /// scan) and covers exactly `seg_doc_count` docs (position alignment);
+ /// - the `search_after` primary cursor value (when present) is numeric.
+ ///
+ /// The caller additionally gates on `is_match_all && !deletes_present`:
+ /// deleted/superseded docs occupy positions in the sorted index, so with
+ /// deletes present the candidate slice could under-fill the page.
+ fn build_sort_candidates_prefilter(
+ &self,
+ segments_dir: &std::path::Path,
+ segment_id: &str,
+ seg_doc_count: u64,
+ topk: &SortTopK,
+ cap: usize,
+ ) -> Option<HashSet<u32>> {
+ use xerj_query::sort::SortOrder;
+ use xerj_storage::doc_values::Column;
+
+ let sf = topk.fields.first()?;
+ if sf.is_score() || sf.is_doc_order() {
+ return None;
+ }
+ let cols = self.dv_columns_for(segments_dir, segment_id)?;
+ let Some(Column::Numeric(n)) = cols.get(&sf.field) else {
+ return None;
+ };
+ if !n.null_bitmap.is_empty() || n.doc_count as u64 != seg_doc_count {
+ return None;
+ }
+ let sorted = &n.sorted;
+ if sorted.is_empty() {
+ return None;
+ }
+
+ // Primary cursor bound (search_after). `None` cursor → page 1.
+ let cursor_v: Option<f64> = match topk.after.as_deref() {
+ Some(vals) => match vals.first() {
+ Some(Value::Number(x)) => Some(x.as_f64()?),
+ // Non-numeric cursor against a numeric column — don't
+ // guess, take the full scan.
+ Some(_) => return None,
+ None => None,
+ },
+ None => None,
+ };
+
+ let len = sorted.len();
+ let cap = cap.max(1);
+ let (start, end) = if sf.order == SortOrder::Asc {
+ let lo = match cursor_v {
+ // Include cursor ties: `< v0` keeps every position whose
+ // value equals the cursor's primary value; the collector's
+ // full-key comparison decides via the secondary fields.
+ Some(v0) => sorted.partition_point(|(b, _)| f64::from_bits(*b as u64) < v0),
+ None => 0,
+ };
+ let mut end = (lo + cap).min(len);
+ // Extend across boundary ties so the secondary sort key can
+ // arbitrate between equal primary values.
+ if end > lo && end < len {
+ let boundary = sorted[end - 1].0;
+ while end < len && sorted[end].0 == boundary {
+ end += 1;
+ }
+ }
+ (lo, end)
+ } else {
+ let hi = match cursor_v {
+ // Include cursor ties from the other side: `<= v0` keeps
+ // positions with value == v0 inside the slice.
+ Some(v0) => sorted.partition_point(|(b, _)| f64::from_bits(*b as u64) <= v0),
+ None => len,
+ };
+ let mut start = hi.saturating_sub(cap);
+ if start < hi && start > 0 {
+ let boundary = sorted[start].0;
+ while start > 0 && sorted[start - 1].0 == boundary {
+ start -= 1;
+ }
+ }
+ (start, hi)
+ };
+
+ Some(sorted[start..end].iter().map(|(_, d)| *d).collect())
+ }
+
+ /// Hydrate the sorted-DV candidate positions of one segment from the
+ /// warm `stored_slices_cache` — the fast twin of the
+ /// `scan_stored_section_into` sort-topk branch (same version-map delete
+ /// check, same dedup, same pre-heap cursor/worst rejection, same
+ /// `total_count` tally for parsed live candidates), minus the per-query
+ /// decompress and the O(segment-bytes) brace scan.
+ ///
+ /// Only called on the `is_match_all` sorted-candidates path, so hits
+ /// score 1.0 and every live candidate matches by construction.
+ fn hydrate_sorted_candidates(
+ &self,
+ slices: &StoredSlices,
+ candidates: &HashSet<u32>,
+ topk: &mut SortTopK,
+ seen_ids: &mut HashSet<String>,
+ total_count: &mut u64,
+ ) {
+ // Ascending positions for forward locality over `slices.bytes`.
+ let mut positions: Vec<u32> = candidates.iter().copied().collect();
+ positions.sort_unstable();
+ for pos in positions {
+ let Some(&(start, end)) = slices.offsets.get(pos as usize) else {
+ continue;
+ };
+ let Some(slice) = slices.bytes.get(start as usize..end as usize) else {
+ continue;
+ };
+ let mut doc_buf = slice.to_vec();
+ let doc: Value = match simd_json::serde::from_slice(&mut doc_buf) {
+ Ok(v) => v,
+ Err(_) => continue,
+ };
+ let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");
+ if let Some(ver) = self.store.version_map.get(id_ref) {
+ if ver.deleted {
+ continue;
+ }
+ }
+ *total_count += 1;
+ if seen_ids.contains(id_ref) {
+ continue;
+ }
+ let source_ref = doc.get("_source").unwrap_or(&doc);
+ let key = compute_sort_values(source_ref, 1.0, id_ref, topk.fields.as_slice());
+ if !topk.would_admit(&key) {
+ continue;
+ }
+ let id = id_ref.to_string();
+ seen_ids.insert(id.clone());
+ topk.offer_keyed(Hit {
+ id,
+ score: 1.0,
+ source: source_ref.clone(),
+ sort: key,
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ });
+ }
+ }
+ /// Regexp pre-filter — the set of internal doc positions whose keyword
+ /// term matches the (compile-once) anchored pattern, used to skip
+ /// non-matching docs in the stored-section scan the same way the
+ /// Range pre-filter does.
+ ///
+ /// The regex runs over the segment's DISTINCT terms (doc-values
+ /// keyword dictionary when present, else the field's FTS `.fst`
+ /// dictionary), not per doc: O(dictionary) matcher work + one ords
+ /// walk / bounded postings merge. Exactness mirrors the
+ /// `try_shortcut_count` Regexp arm: the doc-scan Regexp arm matches
+ /// only `Value::String` sources; the keyword column stores exactly
+ /// those strings (non-strings are null), and the FST route is only
+ /// taken when `fst_ok` (schema says keyword-typed → term == raw
+ /// string, and FTS doc ids == stored positions, see the FTS hit
+ /// materialisation which indexes stored docs by `doc_id`). Fields
+ /// with neither source return `None` → full scan, never a wrong skip.
+ ///
+ /// `cap`: when the caller's `hits.total` is authoritative (F1) the
+ /// scan early-breaks after materialising `cap` hits anyway, so we stop
+ /// collecting positions there instead of building an ~850 k-entry set
+ /// for a broad pattern. Pass `None` when the scan must tally the
+ /// exact total itself — then the set must be complete. The capped
+ /// set holds the FIRST `cap` matching positions in doc order (ords
+ /// walk / ascending postings merge), so the scan materialises the
+ /// same leading hits the uncapped scan would.
+ fn build_regexp_prefilter_cached(
+ &self,
+ segments_dir: &std::path::Path,
+ segment_id: &str,
+ field: &str,
+ pattern: &str,
+ cap: Option<usize>,
+ fst_ok: bool,
+ ) -> Option<HashSet<u32>> {
+ let exp =
+ self.regexp_segment_expansion(segments_dir, segment_id, field, pattern, fst_ok)?;
+ if exp.count == 0 {
+ // No dictionary term matches — whole segment skipped without
+ // decompressing / parsing a single stored doc.
+ return Some(HashSet::new());
+ }
+ match cap {
+ // Bounded (F1 count-authoritative) mode: the scan stops after
+ // materialising `c` hits, so the FIRST `c` matching positions
+ // (ascending == scan order) reproduce exactly the hits the
+ // unfiltered scan would emit. The expansion retains
+ // `REGEXP_EXPANSION_POS_CAP` ≥ any default page, so this only
+ // falls back to a full scan for very deep pagination.
+ Some(c) => {
+ if exp.complete || exp.positions.len() >= c {
+ Some(exp.positions.iter().take(c).copied().collect())
+ } else {
+ None
+ }
+ }
+ // Unbounded mode (scan must tally the exact total itself):
+ // only a COMPLETE position set is a valid filter.
+ None => {
+ if exp.complete {
+ Some(exp.positions.iter().copied().collect())
+ } else {
+ None
+ }
+ }
+ }
+ }
+
+ /// Expand a regexp against one segment's term dictionary, cached per
+ /// `(segment_id, field, pattern)` — see `regexp_expand_cache`.
+ ///
+ /// Sources, in order:
+ /// (a) the `.dv` keyword column (sorted distinct terms + ords) — exact
+ /// for any field type because it stores raw strings only;
+ /// (b) the field's FTS `.fst` term dictionary + postings union —
+ /// taken only when `fst_ok` (schema says the field is
+ /// keyword-typed, so FST terms are the raw un-analyzed strings and
+ /// the expansion agrees with the stored-doc scan it stands in
+ /// for). Flushed segments currently carry no `.dv` sidecar, so
+ /// (b) is the path that serves live traffic; merged segments have
+ /// both.
+ ///
+ /// Returns `None` when neither source can answer exactly (missing
+ /// side-cars, non-keyword field, or a pathological pattern matching
+ /// > 1024 dictionary terms) — callers fall back to the stored scan.
+ fn regexp_segment_expansion(
+ &self,
+ segments_dir: &std::path::Path,
+ segment_id: &str,
+ field: &str,
+ pattern: &str,
+ fst_ok: bool,
+ ) -> Option<Arc<RegexpExpansion>> {
+ let key = (
+ segment_id.to_string(),
+ field.to_string(),
+ pattern.to_string(),
+ );
+ if let Some(hit) = self.regexp_expand_cache.get(&key) {
+ return Some(Arc::clone(hit.value()));
+ }
+
+ // An invalid pattern matches nothing (the doc-scan arm fails the
+ // per-doc compile and returns false) — expansion is empty+complete.
+ let re = compiled_anchored_regex(pattern);
+
+ let mut exp: Option<RegexpExpansion> = None;
+ if let Some(cols) = self.dv_columns_for(segments_dir, segment_id) {
+ if let Some(xerj_storage::doc_values::Column::Keyword(k)) = cols.get(field) {
+ let mut count: u64 = 0;
+ let mut positions: Vec<u32> = Vec::new();
+ if let Some(re) = re.as_ref() {
+ let matched_ords: Vec<bool> =
+ k.terms.iter().map(|t| re.is_match(t)).collect();
+ if matched_ords.iter().any(|&m| m) {
+ for (i, &ord) in k.ords.iter().enumerate() {
+ let pos = i as u32;
+ if k.null_bitmap.contains(pos) {
+ continue;
+ }
+ if matched_ords.get(ord as usize).copied().unwrap_or(false) {
+ count += 1;
+ if positions.len() < REGEXP_EXPANSION_POS_CAP {
+ positions.push(pos);
+ }
+ }
+ }
+ }
+ }
+ let complete = count == positions.len() as u64;
+ exp = Some(RegexpExpansion { count, positions, complete });
+ }
+ }
+
+ let exp = match exp {
+ Some(e) => e,
+ None => {
+ if !fst_ok {
+ return None;
+ }
+ let reader =
+ FtsIndexReader::open(segments_dir, segment_id, &[field]).ok()?;
+ reader.field_stats(field)?;
+ let matched = regexp_matched_fst_terms(&reader, field, re.as_ref())?;
+ if matched.is_empty() {
+ RegexpExpansion { count: 0, positions: Vec::new(), complete: true }
+ } else {
+ let (count, positions) = postings_union_expand(
+ &reader,
+ field,
+ &matched,
+ REGEXP_EXPANSION_POS_CAP,
+ );
+ let complete = count == positions.len() as u64;
+ RegexpExpansion { count, positions, complete }
+ }
+ }
+ };
+
+ if self.regexp_expand_cache.len() >= REGEXP_EXPANSION_CACHE_MAX {
+ self.regexp_expand_cache.clear();
+ }
+ let arc = Arc::new(exp);
+ self.regexp_expand_cache.insert(key, Arc::clone(&arc));
+ Some(arc)
+ }
/// Version of `try_aggs_fast_with_segments` that caches decoded columns
/// per segment on `self.dv_cache`. Segments are immutable once flushed,
diff --cc engine/crates/xerj-storage/src/version_map.rs
index ee1e03e7d,f4e6a180e..a0447aba2
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@@ -103,45 -125,12 +125,48 @@@ impl VersionMap
) {
let doc_id = doc_id.into();
debug!(?doc_id, seq_no, "version_map::set");
- self.inner.insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ let old = self
+ .inner
+ .insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ self.live_delta(old.map_or(false, |e| !e.deleted), !deleted);
}
+ /// Record `doc_id → (seq_no, segment_id)` only if `seq_no` is >= the
+ /// currently recorded seq_no (or the doc is unknown).
+ ///
+ /// Used by the merge task to repoint surviving docs at the merged
+ /// segment with their REAL per-doc seq_nos:
+ /// - `>=` (not `>`): the surviving doc's entry currently points at a
+ /// merged-away INPUT segment with the SAME seq_no — equality must
+ /// win or `apply_merge → remove_segment` would drop the entry (and
+ /// the doc) entirely.
+ /// - guarded (not unconditional `set`): a doc updated concurrently
+ /// while the merge ran has a NEWER entry pointing at the memtable /
+ /// another segment — clobbering it with the merged copy would
+ /// resurrect the stale version.
+ ///
+ /// Atomic per key via the dashmap entry API.
+ pub fn set_if_latest(
+ &self,
+ doc_id: impl Into<String>,
+ seq_no: SeqNo,
+ segment_id: impl Into<Arc<str>>,
+ deleted: bool,
+ ) {
+ let doc_id = doc_id.into();
+ use dashmap::mapref::entry::Entry;
+ match self.inner.entry(doc_id) {
+ Entry::Occupied(mut occ) => {
+ if seq_no >= occ.get().seq_no {
+ occ.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ }
+ }
+ Entry::Vacant(vac) => {
+ vac.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ }
+ }
+ }
+
/// Attempt an atomic compare-and-set.
///
/// - If `expected_seq_no` is `None`, succeeds only if the document does NOT