AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(search): search_after O(from+size) via in-collector cursor + sorted-DV candidates + stored-slices cache
xerj-org21542e54f199
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex eabafb8a0..13a71488b 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -343,6 +343,23 @@ pub struct Index {/// invalidation. Same `Left unbounded for now` caveat as `dv_cache`.stored_value_cache: Arc<dashmap::DashMap<String, Arc<Vec<Value>>>>,+ /// Per-segment cache of the DECOMPRESSED stored section plus per-doc+ /// `(start, end)` byte offsets, used by the sorted-DV candidate path+ /// (field-sorted match_all / search_after). With it, hydrating the+ /// O(from+size) candidate positions of a segment is two vector+ /// lookups + a per-candidate `simd_json` parse — no per-query+ /// zstd/LZ4 decompress and no O(segment-bytes) brace re-scan.+ ///+ /// Unlike `stored_value_cache` (unbounded parsed `Value`s, ~3-6× raw+ /// bytes) this holds the raw bytes once per segment and is BUDGETED:+ /// `stored_slices_cache_bytes` tracks retained size and inserts stop+ /// at `STORED_SLICES_CACHE_BUDGET` (miss → per-query decompress path,+ /// exactly the pre-cache behaviour). Segments are immutable, so+ /// entries stay valid until a merge drops the segment id (evicted at+ /// the merge-completion site alongside dv_cache/stored_value_cache).+ stored_slices_cache: Arc<dashmap::DashMap<String, Arc<StoredSlices>>>,+ stored_slices_cache_bytes: Arc<AtomicU64>,+/// M3 framework — response cache for identical queries. Keyed on/// `(query_body_hash, dataset_version)` and holds the pre-computed/// `SearchResult` ready to serialize. Cache hit → ~10 µs total@@ -481,6 +498,8 @@ impl Index {embedding_proxy: Arc::new(make_embedding_proxy(&config.embedding)),dv_cache: Arc::new(dashmap::DashMap::new()),stored_value_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),query_cache: Arc::new(dashmap::DashMap::new()),dataset_version: Arc::new(AtomicU64::new(0)),flush_signal: Arc::new(SyncFlushCoord::new()),@@ -632,6 +651,8 @@ impl Index {embedding_proxy: Arc::new(make_embedding_proxy(&config.embedding)),dv_cache: Arc::new(dashmap::DashMap::new()),stored_value_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache: Arc::new(dashmap::DashMap::new()),+ stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),query_cache: Arc::new(dashmap::DashMap::new()),dataset_version: Arc::new(AtomicU64::new(0)),flush_signal: Arc::new(SyncFlushCoord::new()),@@ -2358,6 +2379,10 @@ impl Index {for id in &dropped_seg_ids {self.dv_cache.remove(id.as_str());self.stored_value_cache.remove(id.as_str());+ if let Some((_, slices)) = self.stored_slices_cache.remove(id.as_str()) {+ self.stored_slices_cache_bytes+ .fetch_sub(slices.retained_bytes(), Ordering::Relaxed);+ }}}@@ -3744,10 +3769,38 @@ impl Index {// segment loop. `total_count` is tallied independently and stays exact.let field_sort_active: bool = size > 0&& request.sort.iter().any(|sf| !sf.is_score() && !sf.is_doc_order());++ // ── search_after cursor, normalized ONCE up front ─────────────────+ // Date-shaped cursor strings are converted to epoch numbers here so+ // BOTH the bounded sorted collector (pre-heap rejection during+ // collection) and the legacy post-collection filter compare against+ // the same shape the hits' `sort` arrays use (`compute_sort_values`+ // emits epoch numbers for date-shaped strings). Users may supply+ // either the raw epoch number (`search_after: [1571646604828]`) or a+ // formatted date string (`search_after: ["2019-10-21 08:30:04.828"]`);+ // ES accepts both.+ let after_norm: Option<Vec<Value>> = match &request.search_after {+ Some(vals) if !vals.is_empty() => Some(+ vals.iter()+ .enumerate()+ .map(|(i, v)| {+ let fmt = request.sort.get(i).and_then(|s| s.format.as_deref());+ normalize_search_after_value(v, fmt)+ })+ .collect(),+ ),+ _ => None,+ };+let mut sort_topk: Option<SortTopK> = if field_sort_active {Some(SortTopK::new(Arc::new(request.sort.clone()),materialisation_limit,+ // The cursor is applied DURING collection: the collector+ // admits only hits strictly after it, so deep cursors cost+ // the same as page 1 and are correct past the+ // materialisation cap.+ after_norm.clone(),))} else {None@@ -4420,15 +4473,75 @@ impl Index {lte.as_ref(),lt.as_ref(),)+ } else if is_match_all && !deletes_present {+ // Sorted-DV candidate pruning for field-sorted+ // match_all: the segment's sorted numeric+ // doc-values index yields the only positions that+ // can reach the global top-(from+size) page —+ // including the `search_after` cursor bound — so+ // the stored scan parses O(from+size) docs per+ // segment instead of every doc. `None` when the+ // shape isn't provably safe (non-numeric field,+ // nulls/arrays present, dv/stored misalignment,+ // deletes) → full scan, still correct.+ sort_topk.as_ref().and_then(|topk| {+ self.build_sort_candidates_prefilter(+ &segments_dir,+ &seg_id,+ meta.doc_count,+ topk,+ materialisation_limit,+ )+ })} else {None};- if let Ok(seg_reader) = self.store.open_segment_arc(&seg_id) {+ // A `Some(∅)` pre-filter proves no doc in this segment can+ // contribute a hit or a count (Range: no value in range;+ // sorted-DV: no candidate position): skip the open ++ // decompress + brace-scan entirely — the scan would have+ // skipped every position anyway.+ // Sorted-DV candidate path (field-sorted match_all with a+ // candidate pre-filter): serve the candidates from the+ // per-segment `stored_slices_cache` when warm — two vec+ // lookups + O(candidates) simd_json parses, no per-query+ // decompress and no O(segment-bytes) brace re-scan.+ let sorted_candidates_path =+ is_match_all && sort_topk.is_some() && pre_filter.is_some();++ if pre_filter.as_ref().is_some_and(|pf| pf.is_empty()) {+ // A `Some(∅)` pre-filter proves no doc in this segment+ // can contribute a hit or a count (Range: no value in+ // range; sorted-DV: no candidate position): skip the+ // open + decompress + brace-scan entirely — the scan+ // would have skipped every position anyway.+ } else if sorted_candidates_path+ && self.stored_slices_cache.contains_key(seg_id.as_str())+ {+ if let Some(entry) = self.stored_slices_cache.get(seg_id.as_str()) {+ let slices = Arc::clone(entry.value());+ drop(entry);+ self.hydrate_sorted_candidates(+ &slices,+ pre_filter.as_ref().expect("gated on is_some"),+ sort_topk.as_mut().expect("gated on is_some"),+ &mut seen_ids,+ &mut total_count,+ );+ }+ } else if let Ok(seg_reader) = self.store.open_segment_arc(&seg_id) {if let Ok(Some(stored_bytes_raw)) = seg_reader.section(SectionType::Stored) {if let Ok(stored_bytes) =xerj_storage::stored_codec::decode_stored(stored_bytes_raw){+ // Record per-doc offsets on the sorted path so+ // the decompressed section can be cached for+ // the next query (subject to the budget).+ let mut offsets: Vec<(u32, u32)> = Vec::new();+ let want_cache = sorted_candidates_path+ && stored_bytes.len() <= u32::MAX as usize+ && !self.stored_slices_cache.contains_key(seg_id.as_str());self.scan_stored_section_into(&stored_bytes,query,@@ -4441,7 +4554,31 @@ impl Index {&mut seen_ids,pre_filter.as_ref(),sort_topk.as_mut(),+ if want_cache { Some(&mut offsets) } else { None },);+ // Cache only a COMPLETE offsets map (a+ // malformed section bails early) and only+ // within the retained-bytes budget.+ if want_cache && offsets.len() as u64 == meta.doc_count {+ let slices = StoredSlices {+ bytes: stored_bytes,+ offsets,+ };+ let sz = slices.retained_bytes();+ if self+ .stored_slices_cache_bytes+ .load(Ordering::Relaxed)+ .saturating_add(sz)+ <= STORED_SLICES_CACHE_BUDGET+ && self+ .stored_slices_cache+ .insert(seg_id.clone(), Arc::new(slices))+ .is_none()+ {+ self.stored_slices_cache_bytes+ .fetch_add(sz, Ordering::Relaxed);+ }+ }}}}@@ -4795,77 +4932,16 @@ impl Index {}// --- Apply search_after cursor (skip hits up to and including the cursor) ---+ // Field-sorted queries already rejected cursor-≤ hits INSIDE the+ // bounded collector (`SortTopK::would_admit`), so for that path this+ // block is a no-op safety net (position 0 matches immediately). It+ // remains load-bearing for the score-sort cursor path, which has no+ // field collector. The cursor was normalized ONCE into `after_norm`+ // before collection (date strings → epoch numbers).if let Some(ref after_values) = request.search_after {if !after_values.is_empty() {- // Normalize the cursor's date-shaped string values to epoch so- // they align with the normalized hit.sort produced above.- // Users may supply either the raw epoch number- // (`search_after: [1571646604828]`) or a formatted date- // string (`search_after: ["2019-10-21 08:30:04.828"]`); ES- // accepts both.- fn normalize_cursor(v: &Value, fmt_hint: Option<&str>) -> Value {- if let Some(s) = v.as_str() {- // If the SortField.format is a known ES pattern, try- // it first — this catches custom formats like- // `yyyy-MM-dd | HH:mm:ss.SSS` whose cursor strings- // don't match the default date detectors below.- if let Some(fmt) = fmt_hint {- if let Some(epoch) = es_format_to_epoch_ms(s, fmt) {- return Value::Number(serde_json::Number::from(epoch));- }- }- let bytes = s.as_bytes();- let looks_date = bytes.len() >= 5- && bytes[0].is_ascii_digit()- && bytes[1].is_ascii_digit()- && bytes[2].is_ascii_digit()- && bytes[3].is_ascii_digit()- && bytes[4] == b'-';- if looks_date {- let frac_digits = s.rsplit_once('.')- .map(|(_, rest)| rest.chars().take_while(|c| c.is_ascii_digit()).count())- .unwrap_or(0);- let is_nanos = frac_digits >= 4;- let s_utc = s.replace(' ', "T");- let s_utc = if s_utc.ends_with('Z') || s_utc.contains('+') {- s_utc- } else {- format!("{}Z", s_utc)- };- if is_nanos {- if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(&s_utc) {- let secs = dt.timestamp();- let nanos = dt.timestamp_subsec_nanos() as i64;- if let Some(ns) = secs.checked_mul(1_000_000_000).and_then(|x| x.checked_add(nanos)) {- return Value::Number(serde_json::Number::from(ns));- }- }- }- if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(&s_utc) {- return Value::Number(serde_json::Number::from(dt.timestamp_millis()));- }- if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&s_utc.trim_end_matches('Z'), "%Y-%m-%dT%H:%M:%S%.f") {- return Value::Number(serde_json::Number::from(dt.and_utc().timestamp_millis()));- }- if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(&s_utc.trim_end_matches('Z'), "%Y-%m-%dT%H:%M:%S") {- return Value::Number(serde_json::Number::from(dt.and_utc().timestamp_millis()));- }- }- }- v.clone()- }- let after_values_norm: Vec<Value> = after_values- .iter()- .enumerate()- .map(|(i, v)| {- let fmt = request- .sort- .get(i)- .and_then(|s| s.format.as_deref());- normalize_cursor(v, fmt)- })- .collect();- let after_values_ref: &[Value] = &after_values_norm;+ let after_values_norm: &[Value] = after_norm.as_deref().unwrap_or(&[]);+ let after_values_ref: &[Value] = after_values_norm;// Find the first hit whose sort values are strictly "after" the cursor.// For sorted results: skip until we find a hit where sort > after_values.// For score-only sort: after_values[0] is the score cursor.@@ -6443,6 +6519,176 @@ impl Index {Some(matching.into_iter().collect())}+ /// Sorted-DV candidate pruning for field-sorted `match_all` queries+ /// (the search_after hot-path fix).+ ///+ /// For a segment with a numeric doc-values column on the PRIMARY sort+ /// field, the column's sorted `(value, position)` index tells us exactly+ /// which positions can possibly reach the global top-`cap` page:+ /// - `asc`: the first `cap` positions at/after the cursor's primary+ /// value (cursor ties INCLUDED — the secondary sort key may still+ /// qualify them), plus all positions tied with the boundary value so+ /// the secondary key can arbitrate;+ /// - `desc`: symmetric from the top end.+ ///+ /// Every other position in the segment provably sorts outside the page+ /// (worse primary key than `cap` closer candidates) or at/before the+ /// cursor, so the stored scan can skip parsing it entirely. This is+ /// what turns a 1M-doc `search_after` page from an O(N) parse-everything+ /// scan into O(from+size) per segment.+ ///+ /// Returns `None` (→ caller falls back to the full scan, still correct)+ /// unless the shape is provably safe:+ /// - primary sort is a plain field (not `_score`/`_doc`) with a+ /// `Column::Numeric` sidecar in this segment;+ /// - the column has NO nulls (a null means a missing field, an array,+ /// or a non-scalar — `missing`/`mode` semantics would need the full+ /// scan) and covers exactly `seg_doc_count` docs (position alignment);+ /// - the `search_after` primary cursor value (when present) is numeric.+ ///+ /// The caller additionally gates on `is_match_all && !deletes_present`:+ /// deleted/superseded docs occupy positions in the sorted index, so with+ /// deletes present the candidate slice could under-fill the page.+ fn build_sort_candidates_prefilter(+ &self,+ segments_dir: &std::path::Path,+ segment_id: &str,+ seg_doc_count: u64,+ topk: &SortTopK,+ cap: usize,+ ) -> Option<HashSet<u32>> {+ use xerj_query::sort::SortOrder;+ use xerj_storage::doc_values::Column;++ let sf = topk.fields.first()?;+ if sf.is_score() || sf.is_doc_order() {+ return None;+ }+ let cols = self.dv_columns_for(segments_dir, segment_id)?;+ let Some(Column::Numeric(n)) = cols.get(&sf.field) else {+ return None;+ };+ if !n.null_bitmap.is_empty() || n.doc_count as u64 != seg_doc_count {+ return None;+ }+ let sorted = &n.sorted;+ if sorted.is_empty() {+ return None;+ }++ // Primary cursor bound (search_after). `None` cursor → page 1.+ let cursor_v: Option<f64> = match topk.after.as_deref() {+ Some(vals) => match vals.first() {+ Some(Value::Number(x)) => Some(x.as_f64()?),+ // Non-numeric cursor against a numeric column — don't+ // guess, take the full scan.+ Some(_) => return None,+ None => None,+ },+ None => None,+ };++ let len = sorted.len();+ let cap = cap.max(1);+ let (start, end) = if sf.order == SortOrder::Asc {+ let lo = match cursor_v {+ // Include cursor ties: `< v0` keeps every position whose+ // value equals the cursor's primary value; the collector's+ // full-key comparison decides via the secondary fields.+ Some(v0) => sorted.partition_point(|(b, _)| f64::from_bits(*b as u64) < v0),+ None => 0,+ };+ let mut end = (lo + cap).min(len);+ // Extend across boundary ties so the secondary sort key can+ // arbitrate between equal primary values.+ if end > lo && end < len {+ let boundary = sorted[end - 1].0;+ while end < len && sorted[end].0 == boundary {+ end += 1;+ }+ }+ (lo, end)+ } else {+ let hi = match cursor_v {+ // Include cursor ties from the other side: `<= v0` keeps+ // positions with value == v0 inside the slice.+ Some(v0) => sorted.partition_point(|(b, _)| f64::from_bits(*b as u64) <= v0),+ None => len,+ };+ let mut start = hi.saturating_sub(cap);+ if start < hi && start > 0 {+ let boundary = sorted[start].0;+ while start > 0 && sorted[start - 1].0 == boundary {+ start -= 1;+ }+ }+ (start, hi)+ };++ Some(sorted[start..end].iter().map(|(_, d)| *d).collect())+ }++ /// Hydrate the sorted-DV candidate positions of one segment from the+ /// warm `stored_slices_cache` — the fast twin of the+ /// `scan_stored_section_into` sort-topk branch (same version-map delete+ /// check, same dedup, same pre-heap cursor/worst rejection, same+ /// `total_count` tally for parsed live candidates), minus the per-query+ /// decompress and the O(segment-bytes) brace scan.+ ///+ /// Only called on the `is_match_all` sorted-candidates path, so hits+ /// score 1.0 and every live candidate matches by construction.+ fn hydrate_sorted_candidates(+ &self,+ slices: &StoredSlices,+ candidates: &HashSet<u32>,+ topk: &mut SortTopK,+ seen_ids: &mut HashSet<String>,+ total_count: &mut u64,+ ) {+ // Ascending positions for forward locality over `slices.bytes`.+ let mut positions: Vec<u32> = candidates.iter().copied().collect();+ positions.sort_unstable();+ for pos in positions {+ let Some(&(start, end)) = slices.offsets.get(pos as usize) else {+ continue;+ };+ let Some(slice) = slices.bytes.get(start as usize..end as usize) else {+ continue;+ };+ let mut doc_buf = slice.to_vec();+ let doc: Value = match simd_json::serde::from_slice(&mut doc_buf) {+ Ok(v) => v,+ Err(_) => continue,+ };+ let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");+ if let Some(ver) = self.store.version_map.get(id_ref) {+ if ver.deleted {+ continue;+ }+ }+ *total_count += 1;+ if seen_ids.contains(id_ref) {+ continue;+ }+ let source_ref = doc.get("_source").unwrap_or(&doc);+ let key = compute_sort_values(source_ref, 1.0, id_ref, topk.fields.as_slice());+ if !topk.would_admit(&key) {+ continue;+ }+ let id = id_ref.to_string();+ seen_ids.insert(id.clone());+ topk.offer_keyed(Hit {+ id,+ score: 1.0,+ source: source_ref.clone(),+ sort: key,+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ });+ }+ }+/// Version of `try_aggs_fast_with_segments` that caches decoded columns/// per segment on `self.dv_cache`. Segments are immutable once flushed,/// so a decoded `Arc<BTreeMap<String, Column>>` stays valid until the@@ -6503,6 +6749,13 @@ impl Index {seen_ids: &mut HashSet<String>,pre_filter: Option<&HashSet<u32>>,mut sort_topk: Option<&mut SortTopK>,+ // When `Some`, every doc's `(start, end)` byte range is appended as+ // the brace-scan walks the section, so the caller can cache the+ // decompressed bytes + offsets (`stored_slices_cache`) and hydrate+ // future sorted-candidate queries without re-scanning. Callers must+ // verify `offsets_out.len() == segment doc_count` before caching —+ // a malformed section bails early and leaves the vec incomplete.+ mut offsets_out: Option<&mut Vec<(u32, u32)>>,) {// Fast path: scan one doc at a time with a hand-rolled array splitter.// The stored section is always shaped as `[{...}, {...}, ...]` (a@@ -6574,6 +6827,10 @@ impl Index {return;}+ if let Some(out) = offsets_out.as_deref_mut() {+ out.push((start as u32, end as u32));+ }+let cur_pos = doc_pos;doc_pos += 1;@@ -6639,23 +6896,38 @@ impl Index {// heap: no early stop (we must visit ALL matches so the survivors// are the GLOBAL top-N by the sort key), and retention is bounded// by the heap capacity rather than by first-N scan order.+ //+ // Hot-path discipline (search_after OOM fix): compute the sort+ // key on the BORROWED source first and reject pre-heap — hits at+ // or before the `search_after` cursor, and hits that cannot beat+ // the current heap-worst, are dropped WITHOUT cloning `_source`+ // and WITHOUT an id insert. Pre-fix this branch cloned every+ // matching doc's source (~1 KB × 1M docs ≈ 1 GB churn/query) and+ // grew `seen_ids` to one String per match; under piled-up+ // concurrent search_after queries that ballooned RSS to an OOM+ // kill. `seen_ids` now only tracks OFFERED hits, which is+ // bounded by heap-admission traffic (≈ cap·ln(N) typical).if let Some(topk) = sort_topk.as_deref_mut() {- let id: String = id_ref.to_string();- if seen_ids.contains(&id) {+ if seen_ids.contains(id_ref) {continue;}- let source = doc.get("_source").cloned().unwrap_or_else(|| doc.clone());+ let source_ref = doc.get("_source").unwrap_or(&doc);let score = if is_match_all {1.0} else {- score_query_against_doc(query, &source)+ score_query_against_doc(query, source_ref)};+ let key = compute_sort_values(source_ref, score, id_ref, topk.fields.as_slice());+ if !topk.would_admit(&key) {+ continue;+ }+ let id: String = id_ref.to_string();seen_ids.insert(id.clone());- topk.offer(Hit {+ topk.offer_keyed(Hit {id,score,- source,- sort: Vec::new(),+ source: source_ref.clone(),+ sort: key,explain: None,highlight: None,matched_queries: Vec::new(),@@ -11022,20 +11294,73 @@ struct SortTopK {heap: std::collections::BinaryHeap<SortHeapEntry>,fields: Arc<Vec<xerj_query::sort::SortField>>,cap: usize,+ /// Normalized `search_after` cursor. Hits whose sort key is NOT+ /// strictly greater than this key (in the request's sort order) are+ /// rejected BEFORE touching the heap — ES cursor semantics: a page+ /// starts strictly after the cursor. Applying the cursor during+ /// collection (rather than post-hoc on the drained top-N) is what+ /// makes deep cursors both correct (a cursor past the materialisation+ /// cap used to yield an empty/wrong page, because the collector had+ /// already discarded everything past the top-(from+size)) and cheap+ /// (page N costs the same as page 1).+ after: Option<Vec<Value>>,}impl SortTopK {- fn new(fields: Arc<Vec<xerj_query::sort::SortField>>, cap: usize) -> Self {+ fn new(+ fields: Arc<Vec<xerj_query::sort::SortField>>,+ cap: usize,+ after: Option<Vec<Value>>,+ ) -> Self {Self {heap: std::collections::BinaryHeap::new(),fields,cap: cap.max(1),+ after,+ }+ }+ /// Cheap pre-admission check on a computed sort key. Returns `false`+ /// when the hit provably cannot appear in the result page:+ /// - it sorts at or before the `search_after` cursor, or+ /// - the heap is full and the hit sorts strictly after the current+ /// worst (key-only comparison; key ties are admitted and resolved+ /// by the heap's `_id` tie-break, matching the pre-existing+ /// push-then-evict semantics).+ ///+ /// Callers on the hot scan path use this BEFORE cloning `_source`, so+ /// rejected hits cost zero allocations.+ fn would_admit(&self, key: &[Value]) -> bool {+ if let Some(after) = &self.after {+ if xerj_query::sort::compare_sort_keys(key, after, &self.fields)+ != std::cmp::Ordering::Greater+ {+ return false;+ }+ }+ if self.heap.len() >= self.cap {+ if let Some(worst) = self.heap.peek() {+ if xerj_query::sort::compare_sort_keys(key, &worst.hit.sort, &self.fields)+ == std::cmp::Ordering::Greater+ {+ return false;+ }+ }}+ true}/// Offer a fully-sourced hit. Computes its sort key from `_source`,- /// inserts into the bounded heap, and evicts the current worst (the- /// element that sorts last) when over capacity.+ /// then delegates to `offer_keyed`.fn offer(&mut self, mut hit: Hit) {hit.sort = compute_sort_values(&hit.source, hit.score, &hit.id, &self.fields);+ self.offer_keyed(hit);+ }+ /// Offer a hit whose `sort` key is already computed. Applies the+ /// cursor + heap-worst rejection, inserts into the bounded heap, and+ /// evicts the current worst (the element that sorts last) when over+ /// capacity.+ fn offer_keyed(&mut self, hit: Hit) {+ if !self.would_admit(&hit.sort) {+ return;+ }self.heap.push(SortHeapEntry {hit,fields: Arc::clone(&self.fields),@@ -11049,6 +11374,92 @@ impl SortTopK {}}+/// Decompressed stored section of one segment plus per-doc byte ranges,+/// cached so the sorted-DV candidate path can hydrate specific doc+/// positions without a per-query decompress + O(bytes) brace re-scan.+/// See `Index::stored_slices_cache`.+struct StoredSlices {+ bytes: Vec<u8>,+ /// `offsets[pos] = (start, end)` byte range of the doc at stored+ /// position `pos` inside `bytes`.+ offsets: Vec<(u32, u32)>,+}++impl StoredSlices {+ fn retained_bytes(&self) -> u64 {+ (self.bytes.len() + self.offsets.len() * 8 + 64) as u64+ }+}++/// Retained-memory budget for `stored_slices_cache`. Inserts stop once the+/// budget is reached (queries then fall back to the per-query decompress+/// path); merge eviction returns the dropped segment's bytes to the budget.+const STORED_SLICES_CACHE_BUDGET: u64 = 512 * 1024 * 1024;++/// Normalize one `search_after` cursor value so it compares against the+/// normalized `hit.sort` values `compute_sort_values` produces (date-shaped+/// strings → epoch-ms / epoch-ns numbers). `fmt_hint` is the SortField's+/// ES format pattern, tried first for custom formats like+/// `yyyy-MM-dd | HH:mm:ss.SSS` whose cursor strings the default date+/// detectors below don't recognize. Non-date values pass through unchanged.+fn normalize_search_after_value(v: &Value, fmt_hint: Option<&str>) -> Value {+ if let Some(s) = v.as_str() {+ if let Some(fmt) = fmt_hint {+ if let Some(epoch) = es_format_to_epoch_ms(s, fmt) {+ return Value::Number(serde_json::Number::from(epoch));+ }+ }+ let bytes = s.as_bytes();+ let looks_date = bytes.len() >= 5+ && bytes[0].is_ascii_digit()+ && bytes[1].is_ascii_digit()+ && bytes[2].is_ascii_digit()+ && bytes[3].is_ascii_digit()+ && bytes[4] == b'-';+ if looks_date {+ let frac_digits = s+ .rsplit_once('.')+ .map(|(_, rest)| rest.chars().take_while(|c| c.is_ascii_digit()).count())+ .unwrap_or(0);+ let is_nanos = frac_digits >= 4;+ let s_utc = s.replace(' ', "T");+ let s_utc = if s_utc.ends_with('Z') || s_utc.contains('+') {+ s_utc+ } else {+ format!("{}Z", s_utc)+ };+ if is_nanos {+ if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(&s_utc) {+ let secs = dt.timestamp();+ let nanos = dt.timestamp_subsec_nanos() as i64;+ if let Some(ns) = secs+ .checked_mul(1_000_000_000)+ .and_then(|x| x.checked_add(nanos))+ {+ return Value::Number(serde_json::Number::from(ns));+ }+ }+ }+ if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(&s_utc) {+ return Value::Number(serde_json::Number::from(dt.timestamp_millis()));+ }+ if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(+ s_utc.trim_end_matches('Z'),+ "%Y-%m-%dT%H:%M:%S%.f",+ ) {+ return Value::Number(serde_json::Number::from(dt.and_utc().timestamp_millis()));+ }+ if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(+ s_utc.trim_end_matches('Z'),+ "%Y-%m-%dT%H:%M:%S",+ ) {+ return Value::Number(serde_json::Number::from(dt.and_utc().timestamp_millis()));+ }+ }+ }+ v.clone()+}+/// Compute the ES sort-value array for a hit from its `_source` (+ score / id/// for the `_score` / `_doc` pseudo-fields). Shared by the bounded/// `SortTopK` collector (selection) and the final `final_hits.sort_by`