AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf+fix(search): ES-exact columnar scoring for dis_max/boosting/pinned/scored-bool
xerj-orgfc85501e423d
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 83a4648fb..bb4e5e2e1 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -542,6 +542,16 @@ pub struct Index {/// segment id at the merge-completion site alongside the other caches.id_pos_cache: Arc<dashmap::DashMap<String, Arc<std::collections::HashMap<String, u32>>>>,+ /// Per-segment `stored-position → seq_no` map for the scored-family+ /// columnar fast path. Segment ROW order is NOT insertion order (the+ /// 16-shard memtable flush + merges interleave rows), but ES's score+ /// tie-break is Lucene doc id = arrival order = our seq_no — so tied+ /// top-k selection must rank rows by seq, not by row index. Built once+ /// per segment from the (cached) `id_pos_map_for` + version map; only+ /// consulted under `!deletes_present` (seq of a live id is then stable),+ /// and evicted by segment id at the merge-completion site.+ row_seq_cache: Arc<dashmap::DashMap<String, Arc<Vec<u64>>>>,+/// Per-segment cache of decoded `Vec<Value>` from the stored section./// KNN search and segment-scan get-document paths used to call/// `decode_stored` + `simd_json::serde::from_slice` *every* time —@@ -820,6 +830,7 @@ impl Index {stored_slices_build_locks: Arc::new(dashmap::DashMap::new()),range_prefilter_cache: Arc::new(dashmap::DashMap::new()),id_pos_cache: Arc::new(dashmap::DashMap::new()),+ row_seq_cache: Arc::new(dashmap::DashMap::new()),stored_value_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),@@ -996,6 +1007,7 @@ impl Index {stored_slices_build_locks: Arc::new(dashmap::DashMap::new()),range_prefilter_cache: Arc::new(dashmap::DashMap::new()),id_pos_cache: Arc::new(dashmap::DashMap::new()),+ row_seq_cache: Arc::new(dashmap::DashMap::new()),stored_value_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),@@ -3078,6 +3090,7 @@ impl Index {for id in &dropped_seg_ids {self.dv_cache.remove(id.as_str());self.id_pos_cache.remove(id.as_str());+ self.row_seq_cache.remove(id.as_str());self.stored_value_cache.remove(id.as_str());if let Some((_, slices)) = self.stored_slices_cache.remove(id.as_str()) {self.stored_slices_cache_bytes@@ -5553,18 +5566,44 @@ impl Index {// a token like "claude" can never match the keyword term// "claude-haiku-4-5", which made multi_match / query_string /// simple_query_string return 0 hits on keyword-only mappings.- let (text_fields, exact_fields): (Vec<String>, std::collections::HashSet<String>) = {+ // `kw_fields` / `num_fields` / `bool_fields` are the schema-typed+ // field sets the scored-family fast-path gate needs: `exact_fields`+ // alone can't prove "keyword" (it also holds numerics/dates/bools),+ // and Text fields must NEVER enter the columnar scorer (they get+ // Keyword dv columns but ES analyzes them).+ let (text_fields, exact_fields, kw_fields, num_fields, bool_fields): (+ Vec<String>,+ std::collections::HashSet<String>,+ std::collections::HashSet<String>,+ std::collections::HashSet<String>,+ std::collections::HashSet<String>,+ ) = {let schema_guard = self.schema.read().await;let mut tf: Vec<String> = Vec::new();let mut ef: std::collections::HashSet<String> = std::collections::HashSet::new();+ let mut kw: std::collections::HashSet<String> = std::collections::HashSet::new();+ let mut num: std::collections::HashSet<String> = std::collections::HashSet::new();+ let mut bf: std::collections::HashSet<String> = std::collections::HashSet::new();for f in &schema_guard.schema.fields {if matches!(f.field_type, FieldType::Text) {tf.push(f.name.clone());} else {ef.insert(f.name.clone());}+ match f.field_type {+ FieldType::Keyword => {+ kw.insert(f.name.clone());+ }+ FieldType::Long | FieldType::Double => {+ num.insert(f.name.clone());+ }+ FieldType::Boolean => {+ bf.insert(f.name.clone());+ }+ _ => {}+ }}- (tf, ef)+ (tf, ef, kw, num, bf)};// ── Fast path: count-only shortcut ────────────────────────────────@@ -5624,7 +5663,26 @@ impl Index {// instead of walking every match to tally `hits.total`. For query// shapes the shortcut can't resolve it returns `None` (cheaply) and we// fall back to the full counting scan — so this is always safe.- let shortcut_count: Option<u64> = self.try_shortcut_count(query, &snap, is_match_all).await;+ // Probe the scored-family columnar fast path EARLY (the executor+ // itself runs after the agg fast-paths below): when it is going to+ // run, `try_shortcut_count` is dead weight — the executor supplies+ // the exact `hits.total` itself — yet costs several ms on some+ // shapes (live-measured: a boolean-column `term` under `must_not`+ // spent ~6 ms here, dominating the fast path's single-digit budget).+ // If the executor later bails at runtime (missing column, etc.) the+ // stored scan simply runs in full-counting mode — correct, just not+ // F1-bounded.+ let scored_plan: Option<ScoredPlan> =+ scored_fast_plan(query, request, size, &kw_fields, &num_fields, &bool_fields);+ let scored_fast_ready: bool = scored_plan.is_some()+ && !(snap.segments.iter().any(|m| m.has_tombstones)+ || self.store.version_map.ghost_events() > 0)+ && self.memtable.doc_count() == 0;+ let shortcut_count: Option<u64> = if scored_fast_ready {+ None+ } else {+ self.try_shortcut_count(query, &snap, is_match_all).await+ };// Whether this query resolves to an FTS (inverted-index) search. The// FTS scored path already counts authoritatively via `seg_hits.len()`@@ -5754,6 +5812,33 @@ impl Index {}}+ // ── Columnar scored-family fast path ──────────────────────────────+ // `dis_max` / `boosting` / `pinned` / scoring `bool` on keyword+ // fields, scored with EXACT ES BM25 (global term stats from the+ // doc-values columns) instead of the IDF-less brute scorer, with the+ // global top-k selected by (score, insertion order) — see+ // `scored_fast_plan` (shape gate) + `scored_columnar` (executor).+ // Bails (leaving `scored_fast_applied = false`) for any uncovered+ // shape, deletes, non-empty memtable, or missing/nulled columns —+ // the brute path below then behaves byte-for-byte as before.+ let mut scored_fast_applied = false;+ if let Some(plan) = &scored_plan {+ if scored_fast_ready && !deletes_present {+ if let Some((hits, total)) = self.scored_columnar(+ plan,+ &snap.segments,+ &segments_dir,+ materialisation_limit,+ ) {+ seen_ids.clear();+ seen_ids.extend(hits.iter().map(|h| h.id.clone()));+ all_hits = hits;+ total_count = total;+ scored_fast_applied = true;+ }+ }+ }+if precomputed_aggs.is_some() {// Skip the entire segment loop — the fast agg path already// computed everything we need. `total_count` is set above.@@ -5776,10 +5861,10 @@ impl Index {// ever returns None for MatchAll).let seg_docs: u64 = snap.segments.iter().map(|m| m.doc_count).sum();total_count += seg_docs;- } else if fs_fast_applied {- // Columnar function_score fast path already produced the global- // top-k (`all_hits`) and the exact live total (`total_count`);- // skip the stored-doc scan entirely.+ } else if fs_fast_applied || scored_fast_applied {+ // A columnar fast path (function_score fvf or scored-family)+ // already produced the global top-k (`all_hits`) and the exact+ // total (`total_count`); skip the stored-doc scan entirely.} else {// Decide up-front whether this query needs FTS side-cars at all.// For term/range/etc. queries we skip FtsIndexReader::open@@ -6628,7 +6713,14 @@ impl Index {// --- Apply pinned query: boost pinned IDs to the top ---// Assign pinned IDs a score higher than any organic result so they sort first.- if let QueryNode::Pinned { ids, .. } = query {+ // Skipped when the scored-family columnar path ran: it already+ // assigned the EXACT ES pin scores (1.7014…e38 family) and this+ // rewrite would clobber them with the legacy max_organic+1+rank+ // fakes (it also only sees hits already inside the bounded+ // collector, which is why pins never floated on the brute path).+ if scored_fast_applied {+ // exact pin scores already in place+ } else if let QueryNode::Pinned { ids, .. } = query {let max_organic_score = final_hits.iter().map(|h| h.score)@@ -6766,7 +6858,13 @@ impl Index {// docs get 1.0" and "docs with rare terms rank higher" for// diversified_sampler / top_hits / 115_multi / interval_query// ordering.- if !final_hits.is_empty() && request.sort.is_empty() && query_uses_bool_text(&request.query)+ // (`!scored_fast_applied`: the columnar scored path already produced+ // exact ES BM25 scores — this heuristic rescore would rewrite+ // range-less scoring bools with ≥2 clauses.)+ if !scored_fast_applied+ && !final_hits.is_empty()+ && request.sort.is_empty()+ && query_uses_bool_text(&request.query){let field_term_pairs = collect_match_field_terms(&request.query);if !field_term_pairs.is_empty() {@@ -6812,7 +6910,11 @@ impl Index {// --- Score normalization: normalize BM25 scores to [0, max_score] ---// When scores are extremely low (BM25 near-zero), fall back to simple TF-IDF scoring.- if !final_hits.is_empty() && request.sort.is_empty() {+ // (`!scored_fast_applied`: a term with n ≈ N legitimately scores+ // < 0.001 under exact BM25 — e.g. status:ok over the 100k bench+ // corpus scores 0.012563465, but boosting×0.3 shapes go lower — and+ // this data-dependent fallback would silently rewrite exact scores.)+ if !scored_fast_applied && !final_hits.is_empty() && request.sort.is_empty() {let max_score = final_hits.iter().map(|h| h.score)@@ -6928,6 +7030,7 @@ impl Index {total_count = self.live_doc_count();} else if !count_only&& !query_needs_fts+ && !scored_fast_applied&& (scan_hit_cap || sort_candidates_narrowed)&& !deletes_present{@@ -9272,6 +9375,470 @@ impl Index {Some((hits, total))}+ /// Columnar exact-BM25 executor for the scored family+ /// (`dis_max` / `boosting` / `pinned` / scoring `bool`) — the shapes+ /// admitted by [`scored_fast_plan`].+ ///+ /// Walks every segment row straight from the doc-values columns,+ /// computes the EXACT ES score per matching row (per-clause BM25+ /// constants from GLOBAL term stats — see [`bm25_keyword_term_score`] —+ /// combined per shape with ES/Lucene's float widths), keeps the global+ /// top-`cap` by `(score DESC, seq_no ASC)` (= arrival order = ES's+ /// Lucene-doc-id `_doc` tie-break; scores here are massively tied so+ /// score-only selection would return an arbitrary tied subset, and+ /// segment ROW order is NOT arrival order — the sharded memtable flush+ /// interleaves rows — so rows are ranked via the cached per-segment+ /// `row_seqs_for` map, the same key the caller's final sort re-derives+ /// per hit via `lookup_seq_no`), and hydrates+ /// `_source` only for the winners via the F2 random-access slice path.+ /// Also returns the EXACT match total (`track_total_hits: true`).+ ///+ /// Returns `None` — caller falls back to the unchanged brute scan — when+ /// the memtable is non-empty, any referenced column is missing /+ /// kind-mismatched / row-count-mismatched, any SCORING field has nulls+ /// (the null-free gate simultaneously guarantees `N == Σ doc_count` for+ /// the idf and excludes array-valued docs, which the column builder+ /// records as nulls), or a pinned id can't be resolved through a+ /// complete per-segment `_id → position` map. The caller additionally+ /// gates on `!deletes_present`, so every row is live and each `_id`+ /// occupies exactly one segment position.+ fn scored_columnar(+ &self,+ plan: &ScoredPlan,+ segments: &[xerj_storage::segment::SegmentMeta],+ segments_dir: &std::path::Path,+ cap: usize,+ ) -> Option<(Vec<Hit>, u64)> {+ use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn};++ // The memtable has no doc-values columns, and its rows would need+ // their own global-stats treatment — serve fully-flushed data only.+ if self.memtable.doc_count() != 0 {+ return None;+ }++ // Flat clause lists. Order matters: bool scores must-then-should.+ let scoring: Vec<&ScoredLeaf> = match plan {+ ScoredPlan::DisMax { clauses, .. } => clauses.iter().collect(),+ ScoredPlan::Boosting { positive, .. } => vec![positive],+ ScoredPlan::Pinned { organic, .. } => vec![organic],+ ScoredPlan::Bool { must, should, .. } => must.iter().chain(should.iter()).collect(),+ };+ let filters: Vec<&ScoredFilterLeaf> = match plan {+ ScoredPlan::Boosting { negative, .. } => vec![negative],+ ScoredPlan::Bool {+ filter, must_not, ..+ } => filter.iter().chain(must_not.iter()).collect(),+ _ => Vec::new(),+ };++ // ── Phase 1: per-segment columns + GLOBAL term stats ──────────────+ // idf uses the field's docCount as N; the null-free scoring-column+ // gate forces it to equal Σ segment doc_count (bench mapping has no+ // nulls, so this coincides with ES's field docCount).+ let mut seg_cols = Vec::with_capacity(segments.len());+ let mut seg_seqs: Vec<Arc<Vec<u64>>> = Vec::with_capacity(segments.len());+ for meta in segments {+ seg_cols.push(self.dv_columns_for(segments_dir, &meta.id)?);+ seg_seqs.push(self.row_seqs_for(meta.id.as_str(), meta.doc_count)?);+ }+ let mut total_docs: u64 = 0;+ let mut df: Vec<u64> = vec![0; scoring.len()];+ for (meta, cols) in segments.iter().zip(seg_cols.iter()) {+ total_docs += meta.doc_count;+ for (ci, leaf) in scoring.iter().enumerate() {+ let Some(Column::Keyword(k)) = cols.get(leaf.field.as_str()) else {+ return None;+ };+ if k.doc_count as u64 != meta.doc_count || !k.null_bitmap.is_empty() {+ return None;+ }+ if let Some(ord) = k.ord_for_term(&leaf.term) {+ df[ci] += k.per_ord_count.get(ord as usize).copied().unwrap_or(0) as u64;+ }+ }+ for fl in &filters {+ match fl {+ ScoredFilterLeaf::KeywordTerm { field, .. }+ | ScoredFilterLeaf::KeywordTerms { field, .. } => {+ let Some(Column::Keyword(k)) = cols.get(field.as_str()) else {+ return None;+ };+ if k.doc_count as u64 != meta.doc_count {+ return None;+ }+ }+ ScoredFilterLeaf::NumericWindow { field, .. } => {+ let Some(Column::Numeric(n)) = cols.get(field.as_str()) else {+ return None;+ };+ if n.doc_count as u64 != meta.doc_count {+ return None;+ }+ }+ }+ }+ }+ // Per-clause BM25 constant (exact ES float32 op sequence). A clause+ // whose term is absent everywhere (df = 0) never matches a row, so+ // its (unused) score value is irrelevant.+ let clause_scores: Vec<f32> = df+ .iter()+ .map(|&n| bm25_keyword_term_score(n, total_docs))+ .collect();++ // Per-row evaluators over one segment's columns.+ struct ScoreEval<'a> {+ col: &'a KeywordColumn,+ ord: Option<u32>,+ }+ impl ScoreEval<'_> {+ #[inline]+ fn matches(&self, row: u32) -> bool {+ // Scoring columns are verified null-free in phase 1.+ match self.ord {+ Some(o) => self.col.ords[row as usize] == o,+ None => false,+ }+ }+ }+ enum FilterEval<'a> {+ Kw {+ col: &'a KeywordColumn,+ // Hoisted `null_bitmap.is_empty()`: per-row `contains` on a+ // RoaringBitmap is the dominant cost of the walk; null-free+ // columns (the common case) skip it entirely.+ no_nulls: bool,+ ords: Vec<u32>, // sorted; empty = matches nothing here+ },+ Num {+ col: &'a NumericColumn,+ no_nulls: bool,+ min: f64,+ max: f64,+ min_inc: bool,+ max_inc: bool,+ },+ }+ impl FilterEval<'_> {+ #[inline]+ fn matches(&self, row: u32) -> bool {+ match self {+ FilterEval::Kw {+ col,+ no_nulls,+ ords,+ } => {+ (*no_nulls || !col.null_bitmap.contains(row))+ && ords.binary_search(&col.ords[row as usize]).is_ok()+ }+ FilterEval::Num {+ col,+ no_nulls,+ min,+ max,+ min_inc,+ max_inc,+ } => {+ if !*no_nulls && col.null_bitmap.contains(row) {+ return false;+ }+ // Numeric dv columns store the f64 bit-pattern.+ let v = f64::from_bits(col.data[row as usize] as u64);+ (if *min_inc { v >= *min } else { v > *min })+ && (if *max_inc { v <= *max } else { v < *max })+ }+ }+ }+ }++ // ── Phase 2: row walk — exact per-row score + exact total ─────────+ // Candidate = (score, seq_no, segment, row); seq_no is the ES-parity+ // tie-break key, (segment, row) locates the stored slice.+ let mut cands: Vec<(f32, u64, u32, u32)> = Vec::new();+ let mut total: u64 = 0;+ for (si, (meta, cols)) in segments.iter().zip(seg_cols.iter()).enumerate() {+ let seqs: &[u64] = &seg_seqs[si];+ let si = si as u32;+ let mut sev: Vec<ScoreEval> = Vec::with_capacity(scoring.len());+ for leaf in &scoring {+ let Some(Column::Keyword(k)) = cols.get(leaf.field.as_str()) else {+ return None;+ };+ sev.push(ScoreEval {+ col: k,+ ord: k.ord_for_term(&leaf.term),+ });+ }+ let mut fev: Vec<FilterEval> = Vec::with_capacity(filters.len());+ for fl in &filters {+ match fl {+ ScoredFilterLeaf::KeywordTerm { field, term } => {+ let Some(Column::Keyword(k)) = cols.get(field.as_str()) else {+ return None;+ };+ fev.push(FilterEval::Kw {+ col: k,+ no_nulls: k.null_bitmap.is_empty(),+ ords: k.ord_for_term(term).into_iter().collect(),+ });+ }+ ScoredFilterLeaf::KeywordTerms { field, terms } => {+ let Some(Column::Keyword(k)) = cols.get(field.as_str()) else {+ return None;+ };+ let mut ords: Vec<u32> =+ terms.iter().filter_map(|t| k.ord_for_term(t)).collect();+ ords.sort_unstable();+ ords.dedup();+ fev.push(FilterEval::Kw {+ col: k,+ no_nulls: k.null_bitmap.is_empty(),+ ords,+ });+ }+ ScoredFilterLeaf::NumericWindow {+ field,+ min,+ max,+ min_inc,+ max_inc,+ } => {+ let Some(Column::Numeric(n)) = cols.get(field.as_str()) else {+ return None;+ };+ fev.push(FilterEval::Num {+ col: n,+ no_nulls: n.null_bitmap.is_empty(),+ min: *min,+ max: *max,+ min_inc: *min_inc,+ max_inc: *max_inc,+ });+ }+ }+ }+ let doc_count = meta.doc_count as u32;+ match plan {+ ScoredPlan::DisMax { tie_breaker, .. } => {+ // Lucene DisjunctionMaxScorer: scoreSum accumulated in+ // double over the float32 clause scores, scoreMax kept+ // as float32, result+ // (float)(max + (sum − max) · tieBreaker).+ let tie = *tie_breaker as f64;+ for row in 0..doc_count {+ let mut sum = 0.0_f64;+ let mut max = 0.0_f32;+ let mut any = false;+ for (ci, ev) in sev.iter().enumerate() {+ if ev.matches(row) {+ any = true;+ let s = clause_scores[ci];+ sum += s as f64;+ if s > max {+ max = s;+ }+ }+ }+ if !any {+ continue;+ }+ total += 1;+ let score = (max as f64 + (sum - max as f64) * tie) as f32;+ cands.push((score, seqs[row as usize], si, row));+ }+ }+ ScoredPlan::Boosting { negative_boost, .. } => {+ // Docs matching only the negative do NOT match at all;+ // negative-matching positives take a plain f32 multiply+ // (verified bit-exact vs ES `_explain` "product of:").+ let ps = clause_scores[0];+ let (pos, neg) = (&sev[0], &fev[0]);+ for row in 0..doc_count {+ if !pos.matches(row) {+ continue;+ }+ total += 1;+ let score = if neg.matches(row) {+ ps * *negative_boost+ } else {+ ps+ };+ cands.push((score, seqs[row as usize], si, row));+ }+ }+ ScoredPlan::Pinned { .. } => {+ // Organic rows only; pins are overlaid in phase 4.+ let org = &sev[0];+ let s = clause_scores[0];+ for row in 0..doc_count {+ if org.matches(row) {+ total += 1;+ cands.push((s, seqs[row as usize], si, row));+ }+ }+ }+ ScoredPlan::Bool {+ must,+ should,+ filter,+ ..+ } => {+ let n_must = must.len();+ let n_filter = filter.len();+ // ES default minimum_should_match: 1 when the bool has+ // ONLY should (no must/filter — must_not doesn't count),+ // else 0.+ let required_should: usize =+ usize::from(n_must == 0 && n_filter == 0 && !should.is_empty());+ 'rows: for row in 0..doc_count {+ for ev in &sev[..n_must] {+ if !ev.matches(row) {+ continue 'rows;+ }+ }+ for (fi, ev) in fev.iter().enumerate() {+ let m = ev.matches(row);+ if fi < n_filter {+ // filter clause: must match, contributes 0.+ if !m {+ continue 'rows;+ }+ } else if m {+ // must_not clause: must NOT match.+ continue 'rows;+ }+ }+ // Σ in f64 of the f32 clause scores (must, then+ // matching should), single cast to f32 — Lucene's+ // double-accumulating sum scorer.+ let mut sum = 0.0_f64;+ for &cs in &clause_scores[..n_must] {+ sum += cs as f64;+ }+ let mut should_matched = 0usize;+ for ci in n_must..sev.len() {+ if sev[ci].matches(row) {+ should_matched += 1;+ sum += clause_scores[ci] as f64;+ }+ }+ if should_matched < required_should {+ continue 'rows;+ }+ total += 1;+ cands.push((sum as f32, seqs[row as usize], si, row));+ }+ }+ }+ }++ // ── Phase 3: global top-`cap` by (score DESC, seq_no ASC) ─────────+ // The composite key is a TOTAL order (live seq_nos are unique under+ // the caller's `!deletes_present` gate), so the kept SET is exactly+ // ES's: arrival order == Lucene `_doc` order. Final page ordering is+ // re-derived by the caller's (score desc, seq_no asc) sort, which+ // uses the same key per hit via `lookup_seq_no`.+ let cmp = |a: &(f32, u64, u32, u32), b: &(f32, u64, u32, u32)| {+ b.0.partial_cmp(&a.0)+ .unwrap_or(std::cmp::Ordering::Equal)+ .then_with(|| a.1.cmp(&b.1))+ };+ let keep = cap.min(cands.len());+ if keep > 0 && cands.len() > keep {+ cands.select_nth_unstable_by(keep - 1, cmp);+ cands.truncate(keep);+ }++ // ── Phase 4: pinned overlay ───────────────────────────────────────+ // A pinned doc at 0-based position `i` of the P-length ids array+ // gets the CONSTANT score bits `0x7F00_0001 + (P − i)` (base+ // Float.MAX_VALUE/2; verified against ES for P = 1..4). Positions+ // count over the FULL ids array including ids that don't exist; a+ // missing id simply produces no hit. Pinned ids that exist are+ // ALWAYS hits — organic non-matches join the result (total += 1),+ // organic matches are re-scored in place (total unchanged).+ if let ScoredPlan::Pinned { ids, organic } = plan {+ if ids.len() > cap {+ return None;+ }+ let p = ids.len() as u32;+ let mut maps = Vec::with_capacity(segments.len());+ for meta in segments {+ maps.push(self.id_pos_map_for(meta.id.as_str(), meta.doc_count)?);+ }+ for (i, id) in ids.iter().enumerate() {+ let mut found: Option<(u32, u32)> = None;+ for (si, map) in maps.iter().enumerate() {+ if let Some(&pos) = map.get(id) {+ found = Some((si as u32, pos));+ break;+ }+ }+ let Some((si, row)) = found else {+ continue; // id not in the index → no hit, no shift+ };+ let pin_score = f32::from_bits(0x7F00_0001_u32 + (p - i as u32));+ if let Some(c) = cands.iter_mut().find(|c| c.2 == si && c.3 == row) {+ // Organic match that survived selection → re-score.+ c.0 = pin_score;+ } else {+ // Either an organic match truncated out of the top-`cap`+ // (already counted in `total`) or a pure pin (count it).+ let organic_matched = {+ let Some(Column::Keyword(k)) =+ seg_cols[si as usize].get(organic.field.as_str())+ else {+ return None;+ };+ k.ord_for_term(&organic.term)+ .is_some_and(|o| k.ords[row as usize] == o)+ };+ if !organic_matched {+ total += 1;+ }+ cands.push((pin_score, seg_seqs[si as usize][row as usize], si, row));+ }+ }+ // Pins outscore everything finite — re-rank + re-truncate.+ cands.sort_unstable_by(cmp);+ cands.truncate(cap);+ }++ // ── Phase 5: hydrate winners via F2 offset slices ─────────────────+ // Identical to `function_score_columnar` hydration, except each hit+ // carries its COMPUTED score (there is no post-loop re-applier for+ // these shapes; the caller's score sort finalises the order).+ let mut hits: Vec<Hit> = Vec::with_capacity(cands.len());+ let mut seen: HashSet<String> = HashSet::with_capacity(cands.len());+ for (score, _seq, si, row) in cands {+ let meta = &segments[si as usize];+ let slices = self.stored_slices_for(meta.id.as_str(), meta.doc_count)?;+ let &(start, end) = slices.offsets.get(row as usize)?;+ let slice = slices.bytes.get(start as usize..end as usize)?;+ let doc: Value = serde_json::from_slice(slice).ok()?;+ let id = doc+ .get("_id")+ .and_then(Value::as_str)+ .unwrap_or("")+ .to_string();+ if id.is_empty() || !seen.insert(id.clone()) {+ continue;+ }+ let source = doc.get("_source").cloned().unwrap_or(doc);+ hits.push(Hit {+ id,+ score,+ source,+ sort: Vec::new(),+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ });+ }+ Some((hits, total))+ }+/// Cache-backed reader for a segment's parsed stored section.////// First call for a segment opens the segment, decodes the Stored@@ -9545,6 +10112,25 @@ impl Index {Some(arc)}+ /// Lazily-built, cached `stored-position → seq_no` array for a segment+ /// (see `row_seq_cache`). Returns `None` when the segment has no complete+ /// id index or any id has no live version-map entry — callers then bail+ /// to the brute path. Only meaningful under `!deletes_present`.+ fn row_seqs_for(&self, seg_id: &str, expect_docs: u64) -> Option<Arc<Vec<u64>>> {+ if let Some(entry) = self.row_seq_cache.get(seg_id) {+ return Some(Arc::clone(entry.value()));+ }+ let map = self.id_pos_map_for(seg_id, expect_docs)?;+ let mut seqs: Vec<u64> = vec![u64::MAX; expect_docs as usize];+ for (id, &pos) in map.iter() {+ seqs[pos as usize] = self.lookup_seq_no(id)?;+ }+ let arc = Arc::new(seqs);+ self.row_seq_cache+ .insert(seg_id.to_string(), Arc::clone(&arc));+ Some(arc)+ }+/// Position pre-filter for an `ids` query: resolve each requested id to its/// stored position via the cached `_id → position` index. Returns/// `Some(∅)` when none of the ids live in this segment (caller skips it),@@ -18057,6 +18643,319 @@ fn function_score_fast_fvf<'q>(f.field_value_factor.as_ref()}+// ─────────────────────────────────────────────────────────────────────────────+// Scored-family columnar fast path (dis_max / boosting / pinned / scoring bool)+// ─────────────────────────────────────────────────────────────────────────────++/// A scoring leaf accepted by the scored-family columnar fast path: a+/// `term` (or `match`, which the keyword analyzer reduces to the identical+/// whole-value term) on a keyword-typed field with default boost. ES scores+/// it as BM25 with `tf = 1` and norms omitted — see+/// [`bm25_keyword_term_score`].+struct ScoredLeaf {+ field: String,+ term: String,+}++/// A filter-context leaf accepted by the scored-family columnar fast path.+/// Filter leaves never contribute to the score — they only constrain which+/// rows match (`filter` / `must_not` / `boosting.negative`).+enum ScoredFilterLeaf {+ /// `term`/`match` on a keyword field (single whole-value term).+ KeywordTerm { field: String, term: String },+ /// `terms` on a keyword field (OR over whole-value terms).+ KeywordTerms { field: String, terms: Vec<String> },+ /// `term` on a numeric/boolean field, or `range` on a numeric field —+ /// both evaluate as an inclusive/exclusive f64 window over the numeric+ /// doc-values column (a term is the degenerate `[v, v]` window;+ /// booleans map to 1.0/0.0, matching the column encoding).+ NumericWindow {+ field: String,+ min: f64,+ max: f64,+ min_inc: bool,+ max_inc: bool,+ },+}++/// The exact query shape the scored-family columnar executor serves.+/// Built by [`scored_fast_plan`]; anything it can't express bails to the+/// unchanged brute path.+enum ScoredPlan {+ /// `dis_max` over scoring leaves: per doc,+ /// `score = max + tie_breaker · Σ(others)` over MATCHING clauses.+ DisMax {+ clauses: Vec<ScoredLeaf>,+ tie_breaker: f32,+ },+ /// `boosting`: positive must match; a doc also matching `negative` has+ /// its score multiplied by `negative_boost`.+ Boosting {+ positive: ScoredLeaf,+ negative: ScoredFilterLeaf,+ negative_boost: f32,+ },+ /// `pinned`: organic scoring leaf below constant-score pins.+ Pinned {+ ids: Vec<String>,+ organic: ScoredLeaf,+ },+ /// Scoring `bool`: `score = Σ` of matching must+should leaf scores;+ /// `filter`/`must_not` constrain only.+ Bool {+ must: Vec<ScoredLeaf>,+ should: Vec<ScoredLeaf>,+ filter: Vec<ScoredFilterLeaf>,+ must_not: Vec<ScoredFilterLeaf>,+ },+}++/// ES/Lucene-9 exact BM25 score of a `term` query on a keyword field+/// (`boost = 1`), for a term with document frequency `n` in a null-free+/// field over `total_docs` documents.+///+/// Keyword fields omit norms, so `dl = avgdl = 1` and `freq = 1`; with+/// `k1 = 1.2`, `b = 0.75` (ES `LegacyBM25Similarity` defaults) the whole+/// per-doc score collapses to a per-term CONSTANT. Lucene caches the+/// RECIPROCAL norm (`1 / (k1·((1−b)+b·dl/avgdl))`) and computes+/// `weight − weight/(1 + freq·normInv)` in float32 — which is algebraically+/// `weight·tf` but ROUNDS differently: every "equivalent" ordering+/// (weight·tf, double-precision intermediates) came out +1 ULP high on live+/// ES 8.13.4 probes. The op sequence below reproduces ES bit-for-bit+/// (verified: status:ok n=98752/N=100000 → 0.012563465 = 0x3c4dd6fe;+/// model:claude-haiku-4-5 n=27744 → 1.2821424 = 0x3fa41d3e).+fn bm25_keyword_term_score(n: u64, total_docs: u64) -> f32 {+ // idf in f64, single cast to f32 (matches Lucene's `(float)` on the+ // double-precision idf expression).+ let idf_f32 =+ (1.0_f64 + (total_docs as f64 - n as f64 + 0.5) / (n as f64 + 0.5)).ln() as f32;+ // weight = query_boost(1.0) · (k1+1) · idf, all float32.+ let weight = 2.2_f32 * idf_f32;+ // Reciprocal cached norm for the norms-omitted case: 1/(k1·1) = 1/1.2,+ // computed in float32 (bits 0x3f555555).+ let norm_inv = 1.0_f32 / 1.2_f32;+ // denom = 1 + freq·normInv with freq = 1 (bits 0x3feaaaab = 1.8333333).+ let denom = 1.0_f32 + 1.0_f32 * norm_inv;+ weight - weight / denom+}++/// Gate + extractor for the scored-family columnar fast path.+///+/// Returns `Some(plan)` iff the request is exactly a shape+/// [`Index::scored_columnar`] can serve byte-for-byte vs ES:+/// a plain top-k page (same request-plainness bail list as+/// [`function_score_fast_fvf`]) whose query is one of+///+/// 1. `dis_max` over scoring leaves (any `tie_breaker`);+/// 2. `boosting { positive: scoring leaf, negative: filter leaf,+/// negative_boost ≥ 0 }`;+/// 3. `pinned { ids (non-empty, no duplicates), organic: scoring leaf }`;+/// 4. `bool` whose `must`/`should` are scoring leaves (at least one —+/// filter-only bools stay on today's proven path), whose+/// `filter`/`must_not` are filter leaves, with no+/// `minimum_should_match`.+///+/// A scoring leaf is a `term`/`match` on a KEYWORD-typed field (schema+/// `FieldType::Keyword` — `exact_fields` is NOT sufficient, it also holds+/// numerics/dates) with boost absent or exactly 1.0. Filter leaves also+/// admit keyword `terms`, numeric/boolean `term`, and numeric `range` with+/// numeric bounds. Every other shape returns `None` → unchanged brute path.+fn scored_fast_plan(+ query: &QueryNode,+ request: &SearchRequest,+ size: usize,+ keyword_fields: &HashSet<String>,+ numeric_fields: &HashSet<String>,+ boolean_fields: &HashSet<String>,+) -> Option<ScoredPlan> {+ // Request-level plainness — identical to `function_score_fast_fvf`.+ if size == 0+ || request.aggs.is_some()+ || request.min_score.is_some()+ || !request.sort.is_empty()+ || !request.rescore.is_empty()+ || request.collapse.is_some()+ || request.search_after.is_some()+ || request.highlight.is_some()+ || request.explain+ || request.script_fields.is_some()+ || !request.fields.is_empty()+ {+ return None;+ }++ fn boost_ok(b: &Option<f32>) -> bool {+ b.map_or(true, |v| v == 1.0)+ }++ let scoring_leaf = |node: &QueryNode| -> Option<ScoredLeaf> {+ match node {+ QueryNode::Term {+ field,+ value: Value::String(s),+ boost,+ } if keyword_fields.contains(field) && boost_ok(boost) => Some(ScoredLeaf {+ field: field.clone(),+ term: s.clone(),+ }),+ QueryNode::Match {+ field,+ query,+ analyzer: None,+ boost,+ minimum_should_match: None,+ ..+ } if keyword_fields.contains(field) && boost_ok(boost) => Some(ScoredLeaf {+ field: field.clone(),+ term: query.clone(),+ }),+ _ => None,+ }+ };++ let filter_leaf = |node: &QueryNode| -> Option<ScoredFilterLeaf> {+ if let Some(l) = scoring_leaf(node) {+ return Some(ScoredFilterLeaf::KeywordTerm {+ field: l.field,+ term: l.term,+ });+ }+ match node {+ QueryNode::Terms {+ field,+ values,+ boost,+ } if keyword_fields.contains(field) && boost_ok(boost) => {+ let mut terms = Vec::with_capacity(values.len());+ for v in values {+ match v {+ Value::String(s) => terms.push(s.clone()),+ _ => return None,+ }+ }+ Some(ScoredFilterLeaf::KeywordTerms {+ field: field.clone(),+ terms,+ })+ }+ QueryNode::Term {+ field,+ value,+ boost,+ } if boost_ok(boost) => {+ let v: f64 = if numeric_fields.contains(field) {+ // `as_f64` is None for Bool/String values → bail (the+ // brute path keeps ES's coercion semantics).+ value.as_f64()?+ } else if boolean_fields.contains(field) {+ // Boolean dv columns store 1.0/0.0 — map the query value+ // ourselves (`as_f64` is None for `Value::Bool`).+ match value {+ Value::Bool(true) => 1.0,+ Value::Bool(false) => 0.0,+ _ => return None,+ }+ } else {+ return None;+ };+ Some(ScoredFilterLeaf::NumericWindow {+ field: field.clone(),+ min: v,+ max: v,+ min_inc: true,+ max_inc: true,+ })+ }+ QueryNode::Range {+ field,+ gte,+ gt,+ lte,+ lt,+ boost,+ } if numeric_fields.contains(field) && boost_ok(boost) => {+ let (min, min_inc) = match (gte, gt) {+ (Some(v), None) => (v.as_f64()?, true),+ (None, Some(v)) => (v.as_f64()?, false),+ (None, None) => (f64::NEG_INFINITY, true),+ _ => return None,+ };+ let (max, max_inc) = match (lte, lt) {+ (Some(v), None) => (v.as_f64()?, true),+ (None, Some(v)) => (v.as_f64()?, false),+ (None, None) => (f64::INFINITY, true),+ _ => return None,+ };+ Some(ScoredFilterLeaf::NumericWindow {+ field: field.clone(),+ min,+ max,+ min_inc,+ max_inc,+ })+ }+ _ => None,+ }+ };++ match query {+ QueryNode::DisMax {+ queries,+ tie_breaker,+ } if !queries.is_empty() => {+ let clauses: Vec<ScoredLeaf> =+ queries.iter().map(&scoring_leaf).collect::<Option<_>>()?;+ Some(ScoredPlan::DisMax {+ clauses,+ tie_breaker: *tie_breaker,+ })+ }+ QueryNode::Boosting {+ positive,+ negative,+ negative_boost,+ } if negative_boost.is_finite() && *negative_boost >= 0.0 => {+ Some(ScoredPlan::Boosting {+ positive: scoring_leaf(positive)?,+ negative: filter_leaf(negative)?,+ negative_boost: *negative_boost,+ })+ }+ QueryNode::Pinned { ids, organic } if !ids.is_empty() => {+ // Duplicate pinned ids would need ES's dedup semantics — bail.+ let mut seen: HashSet<&str> = HashSet::with_capacity(ids.len());+ if !ids.iter().all(|id| seen.insert(id.as_str())) {+ return None;+ }+ Some(ScoredPlan::Pinned {+ ids: ids.clone(),+ organic: scoring_leaf(organic)?,+ })+ }+ QueryNode::Bool {+ must,+ should,+ must_not,+ filter,+ minimum_should_match: None,+ } if !must.is_empty() || !should.is_empty() => {+ let m: Vec<ScoredLeaf> = must.iter().map(&scoring_leaf).collect::<Option<_>>()?;+ let s: Vec<ScoredLeaf> = should.iter().map(&scoring_leaf).collect::<Option<_>>()?;+ let f: Vec<ScoredFilterLeaf> =+ filter.iter().map(&filter_leaf).collect::<Option<_>>()?;+ let mn: Vec<ScoredFilterLeaf> =+ must_not.iter().map(&filter_leaf).collect::<Option<_>>()?;+ Some(ScoredPlan::Bool {+ must: m,+ should: s,+ filter: f,+ must_not: mn,+ })+ }+ _ => None,+ }+}+/// Default chunk size (characters) for auto-embed-on-ingest chunking./// `EmbeddingConfig` has no per-field chunk parameter yet, so this is a/// sensible fixed default (roughly a paragraph).