Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(query): pinned closed-form organic lane + ES-exact deep-page max_score & organic-boost drop

xerj-orgef10b029d8fa
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 03cfa2813..b738724f4 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -6880,6 +6880,25 @@ pub async fn search(
},
)
})
+ } else if search_req.from > 0 && search_req.size > 0 {
+ // ES's no-sort `max_score` is the top COLLECTED doc's score —
+ // the population max — which `from` skips past but does not
+ // erase (live-verified on ES 8.13.4: pinned/bool deep pages
+ // keep the global max even when every higher scorer sits
+ // before `from`, and a beyond-end page reports it with 0
+ // hits). xerj used to report the PAGE head, under-reporting
+ // whenever the head of the ranking was skipped. from == 0
+ // keeps the page-head read (it IS the max under score-DESC
+ // order — byte-identical); size == 0 keeps ES's null.
+ let page_head = merged_hits.first().map(|(_, h)| score_wire(h.score));
+ match (
+ merged_population_max.map(|m| score_wire(m as f32)),
+ page_head,
+ ) {
+ (Some(p), Some(h)) => Some(if p >= h { p } else { h }),
+ (Some(p), None) => Some(p),
+ (None, h) => h,
+ }
} else {
merged_hits.first().map(|(_, h)| score_wire(h.score))
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index aeba414f9..caf2a734d 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -10761,7 +10761,8 @@ impl Index {
let seq_asc = (break_bound.is_some()
|| constant_plan
|| flat_conjunction.is_some()
- || flat_bool2.is_some())
+ || flat_bool2.is_some()
+ || matches!(plan, ScoredPlan::Pinned { .. }))
&& seqs.windows(2).all(|w| w[0] <= w[1]);
let later_min_seq: u64 = segments[si + 1..]
.iter()
@@ -11115,6 +11116,47 @@ impl Index {
// Organic rows only; pins are overlaid in phase 4.
let org = &sev[0];
let s = clause_scores[0];
+ if seq_asc {
+ // Constant-organic fast lane (mirror of the
+ // `constant_plan` lane): Pinned bails on entry
+ // whenever any ghost event was recorded, so the
+ // closed-form column count IS the exact live
+ // total, and — every organic hit scoring the same
+ // per-term constant with rows seq-ascending — the
+ // first keep_target matches ARE this segment's
+ // k-best contribution (identical to what the
+ // full-walk heap kept: the seq-smallest cap
+ // matches). Phase 4 re-scores/adds pins with its
+ // own independent organic-match test on the
+ // columns, so stopping the walk early never
+ // loses a pin. Replaces the profiled 100k-row
+ // walk + ~98.7k heap pushes for a 10-doc page.
+ let seg_matches = org.count();
+ total += seg_matches;
+ if seg_matches > 0 {
+ let mut kept = 0usize;
+ for row in 0..doc_count {
+ if !org.matches(row) {
+ continue;
+ }
+ push_topk(
+ &mut heap,
+ keep_target,
+ TopKCand {
+ score: s,
+ seq: seqs[row as usize],
+ si,
+ row,
+ },
+ );
+ kept += 1;
+ if kept >= keep_target {
+ break;
+ }
+ }
+ }
+ continue;
+ }
for row in 0..doc_count {
if is_ghost(row) {
continue;
@@ -21630,9 +21672,17 @@ fn scored_fast_plan(
if !ids.iter().all(|id| seen.insert(id.as_str())) {
return None;
}
+ // ES's pinned DROPS every boost inside the organic query —
+ // Lucene's CappedScoreQuery wrapper does not propagate the
+ // boost into the organic weight (live-verified on ES 8.13.4:
+ // organic term^2.5, bool>term^2.5 and constant_score boost
+ // 1.7 all score UNBOOSTED 0.012563465 / 1.0). Mirror it by
+ // scoring the admitted organic leaf at boost 1.0.
+ let mut organic = scoring_leaf(organic, 1.0, &fs)?;
+ organic.boost = 1.0;
Some(ScoredPlan::Pinned {
ids: ids.clone(),
- organic: scoring_leaf(organic, 1.0, &fs)?,
+ organic,
})
}
// match_all / terms / exists as constant-score Filtered plans