Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): min_score opens the scoring path at size:0; external scalar N keeps ghosts (#193)

xerj-org22c654cc2c78
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 590ef201e..3626fdecc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -222,8 +222,10 @@ jobs:
done
echo "xerj failed to become healthy"; cat "$DATA/server.log"; exit 1
- # The runner exits non-zero if any case fails. The 3 skipped cases do not
- # count as failures, so a clean run gates the PR at 1326 passing / 0 failing.
+ # The runner exits non-zero if any case fails. Skipped cases do not count
+ # as failures, so the gate is 0 failures — not an exact pass total, which
+ # grows as cases are added (1365 passed / 3 skipped at the last measured
+ # run, per #189; this comment previously said 1326 and had drifted).
- name: Run ES-compat YAML suite (0 failures required)
working-directory: engine
run: cargo run --release -p es-yaml-runner -- --dir tests/es-compat-yaml/yaml
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index edabbbbb1..792455054 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -13161,6 +13161,18 @@ impl Index {
// post-pass over the rendered page (`apply_highlight`), so at `size: 0`
// — where there is no page — it has nothing to do, and a `size: 0` body
// must execute identically with and without it.
+ //
+ // `min_score` IS in this list (#193): the threshold applies to FINAL
+ // scores, so a `size:0 + min_score` count must score its matches
+ // exactly like `size:N` does — count-only mode would close the
+ // index-wide stats gate below (per-arm scores; a different threshold
+ // scale than the size:N page, user-visible since #192) and skip
+ // materialisation entirely, leaving the post-collection min_score
+ // filter nothing to subtract, i.e. a total that ignores the
+ // threshold. Same call ES makes: `QueryPhaseCollector` refuses the
+ // `Weight#count` shortcut whenever min_score is set and collects
+ // only docs with `scorer.score() >= minScore` (approach only,
+ // QueryPhaseCollector.java:77-79,123 — no code copied).
let need_hits_output: bool = size > 0;
let need_sources_for_post: bool = need_hits_output
|| !request.rescore.is_empty()
@@ -13169,7 +13181,8 @@ impl Index {
.sort
.iter()
.any(|sf| !sf.is_score() && !sf.is_doc_order())
- || request.aggs.is_some();
+ || request.aggs.is_some()
+ || request.min_score.is_some();
let count_only: bool = !need_sources_for_post;
// --- Memtable search ---
@@ -13661,8 +13674,14 @@ impl Index {
}
stats_cell
.get_or_init(|| {
- self.build_collection_stats(&snap, query, &text_fields, &exact_fields)
- .map(Arc::new)
+ self.build_collection_stats(
+ &snap,
+ query,
+ &text_fields,
+ &exact_fields,
+ mem_doc_count,
+ )
+ .map(Arc::new)
})
.clone()
};
@@ -19169,20 +19188,33 @@ impl Index {
/// The caller then keeps the historical per-arm statistics.
///
/// COST. Per segment this opens a STATS-ONLY reader
- /// (`FtsIndexReader::open_stats_only`): the FST is mmap'd and `.meta` is
- /// read, but the postings envelope (whole-file read + zstd decompress) and
- /// the norms table (O(docs)) are skipped — they are the expensive parts of
- /// a full open and the statistics never touch them. The pre-pass also
- /// cannot defeat the two read-path fast paths it sits near: F1's
- /// early-break requires `!query_needs_fts`, and the `term_doc_freq`
- /// count-only shortcut requires `count_only` — the caller gates this
- /// whole block off for both.
+ /// (`FtsIndexReader::open_stats_only`): the FST is mmap'd and the `.meta`
+ /// side-car is read AND ZFM4-decoded — a zstd decompress of the
+ /// `num_terms × 24`-byte records section, so O(field vocabulary), small
+ /// but not free (#193). What IS skipped are the two expensive parts of a
+ /// full open: the postings envelope (whole-file read + zstd decompress of
+ /// every posting list, O(index bytes)) and the norms table (O(docs)) —
+ /// the statistics never touch either. The pre-pass also cannot defeat
+ /// the two read-path fast paths it sits near: F1's early-break requires
+ /// `!query_needs_fts`, and the `term_doc_freq` count-only shortcut
+ /// requires `count_only` — the caller gates this whole block off for
+ /// both.
+ ///
+ /// `mem_doc_count` is the caller's ONE point-in-time memtable count —
+ /// captured next to the segment snapshot and already the input to the
+ /// `stats_gate_open` arm arithmetic (#193). Re-reading
+ /// `self.memtable.doc_count()` here made the include-the-memtable
+ /// decision from a SECOND, later reading that could disagree with the
+ /// gate's (and re-paid the 16-shard lock fan-out the capture exists to
+ /// avoid). The stats CONTENT still reads the live shards — exactly like
+ /// the memtable search arm it feeds, which holds no snapshot either.
fn build_collection_stats(
&self,
snap: &xerj_storage::index_store::IndexSnapshot,
query: &QueryNode,
text_fields: &[String],
exact_fields: &HashSet<String>,
+ mem_doc_count: usize,
) -> Option<xerj_fts::CollectionStats> {
// Widest (field × term) pre-pass we will pay for. A field-less
// `query_string` over a wide mapping can project hundreds of leaves;
@@ -19208,7 +19240,7 @@ impl Index {
// any token the segment projection never named (per-field analyzers
// can differ) joins `pairs` and gets its segment-side df in the single
// pass below, instead of the memtable silently keeping a local df.
- let mem_stats: Option<xerj_fts::CollectionStats> = if self.memtable.doc_count() > 0 {
+ let mem_stats: Option<xerj_fts::CollectionStats> = if mem_doc_count > 0 {
extract_query_text(query)
.and_then(|text| self.memtable.collection_stats(&text, &field_refs))
} else {
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 5b8a46582..0005b41fa 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -1218,8 +1218,27 @@ impl ShardedFtsMemtable {
//
// Delete-aware in both modes (Lucene/ES parity): the scoring `N`
// counts live docs AND tombstoned/superseded versions not yet merged
- // away. This is *only* the BM25 IDF `N` — hits.total and pagination
- // still use the live `doc_count()`.
+ // away — tantivy's scalar N is likewise physical, Σ `max_doc()` over
+ // segments (`Bm25StatisticsProvider for Searcher::total_num_docs`,
+ // tantivy/src/query/bm25.rs:38-45, MIT). This is *only* the BM25
+ // IDF `N` — hits.total and pagination still use the live
+ // `doc_count()`.
+ //
+ // The ghost-inclusive scalar is computed ONCE here because BOTH
+ // modes need the same value (#193): the external mode falls back to
+ // it for any scored field the supplied union does not carry (or
+ // carries with `total_docs == 0`), and using the live-only count
+ // there silently raised IDF on indices holding tombstoned or
+ // superseded versions — an unintended divergence from the `None`
+ // path that the previous comment ("scalar N is unused in this
+ // mode") wrongly waved off.
+ let ghost_inclusive_n: u64 = {
+ let mut n: u64 = self.doc_count() as u64;
+ for shard in &self.shards {
+ n += shard.read().ghost_docs;
+ }
+ n
+ };
let mut global_field_doc_count: std::collections::HashMap<String, u64> =
std::collections::HashMap::new();
let global_doc_count: u64;
@@ -1227,9 +1246,11 @@ impl ShardedFtsMemtable {
let term_global_df: std::collections::HashMap<(String, String), u64>;
match external {
Some(cs) => {
- // Scalar N is unused in this mode (every scored field is in
- // the per-field map), but keep it a sane non-zero fallback.
- global_doc_count = self.doc_count() as u64;
+ // Scalar N backs the per-field fallback (field absent from
+ // the union) — keep it ghost-inclusive, exactly like the
+ // `None` branch, so the two modes agree on IDF wherever
+ // the fallback engages.
+ global_doc_count = ghost_inclusive_n;
let mut avg = std::collections::HashMap::new();
for (fname, fs) in cs.field_iter() {
avg.insert(fname.clone(), fs.avg_field_length());
@@ -1250,11 +1271,7 @@ impl ShardedFtsMemtable {
term_global_df = df;
}
None => {
- let mut n: u64 = self.doc_count() as u64;
- for shard in &self.shards {
- n += shard.read().ghost_docs;
- }
- global_doc_count = n;
+ global_doc_count = ghost_inclusive_n;
let (field_total_len, df) = self.aggregate_shard_stats(&q_tokens, fields);
global_avg_field_len = field_total_len
.into_iter()
@@ -3968,3 +3985,85 @@ mod filtered_docs_arc_tests {
);
}
}
+
+#[cfg(test)]
+mod external_scalar_n_ghost_tests {
+ use super::*;
+ use serde_json::json;
+
+ /// #193 item 2 — the external-mode (`Some(cs)`) scalar `N` must be
+ /// ghost-inclusive, exactly like the historical `None` branch.
+ ///
+ /// The scalar only engages as the per-field fallback: a scored field the
+ /// supplied union does not carry. This test pins the seam directly with
+ /// ONE shard, so the local per-shard fallbacks for `df` and `avgdl`
+ /// coincide with the `None` branch's cross-shard aggregation by
+ /// construction, and the ghost carries NO occurrence of the queried
+ /// field — leaving the scalar `N` as the ONLY quantity the two modes
+ /// could disagree on. Before the fix the external branch used the
+ /// live-only `doc_count()` (2) where the `None` branch used
+ /// live + ghosts (3), so the same query on the same memtable scored
+ /// differently depending on which mode the engine happened to take.
+ #[test]
+ fn external_mode_scalar_n_matches_none_mode_under_ghosts() {
+ let registry = Arc::new(AnalyzerRegistry::default());
+ let mem = ShardedFtsMemtable::with_registry_and_shards(registry, 1);
+ let schema = Schema::empty();
+
+ // Two live docs carrying the queried term in `body`, one doomed doc
+ // that never mentions `body` at all.
+ mem.insert("a".into(), &json!({"body": "quicklist alpha"}), &schema, 1);
+ mem.insert(
+ "b".into(),
+ &json!({"body": "quicklist beta gamma delta"}),
+ &schema,
+ 2,
+ );
+ mem.insert(
+ "ghost".into(),
+ &json!({"other": "unrelated text"}),
+ &schema,
+ 3,
+ );
+ mem.remove("ghost");
+
+ // Union that does NOT carry `body` — the scored field falls back to
+ // the scalar N (plus local df/avgdl, identical across modes here).
+ let mut cs = xerj_fts::CollectionStats::new();
+ cs.add_field(
+ "other",
+ &xerj_fts::FieldStats {
+ total_docs: 1,
+ total_field_length: 2,
+ },
+ );
+
+ let boosts = std::collections::HashMap::new();
+ let (none_hits, none_total) =
+ mem.search_text_boosted_with_total("quicklist", &["body"], 10, &boosts);
+ let (ext_hits, ext_total) = mem.search_text_boosted_with_total_using(
+ "quicklist",
+ &["body"],
+ 10,
+ &boosts,
+ Some(&cs),
+ );
+
+ assert_eq!(none_total, 2, "sanity: two live matches");
+ assert_eq!(ext_total, none_total, "totals must agree between modes");
+ assert_eq!(none_hits.len(), 2);
+ assert_eq!(ext_hits.len(), 2);
+ for (n, e) in none_hits.iter().zip(ext_hits.iter()) {
+ assert_eq!(n.doc_id, e.doc_id, "hit order diverged between modes");
+ assert_eq!(
+ n.score.to_bits(),
+ e.score.to_bits(),
+ "{}: external-mode fallback scored {} vs None-mode {} — the \
+ scalar N dropped the ghost from IDF",
+ n.doc_id,
+ e.score,
+ n.score
+ );
+ }
+ }
+}
diff --git a/engine/crates/xerj-engine/tests/bm25_stats_are_index_wide.rs b/engine/crates/xerj-engine/tests/bm25_stats_are_index_wide.rs
index 3fdf09ce2..e02195781 100644
--- a/engine/crates/xerj-engine/tests/bm25_stats_are_index_wide.rs
+++ b/engine/crates/xerj-engine/tests/bm25_stats_are_index_wide.rs
@@ -199,6 +199,61 @@ async fn overwriting_a_document_does_not_change_its_rank() {
}
}
+/// #193 item 1 — `min_score` must mean the same thing at `size:0` as at
+/// `size:N`.
+///
+/// The threshold applies to FINAL scores, so a `size:0 + min_score` count
+/// (no aggs) must score its matches exactly like the `size:N` page does.
+/// Before the fix that shape was classified `count_only`, which (a) closed
+/// the index-wide stats gate — any scoring that did happen used PER-ARM
+/// statistics, a genuinely different scale since #192 — and (b) skipped
+/// materialisation entirely, so the post-collection min_score filter had
+/// nothing to subtract and `hits.total` ignored the threshold. ES's own
+/// collector refuses the counting shortcut whenever min_score is set
+/// (`QueryPhaseCollector`: collection cannot be shortcut via Weight#count).
+#[tokio::test]
+async fn min_score_total_is_the_same_at_size0_and_sizen() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir, 4);
+ engine.create_index("t", Schema::empty()).unwrap();
+ let idx = engine.get_index("t").unwrap();
+ build_corpus(&idx).await;
+
+ // Unthresholded page: 65 live matches, the 5 short "strong" docs on top.
+ let full = idx.search(&match_body(100)).await.unwrap();
+ let scores = page(&full.hits);
+ assert!(scores.len() >= 6, "sanity: need at least 6 scored hits");
+ let (rank4, rank5) = (scores[4].1, scores[5].1);
+ assert!(
+ rank4 > rank5,
+ "sanity: ranks 4/5 must not tie for a clean threshold ({rank4} vs {rank5})"
+ );
+ let threshold = f64::from(rank4 + rank5) / 2.0;
+
+ let with_min = |size: usize| {
+ let mut req = match_body(size);
+ req.min_score = Some(threshold);
+ req
+ };
+
+ let paged = idx.search(&with_min(100)).await.unwrap();
+ assert_eq!(
+ paged.total.value, 5,
+ "size:100 + min_score must count exactly the 5 docs clearing the threshold"
+ );
+ assert_eq!(paged.hits.len(), 5);
+
+ let counted = idx.search(&with_min(0)).await.unwrap();
+ assert!(counted.hits.is_empty(), "size:0 must render an empty page");
+ assert_eq!(
+ counted.total.value, paged.total.value,
+ "size:0 + min_score counted {} docs where size:100 + min_score counted {} — \
+ the count-only path is not applying the threshold against the same \
+ (index-wide) scores as the paged path",
+ counted.total.value, paged.total.value
+ );
+}
+
/// The union must not leak GHOSTS as live hits or lose the delete-aware
/// accounting: statistics stay physical (Lucene counts deleted docs until a
/// merge purges them) while `hits.total` stays live.
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 7f8022e75..2218b7f6e 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -1011,8 +1011,11 @@ impl FtsIndexReader {
/// * the norms table — a whole-file `fs::read` plus decode, O(docs).
///
/// That makes the per-segment `field_stats` + `term_doc_freq` pre-pass
- /// the index-wide scorer needs (#188) cost one mmap and one small read
- /// per field, instead of re-paying the full open a second time.
+ /// the index-wide scorer needs (#188) cost, per field, one mmap plus one
+ /// `.meta` read-and-decode — for the current ZFM4 format that decode is
+ /// a zstd decompress of the `num_terms × 24`-byte records section, i.e.
+ /// O(field vocabulary), cheap but not free (#193) — instead of re-paying
+ /// the full open a second time.
///
/// The returned reader can answer [`Self::field_stats`],
/// [`Self::term_doc_freq`] and [`Self::lookup_term`]; [`Self::postings_data`]