Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Merge #634 (mvanhorn) into main — rescore stage alone reorders a tied primary (#627)

xerj-org274669653c46
diff --cc engine/crates/xerj-engine/tests/integration.rs
index 211c6e9f8,625e97a6e..09888d24d
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@@ -5355,169 -5355,72 +5355,249 @@@ async fn test_rescore_changes_ranking(
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
+ engine.create_index("rescore_idx", Schema::empty()).unwrap();
+ let idx = engine.get_index("rescore_idx").unwrap();
+
+ // Doc "a": lots of "search", few "engine" mentions → high score for "search"
+ idx.index_document(
+ Some("a".into()),
+ json!({ "title": "search", "body": "search search search" }),
+ )
+ .await
+ .unwrap();
+
+ // Doc "b": lots of "engine" mentions → would rank lower for "search", higher for "engine"
+ idx.index_document(
+ Some("b".into()),
+ json!({ "title": "engine", "body": "engine engine engine engine engine" }),
+ )
+ .await
+ .unwrap();
+
+ // Doc "c": mentions "search engine" once
+ idx.index_document(
+ Some("c".into()),
+ json!({ "title": "search engine", "body": "search engine" }),
+ )
+ .await
+ .unwrap();
+
+ // Primary query: search for "search" — doc "a" should rank highest initially.
+ let primary_req = parse_request(&json!({
+ "query": { "match": { "body": "search" } },
+ "size": 10,
+ }))
+ .unwrap();
+ let primary_result = idx.search(&primary_req).await.unwrap();
+ assert!(!primary_result.hits.is_empty());
+ let primary_top = primary_result.hits[0].id.clone();
+
+ // Now add a rescore that weights `title:engine` matches heavily. NOTE:
+ // `parse_request` does NOT parse the native `rescore` body — it hard-codes
+ // `rescore: Vec::new()` (parser.rs); only the ES-compat handler's
+ // `parse_rescore` does. The previous version of this test passed `rescore`
+ // through `parse_request`, which SILENTLY DROPPED it, then discarded the
+ // scores and asserted only `total.value > 0` — exercising ZERO rescore
+ // behaviour (#627). Populate the rescore stage directly instead.
+ let mut rescore_req = parse_request(&json!({
+ "query": { "match": { "body": "search" } },
+ "size": 10,
+ }))
+ .unwrap();
+ rescore_req.rescore = vec![xerj_query::ast::RescoreQuery {
+ window_size: 10,
+ query: Some(xerj_query::ast::RescoreQueryInner {
+ rescore_query: QueryNode::Match {
+ field: "title".into(),
+ query: "engine".into(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ },
+ query_weight: 0.1,
+ rescore_query_weight: 10.0,
+ }),
+ script: None,
+ }];
+ let rescore_result = idx.search(&rescore_req).await.unwrap();
+ assert!(
+ !rescore_result.hits.is_empty(),
+ "rescore search should return hits"
+ );
+
+ // The rescore must genuinely RE-RANK. Primary (`match body:search`) ranks
+ // doc "a" (3x "search") above doc "c" (1x). The rescore scales the primary
+ // score ×0.1 and adds `title:engine` ×10: doc "c" (title "search engine")
+ // matches and is lifted; doc "a" (title "search") does not. So doc "c" must
+ // rank ABOVE doc "a" after rescore — the exact inversion the primary lacked.
+ assert_eq!(
+ primary_top, "a",
+ "primary must rank doc a first (most `search`)"
+ );
+ let a_pos = rescore_result
+ .hits
+ .iter()
+ .position(|h| h.id == "a")
+ .expect("doc a present after rescore");
+ let c_pos = rescore_result
+ .hits
+ .iter()
+ .position(|h| h.id == "c")
+ .expect("doc c present after rescore");
+ assert!(
+ c_pos < a_pos,
+ "rescore must lift doc c (title matches `engine`, ×10) above doc a — a \
+ no-op rescore would leave a on top: c@{c_pos} a@{a_pos}, hits={:?}",
+ rescore_result
+ .hits
+ .iter()
+ .map(|h| (h.id.as_str(), h.score))
+ .collect::<Vec<_>>()
+ );
+}
+
++/// #627 / #634 (@mvanhorn): the rescore stage ALONE must be able to reorder a
++/// tied primary. `match_all` gives both docs one identical primary score, the
++/// rescore gives the primary zero weight (`query_weight: 0.0`), so the only
++/// score left is the rescore contribution — which the keyword rewrite hands to
++/// the whole-value doc and to nobody else. This isolates the rescore from the
++/// primary in a way `test_rescore_changes_ranking` (primary ×0.1) cannot: there
++/// a rank flip could still be a primary-score artefact; here it cannot be.
++#[tokio::test]
++async fn rescore_alone_reorders_a_tied_primary() {
++ let dir = TempDir::new().unwrap();
++ let engine = make_engine(&dir);
++
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("kw", FieldType::Keyword));
- engine.create_index("rescore_idx", schema).unwrap();
- let idx = engine.get_index("rescore_idx").unwrap();
++ engine.create_index("rescore_tied", schema).unwrap();
++ let idx = engine.get_index("rescore_tied").unwrap();
+
+ idx.index_document(Some("partial".into()), json!({ "kw": "claude" }))
+ .await
+ .unwrap();
+ idx.index_document(Some("whole".into()), json!({ "kw": "claude code" }))
+ .await
+ .unwrap();
+
+ let mut request = parse_request(&json!({
+ "query": { "match_all": {} },
+ "size": 10,
+ }))
+ .unwrap();
+ let primary = idx.search(&request).await.unwrap();
+ assert_eq!(primary.total.value, 2);
+ assert_eq!(primary.hits.len(), 2);
+ assert_eq!(
+ primary
+ .hits
+ .iter()
+ .map(|hit| hit.id.as_str())
+ .collect::<Vec<_>>(),
+ ["partial", "whole"],
+ "tied primary hits should preserve insertion order"
+ );
+ assert_eq!(
+ primary.hits[0].score.to_bits(),
+ primary.hits[1].score.to_bits(),
+ "match_all must establish a tied primary score"
+ );
+
- request.rescore = vec![RescoreQuery {
++ // `parse_request` deliberately leaves `rescore` empty (only the ES-compat
++ // handler parses it), so populate the stage directly — see #627.
++ request.rescore = vec![xerj_query::ast::RescoreQuery {
+ window_size: 10,
- query: Some(RescoreQueryInner {
++ query: Some(xerj_query::ast::RescoreQueryInner {
+ rescore_query: xerj_query::parse_query(&json!({
+ "match": { "kw": "claude code" }
+ }))
+ .unwrap(),
+ query_weight: 0.0,
+ rescore_query_weight: 1.0,
+ }),
+ script: None,
+ }];
+
+ let rescored = idx.search(&request).await.unwrap();
+ assert_eq!(rescored.total.value, 2);
+ assert_eq!(rescored.hits.len(), 2);
+ assert_eq!(rescored.hits[0].id, "whole", "rescore must flip the order");
+ assert_eq!(rescored.hits[1].id, "partial");
+ assert!(
+ rescored.hits[0].score > 0.0,
+ "the whole keyword value must receive a rescore contribution"
+ );
+ assert_eq!(
+ rescored.hits[1].score.to_bits(),
+ 0.0f32.to_bits(),
+ "a partial keyword token must receive no rescore contribution"
+ );
+ }
+
+// ── Rescore keyword rewrite: a keyword `match` in a rescore query is whole-value ──
+
+/// #627 (end-to-end guard for #574): a keyword `match` in a `rescore` query must be
+/// rewritten to a whole-value `Term`, so only the exact-value doc is boosted — not
+/// every doc sharing a token. Without the rewrite, `match {kw:"foo bar"}` tokenizes
+/// (OR) and would boost `{kw:"foo"}` too. The engine rewrites the rescore query at
+/// the search entry (`rewrite_rescore_keyword`), so this exercises that path.
+#[tokio::test]
+async fn rescore_keyword_query_is_mapping_aware() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("kw", FieldType::Keyword));
+ engine.create_index("rescore_kw", schema).unwrap();
+ let idx = engine.get_index("rescore_kw").unwrap();
+ idx.index_document(Some("whole".into()), json!({ "kw": "foo bar" }))
+ .await
+ .unwrap();
+ idx.index_document(Some("token".into()), json!({ "kw": "foo" }))
+ .await
+ .unwrap();
+
+ // Primary: match_all → both docs at equal primary score. Rescore boosts the
+ // keyword value "foo bar" ×10 (query_weight ×0.1).
+ let mut req = parse_request(&json!({ "query": { "match_all": {} }, "size": 10 })).unwrap();
+ req.rescore = vec![xerj_query::ast::RescoreQuery {
+ window_size: 10,
+ query: Some(xerj_query::ast::RescoreQueryInner {
+ rescore_query: QueryNode::Match {
+ field: "kw".into(),
+ query: "foo bar".into(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ },
+ query_weight: 0.1,
+ rescore_query_weight: 10.0,
+ }),
+ script: None,
+ }];
+ let result = idx.search(&req).await.unwrap();
+ let score = |id: &str| result.hits.iter().find(|h| h.id == id).map(|h| h.score);
+ let whole = score("whole").expect("whole-value doc present");
+ let token = score("token").expect("token-sharing doc present");
+ // Whole-value Term matches only "foo bar" → boosted; "foo" does not. A tokenized
+ // (un-rewritten) rescore would boost both roughly equally. So the whole-value doc
+ // must clearly outscore the token-sharing doc.
+ assert!(
+ whole > token * 3.0,
+ "keyword rescore must boost ONLY the whole-value doc (#574/#627): \
+ whole={whole} token={token}; a tokenized rescore would boost both"
+ );
+ assert_eq!(
+ result.hits[0].id, "whole",
+ "the boosted whole-value doc must rank first"
+ );
+}
+
// ── Weighted bool: verify boosted queries rank higher ─────────────────────────
#[tokio::test]