AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Merge #634 (mvanhorn) into main — rescore stage alone reorders a tied primary (#627)
xerj-org274669653c46
diff --cc engine/crates/xerj-engine/tests/integration.rsindex 211c6e9f8,625e97a6e..09888d24d--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@@ -5355,169 -5355,72 +5355,249 @@@ async fn test_rescore_changes_ranking(let dir = TempDir::new().unwrap();let engine = make_engine(&dir);+ engine.create_index("rescore_idx", Schema::empty()).unwrap();+ let idx = engine.get_index("rescore_idx").unwrap();++ // Doc "a": lots of "search", few "engine" mentions → high score for "search"+ idx.index_document(+ Some("a".into()),+ json!({ "title": "search", "body": "search search search" }),+ )+ .await+ .unwrap();++ // Doc "b": lots of "engine" mentions → would rank lower for "search", higher for "engine"+ idx.index_document(+ Some("b".into()),+ json!({ "title": "engine", "body": "engine engine engine engine engine" }),+ )+ .await+ .unwrap();++ // Doc "c": mentions "search engine" once+ idx.index_document(+ Some("c".into()),+ json!({ "title": "search engine", "body": "search engine" }),+ )+ .await+ .unwrap();++ // Primary query: search for "search" — doc "a" should rank highest initially.+ let primary_req = parse_request(&json!({+ "query": { "match": { "body": "search" } },+ "size": 10,+ }))+ .unwrap();+ let primary_result = idx.search(&primary_req).await.unwrap();+ assert!(!primary_result.hits.is_empty());+ let primary_top = primary_result.hits[0].id.clone();++ // Now add a rescore that weights `title:engine` matches heavily. NOTE:+ // `parse_request` does NOT parse the native `rescore` body — it hard-codes+ // `rescore: Vec::new()` (parser.rs); only the ES-compat handler's+ // `parse_rescore` does. The previous version of this test passed `rescore`+ // through `parse_request`, which SILENTLY DROPPED it, then discarded the+ // scores and asserted only `total.value > 0` — exercising ZERO rescore+ // behaviour (#627). Populate the rescore stage directly instead.+ let mut rescore_req = parse_request(&json!({+ "query": { "match": { "body": "search" } },+ "size": 10,+ }))+ .unwrap();+ rescore_req.rescore = vec![xerj_query::ast::RescoreQuery {+ window_size: 10,+ query: Some(xerj_query::ast::RescoreQueryInner {+ rescore_query: QueryNode::Match {+ field: "title".into(),+ query: "engine".into(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ },+ query_weight: 0.1,+ rescore_query_weight: 10.0,+ }),+ script: None,+ }];+ let rescore_result = idx.search(&rescore_req).await.unwrap();+ assert!(+ !rescore_result.hits.is_empty(),+ "rescore search should return hits"+ );++ // The rescore must genuinely RE-RANK. Primary (`match body:search`) ranks+ // doc "a" (3x "search") above doc "c" (1x). The rescore scales the primary+ // score ×0.1 and adds `title:engine` ×10: doc "c" (title "search engine")+ // matches and is lifted; doc "a" (title "search") does not. So doc "c" must+ // rank ABOVE doc "a" after rescore — the exact inversion the primary lacked.+ assert_eq!(+ primary_top, "a",+ "primary must rank doc a first (most `search`)"+ );+ let a_pos = rescore_result+ .hits+ .iter()+ .position(|h| h.id == "a")+ .expect("doc a present after rescore");+ let c_pos = rescore_result+ .hits+ .iter()+ .position(|h| h.id == "c")+ .expect("doc c present after rescore");+ assert!(+ c_pos < a_pos,+ "rescore must lift doc c (title matches `engine`, ×10) above doc a — a \+ no-op rescore would leave a on top: c@{c_pos} a@{a_pos}, hits={:?}",+ rescore_result+ .hits+ .iter()+ .map(|h| (h.id.as_str(), h.score))+ .collect::<Vec<_>>()+ );+}+++/// #627 / #634 (@mvanhorn): the rescore stage ALONE must be able to reorder a++/// tied primary. `match_all` gives both docs one identical primary score, the++/// rescore gives the primary zero weight (`query_weight: 0.0`), so the only++/// score left is the rescore contribution — which the keyword rewrite hands to++/// the whole-value doc and to nobody else. This isolates the rescore from the++/// primary in a way `test_rescore_changes_ranking` (primary ×0.1) cannot: there++/// a rank flip could still be a primary-score artefact; here it cannot be.++#[tokio::test]++async fn rescore_alone_reorders_a_tied_primary() {++ let dir = TempDir::new().unwrap();++ let engine = make_engine(&dir);+++ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("kw", FieldType::Keyword));- engine.create_index("rescore_idx", schema).unwrap();- let idx = engine.get_index("rescore_idx").unwrap();++ engine.create_index("rescore_tied", schema).unwrap();++ let idx = engine.get_index("rescore_tied").unwrap();++ idx.index_document(Some("partial".into()), json!({ "kw": "claude" }))+ .await+ .unwrap();+ idx.index_document(Some("whole".into()), json!({ "kw": "claude code" }))+ .await+ .unwrap();++ let mut request = parse_request(&json!({+ "query": { "match_all": {} },+ "size": 10,+ }))+ .unwrap();+ let primary = idx.search(&request).await.unwrap();+ assert_eq!(primary.total.value, 2);+ assert_eq!(primary.hits.len(), 2);+ assert_eq!(+ primary+ .hits+ .iter()+ .map(|hit| hit.id.as_str())+ .collect::<Vec<_>>(),+ ["partial", "whole"],+ "tied primary hits should preserve insertion order"+ );+ assert_eq!(+ primary.hits[0].score.to_bits(),+ primary.hits[1].score.to_bits(),+ "match_all must establish a tied primary score"+ );+- request.rescore = vec![RescoreQuery {++ // `parse_request` deliberately leaves `rescore` empty (only the ES-compat++ // handler parses it), so populate the stage directly — see #627.++ request.rescore = vec![xerj_query::ast::RescoreQuery {+ window_size: 10,- query: Some(RescoreQueryInner {++ query: Some(xerj_query::ast::RescoreQueryInner {+ rescore_query: xerj_query::parse_query(&json!({+ "match": { "kw": "claude code" }+ }))+ .unwrap(),+ query_weight: 0.0,+ rescore_query_weight: 1.0,+ }),+ script: None,+ }];++ let rescored = idx.search(&request).await.unwrap();+ assert_eq!(rescored.total.value, 2);+ assert_eq!(rescored.hits.len(), 2);+ assert_eq!(rescored.hits[0].id, "whole", "rescore must flip the order");+ assert_eq!(rescored.hits[1].id, "partial");+ assert!(+ rescored.hits[0].score > 0.0,+ "the whole keyword value must receive a rescore contribution"+ );+ assert_eq!(+ rescored.hits[1].score.to_bits(),+ 0.0f32.to_bits(),+ "a partial keyword token must receive no rescore contribution"+ );+ }++// ── Rescore keyword rewrite: a keyword `match` in a rescore query is whole-value ──++/// #627 (end-to-end guard for #574): a keyword `match` in a `rescore` query must be+/// rewritten to a whole-value `Term`, so only the exact-value doc is boosted — not+/// every doc sharing a token. Without the rewrite, `match {kw:"foo bar"}` tokenizes+/// (OR) and would boost `{kw:"foo"}` too. The engine rewrites the rescore query at+/// the search entry (`rewrite_rescore_keyword`), so this exercises that path.+#[tokio::test]+async fn rescore_keyword_query_is_mapping_aware() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("kw", FieldType::Keyword));+ engine.create_index("rescore_kw", schema).unwrap();+ let idx = engine.get_index("rescore_kw").unwrap();+ idx.index_document(Some("whole".into()), json!({ "kw": "foo bar" }))+ .await+ .unwrap();+ idx.index_document(Some("token".into()), json!({ "kw": "foo" }))+ .await+ .unwrap();++ // Primary: match_all → both docs at equal primary score. Rescore boosts the+ // keyword value "foo bar" ×10 (query_weight ×0.1).+ let mut req = parse_request(&json!({ "query": { "match_all": {} }, "size": 10 })).unwrap();+ req.rescore = vec![xerj_query::ast::RescoreQuery {+ window_size: 10,+ query: Some(xerj_query::ast::RescoreQueryInner {+ rescore_query: QueryNode::Match {+ field: "kw".into(),+ query: "foo bar".into(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ },+ query_weight: 0.1,+ rescore_query_weight: 10.0,+ }),+ script: None,+ }];+ let result = idx.search(&req).await.unwrap();+ let score = |id: &str| result.hits.iter().find(|h| h.id == id).map(|h| h.score);+ let whole = score("whole").expect("whole-value doc present");+ let token = score("token").expect("token-sharing doc present");+ // Whole-value Term matches only "foo bar" → boosted; "foo" does not. A tokenized+ // (un-rewritten) rescore would boost both roughly equally. So the whole-value doc+ // must clearly outscore the token-sharing doc.+ assert!(+ whole > token * 3.0,+ "keyword rescore must boost ONLY the whole-value doc (#574/#627): \+ whole={whole} token={token}; a tokenized rescore would boost both"+ );+ assert_eq!(+ result.hits[0].id, "whole",+ "the boosted whole-value doc must rank first"+ );+}+// ── Weighted bool: verify boosted queries rank higher ─────────────────────────#[tokio::test]