AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): passage source stays within the fields the query targets (#1364 review)
xerj-orgd5775d2bea6f
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex f752a99b4..aa1e71000 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -39764,6 +39764,27 @@ fn refine_code_passage(crate::code_blocks::enclosing_block(language, text, start, end)}+/// Whether `source` has the shape autoindex's code extractor writes+/// (`xerj-autoindex/src/extract/code.rs`): a whole-file document carries a+/// numeric `symbol_count` (the signal [`refine_code_passage`] keys on); a+/// per-symbol document carries exactly `name` + `kind` + `code` + a numeric+/// `line` + `language`. Same conservatism as `refine_code_passage`: a field+/// called `name` or `body` on an arbitrary document is not enough.+fn is_autoindex_code_document(source: &Value) -> bool {+ if matches!(+ get_field_value(source, "symbol_count"),+ Some(Value::Number(_))+ ) {+ return true;+ }+ let is_str = |field: &str| matches!(get_field_value(source, field), Some(Value::String(_)));+ is_str("name")+ && is_str("kind")+ && is_str("code")+ && is_str("language")+ && matches!(get_field_value(source, "line"), Some(Value::Number(_)))+}+/// Maximum byte length of a query-time lexical `_passage` window.////// Sized so an agent gets enough surrounding code to read an implementation@@ -40121,26 +40142,6 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {Vec::new();for spec in &field_specs {for (field_name, text) in collect_passage_field_texts(&hit.source, spec) {- // `defs` and `name` are autoindex's synthetic symbol-list- // and identifier fields whose actual source lives in `body`- // (whole-file docs) or `code` (per-symbol docs). Ensure- // `body` and `code` are candidates ahead of `defs` and- // `name` even when a query lists `name^4` / `defs^3` first- // (`xerj search` / `xerj def`, #1364).- if matches!(field_name.as_str(), "defs" | "name") {- for primary in ["body", "code"] {- if !seen_fields.contains(primary) {- for (primary_name, primary_text) in- collect_passage_field_texts(&hit.source, primary)- {- if seen_fields.insert(primary_name.clone()) {- let (lower, map) = build_lower_offset_map(&primary_text);- candidates.push((primary_name, primary_text, lower, map));- }- }- }- }- }if seen_fields.insert(field_name.clone()) {let (lower, map) = build_lower_offset_map(&text);candidates.push((field_name, text, lower, map));@@ -40150,6 +40151,22 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {if candidates.is_empty() {return hit;}+ // #1364: on an autoindex CODE document, `name`/`defs`/`title` are+ // synthetic identifier fields (a one-word symbol name, a symbol+ // list) whose source lives in `code` (per-symbol docs) or `body`+ // (whole-file docs). Cross-field ties go to the first candidate,+ // so `xerj search`'s `name^4`-first field list returned the bare+ // word `de` as the passage of `de <- function(col) {...}`. Move the+ // source fields ahead — a stable reorder of fields the query+ // ALREADY targets, never an added field: a passage must come from+ // a field the query named (the #174 contract), and an ordinary+ // document that merely has `name`/`body` fields keeps the query's+ // own field order.+ if is_autoindex_code_document(&hit.source) {+ candidates.sort_by_key(|(field_name, ..)| {+ !matches!(field_name.as_str(), "code" | "body")+ });+ }// Document-level term rarity across all candidate fields,// squared so one term's total contribution stays bounded (see// the weighting note in this function's doc comment).@@ -56601,6 +56618,112 @@ mod lexical_passage_tests {);}+ fn passage_hit(id: &str, source: serde_json::Value) -> Hit {+ Hit {+ id: id.into(),+ score: 1.0,+ source,+ seq_no: None,+ version: None,+ sort: Vec::new(),+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ passage: None,+ }+ }++ /// #1364 review: the `code`-over-`name` preference must never make a+ /// field the query did NOT target the `_passage` source. An ordinary+ /// (non-autoindex) document that happens to carry `name` and `body`+ /// fields, queried on `name` alone, must get its passage from `name` —+ /// the contract is "the best window across every text field the query+ /// targets". The first cut of #1396 injected `body`/`code` whenever it+ /// met a `name`/`defs` field, so this returned `field: "body"`.+ #[test]+ fn lexical_passage_never_uses_a_field_the_query_did_not_target() {+ let source = json!({+ "name": "alpha widget",+ "body": "alpha alpha alpha\nalpha beta alpha\nalpha",+ });+ let query = QueryNode::Match {+ field: "name".into(),+ query: "alpha".into(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ lenient: false,+ };+ let out = apply_lexical_passages(vec![passage_hit("es-doc", source)], &query);+ let passage = out[0].passage.as_ref().expect("passage filled");+ assert_eq!(passage.field, "name", "got {passage:?}");+ assert_eq!(passage.text, "alpha widget");++ // Same shape through multi_match on `name` + `defs`: neither `body`+ // nor `code` was targeted, so neither may be chosen.+ let source = json!({+ "name": "alpha",+ "defs": "alpha beta",+ "code": "alpha alpha alpha alpha",+ "body": "alpha alpha alpha alpha alpha",+ });+ let query = QueryNode::MultiMatch {+ fields: vec!["name^4".into(), "defs^3".into()],+ query: "alpha".into(),+ match_type: Default::default(),+ operator: None,+ analyzer: None,+ boost: None,+ slop: 0,+ max_expansions: 50,+ lenient: false,+ };+ let out = apply_lexical_passages(vec![passage_hit("es-doc-2", source)], &query);+ let passage = out[0].passage.as_ref().expect("passage filled");+ assert!(+ matches!(passage.field.as_str(), "name" | "defs"),+ "passage must come from a targeted field, got {passage:?}"+ );+ }++ /// #1364: the realistic autoindex per-symbol document (exactly the+ /// fields `extract/code.rs` emits — no `symbol_count`, which only the+ /// whole-file document carries) still gets its passage from `code`+ /// rather than the bare `name`, when `xerj search` targets both.+ #[test]+ fn autoindex_symbol_doc_passage_prefers_code_over_name() {+ let source = json!({+ "title": "de",+ "path": "tests/testthat/test-aplicabilidad.R",+ "language": "r",+ "name": "de",+ "kind": "function",+ "line": 241,+ "code": "de <- function(col) {\n trimws(col)\n}",+ });+ let query = QueryNode::MultiMatch {+ fields: vec![+ "name^4".into(),+ "defs^3".into(),+ "code^2".into(),+ "body".into(),+ "title".into(),+ ],+ query: "de".into(),+ match_type: Default::default(),+ operator: None,+ analyzer: None,+ boost: None,+ slop: 0,+ max_expansions: 50,+ lenient: false,+ };+ let out = apply_lexical_passages(vec![passage_hit("sym", source)], &query);+ let passage = out[0].passage.as_ref().expect("passage filled");+ assert_eq!(passage.field, "code", "got {passage:?}");+ }+/// #174: a document that merely carries a field named `language` (not/// autoindex's code-extractor shape — no `symbol_count`) must keep the/// exact line-snapped window the lexical selector already computed.