Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): passage source stays within the fields the query targets (#1364 review)

xerj-orgd5775d2bea6f
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index f752a99b4..aa1e71000 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -39764,6 +39764,27 @@ fn refine_code_passage(
crate::code_blocks::enclosing_block(language, text, start, end)
}
+/// Whether `source` has the shape autoindex's code extractor writes
+/// (`xerj-autoindex/src/extract/code.rs`): a whole-file document carries a
+/// numeric `symbol_count` (the signal [`refine_code_passage`] keys on); a
+/// per-symbol document carries exactly `name` + `kind` + `code` + a numeric
+/// `line` + `language`. Same conservatism as `refine_code_passage`: a field
+/// called `name` or `body` on an arbitrary document is not enough.
+fn is_autoindex_code_document(source: &Value) -> bool {
+ if matches!(
+ get_field_value(source, "symbol_count"),
+ Some(Value::Number(_))
+ ) {
+ return true;
+ }
+ let is_str = |field: &str| matches!(get_field_value(source, field), Some(Value::String(_)));
+ is_str("name")
+ && is_str("kind")
+ && is_str("code")
+ && is_str("language")
+ && matches!(get_field_value(source, "line"), Some(Value::Number(_)))
+}
+
/// Maximum byte length of a query-time lexical `_passage` window.
///
/// Sized so an agent gets enough surrounding code to read an implementation
@@ -40121,26 +40142,6 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
Vec::new();
for spec in &field_specs {
for (field_name, text) in collect_passage_field_texts(&hit.source, spec) {
- // `defs` and `name` are autoindex's synthetic symbol-list
- // and identifier fields whose actual source lives in `body`
- // (whole-file docs) or `code` (per-symbol docs). Ensure
- // `body` and `code` are candidates ahead of `defs` and
- // `name` even when a query lists `name^4` / `defs^3` first
- // (`xerj search` / `xerj def`, #1364).
- if matches!(field_name.as_str(), "defs" | "name") {
- for primary in ["body", "code"] {
- if !seen_fields.contains(primary) {
- for (primary_name, primary_text) in
- collect_passage_field_texts(&hit.source, primary)
- {
- if seen_fields.insert(primary_name.clone()) {
- let (lower, map) = build_lower_offset_map(&primary_text);
- candidates.push((primary_name, primary_text, lower, map));
- }
- }
- }
- }
- }
if seen_fields.insert(field_name.clone()) {
let (lower, map) = build_lower_offset_map(&text);
candidates.push((field_name, text, lower, map));
@@ -40150,6 +40151,22 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
if candidates.is_empty() {
return hit;
}
+ // #1364: on an autoindex CODE document, `name`/`defs`/`title` are
+ // synthetic identifier fields (a one-word symbol name, a symbol
+ // list) whose source lives in `code` (per-symbol docs) or `body`
+ // (whole-file docs). Cross-field ties go to the first candidate,
+ // so `xerj search`'s `name^4`-first field list returned the bare
+ // word `de` as the passage of `de <- function(col) {...}`. Move the
+ // source fields ahead — a stable reorder of fields the query
+ // ALREADY targets, never an added field: a passage must come from
+ // a field the query named (the #174 contract), and an ordinary
+ // document that merely has `name`/`body` fields keeps the query's
+ // own field order.
+ if is_autoindex_code_document(&hit.source) {
+ candidates.sort_by_key(|(field_name, ..)| {
+ !matches!(field_name.as_str(), "code" | "body")
+ });
+ }
// Document-level term rarity across all candidate fields,
// squared so one term's total contribution stays bounded (see
// the weighting note in this function's doc comment).
@@ -56601,6 +56618,112 @@ mod lexical_passage_tests {
);
}
+ fn passage_hit(id: &str, source: serde_json::Value) -> Hit {
+ Hit {
+ id: id.into(),
+ score: 1.0,
+ source,
+ seq_no: None,
+ version: None,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ }
+ }
+
+ /// #1364 review: the `code`-over-`name` preference must never make a
+ /// field the query did NOT target the `_passage` source. An ordinary
+ /// (non-autoindex) document that happens to carry `name` and `body`
+ /// fields, queried on `name` alone, must get its passage from `name` —
+ /// the contract is "the best window across every text field the query
+ /// targets". The first cut of #1396 injected `body`/`code` whenever it
+ /// met a `name`/`defs` field, so this returned `field: "body"`.
+ #[test]
+ fn lexical_passage_never_uses_a_field_the_query_did_not_target() {
+ let source = json!({
+ "name": "alpha widget",
+ "body": "alpha alpha alpha\nalpha beta alpha\nalpha",
+ });
+ let query = QueryNode::Match {
+ field: "name".into(),
+ query: "alpha".into(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ lenient: false,
+ };
+ let out = apply_lexical_passages(vec![passage_hit("es-doc", source)], &query);
+ let passage = out[0].passage.as_ref().expect("passage filled");
+ assert_eq!(passage.field, "name", "got {passage:?}");
+ assert_eq!(passage.text, "alpha widget");
+
+ // Same shape through multi_match on `name` + `defs`: neither `body`
+ // nor `code` was targeted, so neither may be chosen.
+ let source = json!({
+ "name": "alpha",
+ "defs": "alpha beta",
+ "code": "alpha alpha alpha alpha",
+ "body": "alpha alpha alpha alpha alpha",
+ });
+ let query = QueryNode::MultiMatch {
+ fields: vec!["name^4".into(), "defs^3".into()],
+ query: "alpha".into(),
+ match_type: Default::default(),
+ operator: None,
+ analyzer: None,
+ boost: None,
+ slop: 0,
+ max_expansions: 50,
+ lenient: false,
+ };
+ let out = apply_lexical_passages(vec![passage_hit("es-doc-2", source)], &query);
+ let passage = out[0].passage.as_ref().expect("passage filled");
+ assert!(
+ matches!(passage.field.as_str(), "name" | "defs"),
+ "passage must come from a targeted field, got {passage:?}"
+ );
+ }
+
+ /// #1364: the realistic autoindex per-symbol document (exactly the
+ /// fields `extract/code.rs` emits — no `symbol_count`, which only the
+ /// whole-file document carries) still gets its passage from `code`
+ /// rather than the bare `name`, when `xerj search` targets both.
+ #[test]
+ fn autoindex_symbol_doc_passage_prefers_code_over_name() {
+ let source = json!({
+ "title": "de",
+ "path": "tests/testthat/test-aplicabilidad.R",
+ "language": "r",
+ "name": "de",
+ "kind": "function",
+ "line": 241,
+ "code": "de <- function(col) {\n trimws(col)\n}",
+ });
+ let query = QueryNode::MultiMatch {
+ fields: vec![
+ "name^4".into(),
+ "defs^3".into(),
+ "code^2".into(),
+ "body".into(),
+ "title".into(),
+ ],
+ query: "de".into(),
+ match_type: Default::default(),
+ operator: None,
+ analyzer: None,
+ boost: None,
+ slop: 0,
+ max_expansions: 50,
+ lenient: false,
+ };
+ let out = apply_lexical_passages(vec![passage_hit("sym", source)], &query);
+ let passage = out[0].passage.as_ref().expect("passage filled");
+ assert_eq!(passage.field, "code", "got {passage:?}");
+ }
+
/// #174: a document that merely carries a field named `language` (not
/// autoindex's code-extractor shape — no `symbol_count`) must keep the
/// exact line-snapped window the lexical selector already computed.