AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex,engine): render 1-based passage line on whole-file search/def hits (#1365)
Christian M. Todiedb68bf4f8372
diff --git a/engine/crates/xerj-autoindex/src/def.rs b/engine/crates/xerj-autoindex/src/def.rsindex 69c7c334d..31ea04120 100644--- a/engine/crates/xerj-autoindex/src/def.rs+++ b/engine/crates/xerj-autoindex/src/def.rs@@ -12,7 +12,7 @@//! whole-file dump).use crate::esclient::Es;-use crate::search::{is_identifier, passage_text};+use crate::search::{hit_location, is_identifier, passage_text};use serde_json::{json, Value};const USAGE: &str = "\@@ -80,7 +80,7 @@ fn def_query(query: &str, kind: Option<&str>, lang: Option<&str>, k: usize) -> V"filter": filter,"minimum_should_match": 1} },- "_source": ["ax_path", "path", "line", "kind", "language", "name", "code"],+ "_source": ["ax_path", "path", "line", "start_line", "kind", "language", "name", "code"],"fields": ["_passage"]})}@@ -187,16 +187,7 @@ pub fn run_def_cli() -> i32 {for h in &hits {let src = h.get("_source").cloned().unwrap_or(Value::Null);- let path = src- .get("ax_path")- .or_else(|| src.get("path"))- .and_then(Value::as_str)- .unwrap_or("?");- let line = src.get("line").and_then(Value::as_u64);- let loc = match line {- Some(n) => format!("{path}:{n}"),- None => path.to_string(),- };+ let loc = hit_location(h);let kind = src.get("kind").and_then(Value::as_str).unwrap_or("");let tag = if kind.is_empty() {String::new()diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rsindex 22ebb7f9b..83458c9b0 100644--- a/engine/crates/xerj-autoindex/src/search.rs+++ b/engine/crates/xerj-autoindex/src/search.rs@@ -93,6 +93,39 @@ pub fn search_query_body(query: &str, k: usize) -> Value {})}+/// Extract the 1-based starting line number from `fields._passage[0].ordinal`+/// when `_passage` comes from a document body field (ignoring synthetic+/// single-line or symbol-list metadata fields such as `defs` or `title`).+pub(crate) fn passage_line(hit: &Value) -> Option<u64> {+ let first = hit.pointer("/fields/_passage/0")?.as_object()?;+ if matches!(+ first.get("field").and_then(Value::as_str),+ Some("defs" | "title" | "name")+ ) {+ return None;+ }+ first.get("ordinal").and_then(Value::as_u64).map(|o| o + 1)+}++/// Format the `path[:line]` header token for a search or definition hit.+pub(crate) fn hit_location(hit: &Value) -> String {+ let src = hit.get("_source").unwrap_or(&Value::Null);+ let path = src+ .get("ax_path")+ .or_else(|| src.get("path"))+ .and_then(Value::as_str)+ .unwrap_or("?");+ let line = src+ .get("line")+ .or_else(|| src.get("start_line"))+ .and_then(Value::as_u64)+ .or_else(|| passage_line(hit));+ match line {+ Some(n) => format!("{path}:{n}"),+ None => path.to_string(),+ }+}+/// Entry point for the `search` subcommand. Returns a process exit code.pub fn run_search_cli() -> i32 {let mut url = std::env::var("XERJ_URL")@@ -208,16 +241,8 @@ pub fn run_search_cli() -> i32 {for h in &hits {let src = h.get("_source").cloned().unwrap_or(Value::Null);- let path = src.get("ax_path").and_then(Value::as_str).unwrap_or("?");let score = h.get("_score").and_then(Value::as_f64).unwrap_or(0.0);- let line = src- .get("line")- .or_else(|| src.get("start_line"))- .and_then(Value::as_u64);- let loc = match line {- Some(n) => format!("{path}:{n}"),- None => path.to_string(),- };+ let loc = hit_location(h);let kind = src.get("kind").and_then(Value::as_str).unwrap_or("");let tag = if kind.is_empty() {String::new()@@ -339,4 +364,65 @@ mod tests {});assert_eq!(passage_text(&hit), None);}++ /// #1365: whole-file hits (`ax-code-*`, `ax-docs-md-*`, `ax-text-*`) carry+ /// no `_source.line` or `_source.start_line`, so `hit_location` must+ /// convert `fields._passage[0].ordinal` (0-based line index of the snippet)+ /// to a 1-based `:line` suffix when `_passage` comes from the file body+ /// (and ignore synthetic metadata fields like `defs` or `title`).+ #[test]+ fn whole_file_hit_uses_one_based_passage_ordinal_when_source_has_no_line() {+ let whole_file_hit = json!({+ "_source": {+ "ax_path": "crates/fjall/src/batch.rs",+ "language": "rust"+ },+ "fields": {+ "_passage": [{+ "field": "body",+ "ordinal": 41,+ "start_offset": 1200,+ "end_offset": 1450,+ "text": "pub fn commit(&mut self) -> crate::Result<()> {\n Ok(())\n}"+ }]+ }+ });+ assert_eq!(+ hit_location(&whole_file_hit),+ "crates/fjall/src/batch.rs:42"+ );++ let symbol_hit = json!({+ "_source": {+ "ax_path": "crates/fjall/src/batch.rs",+ "line": 108,+ "kind": "function"+ },+ "fields": {+ "_passage": [{+ "field": "code",+ "ordinal": 0,+ "text": "pub fn commit(&mut self) -> crate::Result<()>"+ }]+ }+ });+ assert_eq!(hit_location(&symbol_hit), "crates/fjall/src/batch.rs:108");++ let synthetic_defs_hit = json!({+ "_source": {+ "ax_path": "crates/fjall/src/batch.rs"+ },+ "fields": {+ "_passage": [{+ "field": "defs",+ "ordinal": 0,+ "text": "fn commit"+ }]+ }+ });+ assert_eq!(+ hit_location(&synthetic_defs_hit),+ "crates/fjall/src/batch.rs"+ );+ }}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex aa1e71000..59b844b4a 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -40142,6 +40142,21 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {Vec::new();for spec in &field_specs {for (field_name, text) in collect_passage_field_texts(&hit.source, spec) {+ // `defs` is autoindex's synthetic symbol-list field on+ // whole-file code docs whose actual source lives in `body`.+ // Ensure `body` is a candidate ahead of `defs` even when a+ // query lists `defs^3` first or queries `defs` without+ // naming `body` (`xerj search` / `xerj def`, #1365).+ if field_name == "defs" && !seen_fields.contains("body") {+ for (body_name, body_text) in+ collect_passage_field_texts(&hit.source, "body")+ {+ if seen_fields.insert(body_name.clone()) {+ let (lower, map) = build_lower_offset_map(&body_text);+ candidates.push((body_name, body_text, lower, map));+ }+ }+ }if seen_fields.insert(field_name.clone()) {let (lower, map) = build_lower_offset_map(&text);candidates.push((field_name, text, lower, map));@@ -40204,7 +40219,7 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {// non-ASCII anchor just skips the refinement, not a// correctness bug.let window_lower = text.get(start..end).map(str::to_ascii_lowercase);- let (start, end) = window_lower+ let (start, end, ordinal) = window_lower.as_deref().and_then(|win| {weighted_terms@@ -40227,7 +40242,12 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {anchor_end,)})- .unwrap_or((start, end));+ .map(|(r_start, r_end)| {+ let r_ordinal =+ text[..r_start].bytes().filter(|&b| b == b'\n').count() as u32;+ (r_start, r_end, r_ordinal)+ })+ .unwrap_or((start, end, ordinal));best = Some((score,PassageMatch {@@ -56501,6 +56521,82 @@ mod lexical_passage_tests {passage.text,&body[passage.start_offset as usize..passage.end_offset as usize]);+ assert_eq!(+ passage.ordinal as usize,+ body[..passage.start_offset as usize].matches('\n').count(),+ "ordinal must be recomputed after refine_code_passage widens/snaps start_offset"+ );+ assert_eq!(passage.ordinal, 80);+ }++ /// #1365: when `defs` is listed before `body` (as in `xerj search`'s+ /// `["name^4", "defs^3", "code^2", "body", "title"]`) or when a query+ /// matches `defs` without naming `body` (as in `xerj def`), a single-symbol+ /// match whose window score ties between `defs` and `body` must extract the+ /// passage from `body` at the declaration's line ordinal, never from the+ /// synthetic `defs` symbol-list field at `ordinal: 0`.+ #[test]+ fn defs_field_never_captures_passage_over_code_body_even_when_listed_first_or_alone() {+ let body = format!(+ "{}pub fn check_invariants(x: i32) -> bool {{\n x > 0\n}}\n{}",+ "// header filler\n".repeat(40),+ "// footer filler\n".repeat(40)+ );+ let source = json!({+ "body": body,+ "defs": "fn check_invariants",+ "language": "rust",+ "symbol_count": 1+ });++ for fields in [+ vec![+ "name^4".to_string(),+ "defs^3".to_string(),+ "code^2".to_string(),+ "body".to_string(),+ "title".to_string(),+ ],+ vec![+ "name^4".to_string(),+ "defs^3".to_string(),+ "code^2".to_string(),+ "title".to_string(),+ ],+ ] {+ let query = QueryNode::MultiMatch {+ fields,+ query: "check_invariants".into(),+ match_type: Default::default(),+ operator: None,+ analyzer: None,+ boost: None,+ slop: 0,+ max_expansions: 50,+ lenient: false,+ };+ let hit = Hit {+ id: "doc-code-1".into(),+ score: 1.0,+ source: source.clone(),+ seq_no: None,+ version: None,+ sort: Vec::new(),+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ passage: None,+ };+ let out = apply_lexical_passages(vec![hit], &query);+ let passage = out[0].passage.as_ref().expect("lexical passage filled");+ assert_eq!(passage.field, "body");+ assert!(+ passage.text.starts_with("pub fn check_invariants"),+ "expected enclosing declaration from body, got {:?}",+ passage.text+ );+ assert_eq!(passage.ordinal, 40);+ }}/// #1364: on a per-symbol code hit (`name: "de"`, `title: "de"`, `code: