Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex,engine): render 1-based passage line on whole-file search/def hits (#1365)

Christian M. Todiedb68bf4f8372
diff --git a/engine/crates/xerj-autoindex/src/def.rs b/engine/crates/xerj-autoindex/src/def.rs
index 69c7c334d..31ea04120 100644
--- a/engine/crates/xerj-autoindex/src/def.rs
+++ b/engine/crates/xerj-autoindex/src/def.rs
@@ -12,7 +12,7 @@
//! whole-file dump).
use crate::esclient::Es;
-use crate::search::{is_identifier, passage_text};
+use crate::search::{hit_location, is_identifier, passage_text};
use serde_json::{json, Value};
const USAGE: &str = "\
@@ -80,7 +80,7 @@ fn def_query(query: &str, kind: Option<&str>, lang: Option<&str>, k: usize) -> V
"filter": filter,
"minimum_should_match": 1
} },
- "_source": ["ax_path", "path", "line", "kind", "language", "name", "code"],
+ "_source": ["ax_path", "path", "line", "start_line", "kind", "language", "name", "code"],
"fields": ["_passage"]
})
}
@@ -187,16 +187,7 @@ pub fn run_def_cli() -> i32 {
for h in &hits {
let src = h.get("_source").cloned().unwrap_or(Value::Null);
- let path = src
- .get("ax_path")
- .or_else(|| src.get("path"))
- .and_then(Value::as_str)
- .unwrap_or("?");
- let line = src.get("line").and_then(Value::as_u64);
- let loc = match line {
- Some(n) => format!("{path}:{n}"),
- None => path.to_string(),
- };
+ let loc = hit_location(h);
let kind = src.get("kind").and_then(Value::as_str).unwrap_or("");
let tag = if kind.is_empty() {
String::new()
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rs
index 22ebb7f9b..83458c9b0 100644
--- a/engine/crates/xerj-autoindex/src/search.rs
+++ b/engine/crates/xerj-autoindex/src/search.rs
@@ -93,6 +93,39 @@ pub fn search_query_body(query: &str, k: usize) -> Value {
})
}
+/// Extract the 1-based starting line number from `fields._passage[0].ordinal`
+/// when `_passage` comes from a document body field (ignoring synthetic
+/// single-line or symbol-list metadata fields such as `defs` or `title`).
+pub(crate) fn passage_line(hit: &Value) -> Option<u64> {
+ let first = hit.pointer("/fields/_passage/0")?.as_object()?;
+ if matches!(
+ first.get("field").and_then(Value::as_str),
+ Some("defs" | "title" | "name")
+ ) {
+ return None;
+ }
+ first.get("ordinal").and_then(Value::as_u64).map(|o| o + 1)
+}
+
+/// Format the `path[:line]` header token for a search or definition hit.
+pub(crate) fn hit_location(hit: &Value) -> String {
+ let src = hit.get("_source").unwrap_or(&Value::Null);
+ let path = src
+ .get("ax_path")
+ .or_else(|| src.get("path"))
+ .and_then(Value::as_str)
+ .unwrap_or("?");
+ let line = src
+ .get("line")
+ .or_else(|| src.get("start_line"))
+ .and_then(Value::as_u64)
+ .or_else(|| passage_line(hit));
+ match line {
+ Some(n) => format!("{path}:{n}"),
+ None => path.to_string(),
+ }
+}
+
/// Entry point for the `search` subcommand. Returns a process exit code.
pub fn run_search_cli() -> i32 {
let mut url = std::env::var("XERJ_URL")
@@ -208,16 +241,8 @@ pub fn run_search_cli() -> i32 {
for h in &hits {
let src = h.get("_source").cloned().unwrap_or(Value::Null);
- let path = src.get("ax_path").and_then(Value::as_str).unwrap_or("?");
let score = h.get("_score").and_then(Value::as_f64).unwrap_or(0.0);
- let line = src
- .get("line")
- .or_else(|| src.get("start_line"))
- .and_then(Value::as_u64);
- let loc = match line {
- Some(n) => format!("{path}:{n}"),
- None => path.to_string(),
- };
+ let loc = hit_location(h);
let kind = src.get("kind").and_then(Value::as_str).unwrap_or("");
let tag = if kind.is_empty() {
String::new()
@@ -339,4 +364,65 @@ mod tests {
});
assert_eq!(passage_text(&hit), None);
}
+
+ /// #1365: whole-file hits (`ax-code-*`, `ax-docs-md-*`, `ax-text-*`) carry
+ /// no `_source.line` or `_source.start_line`, so `hit_location` must
+ /// convert `fields._passage[0].ordinal` (0-based line index of the snippet)
+ /// to a 1-based `:line` suffix when `_passage` comes from the file body
+ /// (and ignore synthetic metadata fields like `defs` or `title`).
+ #[test]
+ fn whole_file_hit_uses_one_based_passage_ordinal_when_source_has_no_line() {
+ let whole_file_hit = json!({
+ "_source": {
+ "ax_path": "crates/fjall/src/batch.rs",
+ "language": "rust"
+ },
+ "fields": {
+ "_passage": [{
+ "field": "body",
+ "ordinal": 41,
+ "start_offset": 1200,
+ "end_offset": 1450,
+ "text": "pub fn commit(&mut self) -> crate::Result<()> {\n Ok(())\n}"
+ }]
+ }
+ });
+ assert_eq!(
+ hit_location(&whole_file_hit),
+ "crates/fjall/src/batch.rs:42"
+ );
+
+ let symbol_hit = json!({
+ "_source": {
+ "ax_path": "crates/fjall/src/batch.rs",
+ "line": 108,
+ "kind": "function"
+ },
+ "fields": {
+ "_passage": [{
+ "field": "code",
+ "ordinal": 0,
+ "text": "pub fn commit(&mut self) -> crate::Result<()>"
+ }]
+ }
+ });
+ assert_eq!(hit_location(&symbol_hit), "crates/fjall/src/batch.rs:108");
+
+ let synthetic_defs_hit = json!({
+ "_source": {
+ "ax_path": "crates/fjall/src/batch.rs"
+ },
+ "fields": {
+ "_passage": [{
+ "field": "defs",
+ "ordinal": 0,
+ "text": "fn commit"
+ }]
+ }
+ });
+ assert_eq!(
+ hit_location(&synthetic_defs_hit),
+ "crates/fjall/src/batch.rs"
+ );
+ }
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index aa1e71000..59b844b4a 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -40142,6 +40142,21 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
Vec::new();
for spec in &field_specs {
for (field_name, text) in collect_passage_field_texts(&hit.source, spec) {
+ // `defs` is autoindex's synthetic symbol-list field on
+ // whole-file code docs whose actual source lives in `body`.
+ // Ensure `body` is a candidate ahead of `defs` even when a
+ // query lists `defs^3` first or queries `defs` without
+ // naming `body` (`xerj search` / `xerj def`, #1365).
+ if field_name == "defs" && !seen_fields.contains("body") {
+ for (body_name, body_text) in
+ collect_passage_field_texts(&hit.source, "body")
+ {
+ if seen_fields.insert(body_name.clone()) {
+ let (lower, map) = build_lower_offset_map(&body_text);
+ candidates.push((body_name, body_text, lower, map));
+ }
+ }
+ }
if seen_fields.insert(field_name.clone()) {
let (lower, map) = build_lower_offset_map(&text);
candidates.push((field_name, text, lower, map));
@@ -40204,7 +40219,7 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
// non-ASCII anchor just skips the refinement, not a
// correctness bug.
let window_lower = text.get(start..end).map(str::to_ascii_lowercase);
- let (start, end) = window_lower
+ let (start, end, ordinal) = window_lower
.as_deref()
.and_then(|win| {
weighted_terms
@@ -40227,7 +40242,12 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
anchor_end,
)
})
- .unwrap_or((start, end));
+ .map(|(r_start, r_end)| {
+ let r_ordinal =
+ text[..r_start].bytes().filter(|&b| b == b'\n').count() as u32;
+ (r_start, r_end, r_ordinal)
+ })
+ .unwrap_or((start, end, ordinal));
best = Some((
score,
PassageMatch {
@@ -56501,6 +56521,82 @@ mod lexical_passage_tests {
passage.text,
&body[passage.start_offset as usize..passage.end_offset as usize]
);
+ assert_eq!(
+ passage.ordinal as usize,
+ body[..passage.start_offset as usize].matches('\n').count(),
+ "ordinal must be recomputed after refine_code_passage widens/snaps start_offset"
+ );
+ assert_eq!(passage.ordinal, 80);
+ }
+
+ /// #1365: when `defs` is listed before `body` (as in `xerj search`'s
+ /// `["name^4", "defs^3", "code^2", "body", "title"]`) or when a query
+ /// matches `defs` without naming `body` (as in `xerj def`), a single-symbol
+ /// match whose window score ties between `defs` and `body` must extract the
+ /// passage from `body` at the declaration's line ordinal, never from the
+ /// synthetic `defs` symbol-list field at `ordinal: 0`.
+ #[test]
+ fn defs_field_never_captures_passage_over_code_body_even_when_listed_first_or_alone() {
+ let body = format!(
+ "{}pub fn check_invariants(x: i32) -> bool {{\n x > 0\n}}\n{}",
+ "// header filler\n".repeat(40),
+ "// footer filler\n".repeat(40)
+ );
+ let source = json!({
+ "body": body,
+ "defs": "fn check_invariants",
+ "language": "rust",
+ "symbol_count": 1
+ });
+
+ for fields in [
+ vec![
+ "name^4".to_string(),
+ "defs^3".to_string(),
+ "code^2".to_string(),
+ "body".to_string(),
+ "title".to_string(),
+ ],
+ vec![
+ "name^4".to_string(),
+ "defs^3".to_string(),
+ "code^2".to_string(),
+ "title".to_string(),
+ ],
+ ] {
+ let query = QueryNode::MultiMatch {
+ fields,
+ query: "check_invariants".into(),
+ match_type: Default::default(),
+ operator: None,
+ analyzer: None,
+ boost: None,
+ slop: 0,
+ max_expansions: 50,
+ lenient: false,
+ };
+ let hit = Hit {
+ id: "doc-code-1".into(),
+ score: 1.0,
+ source: source.clone(),
+ seq_no: None,
+ version: None,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ };
+ let out = apply_lexical_passages(vec![hit], &query);
+ let passage = out[0].passage.as_ref().expect("lexical passage filled");
+ assert_eq!(passage.field, "body");
+ assert!(
+ passage.text.starts_with("pub fn check_invariants"),
+ "expected enclosing declaration from body, got {:?}",
+ passage.text
+ );
+ assert_eq!(passage.ordinal, 40);
+ }
}
/// #1364: on a per-symbol code hit (`name: "de"`, `title: "de"`, `code: