Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(search): resolve `_passage` to the enclosing code block, not a byte window

bugerd2ce7df90b26
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 2c3305b90..311270fa2 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -6621,6 +6621,19 @@ dependencies = [
"tokio",
"tracing",
"tracing-subscriber",
+ "tree-sitter",
+ "tree-sitter-bash",
+ "tree-sitter-c",
+ "tree-sitter-c-sharp",
+ "tree-sitter-cpp",
+ "tree-sitter-go",
+ "tree-sitter-java",
+ "tree-sitter-javascript",
+ "tree-sitter-php",
+ "tree-sitter-python",
+ "tree-sitter-ruby",
+ "tree-sitter-rust",
+ "tree-sitter-typescript",
"uuid",
"xerj-ai",
"xerj-cluster",
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index dbe9b3020..a32ae0e59 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -40,11 +40,31 @@ regex = "1"
# Fast non-cryptographic hash — used to shard the engine FTS memtable
# by doc_id so concurrent bulk ingest batches target different shards.
+# Also seeds the `code_blocks` parsed-tree cache key (#174 follow-up).
xxhash-rust = { workspace = true }
# FxHash for the memtable hot maps (postings, field_lengths, doc-values)
# — SipHash was >21% of on-CPU time under the shard write locks at c8.
rustc-hash = { workspace = true }
+# --- AST-aligned `_passage` for code hits (#174 follow-up, `code_blocks`).
+# Same grammar versions as `xerj-autoindex`'s ingest-time extractor
+# (`xerj-autoindex/src/extract/code.rs`) — this module re-parses at RESPONSE
+# time rather than depending on that crate (autoindex depends on the engine,
+# not the other way around; a reverse dependency would be circular). ---
+tree-sitter = "0.25"
+tree-sitter-python = "0.23"
+tree-sitter-c = "0.23"
+tree-sitter-rust = "0.23"
+tree-sitter-javascript = "0.23"
+tree-sitter-go = "0.23"
+tree-sitter-typescript = "0.23"
+tree-sitter-java = "0.23"
+tree-sitter-cpp = "0.23"
+tree-sitter-ruby = "0.23"
+tree-sitter-php = "0.23"
+tree-sitter-c-sharp = "0.23"
+tree-sitter-bash = "0.23"
+
[features]
default = []
# Built-in neural BERT embedder (candle). Propagates to xerj-ai. Off by
diff --git a/engine/crates/xerj-engine/src/code_blocks.rs b/engine/crates/xerj-engine/src/code_blocks.rs
new file mode 100644
index 000000000..3b8f7cf3e
--- /dev/null
+++ b/engine/crates/xerj-engine/src/code_blocks.rs
@@ -0,0 +1,536 @@
+//! AST-aligned code passages (#174 follow-up).
+//!
+//! `_passage` normally returns an arbitrary byte window (a 512-char
+//! index-time chunk for semantic hits, a line-snapped query-time window for
+//! lexical hits). Neither knows about syntax, so a code hit's passage can
+//! open mid-token — an observed real example began `"\t}\n\n\tif ( isset( "`,
+//! a dangling brace with no idea what function it belongs to.
+//!
+//! This module resolves, at RESPONSE time (not index time — no reindex, no
+//! migration, works on every existing index), the smallest syntactic node
+//! enclosing a byte range that counts as a semantic unit (function, method,
+//! class, ...) for the document's language. The caller (`index.rs`) widens
+//! the passage to that span when one is found; otherwise the existing window
+//! is left untouched.
+//!
+//! ## Node-kind policy tables
+//!
+//! Which node kinds count as an acceptable enclosing block is inherently
+//! per-language and per-grammar. The lists below are adapted from
+//! [probelabs/probe](https://github.com/probelabs/probe)'s
+//! `is_acceptable_parent` policy (`src/language/*.rs`, dual Apache-2.0/MIT),
+//! reduced to just the node-kind data and re-expressed for XERJ's
+//! tree-sitter 0.25 bindings and the language set `xerj-autoindex` already
+//! parses (`xerj-autoindex/src/extract/code.rs`). No probe source is
+//! depended on or copied wholesale — only these kind lists, and narrowed:
+//! probe's tables also include generic containers (`block`, `statement`,
+//! `module`, `object`, ...) it uses to keep code-context expansion from
+//! jumping too far; those are deliberately dropped here so "no acceptable
+//! enclosing node" (e.g. a top-level statement) falls back to the caller's
+//! original window instead of returning a bare block.
+//!
+//! Every kind name below was checked against the installed grammar's
+//! `node-types.json` for the exact pinned version (see `Cargo.toml`).
+//!
+//! Bash has no probe table (probe does not support it); its single kind,
+//! `function_definition`, is picked directly from `tree-sitter-bash`'s own
+//! grammar, analogously to the other C-family languages.
+
+use rustc_hash::FxHashMap;
+use std::collections::VecDeque;
+use std::sync::OnceLock;
+use tree_sitter::{Language, Parser, Tree};
+
+/// Default cap on the byte size of a resolved block. Past this we fall back
+/// to the caller's original window rather than hand back a giant function —
+/// probe itself has no such cap and returned a 64,120-byte / 26,906-token
+/// function in one observed case, which is not a "passage" any more.
+/// Sized generously above a typical function/method/class (a few hundred
+/// bytes to a few KB) while still bounding the worst case. Overridable via
+/// `XERJ_CODE_PASSAGE_MAX_BYTES` for deployments with different tolerances.
+pub const DEFAULT_MAX_BLOCK_BYTES: usize = 16_384;
+
+fn max_block_bytes() -> usize {
+ std::env::var("XERJ_CODE_PASSAGE_MAX_BYTES")
+ .ok()
+ .and_then(|v| v.parse::<usize>().ok())
+ .filter(|&n| n > 0)
+ .unwrap_or(DEFAULT_MAX_BLOCK_BYTES)
+}
+
+/// Bound on how many parsed trees the in-process cache holds. XERJ is a
+/// long-lived server (unlike probe's short-lived CLI process, whose
+/// equivalent cache simply dies with it), so an unbounded cache would leak
+/// memory for every distinct code document ever hit. Overridable via
+/// `XERJ_CODE_PASSAGE_CACHE_CAP`.
+const DEFAULT_CACHE_CAPACITY: usize = 128;
+
+fn cache_capacity() -> usize {
+ std::env::var("XERJ_CODE_PASSAGE_CACHE_CAP")
+ .ok()
+ .and_then(|v| v.parse::<usize>().ok())
+ .filter(|&n| n > 0)
+ .unwrap_or(DEFAULT_CACHE_CAPACITY)
+}
+
+// ── Node-kind policy tables ─────────────────────────────────────────────────
+
+const RUST_KINDS: &[&str] = &[
+ "function_item",
+ "struct_item",
+ "impl_item",
+ "trait_item",
+ "enum_item",
+ "mod_item",
+ "macro_definition",
+];
+
+const PHP_KINDS: &[&str] = &[
+ "function_definition",
+ "method_declaration",
+ "class_declaration",
+ "interface_declaration",
+ "trait_declaration",
+];
+
+const PYTHON_KINDS: &[&str] = &["function_definition", "class_definition", "decorated_definition"];
+
+const JS_KINDS: &[&str] = &[
+ "function_declaration",
+ "function_expression",
+ "method_definition",
+ "class_declaration",
+ "arrow_function",
+ "class",
+];
+
+// TypeScript-specific additions on top of the JS set. `internal_module` and
+// `module` are the actual grammar node kinds for `namespace Foo {}` /
+// `declare module "foo" {}` in tree-sitter-typescript 0.23 — probe's own
+// list names them `namespace_declaration`/`module_declaration`, which do not
+// exist in this grammar version; verified against the pinned crate's
+// `node-types.json`.
+const TS_EXTRA_KINDS: &[&str] = &[
+ "interface_declaration",
+ "type_alias_declaration",
+ "enum_declaration",
+ "internal_module",
+ "module",
+];
+
+const GO_KINDS: &[&str] = &[
+ "function_declaration",
+ "method_declaration",
+ "type_declaration",
+ "struct_type",
+ "interface_type",
+ "type_spec",
+];
+
+const JAVA_KINDS: &[&str] = &[
+ "method_declaration",
+ "class_declaration",
+ "interface_declaration",
+ "enum_declaration",
+ "constructor_declaration",
+ "static_initializer",
+];
+
+const C_KINDS: &[&str] = &["function_definition", "struct_specifier", "enum_specifier"];
+
+const CPP_KINDS: &[&str] = &[
+ "function_definition",
+ "struct_specifier",
+ "class_specifier",
+ "enum_specifier",
+ "namespace_definition",
+];
+
+const RUBY_KINDS: &[&str] = &["class", "module", "method", "singleton_method"];
+
+const CSHARP_KINDS: &[&str] = &[
+ "method_declaration",
+ "class_declaration",
+ "struct_declaration",
+ "interface_declaration",
+ "enum_declaration",
+ "namespace_declaration",
+ "property_declaration",
+ "constructor_declaration",
+ "delegate_declaration",
+ "event_declaration",
+];
+
+const BASH_KINDS: &[&str] = &["function_definition"];
+
+/// One JS+TS combined slice, built once. TypeScript and TSX both accept the
+/// JS kinds plus the TS-only ones.
+fn ts_kinds() -> &'static [&'static str] {
+ static KINDS: OnceLock<Vec<&'static str>> = OnceLock::new();
+ KINDS
+ .get_or_init(|| {
+ let mut v = JS_KINDS.to_vec();
+ v.extend_from_slice(TS_EXTRA_KINDS);
+ v
+ })
+ .as_slice()
+}
+
+/// Resolve a `language` field value (as `xerj-autoindex/src/extract/code.rs`
+/// writes it) to its tree-sitter grammar and acceptable-block kind table.
+/// `None` means "not a language this module can parse" — the caller falls
+/// back to its existing window.
+fn registry_entry(language: &str) -> Option<(Language, &'static [&'static str])> {
+ Some(match language {
+ "python" => (tree_sitter_python::LANGUAGE.into(), PYTHON_KINDS),
+ "javascript" => (tree_sitter_javascript::LANGUAGE.into(), JS_KINDS),
+ "typescript" => (
+ tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
+ ts_kinds(),
+ ),
+ "tsx" => (tree_sitter_typescript::LANGUAGE_TSX.into(), ts_kinds()),
+ "rust" => (tree_sitter_rust::LANGUAGE.into(), RUST_KINDS),
+ "go" => (tree_sitter_go::LANGUAGE.into(), GO_KINDS),
+ "java" => (tree_sitter_java::LANGUAGE.into(), JAVA_KINDS),
+ "c" => (tree_sitter_c::LANGUAGE.into(), C_KINDS),
+ "cpp" => (tree_sitter_cpp::LANGUAGE.into(), CPP_KINDS),
+ "ruby" => (tree_sitter_ruby::LANGUAGE.into(), RUBY_KINDS),
+ "php" => (tree_sitter_php::LANGUAGE_PHP.into(), PHP_KINDS),
+ "csharp" => (tree_sitter_c_sharp::LANGUAGE.into(), CSHARP_KINDS),
+ "bash" => (tree_sitter_bash::LANGUAGE.into(), BASH_KINDS),
+ _ => return None,
+ })
+}
+
+// ── Bounded LRU tree cache ───────────────────────────────────────────────────
+
+struct TreeCache {
+ capacity: usize,
+ map: FxHashMap<u64, Tree>,
+ // Recency order, oldest first. `key` may appear only once; a re-touch
+ // removes and re-appends it. Capacity is small (default 128) and this
+ // runs on the query path, not a hot per-token loop, so O(n) touch is
+ // fine — a hand-rolled intrusive doubly-linked list would buy nothing
+ // observable here.
+ order: VecDeque<u64>,
+}
+
+impl TreeCache {
+ fn new(capacity: usize) -> Self {
+ Self {
+ capacity: capacity.max(1),
+ map: FxHashMap::default(),
+ order: VecDeque::new(),
+ }
+ }
+
+ fn touch(&mut self, key: u64) {
+ if let Some(pos) = self.order.iter().position(|&k| k == key) {
+ self.order.remove(pos);
+ }
+ self.order.push_back(key);
+ }
+
+ /// Cheap ref-counted clone on a cache hit (tree-sitter `Tree` wraps a
+ /// shared node pool); a fresh parse on a miss.
+ fn get_or_parse(&mut self, key: u64, language: &Language, body: &str) -> Option<Tree> {
+ if let Some(tree) = self.map.get(&key) {
+ let tree = tree.clone();
+ self.touch(key);
+ return Some(tree);
+ }
+ let mut parser = Parser::new();
+ parser.set_language(language).ok()?;
+ let tree = parser.parse(body.as_bytes(), None)?;
+ if self.map.len() >= self.capacity {
+ if let Some(oldest) = self.order.pop_front() {
+ self.map.remove(&oldest);
+ }
+ }
+ self.map.insert(key, tree.clone());
+ self.touch(key);
+ Some(tree)
+ }
+}
+
+fn cache() -> &'static parking_lot::Mutex<TreeCache> {
+ static CACHE: OnceLock<parking_lot::Mutex<TreeCache>> = OnceLock::new();
+ CACHE.get_or_init(|| parking_lot::Mutex::new(TreeCache::new(cache_capacity())))
+}
+
+/// Content-addressed cache key: language name mixed into the seed so two
+/// languages never collide on the same source bytes (which cannot happen in
+/// practice — a document has one `language` — but costs nothing to guard).
+fn cache_key(language: &str, body: &str) -> u64 {
+ let seed = xxhash_rust::xxh3::xxh3_64(language.as_bytes());
+ xxhash_rust::xxh3::xxh3_64_with_seed(body.as_bytes(), seed)
+}
+
+// ── Public API ────────────────────────────────────────────────────────────
+
+/// Find the smallest node enclosing byte range `[start, end)` in `body`
+/// whose kind counts as an acceptable semantic block for `language`, and
+/// return its own `[start_byte, end_byte)` span.
+///
+/// Returns `None` — meaning "keep the caller's existing window" — when:
+/// - `language` has no registered grammar/kind table here;
+/// - `[start, end)` is out of range for `body`;
+/// - parsing fails (bad grammar/language mismatch; tree-sitter's `parse`
+/// itself is infallible on well-formed UTF-8 input short of a hard
+/// internal error, but the API is fallible and we honor that);
+/// - no ancestor of the smallest node containing the range has an
+/// acceptable kind before the tree root (e.g. the match sits in a
+/// top-level `use`/`import` statement, not inside any function/class);
+/// - the smallest acceptable block exceeds [`max_block_bytes`] — since any
+/// larger enclosing block is only bigger, this stops at the first
+/// candidate rather than searching further up.
+/// Syntactic wrappers that carry a declaration's modifiers rather than being
+/// blocks in their own right. A resolved block is widened through these so the
+/// passage shows the declaration as it is actually written — `export function
+/// f()`, not a bare `function f()` that reads as un-exported.
+///
+/// Language-agnostic by construction: a kind absent from a grammar simply never
+/// matches, so one table is safe for every language in the registry.
+const WRAPPER_KINDS: &[&str] = &[
+ // JavaScript / TypeScript
+ "export_statement",
+ "export_default_declaration",
+ // Python
+ "decorated_definition",
+];
+
+pub fn enclosing_block(language: &str, body: &str, start: usize, end: usize) -> Option<(usize, usize)> {
+ if start > end || end > body.len() {
+ return None;
+ }
+ let (grammar, kinds) = registry_entry(language)?;
+ let key = cache_key(language, body);
+ let tree = {
+ let mut guard = cache().lock();
+ guard.get_or_parse(key, &grammar, body)?
+ };
+ let root = tree.root_node();
+ let node = root.descendant_for_byte_range(start, end)?;
+ let mut cur = Some(node);
+ while let Some(n) = cur {
+ if kinds.contains(&n.kind()) {
+ // Widen through modifier wrappers so the passage carries the
+ // declaration as written. Returning the bare `function_declaration`
+ // inside `export function f() {}` would show an agent a function
+ // that appears un-exported, and the same applies to a Python
+ // definition whose decorators are part of its contract.
+ let mut outer = n;
+ while let Some(p) = outer.parent() {
+ if WRAPPER_KINDS.contains(&p.kind()) && p.start_byte() <= outer.start_byte() {
+ outer = p;
+ } else {
+ break;
+ }
+ }
+ let n = outer;
+ let (s, e) = (n.start_byte(), n.end_byte());
+ if e.saturating_sub(s) > max_block_bytes() {
+ return None;
+ }
+ if s > e || e > body.len() || !body.is_char_boundary(s) || !body.is_char_boundary(e) {
+ // Defensive only: tree-sitter node boundaries are always
+ // aligned to the UTF-8 byte stream it was fed, so this
+ // should be unreachable for valid input.
+ return None;
+ }
+ return Some((s, e));
+ }
+ cur = n.parent();
+ }
+ None
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn rust_function_resolves_to_its_own_declaration() {
+ let body = "fn before() {\n 1\n}\n\nfn target(x: i32) -> i32 {\n let y = x + 1;\n y * 2\n}\n\nfn after() {\n 2\n}\n";
+ let needle = "y * 2";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let (s, e) = enclosing_block("rust", body, start, end).expect("expected an enclosing block");
+ let block = &body[s..e];
+ assert!(
+ block.starts_with("fn target"),
+ "expected block to start at the function declaration, got: {block:?}"
+ );
+ assert!(block.contains(needle));
+ assert!(!block.contains("before"));
+ assert!(!block.contains("after"));
+ }
+
+ #[test]
+ fn php_method_resolves_to_its_own_declaration() {
+ let body = "<?php\nclass Foo {\n public function bar() {\n if (isset($x)) {\n return 1;\n }\n }\n}\n";
+ let needle = "isset($x)";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let (s, e) = enclosing_block("php", body, start, end).expect("expected an enclosing block");
+ let block = &body[s..e];
+ assert!(
+ block.starts_with("public function bar"),
+ "expected block to start at the method declaration, got: {block:?}"
+ );
+ }
+
+ #[test]
+ fn python_function_resolves_to_its_own_declaration() {
+ let body = "def before():\n pass\n\n\ndef target(x):\n y = x + 1\n return y\n";
+ let needle = "return y";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let (s, e) = enclosing_block("python", body, start, end).expect("expected an enclosing block");
+ let block = &body[s..e];
+ assert!(
+ block.starts_with("def target"),
+ "expected block to start at the function declaration, got: {block:?}"
+ );
+ }
+
+ #[test]
+ fn javascript_function_resolves_to_its_own_declaration() {
+ let body = "function before() {\n return 1;\n}\n\nfunction target(x) {\n const y = x + 1;\n return y * 2;\n}\n\nfunction after() {\n return 2;\n}\n";
+ let needle = "y * 2";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let (s, e) =
+ enclosing_block("javascript", body, start, end).expect("expected an enclosing block");
+ let block = &body[s..e];
+ assert!(
+ block.starts_with("function target"),
+ "expected block to start at the function declaration, got: {block:?}"
+ );
+ }
+
+ #[test]
+ fn typescript_namespace_resolves_via_internal_module_kind() {
+ // Exercises the `internal_module` kind (TS's actual grammar node for
+ // `namespace X {}`, not `namespace_declaration` as probe's own table
+ // names it — see the module-level doc comment).
+ let body = "namespace Widgets {\n export function make(x: number): number {\n return x + 1;\n }\n}\n";
+ let needle = "x + 1";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let (s, e) =
+ enclosing_block("typescript", body, start, end).expect("expected an enclosing block");
+ let block = &body[s..e];
+ assert!(
+ block.starts_with("export function make"),
+ "expected the smallest enclosing block (the function, nested inside the namespace), got: {block:?}"
+ );
+ }
+
+ #[test]
+ fn python_decorators_are_part_of_the_block() {
+ // A decorated definition without its decorators reads as an ordinary
+ // function, but `@property` (or a route/auth decorator) is frequently
+ // the whole contract — dropping it hands an agent a misleading block.
+ let body = "class Cfg:\n @property\n def timeout(self):\n return self._t\n";
+ let needle = "self._t";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let (s, e) =
+ enclosing_block("python", body, start, end).expect("expected an enclosing block");
+ let block = &body[s..e];
+ assert!(
+ block.trim_start().starts_with("@property"),
+ "expected the decorator to be part of the block, got: {block:?}"
+ );
+ }
+
+ #[test]
+ fn go_function_resolves_to_its_own_declaration() {
+ let body = "package main\n\nfunc before() int {\n\treturn 1\n}\n\nfunc target(x int) int {\n\ty := x + 1\n\treturn y * 2\n}\n";
+ let needle = "y * 2";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let (s, e) = enclosing_block("go", body, start, end).expect("expected an enclosing block");
+ let block = &body[s..e];
+ assert!(
+ block.starts_with("func target"),
+ "expected block to start at the function declaration, got: {block:?}"
+ );
+ }
+
+ #[test]
+ fn unsupported_language_falls_back() {
+ assert_eq!(enclosing_block("cobol", "IDENTIFICATION DIVISION.", 0, 5), None);
+ }
+
+ #[test]
+ fn out_of_range_falls_back() {
+ let body = "fn f() {}";
+ assert_eq!(enclosing_block("rust", body, 0, body.len() + 1), None);
+ assert_eq!(enclosing_block("rust", body, 5, 2), None);
+ }
+
+ #[test]
+ fn top_level_statement_has_no_acceptable_enclosing_node() {
+ // A `use` item at module scope is not inside any function/struct/etc,
+ // so the walk reaches the tree root without a match.
+ let body = "use std::fmt;\n\nfn f() {}\n";
+ let start = body.find("use").unwrap();
+ let end = start + "use std::fmt;".len();
+ assert_eq!(enclosing_block("rust", body, start, end), None);
+ }
+
+ #[test]
+ fn oversized_block_falls_back_to_caller_window() {
+ let mut body = String::from("fn huge() {\n");
+ for i in 0..2000 {
+ body.push_str(&format!(" let v{i} = {i};\n"));
+ }
+ body.push_str("}\n");
+ let needle = "v5 = 5";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ // Sanity: the function really is bigger than the default cap.
+ assert!(body.len() > DEFAULT_MAX_BLOCK_BYTES);
+ assert_eq!(enclosing_block("rust", &body, start, end), None);
+ }
+
+ #[test]
+ fn repeated_lookup_on_same_body_hits_the_cache_and_agrees_with_cold_parse() {
+ let body = "fn target(x: i32) -> i32 {\n x + 1\n}\n";
+ let start = body.find("x + 1").unwrap();
+ let end = start + "x + 1".len();
+ let first = enclosing_block("rust", body, start, end);
+ let second = enclosing_block("rust", body, start, end);
+ assert_eq!(first, second);
+ assert!(first.is_some());
+ }
+
+ #[test]
+ fn cache_eviction_does_not_change_results() {
+ // Push more distinct bodies through than the default cache capacity
+ // so the LRU has to evict, then confirm every one still resolves
+ // correctly on re-lookup (a bug in the eviction bookkeeping would
+ // show up as a stale/aliased tree here).
+ let mut bodies = Vec::new();
+ for i in 0..(DEFAULT_CACHE_CAPACITY + 16) {
+ bodies.push(format!("fn f{i}() {{\n let v = {i};\n}}\n"));
+ }
+ for body in &bodies {
+ let start = body.find("let v").unwrap();
+ let end = body.len() - 2; // just before the closing brace
+ let result = enclosing_block("rust", body, start, end);
+ assert!(result.is_some(), "expected a resolved block for {body:?}");
+ let (s, e) = result.unwrap();
+ assert!(body[s..e].starts_with("fn f"));
+ }
+ // Re-touch the first (now evicted) body once more.
+ let body = &bodies[0];
+ let start = body.find("let v").unwrap();
+ let end = start + "let v = 0".len();
+ let result = enclosing_block("rust", body, start, end);
+ assert!(result.is_some());
+ assert!(result.unwrap().0 == 0);
+ }
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 2187c51d7..0ec25bc56 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -27122,11 +27122,13 @@ fn passage_match_from_source(
return None;
}
let page = passage_page_from_source(source);
+ let (start_usize, end_usize) = refine_code_passage(source, field, text, start_usize, end_usize)
+ .unwrap_or((start_usize, end_usize));
Some(PassageMatch {
field: field.to_string(),
ordinal,
- start_offset: start,
- end_offset: end,
+ start_offset: start_usize as u64,
+ end_offset: end_usize as u64,
text: text[start_usize..end_usize].to_string(),
page,
})
@@ -27141,6 +27143,40 @@ fn passage_page_from_source(source: &Value) -> Option<u64> {
.and_then(|value| value.as_u64())
}
+/// For CODE documents only, widen a computed `_passage` window to the
+/// smallest enclosing semantic block (function/method/class/...) instead of
+/// the raw byte window (a 512-char index-time chunk for semantic hits, a
+/// line-snapped query-time window for lexical hits) — see `code_blocks`
+/// (#174 follow-up). `None` means "leave the window exactly as computed",
+/// which is what every non-code document gets: unconditionally.
+///
+/// Deliberately conservative about what counts as "a code document": the
+/// `language` field alone is not enough signal, since an unrelated document
+/// is free to carry a field literally named `language` (e.g. a
+/// `"language": "python"` tag on a course-catalog entry) without being
+/// source code. Only documents that ALSO carry autoindex's code-extractor
+/// signature (`symbol_count`, a number — see
+/// `xerj-autoindex/src/extract/code.rs`) qualify, and only their canonical
+/// `body` field (the one that extractor always populates) is ever handed to
+/// a parser — never an arbitrary field name.
+fn refine_code_passage(
+ source: &Value,
+ field: &str,
+ text: &str,
+ start: usize,
+ end: usize,
+) -> Option<(usize, usize)> {
+ if field != "body" {
+ return None;
+ }
+ if !matches!(get_field_value(source, "symbol_count"), Some(Value::Number(_))) {
+ return None;
+ }
+ let language = get_field_value(source, "language")?;
+ let language = language.as_str()?;
+ crate::code_blocks::enclosing_block(language, text, start, end)
+}
+
/// Maximum byte length of a query-time lexical `_passage` window.
///
/// Sized so an agent gets enough surrounding code to read an implementation
@@ -27527,6 +27563,38 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
select_lexical_passage(text, lower, map, &weighted_terms)
{
if best.as_ref().is_none_or(|(s, _)| score > *s) {
+ // `[start, end)` is the already CONTEXT-PADDED window
+ // (matched line plus surrounding lines up to the
+ // byte budget) — handing that whole span to
+ // `refine_code_passage` would usually force it back
+ // to the tree root (the padding routinely spills
+ // outside the enclosing function) and silently no-op
+ // the refinement. Anchor on one actual query-term
+ // occurrence inside the window instead — any point
+ // inside the matched region resolves to the same
+ // enclosing block. ASCII-lowercase only (not the
+ // Unicode-aware `lower`/`map` used above): it never
+ // changes byte length, so `pos` lines up with `text`
+ // without a second offset table; source identifiers
+ // are ASCII in the overwhelming case, and a missed
+ // non-ASCII anchor just skips the refinement, not a
+ // correctness bug.
+ let (start, end) = weighted_terms
+ .iter()
+ .find_map(|&(term, _)| {
+ if term.is_empty() {
+ return None;
+ }
+ text.get(start..end)?
+ .to_ascii_lowercase()
+ .find(term)
+ .map(|pos| start + pos)
+ })
+ .and_then(|anchor| {
+ let anchor_end = (anchor + 1).min(text.len());
+ refine_code_passage(&hit.source, field_name, text, anchor, anchor_end)
+ })
+ .unwrap_or((start, end));
best = Some((
score,
PassageMatch {
@@ -39017,6 +39085,102 @@ mod lexical_passage_tests {
);
}
+ /// #174: the lexical path must widen a code hit's passage to its
+ /// enclosing function too, exactly like the semantic path.
+ #[test]
+ fn code_hit_passage_widens_to_enclosing_function() {
+ let body = format!(
+ "{}fn target(x: i32) -> i32 {{\n let y = x + 1;\n y * needle_marker\n}}\n{}",
+ "// filler line of no consequence\n".repeat(80),
+ "// trailing filler\n".repeat(80)
+ );
+ let query = QueryNode::Match {
+ field: "body".into(),
+ query: "needle_marker".into(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ };
+ let hit = Hit {
+ id: "doc-1".into(),
+ score: 1.0,
+ source: json!({"body": body, "language": "rust", "symbol_count": 1}),
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ };
+ let out = apply_lexical_passages(vec![hit], &query);
+ let passage = out[0].passage.as_ref().expect("lexical passage filled");
+ assert!(
+ passage.text.starts_with("fn target"),
+ "passage must start at the enclosing function's declaration, got: {:?}",
+ passage.text
+ );
+ assert_eq!(
+ passage.text,
+ &body[passage.start_offset as usize..passage.end_offset as usize]
+ );
+ }
+
+ /// #174: a document that merely carries a field named `language` (not
+ /// autoindex's code-extractor shape — no `symbol_count`) must keep the
+ /// exact line-snapped window the lexical selector already computed.
+ #[test]
+ fn non_code_document_lexical_passage_ignores_stray_language_field() {
+ let body = format!(
+ "{}fn target(x: i32) -> i32 {{\n let y = x + 1;\n y * needle_marker\n}}\n{}",
+ "// filler line of no consequence\n".repeat(80),
+ "// trailing filler\n".repeat(80)
+ );
+ let query = QueryNode::Match {
+ field: "body".into(),
+ query: "needle_marker".into(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ };
+ let plain_hit = Hit {
+ id: "doc-1".into(),
+ score: 1.0,
+ source: json!({"body": body}),
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ };
+ let tagged_hit = Hit {
+ id: "doc-2".into(),
+ score: 1.0,
+ // Same body, but with a stray `language` field and no
+ // `symbol_count` — not autoindex's code-doc shape.
+ source: json!({"body": body, "language": "rust"}),
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ };
+ let plain_out = apply_lexical_passages(vec![plain_hit], &query);
+ let tagged_out = apply_lexical_passages(vec![tagged_hit], &query);
+ let plain_passage = plain_out[0].passage.as_ref().expect("plain passage filled");
+ let tagged_passage = tagged_out[0]
+ .passage
+ .as_ref()
+ .expect("tagged passage filled");
+ assert_eq!(plain_passage.start_offset, tagged_passage.start_offset);
+ assert_eq!(plain_passage.end_offset, tagged_passage.end_offset);
+ assert_eq!(plain_passage.text, tagged_passage.text);
+ assert!(
+ !plain_passage.text.starts_with("fn target"),
+ "without symbol_count this must stay the raw line-snapped window, not a widened block"
+ );
+ }
+
/// A semantic-path passage (exact ingest-time chunk provenance) must
/// never be overwritten by the query-time selector.
#[test]
@@ -39541,6 +39705,69 @@ mod chunk_embed_tests {
);
}
+ /// #174 AST-aligned passages: a document that merely carries a field
+ /// literally named `language` (e.g. a course-catalog entry tagged
+ /// `"language": "rust"`, not a source file) must NOT trigger AST
+ /// refinement — `refine_code_passage` also requires autoindex's
+ /// code-extractor signature (`symbol_count`) and the canonical `body`
+ /// field. Every non-code document must come back byte-identical to the
+ /// window computed from the stored chunk offsets, unconditionally.
+ #[test]
+ fn non_code_document_passage_is_byte_identical_regardless_of_language_field() {
+ // Deliberately shaped like the reported bug (a dangling brace mid
+ // window) so a wrongly-triggered AST pass would have visibly changed
+ // the span rather than silently agreeing with it.
+ let text = "\t}\n\n\tif ( isset( $x ) ) {\n\t\tdo_thing();\n\t}\n";
+ let start = text.find("if (").unwrap();
+ let end = start + "if ( isset( $x ) )".len();
+ let source = serde_json::json!({
+ "body": text,
+ "language": "rust",
+ "__xerj_passage_meta__embedding": {
+ "field": "body",
+ "chunks": [[start, end]]
+ }
+ });
+ let passage = passage_match_from_source(&source, "embedding", Some(0)).unwrap();
+ assert_eq!(passage.start_offset as usize, start);
+ assert_eq!(passage.end_offset as usize, end);
+ assert_eq!(passage.text, &text[start..end]);
+ }
+
+ /// #174: a CODE document (autoindex's `language` + `symbol_count` +
+ /// `body` shape) whose stored chunk offsets land mid-statement inside a
+ /// function — exactly the reported bug, a passage opening on a dangling
+ /// brace — must be widened to the smallest enclosing function.
+ #[test]
+ fn code_document_passage_widens_to_enclosing_function() {
+ let body = "fn before() {\n 1\n}\n\nfn target(x: i32) -> i32 {\n let y = x + 1;\n y * 2\n}\n\nfn after() {\n 2\n}\n";
+ let needle = "y * 2";
+ let start = body.find(needle).unwrap();
+ let end = start + needle.len();
+ let source = serde_json::json!({
+ "body": body,
+ "language": "rust",
+ "symbol_count": 3,
+ "__xerj_passage_meta__embedding": {
+ "field": "body",
+ "chunks": [[start, end]]
+ }
+ });
+ let passage = passage_match_from_source(&source, "embedding", Some(0)).unwrap();
+ assert!(
+ passage.text.starts_with("fn target"),
+ "passage must start at the enclosing function's declaration, got: {:?}",
+ passage.text
+ );
+ assert_eq!(
+ passage.text,
+ &body[passage.start_offset as usize..passage.end_offset as usize],
+ "start/end offsets must still slice exactly onto `text`"
+ );
+ assert!(!passage.text.contains("before"));
+ assert!(!passage.text.contains("after"));
+ }
+
#[test]
fn passage_metadata_has_bounded_stored_overhead_without_text_duplication() {
let content = "quarterly finance evidence with stable page-local wording. ".repeat(24);
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 90c614bb7..05f01711a 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -20,6 +20,7 @@ pub mod aggs;
pub mod audit;
pub mod bulk;
mod cluster_state;
+mod code_blocks;
mod collection_publication;
pub mod engine;
pub mod governor;