AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): index each code declaration as its own retrievable document (#500)
xerj-org4a01596bf379
diff --git a/engine/crates/xerj-autoindex/src/extract/code.rs b/engine/crates/xerj-autoindex/src/extract/code.rsindex 1febf1ae5..89e93f4e1 100644--- a/engine/crates/xerj-autoindex/src/extract/code.rs+++ b/engine/crates/xerj-autoindex/src/extract/code.rs@@ -3,14 +3,19 @@//! Source files used to fall through to the plain-text extractor, so a repo was//! searchable only as prose. This parses each file with the matching tree-sitter//! grammar and captures its DEFINITIONS (functions, classes, methods, structs,-//! traits, interfaces, modules, constants, …). Each file becomes one document-//! carrying:+//! traits, interfaces, modules, constants, …). Each file becomes a file-level+//! document carrying://! - `language` the detected language//! - `symbols` a structured array of {name, kind, line}//! - `defs` "kind name" per symbol, newline-joined so BM25 matches a query//! like `class User` or `def save` to the file that owns it//! - `body` the full source text (still full-text searchable)//!+//! PLUS one document PER DECLARATION (#500): `{name, kind, line, code}` under a+//! unique `code:<line>:<name>` locator, so a constant/field/signature lookup+//! retrieves the ~40–80 B declaration line as its own unit instead of only+//! surviving folded inside the enclosing class/method body.+//!//! The capture-name in each query IS the symbol kind (`@function`, `@class`, …),//! so adding a language is a grammar dep + one registry row. If a grammar fails//! to parse a file, it is indexed as plain text rather than dropped.@@ -339,8 +344,17 @@ pub fn extract(path: &Path, sn: &crate::sniff::Sniffed, sink: Sink) -> Result<ExOk(stats)}-/// (name, kind, 1-based line)-type Symbol = (String, String, usize);+/// (name, kind, 1-based line, declaration line text)+///+/// `code` is the single source line the declaration starts on — for a+/// constant/field/`#define`/signature that IS the whole declaration (~40–80 B),+/// so #500 can promote each declaration to its own retrievable document instead+/// of only reaching it inside the ~2–3 KB enclosing class/method body. (A+/// multi-line body's `code` is its signature line; the full body stays in the+/// file document's `body`.) Language-agnostic on purpose: the capture depth of+/// the name node varies per grammar, so a tree-walk to the declaration node is+/// unreliable, whereas the start line is always exact.+type Symbol = (String, String, usize, String);/// Evaluate a pattern's text predicates against one match. The core library/// exposes `#eq?`/`#any-of?` (and their `not-` forms) as *general* predicates@@ -389,6 +403,9 @@ fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {parser.set_language(&def.language).ok()?;let tree = parser.parse(text.as_bytes(), None)?;let names = def.query.capture_names();+ // Row → source line, built once (not `lines().nth(row)` per symbol, which+ // would be O(symbols · file)).+ let lines: Vec<&str> = text.lines().collect();let mut out: Vec<Symbol> = Vec::new();let mut cursor = QueryCursor::new();let mut it = cursor.matches(&def.query, tree.root_node(), text.as_bytes());@@ -406,11 +423,15 @@ fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {if name.is_empty() || name.len() > 200 {continue;}- out.push((- name.to_string(),- kind.to_string(),- node.start_position().row + 1,- ));+ let row = node.start_position().row;+ // The declaration's start line, trimmed of indentation and capped+ // so a minified/pathological line can't bloat the per-symbol doc.+ let mut code = lines.get(row).copied().unwrap_or("").trim().to_string();+ if code.len() > 400 {+ // Char-boundary-safe cap (String::truncate would panic mid-char).+ code = code.chars().take(400).collect();+ }+ out.push((name.to_string(), kind.to_string(), row + 1, code));if out.len() >= 5000 {return Some(out); // pathological generated file — enough}@@ -443,13 +464,13 @@ fn emit_code_doc(let mut seen = std::collections::HashSet::new();let defs: Vec<String> = symbols.iter()- .filter(|(n, k, _)| seen.insert((k.clone(), n.clone())))- .map(|(n, k, _)| format!("{k} {n}"))+ .filter(|(n, k, _, _)| seen.insert((k.clone(), n.clone())))+ .map(|(n, k, _, _)| format!("{k} {n}")).collect();fields.insert("defs".into(), Value::String(defs.join("\n")));let arr: Vec<Value> = symbols.iter()- .map(|(n, k, line)| {+ .map(|(n, k, line, _)| {let mut m = Map::new();m.insert("name".into(), Value::String(n.clone()));m.insert("kind".into(), Value::String(k.clone()));@@ -467,7 +488,7 @@ fn emit_code_doc(fields.insert("body".into(), Value::String(text.to_string()));stats.records += 1;- sink(RawRecord {+ if !sink(RawRecord {fields,locator: "code".into(),group: None,@@ -475,7 +496,54 @@ fn emit_code_doc(// something, so a better grammar would otherwise move the file to a// different dataset and orphan its old document (#178).origin: FieldOrigin::Extractor,- })+ }) {+ return false;+ }++ // #500: promote each declaration to its OWN retrievable document. A+ // constant/field/`#define`/signature lookup then returns the ~40–80 B+ // declaration line with an exact `name` (mapped keyword), instead of only+ // surviving folded inside the ~2–3 KB enclosing class/method body (the+ // measured 32–48× byte blow-up + the rank-35 recall miss). The file+ // document above keeps `body`/`defs` for full-text and cross-file search.+ let file_path = path+ .file_name()+ .and_then(|n| n.to_str())+ .unwrap_or("source")+ .to_string();+ for (name, kind, line, code) in symbols {+ if code.is_empty() {+ continue;+ }+ let mut sf = Map::new();+ // `title` = the symbol so the hit reads as the declaration, `path` for+ // citation, `name`/`code` searchable (the mapping exposes `name.keyword`+ // for exact-identifier ranking), `kind`/`line` for filtering + citation.+ sf.insert("title".into(), Value::String(name.clone()));+ sf.insert("path".into(), Value::String(file_path.clone()));+ sf.insert("language".into(), Value::String(language.to_string()));+ sf.insert("name".into(), Value::String(name.clone()));+ sf.insert("kind".into(), Value::String(kind.clone()));+ sf.insert("line".into(), Value::Number((*line as u64).into()));+ sf.insert("code".into(), Value::String(code.clone()));+ stats.records += 1;+ if !sink(RawRecord {+ fields: sf,+ // Unique, stable per-declaration locator so `doc_id(dataset,+ // file_key, locator)` gives each symbol its OWN document instead of+ // colliding with the file document (all of which used "code"). Keyed+ // by line+name: unique per declaration, stable across re-index unless+ // the line moves, and it dedups a declaration double-captured under+ // two kinds (e.g. `export const f = function(){}` → @function+@const)+ // to a single doc at the same line.+ locator: format!("code:{line}:{name}"),+ group: None,+ origin: FieldOrigin::Extractor,+ }) {+ return false;+ }+ }+ true}// ── Per-language capture queries. Capture-name == emitted symbol kind. ─────────@@ -1094,7 +1162,7 @@ mod tests {parse_symbols(def, src).unwrap()}fn has(s: &[Symbol], name: &str, kind: &str) -> bool {- s.iter().any(|(n, k, _)| n == name && k == kind)+ s.iter().any(|(n, k, _, _)| n == name && k == kind)}#[test]@@ -1118,8 +1186,16 @@ mod tests {true}).unwrap();- assert_eq!((stats.records, stats.junk), (1, 0));+ // #500: the file-level document + one per-declaration document+ // (`alpha_helper`), no junk. records[0] is the file document.+ assert_eq!((stats.records, stats.junk), (2, 0));assert_eq!(records[0].fields["language"], "python");+ assert!(+ records+ .iter()+ .any(|r| r.fields.get("name").is_some_and(|n| n == "alpha_helper")),+ "the function must be its own symbol document (#500)"+ );// Title comes from the logical name, not the blob ordinal.assert_eq!(records[0].fields["title"], "app.py");}diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 95e5f9a43..b729239ea 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -715,7 +715,9 @@ fn the_legacy_terminal_line_and_run_document_report_code_coverage() {let ast = locked.docs.values()- .find(|doc| doc.get("language").is_some())+ // The file-level AST document carries `defs`; the #500 per-symbol+ // documents also carry `language` but no `defs`, so select on `defs`.+ .find(|doc| doc.get("language").is_some() && doc.get("defs").is_some()).unwrap_or_else(|| panic!("the legacy path indexes an AST document for alpha.rs"));assert_eq!(ast["language"], "rust", "{ast}");assert_eq!(ast["title"], "alpha.rs", "{ast}");diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 1a539c4b1..f6e06a4a5 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -1624,15 +1624,35 @@ fn code_files_index_with_ast_fields_and_the_run_reports_code_coverage() {assert_eq!(summary["code_files_junked"], 0, "{summary}");let docs = endpoint.data_docs();+ // The per-file AST document carries `defs`/`symbols`; the #500 per-symbol+ // documents carry `code`/`name` but no `defs`, so filter on `defs` to keep+ // the one-per-file assertions honest about the file-level document.let mut code_docs: Vec<&Value> = docs.iter()- .filter(|doc| doc.get("language").is_some())+ .filter(|doc| doc.get("language").is_some() && doc.get("defs").is_some()).collect();code_docs.sort_by_key(|doc| doc["language"].as_str().unwrap_or("").to_owned());assert_eq!(code_docs.len(),2,- "one AST document per source file: {docs:?}"+ "one AST (file-level) document per source file: {docs:?}"+ );+ // #500: each declaration is also promoted to its own retrievable document+ // (has `code` + `name`, no `defs`). At least the Rust struct + Python class+ // must each be their own symbol document, not only foldable inside a parent.+ let symbol_docs: Vec<&Value> = docs+ .iter()+ .filter(|doc| doc.get("code").is_some() && doc.get("name").is_some())+ .collect();+ assert!(+ symbol_docs+ .iter()+ .any(|d| d["name"] == "AlphaConfig" && d.get("defs").is_none()),+ "the AlphaConfig struct must be its own symbol document (#500): {symbol_docs:?}"+ );+ assert!(+ symbol_docs.iter().any(|d| d["name"] == "GammaStore"),+ "the GammaStore class must be its own symbol document (#500): {symbol_docs:?}");assert_eq!(code_docs[0]["language"], "python");assert_eq!(code_docs[1]["language"], "rust");diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex 48316b171..df3eeaa1f 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -2298,10 +2298,14 @@ mod tests {).unwrap();let prepared = snapshot.files[0].prepared.as_ref().unwrap();- assert_eq!(- (prepared.records, prepared.junk),- (1, 0),- "a code file must prepare one document, not silent junk"+ // #500: a code file prepares the file-level document PLUS one document+ // per declaration — so records is now ≥1 (was exactly 1). The invariant+ // the test guards is "documents, not silent junk": junk stays 0.+ assert!(+ prepared.records >= 1 && prepared.junk == 0,+ "a code file must prepare documents, not silent junk: records={} junk={}",+ prepared.records,+ prepared.junk);let ndjson = std::fs::read_to_string(state