Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): index each code declaration as its own retrievable document (#500)

xerj-org4a01596bf379
diff --git a/engine/crates/xerj-autoindex/src/extract/code.rs b/engine/crates/xerj-autoindex/src/extract/code.rs
index 1febf1ae5..89e93f4e1 100644
--- a/engine/crates/xerj-autoindex/src/extract/code.rs
+++ b/engine/crates/xerj-autoindex/src/extract/code.rs
@@ -3,14 +3,19 @@
//! Source files used to fall through to the plain-text extractor, so a repo was
//! searchable only as prose. This parses each file with the matching tree-sitter
//! grammar and captures its DEFINITIONS (functions, classes, methods, structs,
-//! traits, interfaces, modules, constants, …). Each file becomes one document
-//! carrying:
+//! traits, interfaces, modules, constants, …). Each file becomes a file-level
+//! document carrying:
//! - `language` the detected language
//! - `symbols` a structured array of {name, kind, line}
//! - `defs` "kind name" per symbol, newline-joined so BM25 matches a query
//! like `class User` or `def save` to the file that owns it
//! - `body` the full source text (still full-text searchable)
//!
+//! PLUS one document PER DECLARATION (#500): `{name, kind, line, code}` under a
+//! unique `code:<line>:<name>` locator, so a constant/field/signature lookup
+//! retrieves the ~40–80 B declaration line as its own unit instead of only
+//! surviving folded inside the enclosing class/method body.
+//!
//! The capture-name in each query IS the symbol kind (`@function`, `@class`, …),
//! so adding a language is a grammar dep + one registry row. If a grammar fails
//! to parse a file, it is indexed as plain text rather than dropped.
@@ -339,8 +344,17 @@ pub fn extract(path: &Path, sn: &crate::sniff::Sniffed, sink: Sink) -> Result<Ex
Ok(stats)
}
-/// (name, kind, 1-based line)
-type Symbol = (String, String, usize);
+/// (name, kind, 1-based line, declaration line text)
+///
+/// `code` is the single source line the declaration starts on — for a
+/// constant/field/`#define`/signature that IS the whole declaration (~40–80 B),
+/// so #500 can promote each declaration to its own retrievable document instead
+/// of only reaching it inside the ~2–3 KB enclosing class/method body. (A
+/// multi-line body's `code` is its signature line; the full body stays in the
+/// file document's `body`.) Language-agnostic on purpose: the capture depth of
+/// the name node varies per grammar, so a tree-walk to the declaration node is
+/// unreliable, whereas the start line is always exact.
+type Symbol = (String, String, usize, String);
/// Evaluate a pattern's text predicates against one match. The core library
/// exposes `#eq?`/`#any-of?` (and their `not-` forms) as *general* predicates
@@ -389,6 +403,9 @@ fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {
parser.set_language(&def.language).ok()?;
let tree = parser.parse(text.as_bytes(), None)?;
let names = def.query.capture_names();
+ // Row → source line, built once (not `lines().nth(row)` per symbol, which
+ // would be O(symbols · file)).
+ let lines: Vec<&str> = text.lines().collect();
let mut out: Vec<Symbol> = Vec::new();
let mut cursor = QueryCursor::new();
let mut it = cursor.matches(&def.query, tree.root_node(), text.as_bytes());
@@ -406,11 +423,15 @@ fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {
if name.is_empty() || name.len() > 200 {
continue;
}
- out.push((
- name.to_string(),
- kind.to_string(),
- node.start_position().row + 1,
- ));
+ let row = node.start_position().row;
+ // The declaration's start line, trimmed of indentation and capped
+ // so a minified/pathological line can't bloat the per-symbol doc.
+ let mut code = lines.get(row).copied().unwrap_or("").trim().to_string();
+ if code.len() > 400 {
+ // Char-boundary-safe cap (String::truncate would panic mid-char).
+ code = code.chars().take(400).collect();
+ }
+ out.push((name.to_string(), kind.to_string(), row + 1, code));
if out.len() >= 5000 {
return Some(out); // pathological generated file — enough
}
@@ -443,13 +464,13 @@ fn emit_code_doc(
let mut seen = std::collections::HashSet::new();
let defs: Vec<String> = symbols
.iter()
- .filter(|(n, k, _)| seen.insert((k.clone(), n.clone())))
- .map(|(n, k, _)| format!("{k} {n}"))
+ .filter(|(n, k, _, _)| seen.insert((k.clone(), n.clone())))
+ .map(|(n, k, _, _)| format!("{k} {n}"))
.collect();
fields.insert("defs".into(), Value::String(defs.join("\n")));
let arr: Vec<Value> = symbols
.iter()
- .map(|(n, k, line)| {
+ .map(|(n, k, line, _)| {
let mut m = Map::new();
m.insert("name".into(), Value::String(n.clone()));
m.insert("kind".into(), Value::String(k.clone()));
@@ -467,7 +488,7 @@ fn emit_code_doc(
fields.insert("body".into(), Value::String(text.to_string()));
stats.records += 1;
- sink(RawRecord {
+ if !sink(RawRecord {
fields,
locator: "code".into(),
group: None,
@@ -475,7 +496,54 @@ fn emit_code_doc(
// something, so a better grammar would otherwise move the file to a
// different dataset and orphan its old document (#178).
origin: FieldOrigin::Extractor,
- })
+ }) {
+ return false;
+ }
+
+ // #500: promote each declaration to its OWN retrievable document. A
+ // constant/field/`#define`/signature lookup then returns the ~40–80 B
+ // declaration line with an exact `name` (mapped keyword), instead of only
+ // surviving folded inside the ~2–3 KB enclosing class/method body (the
+ // measured 32–48× byte blow-up + the rank-35 recall miss). The file
+ // document above keeps `body`/`defs` for full-text and cross-file search.
+ let file_path = path
+ .file_name()
+ .and_then(|n| n.to_str())
+ .unwrap_or("source")
+ .to_string();
+ for (name, kind, line, code) in symbols {
+ if code.is_empty() {
+ continue;
+ }
+ let mut sf = Map::new();
+ // `title` = the symbol so the hit reads as the declaration, `path` for
+ // citation, `name`/`code` searchable (the mapping exposes `name.keyword`
+ // for exact-identifier ranking), `kind`/`line` for filtering + citation.
+ sf.insert("title".into(), Value::String(name.clone()));
+ sf.insert("path".into(), Value::String(file_path.clone()));
+ sf.insert("language".into(), Value::String(language.to_string()));
+ sf.insert("name".into(), Value::String(name.clone()));
+ sf.insert("kind".into(), Value::String(kind.clone()));
+ sf.insert("line".into(), Value::Number((*line as u64).into()));
+ sf.insert("code".into(), Value::String(code.clone()));
+ stats.records += 1;
+ if !sink(RawRecord {
+ fields: sf,
+ // Unique, stable per-declaration locator so `doc_id(dataset,
+ // file_key, locator)` gives each symbol its OWN document instead of
+ // colliding with the file document (all of which used "code"). Keyed
+ // by line+name: unique per declaration, stable across re-index unless
+ // the line moves, and it dedups a declaration double-captured under
+ // two kinds (e.g. `export const f = function(){}` → @function+@const)
+ // to a single doc at the same line.
+ locator: format!("code:{line}:{name}"),
+ group: None,
+ origin: FieldOrigin::Extractor,
+ }) {
+ return false;
+ }
+ }
+ true
}
// ── Per-language capture queries. Capture-name == emitted symbol kind. ─────────
@@ -1094,7 +1162,7 @@ mod tests {
parse_symbols(def, src).unwrap()
}
fn has(s: &[Symbol], name: &str, kind: &str) -> bool {
- s.iter().any(|(n, k, _)| n == name && k == kind)
+ s.iter().any(|(n, k, _, _)| n == name && k == kind)
}
#[test]
@@ -1118,8 +1186,16 @@ mod tests {
true
})
.unwrap();
- assert_eq!((stats.records, stats.junk), (1, 0));
+ // #500: the file-level document + one per-declaration document
+ // (`alpha_helper`), no junk. records[0] is the file document.
+ assert_eq!((stats.records, stats.junk), (2, 0));
assert_eq!(records[0].fields["language"], "python");
+ assert!(
+ records
+ .iter()
+ .any(|r| r.fields.get("name").is_some_and(|n| n == "alpha_helper")),
+ "the function must be its own symbol document (#500)"
+ );
// Title comes from the logical name, not the blob ordinal.
assert_eq!(records[0].fields["title"], "app.py");
}
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 95e5f9a43..b729239ea 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -715,7 +715,9 @@ fn the_legacy_terminal_line_and_run_document_report_code_coverage() {
let ast = locked
.docs
.values()
- .find(|doc| doc.get("language").is_some())
+ // The file-level AST document carries `defs`; the #500 per-symbol
+ // documents also carry `language` but no `defs`, so select on `defs`.
+ .find(|doc| doc.get("language").is_some() && doc.get("defs").is_some())
.unwrap_or_else(|| panic!("the legacy path indexes an AST document for alpha.rs"));
assert_eq!(ast["language"], "rust", "{ast}");
assert_eq!(ast["title"], "alpha.rs", "{ast}");
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 1a539c4b1..f6e06a4a5 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -1624,15 +1624,35 @@ fn code_files_index_with_ast_fields_and_the_run_reports_code_coverage() {
assert_eq!(summary["code_files_junked"], 0, "{summary}");
let docs = endpoint.data_docs();
+ // The per-file AST document carries `defs`/`symbols`; the #500 per-symbol
+ // documents carry `code`/`name` but no `defs`, so filter on `defs` to keep
+ // the one-per-file assertions honest about the file-level document.
let mut code_docs: Vec<&Value> = docs
.iter()
- .filter(|doc| doc.get("language").is_some())
+ .filter(|doc| doc.get("language").is_some() && doc.get("defs").is_some())
.collect();
code_docs.sort_by_key(|doc| doc["language"].as_str().unwrap_or("").to_owned());
assert_eq!(
code_docs.len(),
2,
- "one AST document per source file: {docs:?}"
+ "one AST (file-level) document per source file: {docs:?}"
+ );
+ // #500: each declaration is also promoted to its own retrievable document
+ // (has `code` + `name`, no `defs`). At least the Rust struct + Python class
+ // must each be their own symbol document, not only foldable inside a parent.
+ let symbol_docs: Vec<&Value> = docs
+ .iter()
+ .filter(|doc| doc.get("code").is_some() && doc.get("name").is_some())
+ .collect();
+ assert!(
+ symbol_docs
+ .iter()
+ .any(|d| d["name"] == "AlphaConfig" && d.get("defs").is_none()),
+ "the AlphaConfig struct must be its own symbol document (#500): {symbol_docs:?}"
+ );
+ assert!(
+ symbol_docs.iter().any(|d| d["name"] == "GammaStore"),
+ "the GammaStore class must be its own symbol document (#500): {symbol_docs:?}"
);
assert_eq!(code_docs[0]["language"], "python");
assert_eq!(code_docs[1]["language"], "rust");
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index 48316b171..df3eeaa1f 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -2298,10 +2298,14 @@ mod tests {
)
.unwrap();
let prepared = snapshot.files[0].prepared.as_ref().unwrap();
- assert_eq!(
- (prepared.records, prepared.junk),
- (1, 0),
- "a code file must prepare one document, not silent junk"
+ // #500: a code file prepares the file-level document PLUS one document
+ // per declaration — so records is now ≥1 (was exactly 1). The invariant
+ // the test guards is "documents, not silent junk": junk stays 0.
+ assert!(
+ prepared.records >= 1 && prepared.junk == 0,
+ "a code file must prepare documents, not silent junk: records={} junk={}",
+ prepared.records,
+ prepared.junk
);
let ndjson = std::fs::read_to_string(
state