Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): --code-analyzer standard|code (#1198)

Tom Villani6768ff12fd2d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9939fef5c..c361300bc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Added
+
+- **`xerj autoindex --code-analyzer standard|code`** (issue
+ [#1198](https://github.com/xerj-org/xerj/issues/1198)) — `code` declares the engine's built-in
+ identifier-aware analyzer (standard tokenizer + word-delimiter + lowercase: `snake_case`,
+ `camelCase` and dotted identifiers gain their sub-words, the original token is kept) on newly
+ created datasets that hold at least one source file, so a query that says "connection pool"
+ reaches `get_connection_pool` through BM25. Default `standard` keeps today's election
+ unchanged. With `code` elected the #1059 stemmer defers for that dataset. It rides the #1059
+ create-body path: committed `index_identity` digests do not move and existing indexes keep
+ their analyzer. Measured on SWE-bench Lite file localization: acc@5 +0.11 [+0.03, +0.19] on a
+ pre-registered held-out sample; cost ~15% index size and ~2.4× text-analysis time.
+
### Fixed
- **`xerj corpus index` now proves the store answers a search before it says "searchable"** (issue
diff --git a/docs/recipes/zero-config-indexing.md b/docs/recipes/zero-config-indexing.md
index f43df71cd..debea78c5 100644
--- a/docs/recipes/zero-config-indexing.md
+++ b/docs/recipes/zero-config-indexing.md
@@ -456,7 +456,14 @@ outside the indexed folder is refused, and one inside a hidden directory is
skipped like any dotfile, however visible the link's own name;
`--follow-symlinks-outside-root` waives the folder boundary and nothing else),
`--sample N`
-records per file for inference (default 500).
+records per file for inference (default 500), `--code-analyzer code` to analyze
+newly created datasets that hold source code with the built-in identifier-aware
+`code` analyzer instead of today's election (`standard`, or the stemmer when the
+sample reads as prose) — `get_connection_pool` then also matches `connection`
+and `pool`. It costs about 15% more index size and ~2.4× text-analysis time;
+measured +0.11 file-localization acc@5 on a held-out SWE-bench Lite sample
+([#1198](https://github.com/xerj-org/xerj/issues/1198)). Existing indexes keep
+the analyzer they were created with; rebuild under a new `--prefix` to switch.
Build output never reaches the index in the first place. The walk honours
`.xerjignore`, `.gitignore` (nested files and `!negation` included) and
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index b3e98ba8b..43cd2a374 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -82,6 +82,9 @@ pub struct IndexCfg {
pub max_file_gb: u64,
pub sample: usize,
pub no_semantic: bool,
+ /// `--code-analyzer` (#1198): `code` declares the built-in identifier-aware
+ /// analyzer on newly created datasets that hold source code.
+ pub code_analyzer: crate::infer::CodeAnalyzer,
/// Second-brain name; None derives it from the root folder basename.
pub brain: Option<String>,
/// Disable edge detection entirely (no `.xerj-memory-*-edges` writes).
@@ -294,6 +297,12 @@ pub fn help_text_with(feedback: bool) -> String {
--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\
--sample <N> records sampled per file for inference (default 500)\n\
--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\
+ --code-analyzer <A> standard (default) or code: `code` declares the built-in\n\
+ identifier-aware analyzer (splits snake_case/camelCase,\n\
+ keeps the original token) on NEWLY created datasets\n\
+ holding source code; existing indexes keep theirs.\n\
+ Measured +0.11 file-localization acc@5 (#1198); costs\n\
+ ~15% index size and ~2.4x text-analysis time\n\
--brain <NAME> second-brain name; relationship edges land in\n\
.xerj-memory-<NAME>-edges (default: folder name slug)\n\
--no-graph skip relationship detection (wikilinks, local links,\n\
@@ -666,6 +675,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
let mut max_file_gb = 2u64;
let mut sample = 500usize;
let mut no_semantic = false;
+ let mut code_analyzer = crate::infer::CodeAnalyzer::default();
let mut brain: Option<String> = None;
let mut no_graph = false;
let mut watch = false;
@@ -898,6 +908,12 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
"--dry-run" => dry_run = true,
"--json" => json = true,
"--md" => json = false,
+ "--code-analyzer" => {
+ let raw = it
+ .next()
+ .ok_or("--code-analyzer needs a value: standard or code")?;
+ code_analyzer = crate::infer::CodeAnalyzer::parse(&raw)?;
+ }
"--progress" => {
let raw = it
.next()
@@ -1467,6 +1483,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
max_file_gb,
sample: sample.max(50),
no_semantic,
+ code_analyzer,
brain,
no_graph,
dry_run,
@@ -2096,6 +2113,33 @@ mod tests {
);
}
+ /// #1198: `--code-analyzer` defaults to `standard`, takes `code`, and
+ /// refuses anything else by name instead of silently falling back.
+ #[test]
+ fn code_analyzer_flag_parses_and_refuses_unknown_values() {
+ assert_eq!(
+ index(&["data"]).code_analyzer,
+ crate::infer::CodeAnalyzer::Standard
+ );
+ assert_eq!(
+ index(&["data", "--code-analyzer", "code"]).code_analyzer,
+ crate::infer::CodeAnalyzer::Code
+ );
+ assert_eq!(
+ index(&["data", "--code-analyzer", "standard"]).code_analyzer,
+ crate::infer::CodeAnalyzer::Standard
+ );
+ for args in [
+ vec!["data", "--code-analyzer"],
+ vec!["data", "--code-analyzer", "stemmer"],
+ ] {
+ let rendered = args.join(" ");
+ let err = parse(args.into_iter().map(str::to_string).collect())
+ .expect_err(&format!("`{rendered}` must be refused"));
+ assert!(err.contains("--code-analyzer"), "{err}");
+ }
+ }
+
/// `--approve fast` is an instruction, not a label: the run must really
/// drop semantic fields and edges. Accepting it and indexing everything
/// anyway is the #204 defect class.
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
index c31c5855c..51eece96c 100644
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -365,6 +365,7 @@ pub(super) fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
max_file_gb: 1,
sample: 50,
no_semantic: true,
+ code_analyzer: Default::default(),
brain: Some("notes".into()),
no_graph: false,
// The gate is switched off in these fixtures on purpose: they assert
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 5f762b725..c7c4bd8fa 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -778,6 +778,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
max_file_gb: 1,
sample: 50,
no_semantic: true,
+ code_analyzer: Default::default(),
// This module preserves legacy graph-enabled failure behavior;
// incremental non-graph behavior has its own HTTP suite.
brain: None,
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 4aa75676b..0c47d7faf 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -841,6 +841,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {
max_file_gb: 1,
sample: 50,
no_semantic: !semantic,
+ code_analyzer: Default::default(),
brain: None,
no_graph: true,
// Gate off: these fixtures assert reconcile behaviour, not a
diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rs
index a797106f3..610f1a8b1 100644
--- a/engine/crates/xerj-autoindex/src/infer/mod.rs
+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs
@@ -746,6 +746,69 @@ pub fn elected_default_analyzer(
.then_some(STEMMING_TEXT_ANALYZER)
}
+/// The analyzer `--code-analyzer code` declares (#1198): the engine's built-in
+/// identifier-aware analyzer — standard tokenizer, then a word-delimiter
+/// filter that adds the sub-words of `snake_case`/`camelCase`/dotted
+/// identifiers at the original token's position (the original is kept), then
+/// lowercase (`AnalyzerRegistry::register_defaults`, xerj-fts). Under the
+/// `standard` or `stemmer` analyzer `get_connection_pool` is ONE term, so a
+/// query that says "connection pool" cannot reach it through BM25.
+///
+/// Measured on SWE-bench Lite file localization (#1198): acc@5 +0.11
+/// [+0.03, +0.19] on a pre-registered held-out sample; index size +15–17%;
+/// text analysis ~2.4× slower for whole-file records (seconds per repo).
+pub const CODE_TEXT_ANALYZER: &str = "code";
+
+/// `--code-analyzer`: which analyzer a NEWLY CREATED dataset index that holds
+/// source code gets.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum CodeAnalyzer {
+ /// Today's election, unchanged: `standard`, or the #1059 stemmer when the
+ /// sample carries prose — which source code usually does.
+ #[default]
+ Standard,
+ /// [`CODE_TEXT_ANALYZER`] for any dataset with at least one code member.
+ Code,
+}
+
+impl CodeAnalyzer {
+ pub fn parse(raw: &str) -> Result<Self, String> {
+ match raw {
+ "standard" => Ok(Self::Standard),
+ "code" => Ok(Self::Code),
+ other => Err(format!(
+ "--code-analyzer: unknown value `{other}` (expected `standard` or `code`)"
+ )),
+ }
+ }
+}
+
+/// The dataset's text-analyzer election, `--code-analyzer` included.
+///
+/// Code files are not a dataset of their own: they share their scope's
+/// document dataset with prose and demoted config files (`dataset::cluster`).
+/// So the unit is "a dataset holding at least one file the sniffer routed to
+/// the code extractor" (`has_code`), and when `--code-analyzer code` applies,
+/// every text field of that index is analyzed with [`CODE_TEXT_ANALYZER`] —
+/// prose members included, which the word-delimiter filter leaves mostly as
+/// they were (it keeps every original token).
+///
+/// `code` wins over the stemmer (the maintainer's call on #1198): the flag is
+/// an explicit operator choice, sub-words already reach most of what stemming
+/// would in prose inside code, and layering both is unmeasured. A dataset
+/// without code members keeps [`elected_default_analyzer`]'s answer either way.
+pub fn elect_dataset_analyzer(
+ specs: &[FieldSpec],
+ fields: &HashMap<String, FieldAcc>,
+ has_code: bool,
+ code_analyzer: CodeAnalyzer,
+) -> Option<&'static str> {
+ if has_code && code_analyzer == CodeAnalyzer::Code {
+ return Some(CODE_TEXT_ANALYZER);
+ }
+ elected_default_analyzer(specs, fields)
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1014,6 +1077,73 @@ mod tests {
assert_eq!(elected_default_analyzer(&specs, &fields), Some("stemmer"));
}
+ /// #1198: a sample that elects the stemmer on its own. Source code does —
+ /// the profiler's prose predicate counts it as natural language — so this
+ /// is the common case `--code-analyzer code` has to win against.
+ fn stemmer_electing_sample() -> (Vec<FieldSpec>, HashMap<String, FieldAcc>) {
+ let prose = "The connection pool retries every failed handshake with backoff. \
+ Each worker owns one socket and never shares it across threads.";
+ let mut acc = FieldAcc::default();
+ for _ in 0..20 {
+ acc.add(&Value::String(format!("{prose} {prose}")));
+ }
+ let mut fields = HashMap::new();
+ fields.insert("body".to_string(), acc);
+ let specs = infer_fields(&fields, 40, true);
+ assert_eq!(elected_default_analyzer(&specs, &fields), Some("stemmer"));
+ (specs, fields)
+ }
+
+ /// #1198: with `--code-analyzer code`, a dataset holding code elects the
+ /// `code` analyzer and the stemmer election defers to it.
+ #[test]
+ fn code_analyzer_flag_elects_code_for_a_code_dataset_and_the_stemmer_defers() {
+ let (specs, fields) = stemmer_electing_sample();
+ assert_eq!(
+ elect_dataset_analyzer(&specs, &fields, true, CodeAnalyzer::Code),
+ Some(CODE_TEXT_ANALYZER)
+ );
+ }
+
+ /// #1198: the flag only reaches datasets that hold code; a prose-only
+ /// dataset keeps today's election under either value.
+ #[test]
+ fn code_analyzer_flag_leaves_a_dataset_without_code_on_todays_election() {
+ let (specs, fields) = stemmer_electing_sample();
+ assert_eq!(
+ elect_dataset_analyzer(&specs, &fields, false, CodeAnalyzer::Code),
+ Some(STEMMING_TEXT_ANALYZER)
+ );
+ assert_eq!(
+ elect_dataset_analyzer(&[], &HashMap::new(), false, CodeAnalyzer::Code),
+ None
+ );
+ }
+
+ /// #1198: the default (`standard`) changes nothing, code or not.
+ #[test]
+ fn default_code_analyzer_keeps_todays_election() {
+ let (specs, fields) = stemmer_electing_sample();
+ assert_eq!(CodeAnalyzer::default(), CodeAnalyzer::Standard);
+ for has_code in [true, false] {
+ assert_eq!(
+ elect_dataset_analyzer(&specs, &fields, has_code, CodeAnalyzer::Standard),
+ elected_default_analyzer(&specs, &fields)
+ );
+ }
+ }
+
+ #[test]
+ fn code_analyzer_parses_its_two_values_and_names_a_bad_one() {
+ assert_eq!(CodeAnalyzer::parse("standard"), Ok(CodeAnalyzer::Standard));
+ assert_eq!(CodeAnalyzer::parse("code"), Ok(CodeAnalyzer::Code));
+ let err = CodeAnalyzer::parse("stemmer").unwrap_err();
+ assert!(
+ err.contains("stemmer") && err.contains("standard") && err.contains("code"),
+ "{err}"
+ );
+ }
+
/// #1059: prose sampled on a field the profiler types KEYWORD does not
/// elect the analyzer through that field — the declaration targets text
/// fields, and a keyword field's whole-value term space must not be part
diff --git a/engine/crates/xerj-autoindex/src/label_http_tests.rs b/engine/crates/xerj-autoindex/src/label_http_tests.rs
index 0314f968d..9127007d8 100644
--- a/engine/crates/xerj-autoindex/src/label_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/label_http_tests.rs
@@ -335,6 +335,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, label: &Path) -> IndexCfg {
max_file_gb: 1,
sample: 100,
no_semantic: true,
+ code_analyzer: Default::default(),
brain: None,
no_graph: true,
max_minutes: 0,
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index df31ed4b6..58632907e 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -2945,6 +2945,7 @@ mod phase_a_grouping_tests {
max_file_gb: 2,
sample: 500,
no_semantic: false,
+ code_analyzer: Default::default(),
brain: None,
no_graph: true,
// The gate is switched off in these fixtures on purpose: they assert
@@ -2965,6 +2966,10 @@ mod phase_a_grouping_tests {
}
fn plan_for(root: &Path) -> Plan {
+ plan_for_cfg(root, &cfg_for(root))
+ }
+
+ fn plan_for_cfg(root: &Path, cfg: &IndexCfg) -> Plan {
let files = walk::walk(root, false).unwrap();
let keys: Vec<String> = files
.iter()
@@ -2984,16 +2989,78 @@ mod phase_a_grouping_tests {
meter: &meter,
tally: &ScanTally::default(),
};
- build_phase_a(
- root,
- &files,
- &keys,
- &digests,
- Vec::new(),
- &ctx,
- &cfg_for(root),
+ build_phase_a(root, &files, &keys, &digests, Vec::new(), &ctx, cfg).plan
+ }
+
+ /// #1198 end to end through the real planner: `--code-analyzer code` puts
+ /// `code` on the document dataset that holds source (README included —
+ /// code and prose share the per-scope dataset), while the default leaves
+ /// today's election in place.
+ #[test]
+ fn code_analyzer_code_elects_code_on_a_dataset_holding_source() {
+ let dir = tempfile::tempdir().unwrap();
+ let root = dir.path();
+ std::fs::write(root.join("server.c"), CODE).unwrap();
+ std::fs::write(root.join("README.md"), PROSE).unwrap();
+
+ let mut cfg = cfg_for(root);
+ cfg.code_analyzer = infer::CodeAnalyzer::Code;
+ let with_flag = plan_for_cfg(root, &cfg);
+ let analyzers: Vec<_> = with_flag
+ .datasets
+ .iter()
+ .map(|d| d.text_analyzer.as_deref())
+ .collect();
+ assert_eq!(
+ analyzers,
+ vec![Some(infer::CODE_TEXT_ANALYZER)],
+ "{:?}",
+ with_flag.datasets
+ );
+
+ let default = plan_for(root);
+ assert!(
+ default
+ .datasets
+ .iter()
+ .all(|d| d.text_analyzer.as_deref() != Some(infer::CODE_TEXT_ANALYZER)),
+ "the default must not elect `code`: {:?}",
+ default.datasets
+ );
+ }
+
+ /// #1198: a folder with no file the sniffer routes to the code extractor
+ /// never gets `code`, flag or not — the dataset is labelled
+ /// `Family::Code` regardless of content, so the election must look at the
+ /// members' own families.
+ #[test]
+ fn code_analyzer_code_leaves_a_prose_only_dataset_alone() {
+ let dir = tempfile::tempdir().unwrap();
+ let root = dir.path();
+ std::fs::write(root.join("README.md"), PROSE).unwrap();
+ std::fs::write(
+ root.join("GUIDE.md"),
+ format!("{PROSE}\nSee the handler table.\n"),
)
- .plan
+ .unwrap();
+
+ let mut cfg = cfg_for(root);
+ cfg.code_analyzer = infer::CodeAnalyzer::Code;
+ let with_flag = plan_for_cfg(root, &cfg);
+ assert!(!with_flag.datasets.is_empty());
+ assert_eq!(
+ with_flag
+ .datasets
+ .iter()
+ .map(|d| d.text_analyzer.clone())
+ .collect::<Vec<_>>(),
+ plan_for(root)
+ .datasets
+ .iter()
+ .map(|d| d.text_analyzer.clone())
+ .collect::<Vec<_>>(),
+ "a prose-only dataset keeps today's election"
+ );
}
const CODE: &str = "// The event loop dispatches every ready connection to a worker.\n\
@@ -3552,7 +3619,16 @@ fn build_phase_a(
// #1059: the dataset's text-analyzer election, from the same sampled
// accumulators that elected the semantic body above. Deterministic in
// the sample, so the same bytes elect the same analyzer on every run.
- let text_analyzer = infer::elected_default_analyzer(&specs, &c.fields).map(str::to_string);
+ // #1198: `--code-analyzer code` overrides it for a dataset holding any
+ // file the sniffer routed to the code extractor. Per-file family, not
+ // `c.family`: a docs cluster is labelled `Family::Code` whatever it holds.
+ let has_code = c
+ .members
+ .iter()
+ .any(|&m| matches!(file_meta[m], Some((Family::Code, _))));
+ let text_analyzer =
+ infer::elect_dataset_analyzer(&specs, &c.fields, has_code, cfg.code_analyzer)
+ .map(str::to_string);
datasets.push(PlanDataset {
slug: c.slug.clone(),
index: format!("{}-{}", cfg.prefix, c.slug),
@@ -10761,6 +10837,30 @@ mod stem_default_tests {
assert!(body.pointer("/mappings/properties/body/analyzer").is_none());
}
+ /// #1198: a code dataset elected under `--code-analyzer code` declares the
+ /// built-in `code` analyzer on the CREATE body, through the same #1059
+ /// path: the mapping is byte-identical to an undeclaring dataset's, and
+ /// the frozen identities do not move (existing state dirs never abort).
+ #[test]
+ fn a_code_analyzer_dataset_declares_code_on_the_create_body_outside_the_identities() {
+ let body = dataset_create_body(&dataset("code", Some(infer::CODE_TEXT_ANALYZER)));
+ assert_eq!(
+ body.pointer("/settings/analysis/analyzer/default/type"),
+ Some(&json!("code")),
+ "{body}"
+ );
+ let undeclaring = dataset_create_body(&dataset("code", None));
+ assert_eq!(body["mappings"], undeclaring["mappings"]);
+
+ let mut plan = Plan {
+ datasets: vec![dataset("code", None)],
+ ..Plan::default()
+ };
+ let before = generation_contract_identities(&plan).unwrap();
+ plan.datasets[0].text_analyzer = Some(infer::CODE_TEXT_ANALYZER.to_string());
+ assert_eq!(before, generation_contract_identities(&plan).unwrap());
+ }
+
/// A dataset with no profiler-marked prose sends no ANALYZER settings —
/// the create body's analysis side is empty, so the server keeps
/// `standard`'s exact term space for identifier/enum data. The #1093
diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rs
index 4021dbdd5..5bc32d91d 100644
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@ -627,6 +627,7 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {
max_file_gb: 2,
sample: 500,
no_semantic: false,
+ code_analyzer: Default::default(),
brain: Some(brain.to_string()),
no_graph: false,
dry_run: false,