AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
refactor(common): one glue-token normalizer for xerj search and xerj_search (#1364 review)
xerj-org6474fff06fcb
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rsindex 434d62224..22ebb7f9b 100644--- a/engine/crates/xerj-autoindex/src/search.rs+++ b/engine/crates/xerj-autoindex/src/search.rs@@ -72,190 +72,9 @@ pub(crate) fn is_identifier(q: &str) -> bool {&& q.chars().all(|c| c.is_alphanumeric() || c == '_')}-fn is_two_letter_tech_term(w: &str) -> bool {- matches!(- w,- "db" | "ui"- | "ux"- | "ip"- | "gc"- | "vm"- | "id"- | "kv"- | "ir"- | "ci"- | "pr"- | "ai"- | "ml"- | "tx"- | "rx"- | "fs"- | "fd"- | "pk"- | "fk"- )-}--fn is_three_letter_glue_word(w: &str) -> bool {- matches!(- w,- "del"- | "por"- | "los"- | "las"- | "una"- | "uno"- | "con"- | "sin"- | "sus"- | "mis"- | "tus"- | "nos"- | "les"- | "mas"- | "muy"- | "tan"- | "que"- | "son"- | "fue"- | "era"- | "ser"- | "hay"- | "the"- | "and"- | "for"- | "not"- | "are"- | "but"- | "can"- | "all"- | "has"- | "had"- | "was"- | "one"- | "out"- | "who"- | "how"- | "why"- | "his"- | "her"- | "its"- | "our"- | "you"- | "any"- | "may"- | "via"- | "per"- | "des"- | "aux"- | "par"- | "sur"- | "une"- | "est"- | "ses"- | "vos"- | "ils"- | "pas"- | "qui"- | "der"- | "die"- | "das"- | "den"- | "dem"- | "ein"- | "und"- | "mit"- | "von"- | "aus"- | "bei"- | "vor"- | "zur"- | "zum"- | "wie"- | "wir"- | "sie"- | "ich"- | "uma"- | "dos"- | "nas"- | "com"- | "sem"- | "seu"- | "sua"- | "nao"- | "gli"- | "nel"- | "sul"- | "col"- | "tra"- | "fra"- | "che"- | "non"- | "piu"- )-}--fn is_natural_language_glue_token(clean: &str) -> bool {- if clean.is_empty() || clean.contains('_') {- return false;- }- if !clean.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) {- return false;- }- let char_len = clean.chars().count();- (char_len <= 2 && !is_two_letter_tech_term(clean)) || is_three_letter_glue_word(clean)-}--/// Normalize a plain-words `xerj search` query before expanding it into-/// `multi_match` / `match_phrase` / `term` clauses (#1364):-///-/// 1. Deduplicate tokens case-insensitively while preserving first-seen order-/// so a repeated word (`"de"` ×3) never gets 3× BM25 multiplication.-/// 2. When a query has 3+ distinct tokens, drop lowercase natural-language-/// glue tokens (`"de"`, `"el"`, `"la"`, `"en"`, `"the"`, `"of"`, `"in"`, …)-/// so short function/variable names like `de` or `in` on `name^4` / `title`-/// cannot outscore files matching the query's content words, and unrelated-/// questions that only share `"de"` do not collapse onto the same top hits.-/// Single-token queries (`xerj search de`) and 2-token queries keep all-/// tokens untouched, as do UPPERCASE/camelCase acronyms (`IO`, `DB`, `ID`)-/// and alphanumeric tokens (`v2`, `u8`, `o68`).-pub(crate) fn normalize_code_search_query(query: &str) -> String {- let mut seen = Vec::<String>::new();- let mut dedup = Vec::<&str>::new();- for raw in query.split_whitespace() {- let clean = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_');- if clean.is_empty() {- continue;- }- let lower = clean.to_lowercase();- if !seen.contains(&lower) {- seen.push(lower);- dedup.push(clean);- }- }- // Under three distinct tokens the query is a lookup, not a question:- // return it verbatim. Re-joining the trimmed tokens here would strip- // identifier punctuation — R's `.mensaje_motor` (the #1364 reporter's own- // control query) would lose its leading dot and miss the exact-`name`- // `term` clause.- if dedup.len() < 3 {- return query.trim().to_string();- }- let content: Vec<&str> = dedup- .iter()- .copied()- .filter(|tok| !is_natural_language_glue_token(tok))- .collect();- if content.is_empty() {- dedup.join(" ")- } else {- content.join(" ")- }-}-/// Build the ES search request body used by `xerj search`.pub fn search_query_body(query: &str, k: usize) -> Value {- let shaped = normalize_code_search_query(query);+ let shaped = xerj_common::code_query::normalize_code_search_query(query);json!({"size": k,"query": { "bool": { "should": [@@ -434,6 +253,7 @@ pub fn run_search_cli() -> i32 {#[cfg(test)]mod tests {use super::*;+ use xerj_common::code_query::normalize_code_search_query;/// #1364: multi-word natural-language queries strip repeated/glue particles/// (`de`, `la`, `por`, `en`, `the`, `of`) so a 2-letter function named `de`diff --git a/engine/crates/xerj-common/src/code_query.rs b/engine/crates/xerj-common/src/code_query.rsnew file mode 100644index 000000000..07784b7b3--- /dev/null+++ b/engine/crates/xerj-common/src/code_query.rs@@ -0,0 +1,186 @@+//! Query shaping shared by the code-search surfaces: `xerj search`+//! (xerj-autoindex) and the `xerj_search` MCP tool (xerj-mcp), issue #1364.+//!+//! One implementation, so the CLI and the MCP tool cannot drift on which+//! natural-language glue words a multi-word question drops.++fn is_two_letter_tech_term(w: &str) -> bool {+ matches!(+ w,+ "db" | "ui"+ | "ux"+ | "ip"+ | "gc"+ | "vm"+ | "id"+ | "kv"+ | "ir"+ | "ci"+ | "pr"+ | "ai"+ | "ml"+ | "tx"+ | "rx"+ | "fs"+ | "fd"+ | "pk"+ | "fk"+ )+}++fn is_three_letter_glue_word(w: &str) -> bool {+ matches!(+ w,+ "del"+ | "por"+ | "los"+ | "las"+ | "una"+ | "uno"+ | "con"+ | "sin"+ | "sus"+ | "mis"+ | "tus"+ | "nos"+ | "les"+ | "mas"+ | "muy"+ | "tan"+ | "que"+ | "son"+ | "fue"+ | "era"+ | "ser"+ | "hay"+ | "the"+ | "and"+ | "for"+ | "not"+ | "are"+ | "but"+ | "can"+ | "all"+ | "has"+ | "had"+ | "was"+ | "one"+ | "out"+ | "who"+ | "how"+ | "why"+ | "his"+ | "her"+ | "its"+ | "our"+ | "you"+ | "any"+ | "may"+ | "via"+ | "per"+ | "des"+ | "aux"+ | "par"+ | "sur"+ | "une"+ | "est"+ | "ses"+ | "vos"+ | "ils"+ | "pas"+ | "qui"+ | "der"+ | "die"+ | "das"+ | "den"+ | "dem"+ | "ein"+ | "und"+ | "mit"+ | "von"+ | "aus"+ | "bei"+ | "vor"+ | "zur"+ | "zum"+ | "wie"+ | "wir"+ | "sie"+ | "ich"+ | "uma"+ | "dos"+ | "nas"+ | "com"+ | "sem"+ | "seu"+ | "sua"+ | "nao"+ | "gli"+ | "nel"+ | "sul"+ | "col"+ | "tra"+ | "fra"+ | "che"+ | "non"+ | "piu"+ )+}++fn is_natural_language_glue_token(clean: &str) -> bool {+ if clean.is_empty() || clean.contains('_') {+ return false;+ }+ if !clean.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) {+ return false;+ }+ let char_len = clean.chars().count();+ (char_len <= 2 && !is_two_letter_tech_term(clean)) || is_three_letter_glue_word(clean)+}++/// Normalize a plain-words `xerj search` / `xerj_search` query before expanding it into+/// `multi_match` / `match_phrase` / `term` clauses (#1364):+///+/// 1. Deduplicate tokens case-insensitively while preserving first-seen order+/// so a repeated word (`"de"` ×3) never gets 3× BM25 multiplication.+/// 2. When a query has 3+ distinct tokens, drop lowercase natural-language+/// glue tokens (`"de"`, `"el"`, `"la"`, `"en"`, `"the"`, `"of"`, `"in"`, …)+/// so short function/variable names like `de` or `in` on `name^4` / `title`+/// cannot outscore files matching the query's content words, and unrelated+/// questions that only share `"de"` do not collapse onto the same top hits.+/// Single-token queries (`xerj search de`) and 2-token queries keep all+/// tokens untouched, as do UPPERCASE/camelCase acronyms (`IO`, `DB`, `ID`)+/// and alphanumeric tokens (`v2`, `u8`, `o68`).+pub fn normalize_code_search_query(query: &str) -> String {+ let mut seen = Vec::<String>::new();+ let mut dedup = Vec::<&str>::new();+ for raw in query.split_whitespace() {+ let clean = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_');+ if clean.is_empty() {+ continue;+ }+ let lower = clean.to_lowercase();+ if !seen.contains(&lower) {+ seen.push(lower);+ dedup.push(clean);+ }+ }+ // Under three distinct tokens the query is a lookup, not a question:+ // return it verbatim. Re-joining the trimmed tokens here would strip+ // identifier punctuation — R's `.mensaje_motor` (the #1364 reporter's own+ // control query) would lose its leading dot and miss the exact-`name`+ // `term` clause.+ if dedup.len() < 3 {+ return query.trim().to_string();+ }+ let content: Vec<&str> = dedup+ .iter()+ .copied()+ .filter(|tok| !is_natural_language_glue_token(tok))+ .collect();+ if content.is_empty() {+ dedup.join(" ")+ } else {+ content.join(" ")+ }+}diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rsindex f78dfedfd..0f21e89d8 100644--- a/engine/crates/xerj-common/src/lib.rs+++ b/engine/crates/xerj-common/src/lib.rs@@ -14,6 +14,8 @@//! ## Modules//!//! - [`config`] — TOML-based configuration (132 settings)+//! - [`code_query`] — query shaping shared by `xerj search` and the+//! `xerj_search` MCP tool (multi-word glue-token stripping, #1364)//! - [`feedback`] — the bug/UX-report invitation shared by every `--help`//! - [`error`] — Unified error type ([`XerjError`])//! - [`types`] — Core domain types (documents, fields, IDs)@@ -34,6 +36,7 @@//! definition record, and the [`hub::Answer`] every surface renders (#1374)pub mod calibration;+pub mod code_query;pub mod config;pub mod error;pub mod feedback;diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rsindex dfe93dac1..c9317bf55 100644--- a/engine/crates/xerj-mcp/src/lib.rs+++ b/engine/crates/xerj-mcp/src/lib.rs@@ -1377,177 +1377,6 @@ fn apply_judge(Ok(())}-fn is_two_letter_tech_term(w: &str) -> bool {- matches!(- w,- "db" | "ui"- | "ux"- | "ip"- | "gc"- | "vm"- | "id"- | "kv"- | "ir"- | "ci"- | "pr"- | "ai"- | "ml"- | "tx"- | "rx"- | "fs"- | "fd"- | "pk"- | "fk"- )-}--fn is_three_letter_glue_word(w: &str) -> bool {- matches!(- w,- "del"- | "por"- | "los"- | "las"- | "una"- | "uno"- | "con"- | "sin"- | "sus"- | "mis"- | "tus"- | "nos"- | "les"- | "mas"- | "muy"- | "tan"- | "que"- | "son"- | "fue"- | "era"- | "ser"- | "hay"- | "the"- | "and"- | "for"- | "not"- | "are"- | "but"- | "can"- | "all"- | "has"- | "had"- | "was"- | "one"- | "out"- | "who"- | "how"- | "why"- | "his"- | "her"- | "its"- | "our"- | "you"- | "any"- | "may"- | "via"- | "per"- | "des"- | "aux"- | "par"- | "sur"- | "une"- | "est"- | "ses"- | "vos"- | "ils"- | "pas"- | "qui"- | "der"- | "die"- | "das"- | "den"- | "dem"- | "ein"- | "und"- | "mit"- | "von"- | "aus"- | "bei"- | "vor"- | "zur"- | "zum"- | "wie"- | "wir"- | "sie"- | "ich"- | "uma"- | "dos"- | "nas"- | "com"- | "sem"- | "seu"- | "sua"- | "nao"- | "gli"- | "nel"- | "sul"- | "col"- | "tra"- | "fra"- | "che"- | "non"- | "piu"- )-}--fn is_natural_language_glue_token(clean: &str) -> bool {- if clean.is_empty() || clean.contains('_') {- return false;- }- if !clean.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) {- return false;- }- let char_len = clean.chars().count();- (char_len <= 2 && !is_two_letter_tech_term(clean)) || is_three_letter_glue_word(clean)-}--/// Normalize a plain-words `xerj_search` query before expanding it into-/// `multi_match` / `match_phrase` / `term` clauses (#1364), matching-/// `xerj_autoindex::search::normalize_code_search_query`.-fn normalize_code_search_query(query: &str) -> String {- let mut seen = Vec::<String>::new();- let mut dedup = Vec::<&str>::new();- for raw in query.split_whitespace() {- let clean = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_');- if clean.is_empty() {- continue;- }- let lower = clean.to_lowercase();- if !seen.contains(&lower) {- seen.push(lower);- dedup.push(clean);- }- }- // Under three distinct tokens the query is a lookup, not a question:- // return it verbatim. Re-joining the trimmed tokens here would strip- // identifier punctuation — R's `.mensaje_motor` (the #1364 reporter's own- // control query) would lose its leading dot and miss the exact-`name`- // `term` clause.- if dedup.len() < 3 {- return query.trim().to_string();- }- let content: Vec<&str> = dedup- .iter()- .copied()- .filter(|tok| !is_natural_language_glue_token(tok))- .collect();- if content.is_empty() {- dedup.join(" ")- } else {- content.join(" ")- }-}-fn build_search(args: &Value) -> Result<BuiltRequest, String> {// #962: `index` used to be required, but nothing an agent reads said what// to pass — the first real tool call after `xerj autoindex` was an error@@ -1570,7 +1399,7 @@ fn build_search(args: &Value) -> Result<BuiltRequest, String> {// untouched for structured queries.Some(Value::String(q)) if !q.trim().is_empty() => {plain_string = true;- let shaped = normalize_code_search_query(q);+ let shaped = xerj_common::code_query::normalize_code_search_query(q);body.insert("query".into(),json!({ "bool": { "should": [