Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

refactor(common): one glue-token normalizer for xerj search and xerj_search (#1364 review)

xerj-org6474fff06fcb
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rs
index 434d62224..22ebb7f9b 100644
--- a/engine/crates/xerj-autoindex/src/search.rs
+++ b/engine/crates/xerj-autoindex/src/search.rs
@@ -72,190 +72,9 @@ pub(crate) fn is_identifier(q: &str) -> bool {
&& q.chars().all(|c| c.is_alphanumeric() || c == '_')
}
-fn is_two_letter_tech_term(w: &str) -> bool {
- matches!(
- w,
- "db" | "ui"
- | "ux"
- | "ip"
- | "gc"
- | "vm"
- | "id"
- | "kv"
- | "ir"
- | "ci"
- | "pr"
- | "ai"
- | "ml"
- | "tx"
- | "rx"
- | "fs"
- | "fd"
- | "pk"
- | "fk"
- )
-}
-
-fn is_three_letter_glue_word(w: &str) -> bool {
- matches!(
- w,
- "del"
- | "por"
- | "los"
- | "las"
- | "una"
- | "uno"
- | "con"
- | "sin"
- | "sus"
- | "mis"
- | "tus"
- | "nos"
- | "les"
- | "mas"
- | "muy"
- | "tan"
- | "que"
- | "son"
- | "fue"
- | "era"
- | "ser"
- | "hay"
- | "the"
- | "and"
- | "for"
- | "not"
- | "are"
- | "but"
- | "can"
- | "all"
- | "has"
- | "had"
- | "was"
- | "one"
- | "out"
- | "who"
- | "how"
- | "why"
- | "his"
- | "her"
- | "its"
- | "our"
- | "you"
- | "any"
- | "may"
- | "via"
- | "per"
- | "des"
- | "aux"
- | "par"
- | "sur"
- | "une"
- | "est"
- | "ses"
- | "vos"
- | "ils"
- | "pas"
- | "qui"
- | "der"
- | "die"
- | "das"
- | "den"
- | "dem"
- | "ein"
- | "und"
- | "mit"
- | "von"
- | "aus"
- | "bei"
- | "vor"
- | "zur"
- | "zum"
- | "wie"
- | "wir"
- | "sie"
- | "ich"
- | "uma"
- | "dos"
- | "nas"
- | "com"
- | "sem"
- | "seu"
- | "sua"
- | "nao"
- | "gli"
- | "nel"
- | "sul"
- | "col"
- | "tra"
- | "fra"
- | "che"
- | "non"
- | "piu"
- )
-}
-
-fn is_natural_language_glue_token(clean: &str) -> bool {
- if clean.is_empty() || clean.contains('_') {
- return false;
- }
- if !clean.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) {
- return false;
- }
- let char_len = clean.chars().count();
- (char_len <= 2 && !is_two_letter_tech_term(clean)) || is_three_letter_glue_word(clean)
-}
-
-/// Normalize a plain-words `xerj search` query before expanding it into
-/// `multi_match` / `match_phrase` / `term` clauses (#1364):
-///
-/// 1. Deduplicate tokens case-insensitively while preserving first-seen order
-/// so a repeated word (`"de"` ×3) never gets 3× BM25 multiplication.
-/// 2. When a query has 3+ distinct tokens, drop lowercase natural-language
-/// glue tokens (`"de"`, `"el"`, `"la"`, `"en"`, `"the"`, `"of"`, `"in"`, …)
-/// so short function/variable names like `de` or `in` on `name^4` / `title`
-/// cannot outscore files matching the query's content words, and unrelated
-/// questions that only share `"de"` do not collapse onto the same top hits.
-/// Single-token queries (`xerj search de`) and 2-token queries keep all
-/// tokens untouched, as do UPPERCASE/camelCase acronyms (`IO`, `DB`, `ID`)
-/// and alphanumeric tokens (`v2`, `u8`, `o68`).
-pub(crate) fn normalize_code_search_query(query: &str) -> String {
- let mut seen = Vec::<String>::new();
- let mut dedup = Vec::<&str>::new();
- for raw in query.split_whitespace() {
- let clean = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_');
- if clean.is_empty() {
- continue;
- }
- let lower = clean.to_lowercase();
- if !seen.contains(&lower) {
- seen.push(lower);
- dedup.push(clean);
- }
- }
- // Under three distinct tokens the query is a lookup, not a question:
- // return it verbatim. Re-joining the trimmed tokens here would strip
- // identifier punctuation — R's `.mensaje_motor` (the #1364 reporter's own
- // control query) would lose its leading dot and miss the exact-`name`
- // `term` clause.
- if dedup.len() < 3 {
- return query.trim().to_string();
- }
- let content: Vec<&str> = dedup
- .iter()
- .copied()
- .filter(|tok| !is_natural_language_glue_token(tok))
- .collect();
- if content.is_empty() {
- dedup.join(" ")
- } else {
- content.join(" ")
- }
-}
-
/// Build the ES search request body used by `xerj search`.
pub fn search_query_body(query: &str, k: usize) -> Value {
- let shaped = normalize_code_search_query(query);
+ let shaped = xerj_common::code_query::normalize_code_search_query(query);
json!({
"size": k,
"query": { "bool": { "should": [
@@ -434,6 +253,7 @@ pub fn run_search_cli() -> i32 {
#[cfg(test)]
mod tests {
use super::*;
+ use xerj_common::code_query::normalize_code_search_query;
/// #1364: multi-word natural-language queries strip repeated/glue particles
/// (`de`, `la`, `por`, `en`, `the`, `of`) so a 2-letter function named `de`
diff --git a/engine/crates/xerj-common/src/code_query.rs b/engine/crates/xerj-common/src/code_query.rs
new file mode 100644
index 000000000..07784b7b3
--- /dev/null
+++ b/engine/crates/xerj-common/src/code_query.rs
@@ -0,0 +1,186 @@
+//! Query shaping shared by the code-search surfaces: `xerj search`
+//! (xerj-autoindex) and the `xerj_search` MCP tool (xerj-mcp), issue #1364.
+//!
+//! One implementation, so the CLI and the MCP tool cannot drift on which
+//! natural-language glue words a multi-word question drops.
+
+fn is_two_letter_tech_term(w: &str) -> bool {
+ matches!(
+ w,
+ "db" | "ui"
+ | "ux"
+ | "ip"
+ | "gc"
+ | "vm"
+ | "id"
+ | "kv"
+ | "ir"
+ | "ci"
+ | "pr"
+ | "ai"
+ | "ml"
+ | "tx"
+ | "rx"
+ | "fs"
+ | "fd"
+ | "pk"
+ | "fk"
+ )
+}
+
+fn is_three_letter_glue_word(w: &str) -> bool {
+ matches!(
+ w,
+ "del"
+ | "por"
+ | "los"
+ | "las"
+ | "una"
+ | "uno"
+ | "con"
+ | "sin"
+ | "sus"
+ | "mis"
+ | "tus"
+ | "nos"
+ | "les"
+ | "mas"
+ | "muy"
+ | "tan"
+ | "que"
+ | "son"
+ | "fue"
+ | "era"
+ | "ser"
+ | "hay"
+ | "the"
+ | "and"
+ | "for"
+ | "not"
+ | "are"
+ | "but"
+ | "can"
+ | "all"
+ | "has"
+ | "had"
+ | "was"
+ | "one"
+ | "out"
+ | "who"
+ | "how"
+ | "why"
+ | "his"
+ | "her"
+ | "its"
+ | "our"
+ | "you"
+ | "any"
+ | "may"
+ | "via"
+ | "per"
+ | "des"
+ | "aux"
+ | "par"
+ | "sur"
+ | "une"
+ | "est"
+ | "ses"
+ | "vos"
+ | "ils"
+ | "pas"
+ | "qui"
+ | "der"
+ | "die"
+ | "das"
+ | "den"
+ | "dem"
+ | "ein"
+ | "und"
+ | "mit"
+ | "von"
+ | "aus"
+ | "bei"
+ | "vor"
+ | "zur"
+ | "zum"
+ | "wie"
+ | "wir"
+ | "sie"
+ | "ich"
+ | "uma"
+ | "dos"
+ | "nas"
+ | "com"
+ | "sem"
+ | "seu"
+ | "sua"
+ | "nao"
+ | "gli"
+ | "nel"
+ | "sul"
+ | "col"
+ | "tra"
+ | "fra"
+ | "che"
+ | "non"
+ | "piu"
+ )
+}
+
+fn is_natural_language_glue_token(clean: &str) -> bool {
+ if clean.is_empty() || clean.contains('_') {
+ return false;
+ }
+ if !clean.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) {
+ return false;
+ }
+ let char_len = clean.chars().count();
+ (char_len <= 2 && !is_two_letter_tech_term(clean)) || is_three_letter_glue_word(clean)
+}
+
+/// Normalize a plain-words `xerj search` / `xerj_search` query before expanding it into
+/// `multi_match` / `match_phrase` / `term` clauses (#1364):
+///
+/// 1. Deduplicate tokens case-insensitively while preserving first-seen order
+/// so a repeated word (`"de"` ×3) never gets 3× BM25 multiplication.
+/// 2. When a query has 3+ distinct tokens, drop lowercase natural-language
+/// glue tokens (`"de"`, `"el"`, `"la"`, `"en"`, `"the"`, `"of"`, `"in"`, …)
+/// so short function/variable names like `de` or `in` on `name^4` / `title`
+/// cannot outscore files matching the query's content words, and unrelated
+/// questions that only share `"de"` do not collapse onto the same top hits.
+/// Single-token queries (`xerj search de`) and 2-token queries keep all
+/// tokens untouched, as do UPPERCASE/camelCase acronyms (`IO`, `DB`, `ID`)
+/// and alphanumeric tokens (`v2`, `u8`, `o68`).
+pub fn normalize_code_search_query(query: &str) -> String {
+ let mut seen = Vec::<String>::new();
+ let mut dedup = Vec::<&str>::new();
+ for raw in query.split_whitespace() {
+ let clean = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_');
+ if clean.is_empty() {
+ continue;
+ }
+ let lower = clean.to_lowercase();
+ if !seen.contains(&lower) {
+ seen.push(lower);
+ dedup.push(clean);
+ }
+ }
+ // Under three distinct tokens the query is a lookup, not a question:
+ // return it verbatim. Re-joining the trimmed tokens here would strip
+ // identifier punctuation — R's `.mensaje_motor` (the #1364 reporter's own
+ // control query) would lose its leading dot and miss the exact-`name`
+ // `term` clause.
+ if dedup.len() < 3 {
+ return query.trim().to_string();
+ }
+ let content: Vec<&str> = dedup
+ .iter()
+ .copied()
+ .filter(|tok| !is_natural_language_glue_token(tok))
+ .collect();
+ if content.is_empty() {
+ dedup.join(" ")
+ } else {
+ content.join(" ")
+ }
+}
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rs
index f78dfedfd..0f21e89d8 100644
--- a/engine/crates/xerj-common/src/lib.rs
+++ b/engine/crates/xerj-common/src/lib.rs
@@ -14,6 +14,8 @@
//! ## Modules
//!
//! - [`config`] — TOML-based configuration (132 settings)
+//! - [`code_query`] — query shaping shared by `xerj search` and the
+//! `xerj_search` MCP tool (multi-word glue-token stripping, #1364)
//! - [`feedback`] — the bug/UX-report invitation shared by every `--help`
//! - [`error`] — Unified error type ([`XerjError`])
//! - [`types`] — Core domain types (documents, fields, IDs)
@@ -34,6 +36,7 @@
//! definition record, and the [`hub::Answer`] every surface renders (#1374)
pub mod calibration;
+pub mod code_query;
pub mod config;
pub mod error;
pub mod feedback;
diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rs
index dfe93dac1..c9317bf55 100644
--- a/engine/crates/xerj-mcp/src/lib.rs
+++ b/engine/crates/xerj-mcp/src/lib.rs
@@ -1377,177 +1377,6 @@ fn apply_judge(
Ok(())
}
-fn is_two_letter_tech_term(w: &str) -> bool {
- matches!(
- w,
- "db" | "ui"
- | "ux"
- | "ip"
- | "gc"
- | "vm"
- | "id"
- | "kv"
- | "ir"
- | "ci"
- | "pr"
- | "ai"
- | "ml"
- | "tx"
- | "rx"
- | "fs"
- | "fd"
- | "pk"
- | "fk"
- )
-}
-
-fn is_three_letter_glue_word(w: &str) -> bool {
- matches!(
- w,
- "del"
- | "por"
- | "los"
- | "las"
- | "una"
- | "uno"
- | "con"
- | "sin"
- | "sus"
- | "mis"
- | "tus"
- | "nos"
- | "les"
- | "mas"
- | "muy"
- | "tan"
- | "que"
- | "son"
- | "fue"
- | "era"
- | "ser"
- | "hay"
- | "the"
- | "and"
- | "for"
- | "not"
- | "are"
- | "but"
- | "can"
- | "all"
- | "has"
- | "had"
- | "was"
- | "one"
- | "out"
- | "who"
- | "how"
- | "why"
- | "his"
- | "her"
- | "its"
- | "our"
- | "you"
- | "any"
- | "may"
- | "via"
- | "per"
- | "des"
- | "aux"
- | "par"
- | "sur"
- | "une"
- | "est"
- | "ses"
- | "vos"
- | "ils"
- | "pas"
- | "qui"
- | "der"
- | "die"
- | "das"
- | "den"
- | "dem"
- | "ein"
- | "und"
- | "mit"
- | "von"
- | "aus"
- | "bei"
- | "vor"
- | "zur"
- | "zum"
- | "wie"
- | "wir"
- | "sie"
- | "ich"
- | "uma"
- | "dos"
- | "nas"
- | "com"
- | "sem"
- | "seu"
- | "sua"
- | "nao"
- | "gli"
- | "nel"
- | "sul"
- | "col"
- | "tra"
- | "fra"
- | "che"
- | "non"
- | "piu"
- )
-}
-
-fn is_natural_language_glue_token(clean: &str) -> bool {
- if clean.is_empty() || clean.contains('_') {
- return false;
- }
- if !clean.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) {
- return false;
- }
- let char_len = clean.chars().count();
- (char_len <= 2 && !is_two_letter_tech_term(clean)) || is_three_letter_glue_word(clean)
-}
-
-/// Normalize a plain-words `xerj_search` query before expanding it into
-/// `multi_match` / `match_phrase` / `term` clauses (#1364), matching
-/// `xerj_autoindex::search::normalize_code_search_query`.
-fn normalize_code_search_query(query: &str) -> String {
- let mut seen = Vec::<String>::new();
- let mut dedup = Vec::<&str>::new();
- for raw in query.split_whitespace() {
- let clean = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_');
- if clean.is_empty() {
- continue;
- }
- let lower = clean.to_lowercase();
- if !seen.contains(&lower) {
- seen.push(lower);
- dedup.push(clean);
- }
- }
- // Under three distinct tokens the query is a lookup, not a question:
- // return it verbatim. Re-joining the trimmed tokens here would strip
- // identifier punctuation — R's `.mensaje_motor` (the #1364 reporter's own
- // control query) would lose its leading dot and miss the exact-`name`
- // `term` clause.
- if dedup.len() < 3 {
- return query.trim().to_string();
- }
- let content: Vec<&str> = dedup
- .iter()
- .copied()
- .filter(|tok| !is_natural_language_glue_token(tok))
- .collect();
- if content.is_empty() {
- dedup.join(" ")
- } else {
- content.join(" ")
- }
-}
-
fn build_search(args: &Value) -> Result<BuiltRequest, String> {
// #962: `index` used to be required, but nothing an agent reads said what
// to pass — the first real tool call after `xerj autoindex` was an error
@@ -1570,7 +1399,7 @@ fn build_search(args: &Value) -> Result<BuiltRequest, String> {
// untouched for structured queries.
Some(Value::String(q)) if !q.trim().is_empty() => {
plain_string = true;
- let shaped = normalize_code_search_query(q);
+ let shaped = xerj_common::code_query::normalize_code_search_query(q);
body.insert(
"query".into(),
json!({ "bool": { "should": [