AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
//! Query shaping shared by the code-search surfaces: `xerj search`
//! (xerj-autoindex) and the `xerj_search` MCP tool (xerj-mcp), issue #1364.
//!
//! One implementation, so the CLI and the MCP tool cannot drift on which
//! natural-language glue words a multi-word question drops.
use std::collections::BTreeSet;
use serde_json::{json, Value};
fn is_two_letter_tech_term(w: &str) -> bool {
matches!(
w,
"db" | "ui"
| "ux"
| "ip"
| "gc"
| "vm"
| "id"
| "kv"
| "ir"
| "ci"
| "pr"
| "ai"
| "ml"
| "tx"
| "rx"
| "fs"
| "fd"
| "pk"
| "fk"
)
}
fn is_three_letter_glue_word(w: &str) -> bool {
matches!(
w,
"del"
| "por"
| "los"
| "las"
| "una"
| "uno"
| "con"
| "sin"
| "sus"
| "mis"
| "tus"
| "nos"
| "les"
| "mas"
| "muy"
| "tan"
| "que"
| "son"
| "fue"
| "era"
| "ser"
| "hay"
| "the"
| "and"
| "for"
| "not"
| "are"
| "but"
| "can"
| "all"
| "has"
| "had"
| "was"
| "one"
| "out"
| "who"
| "how"
| "why"
| "his"
| "her"
| "its"
| "our"
| "you"
| "any"
| "may"
| "via"
| "per"
| "des"
| "aux"
| "par"
| "sur"
| "une"
| "est"
| "ses"
| "vos"
| "ils"
| "pas"
| "qui"
| "der"
| "die"
| "das"
| "den"
| "dem"
| "ein"
| "und"
| "mit"
| "von"
| "aus"
| "bei"
| "vor"
| "zur"
| "zum"
| "wie"
| "wir"
| "sie"
| "ich"
| "uma"
| "dos"
| "nas"
| "com"
| "sem"
| "seu"
| "sua"
| "nao"
| "gli"
| "nel"
| "sul"
| "col"
| "tra"
| "fra"
| "che"
| "non"
| "piu"
)
}
fn is_natural_language_glue_token(clean: &str) -> bool {
if clean.is_empty() || clean.contains('_') {
return false;
}
if !clean.chars().all(|c| c.is_alphabetic() && c.is_lowercase()) {
return false;
}
let char_len = clean.chars().count();
(char_len <= 2 && !is_two_letter_tech_term(clean)) || is_three_letter_glue_word(clean)
}
/// Normalize a plain-words `xerj search` / `xerj_search` query before expanding it into
/// `multi_match` / `match_phrase` / `term` clauses (#1364):
///
/// 1. Deduplicate tokens case-insensitively while preserving first-seen order
/// so a repeated word (`"de"` ×3) never gets 3× BM25 multiplication.
/// 2. When a query has 3+ distinct tokens, drop lowercase natural-language
/// glue tokens (`"de"`, `"el"`, `"la"`, `"en"`, `"the"`, `"of"`, `"in"`, …)
/// so short function/variable names like `de` or `in` on `name^4` / `title`
/// cannot outscore files matching the query's content words, and unrelated
/// questions that only share `"de"` do not collapse onto the same top hits.
/// Single-token queries (`xerj search de`) and 2-token queries keep all
/// tokens untouched, as do UPPERCASE/camelCase acronyms (`IO`, `DB`, `ID`)
/// and alphanumeric tokens (`v2`, `u8`, `o68`).
pub fn normalize_code_search_query(query: &str) -> String {
let mut seen = Vec::<String>::new();
let mut dedup = Vec::<&str>::new();
for raw in query.split_whitespace() {
let clean = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_');
if clean.is_empty() {
continue;
}
let lower = clean.to_lowercase();
if !seen.contains(&lower) {
seen.push(lower);
dedup.push(clean);
}
}
// Under three distinct tokens the query is a lookup, not a question:
// return it verbatim. Re-joining the trimmed tokens here would strip
// identifier punctuation — R's `.mensaje_motor` (the #1364 reporter's own
// control query) would lose its leading dot and miss the exact-`name`
// `term` clause.
if dedup.len() < 3 {
return query.trim().to_string();
}
let content: Vec<&str> = dedup
.iter()
.copied()
.filter(|tok| !is_natural_language_glue_token(tok))
.collect();
if content.is_empty() {
dedup.join(" ")
} else {
content.join(" ")
}
}
/// Boost on the exact-phrase clause `xerj search` and the plain-string
/// `xerj_search` MCP tool put on content fields (`body`, and `text` /
/// `message` / `impl` where mapped).
///
/// Below the `defs` phrase boost (8) on purpose: when a query is both a
/// symbol's words and a phrase in some file, the definition still ranks
/// first. It only has to beat the `most_fields` sum a short `defs` list
/// collects from partial term overlap (`MemoryCandidate` matching
/// `memory candidate …`), which on a long file body is several times the
/// body's own BM25 score.
pub const EXACT_PHRASE_BOOST: f64 = 4.0;
/// Score added when a multi-term question matches on coverage, not on one
/// short symbol field (#2327).
///
/// A `name^4` / `code^2` hit whose text is `type Token = string` collects a
/// large BM25 score from field length. Definition pages in this crate's
/// tests sit near 172. One thousand sits above that, so a hit that contains
/// at least half the content terms sorts ahead of a one-term name. A hit
/// that contains every term gets this boost a second time.
pub const COVERAGE_MATCH_BOOST: f64 = 1000.0;
/// Extra `constant_score` clauses for a question with at least three content
/// tokens. Empty for a one- or two-token lookup — that is an identifier
/// search, and the exact `name` term stays the strongest signal.
///
/// The first clause matches at least half the tokens (rounded up). The
/// second matches every token. Both are filters: they do not multiply BM25,
/// and a document that fails them still matches the recall clauses.
pub fn coverage_match_clauses(query: &str, fields: &[String]) -> Vec<Value> {
let shaped = normalize_code_search_query(query);
let tokens: Vec<&str> = shaped.split_whitespace().collect();
if tokens.len() < 3 || fields.is_empty() {
return Vec::new();
}
let need = tokens.len().saturating_add(1) / 2;
let per_term: Vec<Value> = tokens
.iter()
.map(|token| {
json!({
"multi_match": {
"query": token,
"fields": fields,
"type": "most_fields"
}
})
})
.collect();
vec![
json!({
"constant_score": {
"filter": {
"bool": {
"should": per_term,
"minimum_should_match": need
}
},
"boost": COVERAGE_MATCH_BOOST
}
}),
json!({
"constant_score": {
"filter": {
"multi_match": {
"query": shaped,
"fields": fields,
"type": "most_fields",
"operator": "and"
}
},
"boost": COVERAGE_MATCH_BOOST
}
}),
]
}
/// The text an exact-phrase content clause matches, or `None` when the query
/// is a single token.
///
/// This is the trimmed query as typed, not [`normalize_code_search_query`]'s
/// output: dropping glue words or duplicate tokens from a pasted error
/// message would turn it into a phrase the file does not contain. A single
/// token is left to the existing clauses — as a phrase it scores the same
/// documents as the `multi_match` and would only inflate them.
pub fn exact_phrase_query(query: &str) -> Option<&str> {
let trimmed = query.trim();
trimmed.split_whitespace().nth(1).map(|_| trimmed)
}
// ─────────────────────────────────────────────────────────────────────────────
// Mapping-aware query bodies (#1601/#1284)
//
// Shared by `xerj search` and the `xerj_search` MCP tool (#1961): the MCP path
// used to build this body itself, so #1602's mapping-aware fix never reached it
// and a prose corpus answered HTTP 400 `strict_unknown_fields`.
// ─────────────────────────────────────────────────────────────────────────────
/// `name` / `code` / `defs` are the code-shaped legs (#916). A corpus that
/// maps none of them — markdown, logs, anything autoindex accepted that is
/// not source — must not be queried with those legs: autoindex indices are
/// created `strict_unknown_fields`, and a `term` or `match_phrase` on a
/// field no resolved index maps is HTTP 400 (#1601).
pub const CODE_SYMBOL_FIELDS: &[&str] = &["name", "code", "defs"];
/// Historical `xerj search` multi_match weights. Order is part of the
/// code-corpus query: do not reorder.
pub const SEARCH_MM_FIELDS: &[(&str, &str)] = &[
("name", "name^4"),
("code", "code^2"),
("body", "body"),
("defs", "defs^3"),
("title", "title"),
];
/// Content fields a non-code autoindex maps (`text` on prose, `impl` on
/// markdown sections, `message` on the log family). Joined only when the
/// target maps none of [`CODE_SYMBOL_FIELDS`], so a code corpus keeps the
/// #916 shape exactly.
pub const PROSE_EXTRA_FIELDS: &[&str] = &["text", "impl", "message"];
/// Analyzed content fields that get the exact-phrase clause when the
/// `multi_match` names them. `name`, `code` and `title` are keywords and
/// `defs` has its own phrase clause.
pub const PHRASE_CONTENT_FIELDS: &[&str] = &["body", "text", "impl", "message"];
/// Every field `xerj search` might name in `multi_match`, `term`, or
/// `match_phrase`. A numeric mapping of one of these on any index in the
/// wildcard fails the whole request (`For input string`, #1284).
pub const SEARCH_LOOKUP_FIELDS: &[&str] = &[
"name", "code", "body", "defs", "title", "text", "impl", "message",
];
/// Union of property names in a `GET /{pattern}/_mapping` body.
///
/// `None` when `mapping` is not that shape, so a transport failure keeps the
/// historical code-shaped query. An empty object is a real answer (no
/// indices) and comes back empty — callers treat an empty set like an
/// unreadable mapping and do not drop clauses they cannot prove unmapped.
pub fn mapped_property_names(mapping: &Value) -> Option<BTreeSet<String>> {
let obj = mapping.as_object()?;
if obj.is_empty() {
return Some(BTreeSet::new());
}
let mut names = BTreeSet::new();
let mut saw_mapping = false;
for body in obj.values() {
let Some(props) = body
.pointer("/mappings/properties")
.and_then(Value::as_object)
else {
continue;
};
saw_mapping = true;
names.extend(props.keys().cloned());
}
if !saw_mapping {
return None;
}
Some(names)
}
/// Types stored as `long` or `double`. A `multi_match`, `term`, or
/// `match_phrase` that names one fails with `For input string: "<query>"`
/// (#1284). `keyword` is not in this list: `name` stays an exact term.
fn is_numeric_mapping_type(ty: &str) -> bool {
matches!(
ty,
"long"
| "integer"
| "short"
| "byte"
| "unsigned_long"
| "double"
| "float"
| "half_float"
| "scaled_float"
)
}
/// Field names any index in a `GET /{pattern}/_mapping` body maps as numeric.
///
/// One numeric mapping is enough to drop the field from a wildcard query:
/// the request is refused for every index, the way `xerj code` drops a
/// numeric field from its union (#1605). An unreadable body yields an empty
/// set; callers ignore that unless [`mapped_property_names`] also succeeded.
pub fn numeric_property_names(mapping: &Value) -> BTreeSet<String> {
let mut out = BTreeSet::new();
let Some(obj) = mapping.as_object() else {
return out;
};
for body in obj.values() {
let Some(props) = body
.pointer("/mappings/properties")
.and_then(Value::as_object)
else {
continue;
};
for (key, spec) in props {
if spec
.get("type")
.and_then(Value::as_str)
.is_some_and(is_numeric_mapping_type)
{
out.insert(key.clone());
}
}
}
out
}
/// A lookup clause may name `field` when it is mapped (or the mapping was
/// unreadable) and no index maps it as numeric.
pub fn field_lookup_ok(
mapped: Option<&BTreeSet<String>>,
numeric: Option<&BTreeSet<String>>,
field: &str,
) -> bool {
field_mapped(mapped, field) && !numeric.is_some_and(|set| set.contains(field))
}
/// True when a numeric mapping removed a field this command would look up,
/// so the historical query must not be restored — it would name that field
/// again and the wildcard would fail.
fn numeric_drops_a_lookup(numeric: Option<&BTreeSet<String>>, candidates: &[&str]) -> bool {
numeric.is_some_and(|set| candidates.iter().any(|name| set.contains(*name)))
}
/// Whether `field` may be named in a lookup clause.
///
/// An unreadable mapping (`None`) or an empty union keeps every historical
/// clause: dropping them with no evidence would change the #916 ranking on
/// a code corpus whose mapping we failed to read.
pub fn field_mapped(mapped: Option<&BTreeSet<String>>, field: &str) -> bool {
match mapped {
None => true,
Some(set) if set.is_empty() => true,
Some(set) => set.contains(field),
}
}
/// True when the mapping was read and no index maps `name`, `code`, or `defs`.
pub fn code_symbol_fields_absent(mapped: Option<&BTreeSet<String>>) -> bool {
match mapped {
Some(set) if !set.is_empty() => CODE_SYMBOL_FIELDS.iter().all(|f| !set.contains(*f)),
_ => false,
}
}
/// `preferred` entries whose base field is mapped, then `extras` (bare names).
///
/// `prose_beside_code`: also name `text` and `message` when a code field is
/// mapped. A folded `docs` index stores changelog and config lines in `text`
/// next to `name`/`code`/`defs`. Leaving `text` off made `xerj search` miss
/// `tokio/CHANGELOG.md` (122 sections) while a direct `match_phrase` on
/// `text` hit them. `impl` stays off in that case so a code corpus does not
/// grow a second body leg (#916). A corpus with no code field still gets
/// every extra it maps, including `impl`.
pub fn weighted_query_fields(
preferred: &[(&str, &str)],
extras: &[&str],
mapped: Option<&BTreeSet<String>>,
numeric: Option<&BTreeSet<String>>,
prose_beside_code: bool,
) -> Vec<String> {
let mut out: Vec<String> = preferred
.iter()
.filter(|(name, _)| field_lookup_ok(mapped, numeric, name))
.map(|(_, weighted)| (*weighted).to_string())
.collect();
let code_present = !code_symbol_fields_absent(mapped);
if let Some(set) = mapped.filter(|set| !set.is_empty()) {
for name in extras {
if code_present && (*name == "impl" || !prose_beside_code) {
continue;
}
if set.contains(*name)
&& !numeric.is_some_and(|nums| nums.contains(*name))
&& !out.iter().any(|f| f.split('^').next() == Some(*name))
{
out.push((*name).to_string());
}
}
}
out
}
/// Build the ES search request body used by `xerj search`.
///
/// `mapped` is the union of property names under the prefix. `None` (and an
/// empty union) is the historical code-shaped query.
pub fn search_query_body(query: &str, k: usize) -> Value {
search_query_body_for_mapping(query, k, None, None)
}
/// [`search_query_body`] with the target's mapped field names (#1601) and
/// the names any index maps as numeric (#1284).
pub fn search_query_body_for_mapping(
query: &str,
k: usize,
mapped: Option<&BTreeSet<String>>,
numeric: Option<&BTreeSet<String>>,
) -> Value {
let built = build_search_query(query, k, mapped, numeric);
if should_clauses(&built).is_empty() {
if numeric_drops_a_lookup(numeric, SEARCH_LOOKUP_FIELDS) {
// Restoring the historical query would name the numeric field
// again. An empty `should` matches every document; match nothing.
let mut built = built;
built["query"] = json!({ "match_none": {} });
return built;
}
// Nothing we can name is mapped. The historical shape is what the
// server's 400 then explains, listing the searchable fields.
return build_search_query(query, k, None, None);
}
built
}
fn should_clauses(body: &Value) -> &[Value] {
body.pointer("/query/bool/should")
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or(&[])
}
fn build_search_query(
query: &str,
k: usize,
mapped: Option<&BTreeSet<String>>,
numeric: Option<&BTreeSet<String>>,
) -> Value {
let shaped = normalize_code_search_query(query);
let mut should = Vec::new();
let fields = weighted_query_fields(SEARCH_MM_FIELDS, PROSE_EXTRA_FIELDS, mapped, numeric, true);
if !fields.is_empty() {
should.push(json!({ "multi_match": {
"query": shaped,
"fields": fields,
"type": "most_fields"
}}));
}
if field_lookup_ok(mapped, numeric, "defs") {
should.push(json!({ "match_phrase": { "defs": { "query": shaped, "boost": 8 } } }));
}
if field_lookup_ok(mapped, numeric, "name") {
should.push(json!({ "term": { "name": { "value": query.trim(), "boost": 10 } } }));
}
// An exact phrase in a content field. A pasted error message or log
// line is the strongest signal of which file is meant, and the
// bag-of-words `multi_match` alone ranks a short `defs` list sharing two
// of its words above the long file holding all of them in order. Only
// the content fields the `multi_match` already names, so the mapping and
// numeric drops (#1601, #1284) apply unchanged. Appended last so the
// historical clause positions do not move.
if let Some(phrase) = exact_phrase_query(query) {
for field in &fields {
let base = field.split('^').next().unwrap_or(field);
if PHRASE_CONTENT_FIELDS.contains(&base) {
should.push(json!({ "match_phrase": { base: {
"query": phrase,
"boost": EXACT_PHRASE_BOOST
} } }));
}
}
}
// Appended last so the #916 clause positions do not move. A one-word
// symbol still matches the recall `multi_match`; these clauses only
// add score when the document covers the question (#2327).
should.extend(coverage_match_clauses(&shaped, &fields));
json!({
"size": k,
"query": { "bool": { "should": should }},
// No `body` here: whole-file bodies are the token flood (measured ~9x
// the size of the matching passage). `_passage` carries the snippet.
// `ax_paths` lists every name a deduplicated file has, so the hit can
// cite the live name instead of a backup copy (#2402).
"_source": ["ax_path", "ax_paths", "ax_locator", "line", "start_line", "language", "kind", "code", "title", "heading", "page", "cell"],
"fields": ["_passage"]
})
}