AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(mcp): xerj_search reads the target mapping before a plain-string query (#1961) (#1976)
ordinary being8c57af38d483
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rsindex 1f9110129..9e43fc1f2 100644--- a/engine/crates/xerj-autoindex/src/search.rs+++ b/engine/crates/xerj-autoindex/src/search.rs@@ -153,283 +153,13 @@ pub(crate) fn suggests_def(q: &str) -> bool {.all(|c| c.is_alphanumeric() || c == '_' || c == ':')}-/// `name` / `code` / `defs` are the code-shaped legs (#916). A corpus that-/// maps none of them — markdown, logs, anything autoindex accepted that is-/// not source — must not be queried with those legs: autoindex indices are-/// created `strict_unknown_fields`, and a `term` or `match_phrase` on a-/// field no resolved index maps is HTTP 400 (#1601).-const CODE_SYMBOL_FIELDS: &[&str] = &["name", "code", "defs"];--/// Historical `xerj search` multi_match weights. Order is part of the-/// code-corpus query: do not reorder.-const SEARCH_MM_FIELDS: &[(&str, &str)] = &[- ("name", "name^4"),- ("code", "code^2"),- ("body", "body"),- ("defs", "defs^3"),- ("title", "title"),-];--/// Content fields a non-code autoindex maps (`text` on prose, `impl` on-/// markdown sections, `message` on the log family). Joined only when the-/// target maps none of [`CODE_SYMBOL_FIELDS`], so a code corpus keeps the-/// #916 shape exactly.-const PROSE_EXTRA_FIELDS: &[&str] = &["text", "impl", "message"];--/// Analyzed content fields that get the exact-phrase clause when the-/// `multi_match` names them. `name`, `code` and `title` are keywords and-/// `defs` has its own phrase clause.-const PHRASE_CONTENT_FIELDS: &[&str] = &["body", "text", "impl", "message"];--/// Every field `xerj search` might name in `multi_match`, `term`, or-/// `match_phrase`. A numeric mapping of one of these on any index in the-/// wildcard fails the whole request (`For input string`, #1284).-const SEARCH_LOOKUP_FIELDS: &[&str] = &[- "name", "code", "body", "defs", "title", "text", "impl", "message",-];--/// Union of property names in a `GET /{pattern}/_mapping` body.-///-/// `None` when `mapping` is not that shape, so a transport failure keeps the-/// historical code-shaped query. An empty object is a real answer (no-/// indices) and comes back empty — callers treat an empty set like an-/// unreadable mapping and do not drop clauses they cannot prove unmapped.-pub(crate) fn mapped_property_names(mapping: &Value) -> Option<BTreeSet<String>> {- let obj = mapping.as_object()?;- if obj.is_empty() {- return Some(BTreeSet::new());- }- let mut names = BTreeSet::new();- let mut saw_mapping = false;- for body in obj.values() {- let Some(props) = body- .pointer("/mappings/properties")- .and_then(Value::as_object)- else {- continue;- };- saw_mapping = true;- names.extend(props.keys().cloned());- }- if !saw_mapping {- return None;- }- Some(names)-}--/// Types stored as `long` or `double`. A `multi_match`, `term`, or-/// `match_phrase` that names one fails with `For input string: "<query>"`-/// (#1284). `keyword` is not in this list: `name` stays an exact term.-fn is_numeric_mapping_type(ty: &str) -> bool {- matches!(- ty,- "long"- | "integer"- | "short"- | "byte"- | "unsigned_long"- | "double"- | "float"- | "half_float"- | "scaled_float"- )-}--/// Field names any index in a `GET /{pattern}/_mapping` body maps as numeric.-///-/// One numeric mapping is enough to drop the field from a wildcard query:-/// the request is refused for every index, the way `xerj code` drops a-/// numeric field from its union (#1605). An unreadable body yields an empty-/// set; callers ignore that unless [`mapped_property_names`] also succeeded.-pub(crate) fn numeric_property_names(mapping: &Value) -> BTreeSet<String> {- let mut out = BTreeSet::new();- let Some(obj) = mapping.as_object() else {- return out;- };- for body in obj.values() {- let Some(props) = body- .pointer("/mappings/properties")- .and_then(Value::as_object)- else {- continue;- };- for (key, spec) in props {- if spec- .get("type")- .and_then(Value::as_str)- .is_some_and(is_numeric_mapping_type)- {- out.insert(key.clone());- }- }- }- out-}--/// A lookup clause may name `field` when it is mapped (or the mapping was-/// unreadable) and no index maps it as numeric.-pub(crate) fn field_lookup_ok(- mapped: Option<&BTreeSet<String>>,- numeric: Option<&BTreeSet<String>>,- field: &str,-) -> bool {- field_mapped(mapped, field) && !numeric.is_some_and(|set| set.contains(field))-}--/// True when a numeric mapping removed a field this command would look up,-/// so the historical query must not be restored — it would name that field-/// again and the wildcard would fail.-fn numeric_drops_a_lookup(numeric: Option<&BTreeSet<String>>, candidates: &[&str]) -> bool {- numeric.is_some_and(|set| candidates.iter().any(|name| set.contains(*name)))-}--/// Whether `field` may be named in a lookup clause.-///-/// An unreadable mapping (`None`) or an empty union keeps every historical-/// clause: dropping them with no evidence would change the #916 ranking on-/// a code corpus whose mapping we failed to read.-pub(crate) fn field_mapped(mapped: Option<&BTreeSet<String>>, field: &str) -> bool {- match mapped {- None => true,- Some(set) if set.is_empty() => true,- Some(set) => set.contains(field),- }-}--/// True when the mapping was read and no index maps `name`, `code`, or `defs`.-pub(crate) fn code_symbol_fields_absent(mapped: Option<&BTreeSet<String>>) -> bool {- match mapped {- Some(set) if !set.is_empty() => CODE_SYMBOL_FIELDS.iter().all(|f| !set.contains(*f)),- _ => false,- }-}--/// `preferred` entries whose base field is mapped, then `extras` (bare names).-///-/// `prose_beside_code`: also name `text` and `message` when a code field is-/// mapped. A folded `docs` index stores changelog and config lines in `text`-/// next to `name`/`code`/`defs`. Leaving `text` off made `xerj search` miss-/// `tokio/CHANGELOG.md` (122 sections) while a direct `match_phrase` on-/// `text` hit them. `impl` stays off in that case so a code corpus does not-/// grow a second body leg (#916). A corpus with no code field still gets-/// every extra it maps, including `impl`.-pub(crate) fn weighted_query_fields(- preferred: &[(&str, &str)],- extras: &[&str],- mapped: Option<&BTreeSet<String>>,- numeric: Option<&BTreeSet<String>>,- prose_beside_code: bool,-) -> Vec<String> {- let mut out: Vec<String> = preferred- .iter()- .filter(|(name, _)| field_lookup_ok(mapped, numeric, name))- .map(|(_, weighted)| (*weighted).to_string())- .collect();- let code_present = !code_symbol_fields_absent(mapped);- if let Some(set) = mapped.filter(|set| !set.is_empty()) {- for name in extras {- if code_present && (*name == "impl" || !prose_beside_code) {- continue;- }- if set.contains(*name)- && !numeric.is_some_and(|nums| nums.contains(*name))- && !out.iter().any(|f| f.split('^').next() == Some(*name))- {- out.push((*name).to_string());- }- }- }- out-}--/// Build the ES search request body used by `xerj search`.-///-/// `mapped` is the union of property names under the prefix. `None` (and an-/// empty union) is the historical code-shaped query.-pub fn search_query_body(query: &str, k: usize) -> Value {- search_query_body_for_mapping(query, k, None, None)-}--/// [`search_query_body`] with the target's mapped field names (#1601) and-/// the names any index maps as numeric (#1284).-pub fn search_query_body_for_mapping(- query: &str,- k: usize,- mapped: Option<&BTreeSet<String>>,- numeric: Option<&BTreeSet<String>>,-) -> Value {- let built = build_search_query(query, k, mapped, numeric);- if should_clauses(&built).is_empty() {- if numeric_drops_a_lookup(numeric, SEARCH_LOOKUP_FIELDS) {- // Restoring the historical query would name the numeric field- // again. An empty `should` matches every document; match nothing.- let mut built = built;- built["query"] = json!({ "match_none": {} });- return built;- }- // Nothing we can name is mapped. The historical shape is what the- // server's 400 then explains, listing the searchable fields.- return build_search_query(query, k, None, None);- }- built-}--fn should_clauses(body: &Value) -> &[Value] {- body.pointer("/query/bool/should")- .and_then(Value::as_array)- .map(Vec::as_slice)- .unwrap_or(&[])-}--fn build_search_query(- query: &str,- k: usize,- mapped: Option<&BTreeSet<String>>,- numeric: Option<&BTreeSet<String>>,-) -> Value {- let shaped = xerj_common::code_query::normalize_code_search_query(query);- let mut should = Vec::new();- let fields = weighted_query_fields(SEARCH_MM_FIELDS, PROSE_EXTRA_FIELDS, mapped, numeric, true);- if !fields.is_empty() {- should.push(json!({ "multi_match": {- "query": shaped,- "fields": fields,- "type": "most_fields"- }}));- }- if field_lookup_ok(mapped, numeric, "defs") {- should.push(json!({ "match_phrase": { "defs": { "query": shaped, "boost": 8 } } }));- }- if field_lookup_ok(mapped, numeric, "name") {- should.push(json!({ "term": { "name": { "value": query.trim(), "boost": 10 } } }));- }- // An exact phrase in a content field. A pasted error message or log- // line is the strongest signal of which file is meant, and the- // bag-of-words `multi_match` alone ranks a short `defs` list sharing two- // of its words above the long file holding all of them in order. Only- // the content fields the `multi_match` already names, so the mapping and- // numeric drops (#1601, #1284) apply unchanged. Appended last so the- // historical clause positions do not move.- if let Some(phrase) = xerj_common::code_query::exact_phrase_query(query) {- for field in &fields {- let base = field.split('^').next().unwrap_or(field);- if PHRASE_CONTENT_FIELDS.contains(&base) {- should.push(json!({ "match_phrase": { base: {- "query": phrase,- "boost": xerj_common::code_query::EXACT_PHRASE_BOOST- } } }));- }- }- }- json!({- "size": k,- "query": { "bool": { "should": should }},- // No `body` here: whole-file bodies are the token flood (measured ~9x- // the size of the matching passage). `_passage` carries the snippet.- "_source": ["ax_path", "ax_locator", "line", "start_line", "language", "kind", "code", "title", "heading", "page", "cell"],- "fields": ["_passage"]- })-}+pub(crate) use xerj_common::code_query::{+ field_lookup_ok, mapped_property_names, numeric_property_names, weighted_query_fields,+};+/// The mapping-aware query body is shared with the `xerj_search` MCP tool+/// (#1961); the implementation lives in `xerj_common::code_query` so the two+/// surfaces cannot drift again (#1601/#1602).+pub use xerj_common::code_query::{search_query_body, search_query_body_for_mapping};/// Extract the 1-based starting line number from `fields._passage[0].ordinal`/// when `_passage` comes from a document body field (ignoring syntheticdiff --git a/engine/crates/xerj-common/src/code_query.rs b/engine/crates/xerj-common/src/code_query.rsindex 3bce95e8f..6b29069f0 100644--- a/engine/crates/xerj-common/src/code_query.rs+++ b/engine/crates/xerj-common/src/code_query.rs@@ -4,6 +4,10 @@//! One implementation, so the CLI and the MCP tool cannot drift on which//! natural-language glue words a multi-word question drops.+use std::collections::BTreeSet;++use serde_json::{json, Value};+fn is_two_letter_tech_term(w: &str) -> bool {matches!(w,@@ -209,3 +213,289 @@ pub fn exact_phrase_query(query: &str) -> Option<&str> {let trimmed = query.trim();trimmed.split_whitespace().nth(1).map(|_| trimmed)}++// ─────────────────────────────────────────────────────────────────────────────+// Mapping-aware query bodies (#1601/#1284)+//+// Shared by `xerj search` and the `xerj_search` MCP tool (#1961): the MCP path+// used to build this body itself, so #1602's mapping-aware fix never reached it+// and a prose corpus answered HTTP 400 `strict_unknown_fields`.+// ─────────────────────────────────────────────────────────────────────────────++/// `name` / `code` / `defs` are the code-shaped legs (#916). A corpus that+/// maps none of them — markdown, logs, anything autoindex accepted that is+/// not source — must not be queried with those legs: autoindex indices are+/// created `strict_unknown_fields`, and a `term` or `match_phrase` on a+/// field no resolved index maps is HTTP 400 (#1601).+pub const CODE_SYMBOL_FIELDS: &[&str] = &["name", "code", "defs"];++/// Historical `xerj search` multi_match weights. Order is part of the+/// code-corpus query: do not reorder.+pub const SEARCH_MM_FIELDS: &[(&str, &str)] = &[+ ("name", "name^4"),+ ("code", "code^2"),+ ("body", "body"),+ ("defs", "defs^3"),+ ("title", "title"),+];++/// Content fields a non-code autoindex maps (`text` on prose, `impl` on+/// markdown sections, `message` on the log family). Joined only when the+/// target maps none of [`CODE_SYMBOL_FIELDS`], so a code corpus keeps the+/// #916 shape exactly.+pub const PROSE_EXTRA_FIELDS: &[&str] = &["text", "impl", "message"];++/// Analyzed content fields that get the exact-phrase clause when the+/// `multi_match` names them. `name`, `code` and `title` are keywords and+/// `defs` has its own phrase clause.+pub const PHRASE_CONTENT_FIELDS: &[&str] = &["body", "text", "impl", "message"];++/// Every field `xerj search` might name in `multi_match`, `term`, or+/// `match_phrase`. A numeric mapping of one of these on any index in the+/// wildcard fails the whole request (`For input string`, #1284).+pub const SEARCH_LOOKUP_FIELDS: &[&str] = &[+ "name", "code", "body", "defs", "title", "text", "impl", "message",+];++/// Union of property names in a `GET /{pattern}/_mapping` body.+///+/// `None` when `mapping` is not that shape, so a transport failure keeps the+/// historical code-shaped query. An empty object is a real answer (no+/// indices) and comes back empty — callers treat an empty set like an+/// unreadable mapping and do not drop clauses they cannot prove unmapped.+pub fn mapped_property_names(mapping: &Value) -> Option<BTreeSet<String>> {+ let obj = mapping.as_object()?;+ if obj.is_empty() {+ return Some(BTreeSet::new());+ }+ let mut names = BTreeSet::new();+ let mut saw_mapping = false;+ for body in obj.values() {+ let Some(props) = body+ .pointer("/mappings/properties")+ .and_then(Value::as_object)+ else {+ continue;+ };+ saw_mapping = true;+ names.extend(props.keys().cloned());+ }+ if !saw_mapping {+ return None;+ }+ Some(names)+}++/// Types stored as `long` or `double`. A `multi_match`, `term`, or+/// `match_phrase` that names one fails with `For input string: "<query>"`+/// (#1284). `keyword` is not in this list: `name` stays an exact term.+fn is_numeric_mapping_type(ty: &str) -> bool {+ matches!(+ ty,+ "long"+ | "integer"+ | "short"+ | "byte"+ | "unsigned_long"+ | "double"+ | "float"+ | "half_float"+ | "scaled_float"+ )+}++/// Field names any index in a `GET /{pattern}/_mapping` body maps as numeric.+///+/// One numeric mapping is enough to drop the field from a wildcard query:+/// the request is refused for every index, the way `xerj code` drops a+/// numeric field from its union (#1605). An unreadable body yields an empty+/// set; callers ignore that unless [`mapped_property_names`] also succeeded.+pub fn numeric_property_names(mapping: &Value) -> BTreeSet<String> {+ let mut out = BTreeSet::new();+ let Some(obj) = mapping.as_object() else {+ return out;+ };+ for body in obj.values() {+ let Some(props) = body+ .pointer("/mappings/properties")+ .and_then(Value::as_object)+ else {+ continue;+ };+ for (key, spec) in props {+ if spec+ .get("type")+ .and_then(Value::as_str)+ .is_some_and(is_numeric_mapping_type)+ {+ out.insert(key.clone());+ }+ }+ }+ out+}++/// A lookup clause may name `field` when it is mapped (or the mapping was+/// unreadable) and no index maps it as numeric.+pub fn field_lookup_ok(+ mapped: Option<&BTreeSet<String>>,+ numeric: Option<&BTreeSet<String>>,+ field: &str,+) -> bool {+ field_mapped(mapped, field) && !numeric.is_some_and(|set| set.contains(field))+}++/// True when a numeric mapping removed a field this command would look up,+/// so the historical query must not be restored — it would name that field+/// again and the wildcard would fail.+fn numeric_drops_a_lookup(numeric: Option<&BTreeSet<String>>, candidates: &[&str]) -> bool {+ numeric.is_some_and(|set| candidates.iter().any(|name| set.contains(*name)))+}++/// Whether `field` may be named in a lookup clause.+///+/// An unreadable mapping (`None`) or an empty union keeps every historical+/// clause: dropping them with no evidence would change the #916 ranking on+/// a code corpus whose mapping we failed to read.+pub fn field_mapped(mapped: Option<&BTreeSet<String>>, field: &str) -> bool {+ match mapped {+ None => true,+ Some(set) if set.is_empty() => true,+ Some(set) => set.contains(field),+ }+}++/// True when the mapping was read and no index maps `name`, `code`, or `defs`.+pub fn code_symbol_fields_absent(mapped: Option<&BTreeSet<String>>) -> bool {+ match mapped {+ Some(set) if !set.is_empty() => CODE_SYMBOL_FIELDS.iter().all(|f| !set.contains(*f)),+ _ => false,+ }+}++/// `preferred` entries whose base field is mapped, then `extras` (bare names).+///+/// `prose_beside_code`: also name `text` and `message` when a code field is+/// mapped. A folded `docs` index stores changelog and config lines in `text`+/// next to `name`/`code`/`defs`. Leaving `text` off made `xerj search` miss+/// `tokio/CHANGELOG.md` (122 sections) while a direct `match_phrase` on+/// `text` hit them. `impl` stays off in that case so a code corpus does not+/// grow a second body leg (#916). A corpus with no code field still gets+/// every extra it maps, including `impl`.+pub fn weighted_query_fields(+ preferred: &[(&str, &str)],+ extras: &[&str],+ mapped: Option<&BTreeSet<String>>,+ numeric: Option<&BTreeSet<String>>,+ prose_beside_code: bool,+) -> Vec<String> {+ let mut out: Vec<String> = preferred+ .iter()+ .filter(|(name, _)| field_lookup_ok(mapped, numeric, name))+ .map(|(_, weighted)| (*weighted).to_string())+ .collect();+ let code_present = !code_symbol_fields_absent(mapped);+ if let Some(set) = mapped.filter(|set| !set.is_empty()) {+ for name in extras {+ if code_present && (*name == "impl" || !prose_beside_code) {+ continue;+ }+ if set.contains(*name)+ && !numeric.is_some_and(|nums| nums.contains(*name))+ && !out.iter().any(|f| f.split('^').next() == Some(*name))+ {+ out.push((*name).to_string());+ }+ }+ }+ out+}++/// Build the ES search request body used by `xerj search`.+///+/// `mapped` is the union of property names under the prefix. `None` (and an+/// empty union) is the historical code-shaped query.+pub fn search_query_body(query: &str, k: usize) -> Value {+ search_query_body_for_mapping(query, k, None, None)+}++/// [`search_query_body`] with the target's mapped field names (#1601) and+/// the names any index maps as numeric (#1284).+pub fn search_query_body_for_mapping(+ query: &str,+ k: usize,+ mapped: Option<&BTreeSet<String>>,+ numeric: Option<&BTreeSet<String>>,+) -> Value {+ let built = build_search_query(query, k, mapped, numeric);+ if should_clauses(&built).is_empty() {+ if numeric_drops_a_lookup(numeric, SEARCH_LOOKUP_FIELDS) {+ // Restoring the historical query would name the numeric field+ // again. An empty `should` matches every document; match nothing.+ let mut built = built;+ built["query"] = json!({ "match_none": {} });+ return built;+ }+ // Nothing we can name is mapped. The historical shape is what the+ // server's 400 then explains, listing the searchable fields.+ return build_search_query(query, k, None, None);+ }+ built+}++fn should_clauses(body: &Value) -> &[Value] {+ body.pointer("/query/bool/should")+ .and_then(Value::as_array)+ .map(Vec::as_slice)+ .unwrap_or(&[])+}++fn build_search_query(+ query: &str,+ k: usize,+ mapped: Option<&BTreeSet<String>>,+ numeric: Option<&BTreeSet<String>>,+) -> Value {+ let shaped = normalize_code_search_query(query);+ let mut should = Vec::new();+ let fields = weighted_query_fields(SEARCH_MM_FIELDS, PROSE_EXTRA_FIELDS, mapped, numeric, true);+ if !fields.is_empty() {+ should.push(json!({ "multi_match": {+ "query": shaped,+ "fields": fields,+ "type": "most_fields"+ }}));+ }+ if field_lookup_ok(mapped, numeric, "defs") {+ should.push(json!({ "match_phrase": { "defs": { "query": shaped, "boost": 8 } } }));+ }+ if field_lookup_ok(mapped, numeric, "name") {+ should.push(json!({ "term": { "name": { "value": query.trim(), "boost": 10 } } }));+ }+ // An exact phrase in a content field. A pasted error message or log+ // line is the strongest signal of which file is meant, and the+ // bag-of-words `multi_match` alone ranks a short `defs` list sharing two+ // of its words above the long file holding all of them in order. Only+ // the content fields the `multi_match` already names, so the mapping and+ // numeric drops (#1601, #1284) apply unchanged. Appended last so the+ // historical clause positions do not move.+ if let Some(phrase) = exact_phrase_query(query) {+ for field in &fields {+ let base = field.split('^').next().unwrap_or(field);+ if PHRASE_CONTENT_FIELDS.contains(&base) {+ should.push(json!({ "match_phrase": { base: {+ "query": phrase,+ "boost": EXACT_PHRASE_BOOST+ } } }));+ }+ }+ }+ json!({+ "size": k,+ "query": { "bool": { "should": should }},+ // No `body` here: whole-file bodies are the token flood (measured ~9x+ // the size of the matching passage). `_passage` carries the snippet.+ "_source": ["ax_path", "ax_locator", "line", "start_line", "language", "kind", "code", "title", "heading", "page", "cell"],+ "fields": ["_passage"]+ })+}diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rsindex f6b78d851..e40440226 100644--- a/engine/crates/xerj-mcp/src/lib.rs+++ b/engine/crates/xerj-mcp/src/lib.rs@@ -104,6 +104,9 @@ use std::time::Duration;use serde_json::{json, Value};use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};+use xerj_common::code_query::{+ mapped_property_names, numeric_property_names, search_query_body_for_mapping,+};/// MCP protocol revision we default to when the client does not request one.const DEFAULT_PROTOCOL_VERSION: &str = "2025-06-18";@@ -1711,7 +1714,23 @@ async fn call_tool(ctx: &Ctx, msg: &Value) -> Value {};let result = match name {- "xerj_search" => build_search(&args),+ "xerj_search" => {+ // #1961: a plain-string query is shaped from the target's mapping,+ // exactly like `xerj search` (#1602). Without this, a prose corpus+ // answers HTTP 400 `strict_unknown_fields` because the body names+ // `defs` and `name`, which it does not map. A DSL object is+ // forwarded untouched (no mapping read, no rewriting).+ let mapping = match args.get("query") {+ Some(Value::String(q)) if !q.trim().is_empty() => {+ fetch_mapping(ctx, search_index(&args)).await+ }+ _ => None,+ };+ match mapping {+ Some(mapping) => build_search_with_mapping(&args, Some(&mapping)),+ None => build_search(&args),+ }+ }"xerj_semantic_search" => build_semantic(&args),"xerj_vector_search" => build_vector(&args),"xerj_hybrid_search" => build_hybrid(&args),@@ -2040,17 +2059,31 @@ fn apply_judge(}fn build_search(args: &Value) -> Result<BuiltRequest, String> {- // #962: `index` used to be required, but nothing an agent reads said what- // to pass — the first real tool call after `xerj autoindex` was an error- // until the agent guessed `ax-*`. Default to the prefix autoindex writes- // (matching the CLI's `xerj search`, which defaults to "ax"); an explicit- // index still wins.- let index = args- .get("index")+ build_search_with_mapping(args, None)+}++/// The index pattern a tool call targets: the CLI's `ax-*` default (#962); an+/// explicit argument wins.+fn search_index(args: &Value) -> &str {+ args.get("index").and_then(Value::as_str).map(str::trim).filter(|s| !s.is_empty())- .unwrap_or("ax-*");+ .unwrap_or("ax-*")+}++/// [`build_search`] with the target's `GET /{index}/_mapping` body (#1961).+///+/// A plain-string query is shaped by `xerj_common::code_query` — the same+/// implementation `xerj search` uses (#1601/#1602). On a corpus that maps none+/// of `name`/`code`/`defs` the code legs are dropped and the prose fields are+/// searched, instead of the request 400ing on a field the index does not map.+/// `None` (no readable mapping) keeps the historical #916 shape.+fn build_search_with_mapping(+ args: &Value,+ mapping: Option<&Value>,+) -> Result<BuiltRequest, String> {+ let index = search_index(args);let mut body = serde_json::Map::new();let mut plain_string = false;match args.get("query") {@@ -2061,26 +2094,15 @@ fn build_search(args: &Value) -> Result<BuiltRequest, String> {// untouched for structured queries.Some(Value::String(q)) if !q.trim().is_empty() => {plain_string = true;- let shaped = xerj_common::code_query::normalize_code_search_query(q);- let mut should = vec![- json!({ "multi_match": {- "query": shaped,- "fields": ["name^4", "code^2", "body", "defs^3", "title"],- "type": "most_fields"- }}),- json!({ "match_phrase": { "defs": { "query": shaped, "boost": 8 } } }),- json!({ "term": { "name": { "value": q.trim(), "boost": 10 } } }),- ];- // Same exact-phrase leg as `xerj search`: a pasted error string- // must rank the file that contains it, not a `defs` list that- // shares two of its words.- if let Some(phrase) = xerj_common::code_query::exact_phrase_query(q) {- should.push(json!({ "match_phrase": { "body": {- "query": phrase,- "boost": xerj_common::code_query::EXACT_PHRASE_BOOST- } } }));- }- body.insert("query".into(), json!({ "bool": { "should": should } }));+ // #1961: the mapping-aware body the CLI builds (#1601/#1602),+ // including the #1759 exact-phrase leg. A prose corpus drops the+ // `defs`/`name` clauses instead of 400ing on them; a code corpus+ // (or an unreadable mapping) keeps the historical shape. `10`+ // only fills the discarded `size` member.+ let mapped = mapping.and_then(mapped_property_names);+ let numeric = mapping.map(numeric_property_names);+ let built = search_query_body_for_mapping(q, 10, mapped.as_ref(), numeric.as_ref());+ body.insert("query".into(), built["query"].clone());}Some(q) if !q.is_null() => {body.insert("query".into(), q.clone());@@ -2582,6 +2604,26 @@ fn code_root() -> std::path::PathBuf {/// to that token budget by [`token_budget::budget_json_response`]. Errors are/// never budgeted — a refusal the agent cannot read end-to-end is worse than/// a long response, and refusals are small anyway.+/// `GET /{index}/_mapping` for a plain-string `xerj_search` (#1961).+///+/// `None` when the node cannot answer or the body is not JSON: the caller+/// then keeps the historical code-shaped query, exactly like the CLI does+/// when the mapping read fails (#1601). Read on every plain-string call, not+/// cached: `xerj autoindex` can add a dataset (and a field) between two tool+/// calls, and a stale mapping is the failure this exists to prevent.+async fn fetch_mapping(ctx: &Ctx, index: &str) -> Option<Value> {+ let url = format!("{}/{index}/_mapping", ctx.base_url.trim_end_matches('/'));+ let mut req = ctx.client.get(&url);+ if let Some(auth) = &ctx.auth {+ req = req.header("Authorization", auth);+ }+ let resp = req.send().await.ok()?;+ if !resp.status().is_success() {+ return None;+ }+ resp.json::<Value>().await.ok()+}+async fn engine_request(ctx: &Ctx,method: Method,@@ -2767,6 +2809,135 @@ mod tests {assert_eq!(path, "/docs/_search");}+ /// #1961: a plain-string query on a prose corpus must not name the code+ /// legs. Autoindex indices are created `strict_unknown_fields`, so the+ /// old body (`match_phrase` on `defs`, `term` on `name`) was an HTTP 400+ /// on any corpus that maps neither — the MCP path never got #1602's fix.+ #[test]+ fn plain_string_query_drops_unmapped_code_legs_on_a_prose_mapping() {+ let mapping = json!({+ "ax-docs": { "mappings": { "properties": {+ "body": {"type": "text"},+ "title": {"type": "text"},+ "text": {"type": "text"},+ "impl": {"type": "text"},+ "message": {"type": "text"}+ }}}+ });+ let (_, _, body) = built(build_search_with_mapping(+ &json!({"query": "how does the retry work"}),+ Some(&mapping),+ ));+ let should = body["query"]["bool"]["should"]+ .as_array()+ .expect("should clauses");+ let fields: Vec<String> = should+ .iter()+ .find_map(|c| c.get("multi_match"))+ .and_then(|m| m.get("fields"))+ .and_then(Value::as_array)+ .expect("multi_match fields")+ .iter()+ .map(|f| f.as_str().unwrap().to_string())+ .collect();+ assert!(+ fields+ .iter()+ .all(|f| !matches!(f.split('^').next().unwrap(), "name" | "code" | "defs")),+ "a prose corpus must not be queried with code legs: {fields:?}"+ );+ for expected in ["body", "title", "text", "impl", "message"] {+ assert!(+ fields.iter().any(|f| f.split('^').next() == Some(expected)),+ "`{expected}` is mapped and must be searched: {fields:?}"+ );+ }+ for clause in should {+ if let Some(phrase) = clause.get("match_phrase").and_then(Value::as_object) {+ let field = phrase.keys().next().expect("phrase field");+ assert_ne!(+ field, "defs",+ "an unmapped `defs` phrase was sent: {should:?}"+ );+ }+ assert!(+ clause.get("term").is_none(),+ "an unmapped `name` term was sent: {should:?}"+ );+ }+ }++ /// On a code corpus the mapping-aware body is exactly the historical+ /// #916 shape: the builder only removes legs the target cannot answer.+ #[test]+ fn plain_string_query_keeps_the_code_shape_on_a_code_mapping() {+ let mapping = json!({+ "ax-rust": { "mappings": { "properties": {+ "name": {"type": "keyword"},+ "code": {"type": "text"},+ "body": {"type": "text"},+ "defs": {"type": "text"},+ "title": {"type": "text"},+ "line": {"type": "long"}+ }}}+ });+ let (_, _, with) = built(build_search_with_mapping(+ &json!({"query": "flush the tree"}),+ Some(&mapping),+ ));+ let (_, _, without) = built(build_search(&json!({"query": "flush the tree"})));+ assert_eq!(+ with["query"], without["query"],+ "a code corpus must keep the #916 query"+ );+ }++ /// A numeric mapping of a code field drops that leg (#1284 via #1601):+ /// naming it fails the whole wildcard request with `For input string`.+ #[test]+ fn plain_string_query_drops_a_numeric_code_leg() {+ let mapping = json!({+ "ax-mixed": { "mappings": { "properties": {+ "name": {"type": "long"},+ "code": {"type": "text"},+ "body": {"type": "text"},+ "defs": {"type": "text"}+ }}}+ });+ let (_, _, body) = built(build_search_with_mapping(+ &json!({"query": "flush"}),+ Some(&mapping),+ ));+ let should = body["query"]["bool"]["should"].as_array().unwrap();+ assert!(+ should.iter().all(|c| c.get("term").is_none()),+ "a numeric `name` must not get a `term`: {should:?}"+ );+ let fields: Vec<&str> = should+ .iter()+ .find_map(|c| c.get("multi_match"))+ .and_then(|m| m.get("fields"))+ .and_then(Value::as_array)+ .unwrap()+ .iter()+ .map(|f| f.as_str().unwrap())+ .collect();+ assert!(!fields.iter().any(|f| f.starts_with("name")), "{fields:?}");+ }++ /// An unreadable mapping must not narrow the query: dropping clauses with+ /// no evidence would change the #916 ranking (the CLI's #1601 rule).+ #[test]+ fn unreadable_mapping_keeps_the_historical_shape() {+ let mapping = json!({"error": {"reason": "no such index"}});+ let (_, _, with) = built(build_search_with_mapping(+ &json!({"query": "flush the tree"}),+ Some(&mapping),+ ));+ let (_, _, without) = built(build_search(&json!({"query": "flush the tree"})));+ assert_eq!(with["query"], without["query"]);+ }+#[test]fn search_plain_string_strips_multiword_glue_tokens_and_preserves_single_word() {let (_, _, body) = built(build_search(&json!({@@ -3641,6 +3812,105 @@ mod tests {String::from_utf8_lossy(&buf).into_owned()}+ /// Serve exactly two requests — a mapping read, then the search — and+ /// return the SECOND request (headers + body). Proves `xerj_search` reads+ /// the mapping before shaping a plain-string query (#1961) and never+ /// sends a code leg the mapping does not have.+ fn serve_mapping_then_search(+ listener: std::net::TcpListener,+ mapping_body: &str,+ search_body: &str,+ ) -> String {+ use std::io::{Read, Write};+ let mut last = String::new();+ for body in [mapping_body, search_body] {+ let (mut sock, _) = listener.accept().expect("accept");+ sock.set_read_timeout(Some(std::time::Duration::from_secs(5)))+ .unwrap();+ let mut buf = Vec::new();+ let mut tmp = [0u8; 4096];+ loop {+ match sock.read(&mut tmp) {+ Ok(0) => break,+ Ok(n) => {+ buf.extend_from_slice(&tmp[..n]);+ if let Some(header_end) = buf.windows(4).position(|w| w == b"\r\n\r\n") {+ let headers = String::from_utf8_lossy(&buf[..header_end]);+ let length = headers.lines().find_map(|line| {+ let rest = line+ .split_once(':')+ .filter(|(name, _)| {+ name.eq_ignore_ascii_case("content-length")+ })?+ .1;+ rest.trim().parse::<usize>().ok()+ });+ let body_at = header_end + 4;+ if let Some(length) = length {+ while buf.len() < body_at + length {+ match sock.read(&mut tmp) {+ Ok(0) | Err(_) => break,+ Ok(n) => buf.extend_from_slice(&tmp[..n]),+ }+ }+ }+ break;+ }+ }+ Err(_) => break,+ }+ }+ last = String::from_utf8_lossy(&buf).into_owned();+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ );+ let _ = sock.write_all(response.as_bytes());+ }+ last+ }++ /// #1961 end to end: the tool reads `GET /{index}/_mapping` first, so the+ /// search body it posts on a prose corpus names no unmapped code field —+ /// the HTTP 400 `strict_unknown_fields` the field report hit.+ #[test]+ fn search_tool_reads_the_mapping_before_a_plain_string_query() {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let port = listener.local_addr().unwrap().port();+ let mapping = r#"{"ax-docs":{"mappings":{"properties":{"body":{"type":"text"},"title":{"type":"text"},"text":{"type":"text"}}}}}"#;+ let hits = r#"{"took":1,"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"1","_source":{"ax_path":"guide.md"}}]}}"#;+ let server = std::thread::spawn(move || serve_mapping_then_search(listener, mapping, hits));+ let ctx = Ctx {+ client: reqwest::Client::new(),+ base_url: format!("http://127.0.0.1:{port}"),+ auth: None,+ };+ let msg = json!({+ "params": { "name": "xerj_search", "arguments": {+ "query": "how does the retry policy work"+ } }+ });+ let res = block_on(call_tool(&ctx, &msg));+ let request = server.join().expect("fake node thread");+ assert_eq!(res["isError"], false, "{res}");+ assert!(+ request.starts_with("POST /ax-*/_search HTTP/1.1"),+ "the SECOND request must be the search: {request}"+ );+ assert!(+ !request.contains("\"defs\""),+ "an unmapped `defs` leg was sent (HTTP 400 on a prose index): {request}"+ );+ assert!(+ !request.contains("\"name\""),+ "an unmapped `name` leg was sent (HTTP 400 on a prose index): {request}"+ );+ assert!(+ request.contains("\"text\""),+ "the mapped prose field was not searched: {request}"+ );+ }+fn block_on<T>(future: impl std::future::Future<Output = T>) -> T {tokio::runtime::Builder::new_current_thread().enable_all()