AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): xerj search and def drop unmapped code legs (#1601) (#1602)
xerj-org598fb8538405
diff --git a/engine/crates/xerj-autoindex/src/def.rs b/engine/crates/xerj-autoindex/src/def.rsindex 31ea04120..0b42c7459 100644--- a/engine/crates/xerj-autoindex/src/def.rs+++ b/engine/crates/xerj-autoindex/src/def.rs@@ -11,8 +11,13 @@//! agent act in one call instead of re-searching (~570 output tokens vs a//! whole-file dump).+use std::collections::BTreeSet;+use crate::esclient::Es;-use crate::search::{hit_location, is_identifier, passage_text};+use crate::search::{+ field_mapped, hit_location, is_identifier, mapped_property_names, passage_text,+ weighted_query_fields,+};use serde_json::{json, Value};const USAGE: &str = "\@@ -38,9 +43,51 @@ SEE ALSO:xerj search \"<plain words>\" ranked passages when you don't know the name";+/// Historical `xerj def` multi_match weights. Order is part of the+/// code-corpus query.+const DEF_MM_FIELDS: &[(&str, &str)] = &[+ ("name", "name^4"),+ ("defs", "defs^3"),+ ("code", "code^2"),+ ("title", "title"),+];++/// Joined into the recall floor only when the target maps no `name` /+/// `code` / `defs` (#1601). `body` is not in the code-shaped list above.+const DEF_PROSE_EXTRA_FIELDS: &[&str] = &["body", "text", "impl", "message"];+/// The `xerj def` query body: definition-first ranking over the per-symbol/// docs, narrowed by the optional `--kind` / `--lang` filters.-fn def_query(query: &str, kind: Option<&str>, lang: Option<&str>, k: usize) -> Value {+///+/// `mapped` is the union of property names under the prefix. `None` keeps+/// the historical code-shaped clauses. A read mapping drops every lookup+/// clause whose field no index maps, including `--kind` / `--lang` — those+/// `term` filters 400 a strict index the same way `name` does (#1601).+fn def_query(+ query: &str,+ kind: Option<&str>,+ lang: Option<&str>,+ k: usize,+ mapped: Option<&BTreeSet<String>>,+) -> Value {+ let built = build_def_query(query, kind, lang, k, mapped);+ let empty = built+ .pointer("/query/bool/should")+ .and_then(Value::as_array)+ .is_some_and(Vec::is_empty);+ if empty {+ return build_def_query(query, kind, lang, k, None);+ }+ built+}++fn build_def_query(+ query: &str,+ kind: Option<&str>,+ lang: Option<&str>,+ k: usize,+ mapped: Option<&BTreeSet<String>>,+) -> Value {// Definition-first ranking, measured at 94% correct-file-at-rank-1 on an// 80-task cross-file benchmark (vs 38% for body-text search):// - `term name`: the whole query IS the identifier — keyword-exact on@@ -49,26 +96,36 @@ fn def_query(query: &str, kind: Option<&str>, lang: Option<&str>, k: usize) -> V// pre-#500 indexes that have no symbol docs, boost 8.// - `multi_match`: recall floor so plain-words queries ("euler angles to// rotation matrix") still surface the definer via `defs`/`code` text.- let mut should = vec![- json!({ "term": { "name": { "value": query, "boost": 12 } } }),- json!({ "match_phrase": { "defs": { "query": query, "boost": 8 } } }),- json!({ "multi_match": {+ let mut should = Vec::new();+ if field_mapped(mapped, "name") {+ should.push(json!({ "term": { "name": { "value": query, "boost": 12 } } }));+ }+ if field_mapped(mapped, "defs") {+ should.push(json!({ "match_phrase": { "defs": { "query": query, "boost": 8 } } }));+ }+ let fields = weighted_query_fields(DEF_MM_FIELDS, DEF_PROSE_EXTRA_FIELDS, mapped);+ if !fields.is_empty() {+ should.push(json!({ "multi_match": {"query": query,- "fields": ["name^4", "defs^3", "code^2", "title"],+ "fields": fields,"type": "most_fields"- }}),- ];- if is_identifier(query) {+ }}));+ }+ if is_identifier(query) && field_mapped(mapped, "code") {// `code` holds the declaration text; a phrase hit there ("def <name>(",// "class <name>:") is a definition even when `name` casing differs.should.push(json!({ "match_phrase": { "code": { "query": query, "boost": 4 } } }));}let mut filter = Vec::new();if let Some(kd) = kind {- filter.push(json!({ "term": { "kind": kd } }));+ if field_mapped(mapped, "kind") {+ filter.push(json!({ "term": { "kind": kd } }));+ }}if let Some(lg) = lang {- filter.push(json!({ "term": { "language": lg } }));+ if field_mapped(mapped, "language") {+ filter.push(json!({ "term": { "language": lg } }));+ }}json!({"size": k,@@ -151,9 +208,20 @@ pub fn run_def_cli() -> i32 {return 2;}- let body = def_query(&query, kind.as_deref(), lang.as_deref(), k);-let pattern = format!("{prefix}-*");+ let mapped = es+ .get_json(&format!("/{pattern}/_mapping"))+ .ok()+ .and_then(|body| mapped_property_names(&body));+ if let Some(set) = mapped.as_ref().filter(|s| !s.is_empty()) {+ if kind.is_some() && !set.contains("kind") {+ eprintln!("xerj def: --kind ignored; no index under this prefix maps `kind`");+ }+ if lang.is_some() && !set.contains("language") {+ eprintln!("xerj def: --lang ignored; no index under this prefix maps `language`");+ }+ }+ let body = def_query(&query, kind.as_deref(), lang.as_deref(), k, mapped.as_ref());let resp = match es.search(&pattern, &body) {Ok(v) => v,Err(e) => {@@ -233,7 +301,7 @@ mod tests {(Some("class"), Some("rust")),(None, None),] {- let body = def_query("zzqq_nonexistent", kind, lang, 5);+ let body = def_query("zzqq_nonexistent", kind, lang, 5, None);let b = &body["query"]["bool"];assert_eq!(b["minimum_should_match"], 1, "{body}");assert!(!b["should"].as_array().unwrap().is_empty());@@ -241,4 +309,65 @@ mod tests {assert_eq!(filters, kind.is_some() as usize + lang.is_some() as usize);}}++ fn names_of(fields: &[&str]) -> BTreeSet<String> {+ let mut props = serde_json::Map::new();+ for f in fields {+ props.insert((*f).to_string(), json!({ "type": "text" }));+ }+ mapped_property_names(&json!({+ "c-docs": { "mappings": { "properties": props } }+ }))+ .expect("mapping shape")+ }++ /// #1601: `xerj def` on a corpus with no symbol fields must not send+ /// `term name`, `match_phrase defs`, or `match_phrase code`. An identifier+ /// query is the case that used to add the `code` leg.+ #[test]+ fn prose_mapping_drops_unmapped_definition_legs() {+ let prose = names_of(&["body", "title", "text", "impl"]);+ let body = def_query(+ "compressPayload",+ Some("function"),+ Some("markdown"),+ 5,+ Some(&prose),+ );+ let should = body["query"]["bool"]["should"].as_array().unwrap();+ assert_eq!(should.len(), 1, "{body}");+ assert_eq!(+ should[0]["multi_match"]["fields"],+ json!(["title", "body", "text", "impl"])+ );+ assert!(+ body["query"]["bool"]["filter"]+ .as_array()+ .unwrap()+ .is_empty(),+ "kind and language are unmapped, so their term filters must not 400 the query: {body}"+ );+ let rendered = body["query"].to_string();+ assert!(!rendered.contains("\"defs\""), "{rendered}");+ assert!(!rendered.contains("\"name\""), "{rendered}");+ assert!(!rendered.contains("\"code\""), "{rendered}");+ assert!(!rendered.contains("\"kind\""), "{rendered}");+ }++ /// The measured code-corpus query is unchanged when those fields are mapped.+ #[test]+ fn code_mapping_keeps_the_historical_def_query() {+ let code = names_of(&[+ "name", "defs", "code", "title", "body", "kind", "language", "text",+ ]);+ let query = "compressPayload";+ assert_eq!(+ def_query(query, Some("function"), Some("rust"), 5, Some(&code)),+ def_query(query, Some("function"), Some("rust"), 5, None)+ );+ assert_eq!(+ def_query(query, None, None, 5, Some(&BTreeSet::new())),+ def_query(query, None, None, 5, None)+ );+ }}diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rsindex 49a064591..e5d0a48ec 100644--- a/engine/crates/xerj-autoindex/src/search.rs+++ b/engine/crates/xerj-autoindex/src/search.rs@@ -8,6 +8,8 @@//! `multi_match` biased toward symbol definitions, printing a passage to read//! rather than a JSON blob to parse.+use std::collections::BTreeSet;+use crate::esclient::Es;use serde_json::{json, Value};@@ -72,20 +74,153 @@ pub(crate) fn is_identifier(q: &str) -> bool {&& q.chars().all(|c| c.is_alphanumeric() || c == '_')}+/// `name` / `code` / `defs` are the code-shaped legs (#916). A corpus that+/// maps none of them — markdown, logs, anything autoindex accepted that is+/// not source — must not be queried with those legs: autoindex indices are+/// created `strict_unknown_fields`, and a `term` or `match_phrase` on a+/// field no resolved index maps is HTTP 400 (#1601).+const CODE_SYMBOL_FIELDS: &[&str] = &["name", "code", "defs"];++/// Historical `xerj search` multi_match weights. Order is part of the+/// code-corpus query: do not reorder.+const SEARCH_MM_FIELDS: &[(&str, &str)] = &[+ ("name", "name^4"),+ ("code", "code^2"),+ ("body", "body"),+ ("defs", "defs^3"),+ ("title", "title"),+];++/// Content fields a non-code autoindex maps (`text` on prose, `impl` on+/// markdown sections, `message` on the log family). Joined only when the+/// target maps none of [`CODE_SYMBOL_FIELDS`], so a code corpus keeps the+/// #916 shape exactly.+const PROSE_EXTRA_FIELDS: &[&str] = &["text", "impl", "message"];++/// Union of property names in a `GET /{pattern}/_mapping` body.+///+/// `None` when `mapping` is not that shape, so a transport failure keeps the+/// historical code-shaped query. An empty object is a real answer (no+/// indices) and comes back empty — callers treat an empty set like an+/// unreadable mapping and do not drop clauses they cannot prove unmapped.+pub(crate) fn mapped_property_names(mapping: &Value) -> Option<BTreeSet<String>> {+ let obj = mapping.as_object()?;+ if obj.is_empty() {+ return Some(BTreeSet::new());+ }+ let mut names = BTreeSet::new();+ let mut saw_mapping = false;+ for body in obj.values() {+ let Some(props) = body+ .pointer("/mappings/properties")+ .and_then(Value::as_object)+ else {+ continue;+ };+ saw_mapping = true;+ names.extend(props.keys().cloned());+ }+ if !saw_mapping {+ return None;+ }+ Some(names)+}++/// Whether `field` may be named in a lookup clause.+///+/// An unreadable mapping (`None`) or an empty union keeps every historical+/// clause: dropping them with no evidence would change the #916 ranking on+/// a code corpus whose mapping we failed to read.+pub(crate) fn field_mapped(mapped: Option<&BTreeSet<String>>, field: &str) -> bool {+ match mapped {+ None => true,+ Some(set) if set.is_empty() => true,+ Some(set) => set.contains(field),+ }+}++/// True when the mapping was read and no index maps `name`, `code`, or `defs`.+pub(crate) fn code_symbol_fields_absent(mapped: Option<&BTreeSet<String>>) -> bool {+ match mapped {+ Some(set) if !set.is_empty() => CODE_SYMBOL_FIELDS.iter().all(|f| !set.contains(*f)),+ _ => false,+ }+}++/// `preferred` entries whose base field is mapped, then `extras` (bare names)+/// when the corpus maps no code-symbol field.+pub(crate) fn weighted_query_fields(+ preferred: &[(&str, &str)],+ extras: &[&str],+ mapped: Option<&BTreeSet<String>>,+) -> Vec<String> {+ let mut out: Vec<String> = preferred+ .iter()+ .filter(|(name, _)| field_mapped(mapped, name))+ .map(|(_, weighted)| (*weighted).to_string())+ .collect();+ if code_symbol_fields_absent(mapped) {+ if let Some(set) = mapped {+ for name in extras {+ if set.contains(*name) && !out.iter().any(|f| f.split('^').next() == Some(*name)) {+ out.push((*name).to_string());+ }+ }+ }+ }+ out+}+/// Build the ES search request body used by `xerj search`.+///+/// `mapped` is the union of property names under the prefix. `None` (and an+/// empty union) is the historical code-shaped query.pub fn search_query_body(query: &str, k: usize) -> Value {+ search_query_body_for_mapping(query, k, None)+}++/// [`search_query_body`] with the target's mapped field names (#1601).+pub fn search_query_body_for_mapping(+ query: &str,+ k: usize,+ mapped: Option<&BTreeSet<String>>,+) -> Value {+ let built = build_search_query(query, k, mapped);+ if should_clauses(&built).is_empty() {+ // Nothing we can name is mapped. The historical shape is what the+ // server's 400 then explains, listing the searchable fields.+ return build_search_query(query, k, None);+ }+ built+}++fn should_clauses(body: &Value) -> &[Value] {+ body.pointer("/query/bool/should")+ .and_then(Value::as_array)+ .map(Vec::as_slice)+ .unwrap_or(&[])+}++fn build_search_query(query: &str, k: usize, mapped: Option<&BTreeSet<String>>) -> Value {let shaped = xerj_common::code_query::normalize_code_search_query(query);+ let mut should = Vec::new();+ let fields = weighted_query_fields(SEARCH_MM_FIELDS, PROSE_EXTRA_FIELDS, mapped);+ if !fields.is_empty() {+ should.push(json!({ "multi_match": {+ "query": shaped,+ "fields": fields,+ "type": "most_fields"+ }}));+ }+ if field_mapped(mapped, "defs") {+ should.push(json!({ "match_phrase": { "defs": { "query": shaped, "boost": 8 } } }));+ }+ if field_mapped(mapped, "name") {+ should.push(json!({ "term": { "name": { "value": query.trim(), "boost": 10 } } }));+ }json!({"size": k,- "query": { "bool": { "should": [- { "multi_match": {- "query": shaped,- "fields": ["name^4", "code^2", "body", "defs^3", "title"],- "type": "most_fields"- }},- { "match_phrase": { "defs": { "query": shaped, "boost": 8 } } },- { "term": { "name": { "value": query.trim(), "boost": 10 } } }- ]}},+ "query": { "bool": { "should": should }},// No `body` here: whole-file bodies are the token flood (measured ~9x// the size of the matching passage). `_passage` carries the snippet."_source": ["ax_path", "line", "start_line", "language", "kind", "code", "title"],@@ -207,9 +342,17 @@ pub fn run_search_cli() -> i32 {// simply contributes no score). `name` is a keyword field on symbol docs// (issue #500), so the `term` clause fires only when the whole query IS// the identifier — the strongest possible signal.- let body = search_query_body(&query, k);-+ //+ // Those legs name fields a non-code autoindex does not map. The indices+ // are strict, so the clause is HTTP 400 rather than a silent miss+ // (#1601). Read the mapping and drop a leg no resolved index maps; an+ // unreadable mapping keeps the code-shaped query.let pattern = format!("{prefix}-*");+ let mapped = es+ .get_json(&format!("/{pattern}/_mapping"))+ .ok()+ .and_then(|body| mapped_property_names(&body));+ let body = search_query_body_for_mapping(&query, k, mapped.as_ref());let resp = match es.search(&pattern, &body) {Ok(v) => v,Err(e) => {@@ -430,4 +573,83 @@ mod tests {"crates/fjall/src/batch.rs");}++ fn index_mapping(fields: &[&str]) -> Value {+ let mut props = serde_json::Map::new();+ for f in fields {+ props.insert((*f).to_string(), json!({ "type": "text" }));+ }+ json!({ "c-docs": { "mappings": { "properties": props } } })+ }++ fn names_of(fields: &[&str]) -> BTreeSet<String> {+ mapped_property_names(&index_mapping(fields)).expect("mapping shape")+ }++ /// #1601: a markdown/prose corpus maps `body`/`title`/`text`/`impl` and+ /// none of `defs`/`name`/`code`. The code-shaped `term` and `match_phrase`+ /// legs are what strict autoindex indices refuse with HTTP 400.+ #[test]+ fn prose_mapping_drops_unmapped_code_legs() {+ let mapped = names_of(&["body", "title", "text", "impl"]);+ let body = search_query_body_for_mapping("retention ceiling", 5, Some(&mapped));+ let should = body["query"]["bool"]["should"].as_array().unwrap();+ assert_eq!(should.len(), 1, "{body}");+ assert!(should[0].get("match_phrase").is_none(), "{body}");+ assert!(should[0].get("term").is_none(), "{body}");+ assert_eq!(+ should[0]["multi_match"]["fields"],+ json!(["body", "title", "text", "impl"])+ );+ let rendered = body["query"].to_string();+ assert!(!rendered.contains("\"defs\""), "{rendered}");+ assert!(!rendered.contains("\"name\""), "{rendered}");+ assert!(!rendered.contains("\"code\""), "{rendered}");+ }++ /// A log index whose text lives in `message` still gets a query the+ /// strict mapping will accept.+ #[test]+ fn log_mapping_searches_message_and_names_no_code_field() {+ let mapped = names_of(&["message", "ts", "host"]);+ let body = search_query_body_for_mapping("SIP 403 Forbidden", 3, Some(&mapped));+ let should = body["query"]["bool"]["should"].as_array().unwrap();+ assert_eq!(should.len(), 1, "{body}");+ assert_eq!(should[0]["multi_match"]["fields"], json!(["message"]));+ }++ /// A code corpus — and an unreadable mapping — keep the #916 shape,+ /// including when the union also carries prose fields.+ #[test]+ fn code_mapping_and_unreadable_mapping_keep_the_historical_query() {+ let query = "fsync the WAL segment";+ let historical = search_query_body(query, 5);+ assert_eq!(search_query_body_for_mapping(query, 5, None), historical);+ let empty = BTreeSet::new();+ assert_eq!(+ search_query_body_for_mapping(query, 5, Some(&empty)),+ historical+ );+ assert!(mapped_property_names(&json!({"error": "no such index"})).is_none());+ let code = names_of(&["name", "code", "body", "defs", "title", "text", "impl"]);+ assert_eq!(+ search_query_body_for_mapping(query, 5, Some(&code)),+ historical+ );+ // One index in the union mapping `defs` is enough to keep the leg:+ // strict mode refuses a field no resolved index maps, not a field+ // some index does not map.+ let partial = names_of(&["body", "defs", "title"]);+ let body = search_query_body_for_mapping(query, 5, Some(&partial));+ let should = body["query"]["bool"]["should"].as_array().unwrap();+ assert!(+ should.iter().any(|c| c.get("match_phrase").is_some()),+ "{body}"+ );+ assert!(should.iter().all(|c| c.get("term").is_none()), "{body}");+ assert_eq!(+ should[0]["multi_match"]["fields"],+ json!(["body", "defs^3", "title"])+ );+ }}