AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): query text and message on a mixed docs index (#1705)
xerj-orgfe9f1320a0c7
diff --git a/engine/crates/xerj-autoindex/src/def.rs b/engine/crates/xerj-autoindex/src/def.rsindex 3817a29cd..cfdf6a5a9 100644--- a/engine/crates/xerj-autoindex/src/def.rs+++ b/engine/crates/xerj-autoindex/src/def.rs@@ -194,7 +194,13 @@ fn build_def_query(if field_lookup_ok(mapped, numeric, "defs") {should.push(json!({ "match_phrase": { "defs": { "query": query, "boost": 8 } } }));}- let mut fields = weighted_query_fields(DEF_MM_FIELDS, DEF_PROSE_EXTRA_FIELDS, mapped, numeric);+ let mut fields = weighted_query_fields(+ DEF_MM_FIELDS,+ DEF_PROSE_EXTRA_FIELDS,+ mapped,+ numeric,+ false,+ );// A phrase ("flush the tree to disk") lives in the symbol body (`impl`),// not in the declaration. Only when the read mapping actually has `impl`// and no index maps it as numeric: the historical code querydiff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rsindex 44aa55e38..e4671ef4c 100644--- a/engine/crates/xerj-autoindex/src/search.rs+++ b/engine/crates/xerj-autoindex/src/search.rs@@ -264,28 +264,38 @@ pub(crate) fn code_symbol_fields_absent(mapped: Option<&BTreeSet<String>>) -> bo}}-/// `preferred` entries whose base field is mapped, then `extras` (bare names)-/// when the corpus maps no code-symbol field.+/// `preferred` entries whose base field is mapped, then `extras` (bare names).+///+/// `prose_beside_code`: also name `text` and `message` when a code field is+/// mapped. A folded `docs` index stores changelog and config lines in `text`+/// next to `name`/`code`/`defs`. Leaving `text` off made `xerj search` miss+/// `tokio/CHANGELOG.md` (122 sections) while a direct `match_phrase` on+/// `text` hit them. `impl` stays off in that case so a code corpus does not+/// grow a second body leg (#916). A corpus with no code field still gets+/// every extra it maps, including `impl`.pub(crate) fn weighted_query_fields(preferred: &[(&str, &str)],extras: &[&str],mapped: Option<&BTreeSet<String>>,numeric: Option<&BTreeSet<String>>,+ prose_beside_code: bool,) -> Vec<String> {let mut out: Vec<String> = preferred.iter().filter(|(name, _)| field_lookup_ok(mapped, numeric, name)).map(|(_, weighted)| (*weighted).to_string()).collect();- if code_symbol_fields_absent(mapped) {- if let Some(set) = mapped {- for name in extras {- if set.contains(*name)- && !numeric.is_some_and(|nums| nums.contains(*name))- && !out.iter().any(|f| f.split('^').next() == Some(*name))- {- out.push((*name).to_string());- }+ let code_present = !code_symbol_fields_absent(mapped);+ if let Some(set) = mapped.filter(|set| !set.is_empty()) {+ for name in extras {+ if code_present && (*name == "impl" || !prose_beside_code) {+ continue;+ }+ if set.contains(*name)+ && !numeric.is_some_and(|nums| nums.contains(*name))+ && !out.iter().any(|f| f.split('^').next() == Some(*name))+ {+ out.push((*name).to_string());}}}@@ -339,7 +349,7 @@ fn build_search_query() -> Value {let shaped = xerj_common::code_query::normalize_code_search_query(query);let mut should = Vec::new();- let fields = weighted_query_fields(SEARCH_MM_FIELDS, PROSE_EXTRA_FIELDS, mapped, numeric);+ let fields = weighted_query_fields(SEARCH_MM_FIELDS, PROSE_EXTRA_FIELDS, mapped, numeric, true);if !fields.is_empty() {should.push(json!({ "multi_match": {"query": shaped,@@ -1244,11 +1254,30 @@ mod tests {historical);assert!(mapped_property_names(&json!({"error": "no such index"})).is_none());- let code = names_of(&["name", "code", "body", "defs", "title", "text", "impl"]);+ let code_only = names_of(&["name", "code", "body", "defs", "title"]);assert_eq!(- search_query_body_for_mapping(query, 5, Some(&code), Some(&BTreeSet::new())),- historical+ search_query_body_for_mapping(query, 5, Some(&code_only), Some(&BTreeSet::new())),+ historical,+ "a code corpus with no prose field keeps the #916 field list");+ // A folded docs index maps `text` (changelogs, line-oriented files)+ // beside the code fields. Search it. `impl` stays off so the code+ // legs do not grow a second body.+ let mixed = names_of(&[+ "name", "code", "body", "defs", "title", "text", "impl", "message",+ ]);+ let body = search_query_body_for_mapping(query, 5, Some(&mixed), Some(&BTreeSet::new()));+ let fields = body["query"]["bool"]["should"][0]["multi_match"]["fields"]+ .as_array()+ .unwrap();+ let bare: Vec<&str> = fields+ .iter()+ .map(|f| f.as_str().unwrap().split('^').next().unwrap())+ .collect();+ assert!(bare.contains(&"text"), "{bare:?}");+ assert!(bare.contains(&"message"), "{bare:?}");+ assert!(!bare.contains(&"impl"), "{bare:?}");+ assert!(bare.contains(&"body") && bare.contains(&"defs"), "{bare:?}");// One index in the union mapping `defs` is enough to keep the leg:// strict mode refuses a field no resolved index maps, not a field// some index does not map.