Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): xerj search and def drop unmapped code legs (#1601) (#1602)

xerj-org598fb8538405
diff --git a/engine/crates/xerj-autoindex/src/def.rs b/engine/crates/xerj-autoindex/src/def.rs
index 31ea04120..0b42c7459 100644
--- a/engine/crates/xerj-autoindex/src/def.rs
+++ b/engine/crates/xerj-autoindex/src/def.rs
@@ -11,8 +11,13 @@
//! agent act in one call instead of re-searching (~570 output tokens vs a
//! whole-file dump).
+use std::collections::BTreeSet;
+
use crate::esclient::Es;
-use crate::search::{hit_location, is_identifier, passage_text};
+use crate::search::{
+ field_mapped, hit_location, is_identifier, mapped_property_names, passage_text,
+ weighted_query_fields,
+};
use serde_json::{json, Value};
const USAGE: &str = "\
@@ -38,9 +43,51 @@ SEE ALSO:
xerj search \"<plain words>\" ranked passages when you don't know the name
";
+/// Historical `xerj def` multi_match weights. Order is part of the
+/// code-corpus query.
+const DEF_MM_FIELDS: &[(&str, &str)] = &[
+ ("name", "name^4"),
+ ("defs", "defs^3"),
+ ("code", "code^2"),
+ ("title", "title"),
+];
+
+/// Joined into the recall floor only when the target maps no `name` /
+/// `code` / `defs` (#1601). `body` is not in the code-shaped list above.
+const DEF_PROSE_EXTRA_FIELDS: &[&str] = &["body", "text", "impl", "message"];
+
/// The `xerj def` query body: definition-first ranking over the per-symbol
/// docs, narrowed by the optional `--kind` / `--lang` filters.
-fn def_query(query: &str, kind: Option<&str>, lang: Option<&str>, k: usize) -> Value {
+///
+/// `mapped` is the union of property names under the prefix. `None` keeps
+/// the historical code-shaped clauses. A read mapping drops every lookup
+/// clause whose field no index maps, including `--kind` / `--lang` — those
+/// `term` filters 400 a strict index the same way `name` does (#1601).
+fn def_query(
+ query: &str,
+ kind: Option<&str>,
+ lang: Option<&str>,
+ k: usize,
+ mapped: Option<&BTreeSet<String>>,
+) -> Value {
+ let built = build_def_query(query, kind, lang, k, mapped);
+ let empty = built
+ .pointer("/query/bool/should")
+ .and_then(Value::as_array)
+ .is_some_and(Vec::is_empty);
+ if empty {
+ return build_def_query(query, kind, lang, k, None);
+ }
+ built
+}
+
+fn build_def_query(
+ query: &str,
+ kind: Option<&str>,
+ lang: Option<&str>,
+ k: usize,
+ mapped: Option<&BTreeSet<String>>,
+) -> Value {
// Definition-first ranking, measured at 94% correct-file-at-rank-1 on an
// 80-task cross-file benchmark (vs 38% for body-text search):
// - `term name`: the whole query IS the identifier — keyword-exact on
@@ -49,26 +96,36 @@ fn def_query(query: &str, kind: Option<&str>, lang: Option<&str>, k: usize) -> V
// pre-#500 indexes that have no symbol docs, boost 8.
// - `multi_match`: recall floor so plain-words queries ("euler angles to
// rotation matrix") still surface the definer via `defs`/`code` text.
- let mut should = vec![
- json!({ "term": { "name": { "value": query, "boost": 12 } } }),
- json!({ "match_phrase": { "defs": { "query": query, "boost": 8 } } }),
- json!({ "multi_match": {
+ let mut should = Vec::new();
+ if field_mapped(mapped, "name") {
+ should.push(json!({ "term": { "name": { "value": query, "boost": 12 } } }));
+ }
+ if field_mapped(mapped, "defs") {
+ should.push(json!({ "match_phrase": { "defs": { "query": query, "boost": 8 } } }));
+ }
+ let fields = weighted_query_fields(DEF_MM_FIELDS, DEF_PROSE_EXTRA_FIELDS, mapped);
+ if !fields.is_empty() {
+ should.push(json!({ "multi_match": {
"query": query,
- "fields": ["name^4", "defs^3", "code^2", "title"],
+ "fields": fields,
"type": "most_fields"
- }}),
- ];
- if is_identifier(query) {
+ }}));
+ }
+ if is_identifier(query) && field_mapped(mapped, "code") {
// `code` holds the declaration text; a phrase hit there ("def <name>(",
// "class <name>:") is a definition even when `name` casing differs.
should.push(json!({ "match_phrase": { "code": { "query": query, "boost": 4 } } }));
}
let mut filter = Vec::new();
if let Some(kd) = kind {
- filter.push(json!({ "term": { "kind": kd } }));
+ if field_mapped(mapped, "kind") {
+ filter.push(json!({ "term": { "kind": kd } }));
+ }
}
if let Some(lg) = lang {
- filter.push(json!({ "term": { "language": lg } }));
+ if field_mapped(mapped, "language") {
+ filter.push(json!({ "term": { "language": lg } }));
+ }
}
json!({
"size": k,
@@ -151,9 +208,20 @@ pub fn run_def_cli() -> i32 {
return 2;
}
- let body = def_query(&query, kind.as_deref(), lang.as_deref(), k);
-
let pattern = format!("{prefix}-*");
+ let mapped = es
+ .get_json(&format!("/{pattern}/_mapping"))
+ .ok()
+ .and_then(|body| mapped_property_names(&body));
+ if let Some(set) = mapped.as_ref().filter(|s| !s.is_empty()) {
+ if kind.is_some() && !set.contains("kind") {
+ eprintln!("xerj def: --kind ignored; no index under this prefix maps `kind`");
+ }
+ if lang.is_some() && !set.contains("language") {
+ eprintln!("xerj def: --lang ignored; no index under this prefix maps `language`");
+ }
+ }
+ let body = def_query(&query, kind.as_deref(), lang.as_deref(), k, mapped.as_ref());
let resp = match es.search(&pattern, &body) {
Ok(v) => v,
Err(e) => {
@@ -233,7 +301,7 @@ mod tests {
(Some("class"), Some("rust")),
(None, None),
] {
- let body = def_query("zzqq_nonexistent", kind, lang, 5);
+ let body = def_query("zzqq_nonexistent", kind, lang, 5, None);
let b = &body["query"]["bool"];
assert_eq!(b["minimum_should_match"], 1, "{body}");
assert!(!b["should"].as_array().unwrap().is_empty());
@@ -241,4 +309,65 @@ mod tests {
assert_eq!(filters, kind.is_some() as usize + lang.is_some() as usize);
}
}
+
+ fn names_of(fields: &[&str]) -> BTreeSet<String> {
+ let mut props = serde_json::Map::new();
+ for f in fields {
+ props.insert((*f).to_string(), json!({ "type": "text" }));
+ }
+ mapped_property_names(&json!({
+ "c-docs": { "mappings": { "properties": props } }
+ }))
+ .expect("mapping shape")
+ }
+
+ /// #1601: `xerj def` on a corpus with no symbol fields must not send
+ /// `term name`, `match_phrase defs`, or `match_phrase code`. An identifier
+ /// query is the case that used to add the `code` leg.
+ #[test]
+ fn prose_mapping_drops_unmapped_definition_legs() {
+ let prose = names_of(&["body", "title", "text", "impl"]);
+ let body = def_query(
+ "compressPayload",
+ Some("function"),
+ Some("markdown"),
+ 5,
+ Some(&prose),
+ );
+ let should = body["query"]["bool"]["should"].as_array().unwrap();
+ assert_eq!(should.len(), 1, "{body}");
+ assert_eq!(
+ should[0]["multi_match"]["fields"],
+ json!(["title", "body", "text", "impl"])
+ );
+ assert!(
+ body["query"]["bool"]["filter"]
+ .as_array()
+ .unwrap()
+ .is_empty(),
+ "kind and language are unmapped, so their term filters must not 400 the query: {body}"
+ );
+ let rendered = body["query"].to_string();
+ assert!(!rendered.contains("\"defs\""), "{rendered}");
+ assert!(!rendered.contains("\"name\""), "{rendered}");
+ assert!(!rendered.contains("\"code\""), "{rendered}");
+ assert!(!rendered.contains("\"kind\""), "{rendered}");
+ }
+
+ /// The measured code-corpus query is unchanged when those fields are mapped.
+ #[test]
+ fn code_mapping_keeps_the_historical_def_query() {
+ let code = names_of(&[
+ "name", "defs", "code", "title", "body", "kind", "language", "text",
+ ]);
+ let query = "compressPayload";
+ assert_eq!(
+ def_query(query, Some("function"), Some("rust"), 5, Some(&code)),
+ def_query(query, Some("function"), Some("rust"), 5, None)
+ );
+ assert_eq!(
+ def_query(query, None, None, 5, Some(&BTreeSet::new())),
+ def_query(query, None, None, 5, None)
+ );
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rs
index 49a064591..e5d0a48ec 100644
--- a/engine/crates/xerj-autoindex/src/search.rs
+++ b/engine/crates/xerj-autoindex/src/search.rs
@@ -8,6 +8,8 @@
//! `multi_match` biased toward symbol definitions, printing a passage to read
//! rather than a JSON blob to parse.
+use std::collections::BTreeSet;
+
use crate::esclient::Es;
use serde_json::{json, Value};
@@ -72,20 +74,153 @@ pub(crate) fn is_identifier(q: &str) -> bool {
&& q.chars().all(|c| c.is_alphanumeric() || c == '_')
}
+/// `name` / `code` / `defs` are the code-shaped legs (#916). A corpus that
+/// maps none of them — markdown, logs, anything autoindex accepted that is
+/// not source — must not be queried with those legs: autoindex indices are
+/// created `strict_unknown_fields`, and a `term` or `match_phrase` on a
+/// field no resolved index maps is HTTP 400 (#1601).
+const CODE_SYMBOL_FIELDS: &[&str] = &["name", "code", "defs"];
+
+/// Historical `xerj search` multi_match weights. Order is part of the
+/// code-corpus query: do not reorder.
+const SEARCH_MM_FIELDS: &[(&str, &str)] = &[
+ ("name", "name^4"),
+ ("code", "code^2"),
+ ("body", "body"),
+ ("defs", "defs^3"),
+ ("title", "title"),
+];
+
+/// Content fields a non-code autoindex maps (`text` on prose, `impl` on
+/// markdown sections, `message` on the log family). Joined only when the
+/// target maps none of [`CODE_SYMBOL_FIELDS`], so a code corpus keeps the
+/// #916 shape exactly.
+const PROSE_EXTRA_FIELDS: &[&str] = &["text", "impl", "message"];
+
+/// Union of property names in a `GET /{pattern}/_mapping` body.
+///
+/// `None` when `mapping` is not that shape, so a transport failure keeps the
+/// historical code-shaped query. An empty object is a real answer (no
+/// indices) and comes back empty — callers treat an empty set like an
+/// unreadable mapping and do not drop clauses they cannot prove unmapped.
+pub(crate) fn mapped_property_names(mapping: &Value) -> Option<BTreeSet<String>> {
+ let obj = mapping.as_object()?;
+ if obj.is_empty() {
+ return Some(BTreeSet::new());
+ }
+ let mut names = BTreeSet::new();
+ let mut saw_mapping = false;
+ for body in obj.values() {
+ let Some(props) = body
+ .pointer("/mappings/properties")
+ .and_then(Value::as_object)
+ else {
+ continue;
+ };
+ saw_mapping = true;
+ names.extend(props.keys().cloned());
+ }
+ if !saw_mapping {
+ return None;
+ }
+ Some(names)
+}
+
+/// Whether `field` may be named in a lookup clause.
+///
+/// An unreadable mapping (`None`) or an empty union keeps every historical
+/// clause: dropping them with no evidence would change the #916 ranking on
+/// a code corpus whose mapping we failed to read.
+pub(crate) fn field_mapped(mapped: Option<&BTreeSet<String>>, field: &str) -> bool {
+ match mapped {
+ None => true,
+ Some(set) if set.is_empty() => true,
+ Some(set) => set.contains(field),
+ }
+}
+
+/// True when the mapping was read and no index maps `name`, `code`, or `defs`.
+pub(crate) fn code_symbol_fields_absent(mapped: Option<&BTreeSet<String>>) -> bool {
+ match mapped {
+ Some(set) if !set.is_empty() => CODE_SYMBOL_FIELDS.iter().all(|f| !set.contains(*f)),
+ _ => false,
+ }
+}
+
+/// `preferred` entries whose base field is mapped, then `extras` (bare names)
+/// when the corpus maps no code-symbol field.
+pub(crate) fn weighted_query_fields(
+ preferred: &[(&str, &str)],
+ extras: &[&str],
+ mapped: Option<&BTreeSet<String>>,
+) -> Vec<String> {
+ let mut out: Vec<String> = preferred
+ .iter()
+ .filter(|(name, _)| field_mapped(mapped, name))
+ .map(|(_, weighted)| (*weighted).to_string())
+ .collect();
+ if code_symbol_fields_absent(mapped) {
+ if let Some(set) = mapped {
+ for name in extras {
+ if set.contains(*name) && !out.iter().any(|f| f.split('^').next() == Some(*name)) {
+ out.push((*name).to_string());
+ }
+ }
+ }
+ }
+ out
+}
+
/// Build the ES search request body used by `xerj search`.
+///
+/// `mapped` is the union of property names under the prefix. `None` (and an
+/// empty union) is the historical code-shaped query.
pub fn search_query_body(query: &str, k: usize) -> Value {
+ search_query_body_for_mapping(query, k, None)
+}
+
+/// [`search_query_body`] with the target's mapped field names (#1601).
+pub fn search_query_body_for_mapping(
+ query: &str,
+ k: usize,
+ mapped: Option<&BTreeSet<String>>,
+) -> Value {
+ let built = build_search_query(query, k, mapped);
+ if should_clauses(&built).is_empty() {
+ // Nothing we can name is mapped. The historical shape is what the
+ // server's 400 then explains, listing the searchable fields.
+ return build_search_query(query, k, None);
+ }
+ built
+}
+
+fn should_clauses(body: &Value) -> &[Value] {
+ body.pointer("/query/bool/should")
+ .and_then(Value::as_array)
+ .map(Vec::as_slice)
+ .unwrap_or(&[])
+}
+
+fn build_search_query(query: &str, k: usize, mapped: Option<&BTreeSet<String>>) -> Value {
let shaped = xerj_common::code_query::normalize_code_search_query(query);
+ let mut should = Vec::new();
+ let fields = weighted_query_fields(SEARCH_MM_FIELDS, PROSE_EXTRA_FIELDS, mapped);
+ if !fields.is_empty() {
+ should.push(json!({ "multi_match": {
+ "query": shaped,
+ "fields": fields,
+ "type": "most_fields"
+ }}));
+ }
+ if field_mapped(mapped, "defs") {
+ should.push(json!({ "match_phrase": { "defs": { "query": shaped, "boost": 8 } } }));
+ }
+ if field_mapped(mapped, "name") {
+ should.push(json!({ "term": { "name": { "value": query.trim(), "boost": 10 } } }));
+ }
json!({
"size": k,
- "query": { "bool": { "should": [
- { "multi_match": {
- "query": shaped,
- "fields": ["name^4", "code^2", "body", "defs^3", "title"],
- "type": "most_fields"
- }},
- { "match_phrase": { "defs": { "query": shaped, "boost": 8 } } },
- { "term": { "name": { "value": query.trim(), "boost": 10 } } }
- ]}},
+ "query": { "bool": { "should": should }},
// No `body` here: whole-file bodies are the token flood (measured ~9x
// the size of the matching passage). `_passage` carries the snippet.
"_source": ["ax_path", "line", "start_line", "language", "kind", "code", "title"],
@@ -207,9 +342,17 @@ pub fn run_search_cli() -> i32 {
// simply contributes no score). `name` is a keyword field on symbol docs
// (issue #500), so the `term` clause fires only when the whole query IS
// the identifier — the strongest possible signal.
- let body = search_query_body(&query, k);
-
+ //
+ // Those legs name fields a non-code autoindex does not map. The indices
+ // are strict, so the clause is HTTP 400 rather than a silent miss
+ // (#1601). Read the mapping and drop a leg no resolved index maps; an
+ // unreadable mapping keeps the code-shaped query.
let pattern = format!("{prefix}-*");
+ let mapped = es
+ .get_json(&format!("/{pattern}/_mapping"))
+ .ok()
+ .and_then(|body| mapped_property_names(&body));
+ let body = search_query_body_for_mapping(&query, k, mapped.as_ref());
let resp = match es.search(&pattern, &body) {
Ok(v) => v,
Err(e) => {
@@ -430,4 +573,83 @@ mod tests {
"crates/fjall/src/batch.rs"
);
}
+
+ fn index_mapping(fields: &[&str]) -> Value {
+ let mut props = serde_json::Map::new();
+ for f in fields {
+ props.insert((*f).to_string(), json!({ "type": "text" }));
+ }
+ json!({ "c-docs": { "mappings": { "properties": props } } })
+ }
+
+ fn names_of(fields: &[&str]) -> BTreeSet<String> {
+ mapped_property_names(&index_mapping(fields)).expect("mapping shape")
+ }
+
+ /// #1601: a markdown/prose corpus maps `body`/`title`/`text`/`impl` and
+ /// none of `defs`/`name`/`code`. The code-shaped `term` and `match_phrase`
+ /// legs are what strict autoindex indices refuse with HTTP 400.
+ #[test]
+ fn prose_mapping_drops_unmapped_code_legs() {
+ let mapped = names_of(&["body", "title", "text", "impl"]);
+ let body = search_query_body_for_mapping("retention ceiling", 5, Some(&mapped));
+ let should = body["query"]["bool"]["should"].as_array().unwrap();
+ assert_eq!(should.len(), 1, "{body}");
+ assert!(should[0].get("match_phrase").is_none(), "{body}");
+ assert!(should[0].get("term").is_none(), "{body}");
+ assert_eq!(
+ should[0]["multi_match"]["fields"],
+ json!(["body", "title", "text", "impl"])
+ );
+ let rendered = body["query"].to_string();
+ assert!(!rendered.contains("\"defs\""), "{rendered}");
+ assert!(!rendered.contains("\"name\""), "{rendered}");
+ assert!(!rendered.contains("\"code\""), "{rendered}");
+ }
+
+ /// A log index whose text lives in `message` still gets a query the
+ /// strict mapping will accept.
+ #[test]
+ fn log_mapping_searches_message_and_names_no_code_field() {
+ let mapped = names_of(&["message", "ts", "host"]);
+ let body = search_query_body_for_mapping("SIP 403 Forbidden", 3, Some(&mapped));
+ let should = body["query"]["bool"]["should"].as_array().unwrap();
+ assert_eq!(should.len(), 1, "{body}");
+ assert_eq!(should[0]["multi_match"]["fields"], json!(["message"]));
+ }
+
+ /// A code corpus — and an unreadable mapping — keep the #916 shape,
+ /// including when the union also carries prose fields.
+ #[test]
+ fn code_mapping_and_unreadable_mapping_keep_the_historical_query() {
+ let query = "fsync the WAL segment";
+ let historical = search_query_body(query, 5);
+ assert_eq!(search_query_body_for_mapping(query, 5, None), historical);
+ let empty = BTreeSet::new();
+ assert_eq!(
+ search_query_body_for_mapping(query, 5, Some(&empty)),
+ historical
+ );
+ assert!(mapped_property_names(&json!({"error": "no such index"})).is_none());
+ let code = names_of(&["name", "code", "body", "defs", "title", "text", "impl"]);
+ assert_eq!(
+ search_query_body_for_mapping(query, 5, Some(&code)),
+ historical
+ );
+ // One index in the union mapping `defs` is enough to keep the leg:
+ // strict mode refuses a field no resolved index maps, not a field
+ // some index does not map.
+ let partial = names_of(&["body", "defs", "title"]);
+ let body = search_query_body_for_mapping(query, 5, Some(&partial));
+ let should = body["query"]["bool"]["should"].as_array().unwrap();
+ assert!(
+ should.iter().any(|c| c.get("match_phrase").is_some()),
+ "{body}"
+ );
+ assert!(should.iter().all(|c| c.get("term").is_none()), "{body}");
+ assert_eq!(
+ should[0]["multi_match"]["fields"],
+ json!(["body", "defs^3", "title"])
+ );
+ }
}