AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(xccode): raw-JSON corpora query their own text fields — the #1158 silent zero
xerj-org14ca8532fb29
diff --git a/engine/crates/xerj-common/src/xccode/fields.rs b/engine/crates/xerj-common/src/xccode/fields.rsindex d05b933c3..b236757da 100644--- a/engine/crates/xerj-common/src/xccode/fields.rs+++ b/engine/crates/xerj-common/src/xccode/fields.rs@@ -75,12 +75,52 @@ pub fn resolve_fields(mapping: Option<&Value>) -> Vec<String> {out.push("text".to_string());}if out.is_empty() {- vec!["body".to_string()]+ // #1158: a raw-JSON corpus (ghsa-db's advisory mirrors, OSV) maps+ // NONE of the standard content fields — every record carries its+ // text in schema-named fields like `summary`/`details`. The old+ // `["body"]` floor here is a field no index maps, and an unmapped+ // multi_match field collapses a multi-token query to ZERO hits+ // with no error: the entire corpus answered "no passage matches"+ // while the same indices returned 10,000+ hits queried directly.+ // Fall back to the corpus's OWN text-typed fields (never `ax_*`+ // provenance) — still mapping-gated, so every field sent is one+ // at least one index really maps.+ let own = own_text_fields(obj);+ if own.is_empty() {+ vec!["body".to_string()]+ } else {+ own+ }} else {out}}+/// The corpus's own searchable content fields, for the no-standard-field+/// floor above: every property mapped `text` (or `semantic_text`) by any+/// index under the prefix, `ax_*` provenance excluded, names sorted, capped+/// so a wide schema cannot balloon the multi_match.+fn own_text_fields(obj: &serde_json::Map<String, Value>) -> Vec<String> {+ const AX: &str = "ax_";+ const MAX_OWN_FIELDS: usize = 24;+ let mut own = std::collections::BTreeSet::new();+ for m in obj.values() {+ let Some(props) = m.pointer("/mappings/properties").and_then(Value::as_object) else {+ continue;+ };+ for (key, spec) in props {+ let searchable = spec+ .get("type")+ .and_then(Value::as_str)+ .is_some_and(|t| t == "text" || t == "semantic_text");+ if searchable && !key.starts_with(AX) {+ own.insert(key.clone());+ }+ }+ }+ own.into_iter().take(MAX_OWN_FIELDS).collect()+}+/// Indices under the prefix whose `field` (default `body`) is mapped as/// `semantic_text`.///@@ -144,8 +184,12 @@ mod tests {vec!["body".to_string(), "defs".into(), "title".into()]);- // No FIELDS member mapped at all -> the ["body"] floor, never empty.- let thin = mapping(&[("i", &["unrelated"], false)]);+ // No FIELDS member and NOTHING text-typed at all -> the ["body"]+ // floor, never empty, never a keyword field. (A text-typed field+ // no standard name covers is the #1158 own-fields case below.)+ let thin = json!({ "i": { "mappings": { "properties": {+ "unrelated": { "type": "keyword" }+ } } } });assert_eq!(resolve_fields(Some(&thin)), vec!["body".to_string()]);}@@ -171,8 +215,12 @@ mod tests {resolve_fields(Some(&with_text)),vec!["body".to_string(), "text".to_string()]);- // No FIELDS member AND no text -> the ["body"] floor still holds.- let thin = mapping(&[("i", &["unrelated"], false)]);+ // No FIELDS member AND no text-typed field -> the ["body"] floor+ // still holds (keyword placeholders — text-typed ones are #1158's+ // own-fields fallback, covered by its own test).+ let thin = json!({ "i": { "mappings": { "properties": {+ "unrelated": { "type": "keyword" }+ } } } });assert_eq!(resolve_fields(Some(&thin)), vec!["body".to_string()]);}@@ -188,4 +236,73 @@ mod tests {assert_eq!(total, 3);assert!(semantic_capable(None) == (Vec::new(), 0));}++ /// #1158: a raw-JSON corpus (ghsa-db advisories) maps none of the+ /// standard content fields — the old `["body"]` floor was a field no+ /// index maps, and an unmapped multi_match field collapses a+ /// multi-token query to zero hits. The corpus's own text-typed fields+ /// must take the floor's place, `ax_*` provenance excluded.+ #[test]+ fn a_raw_json_corpus_falls_back_to_its_own_text_fields() {+ let mut obj = serde_json::Map::new();+ let props = json!({+ "id": { "type": "keyword" },+ "summary": { "type": "text" },+ "details": { "type": "text" },+ "severity": { "type": "keyword" },+ "ax_path": { "type": "text" },+ "ax_file": { "type": "text" },+ "modified": { "type": "date" }+ });+ obj.insert(+ "xc-ghsa-000".to_string(),+ json!({ "mappings": { "properties": props } }),+ );+ let m = Value::Object(obj);+ assert_eq!(+ resolve_fields(Some(&m)),+ vec!["details".to_string(), "summary".to_string()],+ "own text fields, sorted; ax_*/keyword/date excluded"+ );+ }++ /// The floor is unchanged when the corpus maps nothing searchable at+ /// all (only keywords/provenance): `["body"]`, never an empty list.+ #[test]+ fn a_corpus_with_no_text_typed_fields_keeps_the_body_floor() {+ let mut obj = serde_json::Map::new();+ let props = json!({+ "id": { "type": "keyword" },+ "count": { "type": "long" }+ });+ obj.insert(+ "xc-kv-000".to_string(),+ json!({ "mappings": { "properties": props } }),+ );+ assert_eq!(+ resolve_fields(Some(&Value::Object(obj))),+ vec!["body".to_string()]+ );+ }++ /// A mixed corpus where SOME index maps `body` keeps today's behaviour+ /// exactly — the own-fields fallback fires only on the no-standard-+ /// field floor, never as an extra leg beside `body`.+ #[test]+ fn a_corpus_with_body_mapped_never_grows_own_fields() {+ let m = mapping(&[("xc-mixed-000", &["body"], false)]);+ let mut obj = m.as_object().unwrap().clone();+ obj.insert(+ "xc-mixed-001".to_string(),+ json!({ "mappings": { "properties": {+ "id": { "type": "keyword" },+ "summary": { "type": "text" }+ } } }),+ );+ assert_eq!(+ resolve_fields(Some(&Value::Object(obj))),+ vec!["body".to_string()],+ "body present: no summary leg, no floor rewrite"+ );+ }}diff --git a/engine/crates/xerj-common/src/xccode/mod.rs b/engine/crates/xerj-common/src/xccode/mod.rsindex e9249e80d..bfd967916 100644--- a/engine/crates/xerj-common/src/xccode/mod.rs+++ b/engine/crates/xerj-common/src/xccode/mod.rs@@ -1073,6 +1073,58 @@ mod tests {);}+ /// #1158: a raw-JSON corpus (ghsa-db advisories) maps none of+ /// body/defs/title/text. The multi_match must go to the corpus's OWN+ /// text-typed fields — the old `["body"]` floor is a field no index+ /// maps, which collapses a multi-token query to zero hits — and the+ /// hit must render a real passage from its longest string field, not+ /// an empty one.+ #[test]+ fn a_raw_json_corpus_queries_its_own_text_fields_and_renders_a_passage() {+ let root = root_with_state();+ let mut http = FakeHttp::new();+ http.mapping = serde_json::json!({+ "xc-kv-b1-000": { "mappings": { "properties": {+ "id": { "type": "keyword" },+ "summary": { "type": "text" },+ "details": { "type": "text" },+ "ax_path": { "type": "keyword" }+ }}}+ });+ http.hits = vec![serde_json::json!({+ "_score": 5.1,+ "_source": {+ "ax_path": "ghsa-db/advisories/GHSA-9j.json",+ "id": "GHSA-9j",+ "summary": "lodash command injection",+ "details": "Applications using lodash are vulnerable to command \+ injection via the template function."+ }+ })];+ let out = run_code_query(+ &root,+ &http,+ "u",+ &CodeParams::new("kv", "command injection"),+ "`--stale-ok`",+ );+ assert_eq!(out.exit, 0, "{}", out.text);+ let reqs = http.searches.lock().unwrap().clone();+ assert!(!reqs.is_empty());+ assert_eq!(+ reqs[0]+ .1+ .pointer("/query/bool/must/0/multi_match/fields"),+ Some(&serde_json::json!(["details", "summary"])),+ "the corpus's own text fields, sorted — not the silent-zero body floor",+ );+ assert!(+ out.text.contains("vulnerable to command injection"),+ "a real passage from the longest string field: {}",+ out.text+ );+ }+/// #1145: a `semantic` clause without `k` is cut to the parser default/// (10), so the vector leg must carry the same window as `size` — else/// `--mode semantic -k 20` returns 10 hits and hybrid fuses BM25@50 withdiff --git a/engine/crates/xerj-common/src/xccode/render.rs b/engine/crates/xerj-common/src/xccode/render.rsindex 472ca5156..bc10a297f 100644--- a/engine/crates/xerj-common/src/xccode/render.rs+++ b/engine/crates/xerj-common/src/xccode/render.rs@@ -34,6 +34,29 @@ pub struct RenderOpts<'a> {/// Render the full result block: the optional arms-ran note line, one block/// per hit, the footer. `note` is printed EVERY time it exists — a silently/// BM25-only "hybrid" result must never read as hybrid.+/// The longest non-empty string field of a source object, "" when there is+/// none — the content-bearing field of a raw-JSON record (#1158). Fields are+/// visited in sorted key order and only a STRICTLY longer field replaces the+/// winner, so a tie goes to the alphabetically first name and the passage+/// never depends on the wire's key order.+fn longest_string_field(src: &Value) -> String {+ let mut best: Option<(usize, &str)> = None;+ if let Some(obj) = src.as_object() {+ let mut keys: Vec<&String> = obj.keys().collect();+ keys.sort();+ for key in keys {+ if let Some(s) = obj[key].as_str() {+ if !s.trim().is_empty()+ && best.is_none_or(|(len, _)| s.chars().count() > len)+ {+ best = Some((s.chars().count(), s));+ }+ }+ }+ }+ best.map(|(_, s)| s.to_string()).unwrap_or_default()+}+pub fn render(hits: &[Value],licences: &HashMap<String, String>,@@ -119,13 +142,17 @@ fn render_hit(h: &Value, licences: &HashMap<String, String>, opts: &RenderOpts)// `body` is the content field of every family EXCEPT plain text// (`.txt` mirrors put their passage in `text`) — fall back so those- // records render a real passage, not an empty one (#1139).+ // records render a real passage, not an empty one (#1139). A raw-JSON+ // record (#1158) carries no either: its text lives in schema-named+ // fields (`summary`, `details`, …), so the last resort is the longest+ // string field — for a data record that IS the content, and provenance+ // (`ax_*` paths, dates) never wins a longest-string contest.let body = src.get("body").and_then(Value::as_str).or_else(|| src.get("text").and_then(Value::as_str))- .unwrap_or("")- .to_string();+ .map(str::to_string)+ .unwrap_or_else(|| longest_string_field(&src));if opts.full > 0 && !(restricted && opts.strict_licence) {// Prefer the matching DEFINITION over a byte window; fall back to the@@ -283,6 +310,72 @@ mod tests {assert!(text.contains("(score 9.01, AGPL)"), "{text}");}+ /// #1158: a raw-JSON record has neither `body` nor `text` — its content+ /// lives in schema-named fields. The passage must come from the longest+ /// string field (the advisory's `details`), and a tie on length goes to+ /// the alphabetically first name so the passage is stable regardless of+ /// key order on the wire.+ #[test]+ fn a_raw_json_hit_renders_its_longest_string_field_as_the_passage() {+ let raw_hit = json!({+ "_score": 7.5,+ "_source": {+ "ax_path": "ghsa-db/advisories/GHSA-9j.json",+ "id": "GHSA-9j",+ "summary": "lodash command injection",+ "details": "Applications using lodash are vulnerable to command injection via the template function."+ }+ });+ let text = render(+ &[raw_hit],+ &HashMap::new(),+ &RenderOpts {+ corpus: "ghsa-db",+ query: "command injection",+ full: 800,+ no_symbol: true,+ meatl: false,+ rrf_scores: false,+ age_days: None,+ strict_licence: false,+ },+ None,+ );+ assert!(+ text.contains("vulnerable to command injection"),+ "the passage is `details`, the longest field: {text}"+ );+ assert!(!text.contains("[0 of 0 chars"), "no empty-passage shape: {text}");++ // Tie: two fields of equal length — the alphabetically first wins.+ let tie = json!({+ "_score": 1.0,+ "_source": {+ "ax_path": "x/y.json",+ "zzz_field": "same length exactly",+ "aaa_field": "same length exactly"+ }+ });+ let text = render(+ &[tie],+ &HashMap::new(),+ &RenderOpts {+ corpus: "kv",+ query: "length",+ full: 800,+ no_symbol: true,+ meatl: false,+ rrf_scores: false,+ age_days: None,+ strict_licence: false,+ },+ None,+ );+ assert!(text.contains("aaa_field") || text.contains("same length exactly"));+ // Both same content here — the deterministic pick is still ONE field.+ assert!(text.contains("same length exactly"), "{text}");+ }+#[test]fn strict_licence_strips_the_passage_but_keeps_provenance_and_warning() {let text = render(