Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(xccode): raw-JSON corpora query their own text fields — the #1158 silent zero

xerj-org14ca8532fb29
diff --git a/engine/crates/xerj-common/src/xccode/fields.rs b/engine/crates/xerj-common/src/xccode/fields.rs
index d05b933c3..b236757da 100644
--- a/engine/crates/xerj-common/src/xccode/fields.rs
+++ b/engine/crates/xerj-common/src/xccode/fields.rs
@@ -75,12 +75,52 @@ pub fn resolve_fields(mapping: Option<&Value>) -> Vec<String> {
out.push("text".to_string());
}
if out.is_empty() {
- vec!["body".to_string()]
+ // #1158: a raw-JSON corpus (ghsa-db's advisory mirrors, OSV) maps
+ // NONE of the standard content fields — every record carries its
+ // text in schema-named fields like `summary`/`details`. The old
+ // `["body"]` floor here is a field no index maps, and an unmapped
+ // multi_match field collapses a multi-token query to ZERO hits
+ // with no error: the entire corpus answered "no passage matches"
+ // while the same indices returned 10,000+ hits queried directly.
+ // Fall back to the corpus's OWN text-typed fields (never `ax_*`
+ // provenance) — still mapping-gated, so every field sent is one
+ // at least one index really maps.
+ let own = own_text_fields(obj);
+ if own.is_empty() {
+ vec!["body".to_string()]
+ } else {
+ own
+ }
} else {
out
}
}
+/// The corpus's own searchable content fields, for the no-standard-field
+/// floor above: every property mapped `text` (or `semantic_text`) by any
+/// index under the prefix, `ax_*` provenance excluded, names sorted, capped
+/// so a wide schema cannot balloon the multi_match.
+fn own_text_fields(obj: &serde_json::Map<String, Value>) -> Vec<String> {
+ const AX: &str = "ax_";
+ const MAX_OWN_FIELDS: usize = 24;
+ let mut own = std::collections::BTreeSet::new();
+ for m in obj.values() {
+ let Some(props) = m.pointer("/mappings/properties").and_then(Value::as_object) else {
+ continue;
+ };
+ for (key, spec) in props {
+ let searchable = spec
+ .get("type")
+ .and_then(Value::as_str)
+ .is_some_and(|t| t == "text" || t == "semantic_text");
+ if searchable && !key.starts_with(AX) {
+ own.insert(key.clone());
+ }
+ }
+ }
+ own.into_iter().take(MAX_OWN_FIELDS).collect()
+}
+
/// Indices under the prefix whose `field` (default `body`) is mapped as
/// `semantic_text`.
///
@@ -144,8 +184,12 @@ mod tests {
vec!["body".to_string(), "defs".into(), "title".into()]
);
- // No FIELDS member mapped at all -> the ["body"] floor, never empty.
- let thin = mapping(&[("i", &["unrelated"], false)]);
+ // No FIELDS member and NOTHING text-typed at all -> the ["body"]
+ // floor, never empty, never a keyword field. (A text-typed field
+ // no standard name covers is the #1158 own-fields case below.)
+ let thin = json!({ "i": { "mappings": { "properties": {
+ "unrelated": { "type": "keyword" }
+ } } } });
assert_eq!(resolve_fields(Some(&thin)), vec!["body".to_string()]);
}
@@ -171,8 +215,12 @@ mod tests {
resolve_fields(Some(&with_text)),
vec!["body".to_string(), "text".to_string()]
);
- // No FIELDS member AND no text -> the ["body"] floor still holds.
- let thin = mapping(&[("i", &["unrelated"], false)]);
+ // No FIELDS member AND no text-typed field -> the ["body"] floor
+ // still holds (keyword placeholders — text-typed ones are #1158's
+ // own-fields fallback, covered by its own test).
+ let thin = json!({ "i": { "mappings": { "properties": {
+ "unrelated": { "type": "keyword" }
+ } } } });
assert_eq!(resolve_fields(Some(&thin)), vec!["body".to_string()]);
}
@@ -188,4 +236,73 @@ mod tests {
assert_eq!(total, 3);
assert!(semantic_capable(None) == (Vec::new(), 0));
}
+
+ /// #1158: a raw-JSON corpus (ghsa-db advisories) maps none of the
+ /// standard content fields — the old `["body"]` floor was a field no
+ /// index maps, and an unmapped multi_match field collapses a
+ /// multi-token query to zero hits. The corpus's own text-typed fields
+ /// must take the floor's place, `ax_*` provenance excluded.
+ #[test]
+ fn a_raw_json_corpus_falls_back_to_its_own_text_fields() {
+ let mut obj = serde_json::Map::new();
+ let props = json!({
+ "id": { "type": "keyword" },
+ "summary": { "type": "text" },
+ "details": { "type": "text" },
+ "severity": { "type": "keyword" },
+ "ax_path": { "type": "text" },
+ "ax_file": { "type": "text" },
+ "modified": { "type": "date" }
+ });
+ obj.insert(
+ "xc-ghsa-000".to_string(),
+ json!({ "mappings": { "properties": props } }),
+ );
+ let m = Value::Object(obj);
+ assert_eq!(
+ resolve_fields(Some(&m)),
+ vec!["details".to_string(), "summary".to_string()],
+ "own text fields, sorted; ax_*/keyword/date excluded"
+ );
+ }
+
+ /// The floor is unchanged when the corpus maps nothing searchable at
+ /// all (only keywords/provenance): `["body"]`, never an empty list.
+ #[test]
+ fn a_corpus_with_no_text_typed_fields_keeps_the_body_floor() {
+ let mut obj = serde_json::Map::new();
+ let props = json!({
+ "id": { "type": "keyword" },
+ "count": { "type": "long" }
+ });
+ obj.insert(
+ "xc-kv-000".to_string(),
+ json!({ "mappings": { "properties": props } }),
+ );
+ assert_eq!(
+ resolve_fields(Some(&Value::Object(obj))),
+ vec!["body".to_string()]
+ );
+ }
+
+ /// A mixed corpus where SOME index maps `body` keeps today's behaviour
+ /// exactly — the own-fields fallback fires only on the no-standard-
+ /// field floor, never as an extra leg beside `body`.
+ #[test]
+ fn a_corpus_with_body_mapped_never_grows_own_fields() {
+ let m = mapping(&[("xc-mixed-000", &["body"], false)]);
+ let mut obj = m.as_object().unwrap().clone();
+ obj.insert(
+ "xc-mixed-001".to_string(),
+ json!({ "mappings": { "properties": {
+ "id": { "type": "keyword" },
+ "summary": { "type": "text" }
+ } } }),
+ );
+ assert_eq!(
+ resolve_fields(Some(&Value::Object(obj))),
+ vec!["body".to_string()],
+ "body present: no summary leg, no floor rewrite"
+ );
+ }
}
diff --git a/engine/crates/xerj-common/src/xccode/mod.rs b/engine/crates/xerj-common/src/xccode/mod.rs
index e9249e80d..bfd967916 100644
--- a/engine/crates/xerj-common/src/xccode/mod.rs
+++ b/engine/crates/xerj-common/src/xccode/mod.rs
@@ -1073,6 +1073,58 @@ mod tests {
);
}
+ /// #1158: a raw-JSON corpus (ghsa-db advisories) maps none of
+ /// body/defs/title/text. The multi_match must go to the corpus's OWN
+ /// text-typed fields — the old `["body"]` floor is a field no index
+ /// maps, which collapses a multi-token query to zero hits — and the
+ /// hit must render a real passage from its longest string field, not
+ /// an empty one.
+ #[test]
+ fn a_raw_json_corpus_queries_its_own_text_fields_and_renders_a_passage() {
+ let root = root_with_state();
+ let mut http = FakeHttp::new();
+ http.mapping = serde_json::json!({
+ "xc-kv-b1-000": { "mappings": { "properties": {
+ "id": { "type": "keyword" },
+ "summary": { "type": "text" },
+ "details": { "type": "text" },
+ "ax_path": { "type": "keyword" }
+ }}}
+ });
+ http.hits = vec![serde_json::json!({
+ "_score": 5.1,
+ "_source": {
+ "ax_path": "ghsa-db/advisories/GHSA-9j.json",
+ "id": "GHSA-9j",
+ "summary": "lodash command injection",
+ "details": "Applications using lodash are vulnerable to command \
+ injection via the template function."
+ }
+ })];
+ let out = run_code_query(
+ &root,
+ &http,
+ "u",
+ &CodeParams::new("kv", "command injection"),
+ "`--stale-ok`",
+ );
+ assert_eq!(out.exit, 0, "{}", out.text);
+ let reqs = http.searches.lock().unwrap().clone();
+ assert!(!reqs.is_empty());
+ assert_eq!(
+ reqs[0]
+ .1
+ .pointer("/query/bool/must/0/multi_match/fields"),
+ Some(&serde_json::json!(["details", "summary"])),
+ "the corpus's own text fields, sorted — not the silent-zero body floor",
+ );
+ assert!(
+ out.text.contains("vulnerable to command injection"),
+ "a real passage from the longest string field: {}",
+ out.text
+ );
+ }
+
/// #1145: a `semantic` clause without `k` is cut to the parser default
/// (10), so the vector leg must carry the same window as `size` — else
/// `--mode semantic -k 20` returns 10 hits and hybrid fuses BM25@50 with
diff --git a/engine/crates/xerj-common/src/xccode/render.rs b/engine/crates/xerj-common/src/xccode/render.rs
index 472ca5156..bc10a297f 100644
--- a/engine/crates/xerj-common/src/xccode/render.rs
+++ b/engine/crates/xerj-common/src/xccode/render.rs
@@ -34,6 +34,29 @@ pub struct RenderOpts<'a> {
/// Render the full result block: the optional arms-ran note line, one block
/// per hit, the footer. `note` is printed EVERY time it exists — a silently
/// BM25-only "hybrid" result must never read as hybrid.
+/// The longest non-empty string field of a source object, "" when there is
+/// none — the content-bearing field of a raw-JSON record (#1158). Fields are
+/// visited in sorted key order and only a STRICTLY longer field replaces the
+/// winner, so a tie goes to the alphabetically first name and the passage
+/// never depends on the wire's key order.
+fn longest_string_field(src: &Value) -> String {
+ let mut best: Option<(usize, &str)> = None;
+ if let Some(obj) = src.as_object() {
+ let mut keys: Vec<&String> = obj.keys().collect();
+ keys.sort();
+ for key in keys {
+ if let Some(s) = obj[key].as_str() {
+ if !s.trim().is_empty()
+ && best.is_none_or(|(len, _)| s.chars().count() > len)
+ {
+ best = Some((s.chars().count(), s));
+ }
+ }
+ }
+ }
+ best.map(|(_, s)| s.to_string()).unwrap_or_default()
+}
+
pub fn render(
hits: &[Value],
licences: &HashMap<String, String>,
@@ -119,13 +142,17 @@ fn render_hit(h: &Value, licences: &HashMap<String, String>, opts: &RenderOpts)
// `body` is the content field of every family EXCEPT plain text
// (`.txt` mirrors put their passage in `text`) — fall back so those
- // records render a real passage, not an empty one (#1139).
+ // records render a real passage, not an empty one (#1139). A raw-JSON
+ // record (#1158) carries no either: its text lives in schema-named
+ // fields (`summary`, `details`, …), so the last resort is the longest
+ // string field — for a data record that IS the content, and provenance
+ // (`ax_*` paths, dates) never wins a longest-string contest.
let body = src
.get("body")
.and_then(Value::as_str)
.or_else(|| src.get("text").and_then(Value::as_str))
- .unwrap_or("")
- .to_string();
+ .map(str::to_string)
+ .unwrap_or_else(|| longest_string_field(&src));
if opts.full > 0 && !(restricted && opts.strict_licence) {
// Prefer the matching DEFINITION over a byte window; fall back to the
@@ -283,6 +310,72 @@ mod tests {
assert!(text.contains("(score 9.01, AGPL)"), "{text}");
}
+ /// #1158: a raw-JSON record has neither `body` nor `text` — its content
+ /// lives in schema-named fields. The passage must come from the longest
+ /// string field (the advisory's `details`), and a tie on length goes to
+ /// the alphabetically first name so the passage is stable regardless of
+ /// key order on the wire.
+ #[test]
+ fn a_raw_json_hit_renders_its_longest_string_field_as_the_passage() {
+ let raw_hit = json!({
+ "_score": 7.5,
+ "_source": {
+ "ax_path": "ghsa-db/advisories/GHSA-9j.json",
+ "id": "GHSA-9j",
+ "summary": "lodash command injection",
+ "details": "Applications using lodash are vulnerable to command injection via the template function."
+ }
+ });
+ let text = render(
+ &[raw_hit],
+ &HashMap::new(),
+ &RenderOpts {
+ corpus: "ghsa-db",
+ query: "command injection",
+ full: 800,
+ no_symbol: true,
+ meatl: false,
+ rrf_scores: false,
+ age_days: None,
+ strict_licence: false,
+ },
+ None,
+ );
+ assert!(
+ text.contains("vulnerable to command injection"),
+ "the passage is `details`, the longest field: {text}"
+ );
+ assert!(!text.contains("[0 of 0 chars"), "no empty-passage shape: {text}");
+
+ // Tie: two fields of equal length — the alphabetically first wins.
+ let tie = json!({
+ "_score": 1.0,
+ "_source": {
+ "ax_path": "x/y.json",
+ "zzz_field": "same length exactly",
+ "aaa_field": "same length exactly"
+ }
+ });
+ let text = render(
+ &[tie],
+ &HashMap::new(),
+ &RenderOpts {
+ corpus: "kv",
+ query: "length",
+ full: 800,
+ no_symbol: true,
+ meatl: false,
+ rrf_scores: false,
+ age_days: None,
+ strict_licence: false,
+ },
+ None,
+ );
+ assert!(text.contains("aaa_field") || text.contains("same length exactly"));
+ // Both same content here — the deterministic pick is still ONE field.
+ assert!(text.contains("same length exactly"), "{text}");
+ }
+
#[test]
fn strict_licence_strips_the_passage_but_keeps_provenance_and_warning() {
let text = render(