Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(xerj-autoindex): JSON records with no prose gain a text passage (#1158 build-time half)

xerj-orgf8ae2ed81591
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4b621c6bf..b88fade2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+
+- **A JSON record with no prose now carries a synthesized `text` passage, and a dataset that maps no text-searchable field says so at plan time** (the build-time half of issue [#1158](https://github.com/xerj-org/xerj/issues/1158); the query-time half — falling back to an index's own text-typed fields — shipped in rc.86). Raw-JSON corpora indexed as typed records (ids, enums, numbers, dates) were invisible to `xerj code` passage search: 36,263 GHSA advisories indexed to 193k records that answered `No passage matches` while the same index returned 10,000+ hits over its typed fields. Now a record whose flattened fields carry no prose-like string (≥ 24 chars and ≥ 4 tokens — below that a string is an id, an enum or a date, and maps `keyword`) gains a `text` passage rendered from its own fields (`key: value` lines, values capped at 256 chars, the passage at 4,096, scalar arrays joined so identifiers stay matchable); a record that already carries prose is left alone — the query side already searches it, and a rendered key/value soup would only outrank the real prose — and a record that owns a field named `text` keeps it untouched. Separately, any dataset whose sampled mapping contains no text-searchable field (a csv of enums, a keyword-only XML set) now warns at plan time: `dataset 'x' maps no text-searchable field — 'xerj code' passage search will find nothing in it (sampled fields: …)`, through the progress surface so `--quiet` stays silent. Scoped to the JSON family (whole-file and JSONL) — the measured class is raw-JSON mirrors; the same helper can ride the csv/yaml/xml data sites when a real corpus needs it. Existing state dirs re-provision their JSON datasets once to gain the field; every already-built corpus keeps working, and the 100 hub corpora are unaffected (their graded snapshots map standard fields).
+
## [1.0.0-rc.86] - 2026-10-06
### Security
diff --git a/engine/crates/xerj-autoindex/src/extract/json.rs b/engine/crates/xerj-autoindex/src/extract/json.rs
index 52fb169d1..08751c563 100644
--- a/engine/crates/xerj-autoindex/src/extract/json.rs
+++ b/engine/crates/xerj-autoindex/src/extract/json.rs
@@ -7,8 +7,8 @@
//! - anything else → a single record
use super::{
- emit_document_with_fields, flatten_object, ExtractStats, FieldOrigin, RawRecord, Sink,
- MAX_LINE, MAX_WHOLE_FILE,
+ emit_document_with_fields, ensure_text_passage, flatten_object, ExtractStats, FieldOrigin,
+ RawRecord, Sink, MAX_LINE, MAX_WHOLE_FILE,
};
use anyhow::Result;
use serde_json::{Map, Value};
@@ -85,6 +85,7 @@ pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
for (k, v) in flatten_object(m) {
fields.insert(k, v); // element wins collisions
}
+ ensure_text_passage(&mut fields);
stats.records += 1;
if !sink(RawRecord {
fields,
@@ -279,9 +280,11 @@ fn jsonl_fallback(bytes: &[u8], sink: Sink) -> Result<ExtractStats> {
}
match serde_json::from_slice::<Value>(trimmed) {
Ok(Value::Object(m)) => {
+ let mut fields = flatten_object(m);
+ ensure_text_passage(&mut fields);
stats.records += 1;
if !sink(RawRecord {
- fields: flatten_object(m),
+ fields,
locator: format!("b{start}"),
group: None,
origin: FieldOrigin::Data,
@@ -298,7 +301,7 @@ fn jsonl_fallback(bytes: &[u8], sink: Sink) -> Result<ExtractStats> {
}
fn emit(v: Value, locator: &str, sink: Sink, stats: &mut ExtractStats) -> bool {
- let fields = match v {
+ let mut fields = match v {
Value::Object(m) => flatten_object(m),
other => {
let mut m = Map::new();
@@ -306,6 +309,7 @@ fn emit(v: Value, locator: &str, sink: Sink, stats: &mut ExtractStats) -> bool {
m
}
};
+ ensure_text_passage(&mut fields);
stats.records += 1;
sink(RawRecord {
fields,
@@ -461,7 +465,13 @@ mod tests {
serde_json::json!(["a", "b"]),
"an array of scalars stays an array"
);
- assert_eq!(recs[1].fields.len(), 1, "absent keys are not filled in");
+ assert_eq!(
+ recs[1].fields.len(),
+ 2,
+ "absent keys are not filled in — the record's own fields plus its \
+ synthesized passage (`id: 2`), nothing else"
+ );
+ assert_eq!(recs[1].fields["text"], serde_json::json!("id: 2"));
}
#[test]
@@ -568,4 +578,101 @@ mod tests {
assert_eq!((stats.records, stats.junk), (0, 0));
assert!(recs.is_empty());
}
+
+ // ─── #1158: synthesized text passages ────────────────────────────────
+
+ /// The ghsa-db class, in miniature: a record of ids, enums, numbers and
+ /// dates carries nothing passage search can match. Its own fields become
+ /// the passage, so a query naming any of them reaches the record.
+ #[test]
+ fn a_record_with_no_prose_gains_a_synthesized_text_passage() {
+ let (_, recs) = run(r#"[{"sha256":"a3f19c","size":123456,"severity":"HIGH",
+ "modified":"2026-01-01","aliases":["CVE-2026-0001","GHSA-xx"]}]"#);
+ let text = recs[0].fields["text"].as_str().unwrap();
+ for expect in [
+ "sha256: a3f19c",
+ "size: 123456",
+ "severity: HIGH",
+ "modified: 2026-01-01",
+ "aliases: CVE-2026-0001, GHSA-xx",
+ ] {
+ assert!(text.contains(expect), "passage missing {expect:?}:\n{text}");
+ }
+ }
+
+ /// A record with a prose field is already reachable through it (the
+ /// query side searches text-typed fields), and a rendered key/value soup
+ /// would only outrank the real prose. No synthesis.
+ #[test]
+ fn a_record_with_prose_gains_no_synthesized_passage() {
+ let (_, recs) = run(
+ r#"[{"id":"GHSA-1","summary":"Use after free in the connection pool cleanup path"}]"#,
+ );
+ assert!(
+ recs[0].fields.get("text").is_none(),
+ "a prose record must not gain a synthesized passage: {:?}",
+ recs[0].fields
+ );
+ }
+
+ /// A record that owns a field named `text` keeps it, whatever its length:
+ /// the record named that field, and its dataset already maps it.
+ #[test]
+ fn an_existing_text_field_is_never_clobbered() {
+ let (_, recs) = run(r#"[{"text":"tiny","n":1}]"#);
+ assert_eq!(recs[0].fields["text"], serde_json::json!("tiny"));
+ }
+
+ /// The passage is bounded even for a record with hundreds of fields. The
+ /// final cut goes through `chars()`, so it lands on a character boundary
+ /// and never slices a multi-byte character in half.
+ #[test]
+ fn the_synthesized_passage_is_hard_capped() {
+ let fields: Vec<String> = (0..900)
+ .map(|i| format!(r#""f{i}":"{}""#, "v".repeat(40)))
+ .collect();
+ let (_, recs) = run(&format!("[{{{}}}]", fields.join(",")));
+ let text = recs[0].fields["text"].as_str().unwrap();
+ assert!(
+ text.chars().count() <= super::super::SYNTH_PASSAGE_MAX_CHARS,
+ "passage is {} chars, cap is {}",
+ text.chars().count(),
+ super::super::SYNTH_PASSAGE_MAX_CHARS
+ );
+
+ // A record whose values are multi-byte: 20 fields x the 256-char
+ // per-value cap pushes the pastiche past the total cap, so the FINAL
+ // truncation runs — on 設, a 3-byte character. `chars().take()` cannot
+ // produce an invalid string, which is exactly what is being pinned.
+ let mut m = serde_json::Map::new();
+ for i in 0..20 {
+ m.insert(
+ format!("k{i}"),
+ serde_json::Value::String("設".repeat(9_000)),
+ );
+ }
+ assert!(super::ensure_text_passage(&mut m));
+ let passage = m["text"].as_str().unwrap();
+ assert_eq!(
+ passage.chars().count(),
+ super::super::SYNTH_PASSAGE_MAX_CHARS,
+ "a passage over the cap is cut to exactly the cap"
+ );
+ }
+
+ /// A mixed dataset is handled per record: prose rows stay as they were,
+ /// prose-less rows gain the passage. The prose field still maps text for
+ /// the whole dataset either way.
+ #[test]
+ fn synthesis_is_per_record_inside_one_dataset() {
+ let (_, recs) = run(
+ r#"[{"id":1,"note":"a genuinely long prose sentence about pooling"},
+ {"id":2,"count":7}]"#,
+ );
+ assert!(recs[0].fields.get("text").is_none());
+ assert!(recs[1].fields["text"]
+ .as_str()
+ .unwrap()
+ .contains("count: 7"));
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/extract/jsonl.rs b/engine/crates/xerj-autoindex/src/extract/jsonl.rs
index 71d10e118..3b76dc407 100644
--- a/engine/crates/xerj-autoindex/src/extract/jsonl.rs
+++ b/engine/crates/xerj-autoindex/src/extract/jsonl.rs
@@ -1,6 +1,8 @@
//! JSONL (newline-delimited JSON) — streaming, byte-offset locators.
-use super::{flatten_object, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_LINE};
+use super::{
+ ensure_text_passage, flatten_object, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_LINE,
+};
use anyhow::Result;
use serde_json::Value;
use std::path::Path;
@@ -33,9 +35,11 @@ pub fn extract(
}
match serde_json::from_slice::<Value>(trimmed) {
Ok(Value::Object(m)) => {
+ let mut fields = flatten_object(m);
+ ensure_text_passage(&mut fields);
stats.records += 1;
if !sink(RawRecord {
- fields: flatten_object(m),
+ fields,
locator: format!("b{start}"),
group: None,
origin: FieldOrigin::Data,
@@ -197,4 +201,19 @@ mod tests {
assert_eq!(stats.records, 1, "the stream continues after the cap");
assert_eq!(recs[0].fields["id"], serde_json::json!(2));
}
+
+ /// #1158: a JSONL record with no prose gains a `text` passage from its
+ /// own fields, same as the whole-file family — a `.jsonl` mirror is the
+ /// other raw-JSON corpus shape.
+ #[test]
+ fn a_prose_less_jsonl_record_gains_a_synthesized_text_passage() {
+ let (_, recs) = run(
+ b"{\"id\":\"OSV-1\",\"ecosystem\":\"crates.io\",\"count\":3}\n",
+ None,
+ );
+ let text = recs[0].fields["text"].as_str().unwrap();
+ assert!(text.contains("id: OSV-1"));
+ assert!(text.contains("ecosystem: crates.io"));
+ assert!(text.contains("count: 3"));
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index a181b786c..774878ec9 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -329,6 +329,126 @@ fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>)
}
}
+/// A string field counts as prose at this many characters …
+const PASSAGE_PROSE_MIN_CHARS: usize = 24;
+/// … and this many whitespace-separated tokens. Below both, a string is an
+/// id, an enum or a date — the values that map `keyword` and leave a record
+/// invisible to passage search.
+const PASSAGE_PROSE_MIN_TOKENS: usize = 4;
+/// Cap on a synthesized passage, in characters. The passage exists so a
+/// record's own field names and values are matchable; past this size the
+/// record is prose-like and synthesis never ran anyway.
+pub const SYNTH_PASSAGE_MAX_CHARS: usize = 4 << 10;
+/// Cap per rendered value inside a synthesized passage, in characters.
+const SYNTH_PASSAGE_VALUE_CHARS: usize = 256;
+
+/// Does this record already carry a string field that reads as prose? Such a
+/// field maps `text` and the query side already searches it (`resolve_fields`
+/// includes `text` and, since rc.86, falls back to the index's own text-typed
+/// fields — `xerj-common/src/xccode/fields.rs`), so synthesizing anything
+/// would only shadow it.
+fn carries_prose(fields: &Map<String, Value>) -> bool {
+ fields.values().any(|v| {
+ v.as_str().is_some_and(|s| {
+ let t = s.trim();
+ t.chars().count() >= PASSAGE_PROSE_MIN_CHARS
+ && t.split_whitespace().count() >= PASSAGE_PROSE_MIN_TOKENS
+ })
+ })
+}
+
+/// #1158's invisible-record class, fixed at the source: a JSON record whose
+/// flattened fields carry no prose-like string is a row of ids, enums and
+/// numbers — nothing `xerj code` passage search can match, because it reads
+/// `body`/`defs`/`title`/`text` and none of them exist. Measured on the live
+/// case: `ghsa-db`, 36,263 raw advisory JSONs indexed to 193k typed records
+/// that answered `No passage matches` while the same index returned 10,000+
+/// hits over its typed fields. This synthesizes a `text` passage from the
+/// record's own fields (`key: value` lines, bounded) so the corpus is
+/// passage-searchable by default instead of only after an undocumented
+/// pre-flatten workaround.
+///
+/// Bounds and rules:
+/// - Never clobbers an existing `text` field (even a short one — the record
+/// owns that name, and its dataset already maps it).
+/// - Only runs when the record carries NO prose field: a record with real
+/// prose is already reachable through the query-side field resolution, and
+/// a rendered key/value soup would outrank that prose in the field order.
+/// - Values are capped ([`SYNTH_PASSAGE_VALUE_CHARS`]) and the passage with
+/// them ([`SYNTH_PASSAGE_MAX_CHARS`]); scalar arrays (ids, aliases) render
+/// joined so an identifier query can reach the record.
+/// - Scoped to the JSON family (whole-file and JSONL) — the measured class is
+/// raw-JSON mirrors. The same helper can ride the csv/yaml/xml Data sites
+/// if a real corpus ever needs it; until then those families are covered
+/// by the dataset-level zero-text warning instead.
+///
+/// Returns whether a passage was inserted, so the call sites' tests can pin
+/// the predicate without reading the record back out.
+pub(crate) fn ensure_text_passage(fields: &mut Map<String, Value>) -> bool {
+ if carries_prose(fields) || fields.contains_key("text") {
+ return false;
+ }
+ let mut passage = String::new();
+ for (k, v) in fields.iter() {
+ let render_value = |txt: &mut String, s: &str| {
+ // Char cap, checked in chars: a byte check would stop a multibyte
+ // passage at a third of its budget (the values that trip it are
+ // exactly the ones already capped to 256 chars, so this recount
+ // is bounded and cheap).
+ if txt.chars().count() >= SYNTH_PASSAGE_MAX_CHARS {
+ return;
+ }
+ if !txt.is_empty() {
+ txt.push('\n');
+ }
+ txt.push_str(k);
+ txt.push_str(": ");
+ let cut = s
+ .trim()
+ .chars()
+ .take(SYNTH_PASSAGE_VALUE_CHARS)
+ .collect::<String>();
+ txt.push_str(&cut);
+ };
+ match v {
+ Value::String(s) => render_value(&mut passage, s),
+ Value::Number(n) => render_value(&mut passage, &n.to_string()),
+ Value::Bool(b) => render_value(&mut passage, &b.to_string()),
+ // Scalar arrays (aliases, tags, versions) join with ", " — their
+ // identifiers are exactly what a passage exists to make matchable.
+ Value::Array(a) if a.iter().all(|e| !e.is_object() && !e.is_array()) => {
+ let joined = a
+ .iter()
+ .map(|e| match e {
+ Value::String(s) => s.trim().to_string(),
+ other => other.to_string(),
+ })
+ .collect::<Vec<_>>()
+ .join(", ");
+ render_value(&mut passage, &joined);
+ }
+ _ => {}
+ }
+ if passage.chars().count() >= SYNTH_PASSAGE_MAX_CHARS {
+ break;
+ }
+ }
+ if passage.is_empty() {
+ return false;
+ }
+ // The per-line cap check runs before each append, so a final line can
+ // push past the bound by one line's worth. Enforce it exactly, on a
+ // char boundary (a byte slice could land mid-character: panic = abort).
+ if passage.chars().count() > SYNTH_PASSAGE_MAX_CHARS {
+ passage = passage
+ .chars()
+ .take(SYNTH_PASSAGE_MAX_CHARS)
+ .collect::<String>();
+ }
+ fields.insert("text".to_string(), Value::String(passage));
+ true
+}
+
/// Convert a parsed YAML value into JSON (shared by the YAML and Unity
/// extractors). Tagged values unwrap to their inner value; non-string mapping
/// keys are stringified.
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 1e4cd6f22..df31ed4b6 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3546,6 +3546,9 @@ fn build_phase_a(
.iter()
.find(|s| s.es_type == "semantic_text")
.map(|s| s.name.clone());
+ if let Some(note) = textless_dataset_note(&specs, &c.slug) {
+ pr.note(&note);
+ }
// #1059: the dataset's text-analyzer election, from the same sampled
// accumulators that elected the semantic body above. Deterministic in
// the sample, so the same bytes elect the same analyzer on every run.
@@ -3593,6 +3596,35 @@ fn build_phase_a(
}
}
+/// #1158's zero-text warning, the build-time backstop: a dataset whose
+/// mapping carries no text-searchable field is invisible to `xerj code`
+/// passage search (it reads `body`/`defs`/`title`/`text` and then the index's
+/// own text-typed fields — `xerj-common/src/xccode/fields.rs`). The JSON
+/// family synthesizes a `text` passage for prose-less records, so it stops
+/// hitting this; every other shape (a csv of enums, a keyword-only xml set)
+/// still does, and the operator hears it at plan time instead of wondering
+/// why every query comes back empty. `None` means the dataset is searchable.
+fn textless_dataset_note(specs: &[infer::FieldSpec], slug: &str) -> Option<String> {
+ let searchable = specs
+ .iter()
+ .any(|s| matches!(s.es_type.as_str(), "text" | "semantic_text"));
+ if searchable {
+ return None;
+ }
+ let field_list = specs
+ .iter()
+ .take(6)
+ .map(|s| format!("{}:{}", s.name, s.es_type))
+ .collect::<Vec<_>>()
+ .join(" ");
+ Some(format!(
+ "dataset '{slug}' maps no text-searchable field — `xerj code` passage search will \
+ find nothing in it (sampled fields: {field_list}{}); structured queries over \
+ typed fields still work",
+ if specs.len() > 6 { " …" } else { "" }
+ ))
+}
+
// ─── mapping builder ─────────────────────────────────────────────────────
pub const PROVENANCE_FIELDS: &[&str] = &[
@@ -11055,6 +11087,36 @@ mod unity_pipeline_tests {
(plan, files)
}
+ /// #1158's zero-text warning: a dataset whose sampled fields are all
+ /// keyword/numeric warns, one with any text-searchable spec does not.
+ #[test]
+ fn a_dataset_with_no_text_searchable_field_warns_at_plan_time() {
+ let kw = |name: &str| {
+ let mut s = pipeline_keyword_spec(name);
+ s.es_type = "keyword".into();
+ s
+ };
+ let warned = textless_dataset_note(&[kw("sha256"), kw("size")], "gitsnap");
+ let Some(note) = warned else {
+ panic!("a keyword-only dataset must warn");
+ };
+ assert!(note.contains("gitsnap"), "{note}");
+ assert!(note.contains("sha256:keyword"), "{note}");
+ assert!(note.contains("passage search"), "{note}");
+
+ let mut text = kw("summary");
+ text.es_type = "text".into();
+ assert!(
+ textless_dataset_note(&[kw("id"), text.clone()], "advisories").is_none(),
+ "a dataset with a text field is searchable"
+ );
+ text.es_type = "semantic_text".into();
+ assert!(
+ textless_dataset_note(&[kw("id"), text], "advisories").is_none(),
+ "semantic_text counts as searchable too"
+ );
+ }
+
#[test]
fn the_guid_map_resolves_a_meta_sidecar_to_its_asset_path() {
let dir = tempfile::tempdir().unwrap();