Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): cap a nested JSON value stored as one field (#1640)

xerj-org073ca65d3abf
diff --git a/engine/crates/xerj-autoindex/src/extract/json.rs b/engine/crates/xerj-autoindex/src/extract/json.rs
index 08751c563..7bb1f4a8f 100644
--- a/engine/crates/xerj-autoindex/src/extract/json.rs
+++ b/engine/crates/xerj-autoindex/src/extract/json.rs
@@ -572,6 +572,40 @@ mod tests {
assert_eq!((stats.records, stats.junk), (2, 1));
}
+ /// A nested array of objects is stored as one JSON string. Past
+ /// [`NESTED_JSON_FIELD_MAX`] that string is cut, so a 12 MB nested
+ /// file cannot become one analyzed text field. A value under the cap
+ /// is stored whole.
+ #[test]
+ fn a_large_nested_value_is_cut_before_it_becomes_a_field() {
+ let cap = super::super::NESTED_JSON_FIELD_MAX;
+ let big = "x".repeat(cap + 1024);
+ let text = format!(r#"{{"id":"row","payload":[{{"blob":"{big}"}}]}}"#);
+ let (stats, recs) = run(&text);
+ assert_eq!(stats.records, 1);
+ let payload = recs[0].fields["payload"].as_str().unwrap();
+ assert!(
+ payload.ends_with("…[truncated]"),
+ "over-cap nested JSON must say it was cut: {} bytes",
+ payload.len()
+ );
+ assert!(
+ payload.len() <= cap + "…[truncated]".len(),
+ "field is {} bytes, cap is {cap}",
+ payload.len()
+ );
+ assert_eq!(recs[0].fields["id"], serde_json::json!("row"));
+
+ let (stats, recs) = run(r#"{"id":"row","payload":[{"n":1}]}"#);
+ assert_eq!(stats.records, 1);
+ let payload = recs[0].fields["payload"].as_str().unwrap();
+ assert!(
+ !payload.contains("truncated"),
+ "a small nested value is stored whole: {payload}"
+ );
+ assert!(payload.contains("\"n\":1"), "{payload}");
+ }
+
#[test]
fn an_empty_array_yields_nothing_at_all_without_being_junk() {
let (stats, recs) = run("[]");
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index 60691616d..4ad292db8 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -289,6 +289,12 @@ pub fn extract(
pub const MAX_FIELDS_PER_RECORD: usize = 512;
+/// A nested object or array stored as one JSON string. Past this, the
+/// string is cut. A 12 MB nested value used to become one text field, and
+/// analysis of that field wedged ingest. The cap is the bytes kept before
+/// the truncation marker; the cut is on a char boundary.
+pub const NESTED_JSON_FIELD_MAX: usize = 64 * 1024;
+
/// Flatten a JSON object into a flat field map: up to TWO levels of nesting
/// become `a_b_c` keys; deeper structure and arrays-of-objects are stored as
/// JSON strings; arrays of scalars stay as arrays. Fields named `ax_*` are
@@ -325,20 +331,14 @@ fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>)
flatten_into(&format!("{key}_{k}"), vv, depth + 1, out);
}
} else {
- out.insert(
- key.to_string(),
- Value::String(serde_json::to_string(&m).unwrap_or_default()),
- );
+ out.insert(key.to_string(), Value::String(bounded_json(&m)));
}
}
Value::Array(a) => {
if a.iter().all(|e| !e.is_object() && !e.is_array()) {
out.insert(key.to_string(), Value::Array(a));
} else {
- out.insert(
- key.to_string(),
- Value::String(serde_json::to_string(&a).unwrap_or_default()),
- );
+ out.insert(key.to_string(), Value::String(bounded_json(&a)));
}
}
other => {
@@ -347,6 +347,28 @@ fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>)
}
}
+/// JSON for a nested value that will be stored as one string field.
+///
+/// `serde_json::to_string` of a value already in memory is cheap. What is
+/// not cheap is handing the whole string to analysis: a 12 MB nested
+/// document becomes one text field and ingest stops making progress.
+/// Values under the cap are unchanged, so a small nested object still
+/// round-trips.
+fn bounded_json(value: &impl serde::Serialize) -> String {
+ let full = serde_json::to_string(value).unwrap_or_default();
+ if full.len() <= NESTED_JSON_FIELD_MAX {
+ return full;
+ }
+ let mut end = NESTED_JSON_FIELD_MAX.min(full.len());
+ while end > 0 && !full.is_char_boundary(end) {
+ end -= 1;
+ }
+ let mut cut = String::with_capacity(end + 16);
+ cut.push_str(&full[..end]);
+ cut.push_str("…[truncated]");
+ cut
+}
+
/// A string field counts as prose at this many characters …
const PASSAGE_PROSE_MIN_CHARS: usize = 24;
/// … and this many whitespace-separated tokens. Below both, a string is an