AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): cap a nested JSON value stored as one field (#1640)
xerj-org073ca65d3abf
diff --git a/engine/crates/xerj-autoindex/src/extract/json.rs b/engine/crates/xerj-autoindex/src/extract/json.rsindex 08751c563..7bb1f4a8f 100644--- a/engine/crates/xerj-autoindex/src/extract/json.rs+++ b/engine/crates/xerj-autoindex/src/extract/json.rs@@ -572,6 +572,40 @@ mod tests {assert_eq!((stats.records, stats.junk), (2, 1));}+ /// A nested array of objects is stored as one JSON string. Past+ /// [`NESTED_JSON_FIELD_MAX`] that string is cut, so a 12 MB nested+ /// file cannot become one analyzed text field. A value under the cap+ /// is stored whole.+ #[test]+ fn a_large_nested_value_is_cut_before_it_becomes_a_field() {+ let cap = super::super::NESTED_JSON_FIELD_MAX;+ let big = "x".repeat(cap + 1024);+ let text = format!(r#"{{"id":"row","payload":[{{"blob":"{big}"}}]}}"#);+ let (stats, recs) = run(&text);+ assert_eq!(stats.records, 1);+ let payload = recs[0].fields["payload"].as_str().unwrap();+ assert!(+ payload.ends_with("…[truncated]"),+ "over-cap nested JSON must say it was cut: {} bytes",+ payload.len()+ );+ assert!(+ payload.len() <= cap + "…[truncated]".len(),+ "field is {} bytes, cap is {cap}",+ payload.len()+ );+ assert_eq!(recs[0].fields["id"], serde_json::json!("row"));++ let (stats, recs) = run(r#"{"id":"row","payload":[{"n":1}]}"#);+ assert_eq!(stats.records, 1);+ let payload = recs[0].fields["payload"].as_str().unwrap();+ assert!(+ !payload.contains("truncated"),+ "a small nested value is stored whole: {payload}"+ );+ assert!(payload.contains("\"n\":1"), "{payload}");+ }+#[test]fn an_empty_array_yields_nothing_at_all_without_being_junk() {let (stats, recs) = run("[]");diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex 60691616d..4ad292db8 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -289,6 +289,12 @@ pub fn extract(pub const MAX_FIELDS_PER_RECORD: usize = 512;+/// A nested object or array stored as one JSON string. Past this, the+/// string is cut. A 12 MB nested value used to become one text field, and+/// analysis of that field wedged ingest. The cap is the bytes kept before+/// the truncation marker; the cut is on a char boundary.+pub const NESTED_JSON_FIELD_MAX: usize = 64 * 1024;+/// Flatten a JSON object into a flat field map: up to TWO levels of nesting/// become `a_b_c` keys; deeper structure and arrays-of-objects are stored as/// JSON strings; arrays of scalars stay as arrays. Fields named `ax_*` are@@ -325,20 +331,14 @@ fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>)flatten_into(&format!("{key}_{k}"), vv, depth + 1, out);}} else {- out.insert(- key.to_string(),- Value::String(serde_json::to_string(&m).unwrap_or_default()),- );+ out.insert(key.to_string(), Value::String(bounded_json(&m)));}}Value::Array(a) => {if a.iter().all(|e| !e.is_object() && !e.is_array()) {out.insert(key.to_string(), Value::Array(a));} else {- out.insert(- key.to_string(),- Value::String(serde_json::to_string(&a).unwrap_or_default()),- );+ out.insert(key.to_string(), Value::String(bounded_json(&a)));}}other => {@@ -347,6 +347,28 @@ fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>)}}+/// JSON for a nested value that will be stored as one string field.+///+/// `serde_json::to_string` of a value already in memory is cheap. What is+/// not cheap is handing the whole string to analysis: a 12 MB nested+/// document becomes one text field and ingest stops making progress.+/// Values under the cap are unchanged, so a small nested object still+/// round-trips.+fn bounded_json(value: &impl serde::Serialize) -> String {+ let full = serde_json::to_string(value).unwrap_or_default();+ if full.len() <= NESTED_JSON_FIELD_MAX {+ return full;+ }+ let mut end = NESTED_JSON_FIELD_MAX.min(full.len());+ while end > 0 && !full.is_char_boundary(end) {+ end -= 1;+ }+ let mut cut = String::with_capacity(end + 16);+ cut.push_str(&full[..end]);+ cut.push_str("…[truncated]");+ cut+}+/// A string field counts as prose at this many characters …const PASSAGE_PROSE_MIN_CHARS: usize = 24;/// … and this many whitespace-separated tokens. Below both, a string is an