AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): recurse dynamic mapping into nested object properties
Vincenzo Lombardoeb7ee04c8120
diff --git a/engine/crates/xerj-api/src/memory_api.rs b/engine/crates/xerj-api/src/memory_api.rsindex 2c757b14c..09eebe3d8 100644--- a/engine/crates/xerj-api/src/memory_api.rs+++ b/engine/crates/xerj-api/src/memory_api.rs@@ -183,10 +183,22 @@ async fn ensure_backing_index(// similarity. 384 mirrors the built-in embedder width (xerj_ai DEFAULT_DIMS);// kept as a literal to avoid a build-graph edge from xerj-api onto xerj-ai// (same convention as es_compat's semantic_text mapper).+ //+ // `metadata` is deliberately NOT pre-declared here (unlike `text`/+ // `stored_at`, which are always present and same-shaped). Its own shape+ // is caller-defined and varies per memory, so pinning it to a bare+ // `{"type": "object"}` up front would permanently prevent its keys from+ // ever being dynamically mapped -- once a field has ANY explicit+ // mapping entry, later documents no longer go through dynamic+ // discovery for it, even a shallow one with no `properties`. Leaving it+ // out lets the first stored memory's `metadata` keys flow through the+ // engine's normal dynamic-mapping path instead, which now recurses+ // into nested objects (see xerj-engine's `dynamic_field_config`), so+ // `metadata.kind`/`metadata.topic`/etc. become real, `_mapping`-visible+ // fields instead of being permanently opaque.let mut properties = json!({"text": { "type": "semantic_text", "dimensions": 384 },- "stored_at": { "type": "long" },- "metadata": { "type": "object" }+ "stored_at": { "type": "long" }});if let Some(dims) = vector_dims {if dims == 0 {diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex c4462fe25..501bc2c8b 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -28088,6 +28088,12 @@ fn infer_field_type(val: &Value, date_detection: bool) -> FieldType {/// `KeywordFieldMapper("keyword").ignoreAbove(256)`).const DYNAMIC_KEYWORD_IGNORE_ABOVE: u32 = 256;+/// ES default `index.mapping.depth.limit` — how many `properties` levels a+/// dynamically discovered document may nest before recursion stops. Applies+/// only to the nested-object walk below; it does not change how deep an+/// *explicit* `_mapping` PUT may nest (that path is unaffected by this cap).+const DYNAMIC_MAPPING_DEPTH_LIMIT: usize = 20;+/// Build the [`FieldConfig`] for a dynamically discovered field.////// Wraps [`infer_field_type`] and mirrors the ES default dynamic mapping for@@ -28101,13 +28107,62 @@ const DYNAMIC_KEYWORD_IGNORE_ABOVE: u32 = 256;/// this records that contract in the schema rather than leaving it implicit./// Sub-fields of a leaf parent are NOT top-level `schema.fields` entries, so/// the search path's text/keyword field sets are unaffected.+///+/// A nested JSON object gets the same treatment one level deeper: its own+/// keys are recursively dynamic-mapped into child `FieldConfig`s under+/// `fields`, the same way `Text` gets its `keyword` sibling -- `FieldConfig`+/// and the ES-compat mapping renderer already both understand this dual+/// meaning of `fields` (real `properties` on `Object`/`Nested`, multi-fields+/// on a leaf type, see the doc comment on `FieldConfig::fields`); only the+/// *dynamic-discovery* path was never populating it for `Object`, so a+/// subfield like `metadata.kind` was genuinely queryable (the query path+/// resolves dotted paths against `_source` regardless of the mapping) but+/// invisible to `GET _mapping`/`_field_caps` -- indistinguishable from not+/// existing to any client that discovers fields that way instead of already+/// knowing the field name in advance (Kibana/OSD's own field list included).+/// Array-of-objects is intentionally out of scope here: `infer_field_type`+/// already only inspects an array's first element for its *type*, so this+/// recurses into that same first element for consistency rather than+/// attempting to reconcile a heterogeneous shape across array entries --+/// no behavior change for arrays beyond what recursing into their inferred+/// Object element already implies.fn dynamic_field_config(key: &str, val: &Value, date_detection: bool) -> FieldConfig {+ dynamic_field_config_at_depth(key, val, date_detection, 1)+}++fn dynamic_field_config_at_depth(+ key: &str,+ val: &Value,+ date_detection: bool,+ depth: usize,+) -> FieldConfig {let ft = infer_field_type(val, date_detection);let mut fc = FieldConfig::new(key.to_string(), ft);- if matches!(fc.field_type, FieldType::Text) {- let mut kw = FieldConfig::new("keyword".to_string(), FieldType::Keyword);- kw.options.ignore_above = Some(DYNAMIC_KEYWORD_IGNORE_ABOVE);- fc.fields.push(kw);+ match fc.field_type {+ FieldType::Text => {+ let mut kw = FieldConfig::new("keyword".to_string(), FieldType::Keyword);+ kw.options.ignore_above = Some(DYNAMIC_KEYWORD_IGNORE_ABOVE);+ fc.fields.push(kw);+ }+ FieldType::Object if depth < DYNAMIC_MAPPING_DEPTH_LIMIT => {+ // `val` itself may be the Array whose first element inferred+ // Object (see doc comment) -- unwrap to whichever object is the+ // real source of the nested keys.+ let obj = val+ .as_object()+ .or_else(|| val.as_array()?.iter().find(|v| !v.is_null())?.as_object());+ if let Some(obj) = obj {+ for (sub_key, sub_val) in obj {+ fc.fields.push(dynamic_field_config_at_depth(+ sub_key,+ sub_val,+ date_detection,+ depth + 1,+ ));+ }+ }+ }+ _ => {}}fc}@@ -36340,13 +36395,14 @@ mod date_detection_tests {assert!(matches!(fc.field_type, FieldType::Text));assert_eq!(fc.fields.len(), 1);- // Non-string inferences carry NO multi-field.+ // Non-string leaf inferences carry NO multi-field. Object is+ // exercised separately below (dynamic_object_recurses_into_nested_properties)+ // since it now gets real child sub-fields, not zero.for (val, want_sub) in [(serde_json::json!(42), 0usize),(serde_json::json!(2.5), 0),(serde_json::json!(true), 0),(serde_json::json!("2026-07-25T10:30:00Z"), 0), // date-detected- (serde_json::json!({"a": 1}), 0),] {let fc = dynamic_field_config("f", &val, true);assert_eq!(fc.fields.len(), want_sub, "value {val} sub-field count");@@ -36358,6 +36414,89 @@ mod date_detection_tests {assert_eq!(fc.fields.len(), 1);}+ /// A dynamically discovered nested object must recurse into its own+ /// keys instead of staying an opaque `{"type": "object"}` -- otherwise+ /// a subfield like `metadata.kind` is genuinely queryable (the query+ /// path resolves dotted paths against `_source` regardless of the+ /// mapping) but invisible to `GET _mapping`/`_field_caps`, indistinguishable+ /// from not existing to any client that discovers fields that way. Found+ /// via `/_memory`'s `metadata` field, which is exactly this shape.+ #[test]+ fn dynamic_object_recurses_into_nested_properties() {+ let fc = dynamic_field_config(+ "metadata",+ &serde_json::json!({"kind": "preference", "topic": "language"}),+ true,+ );+ assert!(matches!(fc.field_type, FieldType::Object));+ assert_eq!(fc.fields.len(), 2, "both nested keys should be discovered");++ let kind = fc.fields.iter().find(|f| f.name == "kind").expect("kind");+ assert!(matches!(kind.field_type, FieldType::Text));+ // Recursion composes with #209: a nested Text field gets its own+ // .keyword multi-field too, same as a top-level one would.+ assert_eq!(kind.fields.len(), 1);+ assert_eq!(kind.fields[0].name, "keyword");+ assert!(matches!(kind.fields[0].field_type, FieldType::Keyword));++ // Multi-level nesting: object -> object -> leaf.+ let fc = dynamic_field_config(+ "a",+ &serde_json::json!({"b": {"c": 1}}),+ true,+ );+ let b = &fc.fields[0];+ assert_eq!(b.name, "b");+ assert!(matches!(b.field_type, FieldType::Object));+ let c = &b.fields[0];+ assert_eq!(c.name, "c");+ assert!(matches!(c.field_type, FieldType::Long));++ // Mixed-type nested keys are inferred independently, like top-level+ // dynamic mapping already does.+ let fc = dynamic_field_config(+ "metadata",+ &serde_json::json!({"count": 3, "active": true, "label": "x"}),+ true,+ );+ let by_name = |n: &str| fc.fields.iter().find(|f| f.name == n).unwrap();+ assert!(matches!(by_name("count").field_type, FieldType::Long));+ assert!(matches!(by_name("active").field_type, FieldType::Boolean));+ assert!(matches!(by_name("label").field_type, FieldType::Text));++ // Depth limit: recursion stops at DYNAMIC_MAPPING_DEPTH_LIMIT rather+ // than walking an adversarially deep document forever (ES has the+ // same guard via index.mapping.depth.limit, default 20).+ let mut deep = serde_json::json!(1);+ for _ in 0..DYNAMIC_MAPPING_DEPTH_LIMIT + 5 {+ deep = serde_json::json!({"n": deep});+ }+ let fc = dynamic_field_config("root", &deep, true);+ let mut cur = &fc;+ let mut levels = 0;+ while let Some(next) = cur.fields.first() {+ cur = next;+ levels += 1;+ }+ assert!(+ levels < DYNAMIC_MAPPING_DEPTH_LIMIT + 5,+ "recursion must stop before reaching the artificially deep bottom, got {levels} levels"+ );++ // Array of objects: infer_field_type already only looks at the+ // first non-null element to decide the *type* (Object); the+ // dynamic-mapping walk mirrors that by recursing into the same+ // element rather than reconciling every array entry's shape.+ let fc = dynamic_field_config(+ "tags",+ &serde_json::json!([{"kind": "x"}, {"kind": "y", "extra": true}]),+ true,+ );+ assert!(matches!(fc.field_type, FieldType::Object));+ assert_eq!(fc.fields.len(), 1, "only the first element's keys are discovered");+ assert_eq!(fc.fields[0].name, "kind");+ }+/// Ingest-time acceptance for default-format date fields: lenient on/// everything ES could parse, rejecting only impossible shapes.#[test]