AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): project _source filters inside NESTED arrays of objects (#653)
xerj-org99725d8a3c32
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 1f2777ca6..691e08813 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -26639,6 +26639,38 @@ fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Va(acc, liv)}+ // Project each element of an array at the same dotted `path` — an array+ // index is transparent to a `_source` path at EVERY depth (#644/#653), so a+ // nested array of objects recurses too. An object element is filtered (and+ // an emptied one dropped, mirroring the map-value rule); a nested-array+ // element recurses here; a scalar survives only in an accepting state.+ fn project_array_at(+ arr: &[Value],+ path: &str,+ child_all: bool,+ includes: &[String],+ excludes: &[String],+ ) -> Vec<Value> {+ arr.iter()+ .filter_map(|elem| match elem {+ Value::Object(child_obj) => {+ let f = es_filter(elem, path, child_all, includes, excludes);+ match &f {+ Value::Object(m) if m.is_empty() => {+ (child_obj.is_empty() && child_all).then_some(f)+ }+ _ => Some(f),+ }+ }+ Value::Array(inner) => {+ let p = project_array_at(inner, path, child_all, includes, excludes);+ (!p.is_empty() || (inner.is_empty() && child_all)).then(|| Value::Array(p))+ }+ other => child_all.then(|| other.clone()),+ })+ .collect()+ }+// Filter one value at dotted `path`. `include_all` means an ancestor path// was ACCEPTed (or the include list is empty), so this subtree is included// subject only to excludes.@@ -26708,26 +26740,10 @@ fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Vaif child_all && !exc_live {out.insert(key.clone(), val.clone());} else {- let projected: Vec<Value> = arr- .iter()- .filter_map(|elem| match elem {- Value::Object(child_obj) => {- let f = es_filter(elem, &child, child_all, includes, excludes);- match &f {- // An element that *filtering* emptied is- // dropped from the array; one that was- // ALREADY `{}` survives only in an- // accepting state — same rule ES applies- // to array elements as to map values.- Value::Object(m) if m.is_empty() => {- (child_obj.is_empty() && child_all).then_some(f)- }- _ => Some(f),- }- }- other => child_all.then(|| other.clone()),- })- .collect();+ // Project each element at the same path (recursing into+ // nested arrays — #644/#653).+ let projected =+ project_array_at(arr, &child, child_all, includes, excludes);// A key whose array *filtering* emptied is pruned (had// elements, all dropped); an already-empty source array// survives only in an accepting state — mirrors the@@ -27210,6 +27226,34 @@ mod source_filter_tests {);}+ /// #653 (follow-up to #644): an array is transparent to a `_source` path at+ /// EVERY depth, so a nested array of arrays of objects is projected too —+ /// `a.name` over `{"a":[[{"name":"x","id":1}]]}` keeps `[[{"name":"x"}]]`.+ /// The array arm previously sent a nested-list element to the scalar branch,+ /// dropping it.+ #[test]+ fn source_filter_projects_inside_nested_arrays() {+ assert_eq!(+ filter_object(+ &json!({ "a": [[{ "name": "x", "id": 1 }], [{ "name": "y", "id": 2 }]] }),+ &["a.name".to_string()],+ &[]+ ),+ json!({ "a": [[{ "name": "x" }], [{ "name": "y" }]] }),+ "#653: a nested array of objects must project at depth"+ );+ // Exclude scrubs at depth through the nested array too.+ assert_eq!(+ filter_object(+ &json!({ "logs": [[{ "msg": 1, "secret": 2 }]] }),+ &[],+ &["logs.secret".to_string()]+ ),+ json!({ "logs": [[{ "msg": 1 }]] }),+ "#653: an exclude scrubs inside a nested array of objects"+ );+ }+#[test]fn explicit_source_true_keeps_mapping_derived_companions() {let companions = generated_embedding_companion_fields(&embedded_schema());