Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): project _source filters inside NESTED arrays of objects (#653)

xerj-org99725d8a3c32
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 1f2777ca6..691e08813 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -26639,6 +26639,38 @@ fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Va
(acc, liv)
}
+ // Project each element of an array at the same dotted `path` — an array
+ // index is transparent to a `_source` path at EVERY depth (#644/#653), so a
+ // nested array of objects recurses too. An object element is filtered (and
+ // an emptied one dropped, mirroring the map-value rule); a nested-array
+ // element recurses here; a scalar survives only in an accepting state.
+ fn project_array_at(
+ arr: &[Value],
+ path: &str,
+ child_all: bool,
+ includes: &[String],
+ excludes: &[String],
+ ) -> Vec<Value> {
+ arr.iter()
+ .filter_map(|elem| match elem {
+ Value::Object(child_obj) => {
+ let f = es_filter(elem, path, child_all, includes, excludes);
+ match &f {
+ Value::Object(m) if m.is_empty() => {
+ (child_obj.is_empty() && child_all).then_some(f)
+ }
+ _ => Some(f),
+ }
+ }
+ Value::Array(inner) => {
+ let p = project_array_at(inner, path, child_all, includes, excludes);
+ (!p.is_empty() || (inner.is_empty() && child_all)).then(|| Value::Array(p))
+ }
+ other => child_all.then(|| other.clone()),
+ })
+ .collect()
+ }
+
// Filter one value at dotted `path`. `include_all` means an ancestor path
// was ACCEPTed (or the include list is empty), so this subtree is included
// subject only to excludes.
@@ -26708,26 +26740,10 @@ fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Va
if child_all && !exc_live {
out.insert(key.clone(), val.clone());
} else {
- let projected: Vec<Value> = arr
- .iter()
- .filter_map(|elem| match elem {
- Value::Object(child_obj) => {
- let f = es_filter(elem, &child, child_all, includes, excludes);
- match &f {
- // An element that *filtering* emptied is
- // dropped from the array; one that was
- // ALREADY `{}` survives only in an
- // accepting state — same rule ES applies
- // to array elements as to map values.
- Value::Object(m) if m.is_empty() => {
- (child_obj.is_empty() && child_all).then_some(f)
- }
- _ => Some(f),
- }
- }
- other => child_all.then(|| other.clone()),
- })
- .collect();
+ // Project each element at the same path (recursing into
+ // nested arrays — #644/#653).
+ let projected =
+ project_array_at(arr, &child, child_all, includes, excludes);
// A key whose array *filtering* emptied is pruned (had
// elements, all dropped); an already-empty source array
// survives only in an accepting state — mirrors the
@@ -27210,6 +27226,34 @@ mod source_filter_tests {
);
}
+ /// #653 (follow-up to #644): an array is transparent to a `_source` path at
+ /// EVERY depth, so a nested array of arrays of objects is projected too —
+ /// `a.name` over `{"a":[[{"name":"x","id":1}]]}` keeps `[[{"name":"x"}]]`.
+ /// The array arm previously sent a nested-list element to the scalar branch,
+ /// dropping it.
+ #[test]
+ fn source_filter_projects_inside_nested_arrays() {
+ assert_eq!(
+ filter_object(
+ &json!({ "a": [[{ "name": "x", "id": 1 }], [{ "name": "y", "id": 2 }]] }),
+ &["a.name".to_string()],
+ &[]
+ ),
+ json!({ "a": [[{ "name": "x" }], [{ "name": "y" }]] }),
+ "#653: a nested array of objects must project at depth"
+ );
+ // Exclude scrubs at depth through the nested array too.
+ assert_eq!(
+ filter_object(
+ &json!({ "logs": [[{ "msg": 1, "secret": 2 }]] }),
+ &[],
+ &["logs.secret".to_string()]
+ ),
+ json!({ "logs": [[{ "msg": 1 }]] }),
+ "#653: an exclude scrubs inside a nested array of objects"
+ );
+ }
+
#[test]
fn explicit_source_true_keeps_mapping_derived_companions() {
let companions = generated_embedding_companion_fields(&embedded_schema());