Apache DataFusion SQL Query Engine
Parquet row filter struct access tree (#23217)
Subham Singhala05388e82374
diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rsindex 350ed9596b..0381808511 100644--- a/datafusion/datasource-parquet/src/projection_read_plan.rs+++ b/datafusion/datasource-parquet/src/projection_read_plan.rs@@ -76,6 +76,83 @@ pub(crate) struct StructFieldAccess {pub(crate) field_path: Vec<String>,}+/// Trie of nested struct accesses, keyed at the top by the root column index in+/// the file schema and then by field names down each access path.+///+/// # Example+///+/// For a filter expression+///+/// ```sql+/// WHERE s['outer']['a'] > 10+/// AND s['outer']['b'] < 20+/// AND s['outer']['inner']['c'] IS NOT NULL+/// ```+///+/// where `s` is column index `2` in the file schema, three accesses are+/// recorded — all with `root_index = 2` and paths `["outer","a"]`,+/// `["outer","b"]`, `["outer","inner","c"]`. They produce a trie in which+/// the shared `"outer"` prefix is represented by a single intermediate node:+///+/// ```text+/// roots:+/// 2 ──► node { selected_here: false }+/// children:+/// "outer" ──► node { selected_here: false }+/// children:+/// "a" ──► { selected_here: true, children: {} }+/// "b" ──► { selected_here: true, children: {} }+/// "inner" ──► { selected_here: false,+/// children: {+/// "c" ──► { selected_here: true,+/// children: {} }+/// } }+/// ```+#[derive(Debug, Default)]+struct StructAccessTree<'a> {+ roots: BTreeMap<usize, StructAccessNode<'a>>,+}++/// One node in a [`StructAccessTree`].+///+/// `selected_here` is `true` when at least one access path terminates at this+/// node. Duplicate paths are idempotent.+#[derive(Debug, Default)]+struct StructAccessNode<'a> {+ children: BTreeMap<&'a str, StructAccessNode<'a>>,+ selected_here: bool,+}++impl<'a> StructAccessTree<'a> {+ /// Builds a [`StructAccessTree`] from a flat list of accesses.+ ///+ /// For each [`StructFieldAccess`], walks from the given root index down+ /// the field path, creating intermediate nodes as needed, and sets the+ /// terminal node's `selected_here` to `true`. Paths sharing a prefix+ /// collapse onto common intermediate nodes.+ fn from_accesses(accesses: &'a [StructFieldAccess]) -> Self {+ let mut tree = Self::default();+ for StructFieldAccess {+ root_index,+ field_path,+ } in accesses+ {+ let mut node = tree.roots.entry(*root_index).or_default();+ for component in field_path {+ node = node.children.entry(component.as_str()).or_default();+ }+ node.selected_here = true;+ }+ tree+ }++ /// Returns the node for the given file-schema column index, or `None` if+ /// no access path was recorded under that root.+ fn root(&self, idx: usize) -> Option<&StructAccessNode<'a>> {+ self.roots.get(&idx)+ }+}+/// Traverses a `PhysicalExpr` tree to determine if any column references would/// prevent the expression from being pushed down to the parquet decoder.///@@ -630,12 +707,9 @@ fn build_read_plan_with_cast_clipping(}if !get_field_accesses.is_empty() {- leaf_indices.extend(resolve_struct_field_leaves(- &get_field_accesses,- file_schema,- schema_descr,- ));- let get_field_schema = build_filter_schema(file_schema, &[], &get_field_accesses);+ let get_field_tree = StructAccessTree::from_accesses(&get_field_accesses);+ leaf_indices.extend(resolve_struct_field_leaves(&get_field_tree, schema_descr));+ let get_field_schema = build_filter_schema(file_schema, &[], &get_field_tree);let get_field_roots: BTreeSet<usize> =get_field_accesses.iter().map(|a| a.root_index).collect();// `build_filter_schema` emits one field per accessed root in@@ -692,20 +766,18 @@ pub(crate) fn assemble_read_plan(file_schema: &Schema,schema_descr: &SchemaDescriptor,) -> (ParquetReadPlan, Vec<usize>) {+ let access_tree = StructAccessTree::from_accesses(struct_field_accesses);+let mut leaf_indices =leaf_indices_for_roots(root_indices.iter().copied(), schema_descr);- leaf_indices.extend_from_slice(&resolve_struct_field_leaves(- struct_field_accesses,- file_schema,- schema_descr,- ));+ leaf_indices+ .extend_from_slice(&resolve_struct_field_leaves(&access_tree, schema_descr));leaf_indices.sort_unstable();leaf_indices.dedup();let projection_mask =ProjectionMask::leaves(schema_descr, leaf_indices.iter().copied());- let projected_schema =- build_filter_schema(file_schema, root_indices, struct_field_accesses);+ let projected_schema = build_filter_schema(file_schema, root_indices, &access_tree);(ParquetReadPlan {@@ -762,62 +834,129 @@ where.collect()}-/// Resolves struct field access to specific Parquet leaf column indices+/// Returns the Parquet leaf column indices selected by the access tree.+///+/// # Matching+///+/// Iterates Parquet leaves in ascending order (`0..num_columns()`). For each+/// leaf:+///+/// 1. **Root dispatch.** Look up the leaf's root index — the top-level Arrow+/// column it belongs to — via `SchemaDescriptor::get_column_root_idx`. If+/// that root is absent from the access tree (the filter never touched any+/// field under it), skip the leaf without further work.+///+/// 2. **Path walk.** Otherwise, take the leaf's dotted column path+/// (`col.path().parts()`), drop the first component (the root field name,+/// already used in step 1), and walk the remaining components against the+/// matching trie subtree via [`leaf_under_tree`].+///+/// 3. **Inclusion.** The leaf is added to the result iff the walk reaches a+/// node with `selected_here = true` — either an ancestor along the+/// descent (subsumption: a shallower access subsumes the leaf) or the+/// terminal node reached at the end of the path (exact match).+///+/// # Returns///-/// For every `StructFieldAccess`, finds the leaf columns in the Parquet schema-/// whose path matches the struct root name + field path. This avoids reading all-/// leaves of a struct when only specific fields are needed+/// `Vec<usize>` of Parquet leaf column indices. The scan visits each leaf+/// exactly once and pushes in iteration order, so the result is in ascending+/// order and free of duplicates by construction — callers do not need to+/// sort or dedup.fn resolve_struct_field_leaves(- accesses: &[StructFieldAccess],- file_schema: &Schema,+ access_tree: &StructAccessTree<'_>,schema_descr: &SchemaDescriptor,) -> Vec<usize> {let mut leaf_indices = Vec::new();- for access in accesses {- let root_name = file_schema.field(access.root_index).name();- let prefix = std::iter::once(root_name.as_str())- .chain(access.field_path.iter().map(|p| p.as_str()))- .collect::<Vec<_>>();-- for leaf_idx in 0..schema_descr.num_columns() {- let col = schema_descr.column(leaf_idx);- let col_path = col.path().parts();-- // A leaf matches if its path starts with our prefix.- // e.g., prefix=["s", "value"] matches leaf path ["s", "value"]- // prefix=["s", "outer"] matches ["s", "outer", "inner"]- let leaf_matches_path = col_path.len() >= prefix.len()- && col_path.iter().zip(prefix.iter()).all(|(a, b)| a == b);-- if leaf_matches_path {- leaf_indices.push(leaf_idx);- }+ for leaf_idx in 0..schema_descr.num_columns() {+ let root_idx = schema_descr.get_column_root_idx(leaf_idx);+ let Some(root_node) = access_tree.roots.get(&root_idx) else {+ continue;+ };+ // The first part is the root field name, already used in step 1; walk+ // the rest against the tree.+ let col = schema_descr.column(leaf_idx);+ let Some((_root_name, rest)) = col.path().parts().split_first() else {+ continue;+ };+ if leaf_under_tree(root_node, rest) {+ leaf_indices.push(leaf_idx);}}leaf_indices}-/// Builds a filter schema that includes only the fields actually accessed by the-/// filter expression.+/// True when the leaf path beneath a root is selected by the access tree.+///+/// A shallower `selected_here` node subsumes deeper accesses: once the walk+/// reaches such a node, every leaf below it is included.+fn leaf_under_tree(mut node: &StructAccessNode<'_>, path: &[String]) -> bool {+ for component in path {+ if node.selected_here {+ return true;+ }+ let Some(child) = node.children.get(component.as_str()) else {+ return false;+ };+ node = child;+ }+ node.selected_here+}++/// Builds the Arrow schema used to evaluate the filter expression.+///+/// The returned schema is a **subset** of `file_schema`, restricted to the+/// columns the filter actually touches and (for struct columns accessed+/// only through nested paths) **pruned** to only the accessed fields.+///+/// # Inputs+///+/// - `file_schema` — the full file schema; provides the source `Field`s+/// (names, types, nullability, metadata).+/// - `regular_indices` — file-schema column indices the filter references+/// as **whole columns** (non-struct columns, or struct roots referenced+/// in their entirety). Must be sorted, deduplicated.+/// - `access_tree` — the trie of nested struct field accesses recorded by+/// [`PushdownChecker`].+///+/// # Behavior///-/// For regular (non-struct) columns, the full field type is used.-/// For struct columns accessed via `get_field`, a pruned struct type is created-/// containing only the fields along the access path. Note: it must match the schema-/// that the Parquet reader produces when projecting specific struct leaves+/// The set of columns to include is the union of `regular_indices` and+/// `access_tree.roots.keys()`. For each column index in that union, decide+/// how the field appears in the output:+///+/// 1. **Whole-column reference** (`idx` is in `regular_indices`). Keep the+/// field's full type unchanged. This is the **whole-root override**:+/// pruning is only valid when a column is accessed *exclusively* through+/// nested field accesses; if any predicate references the whole column,+/// the projected schema must preserve the full type for that column.+///+/// 2. **Nested-access-only struct root.** Look up the column's node in the+/// access tree and call [`prune_struct_type`] on the field's `DataType`+/// with that node. Wrap the pruned type in a new `Field` carrying the+/// original name and nullability.+///+/// Column order in the output schema follows ascending file-schema index+/// (via the `BTreeSet` union), matching the order the Parquet reader+/// produces when projecting these columns.+///+/// # Returns+///+/// An `Arc<Schema>` whose fields are a subset of `file_schema`'s, with+/// struct types pruned per the access tree. The schema's metadata is+/// inherited from `file_schema`.fn build_filter_schema(file_schema: &Schema,regular_indices: &[usize],- struct_field_accesses: &[StructFieldAccess],+ access_tree: &StructAccessTree<'_>,) -> SchemaRef {let regular_set: BTreeSet<usize> = regular_indices.iter().copied().collect();- let paths_by_root = group_access_paths_by_root(struct_field_accesses);let all_indices = regular_indices.iter().copied()- .chain(paths_by_root.keys().copied())+ .chain(access_tree.roots.keys().copied()).collect::<BTreeSet<_>>();let fields = all_indices@@ -834,11 +973,11 @@ fn build_filter_schema(return Arc::new(field.clone());}- let Some(field_paths) = paths_by_root.get(&idx) else {+ let Some(node) = access_tree.root(idx) else {return Arc::new(field.clone());};- let pruned_data_type = prune_struct_type(field.data_type(), field_paths);+ let pruned_data_type = prune_struct_type(field.data_type(), node);Arc::new(Field::new(field.name(),pruned_data_type,@@ -853,68 +992,64 @@ fn build_filter_schema())}-/// Groups struct field access paths once for the root schema level.+/// Returns a copy of `dt` with non-accessed struct children removed.///-/// Each map entry contains the complete field paths accessed below a root-/// column. Recursive pruning groups these paths by their next component at each-/// nested struct level.-fn group_access_paths_by_root(- struct_field_accesses: &[StructFieldAccess],-) -> BTreeMap<usize, Vec<&[String]>> {- let mut paths_by_root: BTreeMap<usize, Vec<&[String]>> = BTreeMap::new();- for StructFieldAccess {- root_index,- field_path,- } in struct_field_accesses- {- paths_by_root- .entry(*root_index)- .or_default()- .push(field_path.as_slice());- }-- paths_by_root-}--/// Groups access paths once for the current struct level.+/// # Behavior///-/// The map key is the field name at this level. The map value is the list of-/// remaining path suffixes below that field. An empty suffix means the access-/// path terminates at that field, so the full field must be preserved.-fn group_paths_by_next_field<'a>(- paths: &'a [&'a [String]],-) -> BTreeMap<&'a str, Vec<&'a [String]>> {- let mut paths_by_field: BTreeMap<&str, Vec<&[String]>> = BTreeMap::new();- for path in paths {- if let Some((field, sub_path)) = path.split_first() {- paths_by_field- .entry(field.as_str())- .or_default()- .push(sub_path);- }+/// - If `node.selected_here` is `true`, the input type is returned+/// unchanged. An access path terminates at this node, so the whole+/// subtree (every field of `dt`, recursively) is required. This mirrors+/// the subsumption check in [`leaf_under_tree`] so the projection mask+/// and the projected schema agree even if a producer ever records an+/// access whose `field_path` terminates above a struct.+///+/// - Otherwise, if `dt` is not a `DataType::Struct`, it is cloned and+/// returned unchanged. The trie only ever guides struct-level pruning;+/// other types pass through.+///+/// - Otherwise, `dt` is a struct and its fields are iterated in their+/// original order. For each field `f`:+/// 1. Look up `f.name()` in `node.children`.+/// - **Absent.** No access goes through this field. Drop it.+/// - **Present, child node's `selected_here` is `true`.** An access+/// path terminates at this field. Keep the entire subtree by+/// cloning `f` unchanged (`Arc::clone` — no new `Field`).+/// - **Present, child node's `selected_here` is `false`.** Some+/// access goes through this field to a deeper terminal. Recurse+/// into `f.data_type()` with the matching child node, then wrap+/// the pruned type in a fresh `Field` with `f`'s name and+/// nullability.+///+/// Field ordering is preserved (consumers must match the order the Parquet+/// reader produces when projecting specific leaves). Iterating Arrow's+/// `Fields` directly — rather than iterating `node.children` — is what+/// preserves that order.+///+/// # Returns+///+/// A new `DataType::Struct` whose fields are a subset of `dt`'s, restricted+/// to the paths represented by `node`. The original `dt` is not modified.+fn prune_struct_type(dt: &DataType, node: &StructAccessNode<'_>) -> DataType {+ if node.selected_here {+ // Subsumption: the entire subtree below this node is required.+ return dt.clone();}- paths_by_field-}--fn prune_struct_type(dt: &DataType, paths: &[&[String]]) -> DataType {let DataType::Struct(fields) = dt else {return dt.clone();};- let paths_by_field = group_paths_by_next_field(paths);-let pruned_fields = fields.iter().filter_map(|f| {- let sub_paths = paths_by_field.get(f.name().as_str())?;+ let child = node.children.get(f.name().as_str())?;- let out = if sub_paths.iter().any(|sub| sub.is_empty()) {- // Leaf of access path — keep the field as-is.+ let out = if child.selected_here {+ // Access path terminates at this field — preserve the whole subtree.Arc::clone(f)} else {// Recurse into nested struct.- let pruned = prune_struct_type(f.data_type(), sub_paths);+ let pruned = prune_struct_type(f.data_type(), child);Arc::new(Field::new(f.name(), pruned, f.is_nullable()))};@@ -1457,4 +1592,206 @@ mod test {),);}++ fn access(root: usize, path: &[&str]) -> StructFieldAccess {+ StructFieldAccess {+ root_index: root,+ field_path: path.iter().map(|&s| s.to_string()).collect(),+ }+ }++ #[test]+ fn struct_access_tree_from_empty_input_has_no_roots() {+ let tree = StructAccessTree::from_accesses(&[]);+ assert!(tree.roots.is_empty());+ }++ #[test]+ fn struct_access_tree_groups_paths_by_root() {+ let accesses = [access(0, &["a"]), access(2, &["x"]), access(2, &["y"])];+ let tree = StructAccessTree::from_accesses(&accesses);++ assert_eq!(tree.roots.keys().copied().collect::<Vec<_>>(), vec![0, 2]);+ let root0 = tree.root(0).unwrap();+ assert!(root0.children.contains_key("a"));+ assert!(root0.children["a"].selected_here);++ let root2 = tree.root(2).unwrap();+ assert_eq!(+ root2.children.keys().copied().collect::<Vec<_>>(),+ vec!["x", "y"],+ );+ }++ #[test]+ fn struct_access_tree_shared_prefix_collapses_into_one_node() {+ let accesses = [access(0, &["outer", "a"]), access(0, &["outer", "b"])];+ let tree = StructAccessTree::from_accesses(&accesses);++ let root = tree.root(0).unwrap();+ assert!(!root.selected_here);++ let outer = &root.children["outer"];+ // `outer` itself was never the terminal of an access path.+ assert!(!outer.selected_here);+ // Both leaves below share the single `outer` node.+ assert_eq!(+ outer.children.keys().copied().collect::<Vec<_>>(),+ vec!["a", "b"],+ );+ assert!(outer.children["a"].selected_here);+ assert!(outer.children["b"].selected_here);+ }++ #[test]+ fn struct_access_tree_records_both_shallow_and_deep_selection() {+ // `s['outer']` (whole subtree) and `s['outer']['a']` (specific leaf)+ // both recorded. Consumers honor the shallower selection at walk time;+ // the builder simply records both `selected_here` flags.+ let accesses = [access(0, &["outer"]), access(0, &["outer", "a"])];+ let tree = StructAccessTree::from_accesses(&accesses);++ let outer = &tree.root(0).unwrap().children["outer"];+ assert!(outer.selected_here);+ assert!(outer.children["a"].selected_here);+ }++ /// `prune_struct_type` must honor `selected_here` on the input node+ /// itself, not only on its children — symmetric with `leaf_under_tree`.+ /// Without this guard, a node with `selected_here = true` and no+ /// children produces an empty struct (silent drift from the leaf set).+ #[test]+ fn prune_struct_type_returns_full_type_when_node_is_selected_here() {+ let node = StructAccessNode {+ selected_here: true,+ ..Default::default()+ };++ let s_type = DataType::Struct(+ vec![+ Arc::new(Field::new("outer", DataType::Int32, false)),+ Arc::new(Field::new("other", DataType::Int32, false)),+ ]+ .into(),+ );++ let pruned = prune_struct_type(&s_type, &node);++ assert_eq!(+ pruned, s_type,+ "selected_here on the input node must preserve the full type"+ );+ }++ /// Same guard, but for the case where `selected_here` is set on an+ /// intermediate node that also has children — e.g. both `s['outer']`+ /// and `s['outer']['a']` are recorded. The shallower terminal must+ /// keep the entire `outer` subtree, ignoring the deeper child entry.+ #[test]+ fn prune_struct_type_shallow_selection_subsumes_deeper_children() {+ let accesses = [access(0, &["outer"]), access(0, &["outer", "a"])];+ let tree = StructAccessTree::from_accesses(&accesses);++ let outer_type = DataType::Struct(+ vec![+ Arc::new(Field::new("a", DataType::Int32, false)),+ Arc::new(Field::new("b", DataType::Int32, false)),+ ]+ .into(),+ );++ let outer_node = &tree.root(0).unwrap().children["outer"];+ let pruned = prune_struct_type(&outer_type, outer_node);++ assert_eq!(+ pruned, outer_type,+ "shallow selected_here must preserve the whole subtree, \+ not narrow to the deeper child"+ );+ }++ /// Mixed whole-root and nested access.+ /// Projecting `s` (whole) alongside `get_field(s, 'outer', 'a')` (nested)+ /// must preserve the full `s` struct type AND include all `s` leaves in+ /// the projection mask. The nested access does not narrow the whole-root+ /// reference — `regular_indices` wins over the access tree for that root.+ #[test]+ fn projection_whole_root_plus_nested_access_keeps_full_struct() {+ // Schema: s (Struct{outer: Struct{a, b}})+ // Parquet leaves: s.outer.a=0, s.outer.b=1+ let outer_fields: Fields = vec![+ Arc::new(Field::new("a", DataType::Int32, false)),+ Arc::new(Field::new("b", DataType::Int32, false)),+ ]+ .into();+ let s_fields: Fields = vec![Arc::new(Field::new(+ "outer",+ DataType::Struct(outer_fields.clone()),+ false,+ ))]+ .into();+ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Struct(s_fields.clone()),+ false,+ )]));++ let outer_arr = StructArray::new(+ outer_fields.clone(),+ vec![+ Arc::new(Int32Array::from(vec![1, 2])) as _,+ Arc::new(Int32Array::from(vec![3, 4])) as _,+ ],+ None,+ );+ let s_arr =+ StructArray::new(s_fields.clone(), vec![Arc::new(outer_arr) as _], None);+ let batch =+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(s_arr)]).unwrap();++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)+ .expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let reader_file = file.reopen().expect("reopen file");+ let builder = ParquetRecordBatchReaderBuilder::try_new(reader_file)+ .expect("reader builder");+ let metadata = builder.metadata().clone();+ let file_schema = builder.schema().clone();+ let schema_descr = metadata.file_metadata().schema_descr();++ // Column("s") (whole struct) + get_field(s, 'outer', 'a') (nested access).+ let exprs: Vec<Arc<dyn PhysicalExpr>> = vec![+ Arc::new(PhysicalColumn::new("s", 0)),+ logical2physical(+ &get_field().call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),+ ]),+ &file_schema,+ ),+ ];++ let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr);++ // `s` must keep its full nested type — NOT narrowed to Struct{outer: Struct{a}}.+ let s_field = read_plan.projected_schema.field_with_name("s").unwrap();+ assert_eq!(+ s_field.data_type(),+ &DataType::Struct(s_fields),+ "whole-root reference must preserve the full nested struct type \+ even when a nested access is also recorded"+ );++ // All `s` leaves must be in the projection mask (s.outer.a AND s.outer.b).+ let expected_mask = ProjectionMask::leaves(schema_descr, [0, 1]);+ assert_eq!(+ read_plan.projection_mask, expected_mask,+ "whole-root reference must select every leaf under the root"+ );+ }}diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rsindex a375e6611e..c1a47c896c 100644--- a/datafusion/datasource-parquet/src/row_filter.rs+++ b/datafusion/datasource-parquet/src/row_filter.rs@@ -1419,4 +1419,453 @@ mod test {let batch = RecordBatch::new_empty(Arc::clone(table_schema));expr.evaluate(&batch).is_ok()}++ /// Multiple sibling fields under one struct root: `s['value'] AND s['label']`.+ /// The projection mask should include exactly those two leaves (not the third+ /// sibling), and the projected schema should be pruned to those siblings.+ #[test]+ fn get_field_multiple_fields_under_same_root_uses_only_those_leaves() {+ // Schema: s (Struct{value: Int32, label: Utf8, extra: Int32})+ // Parquet leaves: s.value=0, s.label=1, s.extra=2+ let struct_fields: Fields = vec![+ Arc::new(Field::new("value", DataType::Int32, false)),+ Arc::new(Field::new("label", DataType::Utf8, false)),+ Arc::new(Field::new("extra", DataType::Int32, false)),+ ]+ .into();++ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Struct(struct_fields.clone()),+ false,+ )]));++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(StructArray::new(+ struct_fields.clone(),+ vec![+ Arc::new(Int32Array::from(vec![10, 20, 30])) as _,+ Arc::new(StringArray::from(vec!["a", "b", "c"])) as _,+ Arc::new(Int32Array::from(vec![100, 200, 300])) as _,+ ],+ None,+ ))],+ )+ .unwrap();++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)+ .expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let reader_file = file.reopen().expect("reopen file");+ let builder = ParquetRecordBatchReaderBuilder::try_new(reader_file)+ .expect("reader builder");+ let metadata = builder.metadata().clone();+ let file_schema = builder.schema().clone();++ // s['value'] > 5 AND s['label'] = 'b'+ let value_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("value".to_string())), None),+ ])+ .gt(Expr::Literal(ScalarValue::Int32(Some(5)), None));+ let label_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("label".to_string())), None),+ ])+ .eq(Expr::Literal(+ ScalarValue::Utf8(Some("b".to_string())),+ None,+ ));+ let expr = logical2physical(&value_expr.and(label_expr), &file_schema);++ let candidate = FilterCandidateBuilder::new(expr, Arc::clone(&file_schema))+ .build(&metadata)+ .expect("building candidate")+ .expect("conjunction of two get_field predicates should be pushable");++ // Only s.value (leaf 0) and s.label (leaf 1) should be projected; s.extra (leaf 2) skipped.+ let expected_mask =+ ProjectionMask::leaves(metadata.file_metadata().schema_descr(), [0, 1]);+ assert_eq!(+ candidate.read_plan.projection_mask, expected_mask,+ "projection_mask should include only the two accessed sibling leaves"+ );++ let s_field = candidate+ .read_plan+ .projected_schema+ .field_with_name("s")+ .unwrap();+ let expected_pruned: Fields = vec![+ Arc::new(Field::new("value", DataType::Int32, false)),+ Arc::new(Field::new("label", DataType::Utf8, false)),+ ]+ .into();+ assert_eq!(+ s_field.data_type(),+ &DataType::Struct(expected_pruned),+ "projected struct schema should drop the un-accessed `extra` sibling"+ );+ }++ /// Two predicates share a nested prefix: `s['outer']['a'] AND s['outer']['b']`.+ /// The projection mask should include exactly those two leaves and exclude+ /// the cousin under `s['other']` plus `s['outer']['c']`. The projected+ /// schema must mirror that shape.+ #[test]+ fn get_field_nested_shared_prefix_uses_only_prefix_leaves() {+ // Schema: s (Struct{outer: Struct{a, b, c}, other: Struct{x}})+ // Parquet leaves: s.outer.a=0, s.outer.b=1, s.outer.c=2, s.other.x=3+ let outer_fields: Fields = vec![+ Arc::new(Field::new("a", DataType::Int32, false)),+ Arc::new(Field::new("b", DataType::Int32, false)),+ Arc::new(Field::new("c", DataType::Int32, false)),+ ]+ .into();+ let other_fields: Fields =+ vec![Arc::new(Field::new("x", DataType::Int32, false))].into();+ let s_fields: Fields = vec![+ Arc::new(Field::new(+ "outer",+ DataType::Struct(outer_fields.clone()),+ false,+ )),+ Arc::new(Field::new(+ "other",+ DataType::Struct(other_fields.clone()),+ false,+ )),+ ]+ .into();+ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Struct(s_fields.clone()),+ false,+ )]));++ let outer_arr = StructArray::new(+ outer_fields,+ vec![+ Arc::new(Int32Array::from(vec![1, 2])) as _,+ Arc::new(Int32Array::from(vec![10, 20])) as _,+ Arc::new(Int32Array::from(vec![100, 200])) as _,+ ],+ None,+ );+ let other_arr = StructArray::new(+ other_fields,+ vec![Arc::new(Int32Array::from(vec![7, 8])) as _],+ None,+ );+ let s_arr = StructArray::new(+ s_fields,+ vec![Arc::new(outer_arr) as _, Arc::new(other_arr) as _],+ None,+ );+ let batch =+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(s_arr)]).unwrap();++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)+ .expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let reader_file = file.reopen().expect("reopen file");+ let builder = ParquetRecordBatchReaderBuilder::try_new(reader_file)+ .expect("reader builder");+ let metadata = builder.metadata().clone();+ let file_schema = builder.schema().clone();++ // s['outer']['a'] > 0 AND s['outer']['b'] > 0+ let a_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),+ ])+ .gt(Expr::Literal(ScalarValue::Int32(Some(0)), None));+ let b_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("b".to_string())), None),+ ])+ .gt(Expr::Literal(ScalarValue::Int32(Some(0)), None));+ let expr = logical2physical(&a_expr.and(b_expr), &file_schema);++ let candidate = FilterCandidateBuilder::new(expr, Arc::clone(&file_schema))+ .build(&metadata)+ .expect("building candidate")+ .expect("shared-prefix nested predicates should be pushable");++ // Only s.outer.a (0) and s.outer.b (1) — not s.outer.c (2), not s.other.x (3).+ let expected_mask =+ ProjectionMask::leaves(metadata.file_metadata().schema_descr(), [0, 1]);+ assert_eq!(+ candidate.read_plan.projection_mask, expected_mask,+ "projection_mask should drop cousin and un-accessed sibling leaves"+ );++ let s_field = candidate+ .read_plan+ .projected_schema+ .field_with_name("s")+ .unwrap();+ let expected_inner: Fields = vec![+ Arc::new(Field::new("a", DataType::Int32, false)),+ Arc::new(Field::new("b", DataType::Int32, false)),+ ]+ .into();+ let expected_outer: Fields = vec![Arc::new(Field::new(+ "outer",+ DataType::Struct(expected_inner),+ false,+ ))]+ .into();+ assert_eq!(+ s_field.data_type(),+ &DataType::Struct(expected_outer),+ "projected schema should keep only the shared-prefix subtree"+ );+ }++ /// Two predicates touch disjoint subtrees of the same struct root:+ /// `s['outer']['a'] AND s['other']['x']`. Both subtrees must be retained+ /// in the projection mask and in the projected schema.+ #[test]+ fn get_field_disjoint_subtrees_keep_both() {+ // Schema: s (Struct{outer: Struct{a, b}, other: Struct{x, y}})+ // Parquet leaves: s.outer.a=0, s.outer.b=1, s.other.x=2, s.other.y=3+ let outer_fields: Fields = vec![+ Arc::new(Field::new("a", DataType::Int32, false)),+ Arc::new(Field::new("b", DataType::Int32, false)),+ ]+ .into();+ let other_fields: Fields = vec![+ Arc::new(Field::new("x", DataType::Int32, false)),+ Arc::new(Field::new("y", DataType::Int32, false)),+ ]+ .into();+ let s_fields: Fields = vec![+ Arc::new(Field::new(+ "outer",+ DataType::Struct(outer_fields.clone()),+ false,+ )),+ Arc::new(Field::new(+ "other",+ DataType::Struct(other_fields.clone()),+ false,+ )),+ ]+ .into();+ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Struct(s_fields.clone()),+ false,+ )]));++ let outer_arr = StructArray::new(+ outer_fields,+ vec![+ Arc::new(Int32Array::from(vec![1, 2])) as _,+ Arc::new(Int32Array::from(vec![3, 4])) as _,+ ],+ None,+ );+ let other_arr = StructArray::new(+ other_fields,+ vec![+ Arc::new(Int32Array::from(vec![5, 6])) as _,+ Arc::new(Int32Array::from(vec![7, 8])) as _,+ ],+ None,+ );+ let s_arr = StructArray::new(+ s_fields,+ vec![Arc::new(outer_arr) as _, Arc::new(other_arr) as _],+ None,+ );+ let batch =+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(s_arr)]).unwrap();++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)+ .expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let reader_file = file.reopen().expect("reopen file");+ let builder = ParquetRecordBatchReaderBuilder::try_new(reader_file)+ .expect("reader builder");+ let metadata = builder.metadata().clone();+ let file_schema = builder.schema().clone();++ // s['outer']['a'] > 0 AND s['other']['x'] > 0+ let a_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),+ ])+ .gt(Expr::Literal(ScalarValue::Int32(Some(0)), None));+ let x_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("other".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("x".to_string())), None),+ ])+ .gt(Expr::Literal(ScalarValue::Int32(Some(0)), None));+ let expr = logical2physical(&a_expr.and(x_expr), &file_schema);++ let candidate = FilterCandidateBuilder::new(expr, Arc::clone(&file_schema))+ .build(&metadata)+ .expect("building candidate")+ .expect("disjoint nested predicates should be pushable");++ // s.outer.a (0) and s.other.x (2); not s.outer.b (1), not s.other.y (3).+ let expected_mask =+ ProjectionMask::leaves(metadata.file_metadata().schema_descr(), [0, 2]);+ assert_eq!(+ candidate.read_plan.projection_mask, expected_mask,+ "projection_mask should keep one leaf from each disjoint subtree"+ );++ let s_field = candidate+ .read_plan+ .projected_schema+ .field_with_name("s")+ .unwrap();+ let expected_outer: Fields =+ vec![Arc::new(Field::new("a", DataType::Int32, false))].into();+ let expected_other: Fields =+ vec![Arc::new(Field::new("x", DataType::Int32, false))].into();+ let expected_s: Fields = vec![+ Arc::new(Field::new("outer", DataType::Struct(expected_outer), false)),+ Arc::new(Field::new("other", DataType::Struct(expected_other), false)),+ ]+ .into();+ assert_eq!(+ s_field.data_type(),+ &DataType::Struct(expected_s),+ "projected schema should keep one pruned field from each disjoint subtree"+ );+ }++ /// End-to-end: shared-prefix nested predicates filter rows correctly during+ /// Parquet decoding and report the expected pushdown metrics.+ #[test]+ fn get_field_end_to_end_shared_prefix_filters_rows() {+ // Schema: id (Int32), s (Struct{outer: Struct{a, b}})+ // Parquet leaves: id=0, s.outer.a=1, s.outer.b=2+ let outer_fields: Fields = vec![+ Arc::new(Field::new("a", DataType::Int32, false)),+ Arc::new(Field::new("b", DataType::Int32, false)),+ ]+ .into();+ let s_fields: Fields = vec![Arc::new(Field::new(+ "outer",+ DataType::Struct(outer_fields.clone()),+ false,+ ))]+ .into();+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("s", DataType::Struct(s_fields.clone()), false),+ ]));++ // +----+--------------------------++ // | id | s |+ // +----+--------------------------++ // | 1 | {outer: {a: 10, b: 50}} | <- a>5 and b<100 → match+ // | 2 | {outer: {a: 0, b: 60}} | <- a>5 fails → drop+ // | 3 | {outer: {a: 20, b: 80}} | <- a>5 and b<100 → match+ // | 4 | {outer: {a: 30, b: 200}} | <- b<100 fails → drop+ // +----+--------------------------++ let outer_arr = StructArray::new(+ outer_fields,+ vec![+ Arc::new(Int32Array::from(vec![10, 0, 20, 30])) as _,+ Arc::new(Int32Array::from(vec![50, 60, 80, 200])) as _,+ ],+ None,+ );+ let s_arr = StructArray::new(s_fields, vec![Arc::new(outer_arr) as _], None);+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3, 4])),+ Arc::new(s_arr),+ ],+ )+ .unwrap();++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)+ .expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let reader_file = file.reopen().expect("reopen file");+ let parquet_reader_builder =+ ParquetRecordBatchReaderBuilder::try_new(reader_file)+ .expect("reader builder");+ let metadata = parquet_reader_builder.metadata().clone();+ let file_schema = parquet_reader_builder.schema().clone();++ // s['outer']['a'] > 5 AND s['outer']['b'] < 100+ let a_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),+ ])+ .gt(Expr::Literal(ScalarValue::Int32(Some(5)), None));+ let b_expr = get_field()+ .call(vec![+ col("s"),+ Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("b".to_string())), None),+ ])+ .lt(Expr::Literal(ScalarValue::Int32(Some(100)), None));+ let expr = logical2physical(&a_expr.and(b_expr), &file_schema);++ let metrics = ExecutionPlanMetricsSet::new();+ let file_metrics =+ ParquetFileMetrics::new(0, "shared_prefix_e2e.parquet", &metrics);++ let row_filter =+ build_row_filter(&expr, &file_schema, &metadata, false, &file_metrics)+ .expect("building row filter")+ .expect("row filter should exist");++ let reader = parquet_reader_builder+ .with_row_filter(row_filter)+ .build()+ .expect("build reader");++ let mut total_rows = 0;+ for batch in reader {+ let batch = batch.expect("record batch");+ total_rows += batch.num_rows();+ }++ assert_eq!(+ total_rows, 2,+ "expected 2 rows matching s.outer.a > 5 AND s.outer.b < 100"+ );+ assert_eq!(file_metrics.pushdown_rows_pruned.value(), 2);+ assert_eq!(file_metrics.pushdown_rows_matched.value(), 2);+ }}