Skip to content

Apache DataFusion SQL Query Engine

Project only accessed struct leaves in Parquet row filter pushdown (#20854)

Matthew Kim2c871b2f301d
diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rs
index 0c0d516aea..42bdbfb992 100644
--- a/datafusion/datasource-parquet/src/row_filter.rs
+++ b/datafusion/datasource-parquet/src/row_filter.rs
@@ -75,7 +75,7 @@ use std::collections::BTreeSet;
use std::sync::Arc;
use arrow::array::BooleanArray;
-use arrow::datatypes::{DataType, Schema, SchemaRef};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::error::{ArrowError, Result as ArrowResult};
use arrow::record_batch::RecordBatch;
use datafusion_functions::core::getfield::GetFieldFunc;
@@ -88,7 +88,7 @@ use datafusion_common::Result;
use datafusion_common::cast::as_boolean_array;
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};
use datafusion_physical_expr::ScalarFunctionExpr;
-use datafusion_physical_expr::expressions::Column;
+use datafusion_physical_expr::expressions::{Column, Literal};
use datafusion_physical_expr::utils::reassign_expr_columns;
use datafusion_physical_expr::{PhysicalExpr, split_conjunction};
@@ -251,15 +251,26 @@ impl FilterCandidateBuilder {
return Ok(None);
};
+ let schema_descr = metadata.file_metadata().schema_descr();
let root_indices: Vec<_> =
required_columns.required_columns.into_iter().collect();
- let leaf_indices = leaf_indices_for_roots(
- &root_indices,
- metadata.file_metadata().schema_descr(),
+ let mut leaf_indices = leaf_indices_for_roots(&root_indices, schema_descr);
+
+ let struct_leaf_indices = resolve_struct_field_leaves(
+ &required_columns.struct_field_accesses,
+ &self.file_schema,
+ schema_descr,
);
+ leaf_indices.extend_from_slice(&struct_leaf_indices);
+ leaf_indices.sort_unstable();
+ leaf_indices.dedup();
- let projected_schema = Arc::new(self.file_schema.project(&root_indices)?);
+ let projected_schema = build_filter_schema(
+ &self.file_schema,
+ &root_indices,
+ &required_columns.struct_field_accesses,
+ );
let required_bytes = size_of_columns(&leaf_indices, metadata)?;
let can_use_index = columns_sorted(&leaf_indices, metadata)?;
@@ -290,7 +301,10 @@ struct PushdownChecker<'schema> {
/// Does the expression reference any columns not present in the file schema?
projected_columns: bool,
/// Indices into the file schema of columns required to evaluate the expression.
+ /// Does not include struct columns accessed via `get_field`.
required_columns: Vec<usize>,
+ /// Struct field accesses via `get_field`.
+ struct_field_accesses: Vec<StructFieldAccess>,
/// Whether nested list columns are supported by the predicate semantics.
allow_list_columns: bool,
/// The Arrow schema of the parquet file.
@@ -303,6 +317,7 @@ impl<'schema> PushdownChecker<'schema> {
non_primitive_columns: false,
projected_columns: false,
required_columns: Vec::new(),
+ struct_field_accesses: Vec::new(),
allow_list_columns,
file_schema,
}
@@ -330,16 +345,17 @@ impl<'schema> PushdownChecker<'schema> {
fn check_struct_field_column(
&mut self,
column_name: &str,
+ field_path: Vec<String>,
) -> Option<TreeNodeRecursion> {
- let idx = match self.file_schema.index_of(column_name) {
- Ok(idx) => idx,
- Err(_) => {
- self.projected_columns = true;
- return Some(TreeNodeRecursion::Jump);
- }
+ let Ok(idx) = self.file_schema.index_of(column_name) else {
+ self.projected_columns = true;
+ return Some(TreeNodeRecursion::Jump);
};
- self.required_columns.push(idx);
+ self.struct_field_accesses.push(StructFieldAccess {
+ root_index: idx,
+ field_path,
+ });
None
}
@@ -410,6 +426,7 @@ impl<'schema> PushdownChecker<'schema> {
self.required_columns.dedup();
PushdownColumns {
required_columns: self.required_columns,
+ struct_field_accesses: self.struct_field_accesses,
}
}
}
@@ -448,10 +465,39 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
.and_then(|a| a.as_any().downcast_ref::<Column>())
{
let return_type = func.return_type();
- if !DataType::is_nested(return_type) {
- if let Some(recursion) = self.check_struct_field_column(column.name())
- {
- return Ok(recursion);
+
+ if !DataType::is_nested(return_type)
+ || self.is_nested_type_supported(return_type)
+ {
+ // try to resolve all field name arguments to strinrg literals
+ // if any argument is not a string literal, we can not determine the exact
+ // leaf path so we fall back to reading the entire struct root column
+ let field_path = args[1..]
+ .iter()
+ .map(|arg| {
+ arg.as_any().downcast_ref::<Literal>().and_then(|lit| {
+ lit.value().try_as_str().flatten().map(|s| s.to_string())
+ })
+ })
+ .collect();
+
+ match field_path {
+ Some(path) => {
+ if let Some(recursion) =
+ self.check_struct_field_column(column.name(), path)
+ {
+ return Ok(recursion);
+ }
+ }
+ None => {
+ // Could not resolve field path — fall back to
+ // reading the entire struct root column.
+ if let Some(recursion) =
+ self.check_single_column(column.name())
+ {
+ return Ok(recursion);
+ }
+ }
}
return Ok(TreeNodeRecursion::Jump);
@@ -478,8 +524,24 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
struct PushdownColumns {
/// Sorted, unique column indices into the file schema required to evaluate
/// the filter expression. Must be in ascending order for correct schema
- /// projection matching.
+ /// projection matching. Does not include struct columns accessed via `get_field`.
required_columns: Vec<usize>,
+ /// Struct field accesses via `get_field`. Each entry records the root struct
+ /// column index and the field path being accessed.
+ struct_field_accesses: Vec<StructFieldAccess>,
+}
+
+/// Records a struct field access via `get_field(struct_col, 'field1', 'field2', ...)`.
+///
+/// This allows the row filter to project only the specific Parquet leaf columns
+/// needed by the filter, rather than all leaves of the struct.
+#[derive(Debug, Clone)]
+struct StructFieldAccess {
+ /// Arrow root column index of the struct in the file schema.
+ root_index: usize,
+ /// Field names forming the path into the struct.
+ /// e.g., `["value"]` for `s['value']`, `["outer", "inner"]` for `s['outer']['inner']`.
+ field_path: Vec<String>,
}
/// Checks if a given expression can be pushed down to the parquet decoder.
@@ -519,6 +581,149 @@ fn leaf_indices_for_roots(
.collect()
}
+/// Resolves struct field access to specific Parquet leaf column indices
+///
+/// For every `StructFieldAccess`, finds the leaf columns in the Parquet schema
+/// whose path matches the struct root name + field path. This avoids reading all
+/// leaves of a struct when only specific fields are needed
+fn resolve_struct_field_leaves(
+ accesses: &[StructFieldAccess],
+ file_schema: &Schema,
+ schema_descr: &SchemaDescriptor,
+) -> Vec<usize> {
+ let mut leaf_indices = Vec::new();
+
+ for access in accesses {
+ let root_name = file_schema.field(access.root_index).name();
+ let prefix = std::iter::once(root_name.as_str())
+ .chain(access.field_path.iter().map(|p| p.as_str()))
+ .collect::<Vec<_>>();
+
+ for leaf_idx in 0..schema_descr.num_columns() {
+ let col = schema_descr.column(leaf_idx);
+ let col_path = col.path().parts();
+
+ // A leaf matches if its path starts with our prefix.
+ // e.g., prefix=["s", "value"] matches leaf path ["s", "value"]
+ // prefix=["s", "outer"] matches ["s", "outer", "inner"]
+
+ // a leaf matches iff its path starts with our prefix
+ // for example: prefix=["s", "value"] matches leaf path ["s", "value"]
+ // prefix=["s", "outer"] matches ["s", "outer", "inner"]
+ let leaf_matches_path = col_path.len() >= prefix.len()
+ && col_path.iter().zip(prefix.iter()).all(|(a, b)| a == b);
+
+ if leaf_matches_path {
+ leaf_indices.push(leaf_idx);
+ }
+ }
+ }
+
+ leaf_indices
+}
+
+/// Builds a filter schema that includes only the fields actually accessed by the
+/// filter expression.
+///
+/// For regular (non-struct) columns, the full field type is used.
+/// For struct columns accessed via `get_field`, a pruned struct type is created
+/// containing only the fields along the access path. Note: it must match the schema
+/// that the Parquet reader produces when projecting specific struct leaves
+fn build_filter_schema(
+ file_schema: &Schema,
+ regular_indices: &[usize],
+ struct_field_accesses: &[StructFieldAccess],
+) -> SchemaRef {
+ let all_indices = regular_indices
+ .iter()
+ .copied()
+ .chain(
+ struct_field_accesses
+ .iter()
+ .map(|&StructFieldAccess { root_index, .. }| root_index),
+ )
+ .collect::<BTreeSet<_>>();
+
+ let fields = all_indices
+ .iter()
+ .map(|&idx| {
+ let field = file_schema.field(idx);
+
+ // collect all field paths that access this root struct column
+ let field_paths = struct_field_accesses
+ .iter()
+ .filter_map(
+ |&StructFieldAccess {
+ root_index,
+ ref field_path,
+ }| {
+ (root_index == idx).then_some(field_path.as_slice())
+ },
+ )
+ .collect::<Vec<_>>();
+
+ if field_paths.is_empty() {
+ // its a regular column - use the full type
+ return Arc::new(field.clone());
+ }
+
+ let pruned_data_type = prune_struct_type(field.data_type(), &field_paths);
+ Arc::new(Field::new(
+ field.name(),
+ pruned_data_type,
+ field.is_nullable(),
+ ))
+ })
+ .collect::<Vec<_>>();
+
+ Arc::new(Schema::new(fields))
+}
+
+fn prune_struct_type(dt: &DataType, paths: &[&[String]]) -> DataType {
+ let DataType::Struct(fields) = dt else {
+ return dt.clone();
+ };
+
+ let needed = paths
+ .iter()
+ .filter_map(|p| p.first().map(|s| s.as_str()))
+ .collect::<BTreeSet<_>>();
+
+ let pruned_fields = fields
+ .iter()
+ .filter_map(|f| {
+ if !needed.contains(f.name().as_str()) {
+ return None;
+ }
+
+ let sub_paths = paths
+ .iter()
+ .filter_map(|path| {
+ if path.first().map(|s| s.as_str()) == Some(f.name()) {
+ Some(&path[1..])
+ } else {
+ None
+ }
+ })
+ .filter(|sub| !sub.is_empty())
+ .collect::<Vec<_>>();
+
+ let out = if sub_paths.is_empty() {
+ // Leaf of access path — keep the field as-is.
+ Arc::clone(f)
+ } else {
+ // Recurse into nested struct.
+ let pruned = prune_struct_type(f.data_type(), &sub_paths);
+ Arc::new(Field::new(f.name(), pruned, f.is_nullable()))
+ };
+
+ Some(out)
+ })
+ .collect::<Vec<_>>();
+
+ DataType::Struct(pruned_fields.into())
+}
+
/// Checks if a predicate expression can be pushed down to the parquet decoder.
///
/// Returns `true` if all columns referenced by the expression:
@@ -722,11 +927,15 @@ pub fn build_row_filter(
#[cfg(test)]
mod test {
use super::*;
+ use arrow::datatypes::Fields;
use datafusion_common::ScalarValue;
- use arrow::array::{ListBuilder, StringBuilder};
+ use arrow::array::{
+ Int32Array, ListBuilder, StringArray, StringBuilder, StructArray,
+ };
use arrow::datatypes::{Field, TimeUnit::Nanosecond};
use datafusion_expr::{Expr, col};
+ use datafusion_functions::core::get_field;
use datafusion_functions_nested::array_has::{
array_has_all_udf, array_has_any_udf, array_has_udf,
};
@@ -1250,7 +1459,7 @@ mod test {
)]));
// get_field(struct_col, 'a') > 5
- let get_field_expr = datafusion_functions::core::get_field().call(vec![
+ let get_field_expr = get_field().call(vec![
col("struct_col"),
Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),
]);
@@ -1275,7 +1484,7 @@ mod test {
)]));
// get_field(struct_col, 'nested') IS NOT NULL — the leaf is still a struct
- let get_field_expr = datafusion_functions::core::get_field().call(vec![
+ let get_field_expr = get_field().call(vec![
col("struct_col"),
Expr::Literal(ScalarValue::Utf8(Some("nested".to_string())), None),
]);
@@ -1285,6 +1494,41 @@ mod test {
assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
}
+ /// get_field returning a list inside a struct should allow pushdown when
+ /// wrapped in a supported list predicate like `array_has_any`.
+ /// e.g. `array_has_any(get_field(s, 'items'), make_array('x'))`
+ #[test]
+ fn get_field_list_leaf_with_array_predicate_allows_pushdown() {
+ let item_field = Arc::new(Field::new("item", DataType::Utf8, true));
+ let table_schema = Arc::new(Schema::new(vec![Field::new(
+ "s",
+ DataType::Struct(
+ vec![
+ Arc::new(Field::new("id", DataType::Int32, true)),
+ Arc::new(Field::new("items", DataType::List(item_field), true)),
+ ]
+ .into(),
+ ),
+ true,
+ )]));
+
+ // array_has_any(get_field(s, 'items'), make_array('x'))
+ let get_field_expr = get_field().call(vec![
+ col("s"),
+ Expr::Literal(ScalarValue::Utf8(Some("items".to_string())), None),
+ ]);
+ let expr = array_has_any(
+ get_field_expr,
+ make_array(vec![Expr::Literal(
+ ScalarValue::Utf8(Some("x".to_string())),
+ None,
+ )]),
+ );
+ let expr = logical2physical(&expr, &table_schema);
+
+ assert!(can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
+ }
+
/// get_field on a struct produces correct Parquet leaf indices.
#[test]
fn get_field_filter_candidate_has_correct_leaf_indices() {
@@ -1292,7 +1536,7 @@ mod test {
// Schema: id (Int32), s (Struct{value: Int32, label: Utf8})
// Parquet leaves: id=0, s.value=1, s.label=2
- let struct_fields: arrow::datatypes::Fields = vec![
+ let struct_fields: Fields = vec![
Arc::new(Field::new("value", DataType::Int32, false)),
Arc::new(Field::new("label", DataType::Utf8, false)),
]
@@ -1332,7 +1576,7 @@ mod test {
let file_schema = builder.schema().clone();
// get_field(s, 'value') > 5
- let get_field_expr = datafusion_functions::core::get_field().call(vec![
+ let get_field_expr = get_field().call(vec![
col("s"),
Expr::Literal(ScalarValue::Utf8(Some("value".to_string())), None),
]);
@@ -1344,12 +1588,12 @@ mod test {
.expect("building candidate")
.expect("get_field filter on struct should be pushable");
- // The root column is s (Arrow index 1), which expands to Parquet
- // leaves 1 (s.value) and 2 (s.label).
+ // The filter accesses only s.value, so only Parquet leaf 1 is needed.
+ // Leaf 2 (s.label) is not read, reducing unnecessary I/O.
assert_eq!(
candidate.projection.leaf_indices,
- vec![1, 2],
- "leaf_indices should contain both leaves of struct s"
+ vec![1],
+ "leaf_indices should contain only the accessed struct field leaf"
);
}
@@ -1374,7 +1618,7 @@ mod test {
)]));
// s['outer']['inner'] > 5
- let get_field_expr = datafusion_functions::core::get_field().call(vec![
+ let get_field_expr = get_field().call(vec![
col("s"),
Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),
Expr::Literal(ScalarValue::Utf8(Some("inner".to_string())), None),
@@ -1389,17 +1633,23 @@ mod test {
/// and the filter actually works against a Parquet file.
#[test]
fn get_field_deeply_nested_filter_candidate() {
- use arrow::array::{Int32Array, StructArray};
-
- // Schema: id (Int32), s (Struct{outer: Struct{inner: Int32}})
- // Parquet leaves: id=0, s.outer.inner=1
- let inner_fields: arrow::datatypes::Fields =
- vec![Arc::new(Field::new("inner", DataType::Int32, false))].into();
- let outer_fields: arrow::datatypes::Fields = vec![Arc::new(Field::new(
- "outer",
- DataType::Struct(inner_fields.clone()),
- false,
- ))]
+ use arrow::array::{Int32Array, StringArray, StructArray};
+
+ // Schema: id (Int32), s (Struct{outer: Struct{extra: Int32, inner: Int32}, tag: Utf8})
+ // Parquet leaves: id=0, s.outer.extra=1, s.outer.inner=2, s.tag=3
+ let inner_fields: Fields = vec![
+ Arc::new(Field::new("extra", DataType::Int32, false)),
+ Arc::new(Field::new("inner", DataType::Int32, false)),
+ ]
+ .into();
+ let outer_fields: Fields = vec![
+ Arc::new(Field::new(
+ "outer",
+ DataType::Struct(inner_fields.clone()),
+ false,
+ )),
+ Arc::new(Field::new("tag", DataType::Utf8, false)),
+ ]
.into();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
@@ -1408,11 +1658,20 @@ mod test {
let inner_struct = StructArray::new(
inner_fields,
- vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as _],
+ vec![
+ Arc::new(Int32Array::from(vec![100, 200, 300])) as _,
+ Arc::new(Int32Array::from(vec![10, 20, 30])) as _,
+ ],
+ None,
+ );
+ let outer_struct = StructArray::new(
+ outer_fields,
+ vec![
+ Arc::new(inner_struct) as _,
+ Arc::new(StringArray::from(vec!["x", "y", "z"])) as _,
+ ],
None,
);
- let outer_struct =
- StructArray::new(outer_fields, vec![Arc::new(inner_struct) as _], None);
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![
@@ -1435,11 +1694,12 @@ mod test {
let metadata = builder.metadata().clone();
let file_schema = builder.schema().clone();
- // Parquet should have 2 leaves: id=0, s.outer.inner=1
- assert_eq!(metadata.file_metadata().schema_descr().num_columns(), 2);
+ // Parquet should have 4 leaves: id=0, s.outer.extra=1, s.outer.inner=2, s.tag=3
+ assert_eq!(metadata.file_metadata().schema_descr().num_columns(), 4);
// get_field(s, 'outer', 'inner') > 15
- let get_field_expr = datafusion_functions::core::get_field().call(vec![
+ // Should only need leaf 2 (s.outer.inner), not leaf 1 (s.outer.extra) or leaf 3 (s.tag).
+ let get_field_expr = get_field().call(vec![
col("s"),
Expr::Literal(ScalarValue::Utf8(Some("outer".to_string())), None),
Expr::Literal(ScalarValue::Utf8(Some("inner".to_string())), None),
@@ -1452,14 +1712,101 @@ mod test {
.expect("building candidate")
.expect("deeply nested get_field filter should be pushable");
- // Root column is s (Arrow index 1), which has one leaf: s.outer.inner=1
+ // Only s.outer.inner (leaf 2) should be projected,
assert_eq!(
candidate.projection.leaf_indices,
- vec![1],
- "leaf_indices should be [1] for s.outer.inner"
+ vec![2],
+ "leaf_indices should be [2] for s.outer.inner, skipping sibling and cousin leaves"
);
}
+ /// End-to-end: get_field filter on a struct column with multiple fields
+ /// reads only the needed leaf and correctly filters rows during Parquet decoding.
+ #[test]
+ fn get_field_end_to_end_filters_rows() {
+ // Schema: id (Int32), s (Struct{value: Int32, label: Utf8})
+ // Parquet leaves: id=0, s.value=1, s.label=2
+ let struct_fields: Fields = vec![
+ Arc::new(Field::new("value", DataType::Int32, false)),
+ Arc::new(Field::new("label", DataType::Utf8, false)),
+ ]
+ .into();
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("s", DataType::Struct(struct_fields.clone()), false),
+ ]));
+
+ // +----+--------------------------+
+ // | id | s |
+ // +----+--------------------------+
+ // | 1 | {value: 10, label: "a"} |
+ // | 2 | {value: 20, label: "b"} |
+ // | 3 | {value: 30, label: "c"} |
+ // +----+--------------------------+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3])),
+ Arc::new(StructArray::new(
+ struct_fields,
+ vec![
+ Arc::new(Int32Array::from(vec![10, 20, 30])) as _,
+ Arc::new(StringArray::from(vec!["a", "b", "c"])) as _,
+ ],
+ None,
+ )),
+ ],
+ )
+ .unwrap();
+
+ let file = NamedTempFile::new().expect("temp file");
+ let mut writer =
+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)
+ .expect("writer");
+ writer.write(&batch).expect("write batch");
+ writer.close().expect("close writer");
+
+ let reader_file = file.reopen().expect("reopen file");
+ let parquet_reader_builder =
+ ParquetRecordBatchReaderBuilder::try_new(reader_file)
+ .expect("reader builder");
+ let metadata = parquet_reader_builder.metadata().clone();
+ let file_schema = parquet_reader_builder.schema().clone();
+
+ // get_field(s, 'value') > 15 — should match rows with value=20 and value=30
+ let get_field_expr = get_field().call(vec![
+ col("s"),
+ Expr::Literal(ScalarValue::Utf8(Some("value".to_string())), None),
+ ]);
+ let predicate_expr =
+ get_field_expr.gt(Expr::Literal(ScalarValue::Int32(Some(15)), None));
+ let expr = logical2physical(&predicate_expr, &file_schema);
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let file_metrics = ParquetFileMetrics::new(0, "struct_e2e.parquet", &metrics);
+
+ let row_filter =
+ build_row_filter(&expr, &file_schema, &metadata, false, &file_metrics)
+ .expect("building row filter")
+ .expect("row filter should exist");
+
+ let reader = parquet_reader_builder
+ .with_row_filter(row_filter)
+ .build()
+ .expect("build reader");
+
+ let mut total_rows = 0;
+ for batch in reader {
+ let batch = batch.expect("record batch");
+ total_rows += batch.num_rows();
+ }
+
+ assert_eq!(total_rows, 2, "expected 2 rows matching value > 15");
+ assert_eq!(file_metrics.pushdown_rows_pruned.value(), 1);
+ assert_eq!(file_metrics.pushdown_rows_matched.value(), 2);
+ }
+
/// Sanity check that the given expression could be evaluated against the given schema without any errors.
/// This will fail if the expression references columns that are not in the schema or if the types of the columns are incompatible, etc.
fn check_expression_can_evaluate_against_schema(