Apache DataFusion SQL Query Engine
Parquet: Push down supported list predicates (array_has/any/all) during decoding (#19545)
kosiew566bcde9e129
diff --git a/Cargo.lock b/Cargo.lockindex b0c26e27b4..ce09369b0a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2111,12 +2111,14 @@ dependencies = ["async-trait","bytes","chrono",+ "criterion","datafusion-common","datafusion-common-runtime","datafusion-datasource","datafusion-execution","datafusion-expr","datafusion-functions-aggregate-common",+ "datafusion-functions-nested","datafusion-physical-expr","datafusion-physical-expr-adapter","datafusion-physical-expr-common",@@ -2129,6 +2131,7 @@ dependencies = ["object_store","parking_lot","parquet",+ "tempfile","tokio",]diff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.tomlindex a5f6f56ac6..b865422366 100644--- a/datafusion/datasource-parquet/Cargo.toml+++ b/datafusion/datasource-parquet/Cargo.toml@@ -56,6 +56,9 @@ tokio = { workspace = true }[dev-dependencies]chrono = { workspace = true }+criterion = { workspace = true }+datafusion-functions-nested = { workspace = true }+tempfile = { workspace = true }# Note: add additional linter rules in lib.rs.# Rust does not support workspace + new linter rules in subcrates yet@@ -73,3 +76,7 @@ parquet_encryption = ["datafusion-common/parquet_encryption","datafusion-execution/parquet_encryption",]++[[bench]]+name = "parquet_nested_filter_pushdown"+harness = falsediff --git a/datafusion/datasource-parquet/benches/parquet_nested_filter_pushdown.rs b/datafusion/datasource-parquet/benches/parquet_nested_filter_pushdown.rsnew file mode 100644index 0000000000..ed92031f86--- /dev/null+++ b/datafusion/datasource-parquet/benches/parquet_nested_filter_pushdown.rs@@ -0,0 +1,238 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::path::{Path, PathBuf};+use std::sync::{Arc, LazyLock};++use arrow::array::{+ BinaryBuilder, BooleanArray, ListBuilder, RecordBatch, StringBuilder,+};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use criterion::{Criterion, Throughput, criterion_group, criterion_main};+use datafusion_common::ScalarValue;+use datafusion_datasource_parquet::{ParquetFileMetrics, build_row_filter};+use datafusion_expr::{Expr, col};+use datafusion_functions_nested::expr_fn::array_has;+use datafusion_physical_expr::planner::logical2physical;+use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;+use parquet::arrow::{ArrowWriter, ProjectionMask};+use parquet::file::properties::WriterProperties;+use tempfile::TempDir;++const ROW_GROUP_SIZE: usize = 10_000;+const TOTAL_ROW_GROUPS: usize = 10;+const TOTAL_ROWS: usize = ROW_GROUP_SIZE * TOTAL_ROW_GROUPS;+const TARGET_VALUE: &str = "target_value";+const COLUMN_NAME: &str = "list_col";+const PAYLOAD_COLUMN_NAME: &str = "payload";+// Large binary payload to emphasize decoding overhead when pushdown is disabled.+const PAYLOAD_BYTES: usize = 8 * 1024;++struct BenchmarkDataset {+ _tempdir: TempDir,+ file_path: PathBuf,+}++impl BenchmarkDataset {+ fn path(&self) -> &Path {+ &self.file_path+ }+}++static DATASET: LazyLock<BenchmarkDataset> = LazyLock::new(|| {+ create_dataset().expect("failed to prepare parquet benchmark dataset")+});++fn parquet_nested_filter_pushdown(c: &mut Criterion) {+ let dataset_path = DATASET.path().to_owned();+ let mut group = c.benchmark_group("parquet_nested_filter_pushdown");+ group.throughput(Throughput::Elements(TOTAL_ROWS as u64));++ group.bench_function("no_pushdown", |b| {+ let file_schema = setup_reader(&dataset_path);+ let predicate = logical2physical(&create_predicate(), &file_schema);+ b.iter(|| {+ let matched = scan_with_predicate(&dataset_path, &predicate, false)+ .expect("baseline parquet scan with filter succeeded");+ assert_eq!(matched, ROW_GROUP_SIZE);+ });+ });++ group.bench_function("with_pushdown", |b| {+ let file_schema = setup_reader(&dataset_path);+ let predicate = logical2physical(&create_predicate(), &file_schema);+ b.iter(|| {+ let matched = scan_with_predicate(&dataset_path, &predicate, true)+ .expect("pushdown parquet scan with filter succeeded");+ assert_eq!(matched, ROW_GROUP_SIZE);+ });+ });++ group.finish();+}++fn setup_reader(path: &Path) -> SchemaRef {+ let file = std::fs::File::open(path).expect("failed to open file");+ let builder =+ ParquetRecordBatchReaderBuilder::try_new(file).expect("failed to build reader");+ Arc::clone(builder.schema())+}++fn create_predicate() -> Expr {+ array_has(+ col(COLUMN_NAME),+ Expr::Literal(ScalarValue::Utf8(Some(TARGET_VALUE.to_string())), None),+ )+}++fn scan_with_predicate(+ path: &Path,+ predicate: &Arc<dyn datafusion_physical_expr::PhysicalExpr>,+ pushdown: bool,+) -> datafusion_common::Result<usize> {+ let file = std::fs::File::open(path)?;+ let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;+ let metadata = builder.metadata().clone();+ let file_schema = builder.schema();+ let projection = ProjectionMask::all();++ let metrics = ExecutionPlanMetricsSet::new();+ let file_metrics = ParquetFileMetrics::new(0, &path.display().to_string(), &metrics);++ let builder = if pushdown {+ if let Some(row_filter) =+ build_row_filter(predicate, file_schema, &metadata, false, &file_metrics)?+ {+ builder.with_row_filter(row_filter)+ } else {+ builder+ }+ } else {+ builder+ };++ let reader = builder.with_projection(projection).build()?;++ let mut matched_rows = 0usize;+ for batch in reader {+ let batch = batch?;+ matched_rows += count_matches(predicate, &batch)?;+ }++ if pushdown {+ let pruned_rows = file_metrics.pushdown_rows_pruned.value();+ assert_eq!(+ pruned_rows,+ TOTAL_ROWS - matched_rows,+ "row-level pushdown should prune 90% of rows"+ );+ }++ Ok(matched_rows)+}++fn count_matches(+ expr: &Arc<dyn datafusion_physical_expr::PhysicalExpr>,+ batch: &RecordBatch,+) -> datafusion_common::Result<usize> {+ let values = expr.evaluate(batch)?.into_array(batch.num_rows())?;+ let bools = values+ .as_any()+ .downcast_ref::<BooleanArray>()+ .expect("boolean filter result");++ Ok(bools.iter().filter(|v| matches!(v, Some(true))).count())+}++fn create_dataset() -> datafusion_common::Result<BenchmarkDataset> {+ let tempdir = TempDir::new()?;+ let file_path = tempdir.path().join("nested_lists.parquet");++ let field = Arc::new(Field::new("item", DataType::Utf8, true));+ let schema = Arc::new(Schema::new(vec![+ Field::new(COLUMN_NAME, DataType::List(field), false),+ Field::new(PAYLOAD_COLUMN_NAME, DataType::Binary, false),+ ]));++ let writer_props = WriterProperties::builder()+ .set_max_row_group_size(ROW_GROUP_SIZE)+ .build();++ let mut writer = ArrowWriter::try_new(+ std::fs::File::create(&file_path)?,+ Arc::clone(&schema),+ Some(writer_props),+ )?;++ // Create sorted row groups with distinct values so that min/max statistics+ // allow skipping most groups when applying a selective predicate.+ let sorted_values = [+ "alpha",+ "bravo",+ "charlie",+ "delta",+ "echo",+ "foxtrot",+ "golf",+ "hotel",+ "india",+ TARGET_VALUE,+ ];++ for value in sorted_values {+ let batch = build_list_batch(&schema, value, ROW_GROUP_SIZE)?;+ writer.write(&batch)?;+ }++ writer.close()?;++ // Ensure the writer respected the requested row group size+ let reader =+ ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(&file_path)?)?;+ assert_eq!(reader.metadata().row_groups().len(), TOTAL_ROW_GROUPS);++ Ok(BenchmarkDataset {+ _tempdir: tempdir,+ file_path,+ })+}++fn build_list_batch(+ schema: &SchemaRef,+ value: &str,+ len: usize,+) -> datafusion_common::Result<RecordBatch> {+ let mut builder = ListBuilder::new(StringBuilder::new());+ let mut payload_builder = BinaryBuilder::new();+ let payload = vec![1u8; PAYLOAD_BYTES];+ for _ in 0..len {+ builder.values().append_value(value);+ builder.append(true);+ payload_builder.append_value(&payload);+ }++ let array = builder.finish();+ let payload_array = payload_builder.finish();+ Ok(RecordBatch::try_new(+ Arc::clone(schema),+ vec![Arc::new(array), Arc::new(payload_array)],+ )?)+}++criterion_group!(benches, parquet_nested_filter_pushdown);+criterion_main!(benches);diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rsindex eb4cc9e9ad..d7e92f70af 100644--- a/datafusion/datasource-parquet/src/mod.rs+++ b/datafusion/datasource-parquet/src/mod.rs@@ -32,6 +32,7 @@ mod row_filter;mod row_group_filter;mod sort;pub mod source;+mod supported_predicates;mod writer;pub use access_plan::{ParquetAccessPlan, RowGroupAccess};diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rsindex ba3b29be40..04c11b8875 100644--- a/datafusion/datasource-parquet/src/row_filter.rs+++ b/datafusion/datasource-parquet/src/row_filter.rs@@ -58,6 +58,11 @@//! 8. Build the `RowFilter` with the sorted predicates followed by//! the unsorted predicates. Within each partition, predicates are//! still be sorted by size.+//!+//! List-aware predicates (for example, `array_has`, `array_has_all`, and+//! `array_has_any`) can be evaluated directly during Parquet decoding. Struct+//! columns and other nested projections that are not explicitly supported will+//! continue to be evaluated after the batches are materialized.use std::cmp::Ordering;use std::collections::BTreeSet;@@ -70,6 +75,7 @@ use arrow::record_batch::RecordBatch;use parquet::arrow::ProjectionMask;use parquet::arrow::arrow_reader::{ArrowPredicate, RowFilter};use parquet::file::metadata::ParquetMetaData;+use parquet::schema::types::SchemaDescriptor;use datafusion_common::Result;use datafusion_common::cast::as_boolean_array;@@ -81,6 +87,7 @@ use datafusion_physical_expr::{PhysicalExpr, split_conjunction};use datafusion_physical_plan::metrics;use super::ParquetFileMetrics;+use super::supported_predicates::supports_list_predicates;/// A "compiled" predicate passed to `ParquetRecordBatchStream` to perform/// row-level filtering during parquet decoding.@@ -91,12 +98,14 @@ use super::ParquetFileMetrics;////// An expression can be evaluated as a `DatafusionArrowPredicate` if it:/// * Does not reference any projected columns-/// * Does not reference columns with non-primitive types (e.g. structs / lists)+/// * References either primitive columns or list columns used by+/// supported predicates (such as `array_has_all` or NULL checks). Struct+/// columns are still evaluated after decoding.#[derive(Debug)]pub(crate) struct DatafusionArrowPredicate {/// the filter expressionphysical_expr: Arc<dyn PhysicalExpr>,- /// Path to the columns in the parquet schema required to evaluate the+ /// Path to the leaf columns in the parquet schema required to evaluate the/// expressionprojection_mask: ProjectionMask,/// how many rows were filtered out by this predicate@@ -121,9 +130,12 @@ impl DatafusionArrowPredicate {Ok(Self {physical_expr,- projection_mask: ProjectionMask::roots(+ // Use leaf indices: when nested columns are involved, we must specify+ // leaf (primitive) column indices in the Parquet schema so the decoder+ // can properly project and filter nested structures.+ projection_mask: ProjectionMask::leaves(metadata.file_metadata().schema_descr(),- candidate.projection,+ candidate.projection.leaf_indices.iter().copied(),),rows_pruned,rows_matched,@@ -177,12 +189,23 @@ pub(crate) struct FilterCandidate {/// Can this filter use an index (e.g. a page index) to prune rows?can_use_index: bool,/// Column indices into the parquet file schema required to evaluate this filter.- projection: Vec<usize>,+ projection: LeafProjection,/// The Arrow schema containing only the columns required by this filter,/// projected from the file's Arrow schema.filter_schema: SchemaRef,}+/// Projection specification for nested columns using Parquet leaf column indices.+///+/// For nested types like List and Struct, Parquet stores data in leaf columns+/// (the primitive fields). This struct tracks which leaf columns are needed+/// to evaluate a filter expression.+#[derive(Debug, Clone)]+struct LeafProjection {+ /// Leaf column indices in the Parquet schema descriptor.+ leaf_indices: Vec<usize>,+}+/// Helper to build a `FilterCandidate`.////// This will do several things:@@ -212,23 +235,29 @@ impl FilterCandidateBuilder {/// * `Ok(None)` if the expression cannot be used as an ArrowFilter/// * `Err(e)` if an error occurs while building the candidatepub fn build(self, metadata: &ParquetMetaData) -> Result<Option<FilterCandidate>> {- let Some(required_column_indices) =- pushdown_columns(&self.expr, &self.file_schema)?+ let Some(required_columns) = pushdown_columns(&self.expr, &self.file_schema)?else {return Ok(None);};- let projected_schema =- Arc::new(self.file_schema.project(&required_column_indices)?);+ let root_indices: Vec<_> =+ required_columns.required_columns.into_iter().collect();+ let leaf_indices = leaf_indices_for_roots(+ &root_indices,+ metadata.file_metadata().schema_descr(),+ required_columns.nested,+ );- let required_bytes = size_of_columns(&required_column_indices, metadata)?;- let can_use_index = columns_sorted(&required_column_indices, metadata)?;+ let projected_schema = Arc::new(self.file_schema.project(&root_indices)?);++ let required_bytes = size_of_columns(&leaf_indices, metadata)?;+ let can_use_index = columns_sorted(&leaf_indices, metadata)?;Ok(Some(FilterCandidate {expr: self.expr,required_bytes,can_use_index,- projection: required_column_indices,+ projection: LeafProjection { leaf_indices },filter_schema: projected_schema,}))}@@ -238,7 +267,8 @@ impl FilterCandidateBuilder {/// prevent the expression from being pushed down to the parquet decoder.////// An expression cannot be pushed down if it references:-/// - Non-primitive columns (like structs or lists)+/// - Unsupported nested columns (structs or list fields that are not covered by+/// the supported predicate set)/// - Columns that don't exist in the file schemastruct PushdownChecker<'schema> {/// Does the expression require any non-primitive columns (like structs)?@@ -247,34 +277,78 @@ struct PushdownChecker<'schema> {projected_columns: bool,/// Indices into the file schema of columns required to evaluate the expression.required_columns: BTreeSet<usize>,+ /// Tracks the nested column behavior found during traversal.+ nested_behavior: NestedColumnSupport,+ /// Whether nested list columns are supported by the predicate semantics.+ allow_list_columns: bool,/// The Arrow schema of the parquet file.file_schema: &'schema Schema,}impl<'schema> PushdownChecker<'schema> {- fn new(file_schema: &'schema Schema) -> Self {+ fn new(file_schema: &'schema Schema, allow_list_columns: bool) -> Self {Self {non_primitive_columns: false,projected_columns: false,required_columns: BTreeSet::default(),+ nested_behavior: NestedColumnSupport::PrimitiveOnly,+ allow_list_columns,file_schema,}}fn check_single_column(&mut self, column_name: &str) -> Option<TreeNodeRecursion> {- if let Ok(idx) = self.file_schema.index_of(column_name) {- self.required_columns.insert(idx);- if DataType::is_nested(self.file_schema.field(idx).data_type()) {- self.non_primitive_columns = true;+ let idx = match self.file_schema.index_of(column_name) {+ Ok(idx) => idx,+ Err(_) => {+ // Column does not exist in the file schema, so we can't push this down.+ self.projected_columns = true;return Some(TreeNodeRecursion::Jump);}+ };++ self.required_columns.insert(idx);+ let data_type = self.file_schema.field(idx).data_type();++ if DataType::is_nested(data_type) {+ self.handle_nested_type(data_type)} else {- // Column does not exist in the file schema, so we can't push this down.- self.projected_columns = true;- return Some(TreeNodeRecursion::Jump);+ None}+ }- None+ /// Determines whether a nested data type can be pushed down to Parquet decoding.+ ///+ /// Returns `Some(TreeNodeRecursion::Jump)` if the nested type prevents pushdown,+ /// `None` if the type is supported and pushdown can continue.+ fn handle_nested_type(&mut self, data_type: &DataType) -> Option<TreeNodeRecursion> {+ if self.is_nested_type_supported(data_type) {+ // Update to ListsSupported if we haven't encountered unsupported types yet+ if self.nested_behavior == NestedColumnSupport::PrimitiveOnly {+ self.nested_behavior = NestedColumnSupport::ListsSupported;+ }+ None+ } else {+ // Block pushdown for unsupported nested types:+ // - Structs (regardless of predicate support)+ // - Lists without supported predicates+ self.nested_behavior = NestedColumnSupport::Unsupported;+ self.non_primitive_columns = true;+ Some(TreeNodeRecursion::Jump)+ }+ }++ /// Checks if a nested data type is supported for list column pushdown.+ ///+ /// List columns are only supported if:+ /// 1. The data type is a list variant (List, LargeList, or FixedSizeList)+ /// 2. The expression contains supported list predicates (e.g., array_has_all)+ fn is_nested_type_supported(&self, data_type: &DataType) -> bool {+ let is_list = matches!(+ data_type,+ DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)+ );+ self.allow_list_columns && is_list}#[inline]@@ -297,34 +371,138 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {}}+/// Describes the nested column behavior for filter pushdown.+///+/// This enum makes explicit the different states a predicate can be in+/// with respect to nested column handling during Parquet decoding.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum NestedColumnSupport {+ /// Expression references only primitive (non-nested) columns.+ /// These can always be pushed down to the Parquet decoder.+ PrimitiveOnly,+ /// Expression references list columns with supported predicates+ /// (e.g., array_has, array_has_all, IS NULL).+ /// These can be pushed down to the Parquet decoder.+ ListsSupported,+ /// Expression references unsupported nested types (e.g., structs)+ /// or list columns without supported predicates.+ /// These cannot be pushed down and must be evaluated after decoding.+ Unsupported,+}++#[derive(Debug)]+struct PushdownColumns {+ required_columns: BTreeSet<usize>,+ nested: NestedColumnSupport,+}+/// Checks if a given expression can be pushed down to the parquet decoder.///-/// Returns `Some(column_indices)` if the expression can be pushed down,-/// where `column_indices` are the indices into the file schema of all columns+/// Returns `Some(PushdownColumns)` if the expression can be pushed down,+/// where the struct contains the indices into the file schema of all columns/// required to evaluate the expression.////// Returns `None` if the expression cannot be pushed down (e.g., references-/// non-primitive types or columns not in the file).+/// unsupported nested types or columns not in the file).fn pushdown_columns(expr: &Arc<dyn PhysicalExpr>,file_schema: &Schema,-) -> Result<Option<Vec<usize>>> {- let mut checker = PushdownChecker::new(file_schema);+) -> Result<Option<PushdownColumns>> {+ let allow_list_columns = supports_list_predicates(expr);+ let mut checker = PushdownChecker::new(file_schema, allow_list_columns);expr.visit(&mut checker)?;- Ok((!checker.prevents_pushdown())- .then_some(checker.required_columns.into_iter().collect()))+ Ok((!checker.prevents_pushdown()).then_some(PushdownColumns {+ required_columns: checker.required_columns,+ nested: checker.nested_behavior,+ }))+}++fn leaf_indices_for_roots(+ root_indices: &[usize],+ schema_descr: &SchemaDescriptor,+ nested: NestedColumnSupport,+) -> Vec<usize> {+ // For primitive-only columns, root indices ARE the leaf indices+ if nested == NestedColumnSupport::PrimitiveOnly {+ return root_indices.to_vec();+ }++ // For List columns, expand to the single leaf column (item field)+ // For Struct columns (unsupported), this would expand to multiple leaves+ let root_set: BTreeSet<_> = root_indices.iter().copied().collect();++ (0..schema_descr.num_columns())+ .filter(|leaf_idx| {+ root_set.contains(&schema_descr.get_column_root_idx(*leaf_idx))+ })+ .collect()}/// Checks if a predicate expression can be pushed down to the parquet decoder.////// Returns `true` if all columns referenced by the expression:/// - Exist in the provided schema-/// - Are primitive types (not structs, lists, etc.)+/// - Are primitive types OR list columns with supported predicates+/// (e.g., `array_has`, `array_has_all`, `array_has_any`, IS NULL, IS NOT NULL)+/// - Struct columns are not supported and will prevent pushdown////// # Arguments/// * `expr` - The filter expression to check/// * `file_schema` - The Arrow schema of the parquet file (or table schema when/// the file schema is not yet available during planning)+///+/// # Examples+///+/// Primitive column filters can be pushed down:+/// ```ignore+/// use datafusion_expr::{col, Expr};+/// use datafusion_common::ScalarValue;+/// use arrow::datatypes::{DataType, Field, Schema};+/// use std::sync::Arc;+///+/// let schema = Arc::new(Schema::new(vec![+/// Field::new("age", DataType::Int32, false),+/// ]));+///+/// // Primitive filter: can be pushed down+/// let expr = col("age").gt(Expr::Literal(ScalarValue::Int32(Some(30)), None));+/// let expr = logical2physical(&expr, &schema);+/// assert!(can_expr_be_pushed_down_with_schemas(&expr, &schema));+/// ```+///+/// Struct column filters cannot be pushed down:+/// ```ignore+/// use arrow::datatypes::Fields;+///+/// let schema = Arc::new(Schema::new(vec![+/// Field::new("person", DataType::Struct(+/// Fields::from(vec![Field::new("name", DataType::Utf8, true)])+/// ), true),+/// ]));+///+/// // Struct filter: cannot be pushed down+/// let expr = col("person").is_not_null();+/// let expr = logical2physical(&expr, &schema);+/// assert!(!can_expr_be_pushed_down_with_schemas(&expr, &schema));+/// ```+///+/// List column filters with supported predicates can be pushed down:+/// ```ignore+/// use datafusion_functions_nested::expr_fn::{array_has_all, make_array};+///+/// let schema = Arc::new(Schema::new(vec![+/// Field::new("tags", DataType::List(+/// Arc::new(Field::new("item", DataType::Utf8, true))+/// ), true),+/// ]));+///+/// // Array filter with supported predicate: can be pushed down+/// let expr = array_has_all(col("tags"), make_array(vec![+/// Expr::Literal(ScalarValue::Utf8(Some("rust".to_string())), None)+/// ]));+/// let expr = logical2physical(&expr, &schema);+/// assert!(can_expr_be_pushed_down_with_schemas(&expr, &schema));+/// ```pub fn can_expr_be_pushed_down_with_schemas(expr: &Arc<dyn PhysicalExpr>,file_schema: &Schema,@@ -335,7 +513,7 @@ pub fn can_expr_be_pushed_down_with_schemas(}}-/// Calculate the total compressed size of all `Column`'s required for+/// Calculate the total compressed size of all leaf columns required for/// predicate `Expr`.////// This value represents the total amount of IO required to evaluate the@@ -464,21 +642,30 @@ mod test {use super::*;use datafusion_common::ScalarValue;+ use arrow::array::{ListBuilder, StringBuilder};use arrow::datatypes::{Field, TimeUnit::Nanosecond};use datafusion_expr::{Expr, col};+ use datafusion_functions_nested::array_has::{+ array_has_all_udf, array_has_any_udf, array_has_udf,+ };+ use datafusion_functions_nested::expr_fn::{+ array_has, array_has_all, array_has_any, make_array,+ };use datafusion_physical_expr::planner::logical2physical;use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapterFactory,};- use datafusion_physical_plan::metrics::{Count, Time};+ use datafusion_physical_plan::metrics::{Count, ExecutionPlanMetricsSet, Time};+ use parquet::arrow::ArrowWriter;use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;use parquet::arrow::parquet_to_arrow_schema;use parquet::file::reader::{FileReader, SerializedFileReader};+ use tempfile::NamedTempFile;- // We should ignore predicate that read non-primitive columns+ // List predicates used by the decoder should be accepted for pushdown#[test]- fn test_filter_candidate_builder_ignore_complex_types() {+ fn test_filter_candidate_builder_supports_list_types() {let testdata = datafusion_common::test_util::parquet_test_data();let file = std::fs::File::open(format!("{testdata}/list_columns.parquet")).expect("opening file");@@ -496,11 +683,16 @@ mod test {let table_schema = Arc::new(table_schema.clone());+ let list_index = table_schema+ .index_of("int64_list")+ .expect("list column should exist");+let candidate = FilterCandidateBuilder::new(expr, table_schema).build(metadata)- .expect("building candidate");+ .expect("building candidate")+ .expect("list pushdown should be supported");- assert!(candidate.is_none());+ assert_eq!(candidate.projection.leaf_indices, vec![list_index]);}#[test]@@ -590,14 +782,233 @@ mod test {}#[test]- fn nested_data_structures_prevent_pushdown() {+ fn struct_data_structures_prevent_pushdown() {+ let table_schema = Arc::new(Schema::new(vec![Field::new(+ "struct_col",+ DataType::Struct(+ vec![Arc::new(Field::new("a", DataType::Int32, true))].into(),+ ),+ true,+ )]));++ let expr = col("struct_col").is_not_null();+ let expr = logical2physical(&expr, &table_schema);++ assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));+ }++ #[test]+ fn mixed_primitive_and_struct_prevents_pushdown() {+ // Even when a predicate contains both primitive and unsupported nested columns,+ // the entire predicate should not be pushed down because the struct column+ // cannot be evaluated during Parquet decoding.+ let table_schema = Arc::new(Schema::new(vec![+ Field::new(+ "struct_col",+ DataType::Struct(+ vec![Arc::new(Field::new("a", DataType::Int32, true))].into(),+ ),+ true,+ ),+ Field::new("int_col", DataType::Int32, false),+ ]));++ // Expression: (struct_col IS NOT NULL) AND (int_col = 5)+ // Even though int_col is primitive, the presence of struct_col in the+ // conjunction should prevent pushdown of the entire expression.+ let expr = col("struct_col")+ .is_not_null()+ .and(col("int_col").eq(Expr::Literal(ScalarValue::Int32(Some(5)), None)));+ let expr = logical2physical(&expr, &table_schema);++ // The entire expression should not be pushed down+ assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));++ // However, just the int_col predicate alone should be pushable+ let expr_int_only =+ col("int_col").eq(Expr::Literal(ScalarValue::Int32(Some(5)), None));+ let expr_int_only = logical2physical(&expr_int_only, &table_schema);+ assert!(can_expr_be_pushed_down_with_schemas(+ &expr_int_only,+ &table_schema+ ));+ }++ #[test]+ fn nested_lists_allow_pushdown_checks() {let table_schema = Arc::new(get_lists_table_schema());let expr = col("utf8_list").is_not_null();let expr = logical2physical(&expr, &table_schema);check_expression_can_evaluate_against_schema(&expr, &table_schema);- assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));+ assert!(can_expr_be_pushed_down_with_schemas(&expr, &table_schema));+ }++ #[test]+ fn array_has_all_pushdown_filters_rows() {+ // Test array_has_all: checks if array contains all of ["c"]+ // Rows with "c": row 1 and row 2+ let expr = array_has_all(+ col("letters"),+ make_array(vec![Expr::Literal(+ ScalarValue::Utf8(Some("c".to_string())),+ None,+ )]),+ );+ test_array_predicate_pushdown("array_has_all", expr, 1, 2, true);+ }++ /// Helper function to test array predicate pushdown functionality.+ ///+ /// Creates a Parquet file with a list column, applies the given predicate,+ /// and verifies that rows are correctly filtered during decoding.+ fn test_array_predicate_pushdown(+ func_name: &str,+ predicate_expr: Expr,+ expected_pruned: usize,+ expected_matched: usize,+ expect_list_support: bool,+ ) {+ let item_field = Arc::new(Field::new("item", DataType::Utf8, true));+ let schema = Arc::new(Schema::new(vec![Field::new(+ "letters",+ DataType::List(item_field),+ true,+ )]));++ let mut builder = ListBuilder::new(StringBuilder::new());+ // Row 0: ["a", "b"]+ builder.values().append_value("a");+ builder.values().append_value("b");+ builder.append(true);++ // Row 1: ["c"]+ builder.values().append_value("c");+ builder.append(true);++ // Row 2: ["c", "d"]+ builder.values().append_value("c");+ builder.values().append_value("d");+ builder.append(true);++ let batch =+ RecordBatch::try_new(schema.clone(), vec![Arc::new(builder.finish())])+ .expect("record batch");++ let file = NamedTempFile::new().expect("temp file");+ let mut writer =+ ArrowWriter::try_new(file.reopen().unwrap(), schema, None).expect("writer");+ writer.write(&batch).expect("write batch");+ writer.close().expect("close writer");++ let reader_file = file.reopen().expect("reopen file");+ let parquet_reader_builder =+ ParquetRecordBatchReaderBuilder::try_new(reader_file)+ .expect("reader builder");+ let metadata = parquet_reader_builder.metadata().clone();+ let file_schema = parquet_reader_builder.schema().clone();++ let expr = logical2physical(&predicate_expr, &file_schema);+ if expect_list_support {+ assert!(supports_list_predicates(&expr));+ }++ let metrics = ExecutionPlanMetricsSet::new();+ let file_metrics =+ ParquetFileMetrics::new(0, &format!("{func_name}.parquet"), &metrics);++ let row_filter =+ build_row_filter(&expr, &file_schema, &metadata, false, &file_metrics)+ .expect("building row filter")+ .expect("row filter should exist");++ let reader = parquet_reader_builder+ .with_row_filter(row_filter)+ .build()+ .expect("build reader");++ let mut total_rows = 0;+ for batch in reader {+ let batch = batch.expect("record batch");+ total_rows += batch.num_rows();+ }++ assert_eq!(+ file_metrics.pushdown_rows_pruned.value(),+ expected_pruned,+ "{func_name}: expected {expected_pruned} pruned rows"+ );+ assert_eq!(+ file_metrics.pushdown_rows_matched.value(),+ expected_matched,+ "{func_name}: expected {expected_matched} matched rows"+ );+ assert_eq!(+ total_rows, expected_matched,+ "{func_name}: expected {expected_matched} total rows"+ );+ }++ #[test]+ fn array_has_pushdown_filters_rows() {+ // Test array_has: checks if "c" is in the array+ // Rows with "c": row 1 and row 2+ let expr = array_has(+ col("letters"),+ Expr::Literal(ScalarValue::Utf8(Some("c".to_string())), None),+ );+ test_array_predicate_pushdown("array_has", expr, 1, 2, true);+ }++ #[test]+ fn array_has_any_pushdown_filters_rows() {+ // Test array_has_any: checks if array contains any of ["a", "d"]+ // Row 0 has "a", row 2 has "d" - both should match+ let expr = array_has_any(+ col("letters"),+ make_array(vec![+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("d".to_string())), None),+ ]),+ );+ test_array_predicate_pushdown("array_has_any", expr, 1, 2, true);+ }++ #[test]+ fn array_has_udf_pushdown_filters_rows() {+ let expr = array_has_udf().call(vec![+ col("letters"),+ Expr::Literal(ScalarValue::Utf8(Some("c".to_string())), None),+ ]);++ test_array_predicate_pushdown("array_has_udf", expr, 1, 2, true);+ }++ #[test]+ fn array_has_all_udf_pushdown_filters_rows() {+ let expr = array_has_all_udf().call(vec![+ col("letters"),+ make_array(vec![Expr::Literal(+ ScalarValue::Utf8(Some("c".to_string())),+ None,+ )]),+ ]);++ test_array_predicate_pushdown("array_has_all_udf", expr, 1, 2, true);+ }++ #[test]+ fn array_has_any_udf_pushdown_filters_rows() {+ let expr = array_has_any_udf().call(vec![+ col("letters"),+ make_array(vec![+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),+ Expr::Literal(ScalarValue::Utf8(Some("d".to_string())), None),+ ]),+ ]);++ test_array_predicate_pushdown("array_has_any_udf", expr, 1, 2, true);}#[test]diff --git a/datafusion/datasource-parquet/src/supported_predicates.rs b/datafusion/datasource-parquet/src/supported_predicates.rsnew file mode 100644index 0000000000..a205c12dd0--- /dev/null+++ b/datafusion/datasource-parquet/src/supported_predicates.rs@@ -0,0 +1,144 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Registry of physical expressions that support nested list column pushdown+//! to the Parquet decoder.+//!+//! This module provides a trait-based approach for determining which predicates+//! can be safely evaluated on nested list columns during Parquet decoding.++use std::sync::Arc;++use datafusion_physical_expr::expressions::{IsNotNullExpr, IsNullExpr};+use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr};++/// Trait for physical expressions that support list column pushdown during+/// Parquet decoding.+///+/// This trait provides a type-safe mechanism for identifying expressions that+/// can be safely pushed down to the Parquet decoder for evaluation on nested+/// list columns.+///+/// # Implementation Notes+///+/// Expression types in external crates cannot directly implement this trait+/// due to Rust's orphan rules. Instead, we use a blanket implementation that+/// delegates to a registration mechanism.+///+/// # Examples+///+/// ```ignore+/// use datafusion_physical_expr::PhysicalExpr;+/// use datafusion_datasource_parquet::SupportsListPushdown;+///+/// let expr: Arc<dyn PhysicalExpr> = ...;+/// if expr.supports_list_pushdown() {+/// // Can safely push down to Parquet decoder+/// }+/// ```+pub trait SupportsListPushdown {+ /// Returns `true` if this expression supports list column pushdown.+ fn supports_list_pushdown(&self) -> bool;+}++/// Blanket implementation for all physical expressions.+///+/// This delegates to specialized predicates that check whether the concrete+/// expression type is registered as supporting list pushdown. This design+/// allows the trait to work with expression types defined in external crates.+impl SupportsListPushdown for dyn PhysicalExpr {+ fn supports_list_pushdown(&self) -> bool {+ is_null_check(self) || is_supported_scalar_function(self)+ }+}++/// Checks if an expression is a NULL or NOT NULL check.+///+/// These checks are universally supported for all column types.+fn is_null_check(expr: &dyn PhysicalExpr) -> bool {+ expr.as_any().downcast_ref::<IsNullExpr>().is_some()+ || expr.as_any().downcast_ref::<IsNotNullExpr>().is_some()+}++/// Checks if an expression is a scalar function registered for list pushdown.+///+/// Returns `true` if the expression is a `ScalarFunctionExpr` whose function+/// is in the registry of supported operations.+fn is_supported_scalar_function(expr: &dyn PhysicalExpr) -> bool {+ scalar_function_name(expr).is_some_and(|name| {+ // Registry of verified array functions+ matches!(name, "array_has" | "array_has_all" | "array_has_any")+ })+}++fn scalar_function_name(expr: &dyn PhysicalExpr) -> Option<&str> {+ expr.as_any()+ .downcast_ref::<ScalarFunctionExpr>()+ .map(ScalarFunctionExpr::name)+}++/// Checks whether the given physical expression contains a supported nested+/// predicate (for example, `array_has_all`).+///+/// This function recursively traverses the expression tree to determine if+/// any node contains predicates that support list column pushdown to the+/// Parquet decoder.+///+/// # Supported predicates+///+/// - `IS NULL` and `IS NOT NULL` checks on any column type+/// - Array functions: `array_has`, `array_has_all`, `array_has_any`+///+/// # Returns+///+/// `true` if the expression or any of its children contain supported predicates.+pub fn supports_list_predicates(expr: &Arc<dyn PhysicalExpr>) -> bool {+ expr.supports_list_pushdown()+ || expr+ .children()+ .iter()+ .any(|child| supports_list_predicates(child))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_null_check_detection() {+ use datafusion_physical_expr::expressions::Column;++ let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("test", 0));+ assert!(!is_null_check(col_expr.as_ref()));++ // IsNullExpr and IsNotNullExpr detection requires actual instances+ // which need schema setup - tested in integration tests+ }++ #[test]+ fn test_supported_scalar_functions() {+ use datafusion_physical_expr::expressions::Column;++ let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("test", 0));++ // Non-function expressions should return false+ assert!(!is_supported_scalar_function(col_expr.as_ref()));++ // Testing with actual ScalarFunctionExpr requires function setup+ // and is better suited for integration tests+ }+}diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.sltindex 8bb79d5769..aa94e2e2f2 100644--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt@@ -563,3 +563,114 @@ ORDER BY start_timestamp, trace_idLIMIT 1;----2024-10-01T00:00:00++###+# Array function predicate pushdown tests+# These tests verify that array_has, array_has_all, and array_has_any predicates+# are correctly pushed down to the DataSourceExec node+###++# Create test data with array columns+statement ok+COPY (+ SELECT 1 as id, ['rust', 'performance'] as tags+ UNION ALL+ SELECT 2 as id, ['python', 'javascript'] as tags+ UNION ALL+ SELECT 3 as id, ['rust', 'webassembly'] as tags+)+TO 'test_files/scratch/parquet_filter_pushdown/array_data/data.parquet';++statement ok+CREATE EXTERNAL TABLE array_test STORED AS PARQUET LOCATION 'test_files/scratch/parquet_filter_pushdown/array_data/';++statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++# Test array_has predicate pushdown+query I?+SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;+----+1 [rust, performance]+3 [rust, webassembly]++query TT+EXPLAIN SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;+----+logical_plan+01)Sort: array_test.id ASC NULLS LAST+02)--Filter: array_has(array_test.tags, Utf8("rust"))+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has(array_test.tags, Utf8("rust"))]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust)++# Test array_has_all predicate pushdown+query I?+SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;+----+1 [rust, performance]++query TT+EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;+----+logical_plan+01)Sort: array_test.id ASC NULLS LAST+02)--Filter: array_has_all(array_test.tags, List([rust, performance]))+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust, performance]))]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance])++# Test array_has_any predicate pushdown+query I?+SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;+----+2 [python, javascript]++query TT+EXPLAIN SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;+----+logical_plan+01)Sort: array_test.id ASC NULLS LAST+02)--Filter: array_has_any(array_test.tags, List([python, go]))+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_any(array_test.tags, List([python, go]))]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go])++# Test complex predicate with OR+query I?+SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;+----+1 [rust, performance]+2 [python, javascript]+3 [rust, webassembly]++query TT+EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;+----+logical_plan+01)Sort: array_test.id ASC NULLS LAST+02)--Filter: array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go])++# Test array function with other predicates+query I?+SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;+----+3 [rust, webassembly]++query TT+EXPLAIN SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;+----+logical_plan+01)Sort: array_test.id ASC NULLS LAST+02)--Filter: array_test.id > Int64(1) AND array_has(array_test.tags, Utf8("rust"))+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_test.id > Int64(1), array_has(array_test.tags, Utf8("rust"))]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]