Apache DataFusion SQL Query Engine
Push down projection expressions into ParquetOpener (#19111)
Adrian Garcia Badaracco021188e719ee
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 8f521c3f4e..fb9d27527e 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -23,7 +23,7 @@ use crate::{apply_file_schema_type_coercions, coerce_int96_to_resolution, row_filter,ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory,};-use arrow::array::RecordBatch;+use arrow::array::{RecordBatch, RecordBatchOptions};use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::utils::reassign_expr_columns;@@ -66,8 +66,8 @@ use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};pub(super) struct ParquetOpener {/// Execution partition indexpub partition_index: usize,- /// Column indexes in `table_schema` needed by the query- pub projection: Arc<[usize]>,+ /// Projection to apply on top of the table schema (i.e. can reference partition columns).+ pub projection: ProjectionExprs,/// Target number of rows in each output RecordBatchpub batch_size: usize,/// Optional limit on the number of rows to read@@ -104,7 +104,7 @@ pub(super) struct ParquetOpener {#[cfg(feature = "parquet_encryption")]pub file_decryption_properties: Option<Arc<FileDecryptionProperties>>,/// Rewrite expressions in the context of the file schema- pub(crate) expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,+ pub(crate) expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,/// Optional factory to create file decryption properties dynamically#[cfg(feature = "parquet_encryption")]pub encryption_factory:@@ -137,7 +137,7 @@ impl FileOpener for ParquetOpener {let batch_size = self.batch_size;- // Replace any partition column references in the predicate+ // Build partition values map for replacing partition column references// with their literal values from this file's partition values.//// For example, given@@ -157,29 +157,39 @@ impl FileOpener for ParquetOpener {// there are other cases where partition columns may appear in more// complex predicates that cannot be simplified until we are about to// open the file (such as dynamic predicates)- let mut predicate = {- // Only replace partition columns if there are partition values.- // For non-partitioned tables, skip the clone and replacement traversal.- if partitioned_file.partition_values.is_empty() {- self.predicate.clone()- } else {- let partition_values: HashMap<&str, &ScalarValue> = self- .table_schema- .table_partition_cols()- .iter()- .zip(partitioned_file.partition_values.iter())- .map(|(field, value)| (field.name().as_str(), value))- .collect();-- self.predicate- .clone()- .map(|p| replace_columns_with_literals(p, &partition_values))- .transpose()?- }- };+ let partition_values: HashMap<&str, &ScalarValue> = self+ .table_schema+ .table_partition_cols()+ .iter()+ .zip(partitioned_file.partition_values.iter())+ .map(|(field, value)| (field.name().as_str(), value))+ .collect();++ // Calculate the output schema from the original projection (before literal replacement)+ // so we get correct field names from column references+ let logical_file_schema = Arc::clone(self.table_schema.file_schema());+ let output_schema = Arc::new(+ self.projection+ .project_schema(self.table_schema.table_schema())?,+ );- let projection = Arc::clone(&self.projection);- let logical_file_schema = Arc::clone(self.table_schema.table_schema());+ // Apply partition column replacement to projection expressions+ let mut projection = self.projection.clone();+ if !partition_values.is_empty() {+ projection = projection.try_map_exprs(|expr| {+ replace_columns_with_literals(Arc::clone(&expr), &partition_values)+ })?;+ }++ // Apply partition column replacement to predicate+ let mut predicate = if partition_values.is_empty() {+ self.predicate.clone()+ } else {+ self.predicate+ .clone()+ .map(|p| replace_columns_with_literals(p, &partition_values))+ .transpose()?+ };let reorder_predicates = self.reorder_filters;let pushdown_filters = self.pushdown_filters;let force_filter_selections = self.force_filter_selections;@@ -191,8 +201,7 @@ impl FileOpener for ParquetOpener {let predicate_creation_errors = MetricBuilder::new(&self.metrics).global_counter("num_predicate_creation_errors");- let expr_adapter_factory = self.expr_adapter_factory.clone();- let mut predicate_file_schema = Arc::clone(self.table_schema.file_schema());+ let expr_adapter_factory = Arc::clone(&self.expr_adapter_factory);let enable_page_index = self.enable_page_index;#[cfg(feature = "parquet_encryption")]@@ -311,30 +320,29 @@ impl FileOpener for ParquetOpener {}}- // Adapt the predicate to the physical file schema.+ // Adapt the projection & filter predicate to the physical file schema.// This evaluates missing columns and inserts any necessary casts.- if let Some(expr_adapter_factory) = expr_adapter_factory.as_ref() {- predicate = predicate- .map(|p| {- let expr = expr_adapter_factory- .create(- Arc::clone(&logical_file_schema),- Arc::clone(&physical_file_schema),- )- .rewrite(p)?;- // After rewriting to the file schema, further simplifications may be possible.- // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`- // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).- PhysicalExprSimplifier::new(&physical_file_schema).simplify(expr)- })- .transpose()?;- predicate_file_schema = Arc::clone(&physical_file_schema);- }+ // After rewriting to the file schema, further simplifications may be possible.+ // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`+ // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).+ // Additionally, if any casts were inserted we can move casts from the column to the literal side:+ // `CAST(col AS INT) = 5` can become `col = CAST(5 AS <col type>)`, which can be evaluated statically.+ let rewriter = expr_adapter_factory.create(+ Arc::clone(&logical_file_schema),+ Arc::clone(&physical_file_schema),+ );+ let simplifier = PhysicalExprSimplifier::new(&physical_file_schema);+ predicate = predicate+ .map(|p| simplifier.simplify(rewriter.rewrite(p)?))+ .transpose()?;+ // Adapt projections to the physical file schema as well+ projection = projection+ .try_map_exprs(|p| simplifier.simplify(rewriter.rewrite(p)?))?;// Build predicates for this specific filelet (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(predicate.as_ref(),- &predicate_file_schema,+ &physical_file_schema,&predicate_creation_errors,);@@ -358,15 +366,6 @@ impl FileOpener for ParquetOpener {reader_metadata,);- let mut projection =- ProjectionExprs::from_indices(&projection, &logical_file_schema);- if let Some(expr_adapter_factory) = expr_adapter_factory {- let adapter = expr_adapter_factory.create(- Arc::clone(&logical_file_schema),- Arc::clone(&physical_file_schema),- );- projection = projection.try_map_exprs(|expr| adapter.rewrite(expr))?;- }let indices = projection.column_indices();let mask = ProjectionMask::roots(builder.parquet_schema(), indices);@@ -376,7 +375,6 @@ impl FileOpener for ParquetOpener {let row_filter = row_filter::build_row_filter(&predicate,&physical_file_schema,- &predicate_file_schema,builder.metadata(),reorder_predicates,&file_metrics,@@ -507,6 +505,9 @@ impl FileOpener for ParquetOpener {let predicate_cache_records = file_metrics.predicate_cache_records.clone();let stream_schema = Arc::clone(stream.schema());+ // Check if we need to replace the schema to handle things like differing nullability or metadata.+ // See note below about file vs. output schema.+ let replace_schema = !stream_schema.eq(&output_schema);// Rebase column indices to match the narrowed stream schema.// The projection expressions have indices based on physical_file_schema,@@ -517,13 +518,34 @@ impl FileOpener for ParquetOpener {let projector = projection.make_projector(&stream_schema)?;let stream = stream.map_err(DataFusionError::from).map(move |b| {- b.and_then(|b| {+ b.and_then(|mut b| {copy_arrow_reader_metrics(&arrow_reader_metrics,&predicate_cache_inner_records,&predicate_cache_records,);- projector.project_batch(&b)+ b = projector.project_batch(&b)?;+ if replace_schema {+ // Ensure the output batch has the expected schema.+ // This handles things like schema level and field level metadata, which may not be present+ // in the physical file schema.+ // It is also possible for nullability to differ; some writers create files with+ // OPTIONAL fields even when there are no nulls in the data.+ // In these cases it may make sense for the logical schema to be `NOT NULL`.+ // RecordBatch::try_new_with_options checks that if the schema is NOT NULL+ // the array cannot contain nulls, amongst other checks.+ let (_stream_schema, arrays, num_rows) = b.into_parts();+ let options =+ RecordBatchOptions::new().with_row_count(Some(num_rows));+ RecordBatch::try_new_with_options(+ Arc::clone(&output_schema),+ arrays,+ &options,+ )+ .map_err(Into::into)+ } else {+ Ok(b)+ }})});@@ -826,7 +848,8 @@ mod test {use datafusion_datasource::{file_stream::FileOpener, PartitionedFile, TableSchema};use datafusion_expr::{col, lit};use datafusion_physical_expr::{- expressions::DynamicFilterPhysicalExpr, planner::logical2physical, PhysicalExpr,+ expressions::DynamicFilterPhysicalExpr, planner::logical2physical,+ projection::ProjectionExprs, PhysicalExpr,};use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;@@ -910,7 +933,7 @@ mod test {let make_opener = |predicate| {ParquetOpener {partition_index: 0,- projection: Arc::new([0, 1]),+ projection: ProjectionExprs::from_indices(&[0, 1], &schema),batch_size: 1024,limit: None,predicate: Some(predicate),@@ -929,7 +952,7 @@ mod test {coerce_int96: None,#[cfg(feature = "parquet_encryption")]file_decryption_properties: None,- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,@@ -978,7 +1001,7 @@ mod test {let make_opener = |predicate| {ParquetOpener {partition_index: 0,- projection: Arc::new([0]),+ projection: ProjectionExprs::from_indices(&[0], &file_schema),batch_size: 1024,limit: None,predicate: Some(predicate),@@ -1000,7 +1023,7 @@ mod test {coerce_int96: None,#[cfg(feature = "parquet_encryption")]file_decryption_properties: None,- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,@@ -1065,7 +1088,7 @@ mod test {let make_opener = |predicate| {ParquetOpener {partition_index: 0,- projection: Arc::new([0]),+ projection: ProjectionExprs::from_indices(&[0], &file_schema),batch_size: 1024,limit: None,predicate: Some(predicate),@@ -1087,7 +1110,7 @@ mod test {coerce_int96: None,#[cfg(feature = "parquet_encryption")]file_decryption_properties: None,- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,@@ -1155,7 +1178,7 @@ mod test {let make_opener = |predicate| {ParquetOpener {partition_index: 0,- projection: Arc::new([0]),+ projection: ProjectionExprs::from_indices(&[0], &file_schema),batch_size: 1024,limit: None,predicate: Some(predicate),@@ -1177,7 +1200,7 @@ mod test {coerce_int96: None,#[cfg(feature = "parquet_encryption")]file_decryption_properties: None,- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,@@ -1253,7 +1276,7 @@ mod test {let make_opener = |predicate| {ParquetOpener {partition_index: 0,- projection: Arc::new([0]),+ projection: ProjectionExprs::from_indices(&[0], &file_schema),batch_size: 1024,limit: None,predicate: Some(predicate),@@ -1275,7 +1298,7 @@ mod test {coerce_int96: None,#[cfg(feature = "parquet_encryption")]file_decryption_properties: None,- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rsindex 059663c231..04f8d5533e 100644--- a/datafusion/datasource-parquet/src/row_filter.rs+++ b/datafusion/datasource-parquet/src/row_filter.rs@@ -73,7 +73,7 @@ use parquet::file::metadata::ParquetMetaData;use datafusion_common::cast::as_boolean_array;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};-use datafusion_common::{internal_datafusion_err, Result};+use datafusion_common::Result;use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr::utils::reassign_expr_columns;use datafusion_physical_expr::{split_conjunction, PhysicalExpr};@@ -176,42 +176,32 @@ pub(crate) struct FilterCandidate {required_bytes: usize,/// Can this filter use an index (e.g. a page index) to prune rows?can_use_index: bool,- /// The projection to read from the file schema to get the columns- /// required to evaluate the filter expression.+ /// Column indices into the parquet file schema required to evaluate this filter.projection: Vec<usize>,- /// The projected table schema that this filter references+ /// The Arrow schema containing only the columns required by this filter,+ /// projected from the file's Arrow schema.filter_schema: SchemaRef,}/// Helper to build a `FilterCandidate`.///-/// This will do several things+/// This will do several things:/// 1. Determine the columns required to evaluate the expression/// 2. Calculate data required to estimate the cost of evaluating the filter///-/// Note that this does *not* handle any adaptation of the data schema to the expression schema,-/// it is assumed that the expression has already been adapted to the file schema before being passed in here,-/// generally using [`PhysicalExprAdapter`](datafusion_physical_expr_adapter::PhysicalExprAdapter).+/// Note: This does *not* handle any adaptation of the expression to the file schema.+/// The expression must already be adapted before being passed in here, generally using+/// [`PhysicalExprAdapter`](datafusion_physical_expr_adapter::PhysicalExprAdapter).struct FilterCandidateBuilder {expr: Arc<dyn PhysicalExpr>,- /// The schema of this parquet file.+ /// The Arrow schema of this parquet file (the result of converting the+ /// parquet schema to Arrow, potentially with type coercions applied).file_schema: SchemaRef,- /// The schema of the table (merged schema) -- columns may be in different- /// order than in the file and have columns that are not in the file schema- table_schema: SchemaRef,}impl FilterCandidateBuilder {- pub fn new(- expr: Arc<dyn PhysicalExpr>,- file_schema: Arc<Schema>,- table_schema: Arc<Schema>,- ) -> Self {- Self {- expr,- file_schema,- table_schema,- }+ pub fn new(expr: Arc<dyn PhysicalExpr>, file_schema: Arc<Schema>) -> Self {+ Self { expr, file_schema }}/// Attempt to build a `FilterCandidate` from the expression@@ -222,85 +212,64 @@ impl FilterCandidateBuilder {/// * `Ok(None)` if the expression cannot be used as an ArrowFilter/// * `Err(e)` if an error occurs while building the candidatepub fn build(self, metadata: &ParquetMetaData) -> Result<Option<FilterCandidate>> {- let Some(required_indices_into_table_schema) =- pushdown_columns(&self.expr, &self.table_schema)?+ let Some(required_column_indices) =+ pushdown_columns(&self.expr, &self.file_schema)?else {return Ok(None);};- let projected_table_schema = Arc::new(- self.table_schema- .project(&required_indices_into_table_schema)?,- );-- // Compute the projection into the file schema by matching column names- let mut projection_into_file_schema: Vec<usize> = projected_table_schema- .fields()- .iter()- .filter_map(|f| self.file_schema.index_of(f.name()).ok())- .collect();- // Sort and remove duplicates- let original_len = projection_into_file_schema.len();- projection_into_file_schema.sort_unstable();- projection_into_file_schema.dedup();- if projection_into_file_schema.len() < original_len {- // This should not happen, as we built projected_table_schema from- // the table schema which should not have duplicate column names.- return Err(internal_datafusion_err!(- "Duplicate column names found when building filter candidate: {:?}",- projection_into_file_schema- ));- }+ let projected_schema =+ Arc::new(self.file_schema.project(&required_column_indices)?);- let required_bytes = size_of_columns(&projection_into_file_schema, metadata)?;- let can_use_index = columns_sorted(&projection_into_file_schema, metadata)?;+ let required_bytes = size_of_columns(&required_column_indices, metadata)?;+ let can_use_index = columns_sorted(&required_column_indices, metadata)?;Ok(Some(FilterCandidate {expr: self.expr,required_bytes,can_use_index,- projection: projection_into_file_schema,- filter_schema: Arc::clone(&projected_table_schema),+ projection: required_column_indices,+ filter_schema: projected_schema,}))}}-// a struct that implements TreeNodeRewriter to traverse a PhysicalExpr tree structure to determine-// if any column references in the expression would prevent it from being predicate-pushed-down.-// if non_primitive_columns || projected_columns, it can't be pushed down.-// can't be reused between calls to `rewrite`; each construction must be used only once.+/// Traverses a `PhysicalExpr` tree to determine if any column references would+/// prevent the expression from being pushed down to the parquet decoder.+///+/// An expression cannot be pushed down if it references:+/// - Non-primitive columns (like structs or lists)+/// - Columns that don't exist in the file schemastruct PushdownChecker<'schema> {/// Does the expression require any non-primitive columns (like structs)?non_primitive_columns: bool,- /// Does the expression reference any columns that are in the table- /// schema but not in the file schema?- /// This includes partition columns and projected columns.+ /// Does the expression reference any columns not present in the file schema?projected_columns: bool,- // Indices into the table schema of the columns required to evaluate the expression+ /// Indices into the file schema of columns required to evaluate the expression.required_columns: BTreeSet<usize>,- table_schema: &'schema Schema,+ /// The Arrow schema of the parquet file.+ file_schema: &'schema Schema,}impl<'schema> PushdownChecker<'schema> {- fn new(table_schema: &'schema Schema) -> Self {+ fn new(file_schema: &'schema Schema) -> Self {Self {non_primitive_columns: false,projected_columns: false,required_columns: BTreeSet::default(),- table_schema,+ file_schema,}}fn check_single_column(&mut self, column_name: &str) -> Option<TreeNodeRecursion> {- if let Ok(idx) = self.table_schema.index_of(column_name) {+ if let Ok(idx) = self.file_schema.index_of(column_name) {self.required_columns.insert(idx);- if DataType::is_nested(self.table_schema.field(idx).data_type()) {+ if DataType::is_nested(self.file_schema.field(idx).data_type()) {self.non_primitive_columns = true;return Some(TreeNodeRecursion::Jump);}} else {- // If the column does not exist in the (un-projected) table schema then- // it must be a projected column.+ // Column does not exist in the file schema, so we can't push this down.self.projected_columns = true;return Some(TreeNodeRecursion::Jump);}@@ -328,25 +297,34 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {}}-// Checks if a given expression can be pushed down into `DataSourceExec` as opposed to being evaluated-// post-parquet-scan in a `FilterExec`. If it can be pushed down, this returns all the-// columns in the given expression so that they can be used in the parquet scanning, along with the-// expression rewritten as defined in [`PushdownChecker::f_up`]+/// Checks if a given expression can be pushed down to the parquet decoder.+///+/// Returns `Some(column_indices)` if the expression can be pushed down,+/// where `column_indices` are the indices into the file schema of all columns+/// required to evaluate the expression.+///+/// Returns `None` if the expression cannot be pushed down (e.g., references+/// non-primitive types or columns not in the file).fn pushdown_columns(expr: &Arc<dyn PhysicalExpr>,- table_schema: &Schema,+ file_schema: &Schema,) -> Result<Option<Vec<usize>>> {- let mut checker = PushdownChecker::new(table_schema);+ let mut checker = PushdownChecker::new(file_schema);expr.visit(&mut checker)?;Ok((!checker.prevents_pushdown()).then_some(checker.required_columns.into_iter().collect()))}-/// Recurses through expr as a tree, finds all `column`s, and checks if any of them would prevent-/// this expression from being predicate pushed down. If any of them would, this returns false.-/// Otherwise, true.-/// Note that the schema passed in here is *not* the physical file schema (as it is not available at that point in time);-/// it is the schema of the table that this expression is being evaluated against minus any projected columns and partition columns.+/// Checks if a predicate expression can be pushed down to the parquet decoder.+///+/// Returns `true` if all columns referenced by the expression:+/// - Exist in the provided schema+/// - Are primitive types (not structs, lists, etc.)+///+/// # Arguments+/// * `expr` - The filter expression to check+/// * `file_schema` - The Arrow schema of the parquet file (or table schema when+/// the file schema is not yet available during planning)pub fn can_expr_be_pushed_down_with_schemas(expr: &Arc<dyn PhysicalExpr>,file_schema: &Schema,@@ -384,24 +362,30 @@ fn columns_sorted(_columns: &[usize], _metadata: &ParquetMetaData) -> Result<booOk(false)}-/// Build a [`RowFilter`] from the given predicate `Expr` if possible+/// Build a [`RowFilter`] from the given predicate expression if possible.///-/// # returns-/// * `Ok(Some(row_filter))` if the expression can be used as RowFilter-/// * `Ok(None)` if the expression cannot be used as an RowFilter+/// # Arguments+/// * `expr` - The filter predicate, already adapted to reference columns in `file_schema`+/// * `file_schema` - The Arrow schema of the parquet file (the result of converting+/// the parquet schema to Arrow, potentially with type coercions applied)+/// * `metadata` - Parquet file metadata used for cost estimation+/// * `reorder_predicates` - If true, reorder predicates to minimize I/O+/// * `file_metrics` - Metrics for tracking filter performance+///+/// # Returns+/// * `Ok(Some(row_filter))` if the expression can be used as a RowFilter+/// * `Ok(None)` if the expression cannot be used as a RowFilter/// * `Err(e)` if an error occurs while building the filter///-/// Note that the returned `RowFilter` may not contains all conjuncts in the-/// original expression. This is because some conjuncts may not be able to be-/// evaluated as an `ArrowPredicate` and will be ignored.+/// Note: The returned `RowFilter` may not contain all conjuncts from the original+/// expression. Conjuncts that cannot be evaluated as an `ArrowPredicate` are ignored.////// For example, if the expression is `a = 1 AND b = 2 AND c = 3` and `b = 2`-/// can not be evaluated for some reason, the returned `RowFilter` will contain-/// `a = 1` and `c = 3`.+/// cannot be evaluated for some reason, the returned `RowFilter` will contain+/// only `a = 1` and `c = 3`.pub fn build_row_filter(expr: &Arc<dyn PhysicalExpr>,- physical_file_schema: &SchemaRef,- predicate_file_schema: &SchemaRef,+ file_schema: &SchemaRef,metadata: &ParquetMetaData,reorder_predicates: bool,file_metrics: &ParquetFileMetrics,@@ -418,12 +402,8 @@ pub fn build_row_filter(let mut candidates: Vec<FilterCandidate> = predicates.into_iter().map(|expr| {- FilterCandidateBuilder::new(- Arc::clone(expr),- Arc::clone(physical_file_schema),- Arc::clone(predicate_file_schema),- )- .build(metadata)+ FilterCandidateBuilder::new(Arc::clone(expr), Arc::clone(file_schema))+ .build(metadata)}).collect::<Result<Vec<_>, _>>()?.into_iter()@@ -516,10 +496,9 @@ mod test {let table_schema = Arc::new(table_schema.clone());- let candidate =- FilterCandidateBuilder::new(expr, table_schema.clone(), table_schema)- .build(metadata)- .expect("building candidate");+ let candidate = FilterCandidateBuilder::new(expr, table_schema)+ .build(metadata)+ .expect("building candidate");assert!(candidate.is_none());}@@ -553,12 +532,10 @@ mod test {.create(Arc::new(table_schema.clone()), Arc::clone(&file_schema)).rewrite(expr).expect("rewriting expression");- let table_schema = Arc::new(table_schema.clone());- let candidate =- FilterCandidateBuilder::new(expr, file_schema.clone(), table_schema.clone())- .build(&metadata)- .expect("building candidate")- .expect("candidate expected");+ let candidate = FilterCandidateBuilder::new(expr, file_schema.clone())+ .build(&metadata)+ .expect("building candidate")+ .expect("candidate expected");let mut row_filter = DatafusionArrowPredicate::try_new(candidate,@@ -591,10 +568,10 @@ mod test {let expr = logical2physical(&expr, &table_schema);// Rewrite the expression to add CastExpr for type coercionlet expr = DefaultPhysicalExprAdapterFactory {}- .create(table_schema.clone(), Arc::clone(&file_schema))+ .create(Arc::new(table_schema), Arc::clone(&file_schema)).rewrite(expr).expect("rewriting expression");- let candidate = FilterCandidateBuilder::new(expr, file_schema, table_schema)+ let candidate = FilterCandidateBuilder::new(expr, file_schema).build(&metadata).expect("building candidate").expect("candidate expected");diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex d2a140c4f8..c6a58a7c98 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -31,7 +31,6 @@ use datafusion_common::config::ConfigOptions;use datafusion_common::config::EncryptionFactoryOptions;use datafusion_datasource::as_file_source;use datafusion_datasource::file_stream::FileOpener;-use datafusion_datasource::projection::{ProjectionOpener, SplitProjection};use arrow::datatypes::TimeUnit;use datafusion_common::config::TableParquetOptions;@@ -40,6 +39,7 @@ use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::TableSchema;use datafusion_physical_expr::conjunction;+use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;@@ -49,7 +49,6 @@ use datafusion_physical_plan::filter_pushdown::{};use datafusion_physical_plan::metrics::Count;use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;-use datafusion_physical_plan::projection::ProjectionExprs;use datafusion_physical_plan::DisplayFormatType;#[cfg(feature = "parquet_encryption")]@@ -284,8 +283,8 @@ pub struct ParquetSource {pub(crate) batch_size: Option<usize>,/// Optional hint for the size of the parquet metadatapub(crate) metadata_size_hint: Option<usize>,- /// Projection information for column pushdown- pub(crate) projection: SplitProjection,+ /// Projection to apply to the output.+ pub(crate) projection: ProjectionExprs,#[cfg(feature = "parquet_encryption")]pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,}@@ -298,8 +297,11 @@ impl ParquetSource {/// To set custom options, use [ParquetSource::with_table_parquet_options`].pub fn new(table_schema: impl Into<TableSchema>) -> Self {let table_schema = table_schema.into();+ // Projection over the full table schema (file columns + partition columns)+ let full_schema = table_schema.table_schema();+ let indices: Vec<usize> = (0..full_schema.fields().len()).collect();Self {- projection: SplitProjection::unprojected(&table_schema),+ projection: ProjectionExprs::from_indices(&indices, full_schema),table_schema,table_parquet_options: TableParquetOptions::default(),metrics: ExecutionPlanMetricsSet::new(),@@ -497,12 +499,10 @@ impl FileSource for ParquetSource {base_config: &FileScanConfig,partition: usize,) -> datafusion_common::Result<Arc<dyn FileOpener>> {- let split_projection = self.projection.clone();-let expr_adapter_factory = base_config.expr_adapter_factory.clone()- .or_else(|| Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _));+ .unwrap_or_else(|| Arc::new(DefaultPhysicalExprAdapterFactory) as _);let parquet_file_reader_factory =self.parquet_file_reader_factory.clone().unwrap_or_else(|| {@@ -525,9 +525,9 @@ impl FileSource for ParquetSource {.as_ref().map(|time_unit| parse_coerce_int96_string(time_unit.as_str()).unwrap());- let mut opener = Arc::new(ParquetOpener {+ let opener = Arc::new(ParquetOpener {partition_index: partition,- projection: Arc::from(split_projection.file_indices.clone()),+ projection: self.projection.clone(),batch_size: self.batch_size.expect("Batch size must set before creating ParquetOpener"),@@ -550,12 +550,7 @@ impl FileSource for ParquetSource {#[cfg(feature = "parquet_encryption")]encryption_factory: self.get_encryption_factory_with_config(),max_predicate_cache_size: self.max_predicate_cache_size(),- }) as Arc<dyn FileOpener>;- opener = ProjectionOpener::try_new(- split_projection.clone(),- Arc::clone(&opener),- self.table_schema.file_schema(),- )?;+ });Ok(opener)}@@ -582,15 +577,12 @@ impl FileSource for ParquetSource {projection: &ProjectionExprs,) -> datafusion_common::Result<Option<Arc<dyn FileSource>>> {let mut source = self.clone();- let new_projection = self.projection.source.try_merge(projection)?;- let split_projection =- SplitProjection::new(self.table_schema.file_schema(), &new_projection);- source.projection = split_projection;+ source.projection = self.projection.try_merge(projection)?;Ok(Some(Arc::new(source)))}fn projection(&self) -> Option<&ProjectionExprs> {- Some(&self.projection.source)+ Some(&self.projection)}fn metrics(&self) -> &ExecutionPlanMetricsSet {diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex cecc0ef25b..159bd3e4e7 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -229,6 +229,52 @@ let expr_with_literals = replace_columns_with_literals(expr, &partition_values)?let adapted_expr = adapter.rewrite(expr_with_literals)?;```+### `build_row_filter` signature simplified++The `build_row_filter` function in `datafusion-datasource-parquet` has been simplified to take a single schema parameter instead of two.+The expectation is now that the filter has been adapted to the physical file schema (the arrow representation of the parquet file's schema) before being passed to this function+using a `PhysicalExprAdapter` for example.++**Who is affected:**++- Users who call `build_row_filter` directly++**Breaking changes:**++The function signature changed from:++```rust,ignore+pub fn build_row_filter(+ expr: &Arc<dyn PhysicalExpr>,+ physical_file_schema: &SchemaRef,+ predicate_file_schema: &SchemaRef, // removed+ metadata: &ParquetMetaData,+ reorder_predicates: bool,+ file_metrics: &ParquetFileMetrics,+) -> Result<Option<RowFilter>>+```++To:++```rust,ignore+pub fn build_row_filter(+ expr: &Arc<dyn PhysicalExpr>,+ file_schema: &SchemaRef,+ metadata: &ParquetMetaData,+ reorder_predicates: bool,+ file_metrics: &ParquetFileMetrics,+) -> Result<Option<RowFilter>>+```++**Migration guide:**++Remove the duplicate schema parameter from your call:++```diff+- build_row_filter(&predicate, &file_schema, &file_schema, metadata, reorder, metrics)++ build_row_filter(&predicate, &file_schema, metadata, reorder, metrics)+```+### Planner now requires explicit opt-in for WITHIN GROUP syntaxThe SQL planner now enforces the aggregate UDF contract more strictly: the