Apache DataFusion SQL Query Engine
Move partition handling out of PhysicalExprAdapter (#19128)
Adrian Garcia Badaraccodc786138cfa0
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rsindex ada3f72f45..80b88f7a38 100644--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs@@ -20,12 +20,12 @@use std::sync::Arc;use arrow::array::{record_batch, RecordBatch};-use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::assert_batches_eq;use datafusion::common::not_impl_err;use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion::common::{Result, ScalarValue};+use datafusion::common::Result;use datafusion::datasource::listing::{ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,};@@ -209,14 +209,4 @@ impl PhysicalExprAdapter for CustomCastsPhysicalExprAdapter {}).data()}-- fn with_partition_values(- &self,- partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter> {- Arc::new(Self {- inner: self.inner.with_partition_values(partition_values),- ..self.clone()- })- }}diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rsindex 9fe8165025..7381dc938a 100644--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs@@ -22,13 +22,12 @@ use std::collections::HashMap;use std::sync::Arc;use arrow::array::RecordBatch;-use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use async_trait::async_trait;use datafusion::assert_batches_eq;use datafusion::catalog::memory::DataSourceExec;use datafusion::catalog::{Session, TableProvider};-use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion::common::DFSchema;use datafusion::common::{Result, ScalarValue};use datafusion::datasource::listing::PartitionedFile;@@ -39,12 +38,12 @@ use datafusion::logical_expr::utils::conjunction;use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType};use datafusion::parquet::arrow::ArrowWriter;use datafusion::parquet::file::properties::WriterProperties;-use datafusion::physical_expr::expressions::{CastExpr, Column, Literal};use datafusion::physical_expr::PhysicalExpr;use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::{lit, SessionConfig};use datafusion_physical_expr_adapter::{- DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,+ replace_columns_with_literals, DefaultPhysicalExprAdapterFactory,+ PhysicalExprAdapter, PhysicalExprAdapterFactory,};use futures::StreamExt;use object_store::memory::InMemory;@@ -60,16 +59,16 @@ const DEFAULT_VALUE_METADATA_KEY: &str = "example.default_value";/// This example demonstrates how to:/// 1. Store default values in field metadata using a constant key/// 2. Create a custom PhysicalExprAdapter that reads these defaults-/// 3. Inject default values for missing columns in filter predicates+/// 3. Inject default values for missing columns in filter predicates using `replace_columns_with_literals`/// 4. Use the DefaultPhysicalExprAdapter as a fallback for standard schema adaptation-/// 5. Wrap string default values in cast expressions for proper type conversion+/// 5. Convert string default values to proper types using `ScalarValue::cast_to()` at planning time////// Important: PhysicalExprAdapter is specifically designed for rewriting filter predicates/// that get pushed down to file scans. For handling missing columns in projections,/// other mechanisms in DataFusion are used (like SchemaAdapter).////// The metadata-based approach provides a flexible way to store default values as strings-/// and cast them to the appropriate types at query time.+/// and cast them to the appropriate types at planning time, avoiding runtime overhead.pub async fn default_column_values() -> Result<()> {println!("=== Creating example data with missing columns and default values ===");@@ -138,8 +137,8 @@ pub async fn default_column_values() -> Result<()> {println!("This example demonstrates how PhysicalExprAdapter works:");println!("1. Physical schema only has 'id' and 'name' columns");println!("2. Logical schema has 'id', 'name', 'status', and 'priority' columns with defaults");- println!("3. Our custom adapter intercepts filter expressions on missing columns");- println!("4. Default values from metadata are injected as cast expressions");+ println!("3. Our custom adapter uses replace_columns_with_literals to inject default values");+ println!("4. Default values from metadata are cast to proper types at planning time");println!("5. The DefaultPhysicalExprAdapter handles other schema adaptations");println!("\nNote: PhysicalExprAdapter is specifically for filter predicates.");println!("For projection columns, different mechanisms handle missing columns.");@@ -206,7 +205,7 @@ impl TableProvider for DefaultValueTableProvider {}fn schema(&self) -> SchemaRef {- self.schema.clone()+ Arc::clone(&self.schema)}fn table_type(&self) -> TableType {@@ -227,7 +226,7 @@ impl TableProvider for DefaultValueTableProvider {filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {- let schema = self.schema.clone();+ let schema = Arc::clone(&self.schema);let df_schema = DFSchema::try_from(schema.clone())?;let filter = state.create_physical_expr(conjunction(filters.iter().cloned()).unwrap_or_else(|| lit(true)),@@ -280,14 +279,15 @@ impl PhysicalExprAdapterFactory for DefaultValuePhysicalExprAdapterFactory {physical_file_schema: SchemaRef,) -> Arc<dyn PhysicalExprAdapter> {let default_factory = DefaultPhysicalExprAdapterFactory;- let default_adapter = default_factory- .create(logical_file_schema.clone(), physical_file_schema.clone());+ let default_adapter = default_factory.create(+ Arc::clone(&logical_file_schema),+ Arc::clone(&physical_file_schema),+ );Arc::new(DefaultValuePhysicalExprAdapter {logical_file_schema,physical_file_schema,default_adapter,- partition_values: Vec::new(),})}}@@ -299,98 +299,36 @@ struct DefaultValuePhysicalExprAdapter {logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,default_adapter: Arc<dyn PhysicalExprAdapter>,- partition_values: Vec<(FieldRef, ScalarValue)>,}impl PhysicalExprAdapter for DefaultValuePhysicalExprAdapter {fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {- // First try our custom default value injection for missing columns- let rewritten = expr- .transform(|expr| {- self.inject_default_values(- expr,- &self.logical_file_schema,- &self.physical_file_schema,- )- })- .data()?;-- // Then apply the default adapter as a fallback to handle standard schema differences- // like type casting, partition column handling, etc.- let default_adapter = if !self.partition_values.is_empty() {- self.default_adapter- .with_partition_values(self.partition_values.clone())- } else {- self.default_adapter.clone()- };-- default_adapter.rewrite(rewritten)- }-- fn with_partition_values(- &self,- partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter> {- Arc::new(DefaultValuePhysicalExprAdapter {- logical_file_schema: self.logical_file_schema.clone(),- physical_file_schema: self.physical_file_schema.clone(),- default_adapter: self.default_adapter.clone(),- partition_values,- })- }-}--impl DefaultValuePhysicalExprAdapter {- fn inject_default_values(- &self,- expr: Arc<dyn PhysicalExpr>,- logical_file_schema: &Schema,- physical_file_schema: &Schema,- ) -> Result<Transformed<Arc<dyn PhysicalExpr>>> {- if let Some(column) = expr.as_any().downcast_ref::<Column>() {- let column_name = column.name();-- // Check if this column exists in the physical schema- if physical_file_schema.index_of(column_name).is_err() {- // Column is missing from physical schema, check if logical schema has a default- if let Ok(logical_field) =- logical_file_schema.field_with_name(column_name)- {- if let Some(default_value_str) =- logical_field.metadata().get(DEFAULT_VALUE_METADATA_KEY)- {- // Create a string literal and wrap it in a cast expression- let default_literal = self.create_default_value_expr(- default_value_str,- logical_field.data_type(),- )?;- return Ok(Transformed::yes(default_literal));- }- }+ // Pre-compute replacements for missing columns with default values+ let mut replacements = HashMap::new();+ for field in self.logical_file_schema.fields() {+ // Skip columns that exist in physical schema+ if self.physical_file_schema.index_of(field.name()).is_ok() {+ continue;}- }-- // No transformation needed- Ok(Transformed::no(expr))- }- fn create_default_value_expr(- &self,- value_str: &str,- data_type: &DataType,- ) -> Result<Arc<dyn PhysicalExpr>> {- // Create a string literal with the default value- let string_literal =- Arc::new(Literal::new(ScalarValue::Utf8(Some(value_str.to_string()))));-- // If the target type is already Utf8, return the string literal directly- if matches!(data_type, DataType::Utf8) {- return Ok(string_literal);+ // Check if this missing column has a default value in metadata+ if let Some(default_str) = field.metadata().get(DEFAULT_VALUE_METADATA_KEY) {+ // Create a Utf8 ScalarValue from the string and cast it to the target type+ let string_value = ScalarValue::Utf8(Some(default_str.to_string()));+ let typed_value = string_value.cast_to(field.data_type())?;+ replacements.insert(field.name().as_str(), typed_value);+ }}- // Otherwise, wrap the string literal in a cast expression- let cast_expr = Arc::new(CastExpr::new(string_literal, data_type.clone(), None));+ // Replace columns with their default literals if any+ let rewritten = if !replacements.is_empty() {+ let refs: HashMap<_, _> = replacements.iter().map(|(k, v)| (*k, v)).collect();+ replace_columns_with_literals(expr, &refs)?+ } else {+ expr+ };- Ok(cast_expr)+ // Apply the default adapter as a fallback for other schema adaptations+ self.default_adapter.rewrite(rewritten)}}diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rsindex baa0a2e660..a4b0de9301 100644--- a/datafusion-examples/examples/data_io/json_shredding.rs+++ b/datafusion-examples/examples/data_io/json_shredding.rs@@ -21,7 +21,7 @@ use std::any::Any;use std::sync::Arc;use arrow::array::{RecordBatch, StringArray};-use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::assert_batches_eq;use datafusion::common::tree_node::{@@ -277,14 +277,14 @@ impl PhysicalExprAdapterFactory for ShreddedJsonRewriterFactory {physical_file_schema: SchemaRef,) -> Arc<dyn PhysicalExprAdapter> {let default_factory = DefaultPhysicalExprAdapterFactory;- let default_adapter = default_factory- .create(logical_file_schema.clone(), physical_file_schema.clone());+ let default_adapter = default_factory.create(+ Arc::clone(&logical_file_schema),+ Arc::clone(&physical_file_schema),+ );Arc::new(ShreddedJsonRewriter {- logical_file_schema,physical_file_schema,default_adapter,- partition_values: Vec::new(),})}}@@ -293,10 +293,8 @@ impl PhysicalExprAdapterFactory for ShreddedJsonRewriterFactory {/// and wraps DefaultPhysicalExprAdapter for standard schema adaptation#[derive(Debug)]struct ShreddedJsonRewriter {- logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,default_adapter: Arc<dyn PhysicalExprAdapter>,- partition_values: Vec<(FieldRef, ScalarValue)>,}impl PhysicalExprAdapter for ShreddedJsonRewriter {@@ -307,27 +305,8 @@ impl PhysicalExprAdapter for ShreddedJsonRewriter {.data()?;// Then apply the default adapter as a fallback to handle standard schema differences- // like type casting, missing columns, and partition column handling- let default_adapter = if !self.partition_values.is_empty() {- self.default_adapter- .with_partition_values(self.partition_values.clone())- } else {- self.default_adapter.clone()- };-- default_adapter.rewrite(rewritten)- }-- fn with_partition_values(- &self,- partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter> {- Arc::new(ShreddedJsonRewriter {- logical_file_schema: self.logical_file_schema.clone(),- physical_file_schema: self.physical_file_schema.clone(),- default_adapter: self.default_adapter.clone(),- partition_values,- })+ // like type casting and missing columns+ self.default_adapter.rewrite(rewritten)}}diff --git a/datafusion/common/src/pruning.rs b/datafusion/common/src/pruning.rsindex a91b5176d4..5a7598ea1f 100644--- a/datafusion/common/src/pruning.rs+++ b/datafusion/common/src/pruning.rs@@ -135,6 +135,10 @@ pub trait PruningStatistics {/// This feeds into [`CompositePruningStatistics`] to allow pruning/// with filters that depend both on partition columns and data columns/// (e.g. `WHERE partition_col = data_col`).+#[deprecated(+ since = "52.0.0",+ note = "This struct is no longer used internally. Use `replace_columns_with_literals` from `datafusion-physical-expr-adapter` to substitute partition column values before pruning. It will be removed in 58.0.0 or 6 months after 52.0.0 is released, whichever comes first."+)]#[derive(Clone)]pub struct PartitionPruningStatistics {/// Values for each column for each container.@@ -156,6 +160,7 @@ pub struct PartitionPruningStatistics {partition_schema: SchemaRef,}+#[expect(deprecated)]impl PartitionPruningStatistics {/// Create a new instance of [`PartitionPruningStatistics`].///@@ -232,6 +237,7 @@ impl PartitionPruningStatistics {}}+#[expect(deprecated)]impl PruningStatistics for PartitionPruningStatistics {fn min_values(&self, column: &Column) -> Option<ArrayRef> {let index = self.partition_schema.index_of(column.name()).ok()?;@@ -439,10 +445,15 @@ impl PruningStatistics for PrunableStatistics {/// the first one is returned without any regard for completeness or accuracy./// That is: if the first statistics has information for a column, even if it is incomplete,/// that is returned even if a later statistics has more complete information.+#[deprecated(+ since = "52.0.0",+ note = "This struct is no longer used internally. It may be removed in 58.0.0 or 6 months after 52.0.0 is released, whichever comes first. Please open an issue if you have a use case for it."+)]pub struct CompositePruningStatistics {pub statistics: Vec<Box<dyn PruningStatistics>>,}+#[expect(deprecated)]impl CompositePruningStatistics {/// Create a new instance of [`CompositePruningStatistics`] from/// a vector of [`PruningStatistics`].@@ -457,6 +468,7 @@ impl CompositePruningStatistics {}}+#[expect(deprecated)]impl PruningStatistics for CompositePruningStatistics {fn min_values(&self, column: &Column) -> Option<ArrayRef> {for stats in &self.statistics {@@ -513,6 +525,7 @@ impl PruningStatistics for CompositePruningStatistics {}#[cfg(test)]+#[expect(deprecated)]mod tests {use crate::{ColumnStatistics,diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rsindex 28faea9a68..52564663b9 100644--- a/datafusion/core/src/datasource/mod.rs+++ b/datafusion/core/src/datasource/mod.rs@@ -56,7 +56,7 @@ mod tests {use ::object_store::{path::Path, ObjectMeta};use arrow::{array::Int32Array,- datatypes::{DataType, Field, FieldRef, Schema, SchemaRef},+ datatypes::{DataType, Field, Schema, SchemaRef},record_batch::RecordBatch,};use datafusion_common::{@@ -209,11 +209,10 @@ mod tests {impl PhysicalExprAdapterFactory for TestPhysicalExprAdapterFactory {fn create(&self,- logical_file_schema: SchemaRef,+ _logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,) -> Arc<dyn PhysicalExprAdapter> {Arc::new(TestPhysicalExprAdapter {- logical_file_schema,physical_file_schema,})}@@ -221,7 +220,6 @@ mod tests {#[derive(Debug)]struct TestPhysicalExprAdapter {- logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,}@@ -243,15 +241,5 @@ mod tests {}).data()}-- fn with_partition_values(- &self,- _partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter> {- Arc::new(TestPhysicalExprAdapter {- logical_file_schema: self.logical_file_schema.clone(),- physical_file_schema: self.physical_file_schema.clone(),- })- }}}diff --git a/datafusion/core/tests/parquet/schema_adapter.rs b/datafusion/core/tests/parquet/schema_adapter.rsindex 541785319c..3c12862f38 100644--- a/datafusion/core/tests/parquet/schema_adapter.rs+++ b/datafusion/core/tests/parquet/schema_adapter.rs@@ -18,7 +18,7 @@use std::sync::Arc;use arrow::array::{record_batch, RecordBatch};-use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef};+use arrow_schema::{DataType, Field, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion::assert_batches_eq;use datafusion::common::Result;@@ -123,17 +123,6 @@ impl PhysicalExprAdapter for CustomPhysicalExprAdapter {.data()?;self.inner.rewrite(expr)}-- fn with_partition_values(- &self,- partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter> {- assert!(- partition_values.is_empty(),- "Partition values are not supported in this test"- );- Arc::new(self.clone())- }}#[tokio::test]diff --git a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rsindex e9885e4014..2a0c133312 100644--- a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs+++ b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs@@ -19,7 +19,7 @@ use std::sync::Arc;use arrow::array::RecordBatch;-use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef};+use arrow_schema::{DataType, Field, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion::common::Result;use datafusion::config::{ConfigOptions, TableParquetOptions};@@ -35,7 +35,7 @@ use datafusion::prelude::SessionContext;use datafusion_common::config::CsvOptions;use datafusion_common::record_batch;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::{ColumnStatistics, ScalarValue};+use datafusion_common::ColumnStatistics;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::schema_adapter::{SchemaAdapter, SchemaAdapterFactory, SchemaMapper,@@ -193,11 +193,10 @@ struct UppercasePhysicalExprAdapterFactory;impl PhysicalExprAdapterFactory for UppercasePhysicalExprAdapterFactory {fn create(&self,- logical_file_schema: SchemaRef,+ _logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,) -> Arc<dyn PhysicalExprAdapter> {Arc::new(UppercasePhysicalExprAdapter {- logical_file_schema,physical_file_schema,})}@@ -205,7 +204,6 @@ impl PhysicalExprAdapterFactory for UppercasePhysicalExprAdapterFactory {#[derive(Debug)]struct UppercasePhysicalExprAdapter {- logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,}@@ -226,16 +224,6 @@ impl PhysicalExprAdapter for UppercasePhysicalExprAdapter {}).data()}-- fn with_partition_values(- &self,- _partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter> {- Arc::new(Self {- logical_file_schema: self.logical_file_schema.clone(),- physical_file_schema: self.physical_file_schema.clone(),- })- }}#[derive(Clone)]diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex fe1fab7a0e..8f521c3f4e 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -27,15 +27,17 @@ use arrow::array::RecordBatch;use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::utils::reassign_expr_columns;+use datafusion_physical_expr_adapter::replace_columns_with_literals;+use std::collections::HashMap;use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};-use arrow::datatypes::{FieldRef, SchemaRef, TimeUnit};+use arrow::datatypes::{SchemaRef, TimeUnit};use datafusion_common::encryption::FileDecryptionProperties;-use datafusion_common::{exec_err, DataFusionError, Result};-use datafusion_datasource::PartitionedFile;+use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue};+use datafusion_datasource::{PartitionedFile, TableSchema};use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::{@@ -51,7 +53,6 @@ use datafusion_common::config::EncryptionFactoryOptions;#[cfg(feature = "parquet_encryption")]use datafusion_execution::parquet_encryption::EncryptionFactory;use futures::{ready, Stream, StreamExt, TryStreamExt};-use itertools::Itertools;use log::debug;use parquet::arrow::arrow_reader::metrics::ArrowReaderMetrics;use parquet::arrow::arrow_reader::{@@ -73,11 +74,8 @@ pub(super) struct ParquetOpener {pub limit: Option<usize>,/// Optional predicate to apply during the scanpub predicate: Option<Arc<dyn PhysicalExpr>>,- /// Schema of the output table without partition columns.- /// This is the schema we coerce the physical file schema into.- pub logical_file_schema: SchemaRef,- /// Partition columns- pub partition_fields: Vec<FieldRef>,+ /// Table schema, including partition columns.+ pub table_schema: TableSchema,/// Optional hint for how large the initial request to read parquet metadata/// should bepub metadata_size_hint: Option<usize>,@@ -139,10 +137,49 @@ impl FileOpener for ParquetOpener {let batch_size = self.batch_size;+ // Replace any partition column references in the predicate+ // with their literal values from this file's partition values.+ //+ // For example, given+ // 1. `region` is a partition column,+ // 2. predicate `host IN ('us-east-1', 'eu-central-1')`:+ // 3. The file path is `/data/region=us-west-2/...`+ // (that is the partition column value is `us-west-2`)+ //+ // The predicate would be rewritten to+ // ```sql+ // 'us-west-2` IN ('us-east-1', 'eu-central-1')+ // ```+ // which can be further simplified to `FALSE`, meaning+ // the file can be skipped entirely.+ //+ // While this particular optimization is done during logical planning,+ // there are other cases where partition columns may appear in more+ // complex predicates that cannot be simplified until we are about to+ // open the file (such as dynamic predicates)+ let mut predicate = {+ // Only replace partition columns if there are partition values.+ // For non-partitioned tables, skip the clone and replacement traversal.+ if partitioned_file.partition_values.is_empty() {+ self.predicate.clone()+ } else {+ let partition_values: HashMap<&str, &ScalarValue> = self+ .table_schema+ .table_partition_cols()+ .iter()+ .zip(partitioned_file.partition_values.iter())+ .map(|(field, value)| (field.name().as_str(), value))+ .collect();++ self.predicate+ .clone()+ .map(|p| replace_columns_with_literals(p, &partition_values))+ .transpose()?+ }+ };+let projection = Arc::clone(&self.projection);- let mut predicate = self.predicate.clone();- let logical_file_schema = Arc::clone(&self.logical_file_schema);- let partition_fields = self.partition_fields.clone();+ let logical_file_schema = Arc::clone(self.table_schema.table_schema());let reorder_predicates = self.reorder_filters;let pushdown_filters = self.pushdown_filters;let force_filter_selections = self.force_filter_selections;@@ -155,7 +192,7 @@ impl FileOpener for ParquetOpener {.global_counter("num_predicate_creation_errors");let expr_adapter_factory = self.expr_adapter_factory.clone();- let mut predicate_file_schema = Arc::clone(&self.logical_file_schema);+ let mut predicate_file_schema = Arc::clone(self.table_schema.file_schema());let enable_page_index = self.enable_page_index;#[cfg(feature = "parquet_encryption")]@@ -178,20 +215,33 @@ impl FileOpener for ParquetOpener {// we can end the stream early.let mut file_pruner = predicate.as_ref()- .map(|p| {- Ok::<_, DataFusionError>(- (is_dynamic_physical_expr(p) | partitioned_file.has_statistics())- .then_some(FilePruner::new(- Arc::clone(p),- &logical_file_schema,- partition_fields.clone(),- partitioned_file.clone(),- predicate_creation_errors.clone(),- )?),- )+ .filter(|p| {+ // Make a FilePruner only if there is either+ // 1. a dynamic expr in the predicate+ // 2. the file has file-level statistics.+ //+ // File-level statistics may prune the file without loading+ // any row groups or metadata.+ //+ // Dynamic filters may prune the file after initial+ // planning, as the dynamic filter is updated during+ // execution.+ //+ // The case where there is a dynamic filter but no+ // statistics corresponds to a dynamic filter that+ // references partition columns. While rare, this is possible+ // e.g. `select * from table order by partition_col limit+ // 10` could hit this condition.+ is_dynamic_physical_expr(p) || partitioned_file.has_statistics()})- .transpose()?- .flatten();+ .and_then(|p| {+ FilePruner::try_new(+ Arc::clone(p),+ &logical_file_schema,+ &partitioned_file,+ predicate_creation_errors.clone(),+ )+ });if let Some(file_pruner) = &mut file_pruner {if file_pruner.should_prune()? {@@ -266,17 +316,11 @@ impl FileOpener for ParquetOpener {if let Some(expr_adapter_factory) = expr_adapter_factory.as_ref() {predicate = predicate.map(|p| {- let partition_values = partition_fields- .iter()- .cloned()- .zip(partitioned_file.partition_values.clone())- .collect_vec();let expr = expr_adapter_factory.create(Arc::clone(&logical_file_schema),Arc::clone(&physical_file_schema),)- .with_partition_values(partition_values).rewrite(p)?;// After rewriting to the file schema, further simplifications may be possible.// For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`@@ -317,18 +361,10 @@ impl FileOpener for ParquetOpener {let mut projection =ProjectionExprs::from_indices(&projection, &logical_file_schema);if let Some(expr_adapter_factory) = expr_adapter_factory {- let adapter = expr_adapter_factory- .create(- Arc::clone(&logical_file_schema),- Arc::clone(&physical_file_schema),- )- .with_partition_values(- partition_fields- .iter()- .cloned()- .zip(partitioned_file.partition_values.clone())- .collect_vec(),- );+ let adapter = expr_adapter_factory.create(+ Arc::clone(&logical_file_schema),+ Arc::clone(&physical_file_schema),+ );projection = projection.try_map_exprs(|expr| adapter.rewrite(expr))?;}let indices = projection.column_indices();@@ -550,6 +586,7 @@ impl<S> EarlyStoppingStream<S> {}}}+impl<S> EarlyStoppingStream<S>whereS: Stream<Item = Result<RecordBatch>> + Unpin,@@ -786,7 +823,7 @@ mod test {record_batch, stats::Precision, ColumnStatistics, DataFusionError, ScalarValue,Statistics,};- use datafusion_datasource::{file_stream::FileOpener, PartitionedFile};+ use datafusion_datasource::{file_stream::FileOpener, PartitionedFile, TableSchema};use datafusion_expr::{col, lit};use datafusion_physical_expr::{expressions::DynamicFilterPhysicalExpr, planner::logical2physical, PhysicalExpr,@@ -877,13 +914,12 @@ mod test {batch_size: 1024,limit: None,predicate: Some(predicate),- logical_file_schema: schema.clone(),+ table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),metadata_size_hint: None,metrics: ExecutionPlanMetricsSet::new(),parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(Arc::clone(&store)),),- partition_fields: vec![],pushdown_filters: false, // note that this is false!reorder_filters: false,force_filter_selections: false,@@ -946,17 +982,15 @@ mod test {batch_size: 1024,limit: None,predicate: Some(predicate),- logical_file_schema: file_schema.clone(),+ table_schema: TableSchema::new(+ file_schema.clone(),+ vec![Arc::new(Field::new("part", DataType::Int32, false))],+ ),metadata_size_hint: None,metrics: ExecutionPlanMetricsSet::new(),parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(Arc::clone(&store)),),- partition_fields: vec![Arc::new(Field::new(- "part",- DataType::Int32,- false,- ))],pushdown_filters: false, // note that this is false!reorder_filters: false,force_filter_selections: false,@@ -1035,17 +1069,15 @@ mod test {batch_size: 1024,limit: None,predicate: Some(predicate),- logical_file_schema: file_schema.clone(),+ table_schema: TableSchema::new(+ file_schema.clone(),+ vec![Arc::new(Field::new("part", DataType::Int32, false))],+ ),metadata_size_hint: None,metrics: ExecutionPlanMetricsSet::new(),parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(Arc::clone(&store)),),- partition_fields: vec![Arc::new(Field::new(- "part",- DataType::Int32,- false,- ))],pushdown_filters: false, // note that this is false!reorder_filters: false,force_filter_selections: false,@@ -1127,17 +1159,15 @@ mod test {batch_size: 1024,limit: None,predicate: Some(predicate),- logical_file_schema: file_schema.clone(),+ table_schema: TableSchema::new(+ file_schema.clone(),+ vec![Arc::new(Field::new("part", DataType::Int32, false))],+ ),metadata_size_hint: None,metrics: ExecutionPlanMetricsSet::new(),parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(Arc::clone(&store)),),- partition_fields: vec![Arc::new(Field::new(- "part",- DataType::Int32,- false,- ))],pushdown_filters: true, // note that this is true!reorder_filters: true,force_filter_selections: false,@@ -1206,10 +1236,18 @@ mod test {u64::try_from(data_size).unwrap(),);file.partition_values = vec![ScalarValue::Int32(Some(1))];+ file.statistics = Some(Arc::new(+ Statistics::default().add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_min_value(Precision::Exact(ScalarValue::Int32(Some(1))))+ .with_max_value(Precision::Exact(ScalarValue::Int32(Some(3))))+ .with_null_count(Precision::Exact(0)),+ ),+ ));let table_schema = Arc::new(Schema::new(vec![- Field::new("part", DataType::Int32, false),Field::new("a", DataType::Int32, false),+ Field::new("part", DataType::Int32, false),]));let make_opener = |predicate| {@@ -1219,17 +1257,15 @@ mod test {batch_size: 1024,limit: None,predicate: Some(predicate),- logical_file_schema: file_schema.clone(),+ table_schema: TableSchema::new(+ file_schema.clone(),+ vec![Arc::new(Field::new("part", DataType::Int32, false))],+ ),metadata_size_hint: None,metrics: ExecutionPlanMetricsSet::new(),parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(Arc::clone(&store)),),- partition_fields: vec![Arc::new(Field::new(- "part",- DataType::Int32,- false,- ))],pushdown_filters: false, // note that this is false!reorder_filters: false,force_filter_selections: false,@@ -1246,16 +1282,37 @@ mod test {}};- // Filter should NOT match the stats but the file is never attempted to be pruned because the filters are not dynamic- let expr = col("part").eq(lit(2));+ // This filter could prune based on statistics, but since it's not dynamic it's not applied for pruning+ // (the assumption is this happened already at planning time)+ let expr = col("a").eq(lit(42));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);let stream = opener.open(file.clone()).unwrap().await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;- assert_eq!(num_batches, 1);- assert_eq!(num_rows, 3);+ assert_eq!(num_batches, 0);+ assert_eq!(num_rows, 0);++ // If we make the filter dynamic, it should prune.+ // This allows dynamic filters to prune partitions/files even if they are populated late into execution.+ let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));+ let opener = make_opener(predicate);+ let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let (num_batches, num_rows) = count_batches_and_rows(stream).await;+ assert_eq!(num_batches, 0);+ assert_eq!(num_rows, 0);++ // If we have a filter that touches partition columns only and is dynamic, it should prune even if there are no stats.+ file.statistics = Some(Arc::new(Statistics::new_unknown(&file_schema)));+ let expr = col("part").eq(lit(2));+ let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));+ let opener = make_opener(predicate);+ let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let (num_batches, num_rows) = count_batches_and_rows(stream).await;+ assert_eq!(num_batches, 0);+ assert_eq!(num_rows, 0);- // If we make the filter dynamic, it should prune+ // Similarly a filter that combines partition and data columns should prune even if there are no stats.+ let expr = col("part").eq(lit(2)).and(col("a").eq(lit(42)));let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));let opener = make_opener(predicate);let stream = opener.open(file.clone()).unwrap().await.unwrap();diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 1abe126369..d2a140c4f8 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -533,8 +533,7 @@ impl FileSource for ParquetSource {.expect("Batch size must set before creating ParquetOpener"),limit: base_config.limit,predicate: self.predicate.clone(),- logical_file_schema: Arc::clone(base_config.file_schema()),- partition_fields: base_config.table_partition_cols().clone(),+ table_schema: self.table_schema.clone(),metadata_size_hint: self.metadata_size_hint,metrics: self.metrics().clone(),parquet_file_reader_factory,diff --git a/datafusion/physical-expr-adapter/src/lib.rs b/datafusion/physical-expr-adapter/src/lib.rsindex 9586372803..d7c750e4a1 100644--- a/datafusion/physical-expr-adapter/src/lib.rs+++ b/datafusion/physical-expr-adapter/src/lib.rs@@ -30,5 +30,5 @@ pub mod schema_rewriter;pub use schema_rewriter::{DefaultPhysicalExprAdapter, DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter,- PhysicalExprAdapterFactory,+ PhysicalExprAdapterFactory, replace_columns_with_literals,};diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rsindex cbb4687ab5..6f040c1a05 100644--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs@@ -15,12 +15,15 @@// specific language governing permissions and limitations// under the License.-//! Physical expression schema rewriting utilities+//! Physical expression schema rewriting utilities: [`PhysicalExprAdapter`],+//! [`PhysicalExprAdapterFactory`], default implementations,+//! and [`replace_columns_with_literals`].+use std::collections::HashMap;use std::sync::Arc;use arrow::compute::can_cast_types;-use arrow::datatypes::{DataType, FieldRef, Schema, SchemaRef};+use arrow::datatypes::{DataType, Schema, SchemaRef};use datafusion_common::{Result, ScalarValue, exec_err,nested_struct::validate_struct_compatibility,@@ -34,29 +37,52 @@ use datafusion_physical_expr::{};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;-/// Trait for adapting physical expressions to match a target schema.+/// Replace column references in the given physical expression with literal values.///-/// This is used in file scans to rewrite expressions so that they can be evaluated-/// against the physical schema of the file being scanned. It allows for handling-/// differences between logical and physical schemas, such as type mismatches or missing columns.+/// Some use cases for this include:+/// - Partition column pruning: When scanning partitioned data, partition column references+/// can be replaced with their literal values for the specific partition being scanned.+/// - Constant folding: In some cases, columns that can be proven to be constant+/// from statistical analysis may be replaced with their literal values to optimize expression evaluation.+/// - Filling in non-null default values: in a custom [`PhysicalExprAdapter`] implementation,+/// column references can be replaced with default literal values instead of nulls.///-/// ## Overview+/// # Arguments+/// - `expr`: The physical expression in which to replace column references.+/// - `replacements`: A mapping from column names to their corresponding literal `ScalarValue`s.///-/// The `PhysicalExprAdapter` allows rewriting physical expressions to match different schemas, including:+/// # Returns+/// - `Result<Arc<dyn PhysicalExpr>>`: The rewritten physical expression with columns replaced by literals.+pub fn replace_columns_with_literals(+ expr: Arc<dyn PhysicalExpr>,+ replacements: &HashMap<&str, &ScalarValue>,+) -> Result<Arc<dyn PhysicalExpr>> {+ expr.transform(|expr| {+ if let Some(column) = expr.as_any().downcast_ref::<Column>()+ && let Some(replacement_value) = replacements.get(column.name())+ {+ return Ok(Transformed::yes(expressions::lit(+ (*replacement_value).clone(),+ )));+ }+ Ok(Transformed::no(expr))+ })+ .data()+}++/// Trait for adapting [`PhysicalExpr`] expressions to match a target schema.///-/// - **Type casting**: When logical and physical schemas have different types, expressions are-/// automatically wrapped with cast operations. For example, `lit(ScalarValue::Int32(123)) = int64_column`-/// gets rewritten to `lit(ScalarValue::Int32(123)) = cast(int64_column, 'Int32')`.-/// Note that this does not attempt to simplify such expressions - that is done by shared simplifiers.+/// This is used in file scans to rewrite expressions so that they can be+/// evaluated against the physical schema of the file being scanned. It allows+/// for handling differences between logical and physical schemas, such as type+/// mismatches or missing columns common in [Schema evolution] scenarios.///-/// - **Missing columns**: When a column exists in the logical schema but not in the physical schema,-/// references to it are replaced with null literals.+/// [Schema evolution]: https://www.dremio.com/wiki/schema-evolution////-/// - **Struct field access**: Expressions like `struct_column.field_that_is_missing_in_schema` are-/// rewritten to `null` when the field doesn't exist in the physical schema.+/// ## Default Implementations///-/// - **Partition columns**: Partition column references can be replaced with their literal values-/// when scanning specific partitions.+/// The default implementation [`DefaultPhysicalExprAdapter`] handles common+/// cases.////// ## Custom Implementations///@@ -93,17 +119,6 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;/// Ok(Transformed::no(expr))/// }).data()/// }-///-/// fn with_partition_values(-/// &self,-/// partition_values: Vec<(FieldRef, ScalarValue)>,-/// ) -> Arc<dyn PhysicalExprAdapter> {-/// // For simplicity, this example ignores partition values-/// Arc::new(CustomPhysicalExprAdapter {-/// logical_file_schema: self.logical_file_schema.clone(),-/// physical_file_schema: self.physical_file_schema.clone(),-/// })-/// }/// }////// #[derive(Debug)]@@ -136,14 +151,15 @@ pub trait PhysicalExprAdapter: Send + Sync + std::fmt::Debug {////// Returns:/// - `Arc<dyn PhysicalExpr>`: The rewritten physical expression that can be evaluated against the physical schema.+ ///+ /// See Also:+ /// - [`replace_columns_with_literals`]: for replacing partition column references with their literal values.fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>>;-- fn with_partition_values(- &self,- partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter>;}+/// Creates instances of [`PhysicalExprAdapter`] for given logical and physical schemas.+///+/// See [`DefaultPhysicalExprAdapterFactory`] for the default implementation.pub trait PhysicalExprAdapterFactory: Send + Sync + std::fmt::Debug {/// Create a new instance of the physical expression adapter.fn create(@@ -165,20 +181,39 @@ impl PhysicalExprAdapterFactory for DefaultPhysicalExprAdapterFactory {Arc::new(DefaultPhysicalExprAdapter {logical_file_schema,physical_file_schema,- partition_values: Vec::new(),})}}-/// Default implementation for rewriting physical expressions to match different schemas.+/// Default implementation of [`PhysicalExprAdapter`] for rewriting physical+/// expressions to match different schemas.+///+/// ## Overview+///+/// [`DefaultPhysicalExprAdapter`] rewrites physical expressions to match+/// different schemas, including:+///+/// - **Type casting**: When logical and physical schemas have different types, expressions are+/// automatically wrapped with cast operations. For example, `lit(ScalarValue::Int32(123)) = int64_column`+/// gets rewritten to `lit(ScalarValue::Int32(123)) = cast(int64_column, 'Int32')`.+/// Note that this does not attempt to simplify such expressions - that is done by shared simplifiers.+///+/// - **Missing columns**: When a column exists in the logical schema but not in the physical schema,+/// references to it are replaced with null literals.+///+/// - **Struct field access**: Expressions like `struct_column.field_that_is_missing_in_schema` are+/// rewritten to `null` when the field doesn't exist in the physical schema.+///+/// - **Default column values**: Partition column references can be replaced with their literal values+/// when scanning specific partitions. See [`replace_columns_with_literals`] for more details.////// # Example////// ```rust-/// use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapterFactory};-/// use arrow::datatypes::Schema;-/// use std::sync::Arc;-///+/// # use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapterFactory};+/// # use arrow::datatypes::Schema;+/// # use std::sync::Arc;+/// #/// # fn example(/// # predicate: std::sync::Arc<dyn datafusion_physical_expr_common::physical_expr::PhysicalExpr>,/// # physical_file_schema: &Schema,@@ -194,7 +229,6 @@ impl PhysicalExprAdapterFactory for DefaultPhysicalExprAdapterFactory {pub struct DefaultPhysicalExprAdapter {logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,- partition_values: Vec<(FieldRef, ScalarValue)>,}impl DefaultPhysicalExprAdapter {@@ -206,7 +240,6 @@ impl DefaultPhysicalExprAdapter {Self {logical_file_schema,physical_file_schema,- partition_values: Vec::new(),}}}@@ -216,27 +249,15 @@ impl PhysicalExprAdapter for DefaultPhysicalExprAdapter {let rewriter = DefaultPhysicalExprAdapterRewriter {logical_file_schema: &self.logical_file_schema,physical_file_schema: &self.physical_file_schema,- partition_fields: &self.partition_values,};expr.transform(|expr| rewriter.rewrite_expr(Arc::clone(&expr))).data()}-- fn with_partition_values(- &self,- partition_values: Vec<(FieldRef, ScalarValue)>,- ) -> Arc<dyn PhysicalExprAdapter> {- Arc::new(DefaultPhysicalExprAdapter {- partition_values,- ..self.clone()- })- }}struct DefaultPhysicalExprAdapterRewriter<'a> {logical_file_schema: &'a Schema,physical_file_schema: &'a Schema,- partition_fields: &'a [(FieldRef, ScalarValue)],}impl<'a> DefaultPhysicalExprAdapterRewriter<'a> {@@ -347,10 +368,6 @@ impl<'a> DefaultPhysicalExprAdapterRewriter<'a> {{Ok(field) => field,Err(e) => {- // If the column is a partition field, we can use the partition value- if let Some(partition_value) = self.get_partition_value(column.name()) {- return Ok(Transformed::yes(expressions::lit(partition_value)));- }// This can be hit if a custom rewrite injected a reference to a column that doesn't exist in the logical schema.// For example, a pre-computed column that is kept only in the physical schema.// If the column exists in the physical schema, we can still use it.@@ -446,13 +463,6 @@ impl<'a> DefaultPhysicalExprAdapterRewriter<'a> {Ok(Transformed::yes(cast_expr))}-- fn get_partition_value(&self, column_name: &str) -> Option<ScalarValue> {- self.partition_fields- .iter()- .find(|(field, _)| field.name() == column_name)- .map(|(_, value)| value.clone())- }}#[cfg(test)]@@ -698,30 +708,51 @@ mod tests {}#[test]- fn test_rewrite_partition_column() -> Result<()> {- let (physical_schema, logical_schema) = create_test_schema();-- let partition_field =- Arc::new(Field::new("partition_col", DataType::Utf8, false));+ fn test_replace_columns_with_literals() -> Result<()> {let partition_value = ScalarValue::Utf8(Some("test_value".to_string()));- let partition_values = vec![(partition_field, partition_value)];-- let factory = DefaultPhysicalExprAdapterFactory;- let adapter = factory.create(Arc::new(logical_schema), Arc::new(physical_schema));- let adapter = adapter.with_partition_values(partition_values);+ let replacements = HashMap::from([("partition_col", &partition_value)]);- let column_expr = Arc::new(Column::new("partition_col", 0));- let result = adapter.rewrite(column_expr)?;+ let column_expr =+ Arc::new(Column::new("partition_col", 0)) as Arc<dyn PhysicalExpr>;+ let result = replace_columns_with_literals(column_expr, &replacements)?;// Should be replaced with the partition value- if let Some(literal) = result.as_any().downcast_ref::<expressions::Literal>() {- assert_eq!(- *literal.value(),- ScalarValue::Utf8(Some("test_value".to_string()))- );- } else {- panic!("Expected literal expression");- }+ let literal = result+ .as_any()+ .downcast_ref::<expressions::Literal>()+ .expect("Expected literal expression");+ assert_eq!(*literal.value(), partition_value);++ Ok(())+ }++ #[test]+ fn test_replace_columns_with_literals_no_match() -> Result<()> {+ let value = ScalarValue::Utf8(Some("test_value".to_string()));+ let replacements = HashMap::from([("other_col", &value)]);++ let column_expr =+ Arc::new(Column::new("partition_col", 0)) as Arc<dyn PhysicalExpr>;+ let result = replace_columns_with_literals(column_expr, &replacements)?;++ assert!(result.as_any().downcast_ref::<Column>().is_some());+ Ok(())+ }++ #[test]+ fn test_replace_columns_with_literals_nested_expr() -> Result<()> {+ let value_a = ScalarValue::Int64(Some(10));+ let value_b = ScalarValue::Int64(Some(20));+ let replacements = HashMap::from([("a", &value_a), ("b", &value_b)]);++ let expr = Arc::new(expressions::BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Plus,+ Arc::new(Column::new("b", 1)),+ )) as Arc<dyn PhysicalExpr>;++ let result = replace_columns_with_literals(expr, &replacements)?;+ assert_eq!(result.to_string(), "10 + 20");Ok(())}@@ -996,7 +1027,6 @@ mod tests {let rewriter = DefaultPhysicalExprAdapterRewriter {logical_file_schema: &logical_schema,physical_file_schema: &physical_schema,- partition_fields: &[],};// Test that when a field exists in physical schema, it returns Nonediff --git a/datafusion/pruning/src/file_pruner.rs b/datafusion/pruning/src/file_pruner.rsindex ee86a8cc8c..1d923202b3 100644--- a/datafusion/pruning/src/file_pruner.rs+++ b/datafusion/pruning/src/file_pruner.rs@@ -19,69 +19,84 @@use std::sync::Arc;-use arrow::datatypes::{FieldRef, Schema, SchemaRef};-use datafusion_common::{- pruning::{- CompositePruningStatistics, PartitionPruningStatistics, PrunableStatistics,- PruningStatistics,- },- Result,-};+use arrow::datatypes::{FieldRef, SchemaRef};+use datafusion_common::{internal_datafusion_err, pruning::PrunableStatistics, Result};use datafusion_datasource::PartitionedFile;use datafusion_physical_expr_common::physical_expr::{snapshot_generation, PhysicalExpr};use datafusion_physical_plan::metrics::Count;-use itertools::Itertools;use log::debug;use crate::build_pruning_predicate;-/// Prune based on partition values and file-level statistics.+/// Prune based on file-level statistics.+///+/// Note: Partition column pruning is handled earlier via `replace_columns_with_literals`+/// which substitutes partition column references with their literal values before+/// the predicate reaches this pruner.pub struct FilePruner {predicate_generation: Option<u64>,predicate: Arc<dyn PhysicalExpr>,- /// Schema used for pruning, which combines the file schema and partition fields.- /// Partition fields are always at the end, as they are during scans.- pruning_schema: Arc<Schema>,- partitioned_file: PartitionedFile,- partition_fields: Vec<FieldRef>,+ /// Schema used for pruning (the logical file schema).+ file_schema: SchemaRef,+ file_stats_pruning: PrunableStatistics,predicate_creation_errors: Count,}impl FilePruner {+ #[deprecated(+ since = "52.0.0",+ note = "Use `try_new` instead which returns None if no statistics are available"+ )]+ #[expect(clippy::needless_pass_by_value)]pub fn new(predicate: Arc<dyn PhysicalExpr>,logical_file_schema: &SchemaRef,- partition_fields: Vec<FieldRef>,+ _partition_fields: Vec<FieldRef>,partitioned_file: PartitionedFile,predicate_creation_errors: Count,) -> Result<Self> {- // Build a pruning schema that combines the file fields and partition fields.- // Partition fields are always at the end.- let pruning_schema = Arc::new(- Schema::new(- logical_file_schema- .fields()- .iter()- .cloned()- .chain(partition_fields.iter().cloned())- .collect_vec(),+ Self::try_new(+ predicate,+ logical_file_schema,+ &partitioned_file,+ predicate_creation_errors,+ )+ .ok_or_else(|| {+ internal_datafusion_err!(+ "FilePruner::new called on a file without statistics: {:?}",+ partitioned_file)- .with_metadata(logical_file_schema.metadata().clone()),- );- Ok(Self {- // Initialize the predicate generation to None so that the first time we call `should_prune` we actually check the predicate- // Subsequent calls will only do work if the predicate itself has changed.- // See `snapshot_generation` for more info.+ })+ }++ /// Create a new file pruner if statistics are available.+ /// Returns None if this file does not have statistics.+ pub fn try_new(+ predicate: Arc<dyn PhysicalExpr>,+ file_schema: &SchemaRef,+ partitioned_file: &PartitionedFile,+ predicate_creation_errors: Count,+ ) -> Option<Self> {+ let file_stats = partitioned_file.statistics.as_ref()?;+ let file_stats_pruning =+ PrunableStatistics::new(vec![file_stats.clone()], Arc::clone(file_schema));+ Some(Self {predicate_generation: None,predicate,- pruning_schema,- partitioned_file,- partition_fields,+ file_schema: Arc::clone(file_schema),+ file_stats_pruning,predicate_creation_errors,})}pub fn should_prune(&mut self) -> Result<bool> {+ // Check if the predicate has changed since last invocation by tracking+ // its "generation". Dynamic filter expressions can change their values+ // during query execution, so we use generation tracking to detect when+ // the predicate has been updated and needs to be rebuilt.+ //+ // If the generation hasn't changed, we can skip rebuilding the pruning+ // predicate, which is an expensive operation involving expression analysis.let new_generation = snapshot_generation(&self.predicate);if let Some(current_generation) = self.predicate_generation.as_mut() {if *current_generation == new_generation {@@ -93,39 +108,25 @@ impl FilePruner {}let pruning_predicate = build_pruning_predicate(Arc::clone(&self.predicate),- &self.pruning_schema,+ &self.file_schema,&self.predicate_creation_errors,);- if let Some(pruning_predicate) = pruning_predicate {- // The partition column schema is the schema of the table - the schema of the file- let mut pruning = Box::new(PartitionPruningStatistics::try_new(- vec![self.partitioned_file.partition_values.clone()],- self.partition_fields.clone(),- )?) as Box<dyn PruningStatistics>;- if let Some(stats) = &self.partitioned_file.statistics {- let stats_pruning = Box::new(PrunableStatistics::new(- vec![Arc::clone(stats)],- Arc::clone(&self.pruning_schema),- ));- pruning = Box::new(CompositePruningStatistics::new(vec![- pruning,- stats_pruning,- ]));- }- match pruning_predicate.prune(pruning.as_ref()) {- Ok(values) => {- assert!(values.len() == 1);- // We expect a single container -> if all containers are false skip this file- if values.into_iter().all(|v| !v) {- return Ok(true);- }- }- // Stats filter array could not be built, so we can't prune- Err(e) => {- debug!("Ignoring error building pruning predicate for file: {e}");- self.predicate_creation_errors.add(1);+ let Some(pruning_predicate) = pruning_predicate else {+ return Ok(false);+ };+ match pruning_predicate.prune(&self.file_stats_pruning) {+ Ok(values) => {+ assert!(values.len() == 1);+ // We expect a single container -> if all containers are false skip this file+ if values.into_iter().all(|v| !v) {+ return Ok(true);}}+ // Stats filter array could not be built, so we can't prune+ Err(e) => {+ debug!("Ignoring error building pruning predicate for file: {e}");+ self.predicate_creation_errors.add(1);+ }}Ok(false)diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex 6462961ce6..cecc0ef25b 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -165,6 +165,70 @@ Statistics are now accessed through `FileScanConfig` instead of `FileSource`:Note that `FileScanConfig::statistics()` automatically marks statistics as inexact when filters are present, ensuring correctness when filters are pushed down.+### Partition column handling moved out of `PhysicalExprAdapter`++Partition column replacement is now a separate preprocessing step performed before expression rewriting via `PhysicalExprAdapter`. This change provides better separation of concerns and makes the adapter more focused on schema differences rather than partition value substitution.++**Who is affected:**++- Users who have custom implementations of `PhysicalExprAdapterFactory` that handle partition columns+- Users who directly use the `FilePruner` API++**Breaking changes:**++1. `FilePruner::try_new()` signature changed: the `partition_fields` parameter has been removed since partition column handling is now done separately+2. Partition column replacement must now be done via `replace_columns_with_literals()` before expressions are passed to the adapter++**Migration guide:**++If you have code that creates a `FilePruner` with partition fields:++**Before:**++```rust,ignore+use datafusion_pruning::FilePruner;++let pruner = FilePruner::try_new(+ predicate,+ file_schema,+ partition_fields, // This parameter is removed+ file_stats,+)?;+```++**After:**++```rust,ignore+use datafusion_pruning::FilePruner;++// Partition fields are no longer needed+let pruner = FilePruner::try_new(+ predicate,+ file_schema,+ file_stats,+)?;+```++If you have custom code that relies on `PhysicalExprAdapter` to handle partition columns, you must now call `replace_columns_with_literals()` separately:++**Before:**++```rust,ignore+// Adapter handled partition column replacement internally+let adapted_expr = adapter.rewrite(expr)?;+```++**After:**++```rust,ignore+use datafusion_physical_expr_adapter::replace_columns_with_literals;++// Replace partition columns first+let expr_with_literals = replace_columns_with_literals(expr, &partition_values)?;+// Then apply the adapter+let adapted_expr = adapter.rewrite(expr_with_literals)?;+```+### Planner now requires explicit opt-in for WITHIN GROUP syntaxThe SQL planner now enforces the aggregate UDF contract more strictly: the