Skip to content

Apache DataFusion SQL Query Engine

Move partition handling out of PhysicalExprAdapter (#19128)

Adrian Garcia Badaraccodc786138cfa0
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
index ada3f72f45..80b88f7a38 100644
--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
@@ -20,12 +20,12 @@
use std::sync::Arc;
use arrow::array::{record_batch, RecordBatch};
-use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::assert_batches_eq;
use datafusion::common::not_impl_err;
use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion::common::{Result, ScalarValue};
+use datafusion::common::Result;
use datafusion::datasource::listing::{
ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,
};
@@ -209,14 +209,4 @@ impl PhysicalExprAdapter for CustomCastsPhysicalExprAdapter {
})
.data()
}
-
- fn with_partition_values(
- &self,
- partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter> {
- Arc::new(Self {
- inner: self.inner.with_partition_values(partition_values),
- ..self.clone()
- })
- }
}
diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rs
index 9fe8165025..7381dc938a 100644
--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs
+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs
@@ -22,13 +22,12 @@ use std::collections::HashMap;
use std::sync::Arc;
use arrow::array::RecordBatch;
-use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use async_trait::async_trait;
use datafusion::assert_batches_eq;
use datafusion::catalog::memory::DataSourceExec;
use datafusion::catalog::{Session, TableProvider};
-use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion::common::DFSchema;
use datafusion::common::{Result, ScalarValue};
use datafusion::datasource::listing::PartitionedFile;
@@ -39,12 +38,12 @@ use datafusion::logical_expr::utils::conjunction;
use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType};
use datafusion::parquet::arrow::ArrowWriter;
use datafusion::parquet::file::properties::WriterProperties;
-use datafusion::physical_expr::expressions::{CastExpr, Column, Literal};
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::{lit, SessionConfig};
use datafusion_physical_expr_adapter::{
- DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
+ replace_columns_with_literals, DefaultPhysicalExprAdapterFactory,
+ PhysicalExprAdapter, PhysicalExprAdapterFactory,
};
use futures::StreamExt;
use object_store::memory::InMemory;
@@ -60,16 +59,16 @@ const DEFAULT_VALUE_METADATA_KEY: &str = "example.default_value";
/// This example demonstrates how to:
/// 1. Store default values in field metadata using a constant key
/// 2. Create a custom PhysicalExprAdapter that reads these defaults
-/// 3. Inject default values for missing columns in filter predicates
+/// 3. Inject default values for missing columns in filter predicates using `replace_columns_with_literals`
/// 4. Use the DefaultPhysicalExprAdapter as a fallback for standard schema adaptation
-/// 5. Wrap string default values in cast expressions for proper type conversion
+/// 5. Convert string default values to proper types using `ScalarValue::cast_to()` at planning time
///
/// Important: PhysicalExprAdapter is specifically designed for rewriting filter predicates
/// that get pushed down to file scans. For handling missing columns in projections,
/// other mechanisms in DataFusion are used (like SchemaAdapter).
///
/// The metadata-based approach provides a flexible way to store default values as strings
-/// and cast them to the appropriate types at query time.
+/// and cast them to the appropriate types at planning time, avoiding runtime overhead.
pub async fn default_column_values() -> Result<()> {
println!("=== Creating example data with missing columns and default values ===");
@@ -138,8 +137,8 @@ pub async fn default_column_values() -> Result<()> {
println!("This example demonstrates how PhysicalExprAdapter works:");
println!("1. Physical schema only has 'id' and 'name' columns");
println!("2. Logical schema has 'id', 'name', 'status', and 'priority' columns with defaults");
- println!("3. Our custom adapter intercepts filter expressions on missing columns");
- println!("4. Default values from metadata are injected as cast expressions");
+ println!("3. Our custom adapter uses replace_columns_with_literals to inject default values");
+ println!("4. Default values from metadata are cast to proper types at planning time");
println!("5. The DefaultPhysicalExprAdapter handles other schema adaptations");
println!("\nNote: PhysicalExprAdapter is specifically for filter predicates.");
println!("For projection columns, different mechanisms handle missing columns.");
@@ -206,7 +205,7 @@ impl TableProvider for DefaultValueTableProvider {
}
fn schema(&self) -> SchemaRef {
- self.schema.clone()
+ Arc::clone(&self.schema)
}
fn table_type(&self) -> TableType {
@@ -227,7 +226,7 @@ impl TableProvider for DefaultValueTableProvider {
filters: &[Expr],
limit: Option<usize>,
) -> Result<Arc<dyn ExecutionPlan>> {
- let schema = self.schema.clone();
+ let schema = Arc::clone(&self.schema);
let df_schema = DFSchema::try_from(schema.clone())?;
let filter = state.create_physical_expr(
conjunction(filters.iter().cloned()).unwrap_or_else(|| lit(true)),
@@ -280,14 +279,15 @@ impl PhysicalExprAdapterFactory for DefaultValuePhysicalExprAdapterFactory {
physical_file_schema: SchemaRef,
) -> Arc<dyn PhysicalExprAdapter> {
let default_factory = DefaultPhysicalExprAdapterFactory;
- let default_adapter = default_factory
- .create(logical_file_schema.clone(), physical_file_schema.clone());
+ let default_adapter = default_factory.create(
+ Arc::clone(&logical_file_schema),
+ Arc::clone(&physical_file_schema),
+ );
Arc::new(DefaultValuePhysicalExprAdapter {
logical_file_schema,
physical_file_schema,
default_adapter,
- partition_values: Vec::new(),
})
}
}
@@ -299,98 +299,36 @@ struct DefaultValuePhysicalExprAdapter {
logical_file_schema: SchemaRef,
physical_file_schema: SchemaRef,
default_adapter: Arc<dyn PhysicalExprAdapter>,
- partition_values: Vec<(FieldRef, ScalarValue)>,
}
impl PhysicalExprAdapter for DefaultValuePhysicalExprAdapter {
fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {
- // First try our custom default value injection for missing columns
- let rewritten = expr
- .transform(|expr| {
- self.inject_default_values(
- expr,
- &self.logical_file_schema,
- &self.physical_file_schema,
- )
- })
- .data()?;
-
- // Then apply the default adapter as a fallback to handle standard schema differences
- // like type casting, partition column handling, etc.
- let default_adapter = if !self.partition_values.is_empty() {
- self.default_adapter
- .with_partition_values(self.partition_values.clone())
- } else {
- self.default_adapter.clone()
- };
-
- default_adapter.rewrite(rewritten)
- }
-
- fn with_partition_values(
- &self,
- partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter> {
- Arc::new(DefaultValuePhysicalExprAdapter {
- logical_file_schema: self.logical_file_schema.clone(),
- physical_file_schema: self.physical_file_schema.clone(),
- default_adapter: self.default_adapter.clone(),
- partition_values,
- })
- }
-}
-
-impl DefaultValuePhysicalExprAdapter {
- fn inject_default_values(
- &self,
- expr: Arc<dyn PhysicalExpr>,
- logical_file_schema: &Schema,
- physical_file_schema: &Schema,
- ) -> Result<Transformed<Arc<dyn PhysicalExpr>>> {
- if let Some(column) = expr.as_any().downcast_ref::<Column>() {
- let column_name = column.name();
-
- // Check if this column exists in the physical schema
- if physical_file_schema.index_of(column_name).is_err() {
- // Column is missing from physical schema, check if logical schema has a default
- if let Ok(logical_field) =
- logical_file_schema.field_with_name(column_name)
- {
- if let Some(default_value_str) =
- logical_field.metadata().get(DEFAULT_VALUE_METADATA_KEY)
- {
- // Create a string literal and wrap it in a cast expression
- let default_literal = self.create_default_value_expr(
- default_value_str,
- logical_field.data_type(),
- )?;
- return Ok(Transformed::yes(default_literal));
- }
- }
+ // Pre-compute replacements for missing columns with default values
+ let mut replacements = HashMap::new();
+ for field in self.logical_file_schema.fields() {
+ // Skip columns that exist in physical schema
+ if self.physical_file_schema.index_of(field.name()).is_ok() {
+ continue;
}
- }
-
- // No transformation needed
- Ok(Transformed::no(expr))
- }
- fn create_default_value_expr(
- &self,
- value_str: &str,
- data_type: &DataType,
- ) -> Result<Arc<dyn PhysicalExpr>> {
- // Create a string literal with the default value
- let string_literal =
- Arc::new(Literal::new(ScalarValue::Utf8(Some(value_str.to_string()))));
-
- // If the target type is already Utf8, return the string literal directly
- if matches!(data_type, DataType::Utf8) {
- return Ok(string_literal);
+ // Check if this missing column has a default value in metadata
+ if let Some(default_str) = field.metadata().get(DEFAULT_VALUE_METADATA_KEY) {
+ // Create a Utf8 ScalarValue from the string and cast it to the target type
+ let string_value = ScalarValue::Utf8(Some(default_str.to_string()));
+ let typed_value = string_value.cast_to(field.data_type())?;
+ replacements.insert(field.name().as_str(), typed_value);
+ }
}
- // Otherwise, wrap the string literal in a cast expression
- let cast_expr = Arc::new(CastExpr::new(string_literal, data_type.clone(), None));
+ // Replace columns with their default literals if any
+ let rewritten = if !replacements.is_empty() {
+ let refs: HashMap<_, _> = replacements.iter().map(|(k, v)| (*k, v)).collect();
+ replace_columns_with_literals(expr, &refs)?
+ } else {
+ expr
+ };
- Ok(cast_expr)
+ // Apply the default adapter as a fallback for other schema adaptations
+ self.default_adapter.rewrite(rewritten)
}
}
diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rs
index baa0a2e660..a4b0de9301 100644
--- a/datafusion-examples/examples/data_io/json_shredding.rs
+++ b/datafusion-examples/examples/data_io/json_shredding.rs
@@ -21,7 +21,7 @@ use std::any::Any;
use std::sync::Arc;
use arrow::array::{RecordBatch, StringArray};
-use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::assert_batches_eq;
use datafusion::common::tree_node::{
@@ -277,14 +277,14 @@ impl PhysicalExprAdapterFactory for ShreddedJsonRewriterFactory {
physical_file_schema: SchemaRef,
) -> Arc<dyn PhysicalExprAdapter> {
let default_factory = DefaultPhysicalExprAdapterFactory;
- let default_adapter = default_factory
- .create(logical_file_schema.clone(), physical_file_schema.clone());
+ let default_adapter = default_factory.create(
+ Arc::clone(&logical_file_schema),
+ Arc::clone(&physical_file_schema),
+ );
Arc::new(ShreddedJsonRewriter {
- logical_file_schema,
physical_file_schema,
default_adapter,
- partition_values: Vec::new(),
})
}
}
@@ -293,10 +293,8 @@ impl PhysicalExprAdapterFactory for ShreddedJsonRewriterFactory {
/// and wraps DefaultPhysicalExprAdapter for standard schema adaptation
#[derive(Debug)]
struct ShreddedJsonRewriter {
- logical_file_schema: SchemaRef,
physical_file_schema: SchemaRef,
default_adapter: Arc<dyn PhysicalExprAdapter>,
- partition_values: Vec<(FieldRef, ScalarValue)>,
}
impl PhysicalExprAdapter for ShreddedJsonRewriter {
@@ -307,27 +305,8 @@ impl PhysicalExprAdapter for ShreddedJsonRewriter {
.data()?;
// Then apply the default adapter as a fallback to handle standard schema differences
- // like type casting, missing columns, and partition column handling
- let default_adapter = if !self.partition_values.is_empty() {
- self.default_adapter
- .with_partition_values(self.partition_values.clone())
- } else {
- self.default_adapter.clone()
- };
-
- default_adapter.rewrite(rewritten)
- }
-
- fn with_partition_values(
- &self,
- partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter> {
- Arc::new(ShreddedJsonRewriter {
- logical_file_schema: self.logical_file_schema.clone(),
- physical_file_schema: self.physical_file_schema.clone(),
- default_adapter: self.default_adapter.clone(),
- partition_values,
- })
+ // like type casting and missing columns
+ self.default_adapter.rewrite(rewritten)
}
}
diff --git a/datafusion/common/src/pruning.rs b/datafusion/common/src/pruning.rs
index a91b5176d4..5a7598ea1f 100644
--- a/datafusion/common/src/pruning.rs
+++ b/datafusion/common/src/pruning.rs
@@ -135,6 +135,10 @@ pub trait PruningStatistics {
/// This feeds into [`CompositePruningStatistics`] to allow pruning
/// with filters that depend both on partition columns and data columns
/// (e.g. `WHERE partition_col = data_col`).
+#[deprecated(
+ since = "52.0.0",
+ note = "This struct is no longer used internally. Use `replace_columns_with_literals` from `datafusion-physical-expr-adapter` to substitute partition column values before pruning. It will be removed in 58.0.0 or 6 months after 52.0.0 is released, whichever comes first."
+)]
#[derive(Clone)]
pub struct PartitionPruningStatistics {
/// Values for each column for each container.
@@ -156,6 +160,7 @@ pub struct PartitionPruningStatistics {
partition_schema: SchemaRef,
}
+#[expect(deprecated)]
impl PartitionPruningStatistics {
/// Create a new instance of [`PartitionPruningStatistics`].
///
@@ -232,6 +237,7 @@ impl PartitionPruningStatistics {
}
}
+#[expect(deprecated)]
impl PruningStatistics for PartitionPruningStatistics {
fn min_values(&self, column: &Column) -> Option<ArrayRef> {
let index = self.partition_schema.index_of(column.name()).ok()?;
@@ -439,10 +445,15 @@ impl PruningStatistics for PrunableStatistics {
/// the first one is returned without any regard for completeness or accuracy.
/// That is: if the first statistics has information for a column, even if it is incomplete,
/// that is returned even if a later statistics has more complete information.
+#[deprecated(
+ since = "52.0.0",
+ note = "This struct is no longer used internally. It may be removed in 58.0.0 or 6 months after 52.0.0 is released, whichever comes first. Please open an issue if you have a use case for it."
+)]
pub struct CompositePruningStatistics {
pub statistics: Vec<Box<dyn PruningStatistics>>,
}
+#[expect(deprecated)]
impl CompositePruningStatistics {
/// Create a new instance of [`CompositePruningStatistics`] from
/// a vector of [`PruningStatistics`].
@@ -457,6 +468,7 @@ impl CompositePruningStatistics {
}
}
+#[expect(deprecated)]
impl PruningStatistics for CompositePruningStatistics {
fn min_values(&self, column: &Column) -> Option<ArrayRef> {
for stats in &self.statistics {
@@ -513,6 +525,7 @@ impl PruningStatistics for CompositePruningStatistics {
}
#[cfg(test)]
+#[expect(deprecated)]
mod tests {
use crate::{
ColumnStatistics,
diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rs
index 28faea9a68..52564663b9 100644
--- a/datafusion/core/src/datasource/mod.rs
+++ b/datafusion/core/src/datasource/mod.rs
@@ -56,7 +56,7 @@ mod tests {
use ::object_store::{path::Path, ObjectMeta};
use arrow::{
array::Int32Array,
- datatypes::{DataType, Field, FieldRef, Schema, SchemaRef},
+ datatypes::{DataType, Field, Schema, SchemaRef},
record_batch::RecordBatch,
};
use datafusion_common::{
@@ -209,11 +209,10 @@ mod tests {
impl PhysicalExprAdapterFactory for TestPhysicalExprAdapterFactory {
fn create(
&self,
- logical_file_schema: SchemaRef,
+ _logical_file_schema: SchemaRef,
physical_file_schema: SchemaRef,
) -> Arc<dyn PhysicalExprAdapter> {
Arc::new(TestPhysicalExprAdapter {
- logical_file_schema,
physical_file_schema,
})
}
@@ -221,7 +220,6 @@ mod tests {
#[derive(Debug)]
struct TestPhysicalExprAdapter {
- logical_file_schema: SchemaRef,
physical_file_schema: SchemaRef,
}
@@ -243,15 +241,5 @@ mod tests {
})
.data()
}
-
- fn with_partition_values(
- &self,
- _partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter> {
- Arc::new(TestPhysicalExprAdapter {
- logical_file_schema: self.logical_file_schema.clone(),
- physical_file_schema: self.physical_file_schema.clone(),
- })
- }
}
}
diff --git a/datafusion/core/tests/parquet/schema_adapter.rs b/datafusion/core/tests/parquet/schema_adapter.rs
index 541785319c..3c12862f38 100644
--- a/datafusion/core/tests/parquet/schema_adapter.rs
+++ b/datafusion/core/tests/parquet/schema_adapter.rs
@@ -18,7 +18,7 @@
use std::sync::Arc;
use arrow::array::{record_batch, RecordBatch};
-use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef};
+use arrow_schema::{DataType, Field, Schema, SchemaRef};
use bytes::{BufMut, BytesMut};
use datafusion::assert_batches_eq;
use datafusion::common::Result;
@@ -123,17 +123,6 @@ impl PhysicalExprAdapter for CustomPhysicalExprAdapter {
.data()?;
self.inner.rewrite(expr)
}
-
- fn with_partition_values(
- &self,
- partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter> {
- assert!(
- partition_values.is_empty(),
- "Partition values are not supported in this test"
- );
- Arc::new(self.clone())
- }
}
#[tokio::test]
diff --git a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs
index e9885e4014..2a0c133312 100644
--- a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs
+++ b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
use arrow::array::RecordBatch;
-use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef};
+use arrow_schema::{DataType, Field, Schema, SchemaRef};
use bytes::{BufMut, BytesMut};
use datafusion::common::Result;
use datafusion::config::{ConfigOptions, TableParquetOptions};
@@ -35,7 +35,7 @@ use datafusion::prelude::SessionContext;
use datafusion_common::config::CsvOptions;
use datafusion_common::record_batch;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion_common::{ColumnStatistics, ScalarValue};
+use datafusion_common::ColumnStatistics;
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
use datafusion_datasource::schema_adapter::{
SchemaAdapter, SchemaAdapterFactory, SchemaMapper,
@@ -193,11 +193,10 @@ struct UppercasePhysicalExprAdapterFactory;
impl PhysicalExprAdapterFactory for UppercasePhysicalExprAdapterFactory {
fn create(
&self,
- logical_file_schema: SchemaRef,
+ _logical_file_schema: SchemaRef,
physical_file_schema: SchemaRef,
) -> Arc<dyn PhysicalExprAdapter> {
Arc::new(UppercasePhysicalExprAdapter {
- logical_file_schema,
physical_file_schema,
})
}
@@ -205,7 +204,6 @@ impl PhysicalExprAdapterFactory for UppercasePhysicalExprAdapterFactory {
#[derive(Debug)]
struct UppercasePhysicalExprAdapter {
- logical_file_schema: SchemaRef,
physical_file_schema: SchemaRef,
}
@@ -226,16 +224,6 @@ impl PhysicalExprAdapter for UppercasePhysicalExprAdapter {
})
.data()
}
-
- fn with_partition_values(
- &self,
- _partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter> {
- Arc::new(Self {
- logical_file_schema: self.logical_file_schema.clone(),
- physical_file_schema: self.physical_file_schema.clone(),
- })
- }
}
#[derive(Clone)]
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index fe1fab7a0e..8f521c3f4e 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -27,15 +27,17 @@ use arrow::array::RecordBatch;
use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr::utils::reassign_expr_columns;
+use datafusion_physical_expr_adapter::replace_columns_with_literals;
+use std::collections::HashMap;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
-use arrow::datatypes::{FieldRef, SchemaRef, TimeUnit};
+use arrow::datatypes::{SchemaRef, TimeUnit};
use datafusion_common::encryption::FileDecryptionProperties;
-use datafusion_common::{exec_err, DataFusionError, Result};
-use datafusion_datasource::PartitionedFile;
+use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue};
+use datafusion_datasource::{PartitionedFile, TableSchema};
use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
use datafusion_physical_expr_common::physical_expr::{
@@ -51,7 +53,6 @@ use datafusion_common::config::EncryptionFactoryOptions;
#[cfg(feature = "parquet_encryption")]
use datafusion_execution::parquet_encryption::EncryptionFactory;
use futures::{ready, Stream, StreamExt, TryStreamExt};
-use itertools::Itertools;
use log::debug;
use parquet::arrow::arrow_reader::metrics::ArrowReaderMetrics;
use parquet::arrow::arrow_reader::{
@@ -73,11 +74,8 @@ pub(super) struct ParquetOpener {
pub limit: Option<usize>,
/// Optional predicate to apply during the scan
pub predicate: Option<Arc<dyn PhysicalExpr>>,
- /// Schema of the output table without partition columns.
- /// This is the schema we coerce the physical file schema into.
- pub logical_file_schema: SchemaRef,
- /// Partition columns
- pub partition_fields: Vec<FieldRef>,
+ /// Table schema, including partition columns.
+ pub table_schema: TableSchema,
/// Optional hint for how large the initial request to read parquet metadata
/// should be
pub metadata_size_hint: Option<usize>,
@@ -139,10 +137,49 @@ impl FileOpener for ParquetOpener {
let batch_size = self.batch_size;
+ // Replace any partition column references in the predicate
+ // with their literal values from this file's partition values.
+ //
+ // For example, given
+ // 1. `region` is a partition column,
+ // 2. predicate `host IN ('us-east-1', 'eu-central-1')`:
+ // 3. The file path is `/data/region=us-west-2/...`
+ // (that is the partition column value is `us-west-2`)
+ //
+ // The predicate would be rewritten to
+ // ```sql
+ // 'us-west-2` IN ('us-east-1', 'eu-central-1')
+ // ```
+ // which can be further simplified to `FALSE`, meaning
+ // the file can be skipped entirely.
+ //
+ // While this particular optimization is done during logical planning,
+ // there are other cases where partition columns may appear in more
+ // complex predicates that cannot be simplified until we are about to
+ // open the file (such as dynamic predicates)
+ let mut predicate = {
+ // Only replace partition columns if there are partition values.
+ // For non-partitioned tables, skip the clone and replacement traversal.
+ if partitioned_file.partition_values.is_empty() {
+ self.predicate.clone()
+ } else {
+ let partition_values: HashMap<&str, &ScalarValue> = self
+ .table_schema
+ .table_partition_cols()
+ .iter()
+ .zip(partitioned_file.partition_values.iter())
+ .map(|(field, value)| (field.name().as_str(), value))
+ .collect();
+
+ self.predicate
+ .clone()
+ .map(|p| replace_columns_with_literals(p, &partition_values))
+ .transpose()?
+ }
+ };
+
let projection = Arc::clone(&self.projection);
- let mut predicate = self.predicate.clone();
- let logical_file_schema = Arc::clone(&self.logical_file_schema);
- let partition_fields = self.partition_fields.clone();
+ let logical_file_schema = Arc::clone(self.table_schema.table_schema());
let reorder_predicates = self.reorder_filters;
let pushdown_filters = self.pushdown_filters;
let force_filter_selections = self.force_filter_selections;
@@ -155,7 +192,7 @@ impl FileOpener for ParquetOpener {
.global_counter("num_predicate_creation_errors");
let expr_adapter_factory = self.expr_adapter_factory.clone();
- let mut predicate_file_schema = Arc::clone(&self.logical_file_schema);
+ let mut predicate_file_schema = Arc::clone(self.table_schema.file_schema());
let enable_page_index = self.enable_page_index;
#[cfg(feature = "parquet_encryption")]
@@ -178,20 +215,33 @@ impl FileOpener for ParquetOpener {
// we can end the stream early.
let mut file_pruner = predicate
.as_ref()
- .map(|p| {
- Ok::<_, DataFusionError>(
- (is_dynamic_physical_expr(p) | partitioned_file.has_statistics())
- .then_some(FilePruner::new(
- Arc::clone(p),
- &logical_file_schema,
- partition_fields.clone(),
- partitioned_file.clone(),
- predicate_creation_errors.clone(),
- )?),
- )
+ .filter(|p| {
+ // Make a FilePruner only if there is either
+ // 1. a dynamic expr in the predicate
+ // 2. the file has file-level statistics.
+ //
+ // File-level statistics may prune the file without loading
+ // any row groups or metadata.
+ //
+ // Dynamic filters may prune the file after initial
+ // planning, as the dynamic filter is updated during
+ // execution.
+ //
+ // The case where there is a dynamic filter but no
+ // statistics corresponds to a dynamic filter that
+ // references partition columns. While rare, this is possible
+ // e.g. `select * from table order by partition_col limit
+ // 10` could hit this condition.
+ is_dynamic_physical_expr(p) || partitioned_file.has_statistics()
})
- .transpose()?
- .flatten();
+ .and_then(|p| {
+ FilePruner::try_new(
+ Arc::clone(p),
+ &logical_file_schema,
+ &partitioned_file,
+ predicate_creation_errors.clone(),
+ )
+ });
if let Some(file_pruner) = &mut file_pruner {
if file_pruner.should_prune()? {
@@ -266,17 +316,11 @@ impl FileOpener for ParquetOpener {
if let Some(expr_adapter_factory) = expr_adapter_factory.as_ref() {
predicate = predicate
.map(|p| {
- let partition_values = partition_fields
- .iter()
- .cloned()
- .zip(partitioned_file.partition_values.clone())
- .collect_vec();
let expr = expr_adapter_factory
.create(
Arc::clone(&logical_file_schema),
Arc::clone(&physical_file_schema),
)
- .with_partition_values(partition_values)
.rewrite(p)?;
// After rewriting to the file schema, further simplifications may be possible.
// For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`
@@ -317,18 +361,10 @@ impl FileOpener for ParquetOpener {
let mut projection =
ProjectionExprs::from_indices(&projection, &logical_file_schema);
if let Some(expr_adapter_factory) = expr_adapter_factory {
- let adapter = expr_adapter_factory
- .create(
- Arc::clone(&logical_file_schema),
- Arc::clone(&physical_file_schema),
- )
- .with_partition_values(
- partition_fields
- .iter()
- .cloned()
- .zip(partitioned_file.partition_values.clone())
- .collect_vec(),
- );
+ let adapter = expr_adapter_factory.create(
+ Arc::clone(&logical_file_schema),
+ Arc::clone(&physical_file_schema),
+ );
projection = projection.try_map_exprs(|expr| adapter.rewrite(expr))?;
}
let indices = projection.column_indices();
@@ -550,6 +586,7 @@ impl<S> EarlyStoppingStream<S> {
}
}
}
+
impl<S> EarlyStoppingStream<S>
where
S: Stream<Item = Result<RecordBatch>> + Unpin,
@@ -786,7 +823,7 @@ mod test {
record_batch, stats::Precision, ColumnStatistics, DataFusionError, ScalarValue,
Statistics,
};
- use datafusion_datasource::{file_stream::FileOpener, PartitionedFile};
+ use datafusion_datasource::{file_stream::FileOpener, PartitionedFile, TableSchema};
use datafusion_expr::{col, lit};
use datafusion_physical_expr::{
expressions::DynamicFilterPhysicalExpr, planner::logical2physical, PhysicalExpr,
@@ -877,13 +914,12 @@ mod test {
batch_size: 1024,
limit: None,
predicate: Some(predicate),
- logical_file_schema: schema.clone(),
+ table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),
metadata_size_hint: None,
metrics: ExecutionPlanMetricsSet::new(),
parquet_file_reader_factory: Arc::new(
DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
),
- partition_fields: vec![],
pushdown_filters: false, // note that this is false!
reorder_filters: false,
force_filter_selections: false,
@@ -946,17 +982,15 @@ mod test {
batch_size: 1024,
limit: None,
predicate: Some(predicate),
- logical_file_schema: file_schema.clone(),
+ table_schema: TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ ),
metadata_size_hint: None,
metrics: ExecutionPlanMetricsSet::new(),
parquet_file_reader_factory: Arc::new(
DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
),
- partition_fields: vec![Arc::new(Field::new(
- "part",
- DataType::Int32,
- false,
- ))],
pushdown_filters: false, // note that this is false!
reorder_filters: false,
force_filter_selections: false,
@@ -1035,17 +1069,15 @@ mod test {
batch_size: 1024,
limit: None,
predicate: Some(predicate),
- logical_file_schema: file_schema.clone(),
+ table_schema: TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ ),
metadata_size_hint: None,
metrics: ExecutionPlanMetricsSet::new(),
parquet_file_reader_factory: Arc::new(
DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
),
- partition_fields: vec![Arc::new(Field::new(
- "part",
- DataType::Int32,
- false,
- ))],
pushdown_filters: false, // note that this is false!
reorder_filters: false,
force_filter_selections: false,
@@ -1127,17 +1159,15 @@ mod test {
batch_size: 1024,
limit: None,
predicate: Some(predicate),
- logical_file_schema: file_schema.clone(),
+ table_schema: TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ ),
metadata_size_hint: None,
metrics: ExecutionPlanMetricsSet::new(),
parquet_file_reader_factory: Arc::new(
DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
),
- partition_fields: vec![Arc::new(Field::new(
- "part",
- DataType::Int32,
- false,
- ))],
pushdown_filters: true, // note that this is true!
reorder_filters: true,
force_filter_selections: false,
@@ -1206,10 +1236,18 @@ mod test {
u64::try_from(data_size).unwrap(),
);
file.partition_values = vec![ScalarValue::Int32(Some(1))];
+ file.statistics = Some(Arc::new(
+ Statistics::default().add_column_statistics(
+ ColumnStatistics::new_unknown()
+ .with_min_value(Precision::Exact(ScalarValue::Int32(Some(1))))
+ .with_max_value(Precision::Exact(ScalarValue::Int32(Some(3))))
+ .with_null_count(Precision::Exact(0)),
+ ),
+ ));
let table_schema = Arc::new(Schema::new(vec![
- Field::new("part", DataType::Int32, false),
Field::new("a", DataType::Int32, false),
+ Field::new("part", DataType::Int32, false),
]));
let make_opener = |predicate| {
@@ -1219,17 +1257,15 @@ mod test {
batch_size: 1024,
limit: None,
predicate: Some(predicate),
- logical_file_schema: file_schema.clone(),
+ table_schema: TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ ),
metadata_size_hint: None,
metrics: ExecutionPlanMetricsSet::new(),
parquet_file_reader_factory: Arc::new(
DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
),
- partition_fields: vec![Arc::new(Field::new(
- "part",
- DataType::Int32,
- false,
- ))],
pushdown_filters: false, // note that this is false!
reorder_filters: false,
force_filter_selections: false,
@@ -1246,16 +1282,37 @@ mod test {
}
};
- // Filter should NOT match the stats but the file is never attempted to be pruned because the filters are not dynamic
- let expr = col("part").eq(lit(2));
+ // This filter could prune based on statistics, but since it's not dynamic it's not applied for pruning
+ // (the assumption is this happened already at planning time)
+ let expr = col("a").eq(lit(42));
let predicate = logical2physical(&expr, &table_schema);
let opener = make_opener(predicate);
let stream = opener.open(file.clone()).unwrap().await.unwrap();
let (num_batches, num_rows) = count_batches_and_rows(stream).await;
- assert_eq!(num_batches, 1);
- assert_eq!(num_rows, 3);
+ assert_eq!(num_batches, 0);
+ assert_eq!(num_rows, 0);
+
+ // If we make the filter dynamic, it should prune.
+ // This allows dynamic filters to prune partitions/files even if they are populated late into execution.
+ let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));
+ let opener = make_opener(predicate);
+ let stream = opener.open(file.clone()).unwrap().await.unwrap();
+ let (num_batches, num_rows) = count_batches_and_rows(stream).await;
+ assert_eq!(num_batches, 0);
+ assert_eq!(num_rows, 0);
+
+ // If we have a filter that touches partition columns only and is dynamic, it should prune even if there are no stats.
+ file.statistics = Some(Arc::new(Statistics::new_unknown(&file_schema)));
+ let expr = col("part").eq(lit(2));
+ let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));
+ let opener = make_opener(predicate);
+ let stream = opener.open(file.clone()).unwrap().await.unwrap();
+ let (num_batches, num_rows) = count_batches_and_rows(stream).await;
+ assert_eq!(num_batches, 0);
+ assert_eq!(num_rows, 0);
- // If we make the filter dynamic, it should prune
+ // Similarly a filter that combines partition and data columns should prune even if there are no stats.
+ let expr = col("part").eq(lit(2)).and(col("a").eq(lit(42)));
let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));
let opener = make_opener(predicate);
let stream = opener.open(file.clone()).unwrap().await.unwrap();
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 1abe126369..d2a140c4f8 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -533,8 +533,7 @@ impl FileSource for ParquetSource {
.expect("Batch size must set before creating ParquetOpener"),
limit: base_config.limit,
predicate: self.predicate.clone(),
- logical_file_schema: Arc::clone(base_config.file_schema()),
- partition_fields: base_config.table_partition_cols().clone(),
+ table_schema: self.table_schema.clone(),
metadata_size_hint: self.metadata_size_hint,
metrics: self.metrics().clone(),
parquet_file_reader_factory,
diff --git a/datafusion/physical-expr-adapter/src/lib.rs b/datafusion/physical-expr-adapter/src/lib.rs
index 9586372803..d7c750e4a1 100644
--- a/datafusion/physical-expr-adapter/src/lib.rs
+++ b/datafusion/physical-expr-adapter/src/lib.rs
@@ -30,5 +30,5 @@ pub mod schema_rewriter;
pub use schema_rewriter::{
DefaultPhysicalExprAdapter, DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter,
- PhysicalExprAdapterFactory,
+ PhysicalExprAdapterFactory, replace_columns_with_literals,
};
diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
index cbb4687ab5..6f040c1a05 100644
--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs
+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
@@ -15,12 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-//! Physical expression schema rewriting utilities
+//! Physical expression schema rewriting utilities: [`PhysicalExprAdapter`],
+//! [`PhysicalExprAdapterFactory`], default implementations,
+//! and [`replace_columns_with_literals`].
+use std::collections::HashMap;
use std::sync::Arc;
use arrow::compute::can_cast_types;
-use arrow::datatypes::{DataType, FieldRef, Schema, SchemaRef};
+use arrow::datatypes::{DataType, Schema, SchemaRef};
use datafusion_common::{
Result, ScalarValue, exec_err,
nested_struct::validate_struct_compatibility,
@@ -34,29 +37,52 @@ use datafusion_physical_expr::{
};
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
-/// Trait for adapting physical expressions to match a target schema.
+/// Replace column references in the given physical expression with literal values.
///
-/// This is used in file scans to rewrite expressions so that they can be evaluated
-/// against the physical schema of the file being scanned. It allows for handling
-/// differences between logical and physical schemas, such as type mismatches or missing columns.
+/// Some use cases for this include:
+/// - Partition column pruning: When scanning partitioned data, partition column references
+/// can be replaced with their literal values for the specific partition being scanned.
+/// - Constant folding: In some cases, columns that can be proven to be constant
+/// from statistical analysis may be replaced with their literal values to optimize expression evaluation.
+/// - Filling in non-null default values: in a custom [`PhysicalExprAdapter`] implementation,
+/// column references can be replaced with default literal values instead of nulls.
///
-/// ## Overview
+/// # Arguments
+/// - `expr`: The physical expression in which to replace column references.
+/// - `replacements`: A mapping from column names to their corresponding literal `ScalarValue`s.
///
-/// The `PhysicalExprAdapter` allows rewriting physical expressions to match different schemas, including:
+/// # Returns
+/// - `Result<Arc<dyn PhysicalExpr>>`: The rewritten physical expression with columns replaced by literals.
+pub fn replace_columns_with_literals(
+ expr: Arc<dyn PhysicalExpr>,
+ replacements: &HashMap<&str, &ScalarValue>,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ expr.transform(|expr| {
+ if let Some(column) = expr.as_any().downcast_ref::<Column>()
+ && let Some(replacement_value) = replacements.get(column.name())
+ {
+ return Ok(Transformed::yes(expressions::lit(
+ (*replacement_value).clone(),
+ )));
+ }
+ Ok(Transformed::no(expr))
+ })
+ .data()
+}
+
+/// Trait for adapting [`PhysicalExpr`] expressions to match a target schema.
///
-/// - **Type casting**: When logical and physical schemas have different types, expressions are
-/// automatically wrapped with cast operations. For example, `lit(ScalarValue::Int32(123)) = int64_column`
-/// gets rewritten to `lit(ScalarValue::Int32(123)) = cast(int64_column, 'Int32')`.
-/// Note that this does not attempt to simplify such expressions - that is done by shared simplifiers.
+/// This is used in file scans to rewrite expressions so that they can be
+/// evaluated against the physical schema of the file being scanned. It allows
+/// for handling differences between logical and physical schemas, such as type
+/// mismatches or missing columns common in [Schema evolution] scenarios.
///
-/// - **Missing columns**: When a column exists in the logical schema but not in the physical schema,
-/// references to it are replaced with null literals.
+/// [Schema evolution]: https://www.dremio.com/wiki/schema-evolution/
///
-/// - **Struct field access**: Expressions like `struct_column.field_that_is_missing_in_schema` are
-/// rewritten to `null` when the field doesn't exist in the physical schema.
+/// ## Default Implementations
///
-/// - **Partition columns**: Partition column references can be replaced with their literal values
-/// when scanning specific partitions.
+/// The default implementation [`DefaultPhysicalExprAdapter`] handles common
+/// cases.
///
/// ## Custom Implementations
///
@@ -93,17 +119,6 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
/// Ok(Transformed::no(expr))
/// }).data()
/// }
-///
-/// fn with_partition_values(
-/// &self,
-/// partition_values: Vec<(FieldRef, ScalarValue)>,
-/// ) -> Arc<dyn PhysicalExprAdapter> {
-/// // For simplicity, this example ignores partition values
-/// Arc::new(CustomPhysicalExprAdapter {
-/// logical_file_schema: self.logical_file_schema.clone(),
-/// physical_file_schema: self.physical_file_schema.clone(),
-/// })
-/// }
/// }
///
/// #[derive(Debug)]
@@ -136,14 +151,15 @@ pub trait PhysicalExprAdapter: Send + Sync + std::fmt::Debug {
///
/// Returns:
/// - `Arc<dyn PhysicalExpr>`: The rewritten physical expression that can be evaluated against the physical schema.
+ ///
+ /// See Also:
+ /// - [`replace_columns_with_literals`]: for replacing partition column references with their literal values.
fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>>;
-
- fn with_partition_values(
- &self,
- partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter>;
}
+/// Creates instances of [`PhysicalExprAdapter`] for given logical and physical schemas.
+///
+/// See [`DefaultPhysicalExprAdapterFactory`] for the default implementation.
pub trait PhysicalExprAdapterFactory: Send + Sync + std::fmt::Debug {
/// Create a new instance of the physical expression adapter.
fn create(
@@ -165,20 +181,39 @@ impl PhysicalExprAdapterFactory for DefaultPhysicalExprAdapterFactory {
Arc::new(DefaultPhysicalExprAdapter {
logical_file_schema,
physical_file_schema,
- partition_values: Vec::new(),
})
}
}
-/// Default implementation for rewriting physical expressions to match different schemas.
+/// Default implementation of [`PhysicalExprAdapter`] for rewriting physical
+/// expressions to match different schemas.
+///
+/// ## Overview
+///
+/// [`DefaultPhysicalExprAdapter`] rewrites physical expressions to match
+/// different schemas, including:
+///
+/// - **Type casting**: When logical and physical schemas have different types, expressions are
+/// automatically wrapped with cast operations. For example, `lit(ScalarValue::Int32(123)) = int64_column`
+/// gets rewritten to `lit(ScalarValue::Int32(123)) = cast(int64_column, 'Int32')`.
+/// Note that this does not attempt to simplify such expressions - that is done by shared simplifiers.
+///
+/// - **Missing columns**: When a column exists in the logical schema but not in the physical schema,
+/// references to it are replaced with null literals.
+///
+/// - **Struct field access**: Expressions like `struct_column.field_that_is_missing_in_schema` are
+/// rewritten to `null` when the field doesn't exist in the physical schema.
+///
+/// - **Default column values**: Partition column references can be replaced with their literal values
+/// when scanning specific partitions. See [`replace_columns_with_literals`] for more details.
///
/// # Example
///
/// ```rust
-/// use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapterFactory};
-/// use arrow::datatypes::Schema;
-/// use std::sync::Arc;
-///
+/// # use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapterFactory};
+/// # use arrow::datatypes::Schema;
+/// # use std::sync::Arc;
+/// #
/// # fn example(
/// # predicate: std::sync::Arc<dyn datafusion_physical_expr_common::physical_expr::PhysicalExpr>,
/// # physical_file_schema: &Schema,
@@ -194,7 +229,6 @@ impl PhysicalExprAdapterFactory for DefaultPhysicalExprAdapterFactory {
pub struct DefaultPhysicalExprAdapter {
logical_file_schema: SchemaRef,
physical_file_schema: SchemaRef,
- partition_values: Vec<(FieldRef, ScalarValue)>,
}
impl DefaultPhysicalExprAdapter {
@@ -206,7 +240,6 @@ impl DefaultPhysicalExprAdapter {
Self {
logical_file_schema,
physical_file_schema,
- partition_values: Vec::new(),
}
}
}
@@ -216,27 +249,15 @@ impl PhysicalExprAdapter for DefaultPhysicalExprAdapter {
let rewriter = DefaultPhysicalExprAdapterRewriter {
logical_file_schema: &self.logical_file_schema,
physical_file_schema: &self.physical_file_schema,
- partition_fields: &self.partition_values,
};
expr.transform(|expr| rewriter.rewrite_expr(Arc::clone(&expr)))
.data()
}
-
- fn with_partition_values(
- &self,
- partition_values: Vec<(FieldRef, ScalarValue)>,
- ) -> Arc<dyn PhysicalExprAdapter> {
- Arc::new(DefaultPhysicalExprAdapter {
- partition_values,
- ..self.clone()
- })
- }
}
struct DefaultPhysicalExprAdapterRewriter<'a> {
logical_file_schema: &'a Schema,
physical_file_schema: &'a Schema,
- partition_fields: &'a [(FieldRef, ScalarValue)],
}
impl<'a> DefaultPhysicalExprAdapterRewriter<'a> {
@@ -347,10 +368,6 @@ impl<'a> DefaultPhysicalExprAdapterRewriter<'a> {
{
Ok(field) => field,
Err(e) => {
- // If the column is a partition field, we can use the partition value
- if let Some(partition_value) = self.get_partition_value(column.name()) {
- return Ok(Transformed::yes(expressions::lit(partition_value)));
- }
// This can be hit if a custom rewrite injected a reference to a column that doesn't exist in the logical schema.
// For example, a pre-computed column that is kept only in the physical schema.
// If the column exists in the physical schema, we can still use it.
@@ -446,13 +463,6 @@ impl<'a> DefaultPhysicalExprAdapterRewriter<'a> {
Ok(Transformed::yes(cast_expr))
}
-
- fn get_partition_value(&self, column_name: &str) -> Option<ScalarValue> {
- self.partition_fields
- .iter()
- .find(|(field, _)| field.name() == column_name)
- .map(|(_, value)| value.clone())
- }
}
#[cfg(test)]
@@ -698,30 +708,51 @@ mod tests {
}
#[test]
- fn test_rewrite_partition_column() -> Result<()> {
- let (physical_schema, logical_schema) = create_test_schema();
-
- let partition_field =
- Arc::new(Field::new("partition_col", DataType::Utf8, false));
+ fn test_replace_columns_with_literals() -> Result<()> {
let partition_value = ScalarValue::Utf8(Some("test_value".to_string()));
- let partition_values = vec![(partition_field, partition_value)];
-
- let factory = DefaultPhysicalExprAdapterFactory;
- let adapter = factory.create(Arc::new(logical_schema), Arc::new(physical_schema));
- let adapter = adapter.with_partition_values(partition_values);
+ let replacements = HashMap::from([("partition_col", &partition_value)]);
- let column_expr = Arc::new(Column::new("partition_col", 0));
- let result = adapter.rewrite(column_expr)?;
+ let column_expr =
+ Arc::new(Column::new("partition_col", 0)) as Arc<dyn PhysicalExpr>;
+ let result = replace_columns_with_literals(column_expr, &replacements)?;
// Should be replaced with the partition value
- if let Some(literal) = result.as_any().downcast_ref::<expressions::Literal>() {
- assert_eq!(
- *literal.value(),
- ScalarValue::Utf8(Some("test_value".to_string()))
- );
- } else {
- panic!("Expected literal expression");
- }
+ let literal = result
+ .as_any()
+ .downcast_ref::<expressions::Literal>()
+ .expect("Expected literal expression");
+ assert_eq!(*literal.value(), partition_value);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_replace_columns_with_literals_no_match() -> Result<()> {
+ let value = ScalarValue::Utf8(Some("test_value".to_string()));
+ let replacements = HashMap::from([("other_col", &value)]);
+
+ let column_expr =
+ Arc::new(Column::new("partition_col", 0)) as Arc<dyn PhysicalExpr>;
+ let result = replace_columns_with_literals(column_expr, &replacements)?;
+
+ assert!(result.as_any().downcast_ref::<Column>().is_some());
+ Ok(())
+ }
+
+ #[test]
+ fn test_replace_columns_with_literals_nested_expr() -> Result<()> {
+ let value_a = ScalarValue::Int64(Some(10));
+ let value_b = ScalarValue::Int64(Some(20));
+ let replacements = HashMap::from([("a", &value_a), ("b", &value_b)]);
+
+ let expr = Arc::new(expressions::BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Plus,
+ Arc::new(Column::new("b", 1)),
+ )) as Arc<dyn PhysicalExpr>;
+
+ let result = replace_columns_with_literals(expr, &replacements)?;
+ assert_eq!(result.to_string(), "10 + 20");
Ok(())
}
@@ -996,7 +1027,6 @@ mod tests {
let rewriter = DefaultPhysicalExprAdapterRewriter {
logical_file_schema: &logical_schema,
physical_file_schema: &physical_schema,
- partition_fields: &[],
};
// Test that when a field exists in physical schema, it returns None
diff --git a/datafusion/pruning/src/file_pruner.rs b/datafusion/pruning/src/file_pruner.rs
index ee86a8cc8c..1d923202b3 100644
--- a/datafusion/pruning/src/file_pruner.rs
+++ b/datafusion/pruning/src/file_pruner.rs
@@ -19,69 +19,84 @@
use std::sync::Arc;
-use arrow::datatypes::{FieldRef, Schema, SchemaRef};
-use datafusion_common::{
- pruning::{
- CompositePruningStatistics, PartitionPruningStatistics, PrunableStatistics,
- PruningStatistics,
- },
- Result,
-};
+use arrow::datatypes::{FieldRef, SchemaRef};
+use datafusion_common::{internal_datafusion_err, pruning::PrunableStatistics, Result};
use datafusion_datasource::PartitionedFile;
use datafusion_physical_expr_common::physical_expr::{snapshot_generation, PhysicalExpr};
use datafusion_physical_plan::metrics::Count;
-use itertools::Itertools;
use log::debug;
use crate::build_pruning_predicate;
-/// Prune based on partition values and file-level statistics.
+/// Prune based on file-level statistics.
+///
+/// Note: Partition column pruning is handled earlier via `replace_columns_with_literals`
+/// which substitutes partition column references with their literal values before
+/// the predicate reaches this pruner.
pub struct FilePruner {
predicate_generation: Option<u64>,
predicate: Arc<dyn PhysicalExpr>,
- /// Schema used for pruning, which combines the file schema and partition fields.
- /// Partition fields are always at the end, as they are during scans.
- pruning_schema: Arc<Schema>,
- partitioned_file: PartitionedFile,
- partition_fields: Vec<FieldRef>,
+ /// Schema used for pruning (the logical file schema).
+ file_schema: SchemaRef,
+ file_stats_pruning: PrunableStatistics,
predicate_creation_errors: Count,
}
impl FilePruner {
+ #[deprecated(
+ since = "52.0.0",
+ note = "Use `try_new` instead which returns None if no statistics are available"
+ )]
+ #[expect(clippy::needless_pass_by_value)]
pub fn new(
predicate: Arc<dyn PhysicalExpr>,
logical_file_schema: &SchemaRef,
- partition_fields: Vec<FieldRef>,
+ _partition_fields: Vec<FieldRef>,
partitioned_file: PartitionedFile,
predicate_creation_errors: Count,
) -> Result<Self> {
- // Build a pruning schema that combines the file fields and partition fields.
- // Partition fields are always at the end.
- let pruning_schema = Arc::new(
- Schema::new(
- logical_file_schema
- .fields()
- .iter()
- .cloned()
- .chain(partition_fields.iter().cloned())
- .collect_vec(),
+ Self::try_new(
+ predicate,
+ logical_file_schema,
+ &partitioned_file,
+ predicate_creation_errors,
+ )
+ .ok_or_else(|| {
+ internal_datafusion_err!(
+ "FilePruner::new called on a file without statistics: {:?}",
+ partitioned_file
)
- .with_metadata(logical_file_schema.metadata().clone()),
- );
- Ok(Self {
- // Initialize the predicate generation to None so that the first time we call `should_prune` we actually check the predicate
- // Subsequent calls will only do work if the predicate itself has changed.
- // See `snapshot_generation` for more info.
+ })
+ }
+
+ /// Create a new file pruner if statistics are available.
+ /// Returns None if this file does not have statistics.
+ pub fn try_new(
+ predicate: Arc<dyn PhysicalExpr>,
+ file_schema: &SchemaRef,
+ partitioned_file: &PartitionedFile,
+ predicate_creation_errors: Count,
+ ) -> Option<Self> {
+ let file_stats = partitioned_file.statistics.as_ref()?;
+ let file_stats_pruning =
+ PrunableStatistics::new(vec![file_stats.clone()], Arc::clone(file_schema));
+ Some(Self {
predicate_generation: None,
predicate,
- pruning_schema,
- partitioned_file,
- partition_fields,
+ file_schema: Arc::clone(file_schema),
+ file_stats_pruning,
predicate_creation_errors,
})
}
pub fn should_prune(&mut self) -> Result<bool> {
+ // Check if the predicate has changed since last invocation by tracking
+ // its "generation". Dynamic filter expressions can change their values
+ // during query execution, so we use generation tracking to detect when
+ // the predicate has been updated and needs to be rebuilt.
+ //
+ // If the generation hasn't changed, we can skip rebuilding the pruning
+ // predicate, which is an expensive operation involving expression analysis.
let new_generation = snapshot_generation(&self.predicate);
if let Some(current_generation) = self.predicate_generation.as_mut() {
if *current_generation == new_generation {
@@ -93,39 +108,25 @@ impl FilePruner {
}
let pruning_predicate = build_pruning_predicate(
Arc::clone(&self.predicate),
- &self.pruning_schema,
+ &self.file_schema,
&self.predicate_creation_errors,
);
- if let Some(pruning_predicate) = pruning_predicate {
- // The partition column schema is the schema of the table - the schema of the file
- let mut pruning = Box::new(PartitionPruningStatistics::try_new(
- vec![self.partitioned_file.partition_values.clone()],
- self.partition_fields.clone(),
- )?) as Box<dyn PruningStatistics>;
- if let Some(stats) = &self.partitioned_file.statistics {
- let stats_pruning = Box::new(PrunableStatistics::new(
- vec![Arc::clone(stats)],
- Arc::clone(&self.pruning_schema),
- ));
- pruning = Box::new(CompositePruningStatistics::new(vec![
- pruning,
- stats_pruning,
- ]));
- }
- match pruning_predicate.prune(pruning.as_ref()) {
- Ok(values) => {
- assert!(values.len() == 1);
- // We expect a single container -> if all containers are false skip this file
- if values.into_iter().all(|v| !v) {
- return Ok(true);
- }
- }
- // Stats filter array could not be built, so we can't prune
- Err(e) => {
- debug!("Ignoring error building pruning predicate for file: {e}");
- self.predicate_creation_errors.add(1);
+ let Some(pruning_predicate) = pruning_predicate else {
+ return Ok(false);
+ };
+ match pruning_predicate.prune(&self.file_stats_pruning) {
+ Ok(values) => {
+ assert!(values.len() == 1);
+ // We expect a single container -> if all containers are false skip this file
+ if values.into_iter().all(|v| !v) {
+ return Ok(true);
}
}
+ // Stats filter array could not be built, so we can't prune
+ Err(e) => {
+ debug!("Ignoring error building pruning predicate for file: {e}");
+ self.predicate_creation_errors.add(1);
+ }
}
Ok(false)
diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md
index 6462961ce6..cecc0ef25b 100644
--- a/docs/source/library-user-guide/upgrading.md
+++ b/docs/source/library-user-guide/upgrading.md
@@ -165,6 +165,70 @@ Statistics are now accessed through `FileScanConfig` instead of `FileSource`:
Note that `FileScanConfig::statistics()` automatically marks statistics as inexact when filters are present, ensuring correctness when filters are pushed down.
+### Partition column handling moved out of `PhysicalExprAdapter`
+
+Partition column replacement is now a separate preprocessing step performed before expression rewriting via `PhysicalExprAdapter`. This change provides better separation of concerns and makes the adapter more focused on schema differences rather than partition value substitution.
+
+**Who is affected:**
+
+- Users who have custom implementations of `PhysicalExprAdapterFactory` that handle partition columns
+- Users who directly use the `FilePruner` API
+
+**Breaking changes:**
+
+1. `FilePruner::try_new()` signature changed: the `partition_fields` parameter has been removed since partition column handling is now done separately
+2. Partition column replacement must now be done via `replace_columns_with_literals()` before expressions are passed to the adapter
+
+**Migration guide:**
+
+If you have code that creates a `FilePruner` with partition fields:
+
+**Before:**
+
+```rust,ignore
+use datafusion_pruning::FilePruner;
+
+let pruner = FilePruner::try_new(
+ predicate,
+ file_schema,
+ partition_fields, // This parameter is removed
+ file_stats,
+)?;
+```
+
+**After:**
+
+```rust,ignore
+use datafusion_pruning::FilePruner;
+
+// Partition fields are no longer needed
+let pruner = FilePruner::try_new(
+ predicate,
+ file_schema,
+ file_stats,
+)?;
+```
+
+If you have custom code that relies on `PhysicalExprAdapter` to handle partition columns, you must now call `replace_columns_with_literals()` separately:
+
+**Before:**
+
+```rust,ignore
+// Adapter handled partition column replacement internally
+let adapted_expr = adapter.rewrite(expr)?;
+```
+
+**After:**
+
+```rust,ignore
+use datafusion_physical_expr_adapter::replace_columns_with_literals;
+
+// Replace partition columns first
+let expr_with_literals = replace_columns_with_literals(expr, &partition_values)?;
+// Then apply the adapter
+let adapted_expr = adapter.rewrite(expr_with_literals)?;
+```
+
### Planner now requires explicit opt-in for WITHIN GROUP syntax
The SQL planner now enforces the aggregate UDF contract more strictly: the