Skip to content

Apache DataFusion SQL Query Engine

feat: Add constant column extraction and rewriting for projections in ParquetOpener (#19136)

Alex Huangec11f4250815
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index c426cef3fa..7b4db9e772 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -24,6 +24,7 @@ use crate::{
apply_file_schema_type_coercions, coerce_int96_to_resolution, row_filter,
};
use arrow::array::{RecordBatch, RecordBatchOptions};
+use arrow::datatypes::DataType;
use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -35,8 +36,10 @@ use std::task::{Context, Poll};
use arrow::datatypes::{SchemaRef, TimeUnit};
use datafusion_common::encryption::FileDecryptionProperties;
-
-use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err};
+use datafusion_common::stats::Precision;
+use datafusion_common::{
+ ColumnStatistics, DataFusionError, Result, ScalarValue, Statistics, exec_err,
+};
use datafusion_datasource::{PartitionedFile, TableSchema};
use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
@@ -137,59 +140,60 @@ impl FileOpener for ParquetOpener {
let batch_size = self.batch_size;
- // Build partition values map for replacing partition column references
- // with their literal values from this file's partition values.
+ // Calculate the output schema from the original projection (before literal replacement)
+ // so we get correct field names from column references
+ let logical_file_schema = Arc::clone(self.table_schema.file_schema());
+ let output_schema = Arc::new(
+ self.projection
+ .project_schema(self.table_schema.table_schema())?,
+ );
+
+ // Build a combined map for replacing column references with literal values.
+ // This includes:
+ // 1. Partition column values from the file path (e.g., region=us-west-2)
+ // 2. Constant columns detected from file statistics (where min == max)
//
- // For example, given
- // 1. `region` is a partition column,
- // 2. predicate `host IN ('us-east-1', 'eu-central-1')`:
- // 3. The file path is `/data/region=us-west-2/...`
- // (that is the partition column value is `us-west-2`)
+ // Although partition columns *are* constant columns, we don't want to rely on
+ // statistics for them being populated if we can use the partition values
+ // (which are guaranteed to be present).
//
- // The predicate would be rewritten to
- // ```sql
- // 'us-west-2` IN ('us-east-1', 'eu-central-1')
- // ```
- // which can be further simplified to `FALSE`, meaning
- // the file can be skipped entirely.
+ // For example, given a partition column `region` and predicate
+ // `region IN ('us-east-1', 'eu-central-1')` with file path
+ // `/data/region=us-west-2/...`, the predicate is rewritten to
+ // `'us-west-2' IN ('us-east-1', 'eu-central-1')` which simplifies to FALSE.
//
- // While this particular optimization is done during logical planning,
- // there are other cases where partition columns may appear in more
- // complex predicates that cannot be simplified until we are about to
- // open the file (such as dynamic predicates)
- let partition_values: HashMap<&str, &ScalarValue> = self
+ // While partition column optimization is done during logical planning,
+ // there are cases where partition columns may appear in more complex
+ // predicates that cannot be simplified until we open the file (such as
+ // dynamic predicates).
+ let mut literal_columns: HashMap<String, ScalarValue> = self
.table_schema
.table_partition_cols()
.iter()
.zip(partitioned_file.partition_values.iter())
- .map(|(field, value)| (field.name().as_str(), value))
+ .map(|(field, value)| (field.name().clone(), value.clone()))
.collect();
+ // Add constant columns from file statistics.
+ // Note that if there are statistics for partition columns there will be overlap,
+ // but since we use a HashMap, we'll just overwrite the partition values with the
+ // constant values from statistics (which should be the same).
+ literal_columns.extend(constant_columns_from_stats(
+ partitioned_file.statistics.as_deref(),
+ &logical_file_schema,
+ ));
- // Calculate the output schema from the original projection (before literal replacement)
- // so we get correct field names from column references
- let logical_file_schema = Arc::clone(self.table_schema.file_schema());
- let output_schema = Arc::new(
- self.projection
- .project_schema(self.table_schema.table_schema())?,
- );
-
- // Apply partition column replacement to projection expressions
+ // Apply literal replacements to projection and predicate
let mut projection = self.projection.clone();
- if !partition_values.is_empty() {
+ let mut predicate = self.predicate.clone();
+ if !literal_columns.is_empty() {
projection = projection.try_map_exprs(|expr| {
- replace_columns_with_literals(Arc::clone(&expr), &partition_values)
+ replace_columns_with_literals(Arc::clone(&expr), &literal_columns)
})?;
+ predicate = predicate
+ .map(|p| replace_columns_with_literals(p, &literal_columns))
+ .transpose()?;
}
- // Apply partition column replacement to predicate
- let mut predicate = if partition_values.is_empty() {
- self.predicate.clone()
- } else {
- self.predicate
- .clone()
- .map(|p| replace_columns_with_literals(p, &partition_values))
- .transpose()?
- };
let reorder_predicates = self.reorder_filters;
let pushdown_filters = self.pushdown_filters;
let force_filter_selections = self.force_filter_selections;
@@ -581,6 +585,64 @@ fn copy_arrow_reader_metrics(
}
}
+type ConstantColumns = HashMap<String, ScalarValue>;
+
+/// Extract constant column values from statistics, keyed by column name in the logical file schema.
+fn constant_columns_from_stats(
+ statistics: Option<&Statistics>,
+ file_schema: &SchemaRef,
+) -> ConstantColumns {
+ let mut constants = HashMap::new();
+ let Some(statistics) = statistics else {
+ return constants;
+ };
+
+ let num_rows = match statistics.num_rows {
+ Precision::Exact(num_rows) => Some(num_rows),
+ _ => None,
+ };
+
+ for (idx, column_stats) in statistics
+ .column_statistics
+ .iter()
+ .take(file_schema.fields().len())
+ .enumerate()
+ {
+ let field = file_schema.field(idx);
+ if let Some(value) =
+ constant_value_from_stats(column_stats, num_rows, field.data_type())
+ {
+ constants.insert(field.name().clone(), value);
+ }
+ }
+
+ constants
+}
+
+fn constant_value_from_stats(
+ column_stats: &ColumnStatistics,
+ num_rows: Option<usize>,
+ data_type: &DataType,
+) -> Option<ScalarValue> {
+ if let (Precision::Exact(min), Precision::Exact(max)) =
+ (&column_stats.min_value, &column_stats.max_value)
+ && min == max
+ && !min.is_null()
+ && matches!(column_stats.null_count, Precision::Exact(0))
+ {
+ return Some(min.clone());
+ }
+
+ if let (Some(num_rows), Precision::Exact(nulls)) =
+ (num_rows, &column_stats.null_count)
+ && *nulls == num_rows
+ {
+ return ScalarValue::try_new_null(data_type).ok();
+ }
+
+ None
+}
+
/// Wraps an inner RecordBatchStream and a [`FilePruner`]
///
/// This can terminate the scan early when some dynamic filters is updated after
@@ -841,7 +903,8 @@ fn should_enable_page_index(
mod test {
use std::sync::Arc;
- use arrow::datatypes::{DataType, Field, Schema};
+ use super::{ConstantColumns, constant_columns_from_stats};
+ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use bytes::{BufMut, BytesMut};
use datafusion_common::{
ColumnStatistics, DataFusionError, ScalarValue, Statistics, record_batch,
@@ -850,10 +913,14 @@ mod test {
use datafusion_datasource::{PartitionedFile, TableSchema, file_stream::FileOpener};
use datafusion_expr::{col, lit};
use datafusion_physical_expr::{
- PhysicalExpr, expressions::DynamicFilterPhysicalExpr, planner::logical2physical,
+ PhysicalExpr,
+ expressions::{Column, DynamicFilterPhysicalExpr, Literal},
+ planner::logical2physical,
projection::ProjectionExprs,
};
- use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
+ use datafusion_physical_expr_adapter::{
+ DefaultPhysicalExprAdapterFactory, replace_columns_with_literals,
+ };
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
use futures::{Stream, StreamExt};
use object_store::{ObjectStore, memory::InMemory, path::Path};
@@ -861,6 +928,89 @@ mod test {
use crate::{DefaultParquetFileReaderFactory, opener::ParquetOpener};
+ fn constant_int_stats() -> (Statistics, SchemaRef) {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ ]));
+ let statistics = Statistics {
+ num_rows: Precision::Exact(3),
+ total_byte_size: Precision::Absent,
+ column_statistics: vec![
+ ColumnStatistics {
+ null_count: Precision::Exact(0),
+ max_value: Precision::Exact(ScalarValue::from(5i32)),
+ min_value: Precision::Exact(ScalarValue::from(5i32)),
+ sum_value: Precision::Absent,
+ distinct_count: Precision::Absent,
+ byte_size: Precision::Absent,
+ },
+ ColumnStatistics::new_unknown(),
+ ],
+ };
+ (statistics, schema)
+ }
+
+ #[test]
+ fn extract_constant_columns_non_null() {
+ let (statistics, schema) = constant_int_stats();
+ let constants = constant_columns_from_stats(Some(&statistics), &schema);
+ assert_eq!(constants.len(), 1);
+ assert_eq!(constants.get("a"), Some(&ScalarValue::from(5i32)));
+ assert!(!constants.contains_key("b"));
+ }
+
+ #[test]
+ fn extract_constant_columns_all_null() {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, true)]));
+ let statistics = Statistics {
+ num_rows: Precision::Exact(2),
+ total_byte_size: Precision::Absent,
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Exact(2),
+ max_value: Precision::Absent,
+ min_value: Precision::Absent,
+ sum_value: Precision::Absent,
+ distinct_count: Precision::Absent,
+ byte_size: Precision::Absent,
+ }],
+ };
+
+ let constants = constant_columns_from_stats(Some(&statistics), &schema);
+ assert_eq!(
+ constants.get("a"),
+ Some(&ScalarValue::Utf8(None)),
+ "all-null column should be treated as constant null"
+ );
+ }
+
+ #[test]
+ fn rewrite_projection_to_literals() {
+ let (statistics, schema) = constant_int_stats();
+ let constants = constant_columns_from_stats(Some(&statistics), &schema);
+ let projection = ProjectionExprs::from_indices(&[0, 1], &schema);
+
+ let rewritten = projection
+ .try_map_exprs(|expr| replace_columns_with_literals(expr, &constants))
+ .unwrap();
+ let exprs = rewritten.as_ref();
+ assert!(exprs[0].expr.as_any().downcast_ref::<Literal>().is_some());
+ assert!(exprs[1].expr.as_any().downcast_ref::<Column>().is_some());
+
+ // Only column `b` should remain in the projection mask
+ assert_eq!(rewritten.column_indices(), vec![1]);
+ }
+
+ #[test]
+ fn rewrite_physical_expr_literal() {
+ let mut constants = ConstantColumns::new();
+ constants.insert("a".to_string(), ScalarValue::from(7i32));
+ let expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
+
+ let rewritten = replace_columns_with_literals(expr, &constants).unwrap();
+ assert!(rewritten.as_any().downcast_ref::<Literal>().is_some());
+ }
+
async fn count_batches_and_rows(
mut stream: std::pin::Pin<
Box<
diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
index 6f040c1a05..e9b8ff5c37 100644
--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs
+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
@@ -19,7 +19,9 @@
//! [`PhysicalExprAdapterFactory`], default implementations,
//! and [`replace_columns_with_literals`].
+use std::borrow::Borrow;
use std::collections::HashMap;
+use std::hash::Hash;
use std::sync::Arc;
use arrow::compute::can_cast_types;
@@ -50,19 +52,25 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
/// # Arguments
/// - `expr`: The physical expression in which to replace column references.
/// - `replacements`: A mapping from column names to their corresponding literal `ScalarValue`s.
+/// Accepts various HashMap types including `HashMap<&str, &ScalarValue>`,
+/// `HashMap<String, ScalarValue>`, `HashMap<String, &ScalarValue>`, etc.
///
/// # Returns
/// - `Result<Arc<dyn PhysicalExpr>>`: The rewritten physical expression with columns replaced by literals.
-pub fn replace_columns_with_literals(
+pub fn replace_columns_with_literals<K, V>(
expr: Arc<dyn PhysicalExpr>,
- replacements: &HashMap<&str, &ScalarValue>,
-) -> Result<Arc<dyn PhysicalExpr>> {
- expr.transform(|expr| {
+ replacements: &HashMap<K, V>,
+) -> Result<Arc<dyn PhysicalExpr>>
+where
+ K: Borrow<str> + Eq + Hash,
+ V: Borrow<ScalarValue>,
+{
+ expr.transform_down(|expr| {
if let Some(column) = expr.as_any().downcast_ref::<Column>()
&& let Some(replacement_value) = replacements.get(column.name())
{
return Ok(Transformed::yes(expressions::lit(
- (*replacement_value).clone(),
+ replacement_value.borrow().clone(),
)));
}
Ok(Transformed::no(expr))
diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rs
index 60b00117b7..615d9cbbf6 100644
--- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs
+++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs
@@ -607,4 +607,88 @@ mod test {
// wait_complete should return immediately
dynamic_filter.wait_complete().await;
}
+
+ #[test]
+ fn test_with_new_children_independence() {
+ // Create a schema with columns a, b, c, d
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ Field::new("c", DataType::Int32, false),
+ Field::new("d", DataType::Int32, false),
+ ]));
+
+ // Create expression col(a) + col(b)
+ let col_a = col("a", &schema).unwrap();
+ let col_b = col("b", &schema).unwrap();
+ let col_c = col("c", &schema).unwrap();
+ let col_d = col("d", &schema).unwrap();
+
+ let expr = Arc::new(BinaryExpr::new(
+ Arc::clone(&col_a),
+ datafusion_expr::Operator::Plus,
+ Arc::clone(&col_b),
+ ));
+
+ // Create DynamicFilterPhysicalExpr with children [col_a, col_b]
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::clone(&col_a), Arc::clone(&col_b)],
+ expr as Arc<dyn PhysicalExpr>,
+ ));
+
+ // Clone the Arc (two references to the same DynamicFilterPhysicalExpr)
+ let clone_1 = Arc::clone(&dynamic_filter);
+ let clone_2 = Arc::clone(&dynamic_filter);
+
+ // Call with_new_children with different children on each clone
+ // clone_1: replace [a, b] with [b, c] -> expression becomes b + c
+ let remapped_1 = clone_1
+ .with_new_children(vec![Arc::clone(&col_b), Arc::clone(&col_c)])
+ .unwrap();
+
+ // clone_2: replace [a, b] with [b, d] -> expression becomes b + d
+ let remapped_2 = clone_2
+ .with_new_children(vec![Arc::clone(&col_b), Arc::clone(&col_d)])
+ .unwrap();
+
+ // Create a RecordBatch with columns a=1,2,3 b=10,20,30 c=100,200,300 d=1000,2000,3000
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3])), // a
+ Arc::new(arrow::array::Int32Array::from(vec![10, 20, 30])), // b
+ Arc::new(arrow::array::Int32Array::from(vec![100, 200, 300])), // c
+ Arc::new(arrow::array::Int32Array::from(vec![1000, 2000, 3000])), // d
+ ],
+ )
+ .unwrap();
+
+ // Evaluate both remapped expressions
+ let result_1 = remapped_1.evaluate(&batch).unwrap();
+ let result_2 = remapped_2.evaluate(&batch).unwrap();
+
+ // Extract arrays from results
+ let ColumnarValue::Array(arr_1) = result_1 else {
+ panic!("Expected ColumnarValue::Array for result_1");
+ };
+ let ColumnarValue::Array(arr_2) = result_2 else {
+ panic!("Expected ColumnarValue::Array for result_2");
+ };
+
+ // Verify result_1 = b + c = [110, 220, 330]
+ let expected_1: Arc<dyn arrow::array::Array> =
+ Arc::new(arrow::array::Int32Array::from(vec![110, 220, 330]));
+ assert!(
+ arr_1.eq(&expected_1),
+ "Expected b + c = [110, 220, 330], got {arr_1:?}",
+ );
+
+ // Verify result_2 = b + d = [1010, 2020, 3030]
+ let expected_2: Arc<dyn arrow::array::Array> =
+ Arc::new(arrow::array::Int32Array::from(vec![1010, 2020, 3030]));
+ assert!(
+ arr_2.eq(&expected_2),
+ "Expected b + d = [1010, 2020, 3030], got {arr_2:?}",
+ );
+ }
}