Skip to content

Apache DataFusion SQL Query Engine

Push down projection expressions into ParquetOpener (#19111)

Adrian Garcia Badaracco021188e719ee
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index 8f521c3f4e..fb9d27527e 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -23,7 +23,7 @@ use crate::{
apply_file_schema_type_coercions, coerce_int96_to_resolution, row_filter,
ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory,
};
-use arrow::array::RecordBatch;
+use arrow::array::{RecordBatch, RecordBatchOptions};
use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -66,8 +66,8 @@ use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};
pub(super) struct ParquetOpener {
/// Execution partition index
pub partition_index: usize,
- /// Column indexes in `table_schema` needed by the query
- pub projection: Arc<[usize]>,
+ /// Projection to apply on top of the table schema (i.e. can reference partition columns).
+ pub projection: ProjectionExprs,
/// Target number of rows in each output RecordBatch
pub batch_size: usize,
/// Optional limit on the number of rows to read
@@ -104,7 +104,7 @@ pub(super) struct ParquetOpener {
#[cfg(feature = "parquet_encryption")]
pub file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
/// Rewrite expressions in the context of the file schema
- pub(crate) expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,
+ pub(crate) expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
/// Optional factory to create file decryption properties dynamically
#[cfg(feature = "parquet_encryption")]
pub encryption_factory:
@@ -137,7 +137,7 @@ impl FileOpener for ParquetOpener {
let batch_size = self.batch_size;
- // Replace any partition column references in the predicate
+ // Build partition values map for replacing partition column references
// with their literal values from this file's partition values.
//
// For example, given
@@ -157,29 +157,39 @@ impl FileOpener for ParquetOpener {
// there are other cases where partition columns may appear in more
// complex predicates that cannot be simplified until we are about to
// open the file (such as dynamic predicates)
- let mut predicate = {
- // Only replace partition columns if there are partition values.
- // For non-partitioned tables, skip the clone and replacement traversal.
- if partitioned_file.partition_values.is_empty() {
- self.predicate.clone()
- } else {
- let partition_values: HashMap<&str, &ScalarValue> = self
- .table_schema
- .table_partition_cols()
- .iter()
- .zip(partitioned_file.partition_values.iter())
- .map(|(field, value)| (field.name().as_str(), value))
- .collect();
-
- self.predicate
- .clone()
- .map(|p| replace_columns_with_literals(p, &partition_values))
- .transpose()?
- }
- };
+ let partition_values: HashMap<&str, &ScalarValue> = self
+ .table_schema
+ .table_partition_cols()
+ .iter()
+ .zip(partitioned_file.partition_values.iter())
+ .map(|(field, value)| (field.name().as_str(), value))
+ .collect();
+
+ // Calculate the output schema from the original projection (before literal replacement)
+ // so we get correct field names from column references
+ let logical_file_schema = Arc::clone(self.table_schema.file_schema());
+ let output_schema = Arc::new(
+ self.projection
+ .project_schema(self.table_schema.table_schema())?,
+ );
- let projection = Arc::clone(&self.projection);
- let logical_file_schema = Arc::clone(self.table_schema.table_schema());
+ // Apply partition column replacement to projection expressions
+ let mut projection = self.projection.clone();
+ if !partition_values.is_empty() {
+ projection = projection.try_map_exprs(|expr| {
+ replace_columns_with_literals(Arc::clone(&expr), &partition_values)
+ })?;
+ }
+
+ // Apply partition column replacement to predicate
+ let mut predicate = if partition_values.is_empty() {
+ self.predicate.clone()
+ } else {
+ self.predicate
+ .clone()
+ .map(|p| replace_columns_with_literals(p, &partition_values))
+ .transpose()?
+ };
let reorder_predicates = self.reorder_filters;
let pushdown_filters = self.pushdown_filters;
let force_filter_selections = self.force_filter_selections;
@@ -191,8 +201,7 @@ impl FileOpener for ParquetOpener {
let predicate_creation_errors = MetricBuilder::new(&self.metrics)
.global_counter("num_predicate_creation_errors");
- let expr_adapter_factory = self.expr_adapter_factory.clone();
- let mut predicate_file_schema = Arc::clone(self.table_schema.file_schema());
+ let expr_adapter_factory = Arc::clone(&self.expr_adapter_factory);
let enable_page_index = self.enable_page_index;
#[cfg(feature = "parquet_encryption")]
@@ -311,30 +320,29 @@ impl FileOpener for ParquetOpener {
}
}
- // Adapt the predicate to the physical file schema.
+ // Adapt the projection & filter predicate to the physical file schema.
// This evaluates missing columns and inserts any necessary casts.
- if let Some(expr_adapter_factory) = expr_adapter_factory.as_ref() {
- predicate = predicate
- .map(|p| {
- let expr = expr_adapter_factory
- .create(
- Arc::clone(&logical_file_schema),
- Arc::clone(&physical_file_schema),
- )
- .rewrite(p)?;
- // After rewriting to the file schema, further simplifications may be possible.
- // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`
- // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).
- PhysicalExprSimplifier::new(&physical_file_schema).simplify(expr)
- })
- .transpose()?;
- predicate_file_schema = Arc::clone(&physical_file_schema);
- }
+ // After rewriting to the file schema, further simplifications may be possible.
+ // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`
+ // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).
+ // Additionally, if any casts were inserted we can move casts from the column to the literal side:
+ // `CAST(col AS INT) = 5` can become `col = CAST(5 AS <col type>)`, which can be evaluated statically.
+ let rewriter = expr_adapter_factory.create(
+ Arc::clone(&logical_file_schema),
+ Arc::clone(&physical_file_schema),
+ );
+ let simplifier = PhysicalExprSimplifier::new(&physical_file_schema);
+ predicate = predicate
+ .map(|p| simplifier.simplify(rewriter.rewrite(p)?))
+ .transpose()?;
+ // Adapt projections to the physical file schema as well
+ projection = projection
+ .try_map_exprs(|p| simplifier.simplify(rewriter.rewrite(p)?))?;
// Build predicates for this specific file
let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(
predicate.as_ref(),
- &predicate_file_schema,
+ &physical_file_schema,
&predicate_creation_errors,
);
@@ -358,15 +366,6 @@ impl FileOpener for ParquetOpener {
reader_metadata,
);
- let mut projection =
- ProjectionExprs::from_indices(&projection, &logical_file_schema);
- if let Some(expr_adapter_factory) = expr_adapter_factory {
- let adapter = expr_adapter_factory.create(
- Arc::clone(&logical_file_schema),
- Arc::clone(&physical_file_schema),
- );
- projection = projection.try_map_exprs(|expr| adapter.rewrite(expr))?;
- }
let indices = projection.column_indices();
let mask = ProjectionMask::roots(builder.parquet_schema(), indices);
@@ -376,7 +375,6 @@ impl FileOpener for ParquetOpener {
let row_filter = row_filter::build_row_filter(
&predicate,
&physical_file_schema,
- &predicate_file_schema,
builder.metadata(),
reorder_predicates,
&file_metrics,
@@ -507,6 +505,9 @@ impl FileOpener for ParquetOpener {
let predicate_cache_records = file_metrics.predicate_cache_records.clone();
let stream_schema = Arc::clone(stream.schema());
+ // Check if we need to replace the schema to handle things like differing nullability or metadata.
+ // See note below about file vs. output schema.
+ let replace_schema = !stream_schema.eq(&output_schema);
// Rebase column indices to match the narrowed stream schema.
// The projection expressions have indices based on physical_file_schema,
@@ -517,13 +518,34 @@ impl FileOpener for ParquetOpener {
let projector = projection.make_projector(&stream_schema)?;
let stream = stream.map_err(DataFusionError::from).map(move |b| {
- b.and_then(|b| {
+ b.and_then(|mut b| {
copy_arrow_reader_metrics(
&arrow_reader_metrics,
&predicate_cache_inner_records,
&predicate_cache_records,
);
- projector.project_batch(&b)
+ b = projector.project_batch(&b)?;
+ if replace_schema {
+ // Ensure the output batch has the expected schema.
+ // This handles things like schema level and field level metadata, which may not be present
+ // in the physical file schema.
+ // It is also possible for nullability to differ; some writers create files with
+ // OPTIONAL fields even when there are no nulls in the data.
+ // In these cases it may make sense for the logical schema to be `NOT NULL`.
+ // RecordBatch::try_new_with_options checks that if the schema is NOT NULL
+ // the array cannot contain nulls, amongst other checks.
+ let (_stream_schema, arrays, num_rows) = b.into_parts();
+ let options =
+ RecordBatchOptions::new().with_row_count(Some(num_rows));
+ RecordBatch::try_new_with_options(
+ Arc::clone(&output_schema),
+ arrays,
+ &options,
+ )
+ .map_err(Into::into)
+ } else {
+ Ok(b)
+ }
})
});
@@ -826,7 +848,8 @@ mod test {
use datafusion_datasource::{file_stream::FileOpener, PartitionedFile, TableSchema};
use datafusion_expr::{col, lit};
use datafusion_physical_expr::{
- expressions::DynamicFilterPhysicalExpr, planner::logical2physical, PhysicalExpr,
+ expressions::DynamicFilterPhysicalExpr, planner::logical2physical,
+ projection::ProjectionExprs, PhysicalExpr,
};
use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
@@ -910,7 +933,7 @@ mod test {
let make_opener = |predicate| {
ParquetOpener {
partition_index: 0,
- projection: Arc::new([0, 1]),
+ projection: ProjectionExprs::from_indices(&[0, 1], &schema),
batch_size: 1024,
limit: None,
predicate: Some(predicate),
@@ -929,7 +952,7 @@ mod test {
coerce_int96: None,
#[cfg(feature = "parquet_encryption")]
file_decryption_properties: None,
- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),
+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
#[cfg(feature = "parquet_encryption")]
encryption_factory: None,
max_predicate_cache_size: None,
@@ -978,7 +1001,7 @@ mod test {
let make_opener = |predicate| {
ParquetOpener {
partition_index: 0,
- projection: Arc::new([0]),
+ projection: ProjectionExprs::from_indices(&[0], &file_schema),
batch_size: 1024,
limit: None,
predicate: Some(predicate),
@@ -1000,7 +1023,7 @@ mod test {
coerce_int96: None,
#[cfg(feature = "parquet_encryption")]
file_decryption_properties: None,
- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),
+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
#[cfg(feature = "parquet_encryption")]
encryption_factory: None,
max_predicate_cache_size: None,
@@ -1065,7 +1088,7 @@ mod test {
let make_opener = |predicate| {
ParquetOpener {
partition_index: 0,
- projection: Arc::new([0]),
+ projection: ProjectionExprs::from_indices(&[0], &file_schema),
batch_size: 1024,
limit: None,
predicate: Some(predicate),
@@ -1087,7 +1110,7 @@ mod test {
coerce_int96: None,
#[cfg(feature = "parquet_encryption")]
file_decryption_properties: None,
- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),
+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
#[cfg(feature = "parquet_encryption")]
encryption_factory: None,
max_predicate_cache_size: None,
@@ -1155,7 +1178,7 @@ mod test {
let make_opener = |predicate| {
ParquetOpener {
partition_index: 0,
- projection: Arc::new([0]),
+ projection: ProjectionExprs::from_indices(&[0], &file_schema),
batch_size: 1024,
limit: None,
predicate: Some(predicate),
@@ -1177,7 +1200,7 @@ mod test {
coerce_int96: None,
#[cfg(feature = "parquet_encryption")]
file_decryption_properties: None,
- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),
+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
#[cfg(feature = "parquet_encryption")]
encryption_factory: None,
max_predicate_cache_size: None,
@@ -1253,7 +1276,7 @@ mod test {
let make_opener = |predicate| {
ParquetOpener {
partition_index: 0,
- projection: Arc::new([0]),
+ projection: ProjectionExprs::from_indices(&[0], &file_schema),
batch_size: 1024,
limit: None,
predicate: Some(predicate),
@@ -1275,7 +1298,7 @@ mod test {
coerce_int96: None,
#[cfg(feature = "parquet_encryption")]
file_decryption_properties: None,
- expr_adapter_factory: Some(Arc::new(DefaultPhysicalExprAdapterFactory)),
+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
#[cfg(feature = "parquet_encryption")]
encryption_factory: None,
max_predicate_cache_size: None,
diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rs
index 059663c231..04f8d5533e 100644
--- a/datafusion/datasource-parquet/src/row_filter.rs
+++ b/datafusion/datasource-parquet/src/row_filter.rs
@@ -73,7 +73,7 @@ use parquet::file::metadata::ParquetMetaData;
use datafusion_common::cast::as_boolean_array;
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};
-use datafusion_common::{internal_datafusion_err, Result};
+use datafusion_common::Result;
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::utils::reassign_expr_columns;
use datafusion_physical_expr::{split_conjunction, PhysicalExpr};
@@ -176,42 +176,32 @@ pub(crate) struct FilterCandidate {
required_bytes: usize,
/// Can this filter use an index (e.g. a page index) to prune rows?
can_use_index: bool,
- /// The projection to read from the file schema to get the columns
- /// required to evaluate the filter expression.
+ /// Column indices into the parquet file schema required to evaluate this filter.
projection: Vec<usize>,
- /// The projected table schema that this filter references
+ /// The Arrow schema containing only the columns required by this filter,
+ /// projected from the file's Arrow schema.
filter_schema: SchemaRef,
}
/// Helper to build a `FilterCandidate`.
///
-/// This will do several things
+/// This will do several things:
/// 1. Determine the columns required to evaluate the expression
/// 2. Calculate data required to estimate the cost of evaluating the filter
///
-/// Note that this does *not* handle any adaptation of the data schema to the expression schema,
-/// it is assumed that the expression has already been adapted to the file schema before being passed in here,
-/// generally using [`PhysicalExprAdapter`](datafusion_physical_expr_adapter::PhysicalExprAdapter).
+/// Note: This does *not* handle any adaptation of the expression to the file schema.
+/// The expression must already be adapted before being passed in here, generally using
+/// [`PhysicalExprAdapter`](datafusion_physical_expr_adapter::PhysicalExprAdapter).
struct FilterCandidateBuilder {
expr: Arc<dyn PhysicalExpr>,
- /// The schema of this parquet file.
+ /// The Arrow schema of this parquet file (the result of converting the
+ /// parquet schema to Arrow, potentially with type coercions applied).
file_schema: SchemaRef,
- /// The schema of the table (merged schema) -- columns may be in different
- /// order than in the file and have columns that are not in the file schema
- table_schema: SchemaRef,
}
impl FilterCandidateBuilder {
- pub fn new(
- expr: Arc<dyn PhysicalExpr>,
- file_schema: Arc<Schema>,
- table_schema: Arc<Schema>,
- ) -> Self {
- Self {
- expr,
- file_schema,
- table_schema,
- }
+ pub fn new(expr: Arc<dyn PhysicalExpr>, file_schema: Arc<Schema>) -> Self {
+ Self { expr, file_schema }
}
/// Attempt to build a `FilterCandidate` from the expression
@@ -222,85 +212,64 @@ impl FilterCandidateBuilder {
/// * `Ok(None)` if the expression cannot be used as an ArrowFilter
/// * `Err(e)` if an error occurs while building the candidate
pub fn build(self, metadata: &ParquetMetaData) -> Result<Option<FilterCandidate>> {
- let Some(required_indices_into_table_schema) =
- pushdown_columns(&self.expr, &self.table_schema)?
+ let Some(required_column_indices) =
+ pushdown_columns(&self.expr, &self.file_schema)?
else {
return Ok(None);
};
- let projected_table_schema = Arc::new(
- self.table_schema
- .project(&required_indices_into_table_schema)?,
- );
-
- // Compute the projection into the file schema by matching column names
- let mut projection_into_file_schema: Vec<usize> = projected_table_schema
- .fields()
- .iter()
- .filter_map(|f| self.file_schema.index_of(f.name()).ok())
- .collect();
- // Sort and remove duplicates
- let original_len = projection_into_file_schema.len();
- projection_into_file_schema.sort_unstable();
- projection_into_file_schema.dedup();
- if projection_into_file_schema.len() < original_len {
- // This should not happen, as we built projected_table_schema from
- // the table schema which should not have duplicate column names.
- return Err(internal_datafusion_err!(
- "Duplicate column names found when building filter candidate: {:?}",
- projection_into_file_schema
- ));
- }
+ let projected_schema =
+ Arc::new(self.file_schema.project(&required_column_indices)?);
- let required_bytes = size_of_columns(&projection_into_file_schema, metadata)?;
- let can_use_index = columns_sorted(&projection_into_file_schema, metadata)?;
+ let required_bytes = size_of_columns(&required_column_indices, metadata)?;
+ let can_use_index = columns_sorted(&required_column_indices, metadata)?;
Ok(Some(FilterCandidate {
expr: self.expr,
required_bytes,
can_use_index,
- projection: projection_into_file_schema,
- filter_schema: Arc::clone(&projected_table_schema),
+ projection: required_column_indices,
+ filter_schema: projected_schema,
}))
}
}
-// a struct that implements TreeNodeRewriter to traverse a PhysicalExpr tree structure to determine
-// if any column references in the expression would prevent it from being predicate-pushed-down.
-// if non_primitive_columns || projected_columns, it can't be pushed down.
-// can't be reused between calls to `rewrite`; each construction must be used only once.
+/// Traverses a `PhysicalExpr` tree to determine if any column references would
+/// prevent the expression from being pushed down to the parquet decoder.
+///
+/// An expression cannot be pushed down if it references:
+/// - Non-primitive columns (like structs or lists)
+/// - Columns that don't exist in the file schema
struct PushdownChecker<'schema> {
/// Does the expression require any non-primitive columns (like structs)?
non_primitive_columns: bool,
- /// Does the expression reference any columns that are in the table
- /// schema but not in the file schema?
- /// This includes partition columns and projected columns.
+ /// Does the expression reference any columns not present in the file schema?
projected_columns: bool,
- // Indices into the table schema of the columns required to evaluate the expression
+ /// Indices into the file schema of columns required to evaluate the expression.
required_columns: BTreeSet<usize>,
- table_schema: &'schema Schema,
+ /// The Arrow schema of the parquet file.
+ file_schema: &'schema Schema,
}
impl<'schema> PushdownChecker<'schema> {
- fn new(table_schema: &'schema Schema) -> Self {
+ fn new(file_schema: &'schema Schema) -> Self {
Self {
non_primitive_columns: false,
projected_columns: false,
required_columns: BTreeSet::default(),
- table_schema,
+ file_schema,
}
}
fn check_single_column(&mut self, column_name: &str) -> Option<TreeNodeRecursion> {
- if let Ok(idx) = self.table_schema.index_of(column_name) {
+ if let Ok(idx) = self.file_schema.index_of(column_name) {
self.required_columns.insert(idx);
- if DataType::is_nested(self.table_schema.field(idx).data_type()) {
+ if DataType::is_nested(self.file_schema.field(idx).data_type()) {
self.non_primitive_columns = true;
return Some(TreeNodeRecursion::Jump);
}
} else {
- // If the column does not exist in the (un-projected) table schema then
- // it must be a projected column.
+ // Column does not exist in the file schema, so we can't push this down.
self.projected_columns = true;
return Some(TreeNodeRecursion::Jump);
}
@@ -328,25 +297,34 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
}
}
-// Checks if a given expression can be pushed down into `DataSourceExec` as opposed to being evaluated
-// post-parquet-scan in a `FilterExec`. If it can be pushed down, this returns all the
-// columns in the given expression so that they can be used in the parquet scanning, along with the
-// expression rewritten as defined in [`PushdownChecker::f_up`]
+/// Checks if a given expression can be pushed down to the parquet decoder.
+///
+/// Returns `Some(column_indices)` if the expression can be pushed down,
+/// where `column_indices` are the indices into the file schema of all columns
+/// required to evaluate the expression.
+///
+/// Returns `None` if the expression cannot be pushed down (e.g., references
+/// non-primitive types or columns not in the file).
fn pushdown_columns(
expr: &Arc<dyn PhysicalExpr>,
- table_schema: &Schema,
+ file_schema: &Schema,
) -> Result<Option<Vec<usize>>> {
- let mut checker = PushdownChecker::new(table_schema);
+ let mut checker = PushdownChecker::new(file_schema);
expr.visit(&mut checker)?;
Ok((!checker.prevents_pushdown())
.then_some(checker.required_columns.into_iter().collect()))
}
-/// Recurses through expr as a tree, finds all `column`s, and checks if any of them would prevent
-/// this expression from being predicate pushed down. If any of them would, this returns false.
-/// Otherwise, true.
-/// Note that the schema passed in here is *not* the physical file schema (as it is not available at that point in time);
-/// it is the schema of the table that this expression is being evaluated against minus any projected columns and partition columns.
+/// Checks if a predicate expression can be pushed down to the parquet decoder.
+///
+/// Returns `true` if all columns referenced by the expression:
+/// - Exist in the provided schema
+/// - Are primitive types (not structs, lists, etc.)
+///
+/// # Arguments
+/// * `expr` - The filter expression to check
+/// * `file_schema` - The Arrow schema of the parquet file (or table schema when
+/// the file schema is not yet available during planning)
pub fn can_expr_be_pushed_down_with_schemas(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &Schema,
@@ -384,24 +362,30 @@ fn columns_sorted(_columns: &[usize], _metadata: &ParquetMetaData) -> Result<boo
Ok(false)
}
-/// Build a [`RowFilter`] from the given predicate `Expr` if possible
+/// Build a [`RowFilter`] from the given predicate expression if possible.
///
-/// # returns
-/// * `Ok(Some(row_filter))` if the expression can be used as RowFilter
-/// * `Ok(None)` if the expression cannot be used as an RowFilter
+/// # Arguments
+/// * `expr` - The filter predicate, already adapted to reference columns in `file_schema`
+/// * `file_schema` - The Arrow schema of the parquet file (the result of converting
+/// the parquet schema to Arrow, potentially with type coercions applied)
+/// * `metadata` - Parquet file metadata used for cost estimation
+/// * `reorder_predicates` - If true, reorder predicates to minimize I/O
+/// * `file_metrics` - Metrics for tracking filter performance
+///
+/// # Returns
+/// * `Ok(Some(row_filter))` if the expression can be used as a RowFilter
+/// * `Ok(None)` if the expression cannot be used as a RowFilter
/// * `Err(e)` if an error occurs while building the filter
///
-/// Note that the returned `RowFilter` may not contains all conjuncts in the
-/// original expression. This is because some conjuncts may not be able to be
-/// evaluated as an `ArrowPredicate` and will be ignored.
+/// Note: The returned `RowFilter` may not contain all conjuncts from the original
+/// expression. Conjuncts that cannot be evaluated as an `ArrowPredicate` are ignored.
///
/// For example, if the expression is `a = 1 AND b = 2 AND c = 3` and `b = 2`
-/// can not be evaluated for some reason, the returned `RowFilter` will contain
-/// `a = 1` and `c = 3`.
+/// cannot be evaluated for some reason, the returned `RowFilter` will contain
+/// only `a = 1` and `c = 3`.
pub fn build_row_filter(
expr: &Arc<dyn PhysicalExpr>,
- physical_file_schema: &SchemaRef,
- predicate_file_schema: &SchemaRef,
+ file_schema: &SchemaRef,
metadata: &ParquetMetaData,
reorder_predicates: bool,
file_metrics: &ParquetFileMetrics,
@@ -418,12 +402,8 @@ pub fn build_row_filter(
let mut candidates: Vec<FilterCandidate> = predicates
.into_iter()
.map(|expr| {
- FilterCandidateBuilder::new(
- Arc::clone(expr),
- Arc::clone(physical_file_schema),
- Arc::clone(predicate_file_schema),
- )
- .build(metadata)
+ FilterCandidateBuilder::new(Arc::clone(expr), Arc::clone(file_schema))
+ .build(metadata)
})
.collect::<Result<Vec<_>, _>>()?
.into_iter()
@@ -516,10 +496,9 @@ mod test {
let table_schema = Arc::new(table_schema.clone());
- let candidate =
- FilterCandidateBuilder::new(expr, table_schema.clone(), table_schema)
- .build(metadata)
- .expect("building candidate");
+ let candidate = FilterCandidateBuilder::new(expr, table_schema)
+ .build(metadata)
+ .expect("building candidate");
assert!(candidate.is_none());
}
@@ -553,12 +532,10 @@ mod test {
.create(Arc::new(table_schema.clone()), Arc::clone(&file_schema))
.rewrite(expr)
.expect("rewriting expression");
- let table_schema = Arc::new(table_schema.clone());
- let candidate =
- FilterCandidateBuilder::new(expr, file_schema.clone(), table_schema.clone())
- .build(&metadata)
- .expect("building candidate")
- .expect("candidate expected");
+ let candidate = FilterCandidateBuilder::new(expr, file_schema.clone())
+ .build(&metadata)
+ .expect("building candidate")
+ .expect("candidate expected");
let mut row_filter = DatafusionArrowPredicate::try_new(
candidate,
@@ -591,10 +568,10 @@ mod test {
let expr = logical2physical(&expr, &table_schema);
// Rewrite the expression to add CastExpr for type coercion
let expr = DefaultPhysicalExprAdapterFactory {}
- .create(table_schema.clone(), Arc::clone(&file_schema))
+ .create(Arc::new(table_schema), Arc::clone(&file_schema))
.rewrite(expr)
.expect("rewriting expression");
- let candidate = FilterCandidateBuilder::new(expr, file_schema, table_schema)
+ let candidate = FilterCandidateBuilder::new(expr, file_schema)
.build(&metadata)
.expect("building candidate")
.expect("candidate expected");
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index d2a140c4f8..c6a58a7c98 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -31,7 +31,6 @@ use datafusion_common::config::ConfigOptions;
use datafusion_common::config::EncryptionFactoryOptions;
use datafusion_datasource::as_file_source;
use datafusion_datasource::file_stream::FileOpener;
-use datafusion_datasource::projection::{ProjectionOpener, SplitProjection};
use arrow::datatypes::TimeUnit;
use datafusion_common::config::TableParquetOptions;
@@ -40,6 +39,7 @@ use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
use datafusion_datasource::TableSchema;
use datafusion_physical_expr::conjunction;
+use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
use datafusion_physical_expr_common::physical_expr::fmt_sql;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -49,7 +49,6 @@ use datafusion_physical_plan::filter_pushdown::{
};
use datafusion_physical_plan::metrics::Count;
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
-use datafusion_physical_plan::projection::ProjectionExprs;
use datafusion_physical_plan::DisplayFormatType;
#[cfg(feature = "parquet_encryption")]
@@ -284,8 +283,8 @@ pub struct ParquetSource {
pub(crate) batch_size: Option<usize>,
/// Optional hint for the size of the parquet metadata
pub(crate) metadata_size_hint: Option<usize>,
- /// Projection information for column pushdown
- pub(crate) projection: SplitProjection,
+ /// Projection to apply to the output.
+ pub(crate) projection: ProjectionExprs,
#[cfg(feature = "parquet_encryption")]
pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,
}
@@ -298,8 +297,11 @@ impl ParquetSource {
/// To set custom options, use [ParquetSource::with_table_parquet_options`].
pub fn new(table_schema: impl Into<TableSchema>) -> Self {
let table_schema = table_schema.into();
+ // Projection over the full table schema (file columns + partition columns)
+ let full_schema = table_schema.table_schema();
+ let indices: Vec<usize> = (0..full_schema.fields().len()).collect();
Self {
- projection: SplitProjection::unprojected(&table_schema),
+ projection: ProjectionExprs::from_indices(&indices, full_schema),
table_schema,
table_parquet_options: TableParquetOptions::default(),
metrics: ExecutionPlanMetricsSet::new(),
@@ -497,12 +499,10 @@ impl FileSource for ParquetSource {
base_config: &FileScanConfig,
partition: usize,
) -> datafusion_common::Result<Arc<dyn FileOpener>> {
- let split_projection = self.projection.clone();
-
let expr_adapter_factory = base_config
.expr_adapter_factory
.clone()
- .or_else(|| Some(Arc::new(DefaultPhysicalExprAdapterFactory) as _));
+ .unwrap_or_else(|| Arc::new(DefaultPhysicalExprAdapterFactory) as _);
let parquet_file_reader_factory =
self.parquet_file_reader_factory.clone().unwrap_or_else(|| {
@@ -525,9 +525,9 @@ impl FileSource for ParquetSource {
.as_ref()
.map(|time_unit| parse_coerce_int96_string(time_unit.as_str()).unwrap());
- let mut opener = Arc::new(ParquetOpener {
+ let opener = Arc::new(ParquetOpener {
partition_index: partition,
- projection: Arc::from(split_projection.file_indices.clone()),
+ projection: self.projection.clone(),
batch_size: self
.batch_size
.expect("Batch size must set before creating ParquetOpener"),
@@ -550,12 +550,7 @@ impl FileSource for ParquetSource {
#[cfg(feature = "parquet_encryption")]
encryption_factory: self.get_encryption_factory_with_config(),
max_predicate_cache_size: self.max_predicate_cache_size(),
- }) as Arc<dyn FileOpener>;
- opener = ProjectionOpener::try_new(
- split_projection.clone(),
- Arc::clone(&opener),
- self.table_schema.file_schema(),
- )?;
+ });
Ok(opener)
}
@@ -582,15 +577,12 @@ impl FileSource for ParquetSource {
projection: &ProjectionExprs,
) -> datafusion_common::Result<Option<Arc<dyn FileSource>>> {
let mut source = self.clone();
- let new_projection = self.projection.source.try_merge(projection)?;
- let split_projection =
- SplitProjection::new(self.table_schema.file_schema(), &new_projection);
- source.projection = split_projection;
+ source.projection = self.projection.try_merge(projection)?;
Ok(Some(Arc::new(source)))
}
fn projection(&self) -> Option<&ProjectionExprs> {
- Some(&self.projection.source)
+ Some(&self.projection)
}
fn metrics(&self) -> &ExecutionPlanMetricsSet {
diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md
index cecc0ef25b..159bd3e4e7 100644
--- a/docs/source/library-user-guide/upgrading.md
+++ b/docs/source/library-user-guide/upgrading.md
@@ -229,6 +229,52 @@ let expr_with_literals = replace_columns_with_literals(expr, &partition_values)?
let adapted_expr = adapter.rewrite(expr_with_literals)?;
```
+### `build_row_filter` signature simplified
+
+The `build_row_filter` function in `datafusion-datasource-parquet` has been simplified to take a single schema parameter instead of two.
+The expectation is now that the filter has been adapted to the physical file schema (the arrow representation of the parquet file's schema) before being passed to this function
+using a `PhysicalExprAdapter` for example.
+
+**Who is affected:**
+
+- Users who call `build_row_filter` directly
+
+**Breaking changes:**
+
+The function signature changed from:
+
+```rust,ignore
+pub fn build_row_filter(
+ expr: &Arc<dyn PhysicalExpr>,
+ physical_file_schema: &SchemaRef,
+ predicate_file_schema: &SchemaRef, // removed
+ metadata: &ParquetMetaData,
+ reorder_predicates: bool,
+ file_metrics: &ParquetFileMetrics,
+) -> Result<Option<RowFilter>>
+```
+
+To:
+
+```rust,ignore
+pub fn build_row_filter(
+ expr: &Arc<dyn PhysicalExpr>,
+ file_schema: &SchemaRef,
+ metadata: &ParquetMetaData,
+ reorder_predicates: bool,
+ file_metrics: &ParquetFileMetrics,
+) -> Result<Option<RowFilter>>
+```
+
+**Migration guide:**
+
+Remove the duplicate schema parameter from your call:
+
+```diff
+- build_row_filter(&predicate, &file_schema, &file_schema, metadata, reorder, metrics)
++ build_row_filter(&predicate, &file_schema, metadata, reorder, metrics)
+```
+
### Planner now requires explicit opt-in for WITHIN GROUP syntax
The SQL planner now enforces the aggregate UDF contract more strictly: the