Apache DataFusion SQL Query Engine
Misc improvements to ProjectionExprs (#18719)
Adrian Garcia Badaraccob3ff6d83b2ff
diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rsindex 4e60b1dcc0..74edaa1915 100644--- a/datafusion/physical-expr/src/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -22,6 +22,7 @@ use crate::expressions::Column;use crate::utils::collect_columns;use crate::PhysicalExpr;+use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::datatypes::{Field, Schema, SchemaRef};use datafusion_common::stats::{ColumnStatistics, Precision};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};@@ -30,6 +31,7 @@ use datafusion_common::{};use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use indexmap::IndexMap;use itertools::Itertools;@@ -49,6 +51,15 @@ pub struct ProjectionExpr {pub alias: String,}+impl PartialEq for ProjectionExpr {+ fn eq(&self, other: &Self) -> bool {+ let ProjectionExpr { expr, alias } = self;+ expr.eq(&other.expr) && *alias == other.alias+ }+}++impl Eq for ProjectionExpr {}+impl std::fmt::Display for ProjectionExpr {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {if self.expr.to_string() == self.alias {@@ -101,7 +112,7 @@ impl From<ProjectionExpr> for (Arc<dyn PhysicalExpr>, String) {/// This struct encapsulates multiple `ProjectionExpr` instances,/// representing a complete projection operation and provides/// methods to manipulate and analyze the projection as a whole.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, PartialEq, Eq)]pub struct ProjectionExprs {exprs: Vec<ProjectionExpr>,}@@ -194,7 +205,7 @@ impl ProjectionExprs {/// assert_eq!(projection_with_dups.as_ref()[1].alias, "a"); // duplicate/// assert_eq!(projection_with_dups.as_ref()[2].alias, "b");/// ```- pub fn from_indices(indices: &[usize], schema: &SchemaRef) -> Self {+ pub fn from_indices(indices: &[usize], schema: &Schema) -> Self {let projection_exprs = indices.iter().map(|&i| {let field = schema.field(i);ProjectionExpr {@@ -398,6 +409,22 @@ impl ProjectionExprs {))}+ /// Create a new [`Projector`] from this projection and an input schema.+ ///+ /// A [`Projector`] can be used to apply this projection to record batches.+ ///+ /// # Errors+ /// This function returns an error if the output schema cannot be constructed from the input schema+ /// with the given projection expressions.+ /// For example, if an expression only works with integer columns but the input schema has a string column at that index.+ pub fn make_projector(&self, input_schema: &Schema) -> Result<Projector> {+ let output_schema = Arc::new(self.project_schema(input_schema)?);+ Ok(Projector {+ projection: self.clone(),+ output_schema,+ })+ }+/// Project statistics according to this projection./// For example, for a projection `SELECT a AS x, b + 1 AS y`, where `a` is at index 0 and `b` is at index 1,/// if the input statistics has column statistics for columns `a`, `b`, and `c`, the output statistics would have column statistics for columns `x` and `y`.@@ -446,6 +473,57 @@ impl<'a> IntoIterator for &'a ProjectionExprs {}}+/// Applies a projection to record batches.+///+/// A [`Projector`] uses a set of projection expressions to transform+/// and a pre-computed output schema to project record batches accordingly.+///+/// The main reason to use a `Projector` is to avoid repeatedly computing+/// the output schema for each batch, which can be costly if the projection+/// expressions are complex.+#[derive(Clone, Debug)]+pub struct Projector {+ projection: ProjectionExprs,+ output_schema: SchemaRef,+}++impl Projector {+ /// Project a record batch according to this projector's expressions.+ ///+ /// # Errors+ /// This function returns an error if any expression evaluation fails+ /// or if the output schema of the resulting record batch does not match+ /// the pre-computed output schema of the projector.+ pub fn project_batch(&self, batch: &RecordBatch) -> Result<RecordBatch> {+ let arrays = evaluate_expressions_to_arrays(+ self.projection.exprs.iter().map(|p| &p.expr),+ batch,+ )?;++ if arrays.is_empty() {+ let options =+ RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));+ RecordBatch::try_new_with_options(+ Arc::clone(&self.output_schema),+ arrays,+ &options,+ )+ .map_err(Into::into)+ } else {+ RecordBatch::try_new(Arc::clone(&self.output_schema), arrays)+ .map_err(Into::into)+ }+ }++ pub fn output_schema(&self) -> &SchemaRef {+ &self.output_schema+ }++ pub fn projection(&self) -> &ProjectionExprs {+ &self.projection+ }+}+impl IntoIterator for ProjectionExprs {type Item = ProjectionExpr;type IntoIter = std::vec::IntoIter<ProjectionExpr>;@@ -547,7 +625,13 @@ pub fn update_expr(}).data()?;- Ok((state == RewriteState::RewrittenValid).then_some(new_expr))+ match state {+ RewriteState::RewrittenInvalid => Ok(None),+ // Both Unchanged and RewrittenValid are valid:+ // - Unchanged means no columns to rewrite (e.g., literals)+ // - RewrittenValid means columns were successfully rewritten+ RewriteState::Unchanged | RewriteState::RewrittenValid => Ok(Some(new_expr)),+ }}/// Stores target expressions, along with their indices, that associate with a@@ -2009,6 +2093,94 @@ pub(crate) mod tests {);}+ #[test]+ fn test_merge_empty_projection_with_literal() -> Result<()> {+ // This test reproduces the issue from roundtrip_empty_projection test+ // Query like: SELECT 1 FROM table+ // where the file scan needs no columns (empty projection)+ // but we project a literal on top++ // Empty base projection (no columns needed from file)+ let base_projection = ProjectionExprs::new(vec![]);++ // Top projection with a literal expression: SELECT 1+ let top_projection = ProjectionExprs::new(vec![ProjectionExpr {+ expr: Arc::new(Literal::new(ScalarValue::Int64(Some(1)))),+ alias: "Int64(1)".to_string(),+ }]);++ // This should succeed - literals don't reference columns so they should+ // pass through unchanged when merged with an empty projection+ let merged = base_projection.try_merge(&top_projection)?;+ assert_snapshot!(format!("{merged}"), @"Projection[1 AS Int64(1)]");++ Ok(())+ }++ #[test]+ fn test_update_expr_with_literal() -> Result<()> {+ // Test that update_expr correctly handles expressions without column references+ let literal_expr: Arc<dyn PhysicalExpr> =+ Arc::new(Literal::new(ScalarValue::Int64(Some(42))));+ let empty_projection: Vec<ProjectionExpr> = vec![];++ // Updating a literal with an empty projection should return the literal unchanged+ let result = update_expr(&literal_expr, &empty_projection, true)?;+ assert!(result.is_some(), "Literal expression should be valid");++ let result_expr = result.unwrap();+ assert_eq!(+ result_expr+ .as_any()+ .downcast_ref::<Literal>()+ .unwrap()+ .value(),+ &ScalarValue::Int64(Some(42))+ );++ Ok(())+ }++ #[test]+ fn test_update_expr_with_complex_literal_expr() -> Result<()> {+ // Test update_expr with an expression containing both literals and a column+ // This tests the case where we have: literal + column+ let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(Literal::new(ScalarValue::Int64(Some(10)))),+ Operator::Plus,+ Arc::new(Column::new("x", 0)),+ ));++ // Base projection that maps column 0 to a different expression+ let base_projection = vec![ProjectionExpr {+ expr: Arc::new(Column::new("a", 5)),+ alias: "x".to_string(),+ }];++ // The expression should be updated: 10 + x@0 becomes 10 + a@5+ let result = update_expr(&expr, &base_projection, true)?;+ assert!(result.is_some(), "Expression should be valid");++ let result_expr = result.unwrap();+ let binary = result_expr+ .as_any()+ .downcast_ref::<BinaryExpr>()+ .expect("Should be a BinaryExpr");++ // Left side should still be the literal+ assert!(binary.left().as_any().downcast_ref::<Literal>().is_some());++ // Right side should be updated to reference column at index 5+ let right_col = binary+ .right()+ .as_any()+ .downcast_ref::<Column>()+ .expect("Right should be a Column");+ assert_eq!(right_col.index(), 5);++ Ok(())+ }+#[test]fn test_project_schema_simple_columns() -> Result<()> {// Input schema: [col0: Int64, col1: Utf8, col2: Float32]diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 8bc2bcd6f2..abd8daa3fd 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -40,7 +40,7 @@ use std::sync::Arc;use std::task::{Context, Poll};use arrow::datatypes::SchemaRef;-use arrow::record_batch::{RecordBatch, RecordBatchOptions};+use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,@@ -48,6 +48,7 @@ use datafusion_common::tree_node::{use datafusion_common::{internal_err, JoinSide, Result};use datafusion_execution::TaskContext;use datafusion_physical_expr::equivalence::ProjectionMapping;+use datafusion_physical_expr::projection::Projector;use datafusion_physical_expr::utils::collect_columns;use datafusion_physical_expr_common::physical_expr::{fmt_sql, PhysicalExprRef};use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};@@ -57,7 +58,6 @@ pub use datafusion_physical_expr::projection::{update_expr, ProjectionExpr, ProjectionExprs,};-use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::stream::{Stream, StreamExt};use log::trace;@@ -67,10 +67,9 @@ use log::trace;/// output row for each input row.#[derive(Debug, Clone)]pub struct ProjectionExec {- /// The projection expressions stored as tuples of (expression, output column name)- projection: ProjectionExprs,- /// The schema once the projection has been applied to the input- schema: SchemaRef,+ /// A projector specialized to apply the projection to the input schema from the child node+ /// and produce [`RecordBatch`]es with the output schema of this node.+ projector: Projector,/// The input planinput: Arc<dyn ExecutionPlan>,/// Execution metrics@@ -138,16 +137,17 @@ impl ProjectionExec {// convert argument to Vec<ProjectionExpr>let expr_vec = expr.into_iter().map(Into::into).collect::<Vec<_>>();let projection = ProjectionExprs::new(expr_vec);-- let schema = Arc::new(projection.project_schema(&input_schema)?);+ let projector = projection.make_projector(&input_schema)?;// Construct a map from the input expressions to the output expression of the Projectionlet projection_mapping = projection.projection_mapping(&input_schema)?;- let cache =- Self::compute_properties(&input, &projection_mapping, Arc::clone(&schema))?;+ let cache = Self::compute_properties(+ &input,+ &projection_mapping,+ Arc::clone(projector.output_schema()),+ )?;Ok(Self {- projection,- schema,+ projector,input,metrics: ExecutionPlanMetricsSet::new(),cache,@@ -156,7 +156,7 @@ impl ProjectionExec {/// The projection expressions stored as tuples of (expression, output column name)pub fn expr(&self) -> &[ProjectionExpr] {- self.projection.as_ref()+ self.projector.projection().as_ref()}/// The input plan@@ -196,7 +196,8 @@ impl DisplayAs for ProjectionExec {match t {DisplayFormatType::Default | DisplayFormatType::Verbose => {let expr: Vec<String> = self- .projection+ .projector+ .projection().as_ref().iter().map(|proj_expr| {@@ -247,10 +248,15 @@ impl ExecutionPlan for ProjectionExec {}fn benefits_from_input_partitioning(&self) -> Vec<bool> {- let all_simple_exprs = self.projection.iter().all(|proj_expr| {- proj_expr.expr.as_any().is::<Column>()- || proj_expr.expr.as_any().is::<Literal>()- });+ let all_simple_exprs =+ self.projector+ .projection()+ .as_ref()+ .iter()+ .all(|proj_expr| {+ proj_expr.expr.as_any().is::<Column>()+ || proj_expr.expr.as_any().is::<Literal>()+ });// If expressions are all either column_expr or Literal, then all computations in this projection are reorder or rename,// and projection would not benefit from the repartition, benefits_from_input_partitioning will return false.vec![!all_simple_exprs]@@ -264,8 +270,11 @@ impl ExecutionPlan for ProjectionExec {self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- ProjectionExec::try_new(self.projection.clone(), children.swap_remove(0))- .map(|p| Arc::new(p) as _)+ ProjectionExec::try_new(+ self.projector.projection().clone(),+ children.swap_remove(0),+ )+ .map(|p| Arc::new(p) as _)}fn execute(@@ -275,11 +284,10 @@ impl ExecutionPlan for ProjectionExec {) -> Result<SendableRecordBatchStream> {trace!("Start ProjectionExec::execute for partition {} of context session_id {} and task_id {:?}", partition, context.session_id(), context.task_id());Ok(Box::pin(ProjectionStream::new(- Arc::clone(&self.schema),- self.projection.expr_iter().collect(),+ self.projector.clone(),self.input.execute(partition, context)?,BaselineMetrics::new(&self.metrics, partition),- )))+ )?))}fn metrics(&self) -> Option<MetricsSet> {@@ -292,7 +300,8 @@ impl ExecutionPlan for ProjectionExec {fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {let input_stats = self.input.partition_statistics(partition)?;- self.projection+ self.projector+ .projection().project_statistics(input_stats, &self.input.schema())}@@ -342,39 +351,27 @@ impl ExecutionPlan for ProjectionExec {impl ProjectionStream {/// Create a new projection streamfn new(- schema: SchemaRef,- expr: Vec<Arc<dyn PhysicalExpr>>,+ projector: Projector,input: SendableRecordBatchStream,baseline_metrics: BaselineMetrics,- ) -> Self {- Self {- schema,- expr,+ ) -> Result<Self> {+ Ok(Self {+ projector,input,baseline_metrics,- }+ })}fn batch_project(&self, batch: &RecordBatch) -> Result<RecordBatch> {// Records time on droplet _timer = self.baseline_metrics.elapsed_compute().timer();- let arrays = evaluate_expressions_to_arrays(&self.expr, batch)?;-- if arrays.is_empty() {- let options =- RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));- RecordBatch::try_new_with_options(Arc::clone(&self.schema), arrays, &options)- .map_err(Into::into)- } else {- RecordBatch::try_new(Arc::clone(&self.schema), arrays).map_err(Into::into)- }+ self.projector.project_batch(batch)}}/// Projection iteratorstruct ProjectionStream {- schema: SchemaRef,- expr: Vec<Arc<dyn PhysicalExpr>>,+ projector: Projector,input: SendableRecordBatchStream,baseline_metrics: BaselineMetrics,}@@ -403,7 +400,7 @@ impl Stream for ProjectionStream {impl RecordBatchStream for ProjectionStream {/// Get the schemafn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)+ Arc::clone(self.projector.output_schema())}}