Apache DataFusion SQL Query Engine
Add Projection struct w/ helper methods to manipulate projections (#18176)
Adrian Garcia Badaracco347b2b6cc3ee
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex bea51d31ba..0d784c9179 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -3063,7 +3063,7 @@ mod tests {let execution_plan = plan(&logical_plan).await?;// verify that the plan correctly adds cast from Int64(1) to Utf8, and the const will be evaluated.- let expected = "expr: [ProjectionExpr { expr: BinaryExpr { left: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"a\"), field: Field { name: \"lit\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }, op: Or, right: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"1\"), field: Field { name: \"lit\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }, fail_on_overflow: false }";+ let expected = "exprs: [ProjectionExpr { expr: BinaryExpr { left: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"a\"), field: Field { name: \"lit\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }, op: Or, right: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"1\"), field: Field { name: \"lit\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }, fail_on_overflow: false }";assert_contains!(format!("{execution_plan:?}"), expected);diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rsindex 66ce77ef41..5b64884f65 100644--- a/datafusion/physical-expr/src/equivalence/class.rs+++ b/datafusion/physical-expr/src/equivalence/class.rs@@ -20,10 +20,10 @@ use std::ops::Deref;use std::sync::Arc;use std::vec::IntoIter;-use super::projection::ProjectionTargets;use super::ProjectionMapping;use crate::expressions::Literal;use crate::physical_expr::add_offset_to_expr;+use crate::projection::ProjectionTargets;use crate::{PhysicalExpr, PhysicalExprRef, PhysicalSortExpr, PhysicalSortRequirement};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rsindex bcc6835e2f..a728910380 100644--- a/datafusion/physical-expr/src/equivalence/mod.rs+++ b/datafusion/physical-expr/src/equivalence/mod.rs@@ -25,12 +25,13 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};mod class;mod ordering;-mod projection;mod properties;pub use class::{AcrossPartitions, ConstExpr, EquivalenceClass, EquivalenceGroup};pub use ordering::OrderingEquivalenceClass;-pub use projection::{project_ordering, project_orderings, ProjectionMapping};+// Re-export for backwards compatibility, we recommend importing from+// datafusion_physical_expr::projection instead+pub use crate::projection::{project_ordering, project_orderings, ProjectionMapping};pub use properties::{calculate_union, join_equivalence_properties, EquivalenceProperties,};@@ -61,7 +62,7 @@ mod tests {use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};- use datafusion_common::{plan_err, Result};+ use datafusion_common::Result;use datafusion_physical_expr_common::sort_expr::PhysicalSortRequirement;/// Converts a string to a physical sort expression@@ -95,31 +96,6 @@ mod tests {sort_expr}- pub fn output_schema(- mapping: &ProjectionMapping,- input_schema: &Arc<Schema>,- ) -> Result<SchemaRef> {- // Calculate output schema:- let mut fields = vec![];- for (source, targets) in mapping.iter() {- let data_type = source.data_type(input_schema)?;- let nullable = source.nullable(input_schema)?;- for (target, _) in targets.iter() {- let Some(column) = target.as_any().downcast_ref::<Column>() else {- return plan_err!("Expects to have column");- };- fields.push(Field::new(column.name(), data_type.clone(), nullable));- }- }-- let output_schema = Arc::new(Schema::new_with_metadata(- fields,- input_schema.metadata().clone(),- ));-- Ok(output_schema)- }-// Generate a schema which consists of 8 columns (a, b, c, d, e, f, g, h)pub fn create_test_schema() -> Result<SchemaRef> {let a = Field::new("a", DataType::Int32, true);diff --git a/datafusion/physical-expr/src/equivalence/properties/dependency.rs b/datafusion/physical-expr/src/equivalence/properties/dependency.rsindex 26d5d32c65..8945d18be4 100644--- a/datafusion/physical-expr/src/equivalence/properties/dependency.rs+++ b/datafusion/physical-expr/src/equivalence/properties/dependency.rs@@ -387,11 +387,11 @@ mod tests {use super::*;use crate::equivalence::tests::{- convert_to_sort_reqs, create_test_params, create_test_schema, output_schema,- parse_sort_expr,+ convert_to_sort_reqs, create_test_params, create_test_schema, parse_sort_expr,};use crate::equivalence::{convert_to_sort_exprs, ProjectionMapping};use crate::expressions::{col, BinaryExpr, CastExpr, Column};+ use crate::projection::tests::output_schema;use crate::{ConstExpr, EquivalenceProperties, ScalarFunctionExpr};use arrow::compute::SortOptions;diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rsindex 468591d34d..aa8c9e50fd 100644--- a/datafusion/physical-expr/src/lib.rs+++ b/datafusion/physical-expr/src/lib.rs@@ -37,6 +37,7 @@ pub mod intervals;mod partitioning;mod physical_expr;pub mod planner;+pub mod projection;mod scalar_function;pub mod simplifier;pub mod statistics;diff --git a/datafusion/physical-expr/src/equivalence/projection.rs b/datafusion/physical-expr/src/projection.rssimilarity index 51%rename from datafusion/physical-expr/src/equivalence/projection.rsrename to datafusion/physical-expr/src/projection.rsindex a4ed8187cf..e35bfbb3a2 100644--- a/datafusion/physical-expr/src/equivalence/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -19,14 +19,426 @@ use std::ops::Deref;use std::sync::Arc;use crate::expressions::Column;+use crate::utils::collect_columns;use crate::PhysicalExpr;-use arrow::datatypes::SchemaRef;+use arrow::datatypes::{Field, Schema, SchemaRef};+use datafusion_common::stats::{ColumnStatistics, Precision};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::{internal_err, plan_err, Result};+use datafusion_common::{internal_datafusion_err, internal_err, plan_err, Result};use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use indexmap::IndexMap;+use itertools::Itertools;++/// A projection expression as used by projection operations.+///+/// The expression is evaluated and the result is stored in a column+/// with the name specified by `alias`.+///+/// For example, the SQL expression `a + b AS sum_ab` would be represented+/// as a `ProjectionExpr` where `expr` is the expression `a + b`+/// and `alias` is the string `sum_ab`.+#[derive(Debug, Clone)]+pub struct ProjectionExpr {+ /// The expression that will be evaluated.+ pub expr: Arc<dyn PhysicalExpr>,+ /// The name of the output column for use an output schema.+ pub alias: String,+}++impl std::fmt::Display for ProjectionExpr {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ if self.expr.to_string() == self.alias {+ write!(f, "{}", self.alias)+ } else {+ write!(f, "{} AS {}", self.expr, self.alias)+ }+ }+}++impl ProjectionExpr {+ /// Create a new projection expression+ pub fn new(expr: Arc<dyn PhysicalExpr>, alias: String) -> Self {+ Self { expr, alias }+ }++ /// Create a new projection expression from an expression and a schema using the expression's output field name as alias.+ pub fn new_from_expression(+ expr: Arc<dyn PhysicalExpr>,+ schema: &Schema,+ ) -> Result<Self> {+ let field = expr.return_field(schema)?;+ Ok(Self {+ expr,+ alias: field.name().to_string(),+ })+ }+}++impl From<(Arc<dyn PhysicalExpr>, String)> for ProjectionExpr {+ fn from(value: (Arc<dyn PhysicalExpr>, String)) -> Self {+ Self::new(value.0, value.1)+ }+}++impl From<&(Arc<dyn PhysicalExpr>, String)> for ProjectionExpr {+ fn from(value: &(Arc<dyn PhysicalExpr>, String)) -> Self {+ Self::new(Arc::clone(&value.0), value.1.clone())+ }+}++impl From<ProjectionExpr> for (Arc<dyn PhysicalExpr>, String) {+ fn from(value: ProjectionExpr) -> Self {+ (value.expr, value.alias)+ }+}++/// A collection of projection expressions.+///+/// This struct encapsulates multiple `ProjectionExpr` instances,+/// representing a complete projection operation and provides+/// methods to manipulate and analyze the projection as a whole.+#[derive(Debug, Clone)]+pub struct Projection {+ exprs: Vec<ProjectionExpr>,+}++impl std::fmt::Display for Projection {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ let exprs: Vec<String> = self.exprs.iter().map(|e| e.to_string()).collect();+ write!(f, "Projection[{}]", exprs.join(", "))+ }+}++impl From<Vec<ProjectionExpr>> for Projection {+ fn from(value: Vec<ProjectionExpr>) -> Self {+ Self { exprs: value }+ }+}++impl From<&[ProjectionExpr]> for Projection {+ fn from(value: &[ProjectionExpr]) -> Self {+ Self {+ exprs: value.to_vec(),+ }+ }+}++impl AsRef<[ProjectionExpr]> for Projection {+ fn as_ref(&self) -> &[ProjectionExpr] {+ &self.exprs+ }+}++impl Projection {+ pub fn new(exprs: Vec<ProjectionExpr>) -> Self {+ Self { exprs }+ }++ /// Returns an iterator over the projection expressions+ pub fn iter(&self) -> impl Iterator<Item = &ProjectionExpr> {+ self.exprs.iter()+ }++ /// Creates a ProjectionMapping from this projection+ pub fn projection_mapping(+ &self,+ input_schema: &SchemaRef,+ ) -> Result<ProjectionMapping> {+ ProjectionMapping::try_new(+ self.exprs+ .iter()+ .map(|p| (Arc::clone(&p.expr), p.alias.clone())),+ input_schema,+ )+ }++ /// Iterate over a clone of the projection expressions.+ pub fn expr_iter(&self) -> impl Iterator<Item = Arc<dyn PhysicalExpr>> + '_ {+ self.exprs.iter().map(|e| Arc::clone(&e.expr))+ }++ /// Apply another projection on top of this projection, returning the combined projection.+ /// For example, if this projection is `SELECT c@2 AS x, b@1 AS y, a@0 as z` and the other projection is `SELECT x@0 + 1 AS c1, y@1 + z@2 as c2`,+ /// we return a projection equivalent to `SELECT c@2 + 1 AS c1, b@1 + a@0 as c2`.+ ///+ /// # Example+ ///+ /// ```rust+ /// use std::sync::Arc;+ /// use datafusion_physical_expr::projection::{Projection, ProjectionExpr};+ /// use datafusion_physical_expr::expressions::{Column, BinaryExpr, Literal};+ /// use datafusion_common::{Result, ScalarValue};+ /// use datafusion_expr::Operator;+ ///+ /// fn main() -> Result<()> {+ /// // Example from the docstring:+ /// // Base projection: SELECT c@2 AS x, b@1 AS y, a@0 AS z+ /// let base = Projection::new(vec![+ /// ProjectionExpr {+ /// expr: Arc::new(Column::new("c", 2)),+ /// alias: "x".to_string(),+ /// },+ /// ProjectionExpr {+ /// expr: Arc::new(Column::new("b", 1)),+ /// alias: "y".to_string(),+ /// },+ /// ProjectionExpr {+ /// expr: Arc::new(Column::new("a", 0)),+ /// alias: "z".to_string(),+ /// },+ /// ]);+ ///+ /// // Top projection: SELECT x@0 + 1 AS c1, y@1 + z@2 AS c2+ /// let top = Projection::new(vec![+ /// ProjectionExpr {+ /// expr: Arc::new(BinaryExpr::new(+ /// Arc::new(Column::new("x", 0)),+ /// Operator::Plus,+ /// Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),+ /// )),+ /// alias: "c1".to_string(),+ /// },+ /// ProjectionExpr {+ /// expr: Arc::new(BinaryExpr::new(+ /// Arc::new(Column::new("y", 1)),+ /// Operator::Plus,+ /// Arc::new(Column::new("z", 2)),+ /// )),+ /// alias: "c2".to_string(),+ /// },+ /// ]);+ ///+ /// // Expected result: SELECT c@2 + 1 AS c1, b@1 + a@0 AS c2+ /// let result = base.try_merge(&top)?;+ ///+ /// assert_eq!(result.as_ref().len(), 2);+ /// assert_eq!(result.as_ref()[0].alias, "c1");+ /// assert_eq!(result.as_ref()[1].alias, "c2");+ ///+ /// Ok(())+ /// }+ /// ```+ ///+ /// # Errors+ /// This function returns an error if any expression in the `other` projection cannot be+ /// applied on top of this projection.+ pub fn try_merge(&self, other: &Projection) -> Result<Projection> {+ let mut new_exprs = Vec::with_capacity(other.exprs.len());+ for proj_expr in &other.exprs {+ let new_expr = update_expr(&proj_expr.expr, &self.exprs, true)?+ .ok_or_else(|| {+ internal_datafusion_err!(+ "Failed to combine projections: expression {} could not be applied on top of existing projections {}",+ proj_expr.expr,+ self.exprs.iter().map(|e| format!("{e}")).join(", ")+ )+ })?;+ new_exprs.push(ProjectionExpr {+ expr: new_expr,+ alias: proj_expr.alias.clone(),+ });+ }+ Ok(Projection::new(new_exprs))+ }++ /// Extract the column indices used in this projection.+ /// For example, for a projection `SELECT a AS x, b + 1 AS y`, where `a` is at index 0 and `b` is at index 1,+ /// this function would return `[0, 1]`.+ /// Repeated indices are returned only once, and the order is ascending.+ pub fn column_indices(&self) -> Vec<usize> {+ self.exprs+ .iter()+ .flat_map(|e| collect_columns(&e.expr).into_iter().map(|col| col.index()))+ .sorted_unstable()+ .dedup()+ .collect_vec()+ }++ /// Project a schema according to this projection.+ /// For example, for a projection `SELECT a AS x, b + 1 AS y`, where `a` is at index 0 and `b` is at index 1,+ /// if the input schema is `[a: Int32, b: Int32, c: Int32]`, the output schema would be `[x: Int32, y: Int32]`.+ /// Fields' metadata are preserved from the input schema.+ pub fn project_schema(&self, input_schema: &Schema) -> Result<Schema> {+ let fields: Result<Vec<Field>> = self+ .exprs+ .iter()+ .map(|proj_expr| {+ let metadata = proj_expr+ .expr+ .return_field(input_schema)?+ .metadata()+ .clone();++ let field = Field::new(+ &proj_expr.alias,+ proj_expr.expr.data_type(input_schema)?,+ proj_expr.expr.nullable(input_schema)?,+ )+ .with_metadata(metadata);++ Ok(field)+ })+ .collect();++ Ok(Schema::new_with_metadata(+ fields?,+ input_schema.metadata().clone(),+ ))+ }++ /// Project statistics according to this projection.+ /// For example, for a projection `SELECT a AS x, b + 1 AS y`, where `a` is at index 0 and `b` is at index 1,+ /// if the input statistics has column statistics for columns `a`, `b`, and `c`, the output statistics would have column statistics for columns `x` and `y`.+ pub fn project_statistics(+ &self,+ mut stats: datafusion_common::Statistics,+ input_schema: &Schema,+ ) -> Result<datafusion_common::Statistics> {+ let mut primitive_row_size = 0;+ let mut primitive_row_size_possible = true;+ let mut column_statistics = vec![];++ for proj_expr in &self.exprs {+ let expr = &proj_expr.expr;+ let col_stats = if let Some(col) = expr.as_any().downcast_ref::<Column>() {+ stats.column_statistics[col.index()].clone()+ } else {+ // TODO stats: estimate more statistics from expressions+ // (expressions should compute their statistics themselves)+ ColumnStatistics::new_unknown()+ };+ column_statistics.push(col_stats);+ let data_type = expr.data_type(input_schema)?;+ if let Some(value) = data_type.primitive_width() {+ primitive_row_size += value;+ continue;+ }+ primitive_row_size_possible = false;+ }++ if primitive_row_size_possible {+ stats.total_byte_size =+ Precision::Exact(primitive_row_size).multiply(&stats.num_rows);+ }+ stats.column_statistics = column_statistics;+ Ok(stats)+ }+}++impl<'a> IntoIterator for &'a Projection {+ type Item = &'a ProjectionExpr;+ type IntoIter = std::slice::Iter<'a, ProjectionExpr>;++ fn into_iter(self) -> Self::IntoIter {+ self.exprs.iter()+ }+}++impl IntoIterator for Projection {+ type Item = ProjectionExpr;+ type IntoIter = std::vec::IntoIter<ProjectionExpr>;++ fn into_iter(self) -> Self::IntoIter {+ self.exprs.into_iter()+ }+}++/// The function operates in two modes:+///+/// 1) When `sync_with_child` is `true`:+///+/// The function updates the indices of `expr` if the expression resides+/// in the input plan. For instance, given the expressions `a@1 + b@2`+/// and `c@0` with the input schema `c@2, a@0, b@1`, the expressions are+/// updated to `a@0 + b@1` and `c@2`.+///+/// 2) When `sync_with_child` is `false`:+///+/// The function determines how the expression would be updated if a projection+/// was placed before the plan associated with the expression. If the expression+/// cannot be rewritten after the projection, it returns `None`. For example,+/// given the expressions `c@0`, `a@1` and `b@2`, and the projection with+/// an output schema of `a, c_new`, then `c@0` becomes `c_new@1`, `a@1` becomes+/// `a@0`, but `b@2` results in `None` since the projection does not include `b`.+///+/// # Errors+/// This function returns an error if `sync_with_child` is `true` and if any expression references+/// an index that is out of bounds for `projected_exprs`.+/// For example:+///+/// - `expr` is `a@3`+/// - `projected_exprs` is \[`a@0`, `b@1`\]+///+/// In this case, `a@3` references index 3, which is out of bounds for `projected_exprs` (which has length 2).+pub fn update_expr(+ expr: &Arc<dyn PhysicalExpr>,+ projected_exprs: &[ProjectionExpr],+ sync_with_child: bool,+) -> Result<Option<Arc<dyn PhysicalExpr>>> {+ #[derive(Debug, PartialEq)]+ enum RewriteState {+ /// The expression is unchanged.+ Unchanged,+ /// Some part of the expression has been rewritten+ RewrittenValid,+ /// Some part of the expression has been rewritten, but some column+ /// references could not be.+ RewrittenInvalid,+ }++ let mut state = RewriteState::Unchanged;++ let new_expr = Arc::clone(expr)+ .transform_up(|expr| {+ if state == RewriteState::RewrittenInvalid {+ return Ok(Transformed::no(expr));+ }++ let Some(column) = expr.as_any().downcast_ref::<Column>() else {+ return Ok(Transformed::no(expr));+ };+ if sync_with_child {+ state = RewriteState::RewrittenValid;+ // Update the index of `column`:+ let projected_expr = projected_exprs.get(column.index()).ok_or_else(|| {+ internal_datafusion_err!(+ "Column index {} out of bounds for projected expressions of length {}",+ column.index(),+ projected_exprs.len()+ )+ })?;+ Ok(Transformed::yes(Arc::clone(&projected_expr.expr)))+ } else {+ // default to invalid, in case we can't find the relevant column+ state = RewriteState::RewrittenInvalid;+ // Determine how to update `column` to accommodate `projected_exprs`+ projected_exprs+ .iter()+ .enumerate()+ .find_map(|(index, proj_expr)| {+ proj_expr.expr.as_any().downcast_ref::<Column>().and_then(+ |projected_column| {+ (column.name().eq(projected_column.name())+ && column.index() == projected_column.index())+ .then(|| {+ state = RewriteState::RewrittenValid;+ Arc::new(Column::new(&proj_expr.alias, index)) as _+ })+ },+ )+ })+ .map_or_else(+ || Ok(Transformed::no(expr)),+ |c| Ok(Transformed::yes(c)),+ )+ }+ })+ .data()?;++ Ok((state == RewriteState::RewrittenValid).then_some(new_expr))+}/// Stores target expressions, along with their indices, that associate with a/// source expression in a projection mapping.@@ -249,18 +661,46 @@ pub fn project_ordering(}#[cfg(test)]-mod tests {+pub(crate) mod tests {+ use std::collections::HashMap;+use super::*;- use crate::equivalence::tests::output_schema;use crate::equivalence::{convert_to_orderings, EquivalenceProperties};- use crate::expressions::{col, BinaryExpr};+ use crate::expressions::{col, BinaryExpr, Literal};use crate::utils::tests::TestScalarUDF;use crate::{PhysicalExprRef, ScalarFunctionExpr};use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema, TimeUnit};use datafusion_common::config::ConfigOptions;+ use datafusion_common::{ScalarValue, Statistics};use datafusion_expr::{Operator, ScalarUDF};+ use insta::assert_snapshot;++ pub(crate) fn output_schema(+ mapping: &ProjectionMapping,+ input_schema: &Arc<Schema>,+ ) -> Result<SchemaRef> {+ // Calculate output schema:+ let mut fields = vec![];+ for (source, targets) in mapping.iter() {+ let data_type = source.data_type(input_schema)?;+ let nullable = source.nullable(input_schema)?;+ for (target, _) in targets.iter() {+ let Some(column) = target.as_any().downcast_ref::<Column>() else {+ return plan_err!("Expects to have column");+ };+ fields.push(Field::new(column.name(), data_type.clone(), nullable));+ }+ }++ let output_schema = Arc::new(Schema::new_with_metadata(+ fields,+ input_schema.metadata().clone(),+ ));++ Ok(output_schema)+ }#[test]fn project_orderings() -> Result<()> {@@ -1087,4 +1527,628 @@ mod tests {Ok(())}++ fn get_stats() -> Statistics {+ Statistics {+ num_rows: Precision::Exact(5),+ total_byte_size: Precision::Exact(23),+ column_statistics: vec![+ ColumnStatistics {+ distinct_count: Precision::Exact(5),+ max_value: Precision::Exact(ScalarValue::Int64(Some(21))),+ min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),+ sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),+ null_count: Precision::Exact(0),+ },+ ColumnStatistics {+ distinct_count: Precision::Exact(1),+ max_value: Precision::Exact(ScalarValue::from("x")),+ min_value: Precision::Exact(ScalarValue::from("a")),+ sum_value: Precision::Absent,+ null_count: Precision::Exact(3),+ },+ ColumnStatistics {+ distinct_count: Precision::Absent,+ max_value: Precision::Exact(ScalarValue::Float32(Some(1.1))),+ min_value: Precision::Exact(ScalarValue::Float32(Some(0.1))),+ sum_value: Precision::Exact(ScalarValue::Float32(Some(5.5))),+ null_count: Precision::Absent,+ },+ ],+ }+ }++ fn get_schema() -> Schema {+ let field_0 = Field::new("col0", DataType::Int64, false);+ let field_1 = Field::new("col1", DataType::Utf8, false);+ let field_2 = Field::new("col2", DataType::Float32, false);+ Schema::new(vec![field_0, field_1, field_2])+ }++ #[test]+ fn test_stats_projection_columns_only() {+ let source = get_stats();+ let schema = get_schema();++ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("col1", 1)),+ alias: "col1".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "col0".to_string(),+ },+ ]);++ let result = projection.project_statistics(source, &schema).unwrap();++ let expected = Statistics {+ num_rows: Precision::Exact(5),+ total_byte_size: Precision::Exact(23),+ column_statistics: vec![+ ColumnStatistics {+ distinct_count: Precision::Exact(1),+ max_value: Precision::Exact(ScalarValue::from("x")),+ min_value: Precision::Exact(ScalarValue::from("a")),+ sum_value: Precision::Absent,+ null_count: Precision::Exact(3),+ },+ ColumnStatistics {+ distinct_count: Precision::Exact(5),+ max_value: Precision::Exact(ScalarValue::Int64(Some(21))),+ min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),+ sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),+ null_count: Precision::Exact(0),+ },+ ],+ };++ assert_eq!(result, expected);+ }++ #[test]+ fn test_stats_projection_column_with_primitive_width_only() {+ let source = get_stats();+ let schema = get_schema();++ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("col2", 2)),+ alias: "col2".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "col0".to_string(),+ },+ ]);++ let result = projection.project_statistics(source, &schema).unwrap();++ let expected = Statistics {+ num_rows: Precision::Exact(5),+ total_byte_size: Precision::Exact(60),+ column_statistics: vec![+ ColumnStatistics {+ distinct_count: Precision::Absent,+ max_value: Precision::Exact(ScalarValue::Float32(Some(1.1))),+ min_value: Precision::Exact(ScalarValue::Float32(Some(0.1))),+ sum_value: Precision::Exact(ScalarValue::Float32(Some(5.5))),+ null_count: Precision::Absent,+ },+ ColumnStatistics {+ distinct_count: Precision::Exact(5),+ max_value: Precision::Exact(ScalarValue::Int64(Some(21))),+ min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),+ sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),+ null_count: Precision::Exact(0),+ },+ ],+ };++ assert_eq!(result, expected);+ }++ // Tests for Projection struct++ #[test]+ fn test_projection_new() -> Result<()> {+ let exprs = vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "a".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 1)),+ alias: "b".to_string(),+ },+ ];+ let projection = Projection::new(exprs.clone());+ assert_eq!(projection.as_ref().len(), 2);+ Ok(())+ }++ #[test]+ fn test_projection_from_vec() -> Result<()> {+ let exprs = vec![ProjectionExpr {+ expr: Arc::new(Column::new("x", 0)),+ alias: "x".to_string(),+ }];+ let projection: Projection = exprs.clone().into();+ assert_eq!(projection.as_ref().len(), 1);+ Ok(())+ }++ #[test]+ fn test_projection_as_ref() -> Result<()> {+ let exprs = vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("col1", 0)),+ alias: "col1".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col2", 1)),+ alias: "col2".to_string(),+ },+ ];+ let projection = Projection::new(exprs);+ let as_ref: &[ProjectionExpr] = projection.as_ref();+ assert_eq!(as_ref.len(), 2);+ Ok(())+ }++ #[test]+ fn test_column_indices_multiple_columns() -> Result<()> {+ // Test with reversed column order to ensure proper reordering+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("c", 5)),+ alias: "c".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 2)),+ alias: "b".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "a".to_string(),+ },+ ]);+ // Should return sorted indices regardless of projection order+ assert_eq!(projection.column_indices(), vec![0, 2, 5]);+ Ok(())+ }++ #[test]+ fn test_column_indices_duplicates() -> Result<()> {+ // Test that duplicate column indices appear only once+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 1)),+ alias: "a".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 3)),+ alias: "b".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("a2", 1)), // duplicate index+ alias: "a2".to_string(),+ },+ ]);+ assert_eq!(projection.column_indices(), vec![1, 3]);+ Ok(())+ }++ #[test]+ fn test_column_indices_unsorted() -> Result<()> {+ // Test that column indices are sorted in the output+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("c", 5)),+ alias: "c".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 1)),+ alias: "a".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 3)),+ alias: "b".to_string(),+ },+ ]);+ assert_eq!(projection.column_indices(), vec![1, 3, 5]);+ Ok(())+ }++ #[test]+ fn test_column_indices_complex_expr() -> Result<()> {+ // Test with complex expressions containing multiple columns+ let expr = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 1)),+ Operator::Plus,+ Arc::new(Column::new("b", 4)),+ ));+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr,+ alias: "sum".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("c", 2)),+ alias: "c".to_string(),+ },+ ]);+ // Should return [1, 2, 4] - all columns used, sorted and deduplicated+ assert_eq!(projection.column_indices(), vec![1, 2, 4]);+ Ok(())+ }++ #[test]+ fn test_column_indices_empty() -> Result<()> {+ let projection = Projection::new(vec![]);+ assert_eq!(projection.column_indices(), Vec::<usize>::new());+ Ok(())+ }++ #[test]+ fn test_merge_simple_columns() -> Result<()> {+ // First projection: SELECT c@2 AS x, b@1 AS y, a@0 AS z+ let base_projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("c", 2)),+ alias: "x".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 1)),+ alias: "y".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "z".to_string(),+ },+ ]);++ // Second projection: SELECT y@1 AS col2, x@0 AS col1+ let top_projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("y", 1)),+ alias: "col2".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("x", 0)),+ alias: "col1".to_string(),+ },+ ]);++ // Merge should produce: SELECT b@1 AS col2, c@2 AS col1+ let merged = base_projection.try_merge(&top_projection)?;+ assert_snapshot!(format!("{merged}"), @"Projection[b@1 AS col2, c@2 AS col1]");++ Ok(())+ }++ #[test]+ fn test_merge_with_expressions() -> Result<()> {+ // First projection: SELECT c@2 AS x, b@1 AS y, a@0 AS z+ let base_projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("c", 2)),+ alias: "x".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 1)),+ alias: "y".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "z".to_string(),+ },+ ]);++ // Second projection: SELECT y@1 + z@2 AS c2, x@0 + 1 AS c1+ let top_projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(BinaryExpr::new(+ Arc::new(Column::new("y", 1)),+ Operator::Plus,+ Arc::new(Column::new("z", 2)),+ )),+ alias: "c2".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(BinaryExpr::new(+ Arc::new(Column::new("x", 0)),+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),+ )),+ alias: "c1".to_string(),+ },+ ]);++ // Merge should produce: SELECT b@1 + a@0 AS c2, c@2 + 1 AS c1+ let merged = base_projection.try_merge(&top_projection)?;+ assert_snapshot!(format!("{merged}"), @"Projection[b@1 + a@0 AS c2, c@2 + 1 AS c1]");++ Ok(())+ }++ #[test]+ fn try_merge_error() {+ // Create a base projection+ let base = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "x".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 1)),+ alias: "y".to_string(),+ },+ ]);++ // Create a top projection that references a non-existent column index+ let top = Projection::new(vec![ProjectionExpr {+ expr: Arc::new(Column::new("z", 5)), // Invalid index+ alias: "result".to_string(),+ }]);++ // Attempt to merge and expect an error+ let err_msg = base.try_merge(&top).unwrap_err().to_string();+ assert!(+ err_msg.contains("Internal error: Column index 5 out of bounds for projected expressions of length 2"),+ "Unexpected error message: {err_msg}",+ );+ }++ #[test]+ fn test_project_schema_simple_columns() -> Result<()> {+ // Input schema: [col0: Int64, col1: Utf8, col2: Float32]+ let input_schema = get_schema();++ // Projection: SELECT col2 AS c, col0 AS a+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("col2", 2)),+ alias: "c".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "a".to_string(),+ },+ ]);++ let output_schema = projection.project_schema(&input_schema)?;++ // Should have 2 fields+ assert_eq!(output_schema.fields().len(), 2);++ // First field should be "c" with Float32 type+ assert_eq!(output_schema.field(0).name(), "c");+ assert_eq!(output_schema.field(0).data_type(), &DataType::Float32);++ // Second field should be "a" with Int64 type+ assert_eq!(output_schema.field(1).name(), "a");+ assert_eq!(output_schema.field(1).data_type(), &DataType::Int64);++ Ok(())+ }++ #[test]+ fn test_project_schema_with_expressions() -> Result<()> {+ // Input schema: [col0: Int64, col1: Utf8, col2: Float32]+ let input_schema = get_schema();++ // Projection: SELECT col0 + 1 AS incremented+ let projection = Projection::new(vec![ProjectionExpr {+ expr: Arc::new(BinaryExpr::new(+ Arc::new(Column::new("col0", 0)),+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Int64(Some(1)))),+ )),+ alias: "incremented".to_string(),+ }]);++ let output_schema = projection.project_schema(&input_schema)?;++ // Should have 1 field+ assert_eq!(output_schema.fields().len(), 1);++ // Field should be "incremented" with Int64 type+ assert_eq!(output_schema.field(0).name(), "incremented");+ assert_eq!(output_schema.field(0).data_type(), &DataType::Int64);++ Ok(())+ }++ #[test]+ fn test_project_schema_preserves_metadata() -> Result<()> {+ // Create schema with metadata+ let mut metadata = HashMap::new();+ metadata.insert("key".to_string(), "value".to_string());+ let field_with_metadata =+ Field::new("col0", DataType::Int64, false).with_metadata(metadata.clone());+ let input_schema = Schema::new(vec![+ field_with_metadata,+ Field::new("col1", DataType::Utf8, false),+ ]);++ // Projection: SELECT col0 AS renamed+ let projection = Projection::new(vec![ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "renamed".to_string(),+ }]);++ let output_schema = projection.project_schema(&input_schema)?;++ // Should have 1 field+ assert_eq!(output_schema.fields().len(), 1);++ // Field should be "renamed" with metadata preserved+ assert_eq!(output_schema.field(0).name(), "renamed");+ assert_eq!(output_schema.field(0).metadata(), &metadata);++ Ok(())+ }++ #[test]+ fn test_project_schema_empty() -> Result<()> {+ let input_schema = get_schema();+ let projection = Projection::new(vec![]);++ let output_schema = projection.project_schema(&input_schema)?;++ assert_eq!(output_schema.fields().len(), 0);++ Ok(())+ }++ #[test]+ fn test_project_statistics_columns_only() -> Result<()> {+ let input_stats = get_stats();+ let input_schema = get_schema();++ // Projection: SELECT col1 AS text, col0 AS num+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("col1", 1)),+ alias: "text".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "num".to_string(),+ },+ ]);++ let output_stats = projection.project_statistics(input_stats, &input_schema)?;++ // Row count should be preserved+ assert_eq!(output_stats.num_rows, Precision::Exact(5));++ // Should have 2 column statistics (reordered from input)+ assert_eq!(output_stats.column_statistics.len(), 2);++ // First column (col1 from input)+ assert_eq!(+ output_stats.column_statistics[0].distinct_count,+ Precision::Exact(1)+ );+ assert_eq!(+ output_stats.column_statistics[0].max_value,+ Precision::Exact(ScalarValue::from("x"))+ );++ // Second column (col0 from input)+ assert_eq!(+ output_stats.column_statistics[1].distinct_count,+ Precision::Exact(5)+ );+ assert_eq!(+ output_stats.column_statistics[1].max_value,+ Precision::Exact(ScalarValue::Int64(Some(21)))+ );++ Ok(())+ }++ #[test]+ fn test_project_statistics_with_expressions() -> Result<()> {+ let input_stats = get_stats();+ let input_schema = get_schema();++ // Projection with expression: SELECT col0 + 1 AS incremented, col1 AS text+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(BinaryExpr::new(+ Arc::new(Column::new("col0", 0)),+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Int64(Some(1)))),+ )),+ alias: "incremented".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col1", 1)),+ alias: "text".to_string(),+ },+ ]);++ let output_stats = projection.project_statistics(input_stats, &input_schema)?;++ // Row count should be preserved+ assert_eq!(output_stats.num_rows, Precision::Exact(5));++ // Should have 2 column statistics+ assert_eq!(output_stats.column_statistics.len(), 2);++ // First column (expression) should have unknown statistics+ assert_eq!(+ output_stats.column_statistics[0].distinct_count,+ Precision::Absent+ );+ assert_eq!(+ output_stats.column_statistics[0].max_value,+ Precision::Absent+ );++ // Second column (col1) should preserve statistics+ assert_eq!(+ output_stats.column_statistics[1].distinct_count,+ Precision::Exact(1)+ );++ Ok(())+ }++ #[test]+ fn test_project_statistics_primitive_width_only() -> Result<()> {+ let input_stats = get_stats();+ let input_schema = get_schema();++ // Projection with only primitive width columns: SELECT col2 AS f, col0 AS i+ let projection = Projection::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("col2", 2)),+ alias: "f".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "i".to_string(),+ },+ ]);++ let output_stats = projection.project_statistics(input_stats, &input_schema)?;++ // Row count should be preserved+ assert_eq!(output_stats.num_rows, Precision::Exact(5));++ // Total byte size should be recalculated for primitive types+ // Float32 (4 bytes) + Int64 (8 bytes) = 12 bytes per row, 5 rows = 60 bytes+ assert_eq!(output_stats.total_byte_size, Precision::Exact(60));++ // Should have 2 column statistics+ assert_eq!(output_stats.column_statistics.len(), 2);++ Ok(())+ }++ #[test]+ fn test_project_statistics_empty() -> Result<()> {+ let input_stats = get_stats();+ let input_schema = get_schema();++ let projection = Projection::new(vec![]);++ let output_stats = projection.project_statistics(input_stats, &input_schema)?;++ // Row count should be preserved+ assert_eq!(output_stats.num_rows, Precision::Exact(5));++ // Should have no column statistics+ assert_eq!(output_stats.column_statistics.len(), 0);++ // Total byte size should be 0 for empty projection+ assert_eq!(output_stats.total_byte_size, Precision::Exact(0));++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 6eea70e117..4dc88bc566 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -32,17 +32,16 @@ use crate::filter_pushdown::{FilterPushdownPropagation,};use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn, JoinOnRef};-use crate::{ColumnStatistics, DisplayFormatType, ExecutionPlan, PhysicalExpr};+use crate::{DisplayFormatType, ExecutionPlan, PhysicalExpr};use std::any::Any;use std::collections::HashMap;use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};-use arrow::datatypes::{Field, Schema, SchemaRef};+use arrow::datatypes::SchemaRef;use arrow::record_batch::{RecordBatch, RecordBatchOptions};use datafusion_common::config::ConfigOptions;-use datafusion_common::stats::Precision;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,};@@ -52,6 +51,9 @@ use datafusion_physical_expr::equivalence::ProjectionMapping;use datafusion_physical_expr::utils::collect_columns;use datafusion_physical_expr_common::physical_expr::{fmt_sql, PhysicalExprRef};use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};+// Re-exported from datafusion-physical-expr for backwards compatibility+// We recommend updating your imports to use datafusion-physical-expr directly+pub use datafusion_physical_expr::projection::{update_expr, Projection, ProjectionExpr};use futures::stream::{Stream, StreamExt};use log::trace;@@ -63,7 +65,7 @@ use log::trace;#[derive(Debug, Clone)]pub struct ProjectionExec {/// The projection expressions stored as tuples of (expression, output column name)- pub(crate) expr: Vec<ProjectionExpr>,+ projection: Projection,/// The schema once the projection has been applied to the inputschema: SchemaRef,/// The input plan@@ -127,42 +129,17 @@ impl ProjectionExec {{let input_schema = input.schema();// convert argument to Vec<ProjectionExpr>- let expr = expr.into_iter().map(Into::into).collect::<Vec<_>>();+ let expr_vec = expr.into_iter().map(Into::into).collect::<Vec<_>>();+ let projection = Projection::new(expr_vec);- let fields: Result<Vec<Field>> = expr- .iter()- .map(|proj_expr| {- let metadata = proj_expr- .expr- .return_field(&input_schema)?- .metadata()- .clone();-- let field = Field::new(- &proj_expr.alias,- proj_expr.expr.data_type(&input_schema)?,- proj_expr.expr.nullable(&input_schema)?,- )- .with_metadata(metadata);-- Ok(field)- })- .collect();-- let schema = Arc::new(Schema::new_with_metadata(- fields?,- input_schema.metadata().clone(),- ));+ let schema = Arc::new(projection.project_schema(&input_schema)?);// Construct a map from the input expressions to the output expression of the Projection- let projection_mapping = ProjectionMapping::try_new(- expr.iter().map(|p| (Arc::clone(&p.expr), p.alias.clone())),- &input_schema,- )?;+ let projection_mapping = projection.projection_mapping(&input_schema)?;let cache =Self::compute_properties(&input, &projection_mapping, Arc::clone(&schema))?;Ok(Self {- expr,+ projection,schema,input,metrics: ExecutionPlanMetricsSet::new(),@@ -172,7 +149,7 @@ impl ProjectionExec {/// The projection expressions stored as tuples of (expression, output column name)pub fn expr(&self) -> &[ProjectionExpr] {- &self.expr+ self.projection.as_ref()}/// The input plan@@ -203,35 +180,6 @@ impl ProjectionExec {}}-/// A projection expression that is created by [`ProjectionExec`]-///-/// The expression is evaluated and the result is stored in a column-/// with the name specified by `alias`.-///-/// For example, the SQL expression `a + b AS sum_ab` would be represented-/// as a `ProjectionExpr` where `expr` is the expression `a + b`-/// and `alias` is the string `sum_ab`.-#[derive(Debug, Clone)]-pub struct ProjectionExpr {- /// The expression that will be evaluated.- pub expr: Arc<dyn PhysicalExpr>,- /// The name of the output column for use an output schema.- pub alias: String,-}--impl ProjectionExpr {- /// Create a new projection expression- pub fn new(expr: Arc<dyn PhysicalExpr>, alias: String) -> Self {- Self { expr, alias }- }-}--impl From<(Arc<dyn PhysicalExpr>, String)> for ProjectionExpr {- fn from(value: (Arc<dyn PhysicalExpr>, String)) -> Self {- Self::new(value.0, value.1)- }-}-impl DisplayAs for ProjectionExec {fn fmt_as(&self,@@ -241,7 +189,8 @@ impl DisplayAs for ProjectionExec {match t {DisplayFormatType::Default | DisplayFormatType::Verbose => {let expr: Vec<String> = self- .expr+ .projection+ .as_ref().iter().map(|proj_expr| {let e = proj_expr.expr.to_string();@@ -291,7 +240,7 @@ impl ExecutionPlan for ProjectionExec {}fn benefits_from_input_partitioning(&self) -> Vec<bool> {- let all_simple_exprs = self.expr.iter().all(|proj_expr| {+ let all_simple_exprs = self.projection.iter().all(|proj_expr| {proj_expr.expr.as_any().is::<Column>()|| proj_expr.expr.as_any().is::<Literal>()});@@ -308,7 +257,7 @@ impl ExecutionPlan for ProjectionExec {self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- ProjectionExec::try_new(self.expr.clone(), children.swap_remove(0))+ ProjectionExec::try_new(self.projection.clone(), children.swap_remove(0)).map(|p| Arc::new(p) as _)}@@ -318,12 +267,12 @@ impl ExecutionPlan for ProjectionExec {context: Arc<TaskContext>,) -> Result<SendableRecordBatchStream> {trace!("Start ProjectionExec::execute for partition {} of context session_id {} and task_id {:?}", partition, context.session_id(), context.task_id());- Ok(Box::pin(ProjectionStream {- schema: Arc::clone(&self.schema),- expr: self.expr.iter().map(|x| Arc::clone(&x.expr)).collect(),- input: self.input.execute(partition, context)?,- baseline_metrics: BaselineMetrics::new(&self.metrics, partition),- }))+ Ok(Box::pin(ProjectionStream::new(+ Arc::clone(&self.schema),+ self.projection.expr_iter().collect(),+ self.input.execute(partition, context)?,+ BaselineMetrics::new(&self.metrics, partition),+ )))}fn metrics(&self) -> Option<MetricsSet> {@@ -336,13 +285,8 @@ impl ExecutionPlan for ProjectionExec {fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {let input_stats = self.input.partition_statistics(partition)?;- stats_projection(- input_stats,- self.expr- .iter()- .map(|proj_expr| Arc::clone(&proj_expr.expr)),- Arc::clone(&self.input.schema()),- )+ self.projection+ .project_statistics(input_stats, &self.input.schema())}fn supports_limit_pushdown(&self) -> bool {@@ -388,40 +332,22 @@ impl ExecutionPlan for ProjectionExec {}}-fn stats_projection(- mut stats: Statistics,- exprs: impl Iterator<Item = Arc<dyn PhysicalExpr>>,- schema: SchemaRef,-) -> Result<Statistics> {- let mut primitive_row_size = 0;- let mut primitive_row_size_possible = true;- let mut column_statistics = vec![];- for expr in exprs {- let col_stats = if let Some(col) = expr.as_any().downcast_ref::<Column>() {- stats.column_statistics[col.index()].clone()- } else {- // TODO stats: estimate more statistics from expressions- // (expressions should compute their statistics themselves)- ColumnStatistics::new_unknown()- };- column_statistics.push(col_stats);- let data_type = expr.data_type(&schema)?;- if let Some(value) = data_type.primitive_width() {- primitive_row_size += value;- continue;+impl ProjectionStream {+ /// Create a new projection stream+ fn new(+ schema: SchemaRef,+ expr: Vec<Arc<dyn PhysicalExpr>>,+ input: SendableRecordBatchStream,+ baseline_metrics: BaselineMetrics,+ ) -> Self {+ Self {+ schema,+ expr,+ input,+ baseline_metrics,}- primitive_row_size_possible = false;}- if primitive_row_size_possible {- stats.total_byte_size =- Precision::Exact(primitive_row_size).multiply(&stats.num_rows);- }- stats.column_statistics = column_statistics;- Ok(stats)-}--impl ProjectionStream {fn batch_project(&self, batch: &RecordBatch) -> Result<RecordBatch> {// Records time on droplet _timer = self.baseline_metrics.elapsed_compute().timer();@@ -703,86 +629,6 @@ pub fn all_columns(exprs: &[ProjectionExpr]) -> bool {.all(|proj_expr| proj_expr.expr.as_any().is::<Column>())}-/// The function operates in two modes:-///-/// 1) When `sync_with_child` is `true`:-///-/// The function updates the indices of `expr` if the expression resides-/// in the input plan. For instance, given the expressions `a@1 + b@2`-/// and `c@0` with the input schema `c@2, a@0, b@1`, the expressions are-/// updated to `a@0 + b@1` and `c@2`.-///-/// 2) When `sync_with_child` is `false`:-///-/// The function determines how the expression would be updated if a projection-/// was placed before the plan associated with the expression. If the expression-/// cannot be rewritten after the projection, it returns `None`. For example,-/// given the expressions `c@0`, `a@1` and `b@2`, and the [`ProjectionExec`] with-/// an output schema of `a, c_new`, then `c@0` becomes `c_new@1`, `a@1` becomes-/// `a@0`, but `b@2` results in `None` since the projection does not include `b`.-pub fn update_expr(- expr: &Arc<dyn PhysicalExpr>,- projected_exprs: &[ProjectionExpr],- sync_with_child: bool,-) -> Result<Option<Arc<dyn PhysicalExpr>>> {- #[derive(Debug, PartialEq)]- enum RewriteState {- /// The expression is unchanged.- Unchanged,- /// Some part of the expression has been rewritten- RewrittenValid,- /// Some part of the expression has been rewritten, but some column- /// references could not be.- RewrittenInvalid,- }-- let mut state = RewriteState::Unchanged;-- let new_expr = Arc::clone(expr)- .transform_up(|expr| {- if state == RewriteState::RewrittenInvalid {- return Ok(Transformed::no(expr));- }-- let Some(column) = expr.as_any().downcast_ref::<Column>() else {- return Ok(Transformed::no(expr));- };- if sync_with_child {- state = RewriteState::RewrittenValid;- // Update the index of `column`:- Ok(Transformed::yes(Arc::clone(- &projected_exprs[column.index()].expr,- )))- } else {- // default to invalid, in case we can't find the relevant column- state = RewriteState::RewrittenInvalid;- // Determine how to update `column` to accommodate `projected_exprs`- projected_exprs- .iter()- .enumerate()- .find_map(|(index, proj_expr)| {- proj_expr.expr.as_any().downcast_ref::<Column>().and_then(- |projected_column| {- (column.name().eq(projected_column.name())- && column.index() == projected_column.index())- .then(|| {- state = RewriteState::RewrittenValid;- Arc::new(Column::new(&proj_expr.alias, index)) as _- })- },- )- })- .map_or_else(- || Ok(Transformed::no(expr)),- |c| Ok(Transformed::yes(c)),- )- }- })- .data();-- new_expr.map(|e| (state == RewriteState::RewrittenValid).then_some(e))-}-/// Updates the given lexicographic ordering according to given projected/// expressions using the [`update_expr`] function.pub fn update_ordering(@@ -1268,116 +1114,6 @@ mod tests {.unwrap();}- fn get_stats() -> Statistics {- Statistics {- num_rows: Precision::Exact(5),- total_byte_size: Precision::Exact(23),- column_statistics: vec![- ColumnStatistics {- distinct_count: Precision::Exact(5),- max_value: Precision::Exact(ScalarValue::Int64(Some(21))),- min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),- sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),- null_count: Precision::Exact(0),- },- ColumnStatistics {- distinct_count: Precision::Exact(1),- max_value: Precision::Exact(ScalarValue::from("x")),- min_value: Precision::Exact(ScalarValue::from("a")),- sum_value: Precision::Absent,- null_count: Precision::Exact(3),- },- ColumnStatistics {- distinct_count: Precision::Absent,- max_value: Precision::Exact(ScalarValue::Float32(Some(1.1))),- min_value: Precision::Exact(ScalarValue::Float32(Some(0.1))),- sum_value: Precision::Exact(ScalarValue::Float32(Some(5.5))),- null_count: Precision::Absent,- },- ],- }- }-- fn get_schema() -> Schema {- let field_0 = Field::new("col0", DataType::Int64, false);- let field_1 = Field::new("col1", DataType::Utf8, false);- let field_2 = Field::new("col2", DataType::Float32, false);- Schema::new(vec![field_0, field_1, field_2])- }- #[tokio::test]- async fn test_stats_projection_columns_only() {- let source = get_stats();- let schema = get_schema();-- let exprs: Vec<Arc<dyn PhysicalExpr>> = vec![- Arc::new(Column::new("col1", 1)),- Arc::new(Column::new("col0", 0)),- ];-- let result =- stats_projection(source, exprs.into_iter(), Arc::new(schema)).unwrap();-- let expected = Statistics {- num_rows: Precision::Exact(5),- total_byte_size: Precision::Exact(23),- column_statistics: vec![- ColumnStatistics {- distinct_count: Precision::Exact(1),- max_value: Precision::Exact(ScalarValue::from("x")),- min_value: Precision::Exact(ScalarValue::from("a")),- sum_value: Precision::Absent,- null_count: Precision::Exact(3),- },- ColumnStatistics {- distinct_count: Precision::Exact(5),- max_value: Precision::Exact(ScalarValue::Int64(Some(21))),- min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),- sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),- null_count: Precision::Exact(0),- },- ],- };-- assert_eq!(result, expected);- }-- #[tokio::test]- async fn test_stats_projection_column_with_primitive_width_only() {- let source = get_stats();- let schema = get_schema();-- let exprs: Vec<Arc<dyn PhysicalExpr>> = vec![- Arc::new(Column::new("col2", 2)),- Arc::new(Column::new("col0", 0)),- ];-- let result =- stats_projection(source, exprs.into_iter(), Arc::new(schema)).unwrap();-- let expected = Statistics {- num_rows: Precision::Exact(5),- total_byte_size: Precision::Exact(60),- column_statistics: vec![- ColumnStatistics {- distinct_count: Precision::Absent,- max_value: Precision::Exact(ScalarValue::Float32(Some(1.1))),- min_value: Precision::Exact(ScalarValue::Float32(Some(0.1))),- sum_value: Precision::Exact(ScalarValue::Float32(Some(5.5))),- null_count: Precision::Absent,- },- ColumnStatistics {- distinct_count: Precision::Exact(5),- max_value: Precision::Exact(ScalarValue::Int64(Some(21))),- min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),- sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),- null_count: Precision::Exact(0),- },- ],- };-- assert_eq!(result, expected);- }-#[test]fn test_projection_statistics_uses_input_schema() {let input_schema = Schema::new(vec![