Apache DataFusion SQL Query Engine
perf: reuse projection schema in `OptimizeProjections` instead of recomputing it (#24281)
Qi Zhud4c2a329a8fc
diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex 29093d3352..7b9c4586bc 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -544,6 +544,9 @@ impl LogicalPlan {/// # Notes/// * Similar to [`TreeNode::map_children`] but for this node's expressions./// * Visits only the top level expressions (Does not recurse into each expression)+ /// * A `Projection` whose expressions were rewritten gets its schema derived+ /// again from the new expressions, so the new expressions must resolve+ /// against the current input.pub fn map_expressions<F: FnMut(Expr) -> Result<Transformed<Expr>>>(self,mut f: F,@@ -553,13 +556,25 @@ impl LogicalPlan {expr,input,schema,- }) => expr.map_elements(f)?.update_data(|expr| {- LogicalPlan::Projection(Projection {- expr,- input,- schema,- })- }),+ }) => {+ let expr = expr.map_elements(f)?;+ if expr.transformed {+ // A projection's output fields are derived from its+ // expressions, so rewritten expressions need a freshly+ // derived schema.+ expr.map_data(|expr| {+ Projection::try_new(expr, input).map(LogicalPlan::Projection)+ })?+ } else {+ expr.update_data(|expr| {+ LogicalPlan::Projection(Projection {+ expr,+ input,+ schema,+ })+ })+ }+ }LogicalPlan::Values(Values { schema, values }) => values.map_elements(f)?.update_data(|values| LogicalPlan::Values(Values { schema, values })),diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rsindex 89f0cd30dc..08df521f6e 100644--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs@@ -76,11 +76,22 @@ impl CommonSubexprEliminate {..} = projection;let input = Arc::unwrap_or_clone(input);- self.try_unary_plan(expr, input, config)?- .map_data(|(new_expr, new_input)| {- Projection::try_new_with_schema(new_expr, Arc::new(new_input), schema)- .map(LogicalPlan::Projection)- })+ let rewritten = self.try_unary_plan(expr, input, config)?;+ let transformed = rewritten.transformed;+ rewritten.map_data(|(new_expr, new_input)| {+ let new_input = Arc::new(new_input);+ // The rewritten expressions keep their names, but their fields can+ // differ: a CASE whose `WHEN x IS NOT NULL THEN x` guard is split+ // into two common subexpressions can no longer be proven non-null.+ // Derive the schema from the new expressions so it matches what+ // the physical planner will compute from them.+ if transformed {+ Projection::try_new(new_expr, new_input)+ } else {+ Projection::try_new_with_schema(new_expr, new_input, schema)+ }+ .map(LogicalPlan::Projection)+ })}fn try_optimize_sort(@@ -889,11 +900,11 @@ mod test {use std::iter;use arrow::datatypes::{DataType, Field, Schema};- use datafusion_expr::logical_plan::{JoinType, table_scan};+ use datafusion_expr::logical_plan::{JoinType, projection_schema, table_scan};use datafusion_expr::{AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, ExprFunctionExt,ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,- Volatility, WindowFunctionDefinition, grouping_set, is_null, not,+ Volatility, WindowFunctionDefinition, grouping_set, is_null, not, when,};use datafusion_expr::{lit, logical_plan::builder::LogicalPlanBuilder};@@ -2157,4 +2168,31 @@ mod test {")}++ /// A projection rewritten by CSE carries the schema its new expressions+ /// derive, not the one derived before the rewrite.+ #[test]+ fn projection_schema_follows_rewritten_exprs() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ let table_scan = table_scan(Some("t"), &schema, None)?.build()?;+ let a = || col("a") + lit(1);+ let expr = when(a().is_not_null(), a())+ .when(a().is_not_null(), a())+ .otherwise(lit(0))?;+ let plan = LogicalPlanBuilder::from(table_scan)+ .project(vec![expr])?+ .build()?;+ // Before the rewrite the guard proves the CASE non-null.+ assert!(!plan.schema().field(0).is_nullable());++ let optimized = CommonSubexprEliminate::new()+ .rewrite(plan, &OptimizerContext::new())?+ .data;+ let LogicalPlan::Projection(projection) = &optimized else {+ panic!("expected a projection, got {optimized}");+ };+ let derived = projection_schema(&projection.input, &projection.expr)?;+ assert_eq!(projection.schema.as_ref(), derived.as_ref());+ Ok(())+ }}diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex 0b402795d3..f2191fc6df 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -23,7 +23,7 @@ use datafusion_expr::logical_plan::MaterializedCtePlan;use std::sync::Arc;use datafusion_common::{- Column, DFSchema, HashMap, JoinType, Result, assert_eq_or_internal_err,+ Column, DFSchema, DFSchemaRef, HashMap, JoinType, Result, assert_eq_or_internal_err,get_required_group_by_exprs_indices, internal_datafusion_err, internal_err,};use datafusion_expr::expr::Alias;@@ -861,12 +861,19 @@ fn rewrite_projection_given_requirements(config: &dyn OptimizerConfig,indices: &RequiredIndices,) -> Result<Transformed<LogicalPlan>> {- let Projection { expr, input, .. } = proj;+ let Projection {+ expr,+ input,+ schema,+ ..+ } = proj;let exprs_used = indices.get_at_indices(&expr);let required_indices =RequiredIndices::new().with_exprs(input.schema(), exprs_used.iter());+ let old_input_schema = Arc::clone(input.schema());+ let read_indices = required_indices.indices().to_vec();// rewrite the children projection, and if they are changed rewrite the// projection down@@ -874,14 +881,111 @@ fn rewrite_projection_given_requirements(.transform_data(|input| {if is_projection_unnecessary(&input, &exprs_used)? {Ok(Transformed::yes(input))- } else {+ } else if exprs_used.iter().any(contains_subquery)+ || !read_fields_unchanged(+ &old_input_schema,+ input.schema(),+ &read_indices,+ )+ {+ // A subquery is optimized on its own, and rewriting the input+ // can change the fields this projection reads, so the stored+ // schema may be stale. Recompute it rather than reuse it.Projection::try_new(exprs_used, Arc::new(input)).map(LogicalPlan::Projection).map(Transformed::yes)+ } else {+ // The retained expressions are a subset of the original ones and+ // read unchanged input fields, so their output fields are+ // unchanged too. Select them from the existing schema instead of+ // recomputing each one via `Expr::to_field` (see+ // `projection_schema`), which costs O(exprs * schema_width).+ // `merge_consecutive_projections` reuses the schema the same way.+ let projected_schema =+ project_schema_by_indices(&schema, indices.indices())?;+ Projection::try_new_with_schema(+ exprs_used,+ Arc::new(input),+ projected_schema,+ )+ .map(LogicalPlan::Projection)+ .map(Transformed::yes)}})}+/// Returns true if the input fields at `read_indices` of `old` are unchanged in+/// `new`, the input after it was rewritten for those indices.+///+/// The rewritten input either keeps every field of `old` or only the fields at+/// `read_indices`, in order. Any other shape is treated as changed.+fn read_fields_unchanged(old: &DFSchema, new: &DFSchema, read_indices: &[usize]) -> bool {+ if new.fields().len() == old.fields().len() {+ read_indices.iter().all(|&i| old.field(i) == new.field(i))+ } else if new.fields().len() == read_indices.len() {+ read_indices+ .iter()+ .enumerate()+ .all(|(j, &i)| old.field(i) == new.field(j))+ } else {+ false+ }+}++/// Returns true if `expr` contains a subquery expression.+fn contains_subquery(expr: &Expr) -> bool {+ expr.exists(|e| {+ Ok(matches!(+ e,+ Expr::Exists(_)+ | Expr::InSubquery(_)+ | Expr::SetComparison(_)+ | Expr::ScalarSubquery(_)+ ))+ })+ .expect("exists closure is infallible")+}++/// Builds the output schema of a projection that keeps only the fields at+/// `indices` (a sorted, deduplicated subset produced by [`RequiredIndices`]) of+/// `schema`, reusing the already-computed fields instead of recomputing each+/// field's type/nullability from the expressions.+///+/// Pruning unreferenced sibling columns cannot change the retained fields, so+/// as long as `schema` is current for the projection's expressions and input,+/// the sliced fields are identical to the ones [`projection_schema`] would+/// recompute, at O(k) instead of O(exprs * schema_width). Functional+/// dependencies can be weaker than recomputed ones, never stronger.+///+/// [`projection_schema`]: datafusion_expr::logical_plan::projection_schema+fn project_schema_by_indices(+ schema: &DFSchemaRef,+ indices: &[usize],+) -> Result<DFSchemaRef> {+ // Nothing pruned: the output schema is unchanged, reuse it as-is.+ if indices.len() == schema.fields().len() {+ return Ok(Arc::clone(schema));+ }++ let qualified_fields = indices+ .iter()+ .map(|&i| {+ let (qualifier, field) = schema.qualified_field(i);+ (qualifier.cloned(), Arc::clone(field))+ })+ .collect::<Vec<_>>();++ let func_deps = schema+ .functional_dependencies()+ .project_functional_dependencies(indices, indices.len());++ let projected =+ DFSchema::new_with_metadata(qualified_fields, schema.metadata().clone())?+ .with_functional_dependencies(func_deps)?;++ Ok(Arc::new(projected))+}+/// Projection is unnecessary, when/// - input schema of the projection, output schema of the projection are same, and/// - all projection expressions are either Column or Literal@@ -915,15 +1019,16 @@ mod tests {use std::vec;use crate::optimize_projections::OptimizeProjections;- use crate::optimizer::Optimizer;+ use crate::optimizer::{ApplyOrder, Optimizer};use crate::test::{assert_fields_eq, scan_empty, test_table_scan, test_table_scan_fields,test_table_scan_with_name,};- use crate::{OptimizerContext, OptimizerRule};+ use crate::{OptimizerConfig, OptimizerContext, OptimizerRule};use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};use datafusion_common::{- Column, DFSchema, DFSchemaRef, JoinType, Result, TableReference,+ Column, DFSchema, DFSchemaRef, JoinType, Result, ScalarValue, TableReference,};use datafusion_expr::ExprFunctionExt;use datafusion_expr::{@@ -932,7 +1037,7 @@ mod tests {build_join_schema,builder::table_scan_with_filters,col,- expr::{self, Cast},+ expr::{self, Alias, Cast},lit,logical_plan::{builder::LogicalPlanBuilder, table_scan},not, try_cast, when,@@ -1193,6 +1298,107 @@ mod tests {}}+ /// `project_schema_by_indices` must produce exactly the schema that+ /// recomputing it from scratch (`projection_schema`) would, for every subset+ /// of a projection's expressions. This is the correctness invariant that lets+ /// `rewrite_projection_given_requirements` reuse the parent schema instead of+ /// re-deriving each field's type via `Expr::to_field`.+ #[test]+ fn project_schema_by_indices_matches_recompute() -> Result<()> {+ use super::project_schema_by_indices;+ use datafusion_expr::logical_plan::projection_schema;++ let input = Arc::new(test_table_scan()?); // columns: a, b, c (UInt32, NOT NULL)++ // A deliberately mixed expression list: plain column, computed binary+ // expr, alias, nullable literal, and a qualified column. Every input+ // column is NOT NULL, so the NULL literal is what makes nullability+ // actually vary across the schema.+ let exprs = vec![+ col("a"),+ binary_expr(col("b"), Operator::Plus, col("c")),+ col("c").alias("c_alias"),+ lit(ScalarValue::Int64(None)).alias("null_one"),+ Expr::Column(Column::new(Some(TableReference::bare("test")), "b")),+ ];++ let full_schema =+ Arc::clone(&Projection::try_new(exprs.clone(), Arc::clone(&input))?.schema);++ // Guard the premise above: if this ever stops holding, the subsets below+ // would no longer exercise nullability propagation at all.+ assert!(+ full_schema.field(3).is_nullable(),+ "the literal must be nullable for this test to cover nullability"+ );+ assert!(+ !full_schema.field(0).is_nullable(),+ "input columns are expected to be NOT NULL"+ );++ // A sample of the sorted, deduplicated subsets RequiredIndices can+ // produce, including the "nothing pruned" identity case.+ let subsets: Vec<Vec<usize>> = vec![+ vec![0, 1, 2, 3, 4], // identity (SELECT * fast path)+ vec![0],+ vec![1], // computed expr only+ vec![2], // alias only+ vec![3], // literal only+ vec![0, 2],+ vec![1, 3],+ vec![0, 1, 4],+ vec![2, 3, 4],+ ];++ for indices in subsets {+ let exprs_used: Vec<Expr> =+ indices.iter().map(|&i| exprs[i].clone()).collect();++ let reused = project_schema_by_indices(&full_schema, &indices)?;+ let recomputed = projection_schema(input.as_ref(), &exprs_used)?;++ // Output fields (name, data type, nullability, field metadata) must+ // match the from-scratch computation exactly.+ assert_eq!(+ reused.fields(),+ recomputed.fields(),+ "fields differ for indices {indices:?}"+ );++ // Column qualifiers (table references) must be preserved too.+ let reused_quals: Vec<_> = reused.iter().map(|(q, _)| q.cloned()).collect();+ let recomputed_quals: Vec<_> =+ recomputed.iter().map(|(q, _)| q.cloned()).collect();+ assert_eq!(+ reused_quals, recomputed_quals,+ "qualifiers differ for indices {indices:?}"+ );++ // `project_schema_by_indices` also carries over schema-level+ // metadata and projects functional dependencies through the kept+ // indices, so both must match the from-scratch computation as well.+ assert_eq!(+ reused.metadata(),+ recomputed.metadata(),+ "schema metadata differs for indices {indices:?}"+ );+ assert_eq!(+ reused.functional_dependencies(),+ recomputed.functional_dependencies(),+ "functional dependencies differ for indices {indices:?}"+ );+ }++ // The identity subset must reuse the very same Arc, not rebuild it.+ let identity = project_schema_by_indices(&full_schema, &[0, 1, 2, 3, 4])?;+ assert!(+ Arc::ptr_eq(&identity, &full_schema),+ "identity projection should reuse the existing schema Arc"+ );++ Ok(())+ }+#[test]fn merge_two_projection() -> Result<()> {let table_scan = test_table_scan()?;@@ -2536,6 +2742,193 @@ mod tests {)}+ /// Returns `schema` with the field called `name` marked nullable.+ fn with_nullable_field(schema: &DFSchema, name: &str) -> Result<DFSchemaRef> {+ let fields = schema+ .iter()+ .map(|(qualifier, field)| {+ let field = if field.name() == name {+ Arc::new(field.as_ref().clone().with_nullable(true))+ } else {+ Arc::clone(field)+ };+ (qualifier.cloned(), field)+ })+ .collect();+ Ok(Arc::new(DFSchema::new_with_metadata(+ fields,+ HashMap::new(),+ )?))+ }++ /// Asserts that `plan` is a projection whose schema is the one its+ /// expressions derive.+ fn assert_projection_schema_derived(plan: &LogicalPlan) -> Result<()> {+ use datafusion_expr::logical_plan::projection_schema;++ let LogicalPlan::Projection(projection) = plan else {+ panic!("expected a projection, got {plan}");+ };+ let derived = projection_schema(&projection.input, &projection.expr)?;+ assert_eq!(projection.schema.as_ref(), derived.as_ref());+ Ok(())+ }++ /// When rewriting the input changes a field the projection reads, the+ /// projection's schema is derived again instead of reused.+ #[test]+ fn projection_schema_follows_rewritten_input() -> Result<()> {+ use datafusion_expr::Aggregate;++ let table_scan = Arc::new(test_table_scan()?);+ let group_expr = vec![lit(1u32).alias("g")];+ let aggr_expr = vec![count(col("b")).alias("cnt")];+ let aggregate = Aggregate::try_new(+ Arc::clone(&table_scan),+ group_expr.clone(),+ aggr_expr.clone(),+ )?;+ // A non-null group column stored as nullable, as a rule that keeps a+ // node's schema while folding its expressions can leave it.+ let stale = with_nullable_field(&aggregate.schema, "g")?;+ let aggregate =+ Aggregate::try_new_with_schema(table_scan, group_expr, aggr_expr, stale)?;+ let plan = LogicalPlanBuilder::from(LogicalPlan::Aggregate(aggregate))+ .project(vec![col("g").alias("x")])?+ .build()?;+ assert!(plan.schema().field(0).is_nullable());++ // Pruning `cnt` rebuilds the aggregate, which derives `g` as non-null.+ let optimized = optimize(plan)?;+ assert!(!optimized.schema().field(0).is_nullable());+ assert_projection_schema_derived(&optimized)+ }++ /// A projection over a subquery derives its schema again instead of+ /// reusing one that may be stale.+ #[test]+ fn projection_schema_follows_subquery() -> Result<()> {+ let subquery = LogicalPlanBuilder::from(test_table_scan()?)+ .project(vec![col("a")])?+ .limit(0, Some(1))?+ .build()?;+ let input = Arc::new(test_table_scan()?);+ let exprs = vec![+ col("b"),+ datafusion_expr::scalar_subquery(Arc::new(subquery)).alias("s"),+ ];+ let derived = Projection::try_new(exprs.clone(), Arc::clone(&input))?;+ // Mark `b` nullable so a reused schema would differ from the derived one.+ let stale = with_nullable_field(&derived.schema, "b")?;+ let plan = LogicalPlan::Projection(Projection::try_new_with_schema(+ exprs, input, stale,+ )?);++ let optimized = optimize(plan)?;+ assert_projection_schema_derived(&optimized)+ }++ /// A downstream rule that rewrites `a AS x` into+ /// `CASE WHEN a > 0 THEN a END AS x` via `map_expressions`, so `x` becomes+ /// nullable. With `transform` false it rewrites nothing.+ #[derive(Debug)]+ struct RewriteToCase {+ transform: bool,+ }++ impl OptimizerRule for RewriteToCase {+ fn name(&self) -> &str {+ "rewrite_to_case"+ }++ fn apply_order(&self) -> Option<ApplyOrder> {+ Some(ApplyOrder::BottomUp)+ }++ fn rewrite(+ &self,+ plan: LogicalPlan,+ _config: &dyn OptimizerConfig,+ ) -> Result<Transformed<LogicalPlan>> {+ if !self.transform || !matches!(plan, LogicalPlan::Projection(_)) {+ return Ok(Transformed::no(plan));+ }+ plan.map_expressions(|e| match e {+ Expr::Alias(Alias {+ expr,+ relation,+ name,+ metadata,+ }) if name == "x" && matches!(*expr, Expr::Column(_)) => {+ let new = when(expr.as_ref().clone().gt(lit(0u32)), *expr).end()?;+ Ok(Transformed::yes(Expr::Alias(Alias {+ expr: Box::new(new),+ relation,+ name,+ metadata,+ })))+ }+ e => Ok(Transformed::no(e)),+ })+ }+ }++ /// `SELECT DISTINCT x FROM (SELECT a AS x, b + 1 AS y FROM test)`+ fn distinct_over_x_and_y() -> Result<LogicalPlan> {+ LogicalPlanBuilder::from(test_table_scan()?)+ .project(vec![col("a").alias("x"), (col("b") + lit(1u32)).alias("y")])?+ .distinct()?+ .project(vec![col("x")])?+ .build()+ }++ /// A projection whose expressions a downstream rule rewrote carries the+ /// schema the new expressions derive, even after `OptimizeProjections`+ /// prunes it and reuses its schema.+ #[test]+ fn projection_schema_follows_rule_rewrite() -> Result<()> {+ let plan = distinct_over_x_and_y()?;+ assert!(!plan.schema().field(0).is_nullable());++ let optimizer = Optimizer::with_rules(vec![+ Arc::new(RewriteToCase { transform: true }),+ Arc::new(OptimizeProjections::new()),+ ]);+ let optimized = optimizer.optimize(plan, &OptimizerContext::new(), observe)?;++ let mut projections = 0;+ optimized.apply(|plan| {+ if let LogicalPlan::Projection(_) = plan {+ assert_projection_schema_derived(plan)?;+ projections += 1;+ }+ Ok(TreeNodeRecursion::Continue)+ })?;+ assert!(projections > 0, "expected a projection in {optimized}");+ assert!(optimized.schema().field(0).is_nullable());+ Ok(())+ }++ /// A rule that rewrites nothing keeps the projection's schema as is.+ #[test]+ fn projection_schema_kept_without_rule_rewrite() -> Result<()> {+ let plan = LogicalPlanBuilder::from(test_table_scan()?)+ .project(vec![col("a").alias("x"), (col("b") + lit(1u32)).alias("y")])?+ .build()?;+ let schema = Arc::clone(plan.schema());++ let rewritten =+ RewriteToCase { transform: false }.rewrite(plan, &OptimizerContext::new())?;+ assert!(!rewritten.transformed);+ assert!(Arc::ptr_eq(&schema, rewritten.data.schema()));++ // Expressions visited but left unchanged also keep the schema.+ let rewritten = rewritten.data.map_expressions(|e| Ok(Transformed::no(e)))?;+ assert!(!rewritten.transformed);+ assert!(Arc::ptr_eq(&schema, rewritten.data.schema()));+ Ok(())+ }+fn observe(_plan: &LogicalPlan, _rule: &dyn OptimizerRule) {}fn optimize(plan: LogicalPlan) -> Result<LogicalPlan> {diff --git a/datafusion/sqllogictest/test_files/cse.slt b/datafusion/sqllogictest/test_files/cse.sltindex f8d67020be..ff526a694b 100644--- a/datafusion/sqllogictest/test_files/cse.slt+++ b/datafusion/sqllogictest/test_files/cse.slt@@ -355,3 +355,18 @@ FROM cse_agg_filterstatement okDROP TABLE cse_agg_filter++# A CASE guard split into common subexpressions loses its non-null proof, so+# the projection schema has to follow the rewritten expressions.+statement ok+CREATE TABLE cse_coalesce(col1 INT) AS VALUES (1), (NULL), (3)++query I rowsort+SELECT DISTINCT COALESCE(+col1, col1, 2, col1) FROM cse_coalesce GROUP BY col1+----+1+2+3++statement ok+DROP TABLE cse_coalescediff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 4665bfa52e..2b11283bd0 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -1435,6 +1435,29 @@ Code that reads `FileScanConfig::preserve_order` to decide whether files may bereordered should call `FileScanConfig::preserves_file_order()` instead, whichalso accounts for the output ordering.+### `LogicalPlan::map_expressions` re-derives a `Projection`'s schema++When `LogicalPlan::map_expressions` rewrites any expression of a `Projection`,+it now derives the output schema again from the new expressions instead of+keeping the old one. See [#24281].++**Who is affected:**++- Renaming rewrites, such as `unalias`, now change the returned plan's+ `schema()`.+- Rewrites whose new expressions reference columns of an input that is only+ installed afterwards (for example with `with_new_exprs`) now fail with a+ schema error on a `Projection`.++**Migration guide:**++For the second case, rewrite the `Projection`'s `expr` directly instead of+through `map_expressions`, and build the projection once the new input is in+place, with `Projection::try_new`, or with `Projection::try_new_with_schema` to+keep a schema you already have.++[#24281]: https://github.com/apache/datafusion/pull/24281+### `SanityCheckPlan` is deprecated and no longer in the default physical optimizer rulesThe physical planner now runs the `SanityCheckPlan` check itself on the final