Apache DataFusion SQL Query Engine
Consolidate `EliminateNestedUnion` and `EliminateOneUnion` optimizer rules' (#18678)
Andrew Lamb7fa63789f8aa
diff --git a/datafusion/optimizer/src/eliminate_one_union.rs b/datafusion/optimizer/src/eliminate_one_union.rsdeleted file mode 100644index 3e02781142..0000000000--- a/datafusion/optimizer/src/eliminate_one_union.rs+++ /dev/null@@ -1,121 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--//! [`EliminateOneUnion`] eliminates single element `Union`--use crate::{OptimizerConfig, OptimizerRule};-use datafusion_common::{tree_node::Transformed, Result};-use datafusion_expr::logical_plan::{LogicalPlan, Union};-use std::sync::Arc;--use crate::optimizer::ApplyOrder;--#[derive(Default, Debug)]-/// An optimization rule that eliminates union with one element.-pub struct EliminateOneUnion;--impl EliminateOneUnion {- #[allow(missing_docs)]- pub fn new() -> Self {- Self {}- }-}--impl OptimizerRule for EliminateOneUnion {- fn name(&self) -> &str {- "eliminate_one_union"- }-- fn supports_rewrite(&self) -> bool {- true- }-- fn rewrite(- &self,- plan: LogicalPlan,- _config: &dyn OptimizerConfig,- ) -> Result<Transformed<LogicalPlan>> {- match plan {- LogicalPlan::Union(Union { mut inputs, .. }) if inputs.len() == 1 => Ok(- Transformed::yes(Arc::unwrap_or_clone(inputs.pop().unwrap())),- ),- _ => Ok(Transformed::no(plan)),- }- }-- fn apply_order(&self) -> Option<ApplyOrder> {- Some(ApplyOrder::TopDown)- }-}--#[cfg(test)]-mod tests {- use super::*;- use crate::test::*;- use arrow::datatypes::{DataType, Field, Schema};- use datafusion_common::ToDFSchema;- use datafusion_expr::{- expr_rewriter::coerce_plan_expr_for_schema, logical_plan::table_scan,- };- use std::sync::Arc;-- fn schema() -> Schema {- Schema::new(vec![- Field::new("id", DataType::Int32, false),- Field::new("key", DataType::Utf8, false),- Field::new("value", DataType::Int32, false),- ])- }-- fn assert_optimized_plan_equal(plan: LogicalPlan, expected: &str) -> Result<()> {- assert_optimized_plan_with_rules(- vec![Arc::new(EliminateOneUnion::new())],- plan,- expected,- true,- )- }-- #[test]- fn eliminate_nothing() -> Result<()> {- let plan_builder = table_scan(Some("table"), &schema(), None)?;-- let plan = plan_builder.clone().union(plan_builder.build()?)?.build()?;-- let expected = "\- Union\- \n TableScan: table\- \n TableScan: table";- assert_optimized_plan_equal(plan, expected)- }-- #[test]- fn eliminate_one_union() -> Result<()> {- let table_plan = coerce_plan_expr_for_schema(- table_scan(Some("table"), &schema(), None)?.build()?,- &schema().to_dfschema()?,- )?;- let schema = Arc::clone(table_plan.schema());- let single_union_plan = LogicalPlan::Union(Union {- inputs: vec![Arc::new(table_plan)],- schema,- });-- let expected = "TableScan: table";- assert_optimized_plan_equal(single_union_plan, expected)- }-}diff --git a/datafusion/optimizer/src/lib.rs b/datafusion/optimizer/src/lib.rsindex 07ef2a46cb..7632ff858d 100644--- a/datafusion/optimizer/src/lib.rs+++ b/datafusion/optimizer/src/lib.rs@@ -51,12 +51,17 @@ pub mod eliminate_filter;pub mod eliminate_group_by_constant;pub mod eliminate_join;pub mod eliminate_limit;-pub mod eliminate_nested_union;-pub mod eliminate_one_union;+#[deprecated(since = "52.0.0", note = "Please use OptimizeUnions instead")]+pub mod eliminate_nested_union {+ use crate::optimize_unions::OptimizeUnions;+ #[deprecated(since = "52.0.0", note = "Please use OptimizeUnions instead")]+ pub type EliminateNestedUnion = OptimizeUnions;+}pub mod eliminate_outer_join;pub mod extract_equijoin_predicate;pub mod filter_null_join_keys;pub mod optimize_projections;+pub mod optimize_unions;pub mod optimizer;pub mod propagate_empty_relation;pub mod push_down_filter;diff --git a/datafusion/optimizer/src/eliminate_nested_union.rs b/datafusion/optimizer/src/optimize_unions.rssimilarity index 90%rename from datafusion/optimizer/src/eliminate_nested_union.rsrename to datafusion/optimizer/src/optimize_unions.rsindex f8f93727cd..cfabd512b4 100644--- a/datafusion/optimizer/src/eliminate_nested_union.rs+++ b/datafusion/optimizer/src/optimize_unions.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-//! [`EliminateNestedUnion`]: flattens nested `Union` to a single `Union`+//! [`OptimizeUnions`]: removes `Union` nodes in the logical plan.use crate::optimizer::ApplyOrder;use crate::{OptimizerConfig, OptimizerRule};use datafusion_common::tree_node::Transformed;@@ -26,19 +26,21 @@ use itertools::Itertools;use std::sync::Arc;#[derive(Default, Debug)]-/// An optimization rule that replaces nested unions with a single union.-pub struct EliminateNestedUnion;+/// An optimization rule that+/// 1. replaces nested unions with a single union.+/// 2. removes unions with a single input.+pub struct OptimizeUnions;-impl EliminateNestedUnion {+impl OptimizeUnions {#[allow(missing_docs)]pub fn new() -> Self {Self {}}}-impl OptimizerRule for EliminateNestedUnion {+impl OptimizerRule for OptimizeUnions {fn name(&self) -> &str {- "eliminate_nested_union"+ "optimize_unions"}fn apply_order(&self) -> Option<ApplyOrder> {@@ -55,6 +57,9 @@ impl OptimizerRule for EliminateNestedUnion {_config: &dyn OptimizerConfig,) -> Result<Transformed<LogicalPlan>> {match plan {+ LogicalPlan::Union(Union { mut inputs, .. }) if inputs.len() == 1 => Ok(+ Transformed::yes(Arc::unwrap_or_clone(inputs.pop().unwrap())),+ ),LogicalPlan::Union(Union { inputs, schema }) => {let inputs = inputs.into_iter()@@ -139,7 +144,7 @@ mod tests {let analyzed_plan = Analyzer::with_rules(vec![Arc::new(TypeCoercion::new())]).execute_and_check($plan, &options, |_, _| {})?;let optimizer_ctx = OptimizerContext::new().with_max_passes(1);- let rules: Vec<Arc<dyn crate::OptimizerRule + Send + Sync>> = vec![Arc::new(EliminateNestedUnion::new())];+ let rules: Vec<Arc<dyn crate::OptimizerRule + Send + Sync>> = vec![Arc::new(OptimizeUnions::new())];assert_optimized_plan_eq_snapshot!(optimizer_ctx,rules,@@ -420,4 +425,28 @@ mod tests {TableScan: table_1")}++ #[test]+ fn eliminate_one_union() -> Result<()> {+ let plan = table_scan(Some("table"), &schema(), None)?.build()?;+ let schema = Arc::clone(plan.schema());+ // note it is not possible to create a single input union via+ // LogicalPlanBuilder so create it manually here+ let plan = LogicalPlan::Union(Union {+ inputs: vec![Arc::new(plan)],+ schema,+ });++ // Note we can't use the same assert_optimized_plan_equal as creating a+ // single input union is not possible via LogicalPlanBuilder and other passes+ // throw errors / don't handle the schema correctly.+ assert_optimized_plan_eq_snapshot!(+ OptimizerContext::new().with_max_passes(1),+ vec![Arc::new(OptimizeUnions::new())],+ plan,+ @r"+ TableScan: table+ "+ )+ }}diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rsindex 084152d40e..421563d5e7 100644--- a/datafusion/optimizer/src/optimizer.rs+++ b/datafusion/optimizer/src/optimizer.rs@@ -41,12 +41,11 @@ use crate::eliminate_filter::EliminateFilter;use crate::eliminate_group_by_constant::EliminateGroupByConstant;use crate::eliminate_join::EliminateJoin;use crate::eliminate_limit::EliminateLimit;-use crate::eliminate_nested_union::EliminateNestedUnion;-use crate::eliminate_one_union::EliminateOneUnion;use crate::eliminate_outer_join::EliminateOuterJoin;use crate::extract_equijoin_predicate::ExtractEquijoinPredicate;use crate::filter_null_join_keys::FilterNullJoinKeys;use crate::optimize_projections::OptimizeProjections;+use crate::optimize_unions::OptimizeUnions;use crate::plan_signature::LogicalPlanSignature;use crate::propagate_empty_relation::PropagateEmptyRelation;use crate::push_down_filter::PushDownFilter;@@ -228,7 +227,7 @@ impl Optimizer {/// Create a new optimizer using the recommended list of rulespub fn new() -> Self {let rules: Vec<Arc<dyn OptimizerRule + Sync + Send>> = vec![- Arc::new(EliminateNestedUnion::new()),+ Arc::new(OptimizeUnions::new()),Arc::new(SimplifyExpressions::new()),Arc::new(ReplaceDistinctWithAggregate::new()),Arc::new(EliminateJoin::new()),@@ -241,8 +240,6 @@ impl Optimizer {Arc::new(EliminateCrossJoin::new()),Arc::new(EliminateLimit::new()),Arc::new(PropagateEmptyRelation::new()),- // Must be after PropagateEmptyRelation- Arc::new(EliminateOneUnion::new()),Arc::new(FilterNullJoinKeys::default()),Arc::new(EliminateOuterJoin::new()),// Filters can't be pushed down past Limits, we should do PushDownFilter after PushDownLimitdiff --git a/datafusion/optimizer/src/propagate_empty_relation.rs b/datafusion/optimizer/src/propagate_empty_relation.rsindex 4db3215dfb..629b13e400 100644--- a/datafusion/optimizer/src/propagate_empty_relation.rs+++ b/datafusion/optimizer/src/propagate_empty_relation.rs@@ -244,7 +244,7 @@ mod tests {use crate::assert_optimized_plan_eq_snapshot;use crate::eliminate_filter::EliminateFilter;- use crate::eliminate_nested_union::EliminateNestedUnion;+ use crate::optimize_unions::OptimizeUnions;use crate::test::{assert_optimized_plan_with_rules, test_table_scan, test_table_scan_fields,test_table_scan_with_name,@@ -277,7 +277,7 @@ mod tests {assert_optimized_plan_with_rules(vec![Arc::new(EliminateFilter::new()),- Arc::new(EliminateNestedUnion::new()),+ Arc::new(OptimizeUnions::new()),Arc::new(PropagateEmptyRelation::new()),],plan,diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex bfd693e6a0..989589dfb8 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -29,7 +29,7 @@ use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};use datafusion::execution::options::ArrowReadOptions;-use datafusion::optimizer::eliminate_nested_union::EliminateNestedUnion;+use datafusion::optimizer::optimize_unions::OptimizeUnions;use datafusion::optimizer::Optimizer;use datafusion_common::parsers::CompressionTypeVariant;use datafusion_functions_aggregate::sum::sum_distinct;@@ -2744,7 +2744,7 @@ async fn roundtrip_union_query() -> Result<()> {let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;// proto deserialization only supports 2-way union, hence this plan has nested unions// apply the flatten unions optimizer rule to be able to compare- let optimizer = Optimizer::with_rules(vec![Arc::new(EliminateNestedUnion::new())]);+ let optimizer = Optimizer::with_rules(vec![Arc::new(OptimizeUnions::new())]);let unnested = optimizer.optimize(logical_round_trip, &(ctx.state()), |_x, _y| {})?;assert_eq!(format!("{}", plan.display_indent_schema()),diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex d7af5ff4b9..918c01b561 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -176,7 +176,7 @@ initial_logical_planlogical_plan after resolve_grouping_function SAME TEXT AS ABOVElogical_plan after type_coercion SAME TEXT AS ABOVEanalyzed_logical_plan SAME TEXT AS ABOVE-logical_plan after eliminate_nested_union SAME TEXT AS ABOVE+logical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVElogical_plan after eliminate_join SAME TEXT AS ABOVE@@ -189,7 +189,6 @@ logical_plan after eliminate_filter SAME TEXT AS ABOVElogical_plan after eliminate_cross_join SAME TEXT AS ABOVElogical_plan after eliminate_limit SAME TEXT AS ABOVElogical_plan after propagate_empty_relation SAME TEXT AS ABOVE-logical_plan after eliminate_one_union SAME TEXT AS ABOVElogical_plan after filter_null_join_keys SAME TEXT AS ABOVElogical_plan after eliminate_outer_join SAME TEXT AS ABOVElogical_plan after push_down_limit SAME TEXT AS ABOVE@@ -198,7 +197,7 @@ logical_plan after single_distinct_aggregation_to_group_by SAME TEXT AS ABOVElogical_plan after eliminate_group_by_constant SAME TEXT AS ABOVElogical_plan after common_sub_expression_eliminate SAME TEXT AS ABOVElogical_plan after optimize_projections TableScan: simple_explain_test projection=[a, b, c]-logical_plan after eliminate_nested_union SAME TEXT AS ABOVE+logical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVElogical_plan after eliminate_join SAME TEXT AS ABOVE@@ -211,7 +210,6 @@ logical_plan after eliminate_filter SAME TEXT AS ABOVElogical_plan after eliminate_cross_join SAME TEXT AS ABOVElogical_plan after eliminate_limit SAME TEXT AS ABOVElogical_plan after propagate_empty_relation SAME TEXT AS ABOVE-logical_plan after eliminate_one_union SAME TEXT AS ABOVElogical_plan after filter_null_join_keys SAME TEXT AS ABOVElogical_plan after eliminate_outer_join SAME TEXT AS ABOVElogical_plan after push_down_limit SAME TEXT AS ABOVE@@ -537,7 +535,7 @@ initial_logical_planlogical_plan after resolve_grouping_function SAME TEXT AS ABOVElogical_plan after type_coercion SAME TEXT AS ABOVEanalyzed_logical_plan SAME TEXT AS ABOVE-logical_plan after eliminate_nested_union SAME TEXT AS ABOVE+logical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVElogical_plan after eliminate_join SAME TEXT AS ABOVE@@ -550,7 +548,6 @@ logical_plan after eliminate_filter SAME TEXT AS ABOVElogical_plan after eliminate_cross_join SAME TEXT AS ABOVElogical_plan after eliminate_limit SAME TEXT AS ABOVElogical_plan after propagate_empty_relation SAME TEXT AS ABOVE-logical_plan after eliminate_one_union SAME TEXT AS ABOVElogical_plan after filter_null_join_keys SAME TEXT AS ABOVElogical_plan after eliminate_outer_join SAME TEXT AS ABOVElogical_plan after push_down_limit SAME TEXT AS ABOVE@@ -559,7 +556,7 @@ logical_plan after single_distinct_aggregation_to_group_by SAME TEXT AS ABOVElogical_plan after eliminate_group_by_constant SAME TEXT AS ABOVElogical_plan after common_sub_expression_eliminate SAME TEXT AS ABOVElogical_plan after optimize_projections TableScan: simple_explain_test projection=[a, b, c]-logical_plan after eliminate_nested_union SAME TEXT AS ABOVE+logical_plan after optimize_unions SAME TEXT AS ABOVElogical_plan after simplify_expressions SAME TEXT AS ABOVElogical_plan after replace_distinct_aggregate SAME TEXT AS ABOVElogical_plan after eliminate_join SAME TEXT AS ABOVE@@ -572,7 +569,6 @@ logical_plan after eliminate_filter SAME TEXT AS ABOVElogical_plan after eliminate_cross_join SAME TEXT AS ABOVElogical_plan after eliminate_limit SAME TEXT AS ABOVElogical_plan after propagate_empty_relation SAME TEXT AS ABOVE-logical_plan after eliminate_one_union SAME TEXT AS ABOVElogical_plan after filter_null_join_keys SAME TEXT AS ABOVElogical_plan after eliminate_outer_join SAME TEXT AS ABOVElogical_plan after push_down_limit SAME TEXT AS ABOVE