Apache DataFusion SQL Query Engine
refactor: merge CoalesceAsyncExecInput into CoalesceBatches (#18540)
Tim-539029ff183428
diff --git a/datafusion/physical-optimizer/src/coalesce_async_exec_input.rs b/datafusion/physical-optimizer/src/coalesce_async_exec_input.rsdeleted file mode 100644index 0b46c68f2d..0000000000--- a/datafusion/physical-optimizer/src/coalesce_async_exec_input.rs+++ /dev/null@@ -1,71 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--use crate::PhysicalOptimizerRule;-use datafusion_common::config::ConfigOptions;-use datafusion_common::internal_err;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_physical_plan::async_func::AsyncFuncExec;-use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;-use datafusion_physical_plan::ExecutionPlan;-use std::sync::Arc;--/// Optimizer rule that introduces CoalesceAsyncExec to reduce the number of async executions.-#[derive(Default, Debug)]-pub struct CoalesceAsyncExecInput {}--impl CoalesceAsyncExecInput {- #[allow(missing_docs)]- pub fn new() -> Self {- Self::default()- }-}--impl PhysicalOptimizerRule for CoalesceAsyncExecInput {- fn optimize(- &self,- plan: Arc<dyn ExecutionPlan>,- config: &ConfigOptions,- ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {- let target_batch_size = config.execution.batch_size;- plan.transform(|plan| {- if let Some(async_exec) = plan.as_any().downcast_ref::<AsyncFuncExec>() {- if async_exec.children().len() != 1 {- return internal_err!(- "Expected AsyncFuncExec to have exactly one child"- );- }- let child = Arc::clone(async_exec.children()[0]);- let coalesce_exec =- Arc::new(CoalesceBatchesExec::new(child, target_batch_size));- let coalesce_async_exec = plan.with_new_children(vec![coalesce_exec])?;- Ok(Transformed::yes(coalesce_async_exec))- } else {- Ok(Transformed::no(plan))- }- })- .data()- }-- fn name(&self) -> &str {- "coalesce_async_exec_input"- }-- fn schema_check(&self) -> bool {- true- }-}diff --git a/datafusion/physical-optimizer/src/coalesce_batches.rs b/datafusion/physical-optimizer/src/coalesce_batches.rsindex 5cf2c877c6..b19d8d9518 100644--- a/datafusion/physical-optimizer/src/coalesce_batches.rs+++ b/datafusion/physical-optimizer/src/coalesce_batches.rs@@ -22,12 +22,12 @@ use crate::PhysicalOptimizerRule;use std::sync::Arc;-use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;+use datafusion_common::{config::ConfigOptions, internal_err};use datafusion_physical_expr::Partitioning;use datafusion_physical_plan::{- coalesce_batches::CoalesceBatchesExec, filter::FilterExec, joins::HashJoinExec,- repartition::RepartitionExec, ExecutionPlan,+ async_func::AsyncFuncExec, coalesce_batches::CoalesceBatchesExec, filter::FilterExec,+ joins::HashJoinExec, repartition::RepartitionExec, ExecutionPlan,};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};@@ -72,11 +72,27 @@ impl PhysicalOptimizerRule for CoalesceBatches {)}).unwrap_or(false);+if wrap_in_coalesce {Ok(Transformed::yes(Arc::new(CoalesceBatchesExec::new(plan,target_batch_size,))))+ } else if let Some(async_exec) = plan_any.downcast_ref::<AsyncFuncExec>() {+ // Coalesce inputs to async functions to reduce number of async function invocations+ let children = async_exec.children();+ if children.len() != 1 {+ return internal_err!(+ "Expected AsyncFuncExec to have exactly one child"+ );+ }++ let coalesce_exec = Arc::new(CoalesceBatchesExec::new(+ Arc::clone(children[0]),+ target_batch_size,+ ));+ let new_plan = plan.with_new_children(vec![coalesce_exec])?;+ Ok(Transformed::yes(new_plan))} else {Ok(Transformed::no(plan))}diff --git a/datafusion/physical-optimizer/src/lib.rs b/datafusion/physical-optimizer/src/lib.rsindex d238a4264f..f4b82eed3c 100644--- a/datafusion/physical-optimizer/src/lib.rs+++ b/datafusion/physical-optimizer/src/lib.rs@@ -28,7 +28,6 @@#![cfg_attr(test, allow(clippy::needless_pass_by_value))]pub mod aggregate_statistics;-pub mod coalesce_async_exec_input;pub mod coalesce_batches;pub mod combine_partial_final_agg;pub mod enforce_distribution;diff --git a/datafusion/physical-optimizer/src/optimizer.rs b/datafusion/physical-optimizer/src/optimizer.rsindex 4d00f1029d..03c83bb5a0 100644--- a/datafusion/physical-optimizer/src/optimizer.rs+++ b/datafusion/physical-optimizer/src/optimizer.rs@@ -36,7 +36,6 @@ use crate::sanity_checker::SanityCheckPlan;use crate::topk_aggregation::TopKAggregation;use crate::update_aggr_exprs::OptimizeAggregateOrder;-use crate::coalesce_async_exec_input::CoalesceAsyncExecInput;use crate::limit_pushdown_past_window::LimitPushPastWindows;use datafusion_common::config::ConfigOptions;use datafusion_common::Result;@@ -123,7 +122,6 @@ impl PhysicalOptimizer {// The CoalesceBatches rule will not influence the distribution and ordering of the// whole plan tree. Therefore, to avoid influencing other rules, it should run last.Arc::new(CoalesceBatches::new()),- Arc::new(CoalesceAsyncExecInput::new()),// Remove the ancillary output requirement operator since we are done with the planning// phase.Arc::new(OutputRequirements::new_remove_mode()),diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex a3b6d40aea..ec3d9f7465 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -238,7 +238,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVEphysical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVEphysical_plan after ProjectionPushdown SAME TEXT AS ABOVEphysical_plan after coalesce_batches SAME TEXT AS ABOVE-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVEphysical_plan after OutputRequirements DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.csv]]}, projection=[a, b, c], file_type=csv, has_header=truephysical_plan after LimitAggregation SAME TEXT AS ABOVEphysical_plan after LimitPushPastWindows SAME TEXT AS ABOVE@@ -317,7 +316,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVEphysical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVEphysical_plan after ProjectionPushdown SAME TEXT AS ABOVEphysical_plan after coalesce_batches SAME TEXT AS ABOVE-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVEphysical_plan after OutputRequirements01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]@@ -362,7 +360,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVEphysical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVEphysical_plan after ProjectionPushdown SAME TEXT AS ABOVEphysical_plan after coalesce_batches SAME TEXT AS ABOVE-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVEphysical_plan after OutputRequirements01)GlobalLimitExec: skip=0, fetch=1002)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet@@ -602,7 +599,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVEphysical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVEphysical_plan after ProjectionPushdown SAME TEXT AS ABOVEphysical_plan after coalesce_batches SAME TEXT AS ABOVE-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVEphysical_plan after OutputRequirements DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.csv]]}, projection=[a, b, c], file_type=csv, has_header=truephysical_plan after LimitAggregation SAME TEXT AS ABOVEphysical_plan after LimitPushPastWindows SAME TEXT AS ABOVE