Skip to content

Apache DataFusion SQL Query Engine

refactor: merge CoalesceAsyncExecInput into CoalesceBatches (#18540)

Tim-539029ff183428
diff --git a/datafusion/physical-optimizer/src/coalesce_async_exec_input.rs b/datafusion/physical-optimizer/src/coalesce_async_exec_input.rs
deleted file mode 100644
index 0b46c68f2d..0000000000
--- a/datafusion/physical-optimizer/src/coalesce_async_exec_input.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::PhysicalOptimizerRule;
-use datafusion_common::config::ConfigOptions;
-use datafusion_common::internal_err;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion_physical_plan::async_func::AsyncFuncExec;
-use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;
-use datafusion_physical_plan::ExecutionPlan;
-use std::sync::Arc;
-
-/// Optimizer rule that introduces CoalesceAsyncExec to reduce the number of async executions.
-#[derive(Default, Debug)]
-pub struct CoalesceAsyncExecInput {}
-
-impl CoalesceAsyncExecInput {
- #[allow(missing_docs)]
- pub fn new() -> Self {
- Self::default()
- }
-}
-
-impl PhysicalOptimizerRule for CoalesceAsyncExecInput {
- fn optimize(
- &self,
- plan: Arc<dyn ExecutionPlan>,
- config: &ConfigOptions,
- ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
- let target_batch_size = config.execution.batch_size;
- plan.transform(|plan| {
- if let Some(async_exec) = plan.as_any().downcast_ref::<AsyncFuncExec>() {
- if async_exec.children().len() != 1 {
- return internal_err!(
- "Expected AsyncFuncExec to have exactly one child"
- );
- }
- let child = Arc::clone(async_exec.children()[0]);
- let coalesce_exec =
- Arc::new(CoalesceBatchesExec::new(child, target_batch_size));
- let coalesce_async_exec = plan.with_new_children(vec![coalesce_exec])?;
- Ok(Transformed::yes(coalesce_async_exec))
- } else {
- Ok(Transformed::no(plan))
- }
- })
- .data()
- }
-
- fn name(&self) -> &str {
- "coalesce_async_exec_input"
- }
-
- fn schema_check(&self) -> bool {
- true
- }
-}
diff --git a/datafusion/physical-optimizer/src/coalesce_batches.rs b/datafusion/physical-optimizer/src/coalesce_batches.rs
index 5cf2c877c6..b19d8d9518 100644
--- a/datafusion/physical-optimizer/src/coalesce_batches.rs
+++ b/datafusion/physical-optimizer/src/coalesce_batches.rs
@@ -22,12 +22,12 @@ use crate::PhysicalOptimizerRule;
use std::sync::Arc;
-use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
+use datafusion_common::{config::ConfigOptions, internal_err};
use datafusion_physical_expr::Partitioning;
use datafusion_physical_plan::{
- coalesce_batches::CoalesceBatchesExec, filter::FilterExec, joins::HashJoinExec,
- repartition::RepartitionExec, ExecutionPlan,
+ async_func::AsyncFuncExec, coalesce_batches::CoalesceBatchesExec, filter::FilterExec,
+ joins::HashJoinExec, repartition::RepartitionExec, ExecutionPlan,
};
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
@@ -72,11 +72,27 @@ impl PhysicalOptimizerRule for CoalesceBatches {
)
})
.unwrap_or(false);
+
if wrap_in_coalesce {
Ok(Transformed::yes(Arc::new(CoalesceBatchesExec::new(
plan,
target_batch_size,
))))
+ } else if let Some(async_exec) = plan_any.downcast_ref::<AsyncFuncExec>() {
+ // Coalesce inputs to async functions to reduce number of async function invocations
+ let children = async_exec.children();
+ if children.len() != 1 {
+ return internal_err!(
+ "Expected AsyncFuncExec to have exactly one child"
+ );
+ }
+
+ let coalesce_exec = Arc::new(CoalesceBatchesExec::new(
+ Arc::clone(children[0]),
+ target_batch_size,
+ ));
+ let new_plan = plan.with_new_children(vec![coalesce_exec])?;
+ Ok(Transformed::yes(new_plan))
} else {
Ok(Transformed::no(plan))
}
diff --git a/datafusion/physical-optimizer/src/lib.rs b/datafusion/physical-optimizer/src/lib.rs
index d238a4264f..f4b82eed3c 100644
--- a/datafusion/physical-optimizer/src/lib.rs
+++ b/datafusion/physical-optimizer/src/lib.rs
@@ -28,7 +28,6 @@
#![cfg_attr(test, allow(clippy::needless_pass_by_value))]
pub mod aggregate_statistics;
-pub mod coalesce_async_exec_input;
pub mod coalesce_batches;
pub mod combine_partial_final_agg;
pub mod enforce_distribution;
diff --git a/datafusion/physical-optimizer/src/optimizer.rs b/datafusion/physical-optimizer/src/optimizer.rs
index 4d00f1029d..03c83bb5a0 100644
--- a/datafusion/physical-optimizer/src/optimizer.rs
+++ b/datafusion/physical-optimizer/src/optimizer.rs
@@ -36,7 +36,6 @@ use crate::sanity_checker::SanityCheckPlan;
use crate::topk_aggregation::TopKAggregation;
use crate::update_aggr_exprs::OptimizeAggregateOrder;
-use crate::coalesce_async_exec_input::CoalesceAsyncExecInput;
use crate::limit_pushdown_past_window::LimitPushPastWindows;
use datafusion_common::config::ConfigOptions;
use datafusion_common::Result;
@@ -123,7 +122,6 @@ impl PhysicalOptimizer {
// The CoalesceBatches rule will not influence the distribution and ordering of the
// whole plan tree. Therefore, to avoid influencing other rules, it should run last.
Arc::new(CoalesceBatches::new()),
- Arc::new(CoalesceAsyncExecInput::new()),
// Remove the ancillary output requirement operator since we are done with the planning
// phase.
Arc::new(OutputRequirements::new_remove_mode()),
diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt
index a3b6d40aea..ec3d9f7465 100644
--- a/datafusion/sqllogictest/test_files/explain.slt
+++ b/datafusion/sqllogictest/test_files/explain.slt
@@ -238,7 +238,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVE
physical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVE
physical_plan after ProjectionPushdown SAME TEXT AS ABOVE
physical_plan after coalesce_batches SAME TEXT AS ABOVE
-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVE
physical_plan after OutputRequirements DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.csv]]}, projection=[a, b, c], file_type=csv, has_header=true
physical_plan after LimitAggregation SAME TEXT AS ABOVE
physical_plan after LimitPushPastWindows SAME TEXT AS ABOVE
@@ -317,7 +316,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVE
physical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVE
physical_plan after ProjectionPushdown SAME TEXT AS ABOVE
physical_plan after coalesce_batches SAME TEXT AS ABOVE
-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVE
physical_plan after OutputRequirements
01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]
@@ -362,7 +360,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVE
physical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVE
physical_plan after ProjectionPushdown SAME TEXT AS ABOVE
physical_plan after coalesce_batches SAME TEXT AS ABOVE
-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVE
physical_plan after OutputRequirements
01)GlobalLimitExec: skip=0, fetch=10
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet
@@ -602,7 +599,6 @@ physical_plan after EnforceSorting SAME TEXT AS ABOVE
physical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVE
physical_plan after ProjectionPushdown SAME TEXT AS ABOVE
physical_plan after coalesce_batches SAME TEXT AS ABOVE
-physical_plan after coalesce_async_exec_input SAME TEXT AS ABOVE
physical_plan after OutputRequirements DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.csv]]}, projection=[a, b, c], file_type=csv, has_header=true
physical_plan after LimitAggregation SAME TEXT AS ABOVE
physical_plan after LimitPushPastWindows SAME TEXT AS ABOVE