Apache DataFusion SQL Query Engine
feat: Push limit into hash join (#20228)
Jonathan Chen682da846b599
diff --git a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs b/datafusion/core/tests/physical_optimizer/limit_pushdown.rsindex 0c41fc8e9e..b8c4d6d6f0 100644--- a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs@@ -18,8 +18,8 @@use std::sync::Arc;use crate::physical_optimizer::test_utils::{- coalesce_partitions_exec, global_limit_exec, local_limit_exec, sort_exec,- sort_preserving_merge_exec, stream_exec,+ coalesce_partitions_exec, global_limit_exec, hash_join_exec, local_limit_exec,+ sort_exec, sort_preserving_merge_exec, stream_exec,};use arrow::compute::SortOptions;@@ -29,6 +29,7 @@ use datafusion_common::error::Result;use datafusion_expr::{JoinType, Operator};use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::expressions::{BinaryExpr, col, lit};+use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;@@ -161,6 +162,168 @@ fn transforms_streaming_table_exec_into_fetching_version_and_keeps_the_global_liOk(())}+fn join_on_columns(+ left_col: &str,+ right_col: &str,+) -> Vec<(PhysicalExprRef, PhysicalExprRef)> {+ vec![(+ Arc::new(datafusion_physical_expr::expressions::Column::new(+ left_col, 0,+ )) as _,+ Arc::new(datafusion_physical_expr::expressions::Column::new(+ right_col, 0,+ )) as _,+ )]+}++#[test]+fn absorbs_limit_into_hash_join_inner() -> Result<()> {+ // HashJoinExec with Inner join should absorb limit via with_fetch+ let schema = create_schema();+ let left = empty_exec(Arc::clone(&schema));+ let right = empty_exec(Arc::clone(&schema));+ let on = join_on_columns("c1", "c1");+ let hash_join = hash_join_exec(left, right, on, None, &JoinType::Inner)?;+ let global_limit = global_limit_exec(hash_join, 0, Some(5));++ let initial = format_plan(&global_limit);+ insta::assert_snapshot!(+ initial,+ @r"+ GlobalLimitExec: skip=0, fetch=5+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c1@0, c1@0)]+ EmptyExec+ EmptyExec+ "+ );++ let after_optimize =+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;+ let optimized = format_plan(&after_optimize);+ // The limit should be absorbed by the hash join (not pushed to children)+ insta::assert_snapshot!(+ optimized,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c1@0, c1@0)], fetch=5+ EmptyExec+ EmptyExec+ "+ );++ Ok(())+}++#[test]+fn absorbs_limit_into_hash_join_right() -> Result<()> {+ // HashJoinExec with Right join should absorb limit via with_fetch+ let schema = create_schema();+ let left = empty_exec(Arc::clone(&schema));+ let right = empty_exec(Arc::clone(&schema));+ let on = join_on_columns("c1", "c1");+ let hash_join = hash_join_exec(left, right, on, None, &JoinType::Right)?;+ let global_limit = global_limit_exec(hash_join, 0, Some(10));++ let initial = format_plan(&global_limit);+ insta::assert_snapshot!(+ initial,+ @r"+ GlobalLimitExec: skip=0, fetch=10+ HashJoinExec: mode=Partitioned, join_type=Right, on=[(c1@0, c1@0)]+ EmptyExec+ EmptyExec+ "+ );++ let after_optimize =+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;+ let optimized = format_plan(&after_optimize);+ // The limit should be absorbed by the hash join+ insta::assert_snapshot!(+ optimized,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Right, on=[(c1@0, c1@0)], fetch=10+ EmptyExec+ EmptyExec+ "+ );++ Ok(())+}++#[test]+fn absorbs_limit_into_hash_join_left() -> Result<()> {+ // during probing, then unmatched rows at the end, stopping when limit is reached+ let schema = create_schema();+ let left = empty_exec(Arc::clone(&schema));+ let right = empty_exec(Arc::clone(&schema));+ let on = join_on_columns("c1", "c1");+ let hash_join = hash_join_exec(left, right, on, None, &JoinType::Left)?;+ let global_limit = global_limit_exec(hash_join, 0, Some(5));++ let initial = format_plan(&global_limit);+ insta::assert_snapshot!(+ initial,+ @r"+ GlobalLimitExec: skip=0, fetch=5+ HashJoinExec: mode=Partitioned, join_type=Left, on=[(c1@0, c1@0)]+ EmptyExec+ EmptyExec+ "+ );++ let after_optimize =+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;+ let optimized = format_plan(&after_optimize);+ // Left join now absorbs the limit+ insta::assert_snapshot!(+ optimized,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Left, on=[(c1@0, c1@0)], fetch=5+ EmptyExec+ EmptyExec+ "+ );++ Ok(())+}++#[test]+fn absorbs_limit_with_skip_into_hash_join() -> Result<()> {+ let schema = create_schema();+ let left = empty_exec(Arc::clone(&schema));+ let right = empty_exec(Arc::clone(&schema));+ let on = join_on_columns("c1", "c1");+ let hash_join = hash_join_exec(left, right, on, None, &JoinType::Inner)?;+ let global_limit = global_limit_exec(hash_join, 3, Some(5));++ let initial = format_plan(&global_limit);+ insta::assert_snapshot!(+ initial,+ @r"+ GlobalLimitExec: skip=3, fetch=5+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c1@0, c1@0)]+ EmptyExec+ EmptyExec+ "+ );++ let after_optimize =+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;+ let optimized = format_plan(&after_optimize);+ // With skip, GlobalLimit is kept but fetch (skip + limit = 8) is absorbed by the join+ insta::assert_snapshot!(+ optimized,+ @r"+ GlobalLimitExec: skip=3, fetch=5+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c1@0, c1@0)], fetch=8+ EmptyExec+ EmptyExec+ "+ );++ Ok(())+}+#[test]fn pushes_global_limit_exec_through_projection_exec() -> Result<()> {let schema = create_schema();diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 77d736e938..eb2e841791 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -258,6 +258,11 @@ pub struct HashJoinExecBuilder {partition_mode: PartitionMode,null_equality: NullEquality,null_aware: bool,+ /// Maximum number of rows to return+ ///+ /// If the operator produces `< fetch` rows, it returns all available rows.+ /// If it produces `>= fetch` rows, it returns exactly `fetch` rows and stops early.+ fetch: Option<usize>,}impl HashJoinExecBuilder {@@ -278,6 +283,7 @@ impl HashJoinExecBuilder {join_type,null_equality: NullEquality::NullEqualsNothing,null_aware: false,+ fetch: None,}}@@ -316,6 +322,12 @@ impl HashJoinExecBuilder {self}+ /// Set fetch limit.+ pub fn with_fetch(mut self, fetch: Option<usize>) -> Self {+ self.fetch = fetch;+ self+ }+/// Build resulting execution plan.pub fn build(self) -> Result<HashJoinExec> {let Self {@@ -328,6 +340,7 @@ impl HashJoinExecBuilder {partition_mode,null_equality,null_aware,+ fetch,} = self;let left_schema = left.schema();@@ -393,6 +406,7 @@ impl HashJoinExecBuilder {null_aware,cache,dynamic_filter: None,+ fetch,})}}@@ -409,6 +423,7 @@ impl From<&HashJoinExec> for HashJoinExecBuilder {partition_mode: exec.mode,null_equality: exec.null_equality,null_aware: exec.null_aware,+ fetch: exec.fetch,}}}@@ -646,6 +661,8 @@ pub struct HashJoinExec {/// Set when dynamic filter pushdown is detected in handle_child_pushdown_result./// HashJoinExec also needs to keep a shared bounds accumulator for coordinating updates.dynamic_filter: Option<HashJoinExecDynamicFilter>,+ /// Maximum number of rows to return+ fetch: Option<usize>,}#[derive(Clone)]@@ -930,25 +947,27 @@ impl HashJoinExec {) -> Result<Arc<dyn ExecutionPlan>> {let left = self.left();let right = self.right();- let new_join = HashJoinExec::try_new(+ let new_join = HashJoinExecBuilder::new(Arc::clone(right),Arc::clone(left),self.on().iter().map(|(l, r)| (Arc::clone(r), Arc::clone(l))).collect(),- self.filter().map(JoinFilter::swap),- &self.join_type().swap(),- swap_join_projection(- left.schema().fields().len(),- right.schema().fields().len(),- self.projection.as_deref(),- self.join_type(),- ),- partition_mode,- self.null_equality(),- self.null_aware,- )?;+ self.join_type().swap(),+ )+ .with_filter(self.filter().map(JoinFilter::swap))+ .with_projection(swap_join_projection(+ left.schema().fields().len(),+ right.schema().fields().len(),+ self.projection.as_deref(),+ self.join_type(),+ ))+ .with_partition_mode(partition_mode)+ .with_null_equality(self.null_equality())+ .with_null_aware(self.null_aware)+ .with_fetch(self.fetch)+ .build()?;// In case of anti / semi joins or if there is embedded projection in HashJoinExec, output column order is preserved, no need to add projection againif matches!(self.join_type(),@@ -999,6 +1018,9 @@ impl DisplayAs for HashJoinExec {} else {""};+ let display_fetch = self+ .fetch+ .map_or_else(String::new, |f| format!(", fetch={f}"));let on = self.on.iter()@@ -1007,13 +1029,14 @@ impl DisplayAs for HashJoinExec {.join(", ");write!(f,- "HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}",+ "HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}{}",self.mode,self.join_type,on,display_filter,display_projections,display_null_equality,+ display_fetch,)}DisplayFormatType::TreeRender => {@@ -1040,6 +1063,10 @@ impl DisplayAs for HashJoinExec {writeln!(f, "filter={filter}")?;}+ if let Some(fetch) = self.fetch {+ writeln!(f, "fetch={fetch}")?;+ }+Ok(())}}@@ -1142,6 +1169,7 @@ impl ExecutionPlan for HashJoinExec {)?,// Keep the dynamic filter, bounds accumulator will be resetdynamic_filter: self.dynamic_filter.clone(),+ fetch: self.fetch,}))}@@ -1165,6 +1193,7 @@ impl ExecutionPlan for HashJoinExec {cache: self.cache.clone(),// Reset dynamic filter and bounds accumulator to initial statedynamic_filter: None,+ fetch: self.fetch,}))}@@ -1329,6 +1358,7 @@ impl ExecutionPlan for HashJoinExec {build_accumulator,self.mode,self.null_aware,+ self.fetch,)))}@@ -1351,7 +1381,9 @@ impl ExecutionPlan for HashJoinExec {&self.join_schema,)?;// Project statistics if there is a projection- Ok(stats.project(self.projection.as_ref()))+ let stats = stats.project(self.projection.as_ref());+ // Apply fetch limit to statistics+ stats.with_fetch(self.fetch, 0, 1)}/// Tries to push `projection` down through `hash_join`. If possible, performs the@@ -1380,18 +1412,22 @@ impl ExecutionPlan for HashJoinExec {&schema,self.filter(),)? {- Ok(Some(Arc::new(HashJoinExec::try_new(- Arc::new(projected_left_child),- Arc::new(projected_right_child),- join_on,- join_filter,- self.join_type(),+ Ok(Some(Arc::new(+ HashJoinExecBuilder::new(+ Arc::new(projected_left_child),+ Arc::new(projected_right_child),+ join_on,+ *self.join_type(),+ )+ .with_filter(join_filter)// Returned early if projection is not None- None,- *self.partition_mode(),- self.null_equality,- self.null_aware,- )?)))+ .with_projection(None)+ .with_partition_mode(*self.partition_mode())+ .with_null_equality(self.null_equality)+ .with_null_aware(self.null_aware)+ .with_fetch(self.fetch)+ .build()?,+ )))} else {try_embed_projection(projection, self)}@@ -1489,12 +1525,32 @@ impl ExecutionPlan for HashJoinExec {filter: dynamic_filter,build_accumulator: OnceLock::new(),}),+ fetch: self.fetch,});result = result.with_updated_node(new_node as Arc<dyn ExecutionPlan>);}}Ok(result)}++ fn supports_limit_pushdown(&self) -> bool {+ // Hash join execution plan does not support pushing limit down through to children+ // because the children don't know about the join condition and can't+ // determine how many rows to produce+ false+ }++ fn fetch(&self) -> Option<usize> {+ self.fetch+ }++ fn with_fetch(&self, limit: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {+ HashJoinExecBuilder::from(self)+ .with_fetch(limit)+ .build()+ .ok()+ .map(|exec| Arc::new(exec) as _)+ }}/// Accumulator for collecting min/max bounds from build-side data during hash join.diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 54e620f99d..8af26c1b8a 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -24,6 +24,7 @@ use std::sync::Arc;use std::sync::atomic::Ordering;use std::task::Poll;+use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};use crate::joins::Map;use crate::joins::MapOffset;use crate::joins::PartitionMode;@@ -46,7 +47,6 @@ use crate::{};use arrow::array::{Array, ArrayRef, UInt32Array, UInt64Array};-use arrow::compute::BatchCoalescer;use arrow::datatypes::{Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::{@@ -221,10 +221,9 @@ pub(super) struct HashJoinStream {build_waiter: Option<OnceFut<()>>,/// Partitioning mode to usemode: PartitionMode,- /// Output buffer for coalescing small batches into larger ones.- /// Uses `BatchCoalescer` from arrow to efficiently combine batches.- /// When batches are already close to target size, they bypass coalescing.- output_buffer: Box<BatchCoalescer>,+ /// Output buffer for coalescing small batches into larger ones with optional fetch limit.+ /// Uses `LimitedBatchCoalescer` to efficiently combine batches and absorb limit with 'fetch'+ output_buffer: LimitedBatchCoalescer,/// Whether this is a null-aware anti joinnull_aware: bool,}@@ -375,14 +374,11 @@ impl HashJoinStream {build_accumulator: Option<Arc<SharedBuildAccumulator>>,mode: PartitionMode,null_aware: bool,+ fetch: Option<usize>,) -> Self {- // Create output buffer with coalescing.- // Use biggest_coalesce_batch_size to bypass coalescing for batches- // that are already close to target size (within 50%).- let output_buffer = Box::new(- BatchCoalescer::new(Arc::clone(&schema), batch_size)- .with_biggest_coalesce_batch_size(Some(batch_size / 2)),- );+ // Create output buffer with coalescing and optional fetch limit.+ let output_buffer =+ LimitedBatchCoalescer::new(Arc::clone(&schema), batch_size, fetch);Self {partition,@@ -425,6 +421,11 @@ impl HashJoinStream {.record_poll(Poll::Ready(Some(Ok(batch))));}+ // Check if the coalescer has finished (limit reached and flushed)+ if self.output_buffer.is_finished() {+ return Poll::Ready(None);+ }+return match self.state {HashJoinStreamState::WaitBuildSide => {handle_state!(ready!(self.collect_build_side(cx)))@@ -443,7 +444,7 @@ impl HashJoinStream {}HashJoinStreamState::Completed if !self.output_buffer.is_empty() => {// Flush any remaining buffered data- self.output_buffer.finish_buffered_batch()?;+ self.output_buffer.finish()?;// Continue loop to emit the flushed batchcontinue;}@@ -782,10 +783,17 @@ impl HashJoinStream {join_side,)?;- self.output_buffer.push_batch(batch)?;+ let push_status = self.output_buffer.push_batch(batch)?;timer.done();+ // If limit reached, finish and move to Completed state+ if push_status == PushBatchStatus::LimitReached {+ self.output_buffer.finish()?;+ self.state = HashJoinStreamState::Completed;+ return Ok(StatefulStreamResult::Continue);+ }+if next_offset.is_none() {self.state = HashJoinStreamState::FetchProbeBatch;} else {@@ -892,7 +900,12 @@ impl HashJoinStream {&self.column_indices,JoinSide::Left,)?;- self.output_buffer.push_batch(batch)?;+ let push_status = self.output_buffer.push_batch(batch)?;++ // If limit reached, finish the coalescer+ if push_status == PushBatchStatus::LimitReached {+ self.output_buffer.finish()?;+ }}Ok(StatefulStreamResult::Continue)diff --git a/datafusion/sqllogictest/test_files/join_disable_repartition_joins.slt b/datafusion/sqllogictest/test_files/join_disable_repartition_joins.sltindex 1b25a01d62..59f3d8285a 100644--- a/datafusion/sqllogictest/test_files/join_disable_repartition_joins.slt+++ b/datafusion/sqllogictest/test_files/join_disable_repartition_joins.slt@@ -55,7 +55,7 @@ logical_plan07)--------TableScan: annotated_data projection=[a, c]physical_plan01)SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=5-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c@0, c@1)], projection=[a@1]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c@0, c@1)], projection=[a@1], fetch=503)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], file_type=csv, has_header=true04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a, c], output_ordering=[a@0 ASC NULLS LAST], file_type=csv, has_header=true@@ -96,7 +96,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [a2@0 ASC NULLS LAST, b@1 ASC NULLS LAST], fetch=1002)--ProjectionExec: expr=[a@0 as a2, b@1 as b]-03)----HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d@1, d@3), (c@0, c@2)], projection=[a@0, b@1]+03)----HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d@1, d@3), (c@0, c@2)], projection=[a@0, b@1], fetch=1004)------CoalescePartitionsExec05)--------FilterExec: d@1 = 306)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1diff --git a/datafusion/sqllogictest/test_files/join_limit_pushdown.slt b/datafusion/sqllogictest/test_files/join_limit_pushdown.sltnew file mode 100644index 0000000000..6bb23c1b4c--- /dev/null+++ b/datafusion/sqllogictest/test_files/join_limit_pushdown.slt@@ -0,0 +1,269 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Tests for limit pushdown into joins++# need to use a single partition for deterministic results+statement ok+set datafusion.execution.target_partitions = 1;++statement ok+set datafusion.explain.logical_plan_only = false;++statement ok+set datafusion.optimizer.prefer_hash_join = true;++# Create test tables+statement ok+CREATE TABLE t1 (a INT, b VARCHAR) AS VALUES+ (1, 'one'),+ (2, 'two'),+ (3, 'three'),+ (4, 'four'),+ (5, 'five');++statement ok+CREATE TABLE t2 (x INT, y VARCHAR) AS VALUES+ (1, 'alpha'),+ (2, 'beta'),+ (3, 'gamma'),+ (6, 'delta'),+ (7, 'epsilon');++query TT+EXPLAIN SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 2;+----+logical_plan+01)Limit: skip=0, fetch=2+02)--Inner Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----TableScan: t2 projection=[x]+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, x@0)], fetch=2+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--DataSourceExec: partitions=1, partition_sizes=[1]++query II+SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 2;+----+1 1+2 2++# Right join is converted to Left join with projection - fetch pushdown is supported+query TT+EXPLAIN SELECT t1.a, t2.x FROM t1 RIGHT JOIN t2 ON t1.a = t2.x LIMIT 3;+----+logical_plan+01)Limit: skip=0, fetch=3+02)--Right Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----Limit: skip=0, fetch=3+05)------TableScan: t2 projection=[x], fetch=3+physical_plan+01)ProjectionExec: expr=[a@1 as a, x@0 as x]+02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(x@0, a@0)], fetch=3+03)----DataSourceExec: partitions=1, partition_sizes=[1], fetch=3+04)----DataSourceExec: partitions=1, partition_sizes=[1]++query II+SELECT t1.a, t2.x FROM t1 RIGHT JOIN t2 ON t1.a = t2.x LIMIT 3;+----+1 1+2 2+3 3++# Left join supports fetch pushdown+query TT+EXPLAIN SELECT t1.a, t2.x FROM t1 LEFT JOIN t2 ON t1.a = t2.x LIMIT 3;+----+logical_plan+01)Limit: skip=0, fetch=3+02)--Left Join: t1.a = t2.x+03)----Limit: skip=0, fetch=3+04)------TableScan: t1 projection=[a], fetch=3+05)----TableScan: t2 projection=[x]+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, x@0)], fetch=3+02)--DataSourceExec: partitions=1, partition_sizes=[1], fetch=3+03)--DataSourceExec: partitions=1, partition_sizes=[1]++query II+SELECT t1.a, t2.x FROM t1 LEFT JOIN t2 ON t1.a = t2.x LIMIT 3;+----+1 1+2 2+3 3+++# Full join supports fetch pushdown+query TT+EXPLAIN SELECT t1.a, t2.x FROM t1 FULL OUTER JOIN t2 ON t1.a = t2.x LIMIT 4;+----+logical_plan+01)Limit: skip=0, fetch=4+02)--Full Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----TableScan: t2 projection=[x]+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=Full, on=[(a@0, x@0)], fetch=4+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--DataSourceExec: partitions=1, partition_sizes=[1]++# Note: FULL OUTER JOIN order is not deterministic, so we just check count+query I+SELECT COUNT(*) FROM (SELECT t1.a, t2.x FROM t1 FULL OUTER JOIN t2 ON t1.a = t2.x LIMIT 4);+----+4++# EXISTS becomes left semi join - fetch pushdown is supported+query TT+EXPLAIN SELECT t2.x FROM t2 WHERE EXISTS (SELECT 1 FROM t1 WHERE t1.a = t2.x) LIMIT 2;+----+logical_plan+01)Limit: skip=0, fetch=2+02)--LeftSemi Join: t2.x = __correlated_sq_1.a+03)----TableScan: t2 projection=[x]+04)----SubqueryAlias: __correlated_sq_1+05)------TableScan: t1 projection=[a]+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(x@0, a@0)], fetch=2+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--DataSourceExec: partitions=1, partition_sizes=[1]++query I+SELECT t2.x FROM t2 WHERE EXISTS (SELECT 1 FROM t1 WHERE t1.a = t2.x) LIMIT 2;+----+1+2++# NOT EXISTS becomes LeftAnti - fetch pushdown is supported+query TT+EXPLAIN SELECT t2.x FROM t2 WHERE NOT EXISTS (SELECT 1 FROM t1 WHERE t1.a = t2.x) LIMIT 1;+----+logical_plan+01)Limit: skip=0, fetch=1+02)--LeftAnti Join: t2.x = __correlated_sq_1.a+03)----TableScan: t2 projection=[x]+04)----SubqueryAlias: __correlated_sq_1+05)------TableScan: t1 projection=[a]+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(x@0, a@0)], fetch=1+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--DataSourceExec: partitions=1, partition_sizes=[1]++query I+SELECT t2.x FROM t2 WHERE NOT EXISTS (SELECT 1 FROM t1 WHERE t1.a = t2.x) LIMIT 1;+----+6++# Inner join should push+query TT+EXPLAIN SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 1 OFFSET 1;+----+logical_plan+01)Limit: skip=1, fetch=1+02)--Inner Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----TableScan: t2 projection=[x]+physical_plan+01)GlobalLimitExec: skip=1, fetch=1+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, x@0)], fetch=2+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)----DataSourceExec: partitions=1, partition_sizes=[1]++query II+SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 1 OFFSET 1;+----+2 2++query TT+EXPLAIN SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 0;+----+logical_plan EmptyRelation: rows=0+physical_plan EmptyExec++query II+SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 0;+----++statement ok+CREATE TABLE t3 (p INT, q VARCHAR) AS VALUES+ (1, 'foo'),+ (2, 'bar'),+ (3, 'baz');++query TT+EXPLAIN SELECT t1.a, t2.x, t3.p+FROM t1+INNER JOIN t2 ON t1.a = t2.x+INNER JOIN t3 ON t2.x = t3.p+LIMIT 2;+----+logical_plan+01)Limit: skip=0, fetch=2+02)--Inner Join: t2.x = t3.p+03)----Inner Join: t1.a = t2.x+04)------TableScan: t1 projection=[a]+05)------TableScan: t2 projection=[x]+06)----TableScan: t3 projection=[p]+physical_plan+01)ProjectionExec: expr=[a@1 as a, x@2 as x, p@0 as p]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p@0, x@1)], fetch=2+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, x@0)]+05)------DataSourceExec: partitions=1, partition_sizes=[1]+06)------DataSourceExec: partitions=1, partition_sizes=[1]++query III+SELECT t1.a, t2.x, t3.p+FROM t1+INNER JOIN t2 ON t1.a = t2.x+INNER JOIN t3 ON t2.x = t3.p+LIMIT 2;+----+1 1 1+2 2 2++# Try larger limit+query TT+EXPLAIN SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 100;+----+logical_plan+01)Limit: skip=0, fetch=100+02)--Inner Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----TableScan: t2 projection=[x]+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, x@0)], fetch=100+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--DataSourceExec: partitions=1, partition_sizes=[1]++query II+SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 100;+----+1 1+2 2+3 3++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++statement ok+DROP TABLE t3;diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex 3edc721afc..2fb544a638 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -4161,10 +4161,9 @@ logical_plan03)----TableScan: t0 projection=[c1, c2]04)----TableScan: t1 projection=[c1, c2, c3]physical_plan-01)GlobalLimitExec: skip=0, fetch=2-02)--HashJoinExec: mode=CollectLeft, join_type=Full, on=[(c1@0, c1@0)]-03)----DataSourceExec: partitions=1, partition_sizes=[2]-04)----DataSourceExec: partitions=1, partition_sizes=[2]+01)HashJoinExec: mode=CollectLeft, join_type=Full, on=[(c1@0, c1@0)], fetch=2+02)--DataSourceExec: partitions=1, partition_sizes=[2]+03)--DataSourceExec: partitions=1, partition_sizes=[2]## Test join.on.is_empty() && join.filter.is_some() -> single filter now a PWMJquery TT@@ -4191,10 +4190,9 @@ logical_plan03)----TableScan: t0 projection=[c1, c2]04)----TableScan: t1 projection=[c1, c2, c3]physical_plan-01)GlobalLimitExec: skip=0, fetch=2-02)--HashJoinExec: mode=CollectLeft, join_type=Full, on=[(c1@0, c1@0)], filter=c2@0 >= c2@1-03)----DataSourceExec: partitions=1, partition_sizes=[2]-04)----DataSourceExec: partitions=1, partition_sizes=[2]+01)HashJoinExec: mode=CollectLeft, join_type=Full, on=[(c1@0, c1@0)], filter=c2@0 >= c2@1, fetch=2+02)--DataSourceExec: partitions=1, partition_sizes=[2]+03)--DataSourceExec: partitions=1, partition_sizes=[2]## Add more test cases for join limit pushdownstatement ok@@ -4245,6 +4243,7 @@ select * from t1 LEFT JOIN t2 ON t1.a = t2.b LIMIT 2;1 1# can only push down to t1 (preserved side)+# limit pushdown supported for left join - both to join and probe sidequery TTexplain select * from t1 LEFT JOIN t2 ON t1.a = t2.b LIMIT 2;----@@ -4255,10 +4254,9 @@ logical_plan04)------TableScan: t1 projection=[a], fetch=205)----TableScan: t2 projection=[b]physical_plan-01)GlobalLimitExec: skip=0, fetch=2-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, b@0)]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], limit=2, file_type=csv, has_header=true-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], file_type=csv, has_header=true+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, b@0)], fetch=2+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], limit=2, file_type=csv, has_header=true+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], file_type=csv, has_header=true######## RIGHT JOIN w/ LIMIT@@ -4289,10 +4287,9 @@ logical_plan04)----Limit: skip=0, fetch=205)------TableScan: t2 projection=[b], fetch=2physical_plan-01)GlobalLimitExec: skip=0, fetch=2-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@0, b@0)]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], file_type=csv, has_header=true-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], limit=2, file_type=csv, has_header=true+01)HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@0, b@0)], fetch=2+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], file_type=csv, has_header=true+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], limit=2, file_type=csv, has_header=true######## FULL JOIN w/ LIMIT@@ -4316,7 +4313,7 @@ select * from t1 FULL JOIN t2 ON t1.a = t2.b LIMIT 2;4 4-# can't push limit for full outer join+# full outer join supports fetch pushdownquery TTexplain select * from t1 FULL JOIN t2 ON t1.a = t2.b LIMIT 2;----@@ -4326,10 +4323,9 @@ logical_plan03)----TableScan: t1 projection=[a]04)----TableScan: t2 projection=[b]physical_plan-01)GlobalLimitExec: skip=0, fetch=2-02)--HashJoinExec: mode=CollectLeft, join_type=Full, on=[(a@0, b@0)]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], file_type=csv, has_header=true-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], file_type=csv, has_header=true+01)HashJoinExec: mode=CollectLeft, join_type=Full, on=[(a@0, b@0)], fetch=2+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], file_type=csv, has_header=true+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], file_type=csv, has_header=truestatement okdrop table t1;