Skip to content

Apache DataFusion SQL Query Engine

fix: impl `handle_child_pushdown_result` for `SortExec` (#21527)

Huaijin3b5008a47c8a
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
index 4fa7b637ca..4ff1fad8f5 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
@@ -3046,3 +3046,331 @@ async fn test_discover_dynamic_filters_via_expressions_api() {
"After optimization, should discover exactly 2 dynamic filters (1 in HashJoinExec, 1 in DataSourceExec), found {count_after}"
);
}
+
+// ==== Filter pushdown through SortExec tests ====
+
+/// FilterExec above a plain SortExec (no fetch) should be pushed below it.
+/// The scan supports pushdown, so the filter lands in the DataSourceExec.
+#[test]
+fn test_filter_pushdown_through_sort_into_scan() {
+ let scan = TestScanBuilder::new(schema()).with_support(true).build();
+ let sort = Arc::new(SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ ));
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(FilterExec::try_new(predicate, sort).unwrap());
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ output:
+ Ok:
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ "
+ );
+}
+
+/// FilterExec above a plain SortExec (no fetch) when the scan does NOT
+/// support pushdown. The filter should still move below the sort, landing
+/// as a new FilterExec between SortExec and DataSourceExec.
+#[test]
+fn test_filter_pushdown_through_sort_no_scan_support() {
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let sort = Arc::new(SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ ));
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(FilterExec::try_new(predicate, sort).unwrap());
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - FilterExec: a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+/// Multiple conjunctive filters above a plain SortExec should all be
+/// pushed below the sort as a single FilterExec.
+#[test]
+fn test_multiple_filters_pushdown_through_sort() {
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let sort = Arc::new(SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ ));
+ // WHERE a = 'foo' AND b = 'bar'
+ let predicate = Arc::new(BinaryExpr::new(
+ col_lit_predicate("a", "foo", &schema()),
+ Operator::And,
+ col_lit_predicate("b", "bar", &schema()),
+ )) as Arc<dyn PhysicalExpr>;
+ let plan = Arc::new(FilterExec::try_new(predicate, sort).unwrap());
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo AND b@1 = bar
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - FilterExec: a@0 = foo AND b@1 = bar
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+/// FilterExec above a SortExec with fetch (TopK) must NOT be pushed below,
+/// because limiting happens after filtering — changing the order would alter
+/// the result set.
+#[test]
+fn test_filter_not_pushed_through_sort_with_fetch() {
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let sort = Arc::new(
+ SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ )
+ .with_fetch(Some(10)),
+ );
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(FilterExec::try_new(predicate, sort).unwrap());
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo
+ - SortExec: TopK(fetch=10), expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - FilterExec: a@0 = foo
+ - SortExec: TopK(fetch=10), expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+/// FilterExec above a SortExec with fetch (TopK) must NOT be pushed below,
+/// because limiting happens after filtering — changing the order would alter
+/// the result set.
+#[test]
+fn test_filter_pushed_through_sort_with_fetch() {
+ let scan = TestScanBuilder::new(schema()).with_support(true).build();
+ let sort = Arc::new(
+ SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ )
+ .with_fetch(Some(10)),
+ );
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(FilterExec::try_new(predicate, sort).unwrap());
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo
+ - SortExec: TopK(fetch=10), expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ output:
+ Ok:
+ - FilterExec: a@0 = foo
+ - SortExec: TopK(fetch=10), expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ "
+ );
+}
+
+/// FilterExec with a projection above SortExec. The filter should be pushed
+/// below the sort, and the projection should be preserved as a
+/// ProjectionExec on top.
+#[test]
+fn test_filter_with_projection_pushdown_through_sort() {
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let sort = Arc::new(SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ ));
+ // FilterExec: b = 'bar', projection=[a] (only output column a)
+ let predicate = col_lit_predicate("b", "bar", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, sort)
+ .apply_projection(Some(vec![0]))
+ .unwrap()
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: b@1 = bar, projection=[a@0]
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - ProjectionExec: expr=[a@0 as a]
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - FilterExec: b@1 = bar
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+/// SortExec with preserve_partitioning=true should keep that setting after
+/// filters are pushed below it.
+#[test]
+fn test_filter_pushdown_through_sort_preserves_partitioning() {
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let sort = Arc::new(
+ SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ )
+ .with_preserve_partitioning(true),
+ );
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(FilterExec::try_new(predicate, sort).unwrap());
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
+ - FilterExec: a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+/// FilterExec **with a fetch limit** above a plain SortExec. When the filter
+/// is pushed below the sort the fetch should be propagated to the SortExec
+/// (turning it into a TopK).
+#[test]
+fn test_filter_with_fetch_pushdown_through_sort() {
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let sort = Arc::new(SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new_default(
+ col("a", &schema()).unwrap(),
+ )])
+ .unwrap(),
+ scan,
+ ));
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, sort)
+ .with_fetch(Some(10))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo, fetch=10
+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - SortExec: TopK(fetch=10), expr=[a@0 ASC], preserve_partitioning=[false]
+ - FilterExec: a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+#[test]
+fn test_filter_pushdown_through_sort_with_projection() {
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let sort = Arc::new(SortExec::new(
+ LexOrdering::new(vec![PhysicalSortExpr::new(
+ col("a", &schema()).unwrap(),
+ SortOptions::new(true, false), // descending, nulls_last
+ )])
+ .unwrap(),
+ scan,
+ ));
+ // FilterExec: b = 'bar', projection=[a] (only output column a)
+ let predicate = col_lit_predicate("b", "bar", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, sort)
+ .apply_projection(Some(vec![0]))
+ .unwrap()
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), false),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: b@1 = bar, projection=[a@0]
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - ProjectionExec: expr=[a@0 as a]
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - FilterExec: b@1 = bar
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index 583bfa29b0..9afe92ba0a 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -30,8 +30,10 @@ use crate::execution_plan::{
Boundedness, CardinalityEffect, EmissionType, has_same_children_properties,
};
use crate::expressions::PhysicalSortExpr;
+use crate::filter::FilterExec;
use crate::filter_pushdown::{
- ChildFilterDescription, FilterDescription, FilterPushdownPhase,
+ ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,
+ FilterPushdownPropagation, PushedDown,
};
use crate::limit::LimitStream;
use crate::metrics::{
@@ -1424,6 +1426,58 @@ impl ExecutionPlan for SortExec {
Ok(FilterDescription::new().with_child(child))
}
+
+ fn handle_child_pushdown_result(
+ &self,
+ _phase: FilterPushdownPhase,
+ child_pushdown_result: ChildPushdownResult,
+ _config: &datafusion_common::config::ConfigOptions,
+ ) -> Result<FilterPushdownPropagation<Arc<dyn ExecutionPlan>>> {
+ // For a plain sort (no fetch) we intercept any unsupported filters
+ // by inserting a FilterExec below this Sort. Moving the filter below
+ // Sort is safe because Sort preserves all rows.
+ //
+ // Why not fetch (TopK)?
+ // A sort with fetch limits the number of output rows. Inserting a
+ // FilterExec *below* the TopK would change semantics. A filter *above*
+ // the TopK is supposed to post-filter its output (e.g. "take the top 10
+ // rows, then keep only those with a > 5"). Pushing the filter below
+ // Sort changes the meaning to "filter first, then take top 10", which
+ // produces a different result.
+ if self.fetch.is_some() {
+ return Ok(FilterPushdownPropagation::if_all(child_pushdown_result));
+ }
+
+ // Collect parent filters that were NOT successfully pushed to our child.
+ let unsupported_filters: Vec<Arc<dyn PhysicalExpr>> = child_pushdown_result
+ .parent_filters
+ .iter()
+ .filter(|&f| matches!(f.all(), PushedDown::No))
+ .map(|f| Arc::clone(&f.filter))
+ .collect();
+
+ if unsupported_filters.is_empty() {
+ // All filters were pushed — nothing extra to do.
+ return Ok(FilterPushdownPropagation::if_all(child_pushdown_result));
+ }
+
+ // Build a single conjunctive predicate from the unsupported filters
+ // and insert a FilterExec between this SortExec and its child.
+ let predicate = datafusion_physical_expr::conjunction(unsupported_filters);
+ let new_child =
+ Arc::new(FilterExec::try_new(predicate, Arc::clone(self.input()))?)
+ as Arc<dyn ExecutionPlan>;
+ let new_sort = Arc::new(
+ SortExec::new(self.expr.clone(), new_child)
+ .with_fetch(self.fetch())
+ .with_preserve_partitioning(self.preserve_partitioning()),
+ ) as Arc<dyn ExecutionPlan>;
+
+ Ok(FilterPushdownPropagation {
+ filters: vec![PushedDown::Yes; child_pushdown_result.parent_filters.len()],
+ updated_node: Some(new_sort),
+ })
+ }
}
#[cfg(test)]
diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
index 0a30118493..179a611d37 100644
--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
@@ -316,6 +316,56 @@ WHERE r.id IN (SELECT l.id FROM left_parquet l);
3 right3
5 right5
+# LEFT SEMI JOIN with ORDER BY: the join preserves probe-side ordering, so the
+# sort is pushed below the join. The join-generated dynamic filter should then
+# be pushed through the SortExec into the parquet scan predicate.
+query TT
+EXPLAIN SELECT l.*
+FROM left_parquet l
+WHERE l.id IN (SELECT r.id FROM right_parquet r)
+ORDER BY l.data DESC;
+----
+logical_plan
+01)Sort: l.data DESC NULLS FIRST
+02)--LeftSemi Join: l.id = __correlated_sq_1.id
+03)----SubqueryAlias: l
+04)------TableScan: left_parquet projection=[id, data]
+05)----SubqueryAlias: __correlated_sq_1
+06)------SubqueryAlias: r
+07)--------TableScan: right_parquet projection=[id]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(id@0, id@0)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet
+03)--SortExec: expr=[data@1 DESC], preserve_partitioning=[false]
+04)----FilterExec: DynamicFilter [ empty ]
+05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]
+
+statement count 0
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+query TT
+EXPLAIN SELECT l.*
+FROM left_parquet l
+WHERE l.id IN (SELECT r.id FROM right_parquet r)
+ORDER BY l.data DESC;
+----
+logical_plan
+01)Sort: l.data DESC NULLS FIRST
+02)--LeftSemi Join: l.id = __correlated_sq_1.id
+03)----SubqueryAlias: l
+04)------TableScan: left_parquet projection=[id, data]
+05)----SubqueryAlias: __correlated_sq_1
+06)------SubqueryAlias: r
+07)--------TableScan: right_parquet projection=[id]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(id@0, id@0)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet
+03)--SortExec: expr=[data@1 DESC], preserve_partitioning=[false]
+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]
+
+statement count 0
+RESET datafusion.execution.parquet.pushdown_filters;
+
# LEFT ANTI JOIN: both self generated and parent filters can push to the
# preserved (left/build) side.
query TT
diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt
index 1ed3fc89b2..c6c8f9b6bc 100644
--- a/datafusion/sqllogictest/test_files/joins.slt
+++ b/datafusion/sqllogictest/test_files/joins.slt
@@ -2865,8 +2865,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query IT rowsort
SELECT t1_id, t1_name FROM left_semi_anti_join_table_t1 t1 WHERE t1_id IN (SELECT t2_id FROM left_semi_anti_join_table_t2 t2) ORDER BY t1_id
@@ -2900,8 +2901,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query IT
SELECT t1_id, t1_name FROM left_semi_anti_join_table_t1 t1 LEFT SEMI JOIN left_semi_anti_join_table_t2 t2 ON (t1_id = t2_id) ORDER BY t1_id
@@ -2956,8 +2958,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query IT rowsort
SELECT t1_id, t1_name FROM left_semi_anti_join_table_t1 t1 WHERE t1_id IN (SELECT t2_id FROM left_semi_anti_join_table_t2 t2) ORDER BY t1_id
@@ -2991,8 +2994,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query IT
SELECT t1_id, t1_name FROM left_semi_anti_join_table_t1 t1 LEFT SEMI JOIN left_semi_anti_join_table_t2 t2 ON (t1_id = t2_id) ORDER BY t1_id
@@ -3048,8 +3052,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)], filter=t2_name@1 != t1_name@0
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query ITI rowsort
SELECT t1_id, t1_name, t1_int FROM right_semi_anti_join_table_t1 t1 WHERE EXISTS (SELECT * FROM right_semi_anti_join_table_t2 t2 where t2.t2_id = t1.t1_id and t2.t2_name <> t1.t1_name) ORDER BY t1_id
@@ -3064,8 +3069,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)], filter=t2_name@0 != t1_name@1
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query ITI rowsort
SELECT t1_id, t1_name, t1_int FROM right_semi_anti_join_table_t2 t2 RIGHT SEMI JOIN right_semi_anti_join_table_t1 t1 on (t2.t2_id = t1.t1_id and t2.t2_name <> t1.t1_name) ORDER BY t1_id
@@ -3118,8 +3124,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)], filter=t2_name@1 != t1_name@0
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query ITI rowsort
SELECT t1_id, t1_name, t1_int FROM right_semi_anti_join_table_t1 t1 WHERE EXISTS (SELECT * FROM right_semi_anti_join_table_t2 t2 where t2.t2_id = t1.t1_id and t2.t2_name <> t1.t1_name) ORDER BY t1_id
@@ -3134,8 +3141,9 @@ physical_plan
02)--HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(t2_id@0, t1_id@0)], filter=t2_name@0 != t1_name@1
03)----DataSourceExec: partitions=1, partition_sizes=[1]
04)----SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]
-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------FilterExec: DynamicFilter [ empty ]
+06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
+07)----------DataSourceExec: partitions=1, partition_sizes=[1]
query ITI rowsort
SELECT t1_id, t1_name, t1_int FROM right_semi_anti_join_table_t2 t2 RIGHT SEMI JOIN right_semi_anti_join_table_t1 t1 on (t2.t2_id = t1.t1_id and t2.t2_name <> t1.t1_name) ORDER BY t1_id
@@ -4141,7 +4149,8 @@ physical_plan
01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(b@1, y@1)], filter=a@0 < x@1
02)--DataSourceExec: partitions=1, partition_sizes=[0]
03)--SortExec: expr=[x@0 ASC NULLS LAST], preserve_partitioning=[false]
-04)----DataSourceExec: partitions=1, partition_sizes=[0]
+04)----FilterExec: DynamicFilter [ empty ]
+05)------DataSourceExec: partitions=1, partition_sizes=[0]
# Test full join with limit
statement ok
@@ -4443,7 +4452,8 @@ physical_plan
04)------FilterExec: b@1 > 3, projection=[a@0]
05)--------DataSourceExec: partitions=2, partition_sizes=[1, 1]
06)----SortExec: expr=[c@2 DESC], preserve_partitioning=[true]
-07)------DataSourceExec: partitions=2, partition_sizes=[1, 1]
+07)------FilterExec: DynamicFilter [ empty ]
+08)--------DataSourceExec: partitions=2, partition_sizes=[1, 1]
query TT
explain select * from test where a in (select a from test where b > 3) order by c desc nulls last;
@@ -4463,7 +4473,8 @@ physical_plan
04)------FilterExec: b@1 > 3, projection=[a@0]
05)--------DataSourceExec: partitions=2, partition_sizes=[1, 1]
06)----SortExec: expr=[c@2 DESC NULLS LAST], preserve_partitioning=[true]
-07)------DataSourceExec: partitions=2, partition_sizes=[1, 1]
+07)------FilterExec: DynamicFilter [ empty ]
+08)--------DataSourceExec: partitions=2, partition_sizes=[1, 1]
query III
select * from test where a in (select a from test where b > 3) order by c desc nulls first;