Apache DataFusion SQL Query Engine
Allow Range partitioned inputs to PartitionedTopK (#23355)
Stu Hoode123e6fc7b04
diff --git a/datafusion/physical-optimizer/src/window_topn.rs b/datafusion/physical-optimizer/src/window_topn.rsindex 3f88e86c67..c668608ca2 100644--- a/datafusion/physical-optimizer/src/window_topn.rs+++ b/datafusion/physical-optimizer/src/window_topn.rs@@ -61,6 +61,7 @@ use datafusion_physical_expr::window::StandardWindowExpr;use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::projection::ProjectionExec;+use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::sorts::partitioned_topk::{PartitionedTopKExec, WindowFnKind,};@@ -140,24 +141,25 @@ impl WindowTopN {// Step 2: Extract limit from predicate (rn <= K, rn < K, etc.)let (col_idx, limit_n) = extract_window_limit(filter.predicate())?;- // Step 3: Walk through optional ProjectionExec to find BoundedWindowAggExec+ // Step 3: Walk through optional ProjectionExec and RepartitionExec to find BoundedWindowAggExeclet child = filter.input();- let (window_exec, proj_between) = find_window_below(child)?;+ let (window_exec, intermediates) = find_window_below(child)?;// Step 4: Verify col_idx references a supported window function output column- let input_field_count = window_exec.input().schema().fields().len();+ let window_exec_typed = window_exec.downcast_ref::<BoundedWindowAggExec>()?;+ let sort_exec = window_exec_typed.input().downcast_ref::<SortExec>()?;+ let input_field_count = window_exec_typed.input().schema().fields().len();if col_idx < input_field_count {return None; // Filter is on an input column, not a window column}let window_expr_idx = col_idx - input_field_count;- let window_exprs = window_exec.window_expr();+ let window_exprs = window_exec_typed.window_expr();if window_expr_idx >= window_exprs.len() {return None;}let fn_kind = supported_window_fn(&window_exprs[window_expr_idx])?;- // Step 5: Verify child of window is SortExec- let sort_exec = window_exec.input().downcast_ref::<SortExec>()?;+ // Step 5: child of window is SortExec (verified above)let sort_child = sort_exec.input();// Step 6: Determine partition_prefix_len from the window expression@@ -190,28 +192,19 @@ impl WindowTopN {.ok()?;// Step 8: Rebuild window with new child- let new_window = Arc::clone(&child_as_arc(window_exec))+ let mut result = window_exec.with_new_children(vec![Arc::new(partitioned_topk)]).ok()?;- // Step 9: If ProjectionExec was between Filter and Window, rebuild it- let result = match proj_between {- Some(proj) => Arc::clone(&child_as_arc(proj))- .with_new_children(vec![new_window])- .ok()?,- None => new_window,- };+ // Step 9: Rebuild intermediate nodes (ProjectionExec/RepartitionExec)+ for node in intermediates.into_iter().rev() {+ result = node.with_new_children(vec![result]).ok()?;+ }Some(result)}}-/// Helper to get an `Arc<dyn ExecutionPlan>` from a reference.-/// We need this because `with_new_children` takes `Arc<Self>`.-fn child_as_arc<T: ExecutionPlan + Clone + 'static>(plan: &T) -> Arc<dyn ExecutionPlan> {- Arc::new(plan.clone())-}-impl PhysicalOptimizerRule for WindowTopN {fn optimize(&self,@@ -336,29 +329,32 @@ fn supported_window_fn(}}+type PlanAndIntermediates = (Arc<dyn ExecutionPlan>, Vec<Arc<dyn ExecutionPlan>>);+/// Walk below a plan node looking for a [`BoundedWindowAggExec`].///-/// Handles two cases:-/// - Direct child: `FilterExec → BoundedWindowAggExec`-/// - With projection: `FilterExec → ProjectionExec → BoundedWindowAggExec`+/// Handles sequences of `ProjectionExec` and `RepartitionExec`.+/// This is safe because `PartitionedTopKExec` can be pushed below them:+/// projections only provide aliases, and pushing the limit below repartitions+/// is safe because the limit is computed per-partition.///-/// Returns the window exec and an optional `ProjectionExec` in between,-/// or `None` if no `BoundedWindowAggExec` is found within one or two levels.-fn find_window_below(- plan: &Arc<dyn ExecutionPlan>,-) -> Option<(&BoundedWindowAggExec, Option<&ProjectionExec>)> {- // Direct child is BoundedWindowAggExec- if let Some(window) = plan.downcast_ref::<BoundedWindowAggExec>() {- return Some((window, None));- }-- // Child is ProjectionExec with BoundedWindowAggExec below- if let Some(proj) = plan.downcast_ref::<ProjectionExec>() {- let proj_child = proj.input();- if let Some(window) = proj_child.downcast_ref::<BoundedWindowAggExec>() {- return Some((window, Some(proj)));+/// Returns the window exec and a list of intermediate nodes to rebuild,+/// or `None` if no `BoundedWindowAggExec` is found.+fn find_window_below(plan: &Arc<dyn ExecutionPlan>) -> Option<PlanAndIntermediates> {+ let mut current = Arc::clone(plan);+ let mut intermediates = Vec::new();++ loop {+ if current.downcast_ref::<BoundedWindowAggExec>().is_some() {+ return Some((current, intermediates));+ } else if current.downcast_ref::<ProjectionExec>().is_some()+ || current.downcast_ref::<RepartitionExec>().is_some()+ {+ let next = Arc::clone(current.children().first()?);+ intermediates.push(current);+ current = next;+ } else {+ return None;}}-- None}diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rsindex 730440a429..c250130341 100644--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs@@ -355,6 +355,7 @@ impl ExecutionPlan for PartitionedTopKExec {crate::InputDistributionRequirements::new(vec![Distribution::KeyPartitioned(partition_exprs,)])+ .allow_range_satisfaction_for_key_partitioning()}fn maintains_input_order(&self) -> Vec<bool> {diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex 2cf3b87e3c..6c456fe363 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -907,5 +907,162 @@ SELECT range_key, SUM(value) OVER (ORDER BY value) FROM range_partitioned ORDER30 106035 1410+++##########+# TEST 26: PartitionedTopK on Range Partition Column+# Exact Range([range_key]) satisfies the TopK partition key and avoids repartitioning.+##########++statement ok+set datafusion.optimizer.enable_window_topn = true;++statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT * FROM (+ SELECT range_key, value, ROW_NUMBER() OVER (PARTITION BY range_key ORDER BY value DESC) as rn+ FROM range_partitioned+) WHERE rn <= 1;+----+physical_plan+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rn]+02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0], order=[value@1 DESC]+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++query III+SELECT * FROM (+ SELECT range_key, value, ROW_NUMBER() OVER (PARTITION BY range_key ORDER BY value DESC) as rn+ FROM range_partitioned+) WHERE rn <= 1+ORDER BY range_key;+----+1 10 1+5 50 1+10 100 1+15 150 1+20 200 1+25 250 1+30 300 1+35 350 1+++##########+# TEST 27: PartitionedTopK on Non-Range Column+# Partitioning on a non-range key cannot reuse Range([range_key]) and+# requires hash repartitioning.+##########++statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT * FROM (+ SELECT non_range_key, value, ROW_NUMBER() OVER (PARTITION BY non_range_key ORDER BY value DESC) as rn+ FROM range_partitioned+) WHERE rn <= 1;+----+physical_plan+01)ProjectionExec: expr=[non_range_key@0 as non_range_key, value@1 as value, row_number() PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rn]+02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[non_range_key@0], order=[value@1 DESC]+04)------RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false++query III+SELECT * FROM (+ SELECT non_range_key, value, ROW_NUMBER() OVER (PARTITION BY non_range_key ORDER BY value DESC) as rn+ FROM range_partitioned+) WHERE rn <= 1+ORDER BY non_range_key;+----+1 300 1+2 350 1+++##########+# TEST 28: PartitionedTopK Reuses Range Subset Partitioning+# With subset threshold met and preserve-file disabled, Range([range_key])+# satisfies partitioning by (range_key, non_range_key).+##########++statement ok+set datafusion.optimizer.subset_repartition_threshold = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT * FROM (+ SELECT range_key, non_range_key, value, ROW_NUMBER() OVER (PARTITION BY range_key, non_range_key ORDER BY value DESC) as rn+ FROM range_partitioned+) WHERE rn <= 1;+----+physical_plan+01)ProjectionExec: expr=[range_key@0 as range_key, non_range_key@1 as non_range_key, value@2 as value, row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn]+02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0, non_range_key@1], order=[value@2 DESC]+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++query IIII+SELECT * FROM (+ SELECT range_key, non_range_key, value, ROW_NUMBER() OVER (PARTITION BY range_key, non_range_key ORDER BY value DESC) as rn+ FROM range_partitioned+) WHERE rn <= 1+ORDER BY range_key, non_range_key;+----+1 1 10 1+5 2 50 1+10 1 100 1+15 2 150 1+20 1 200 1+25 2 250 1+30 1 300 1+35 2 350 1+++##########+# TEST 29: Range Subset PartitionedTopK Rehashes Below Subset Threshold+# Range([range_key]) is only a subset of PARTITION BY (range_key, non_range_key),+# so it should not satisfy the TopK partition key when subset satisfaction is+# disabled.+##########++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+set datafusion.optimizer.subset_repartition_threshold = 5;++statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TT+EXPLAIN SELECT * FROM (+ SELECT range_key, non_range_key, value, ROW_NUMBER() OVER (PARTITION BY range_key, non_range_key ORDER BY value DESC) as rn+ FROM range_partitioned+) WHERE rn <= 1;+----+physical_plan+01)ProjectionExec: expr=[range_key@0 as range_key, non_range_key@1 as non_range_key, value@2 as value, row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn]+02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0, non_range_key@1], order=[value@2 DESC]+04)------RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++statement ok+reset datafusion.optimizer.subset_repartition_threshold;+statement okreset datafusion.explain.physical_plan_only;++statement ok+reset datafusion.optimizer.enable_window_topn;diff --git a/datafusion/sqllogictest/test_files/window_topn.slt b/datafusion/sqllogictest/test_files/window_topn.sltindex 2eb72f519b..4dff4a779b 100644--- a/datafusion/sqllogictest/test_files/window_topn.slt+++ b/datafusion/sqllogictest/test_files/window_topn.slt@@ -1044,3 +1044,48 @@ DROP TABLE window_topn_rank_null_t;# Reset config to default (false)statement okSET datafusion.optimizer.enable_window_topn = false;++statement ok+create table t(c1 int, c2 int) as values (1, 2), (3, 4);++statement ok+set datafusion.execution.target_partitions = 5;++statement ok+set datafusion.optimizer.repartition_windows = false;++statement ok+set datafusion.execution.batch_size = 1;++statement ok+set datafusion.optimizer.enable_window_topn = true;++query TT+EXPLAIN SELECT * FROM (+ SELECT c1, c2, ROW_NUMBER() OVER (PARTITION BY c1 ORDER BY c2 DESC) as rn+ FROM t+) WHERE rn <= 1;+----+logical_plan+01)Projection: t.c1, t.c2, row_number() PARTITION BY [t.c1] ORDER BY [t.c2 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rn+02)--Filter: row_number() PARTITION BY [t.c1] ORDER BY [t.c2 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW <= UInt64(1)+03)----WindowAggr: windowExpr=[[row_number() PARTITION BY [t.c1] ORDER BY [t.c2 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]+04)------TableScan: t projection=[c1, c2]+physical_plan+01)ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, row_number() PARTITION BY [t.c1] ORDER BY [t.c2 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rn]+02)--RepartitionExec: partitioning=RoundRobinBatch(5), input_partitions=1, maintains_sort_order=true+03)----BoundedWindowAggExec: wdw=[row_number() PARTITION BY [t.c1] ORDER BY [t.c2 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [t.c1] ORDER BY [t.c2 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+04)------PartitionedTopKExec: fn=row_number, fetch=1, partition=[c1@0], order=[c2@1 DESC]+05)--------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+set datafusion.optimizer.repartition_windows = true;++statement ok+set datafusion.execution.batch_size = 8192;++statement ok+set datafusion.optimizer.enable_window_topn = false;