Apache DataFusion SQL Query Engine
fix: The limit_pushdown physical optimization rule removes limits in some cases leading to incorrect results (#20048)
Mason2860adaf4b79
diff --git a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs b/datafusion/core/tests/physical_optimizer/limit_pushdown.rsindex e8d06d69df..b30092159f 100644--- a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs@@ -26,7 +26,7 @@ use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;-use datafusion_expr::Operator;+use datafusion_expr::{JoinType, Operator};use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::expressions::{BinaryExpr, col, lit};use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};@@ -34,6 +34,7 @@ use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::filter::FilterExec;+use datafusion_physical_plan::joins::NestedLoopJoinExec;use datafusion_physical_plan::projection::ProjectionExec;use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::{ExecutionPlan, get_plan_string};@@ -87,6 +88,16 @@ fn empty_exec(schema: SchemaRef) -> Arc<dyn ExecutionPlan> {Arc::new(EmptyExec::new(schema))}+fn nested_loop_join_exec(+ left: Arc<dyn ExecutionPlan>,+ right: Arc<dyn ExecutionPlan>,+ join_type: JoinType,+) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(NestedLoopJoinExec::try_new(+ left, right, None, &join_type, None,+ )?))+}+#[test]fn transforms_streaming_table_exec_into_fetching_version_when_skip_is_zero() -> Result<()>{@@ -343,3 +354,104 @@ fn merges_local_limit_with_global_limit() -> Result<()> {Ok(())}++#[test]+fn preserves_nested_global_limit() -> Result<()> {+ // If there are multiple limits in an execution plan, they all need to be+ // preserved in the optimized plan.+ //+ // Plan structure:+ // GlobalLimitExec: skip=1, fetch=1+ // NestedLoopJoinExec (Left)+ // EmptyExec (left side)+ // GlobalLimitExec: skip=2, fetch=1+ // NestedLoopJoinExec (Right)+ // EmptyExec (left side)+ // EmptyExec (right side)+ let schema = create_schema();++ // Build inner join: NestedLoopJoin(Empty, Empty)+ let inner_left = empty_exec(Arc::clone(&schema));+ let inner_right = empty_exec(Arc::clone(&schema));+ let inner_join = nested_loop_join_exec(inner_left, inner_right, JoinType::Right)?;++ // Add inner limit: GlobalLimitExec: skip=2, fetch=1+ let inner_limit = global_limit_exec(inner_join, 2, Some(1));++ // Build outer join: NestedLoopJoin(Empty, GlobalLimit)+ let outer_left = empty_exec(Arc::clone(&schema));+ let outer_join = nested_loop_join_exec(outer_left, inner_limit, JoinType::Left)?;++ // Add outer limit: GlobalLimitExec: skip=1, fetch=1+ let outer_limit = global_limit_exec(outer_join, 1, Some(1));++ let initial = get_plan_string(&outer_limit);+ let expected_initial = [+ "GlobalLimitExec: skip=1, fetch=1",+ " NestedLoopJoinExec: join_type=Left",+ " EmptyExec",+ " GlobalLimitExec: skip=2, fetch=1",+ " NestedLoopJoinExec: join_type=Right",+ " EmptyExec",+ " EmptyExec",+ ];+ assert_eq!(initial, expected_initial);++ let after_optimize =+ LimitPushdown::new().optimize(outer_limit, &ConfigOptions::new())?;+ let expected = [+ "GlobalLimitExec: skip=1, fetch=1",+ " NestedLoopJoinExec: join_type=Left",+ " EmptyExec",+ " GlobalLimitExec: skip=2, fetch=1",+ " NestedLoopJoinExec: join_type=Right",+ " EmptyExec",+ " EmptyExec",+ ];+ assert_eq!(get_plan_string(&after_optimize), expected);++ Ok(())+}++#[test]+fn preserves_skip_before_sort() -> Result<()> {+ // If there's a limit with skip before a node that (1) supports fetch but+ // (2) does not support limit pushdown, that limit should not be removed.+ //+ // Plan structure:+ // GlobalLimitExec: skip=1, fetch=None+ // SortExec: TopK(fetch=4)+ // EmptyExec+ let schema = create_schema();++ let empty = empty_exec(Arc::clone(&schema));++ let ordering = [PhysicalSortExpr {+ expr: col("c1", &schema)?,+ options: SortOptions::default(),+ }];+ let sort = sort_exec(ordering.into(), empty)+ .with_fetch(Some(4))+ .unwrap();++ let outer_limit = global_limit_exec(sort, 1, None);++ let initial = get_plan_string(&outer_limit);+ let expected_initial = [+ "GlobalLimitExec: skip=1, fetch=None",+ " SortExec: TopK(fetch=4), expr=[c1@0 ASC], preserve_partitioning=[false]",+ " EmptyExec",+ ];+ assert_eq!(initial, expected_initial);++ let after_optimize =+ LimitPushdown::new().optimize(outer_limit, &ConfigOptions::new())?;+ let expected = [+ "GlobalLimitExec: skip=1, fetch=3",+ " SortExec: TopK(fetch=4), expr=[c1@0 ASC], preserve_partitioning=[false]",+ " EmptyExec",+ ];+ assert_eq!(get_plan_string(&after_optimize), expected);++ Ok(())+}diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rsindex a4dac81dba..e7bede494d 100644--- a/datafusion/physical-optimizer/src/limit_pushdown.rs+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs@@ -155,6 +155,7 @@ pub fn pushdown_limit_helper(global_state.skip = skip;global_state.fetch = fetch;global_state.preserve_order = limit_exec.preserve_order();+ global_state.satisfied = false;// Now the global state has the most recent information, we can remove// the `LimitExec` plan. We will decide later if we should add it again@@ -172,7 +173,7 @@ pub fn pushdown_limit_helper(// If we have a non-limit operator with fetch capability, update global// state as necessary:if pushdown_plan.fetch().is_some() {- if global_state.fetch.is_none() {+ if global_state.skip == 0 {global_state.satisfied = true;}(global_state.skip, global_state.fetch) = combine_limit(diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.sltindex 96471411e0..429181a2d3 100644--- a/datafusion/sqllogictest/test_files/limit.slt+++ b/datafusion/sqllogictest/test_files/limit.slt@@ -706,8 +706,8 @@ ON t1.b = t2.bORDER BY t1.b desc, c desc, c2 descOFFSET 3 LIMIT 2;-----3 99 82-3 99 79+3 98 79+3 97 96statement okdrop table ordered_table;diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.sltindex b79b6d2fe5..d858d0ae3e 100644--- a/datafusion/sqllogictest/test_files/union.slt+++ b/datafusion/sqllogictest/test_files/union.slt@@ -494,22 +494,25 @@ physical_plan01)CoalescePartitionsExec: fetch=302)--UnionExec03)----ProjectionExec: expr=[count(Int64(1))@0 as cnt]-04)------AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]-05)--------CoalescePartitionsExec-06)----------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]-07)------------ProjectionExec: expr=[]-08)--------------AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[]-09)----------------RepartitionExec: partitioning=Hash([c1@0], 4), input_partitions=4-10)------------------AggregateExec: mode=Partial, gby=[c1@0 as c1], aggr=[]-11)--------------------FilterExec: c13@1 != C2GT5KVyOPZpgKVl110TyZO0NcJ434, projection=[c1@0]-12)----------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-13)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c1, c13], file_type=csv, has_header=true-14)----ProjectionExec: expr=[1 as cnt]-15)------PlaceholderRowExec-16)----ProjectionExec: expr=[lead(b.c1,Int64(1)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as cnt]-17)------BoundedWindowAggExec: wdw=[lead(b.c1,Int64(1)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Field { "lead(b.c1,Int64(1)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING], mode=[Sorted]-18)--------ProjectionExec: expr=[1 as c1]-19)----------PlaceholderRowExec+04)------GlobalLimitExec: skip=0, fetch=3+05)--------AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]+06)----------CoalescePartitionsExec+07)------------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]+08)--------------ProjectionExec: expr=[]+09)----------------AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[]+10)------------------RepartitionExec: partitioning=Hash([c1@0], 4), input_partitions=4+11)--------------------AggregateExec: mode=Partial, gby=[c1@0 as c1], aggr=[]+12)----------------------FilterExec: c13@1 != C2GT5KVyOPZpgKVl110TyZO0NcJ434, projection=[c1@0]+13)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+14)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c1, c13], file_type=csv, has_header=true+15)----ProjectionExec: expr=[1 as cnt]+16)------GlobalLimitExec: skip=0, fetch=3+17)--------PlaceholderRowExec+18)----ProjectionExec: expr=[lead(b.c1,Int64(1)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as cnt]+19)------GlobalLimitExec: skip=0, fetch=3+20)--------BoundedWindowAggExec: wdw=[lead(b.c1,Int64(1)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Field { "lead(b.c1,Int64(1)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING], mode=[Sorted]+21)----------ProjectionExec: expr=[1 as c1]+22)------------PlaceholderRowExec########