Apache DataFusion SQL Query Engine
fix: saturate window frame lookahead instead of overflowing on huge FOLLOWING bounds (#26662)
ump45nose6286f5e6053a
diff --git a/datafusion/expr/src/window_state.rs b/datafusion/expr/src/window_state.rsindex 514fdc839c..f301c91253 100644--- a/datafusion/expr/src/window_state.rs+++ b/datafusion/expr/src/window_state.rs@@ -263,8 +263,9 @@ impl WindowFrameContext {"Frame start cannot be UNBOUNDED FOLLOWING '{window_frame:?}'");}+ // `n` comes from the query and can be as large as `u64::MAX`WindowFrameBound::Following(ScalarValue::UInt64(Some(n))) => {- std::cmp::min(idx + n as usize, length)+ std::cmp::min(idx.saturating_add(n as usize), length)}// ERRONEOUS FRAMESWindowFrameBound::Preceding(_) | WindowFrameBound::Following(_) => {@@ -289,7 +290,7 @@ impl WindowFrameContext {// UNBOUNDED FOLLOWINGWindowFrameBound::Following(ScalarValue::UInt64(None)) => length,WindowFrameBound::Following(ScalarValue::UInt64(Some(n))) => {- std::cmp::min(idx + n as usize + 1, length)+ std::cmp::min(idx.saturating_add(n as usize).saturating_add(1), length)}// ERRONEOUS FRAMESWindowFrameBound::Preceding(_) | WindowFrameBound::Following(_) => {diff --git a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rsindex 092570b051..67cf7ef91a 100644--- a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs+++ b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs@@ -177,7 +177,10 @@ fn grow_limit(window: &BoundedWindowAggExec, ctx: &mut TraverseState) -> bool {}// finish grow- ctx.max_lookahead(ctx.lookahead + max_rel);+ // Frame bounds come straight from the query and can be as large as+ // `u64::MAX`, so saturate instead of overflowing. A saturated lookahead+ // makes the `checked_add` in `apply_limit` fail, which skips the pushdown.+ ctx.max_lookahead(ctx.lookahead.saturating_add(max_rel));true}@@ -194,16 +197,26 @@ fn apply_limit(ctx.lookahead = 0;return Some(Transformed::no(Arc::clone(node)));};+ // A fetch that overflows can't drop any rows, so skip the pushdown rather+ // than turning the sort into a TopK that holds every row and can't spill.+ let Some(fetch) = fetch.checked_add(ctx.lookahead) else {+ ctx.lookahead = 0;+ return Some(Transformed::no(Arc::clone(node)));+ };let fetch = match node.fetch() {- None => fetch + ctx.lookahead,- Some(existing) => cmp::min(existing, fetch + ctx.lookahead),+ None => fetch,+ Some(existing) => cmp::min(existing, fetch),};Some(Transformed::complete(node.with_fetch(Some(fetch)).unwrap()))}fn get_limit(node: &Arc<dyn ExecutionPlan>, ctx: &mut TraverseState) -> bool {if let Some(limit) = node.downcast_ref::<GlobalLimitExec>() {- ctx.reset_limit(limit.fetch().map(|fetch| fetch + limit.skip()));+ ctx.reset_limit(+ limit+ .fetch()+ .map(|fetch| fetch.saturating_add(limit.skip())),+ );return true;}// In distributed execution, GlobalLimitExec becomes LocalLimitExecdiff --git a/datafusion/sqllogictest/test_files/window_limits.slt b/datafusion/sqllogictest/test_files/window_limits.sltindex 11c96904f4..078d30805f 100644--- a/datafusion/sqllogictest/test_files/window_limits.slt+++ b/datafusion/sqllogictest/test_files/window_limits.slt@@ -862,3 +862,51 @@ physical_plan03)----BoundedWindowAggExec: wdw=[lead(employees.salary,Int64(2)) ORDER BY [employees.empno ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "lead(employees.salary,Int64(2)) ORDER BY [employees.empno ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable UInt64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]04)------SortExec: TopK(fetch=5), expr=[empno@0 ASC NULLS LAST], preserve_partitioning=[false]05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100_with_dates.csv]]}, projection=[empno, salary], file_type=csv, has_header=true++# A huge FOLLOWING bound must not overflow when the limit is grown by the+# window's lookahead (https://github.com/apache/datafusion/issues/26658)+statement ok+CREATE TABLE huge_frame_t(a INT, x INT) AS VALUES (1, 1), (2, 2), (3, 3), (4, 4), (5, 5);++query II+SELECT a, sum(x) OVER (ORDER BY a ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING) s+FROM huge_frame_t ORDER BY a LIMIT 2;+----+1 15+2 14++# Same, with a window function that adds its own lookahead on top of the frame+query II+SELECT a, lead(x, 1) OVER (ORDER BY a ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING) l+FROM huge_frame_t ORDER BY a LIMIT 2;+----+1 2+2 3++# A huge FOLLOWING start bound must not overflow either+query II+SELECT a, sum(x) OVER (ORDER BY a ROWS BETWEEN 18446744073709551615 FOLLOWING AND 18446744073709551615 FOLLOWING) s+FROM huge_frame_t ORDER BY a LIMIT 2;+----+1 NULL+2 NULL++# The pushdown is skipped, so the sort stays a full sort instead of a TopK+query TT+EXPLAIN SELECT a, sum(x) OVER (ORDER BY a ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING) s+FROM huge_frame_t ORDER BY a LIMIT 2;+----+logical_plan+01)Sort: huge_frame_t.a ASC NULLS LAST, fetch=2+02)--Projection: huge_frame_t.a, sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING AS s+03)----WindowAggr: windowExpr=[[sum(CAST(huge_frame_t.x AS Int64)) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING]]+04)------TableScan: huge_frame_t projection=[a, x]+physical_plan+01)ProjectionExec: expr=[a@0 as a, sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING@2 as s]+02)--GlobalLimitExec: skip=0, fetch=2+03)----BoundedWindowAggExec: wdw=[sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING: Field { "sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING": nullable Int64 }, frame: ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING], mode=[Sorted]+04)------SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+05)--------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+DROP TABLE huge_frame_t;