Skip to content

Apache DataFusion SQL Query Engine

fix: saturate window frame lookahead instead of overflowing on huge FOLLOWING bounds (#26662)

ump45nose6286f5e6053a
diff --git a/datafusion/expr/src/window_state.rs b/datafusion/expr/src/window_state.rs
index 514fdc839c..f301c91253 100644
--- a/datafusion/expr/src/window_state.rs
+++ b/datafusion/expr/src/window_state.rs
@@ -263,8 +263,9 @@ impl WindowFrameContext {
"Frame start cannot be UNBOUNDED FOLLOWING '{window_frame:?}'"
);
}
+ // `n` comes from the query and can be as large as `u64::MAX`
WindowFrameBound::Following(ScalarValue::UInt64(Some(n))) => {
- std::cmp::min(idx + n as usize, length)
+ std::cmp::min(idx.saturating_add(n as usize), length)
}
// ERRONEOUS FRAMES
WindowFrameBound::Preceding(_) | WindowFrameBound::Following(_) => {
@@ -289,7 +290,7 @@ impl WindowFrameContext {
// UNBOUNDED FOLLOWING
WindowFrameBound::Following(ScalarValue::UInt64(None)) => length,
WindowFrameBound::Following(ScalarValue::UInt64(Some(n))) => {
- std::cmp::min(idx + n as usize + 1, length)
+ std::cmp::min(idx.saturating_add(n as usize).saturating_add(1), length)
}
// ERRONEOUS FRAMES
WindowFrameBound::Preceding(_) | WindowFrameBound::Following(_) => {
diff --git a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
index 092570b051..67cf7ef91a 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
@@ -177,7 +177,10 @@ fn grow_limit(window: &BoundedWindowAggExec, ctx: &mut TraverseState) -> bool {
}
// finish grow
- ctx.max_lookahead(ctx.lookahead + max_rel);
+ // Frame bounds come straight from the query and can be as large as
+ // `u64::MAX`, so saturate instead of overflowing. A saturated lookahead
+ // makes the `checked_add` in `apply_limit` fail, which skips the pushdown.
+ ctx.max_lookahead(ctx.lookahead.saturating_add(max_rel));
true
}
@@ -194,16 +197,26 @@ fn apply_limit(
ctx.lookahead = 0;
return Some(Transformed::no(Arc::clone(node)));
};
+ // A fetch that overflows can't drop any rows, so skip the pushdown rather
+ // than turning the sort into a TopK that holds every row and can't spill.
+ let Some(fetch) = fetch.checked_add(ctx.lookahead) else {
+ ctx.lookahead = 0;
+ return Some(Transformed::no(Arc::clone(node)));
+ };
let fetch = match node.fetch() {
- None => fetch + ctx.lookahead,
- Some(existing) => cmp::min(existing, fetch + ctx.lookahead),
+ None => fetch,
+ Some(existing) => cmp::min(existing, fetch),
};
Some(Transformed::complete(node.with_fetch(Some(fetch)).unwrap()))
}
fn get_limit(node: &Arc<dyn ExecutionPlan>, ctx: &mut TraverseState) -> bool {
if let Some(limit) = node.downcast_ref::<GlobalLimitExec>() {
- ctx.reset_limit(limit.fetch().map(|fetch| fetch + limit.skip()));
+ ctx.reset_limit(
+ limit
+ .fetch()
+ .map(|fetch| fetch.saturating_add(limit.skip())),
+ );
return true;
}
// In distributed execution, GlobalLimitExec becomes LocalLimitExec
diff --git a/datafusion/sqllogictest/test_files/window_limits.slt b/datafusion/sqllogictest/test_files/window_limits.slt
index 11c96904f4..078d30805f 100644
--- a/datafusion/sqllogictest/test_files/window_limits.slt
+++ b/datafusion/sqllogictest/test_files/window_limits.slt
@@ -862,3 +862,51 @@ physical_plan
03)----BoundedWindowAggExec: wdw=[lead(employees.salary,Int64(2)) ORDER BY [employees.empno ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "lead(employees.salary,Int64(2)) ORDER BY [employees.empno ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable UInt64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
04)------SortExec: TopK(fetch=5), expr=[empno@0 ASC NULLS LAST], preserve_partitioning=[false]
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100_with_dates.csv]]}, projection=[empno, salary], file_type=csv, has_header=true
+
+# A huge FOLLOWING bound must not overflow when the limit is grown by the
+# window's lookahead (https://github.com/apache/datafusion/issues/26658)
+statement ok
+CREATE TABLE huge_frame_t(a INT, x INT) AS VALUES (1, 1), (2, 2), (3, 3), (4, 4), (5, 5);
+
+query II
+SELECT a, sum(x) OVER (ORDER BY a ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING) s
+FROM huge_frame_t ORDER BY a LIMIT 2;
+----
+1 15
+2 14
+
+# Same, with a window function that adds its own lookahead on top of the frame
+query II
+SELECT a, lead(x, 1) OVER (ORDER BY a ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING) l
+FROM huge_frame_t ORDER BY a LIMIT 2;
+----
+1 2
+2 3
+
+# A huge FOLLOWING start bound must not overflow either
+query II
+SELECT a, sum(x) OVER (ORDER BY a ROWS BETWEEN 18446744073709551615 FOLLOWING AND 18446744073709551615 FOLLOWING) s
+FROM huge_frame_t ORDER BY a LIMIT 2;
+----
+1 NULL
+2 NULL
+
+# The pushdown is skipped, so the sort stays a full sort instead of a TopK
+query TT
+EXPLAIN SELECT a, sum(x) OVER (ORDER BY a ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING) s
+FROM huge_frame_t ORDER BY a LIMIT 2;
+----
+logical_plan
+01)Sort: huge_frame_t.a ASC NULLS LAST, fetch=2
+02)--Projection: huge_frame_t.a, sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING AS s
+03)----WindowAggr: windowExpr=[[sum(CAST(huge_frame_t.x AS Int64)) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING]]
+04)------TableScan: huge_frame_t projection=[a, x]
+physical_plan
+01)ProjectionExec: expr=[a@0 as a, sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING@2 as s]
+02)--GlobalLimitExec: skip=0, fetch=2
+03)----BoundedWindowAggExec: wdw=[sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING: Field { "sum(huge_frame_t.x) ORDER BY [huge_frame_t.a ASC NULLS LAST] ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING": nullable Int64 }, frame: ROWS BETWEEN CURRENT ROW AND 18446744073709551615 FOLLOWING], mode=[Sorted]
+04)------SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+DROP TABLE huge_frame_t;