Skip to content

Apache DataFusion SQL Query Engine

fix: wait for the whole NULL peer group before emitting RANGE `N PRECEDING` window results (#25860)

Kita Renji3470bd0fbc01
diff --git a/datafusion/expr/src/window_frame.rs b/datafusion/expr/src/window_frame.rs
index 018ae8216d..68a722eaf6 100644
--- a/datafusion/expr/src/window_frame.rs
+++ b/datafusion/expr/src/window_frame.rs
@@ -224,7 +224,13 @@ impl WindowFrame {
}
_ => true,
},
- WindowFrameUnits::Range | WindowFrameUnits::Groups => match &end_bound {
+ // For a row whose ORDER BY value is NULL or NaN, an offset bound
+ // ends the RANGE frame at its last peer, which may be a later row.
+ WindowFrameUnits::Range => matches!(
+ &end_bound,
+ WindowFrameBound::Preceding(value) if value.is_null()
+ ),
+ WindowFrameUnits::Groups => match &end_bound {
WindowFrameBound::Preceding(value) => {
if value.is_null() {
// Unbounded preceding
diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs
index 690731f4a9..5f6bbc46c2 100644
--- a/datafusion/physical-expr/src/window/window_expr.rs
+++ b/datafusion/physical-expr/src/window/window_expr.rs
@@ -507,15 +507,10 @@ fn is_end_bound_safe_for_range(
idx: usize,
) -> Result<bool> {
match end_bound {
- WindowFrameBound::Preceding(value) => {
- let zero = ScalarValue::new_zero(&value.data_type())?;
- if value.eq(&zero) {
- is_row_ahead(orderby_col, most_recent_ob_col, sort_options)
- } else {
- Ok(true)
- }
- }
- WindowFrameBound::CurrentRow => {
+ // A frame ending in `N PRECEDING` can only reach the end of the buffer
+ // when the current row's frame ends at its last peer, as for a NULL or
+ // NaN ORDER BY value, so it is safe once a row past its peers arrives.
+ WindowFrameBound::Preceding(_) | WindowFrameBound::CurrentRow => {
is_row_ahead(orderby_col, most_recent_ob_col, sort_options)
}
WindowFrameBound::Following(delta) => {
diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt
index 254fdf20e5..7c92514e72 100644
--- a/datafusion/sqllogictest/test_files/window.slt
+++ b/datafusion/sqllogictest/test_files/window.slt
@@ -7345,3 +7345,80 @@ SELECT a FROM window_in_filter_t LIMIT count(*);
statement ok
DROP TABLE window_in_filter_t;
+
+# RANGE frames ending in `N PRECEDING`: for a row whose ORDER BY value is NULL
+# the frame is its NULL peer group, so the result must not depend on how the
+# NULL peers are split across input batches.
+statement ok
+CREATE TABLE range_null_peers(id INT, x INT, v INT) AS VALUES
+ (1, 1, 10), (2, 2, 20), (3, NULL, 30), (4, NULL, 40), (5, NULL, 50);
+
+statement ok
+set datafusion.execution.batch_size = 2;
+
+query IIIII
+SELECT id,
+ sum(v) OVER (ORDER BY x RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),
+ count(*) OVER (ORDER BY x RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),
+ max(v) OVER (ORDER BY x RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),
+ sum(v) OVER (ORDER BY x RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)
+FROM range_null_peers ORDER BY id;
+----
+1 NULL 0 NULL NULL
+2 10 1 10 10
+3 120 3 50 150
+4 120 3 50 150
+5 120 3 50 150
+
+query III
+SELECT id,
+ sum(v) OVER (ORDER BY x NULLS FIRST RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),
+ sum(v) OVER (ORDER BY x DESC RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING)
+FROM range_null_peers ORDER BY id;
+----
+1 NULL 20
+2 10 NULL
+3 120 120
+4 120 120
+5 120 120
+
+# The FOLLOWING frames are reversed to `5 PRECEDING AND 1 PRECEDING` so they can
+# reuse the sort of the first window
+query IIII
+SELECT id,
+ count(*) OVER (ORDER BY x DESC NULLS LAST, id),
+ sum(v) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN 1 FOLLOWING AND 5 FOLLOWING),
+ nth_value(1, 3) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN 1 FOLLOWING AND 5 FOLLOWING)
+FROM range_null_peers ORDER BY id;
+----
+1 2 20 NULL
+2 1 NULL NULL
+3 3 120 1
+4 4 120 1
+5 5 120 1
+
+# NaN sorts as a single peer group too, and `NaN - 1` is NaN
+statement ok
+CREATE TABLE range_nan_peers(id INT, x DOUBLE) AS VALUES
+ (1, 1.0), (2, 'NaN'), (3, 'NaN'), (4, 'NaN'), (5, 2.0);
+
+query III
+SELECT id,
+ count(*) OVER (ORDER BY x RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING),
+ count(*) OVER (ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)
+FROM range_nan_peers ORDER BY id;
+----
+1 0 4
+2 5 3
+3 5 3
+4 5 3
+5 1 3
+
+statement ok
+reset datafusion.execution.batch_size;
+
+statement ok
+DROP TABLE range_null_peers;
+
+statement ok
+DROP TABLE range_nan_peers;