Apache DataFusion SQL Query Engine
fix: wait for the whole NULL peer group before emitting RANGE `N PRECEDING` window results (#25860)
Kita Renji3470bd0fbc01
diff --git a/datafusion/expr/src/window_frame.rs b/datafusion/expr/src/window_frame.rsindex 018ae8216d..68a722eaf6 100644--- a/datafusion/expr/src/window_frame.rs+++ b/datafusion/expr/src/window_frame.rs@@ -224,7 +224,13 @@ impl WindowFrame {}_ => true,},- WindowFrameUnits::Range | WindowFrameUnits::Groups => match &end_bound {+ // For a row whose ORDER BY value is NULL or NaN, an offset bound+ // ends the RANGE frame at its last peer, which may be a later row.+ WindowFrameUnits::Range => matches!(+ &end_bound,+ WindowFrameBound::Preceding(value) if value.is_null()+ ),+ WindowFrameUnits::Groups => match &end_bound {WindowFrameBound::Preceding(value) => {if value.is_null() {// Unbounded precedingdiff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rsindex 690731f4a9..5f6bbc46c2 100644--- a/datafusion/physical-expr/src/window/window_expr.rs+++ b/datafusion/physical-expr/src/window/window_expr.rs@@ -507,15 +507,10 @@ fn is_end_bound_safe_for_range(idx: usize,) -> Result<bool> {match end_bound {- WindowFrameBound::Preceding(value) => {- let zero = ScalarValue::new_zero(&value.data_type())?;- if value.eq(&zero) {- is_row_ahead(orderby_col, most_recent_ob_col, sort_options)- } else {- Ok(true)- }- }- WindowFrameBound::CurrentRow => {+ // A frame ending in `N PRECEDING` can only reach the end of the buffer+ // when the current row's frame ends at its last peer, as for a NULL or+ // NaN ORDER BY value, so it is safe once a row past its peers arrives.+ WindowFrameBound::Preceding(_) | WindowFrameBound::CurrentRow => {is_row_ahead(orderby_col, most_recent_ob_col, sort_options)}WindowFrameBound::Following(delta) => {diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex 254fdf20e5..7c92514e72 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -7345,3 +7345,80 @@ SELECT a FROM window_in_filter_t LIMIT count(*);statement okDROP TABLE window_in_filter_t;++# RANGE frames ending in `N PRECEDING`: for a row whose ORDER BY value is NULL+# the frame is its NULL peer group, so the result must not depend on how the+# NULL peers are split across input batches.+statement ok+CREATE TABLE range_null_peers(id INT, x INT, v INT) AS VALUES+ (1, 1, 10), (2, 2, 20), (3, NULL, 30), (4, NULL, 40), (5, NULL, 50);++statement ok+set datafusion.execution.batch_size = 2;++query IIIII+SELECT id,+ sum(v) OVER (ORDER BY x RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),+ count(*) OVER (ORDER BY x RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),+ max(v) OVER (ORDER BY x RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),+ sum(v) OVER (ORDER BY x RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)+FROM range_null_peers ORDER BY id;+----+1 NULL 0 NULL NULL+2 10 1 10 10+3 120 3 50 150+4 120 3 50 150+5 120 3 50 150++query III+SELECT id,+ sum(v) OVER (ORDER BY x NULLS FIRST RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING),+ sum(v) OVER (ORDER BY x DESC RANGE BETWEEN 5 PRECEDING AND 1 PRECEDING)+FROM range_null_peers ORDER BY id;+----+1 NULL 20+2 10 NULL+3 120 120+4 120 120+5 120 120++# The FOLLOWING frames are reversed to `5 PRECEDING AND 1 PRECEDING` so they can+# reuse the sort of the first window+query IIII+SELECT id,+ count(*) OVER (ORDER BY x DESC NULLS LAST, id),+ sum(v) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN 1 FOLLOWING AND 5 FOLLOWING),+ nth_value(1, 3) OVER (ORDER BY x ASC NULLS FIRST RANGE BETWEEN 1 FOLLOWING AND 5 FOLLOWING)+FROM range_null_peers ORDER BY id;+----+1 2 20 NULL+2 1 NULL NULL+3 3 120 1+4 4 120 1+5 5 120 1++# NaN sorts as a single peer group too, and `NaN - 1` is NaN+statement ok+CREATE TABLE range_nan_peers(id INT, x DOUBLE) AS VALUES+ (1, 1.0), (2, 'NaN'), (3, 'NaN'), (4, 'NaN'), (5, 2.0);++query III+SELECT id,+ count(*) OVER (ORDER BY x RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING),+ count(*) OVER (ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)+FROM range_nan_peers ORDER BY id;+----+1 0 4+2 5 3+3 5 3+4 5 3+5 1 3++statement ok+reset datafusion.execution.batch_size;++statement ok+DROP TABLE range_null_peers;++statement ok+DROP TABLE range_nan_peers;