Apache DataFusion SQL Query Engine
test: Fix flaky window `slt` by adding `rowsort` (#26764)
Yongting You5a1c67bd68de
diff --git a/datafusion/sqllogictest/test_files/array_agg_sliding_window.slt b/datafusion/sqllogictest/test_files/array_agg_sliding_window.sltindex c828794b1d..32144bbdd8 100644--- a/datafusion/sqllogictest/test_files/array_agg_sliding_window.slt+++ b/datafusion/sqllogictest/test_files/array_agg_sliding_window.slt@@ -20,54 +20,57 @@# Validates the retract_batch implementation on ArrayAggAccumulator.#######+# Note: `ORDER BY` in window SQL controls frame evaluation, not output order.+# Use rowsort to compare results independently of output row order.+# Setup test datastatement okCREATE TABLE t(ts INT, val TEXT) AS VALUES(1, 'A'), (2, 'B'), (3, 'C'), (4, 'D'), (5, 'E');# Basic ROWS sliding window-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)FROM t;-----[A][A, B]+[A][B, C][C, D][D, E]# Wider ROWS frame-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)FROM t;-----[A]-[A, B][A, B, C]+[A, B]+[A][B, C, D][C, D, E]# ROWS with FOLLOWING-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)FROM t;-----[A, B][A, B, C]+[A, B][B, C, D][C, D, E][D, E]# Regression: unbounded frame still works-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)FROM t;-----[A]-[A, B]-[A, B, C]-[A, B, C, D][A, B, C, D, E]+[A, B, C, D]+[A, B, C]+[A, B]+[A]# Setup data with NULLsstatement ok@@ -75,14 +78,14 @@ CREATE TABLE t_nulls(ts INT, val TEXT) AS VALUES(1, 'A'), (2, NULL), (3, 'C'), (4, NULL), (5, 'E');# Sliding window with NULLs (nulls preserved)-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)FROM t_nulls;-----[A][A, NULL]-[NULL, C]+[A][C, NULL]+[NULL, C][NULL, E]# Setup data with value gaps for RANGE frame@@ -91,18 +94,18 @@ CREATE TABLE t_range(ts INT, val TEXT) AS VALUES(1, 'A'), (2, 'B'), (3, 'C'), (4, 'D'), (100, 'E');# RANGE frame with value gap causes multi-element retract-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING)FROM t_range;-----[A, B, C][A, B, C, D][A, B, C, D]+[A, B, C][B, C, D][E]# Single-row frame-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts ROWS BETWEEN CURRENT ROW AND CURRENT ROW)FROM t;----@@ -117,12 +120,12 @@ statement okCREATE TABLE t_int(ts INT, val INT) AS VALUES(1, 10), (2, 20), (3, 30), (4, 40), (5, 50);-query ?+query ? rowsortSELECT array_agg(val) OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)FROM t_int;-----[10][10, 20]+[10][20, 30][30, 40][40, 50]@@ -135,19 +138,19 @@ CREATE TABLE t_groups(ts INT, val TEXT) AS VALUES# GROUPS frame: each "group" = rows with same ORDER BY value# Group 0: ts=1 (A,B), Group 1: ts=2 (C,D), Group 2: ts=3 (E)# Frame: 1 PRECEDING group + CURRENT group-query ?-SELECT array_agg(val) OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW)+query ? rowsort+SELECT array_sort(array_agg(val) OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW))FROM t_groups;-----[A, B]-[A, B][A, B, C, D][A, B, C, D]+[A, B]+[A, B][C, D, E]# IGNORE NULLS: nulls are filtered out before storage and retract# t_nulls has: (1,'A'), (2,NULL), (3,'C'), (4,NULL), (5,'E')-query ?+query ? rowsortSELECT array_agg(val) IGNORE NULLS OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)FROM t_nulls;----@@ -158,15 +161,15 @@ FROM t_nulls;[E]# IGNORE NULLS with wider frame-query ?+query ? rowsortSELECT array_agg(val) IGNORE NULLS OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)FROM t_nulls;----+[A, C][A][A]-[A, C]-[C][C, E]+[C]######## DISTINCT sliding window tests@@ -187,71 +190,71 @@ CREATE TABLE t_dist(ts INT, val TEXT) AS VALUES# [A,A,B] -> {A,B}# [A,B,C] -> {A,B,C} (one A retracted, one A remains)# [B,C,B] -> {B,C} (last A retracted, B duplicate stays)-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val)OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW))FROM t_dist;----+[A, B, C]+[A, B][A][A]-[A, B]-[A, B, C][B, C]# Narrower ROWS frame-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val)OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW))FROM t_dist;----+[A, B][A][A]-[A, B][B, C][B, C]# DESC window ORDER BY: frame walks input in reverse temporal order, so# update/retract are called against the reversed row stream. Validates# retract still tracks duplicates correctly when rows arrive in DESC order.-# Output rows are emitted in ts DESC order (ts=5,4,3,2,1).+# The frame follows ts DESC order (ts=5,4,3,2,1).# ts=5 (B): frame [B] -> {B}# ts=4 (C): frame [B,C] -> {B,C} (1 preceding in DESC = ts=5)# ts=3 (B): frame [C,B] -> {B,C}# ts=2 (A): frame [B,A] -> {A,B}# ts=1 (A): frame [A,A] -> {A} (duplicate A in frame)-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val)OVER (ORDER BY ts DESC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW))FROM t_dist;-----[B]-[B, C]-[B, C][A, B][A]+[B, C]+[B, C]+[B]# RANGE frame with value gaps -> multi-row retract on shiftstatement okCREATE TABLE t_dist_range(ts INT, val TEXT) AS VALUES(1,'A'),(2,'A'),(3,'B'),(10,'A'),(11,'C');-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val)OVER (ORDER BY ts RANGE BETWEEN 2 PRECEDING AND CURRENT ROW))FROM t_dist_range;----+[A, B]+[A, C][A][A]-[A, B][A]-[A, C]# DISTINCT + IGNORE NULLS in sliding frame: nulls never enter state.statement okCREATE TABLE t_dist_nulls(ts INT, val TEXT) AS VALUES(1,'A'),(2,NULL),(3,'A'),(4,NULL),(5,'B');-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val) IGNORE NULLSOVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW))FROM t_dist_nulls;@@ -272,7 +275,7 @@ FROM t_dist_nulls;# ts=4 (NULL): frame [A,NULL] -> {A,NULL} sorted [NULL, A]# (the ts=2 NULL retracts but the ts=4 NULL is still present)# ts=5 (B): frame [NULL,B] -> {B,NULL} sorted [NULL, B]-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val)OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW))FROM t_dist_nulls;@@ -290,17 +293,17 @@ statement okCREATE TABLE t_dist_groups(ts INT, val TEXT) AS VALUES(1,'A'),(1,'A'),(2,'B'),(2,'C'),(3,'A'),(3,'D');-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val)OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW))FROM t_dist_groups;-----[A]-[A]-[A, B, C]-[A, B, C][A, B, C, D][A, B, C, D]+[A, B, C]+[A, B, C]+[A]+[A]# PARTITION BY: each partition retracts against its own state only. A leak of# one partition's state into the next would surface as the next partition's@@ -339,15 +342,15 @@ statement okCREATE TABLE t_dist_int(ts INT, val INT) AS VALUES(1,10),(2,10),(3,20),(4,30),(5,20);-query ?+query ? rowsortSELECT array_sort(array_agg(DISTINCT val)OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW))FROM t_dist_int;----+[10, 20, 30]+[10, 20][10][10]-[10, 20]-[10, 20, 30][20, 30]# ORDER BY interaction — window context.