Apache DataFusion SQL Query Engine
fix: allow free RANGE window frames over Duration and Interval ORDER BY types (#24916)
Eduardo Alcantara0e292dcbfdab
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 767a77e3bb..bf5767c3ca 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -1091,7 +1091,11 @@ fn coerce_frame_bound(}}-fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {+/// The type that RANGE frame offsets are coerced to for an ORDER BY column of+/// `col_type`, or `None` if there is no offset type to coerce to (the column+/// type has no arithmetic). `None` does not mean the type is unusable in a+/// RANGE frame: a free frame has no offsets, see `supports_free_range_frame`.+fn extract_window_frame_target_type(col_type: &DataType) -> Option<DataType> {if col_type.is_numeric()|| col_type.is_string()|| col_type.is_binary()@@ -1106,18 +1110,80 @@ fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {| DataType::Time64(_)){- Ok(col_type.clone())+ Some(col_type.clone())} else if is_datetime(col_type) {- Ok(DataType::Interval(IntervalUnit::MonthDayNano))+ Some(DataType::Interval(IntervalUnit::MonthDayNano))} else if let DataType::Dictionary(_, value_type) = col_type {extract_window_frame_target_type(value_type)} else if let DataType::RunEndEncoded(_, value_type) = col_type {extract_window_frame_target_type(value_type.data_type())} else {- internal_err!("Cannot run range queries on datatype: {col_type}")+ None}}+/// Whether a free RANGE frame (all bounds `UNBOUNDED` or `CURRENT ROW`) can+/// run over an ORDER BY column of `col_type` even though the type has no+/// arithmetic for finite offsets.+///+/// Such a frame only compares rows to find peers, so the type must compare+/// the same way in the RANGE peer check (`ScalarValue::partial_cmp`) as in+/// the sort that produced the input order. That holds for durations and+/// intervals; it does not for structs and maps, whose `ScalarValue`+/// comparison differs from the sorter's, so they stay unsupported.+fn supports_free_range_frame(col_type: &DataType) -> bool {+ match col_type {+ DataType::Duration(_) | DataType::Interval(_) => true,+ DataType::Dictionary(_, value_type) => supports_free_range_frame(value_type),+ DataType::RunEndEncoded(_, value_type) => {+ supports_free_range_frame(value_type.data_type())+ }+ _ => false,+ }+}++/// Whether `col_type` is a list, possibly behind dictionary or run-end+/// encoding.+///+/// Lists are kept out of the free-range fallback because their peer+/// comparison and the sort do not agree on element NULLs: `compare_rows`+/// applies the NULLS FIRST / NULLS LAST option to the top-level value only and+/// then calls `ScalarValue::partial_cmp`, whose `partial_cmp_list` always+/// orders a NULL element after a non-NULL one (Postgres semantics), while the+/// sorter's `make_comparator` applies the option to the elements as well. Under+/// `ORDER BY d, l NULLS FIRST` with tied `d`, the sort puts `[NULL]` before+/// `[1]` and the peer check orders them the other way round.+fn is_list_type(col_type: &DataType) -> bool {+ match col_type {+ DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => {+ true+ }+ DataType::Dictionary(_, value_type) => is_list_type(value_type),+ DataType::RunEndEncoded(_, value_type) => is_list_type(value_type.data_type()),+ _ => false,+ }+}++/// Errors if any ORDER BY expression has a type not supported in a free RANGE+/// frame: a type with neither an offset target nor a sound peer comparison, or+/// a list (see `is_list_type`).+fn check_free_range_order_by_types(+ expressions: &[Sort],+ schema: &DFSchema,+) -> Result<()> {+ for sort in expressions {+ let t = sort.expr.get_type(schema)?;+ let supported = supports_free_range_frame(&t)+ || (extract_window_frame_target_type(&t).is_some() && !is_list_type(&t));+ if !supported {+ return plan_err!(+ "RANGE window frames are not supported for ORDER BY type {t}"+ );+ }+ }+ Ok(())+}+// Coerces the given `window_frame` to use appropriate natural types.// For example, ROWS and GROUPS frames use `UInt64` during calculations.fn coerce_window_frame(@@ -1133,7 +1199,21 @@ fn coerce_window_frame(.map(|s| s.expr.get_type(schema)).transpose()?;if let Some(col_type) = current_types {- let target_type = extract_window_frame_target_type(&col_type)?;+ let target_type = match extract_window_frame_target_type(&col_type) {+ Some(target_type) => target_type,+ // A free range frame has no offsets to coerce, so ORDER BY+ // types without arithmetic are fine as long as their peer+ // comparison is sound (see `supports_free_range_frame`).+ None if window_frame.free_range() => {+ check_free_range_order_by_types(expressions, schema)?;+ return Ok(window_frame);+ }+ None => {+ return plan_err!(+ "RANGE window frames are not supported for ORDER BY type {col_type}"+ );+ }+ };// A finite offset bound (e.g. `5 PRECEDING`) is computed as// `current_value ± offset`, so it is only meaningful for target// types that support arithmetic. Other orderable target types candiff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex b0eac5528a..af12fad670 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -7018,8 +7018,8 @@ query error DataFusion error: type_coercion\ncaused by\nError during planning: RSELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND INTERVAL '1' HOUR FOLLOWING)FROM (VALUES (arrow_cast('01:00:00', 'Time64(Microsecond)'))) t(x)-# Unsupported RANGE ORDER BY types still propagate the type-coercion error.-query error DataFusion error: type_coercion\ncaused by\nInternal error: Cannot run range queries on datatype: Struct\("c0": Int64\)+# Unsupported RANGE ORDER BY types are rejected during planning.+query error Error during planning: RANGE window frames are not supported for ORDER BY type StructSELECT COUNT(*) OVER (ORDER BY x)FROM (VALUES (struct(1))) t(x)@@ -7094,6 +7094,84 @@ SELECT last_value() OVER () FROM (VALUES (1), (2), (3)) t(x)statement error DataFusion error: Error during planning: 'nth_value' does not support zero argumentsSELECT nth_value() OVER () FROM (VALUES (1), (2), (3)) t(x)+# The default frame of `OVER (ORDER BY ...)` is a free RANGE frame, which+# needs no offset arithmetic on the ORDER BY type. Durations and intervals+# used to fail with "Internal error: Cannot run range queries on datatype".+query ?I+select x, row_number() over (order by x) from (values (arrow_cast(2, 'Duration(Second)')), (arrow_cast(1, 'Duration(Second)'))) t(x) order by 2+----+0 days 0 hours 0 mins 1 secs 1+0 days 0 hours 0 mins 2 secs 2++query ?I+select x, rank() over (order by x) from (values (interval '2 months'), (interval '1 month')) t(x) order by 2+----+1 mons 1+2 mons 2++query ?I+select x, row_number() over (order by x range between unbounded preceding and current row) from (values (arrow_cast(2, 'Duration(Millisecond)')), (arrow_cast(1, 'Duration(Millisecond)'))) t(x) order by 2+----+0 days 0 hours 0 mins 0.001 secs 1+0 days 0 hours 0 mins 0.002 secs 2++# offset frames still need an arithmetic ORDER BY type; the map and struct+# types are not accepted in free frames either, see `supports_free_range_frame`+statement error Error during planning: RANGE window frames are not supported for ORDER BY type Struct+select count(*) over (order by x range between 1 preceding and current row) from (values (struct(1))) t(x)++statement error Error during planning: RANGE window frames are not supported for ORDER BY type Map+select count(*) over (order by x) from (values (map(['a'], [1])), (map(['a'], [2]))) t(x)++# every ORDER BY expression takes part in the peer comparison, so a map after+# a duration is rejected as well, while two durations are fine; the error names+# the offending type, not the first ORDER BY type+statement error Error during planning: RANGE window frames are not supported for ORDER BY type Map+select count(*) over (order by d, m) from (values (arrow_cast(1, 'Duration(Second)'), map(['a'], [1])), (arrow_cast(1, 'Duration(Second)'), map(['a'], [2]))) t(d, m)++query ??I+select d, e, count(*) over (order by d, e) from (values (arrow_cast(1, 'Duration(Second)'), arrow_cast(2, 'Duration(Millisecond)')), (arrow_cast(1, 'Duration(Second)'), arrow_cast(1, 'Duration(Millisecond)'))) t(d, e) order by 3+----+0 days 0 hours 0 mins 1 secs 0 days 0 hours 0 mins 0.001 secs 1+0 days 0 hours 0 mins 1 secs 0 days 0 hours 0 mins 0.002 secs 2++# List ORDER BY keys were already accepted in RANGE frames before the free-range+# fallback existed (`extract_window_frame_target_type` returns the list type+# itself), and this PR does not change that. A list of primitives works:+query I+select count(*) over (order by l) from (values ([1]), ([2])) t(l)+----+1+2++# but a list whose elements are nested (struct, list) plans and then fails at+# execution: the peer comparison in the RANGE frame goes through+# `ScalarValue::partial_cmp`, which cannot compare nested list elements, while+# the sort that produced the input uses `make_comparator` and can. Documented+# here so the fix to the scalar comparison can flip these expectations.+query error DataFusion error: Internal error: Uncomparable values: List\(\[\{c0: 1\}\]\), List\(\[\{c0: 1\}\]\)+select count(*) over (order by l) from (values ([struct(1)]), ([struct(2)])) t(l)++query error DataFusion error: Internal error: Uncomparable values: List\(\[\[1\]\]\), List\(\[\[1\]\]\)+select count(*) over (order by l) from (values ([[1]]), ([[2]])) t(l)++# after a duration key the free-range check looks at every ORDER BY type, and+# a list is rejected at planning even though it sorts fine: the RANGE peer+# check (`compare_rows`, then `ScalarValue::partial_cmp`) always orders a NULL+# list element after a non-NULL one, while the sort applies NULLS FIRST to the+# elements as well, so with tied durations `[NULL]` and `[1]` would be sorted+# one way and compared the other+query error Error during planning: RANGE window frames are not supported for ORDER BY type List+select count(*) over (order by d, l nulls first) from (values (arrow_cast(1, 'Duration(Second)'), [NULL]), (arrow_cast(1, 'Duration(Second)'), [1])) t(d, l)++# the check is on the type, not the values: a list without NULLs and a+# dictionary-encoded list are rejected the same way+query error Error during planning: RANGE window frames are not supported for ORDER BY type List+select count(*) over (order by d, l) from (values (arrow_cast(1, 'Duration(Second)'), [1]), (arrow_cast(1, 'Duration(Second)'), [2])) t(d, l)++query error Error during planning: RANGE window frames are not supported for ORDER BY type Dictionary\(Int32, List+select count(*) over (order by d, l) from (values (arrow_cast(1, 'Duration(Second)'), arrow_cast([1], 'Dictionary(Int32, List(Int64))')), (arrow_cast(1, 'Duration(Second)'), arrow_cast([2], 'Dictionary(Int32, List(Int64))'))) t(d, l)+# Window functions are not allowed in WHERE or HAVING: both are evaluated before# window functions are computed, so they are rejected during planning# issue: https://github.com/apache/datafusion/issues/4610