Skip to content

Apache DataFusion SQL Query Engine

fix: allow free RANGE window frames over Duration and Interval ORDER BY types (#24916)

Eduardo Alcantara0e292dcbfdab
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index 767a77e3bb..bf5767c3ca 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -1091,7 +1091,11 @@ fn coerce_frame_bound(
}
}
-fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {
+/// The type that RANGE frame offsets are coerced to for an ORDER BY column of
+/// `col_type`, or `None` if there is no offset type to coerce to (the column
+/// type has no arithmetic). `None` does not mean the type is unusable in a
+/// RANGE frame: a free frame has no offsets, see `supports_free_range_frame`.
+fn extract_window_frame_target_type(col_type: &DataType) -> Option<DataType> {
if col_type.is_numeric()
|| col_type.is_string()
|| col_type.is_binary()
@@ -1106,18 +1110,80 @@ fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {
| DataType::Time64(_)
)
{
- Ok(col_type.clone())
+ Some(col_type.clone())
} else if is_datetime(col_type) {
- Ok(DataType::Interval(IntervalUnit::MonthDayNano))
+ Some(DataType::Interval(IntervalUnit::MonthDayNano))
} else if let DataType::Dictionary(_, value_type) = col_type {
extract_window_frame_target_type(value_type)
} else if let DataType::RunEndEncoded(_, value_type) = col_type {
extract_window_frame_target_type(value_type.data_type())
} else {
- internal_err!("Cannot run range queries on datatype: {col_type}")
+ None
}
}
+/// Whether a free RANGE frame (all bounds `UNBOUNDED` or `CURRENT ROW`) can
+/// run over an ORDER BY column of `col_type` even though the type has no
+/// arithmetic for finite offsets.
+///
+/// Such a frame only compares rows to find peers, so the type must compare
+/// the same way in the RANGE peer check (`ScalarValue::partial_cmp`) as in
+/// the sort that produced the input order. That holds for durations and
+/// intervals; it does not for structs and maps, whose `ScalarValue`
+/// comparison differs from the sorter's, so they stay unsupported.
+fn supports_free_range_frame(col_type: &DataType) -> bool {
+ match col_type {
+ DataType::Duration(_) | DataType::Interval(_) => true,
+ DataType::Dictionary(_, value_type) => supports_free_range_frame(value_type),
+ DataType::RunEndEncoded(_, value_type) => {
+ supports_free_range_frame(value_type.data_type())
+ }
+ _ => false,
+ }
+}
+
+/// Whether `col_type` is a list, possibly behind dictionary or run-end
+/// encoding.
+///
+/// Lists are kept out of the free-range fallback because their peer
+/// comparison and the sort do not agree on element NULLs: `compare_rows`
+/// applies the NULLS FIRST / NULLS LAST option to the top-level value only and
+/// then calls `ScalarValue::partial_cmp`, whose `partial_cmp_list` always
+/// orders a NULL element after a non-NULL one (Postgres semantics), while the
+/// sorter's `make_comparator` applies the option to the elements as well. Under
+/// `ORDER BY d, l NULLS FIRST` with tied `d`, the sort puts `[NULL]` before
+/// `[1]` and the peer check orders them the other way round.
+fn is_list_type(col_type: &DataType) -> bool {
+ match col_type {
+ DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => {
+ true
+ }
+ DataType::Dictionary(_, value_type) => is_list_type(value_type),
+ DataType::RunEndEncoded(_, value_type) => is_list_type(value_type.data_type()),
+ _ => false,
+ }
+}
+
+/// Errors if any ORDER BY expression has a type not supported in a free RANGE
+/// frame: a type with neither an offset target nor a sound peer comparison, or
+/// a list (see `is_list_type`).
+fn check_free_range_order_by_types(
+ expressions: &[Sort],
+ schema: &DFSchema,
+) -> Result<()> {
+ for sort in expressions {
+ let t = sort.expr.get_type(schema)?;
+ let supported = supports_free_range_frame(&t)
+ || (extract_window_frame_target_type(&t).is_some() && !is_list_type(&t));
+ if !supported {
+ return plan_err!(
+ "RANGE window frames are not supported for ORDER BY type {t}"
+ );
+ }
+ }
+ Ok(())
+}
+
// Coerces the given `window_frame` to use appropriate natural types.
// For example, ROWS and GROUPS frames use `UInt64` during calculations.
fn coerce_window_frame(
@@ -1133,7 +1199,21 @@ fn coerce_window_frame(
.map(|s| s.expr.get_type(schema))
.transpose()?;
if let Some(col_type) = current_types {
- let target_type = extract_window_frame_target_type(&col_type)?;
+ let target_type = match extract_window_frame_target_type(&col_type) {
+ Some(target_type) => target_type,
+ // A free range frame has no offsets to coerce, so ORDER BY
+ // types without arithmetic are fine as long as their peer
+ // comparison is sound (see `supports_free_range_frame`).
+ None if window_frame.free_range() => {
+ check_free_range_order_by_types(expressions, schema)?;
+ return Ok(window_frame);
+ }
+ None => {
+ return plan_err!(
+ "RANGE window frames are not supported for ORDER BY type {col_type}"
+ );
+ }
+ };
// A finite offset bound (e.g. `5 PRECEDING`) is computed as
// `current_value ± offset`, so it is only meaningful for target
// types that support arithmetic. Other orderable target types can
diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt
index b0eac5528a..af12fad670 100644
--- a/datafusion/sqllogictest/test_files/window.slt
+++ b/datafusion/sqllogictest/test_files/window.slt
@@ -7018,8 +7018,8 @@ query error DataFusion error: type_coercion\ncaused by\nError during planning: R
SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND INTERVAL '1' HOUR FOLLOWING)
FROM (VALUES (arrow_cast('01:00:00', 'Time64(Microsecond)'))) t(x)
-# Unsupported RANGE ORDER BY types still propagate the type-coercion error.
-query error DataFusion error: type_coercion\ncaused by\nInternal error: Cannot run range queries on datatype: Struct\("c0": Int64\)
+# Unsupported RANGE ORDER BY types are rejected during planning.
+query error Error during planning: RANGE window frames are not supported for ORDER BY type Struct
SELECT COUNT(*) OVER (ORDER BY x)
FROM (VALUES (struct(1))) t(x)
@@ -7094,6 +7094,84 @@ SELECT last_value() OVER () FROM (VALUES (1), (2), (3)) t(x)
statement error DataFusion error: Error during planning: 'nth_value' does not support zero arguments
SELECT nth_value() OVER () FROM (VALUES (1), (2), (3)) t(x)
+# The default frame of `OVER (ORDER BY ...)` is a free RANGE frame, which
+# needs no offset arithmetic on the ORDER BY type. Durations and intervals
+# used to fail with "Internal error: Cannot run range queries on datatype".
+query ?I
+select x, row_number() over (order by x) from (values (arrow_cast(2, 'Duration(Second)')), (arrow_cast(1, 'Duration(Second)'))) t(x) order by 2
+----
+0 days 0 hours 0 mins 1 secs 1
+0 days 0 hours 0 mins 2 secs 2
+
+query ?I
+select x, rank() over (order by x) from (values (interval '2 months'), (interval '1 month')) t(x) order by 2
+----
+1 mons 1
+2 mons 2
+
+query ?I
+select x, row_number() over (order by x range between unbounded preceding and current row) from (values (arrow_cast(2, 'Duration(Millisecond)')), (arrow_cast(1, 'Duration(Millisecond)'))) t(x) order by 2
+----
+0 days 0 hours 0 mins 0.001 secs 1
+0 days 0 hours 0 mins 0.002 secs 2
+
+# offset frames still need an arithmetic ORDER BY type; the map and struct
+# types are not accepted in free frames either, see `supports_free_range_frame`
+statement error Error during planning: RANGE window frames are not supported for ORDER BY type Struct
+select count(*) over (order by x range between 1 preceding and current row) from (values (struct(1))) t(x)
+
+statement error Error during planning: RANGE window frames are not supported for ORDER BY type Map
+select count(*) over (order by x) from (values (map(['a'], [1])), (map(['a'], [2]))) t(x)
+
+# every ORDER BY expression takes part in the peer comparison, so a map after
+# a duration is rejected as well, while two durations are fine; the error names
+# the offending type, not the first ORDER BY type
+statement error Error during planning: RANGE window frames are not supported for ORDER BY type Map
+select count(*) over (order by d, m) from (values (arrow_cast(1, 'Duration(Second)'), map(['a'], [1])), (arrow_cast(1, 'Duration(Second)'), map(['a'], [2]))) t(d, m)
+
+query ??I
+select d, e, count(*) over (order by d, e) from (values (arrow_cast(1, 'Duration(Second)'), arrow_cast(2, 'Duration(Millisecond)')), (arrow_cast(1, 'Duration(Second)'), arrow_cast(1, 'Duration(Millisecond)'))) t(d, e) order by 3
+----
+0 days 0 hours 0 mins 1 secs 0 days 0 hours 0 mins 0.001 secs 1
+0 days 0 hours 0 mins 1 secs 0 days 0 hours 0 mins 0.002 secs 2
+
+# List ORDER BY keys were already accepted in RANGE frames before the free-range
+# fallback existed (`extract_window_frame_target_type` returns the list type
+# itself), and this PR does not change that. A list of primitives works:
+query I
+select count(*) over (order by l) from (values ([1]), ([2])) t(l)
+----
+1
+2
+
+# but a list whose elements are nested (struct, list) plans and then fails at
+# execution: the peer comparison in the RANGE frame goes through
+# `ScalarValue::partial_cmp`, which cannot compare nested list elements, while
+# the sort that produced the input uses `make_comparator` and can. Documented
+# here so the fix to the scalar comparison can flip these expectations.
+query error DataFusion error: Internal error: Uncomparable values: List\(\[\{c0: 1\}\]\), List\(\[\{c0: 1\}\]\)
+select count(*) over (order by l) from (values ([struct(1)]), ([struct(2)])) t(l)
+
+query error DataFusion error: Internal error: Uncomparable values: List\(\[\[1\]\]\), List\(\[\[1\]\]\)
+select count(*) over (order by l) from (values ([[1]]), ([[2]])) t(l)
+
+# after a duration key the free-range check looks at every ORDER BY type, and
+# a list is rejected at planning even though it sorts fine: the RANGE peer
+# check (`compare_rows`, then `ScalarValue::partial_cmp`) always orders a NULL
+# list element after a non-NULL one, while the sort applies NULLS FIRST to the
+# elements as well, so with tied durations `[NULL]` and `[1]` would be sorted
+# one way and compared the other
+query error Error during planning: RANGE window frames are not supported for ORDER BY type List
+select count(*) over (order by d, l nulls first) from (values (arrow_cast(1, 'Duration(Second)'), [NULL]), (arrow_cast(1, 'Duration(Second)'), [1])) t(d, l)
+
+# the check is on the type, not the values: a list without NULLs and a
+# dictionary-encoded list are rejected the same way
+query error Error during planning: RANGE window frames are not supported for ORDER BY type List
+select count(*) over (order by d, l) from (values (arrow_cast(1, 'Duration(Second)'), [1]), (arrow_cast(1, 'Duration(Second)'), [2])) t(d, l)
+
+query error Error during planning: RANGE window frames are not supported for ORDER BY type Dictionary\(Int32, List
+select count(*) over (order by d, l) from (values (arrow_cast(1, 'Duration(Second)'), arrow_cast([1], 'Dictionary(Int32, List(Int64))')), (arrow_cast(1, 'Duration(Second)'), arrow_cast([2], 'Dictionary(Int32, List(Int64))'))) t(d, l)
+
# Window functions are not allowed in WHERE or HAVING: both are evaluated before
# window functions are computed, so they are rejected during planning
# issue: https://github.com/apache/datafusion/issues/4610