Skip to content

Apache DataFusion SQL Query Engine

fix: Support RANGE window frames over binary ORDER BY keys (#24357)

Fredrik Fornwallf1f0449a5327
diff --git a/datafusion/expr/src/window_frame.rs b/datafusion/expr/src/window_frame.rs
index a61d9d689a..d9db35875a 100644
--- a/datafusion/expr/src/window_frame.rs
+++ b/datafusion/expr/src/window_frame.rs
@@ -282,7 +282,13 @@ impl WindowFrame {
/// Returns whether the window frame is "free range"; i.e. its start/end
/// bounds are UNBOUNDED or CURRENT ROW.
- fn free_range(&self) -> bool {
+ ///
+ /// This inspects only the bounds, not the frame units, so it returns `true`
+ /// for ROWS and GROUPS frames with such bounds as well. For RANGE frames,
+ /// such bounds are located by comparing order key values, whereas a finite
+ /// offset bound (e.g. `5 PRECEDING`) has to be computed arithmetically from
+ /// the current row's order key value.
+ pub fn free_range(&self) -> bool {
(self.start_bound.is_unbounded()
|| self.start_bound == WindowFrameBound::CurrentRow)
&& (self.end_bound.is_unbounded()
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index ca1527fac8..feadc0370b 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -1068,6 +1068,7 @@ fn coerce_frame_bound(
fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {
if col_type.is_numeric()
|| col_type.is_string()
+ || col_type.is_binary()
|| col_type.is_null()
|| matches!(
col_type,
@@ -1102,7 +1103,20 @@ fn coerce_window_frame(
.map(|s| s.expr.get_type(schema))
.transpose()?;
if let Some(col_type) = current_types {
- extract_window_frame_target_type(&col_type)?
+ let target_type = extract_window_frame_target_type(&col_type)?;
+ // A finite offset bound (e.g. `5 PRECEDING`) is computed as
+ // `current_value ± offset`, so it is only meaningful for target
+ // types that support arithmetic. Strings, binaries, booleans
+ // and lists are orderable -- which is all a free range frame
+ // needs -- but have no such arithmetic.
+ let supports_offset_arithmetic =
+ target_type.is_numeric() || is_interval(&target_type);
+ if !supports_offset_arithmetic && !window_frame.free_range() {
+ return plan_err!(
+ "RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type {target_type}"
+ );
+ }
+ target_type
} else {
return internal_err!("ORDER BY column cannot be empty");
}
diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt
index 6374cbf4f4..fe8bbdc345 100644
--- a/datafusion/sqllogictest/test_files/window.slt
+++ b/datafusion/sqllogictest/test_files/window.slt
@@ -6874,3 +6874,143 @@ ORDER BY id
2 NULL 3 3 3 3
3 NULL NULL NULL NULL NULL
4 7 7 7 7 7
+
+# RANGE window frame over a binary ORDER BY key. The default frame for an
+# ORDER BY without an explicit frame is RANGE BETWEEN UNBOUNDED PRECEDING AND
+# CURRENT ROW, which used to fail with
+# "Internal error: Cannot run range queries on datatype: Binary".
+# Binary is orderable, so peer/range comparison is well defined just like Utf8.
+query ?I
+SELECT x, COUNT(*) OVER (ORDER BY x)
+FROM (VALUES (arrow_cast('a', 'Binary')),
+ (arrow_cast('b', 'Binary')),
+ (arrow_cast('b', 'Binary')),
+ (arrow_cast('c', 'Binary'))) t(x)
+ORDER BY x
+----
+61 1
+62 3
+62 3
+63 4
+
+query ?I
+SELECT x, COUNT(*) OVER (ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+FROM (VALUES (arrow_cast('a', 'LargeBinary')),
+ (arrow_cast('b', 'LargeBinary')),
+ (arrow_cast('b', 'LargeBinary'))) t(x)
+ORDER BY x
+----
+61 3
+62 2
+62 2
+
+query ?I
+SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
+FROM (VALUES (arrow_cast('a', 'BinaryView')),
+ (arrow_cast('b', 'BinaryView')),
+ (arrow_cast('b', 'BinaryView'))) t(x)
+ORDER BY x
+----
+61 3
+62 2
+62 2
+
+query ?I
+SELECT x, COUNT(*) OVER (ORDER BY x)
+FROM (VALUES (arrow_cast(arrow_cast('a', 'Binary'), 'FixedSizeBinary(1)')),
+ (arrow_cast(arrow_cast('b', 'Binary'), 'FixedSizeBinary(1)'))) t(x)
+ORDER BY x
+----
+61 1
+62 2
+
+# A binary ORDER BY key nested in a dictionary resolves through the same arm.
+query ?I
+SELECT x, COUNT(*) OVER (ORDER BY x)
+FROM (VALUES (arrow_cast(arrow_cast('a', 'Binary'), 'Dictionary(Int32, Binary)')),
+ (arrow_cast(arrow_cast('b', 'Binary'), 'Dictionary(Int32, Binary)')),
+ (arrow_cast(arrow_cast('b', 'Binary'), 'Dictionary(Int32, Binary)'))) t(x)
+ORDER BY x
+----
+61 1
+62 3
+62 3
+
+# A non-aggregate window function over a binary ORDER BY key.
+query ?I
+SELECT x, RANK() OVER (ORDER BY x)
+FROM (VALUES (arrow_cast('a', 'Binary')),
+ (arrow_cast('b', 'Binary')),
+ (arrow_cast('b', 'Binary'))) t(x)
+ORDER BY x
+----
+61 1
+62 2
+62 2
+
+# Unsupported RANGE ORDER BY types still propagate the type-coercion error.
+query error DataFusion error: type_coercion\ncaused by\nInternal error: Cannot run range queries on datatype: Struct\("c0": Int64\)
+SELECT COUNT(*) OVER (ORDER BY x)
+FROM (VALUES (struct(1))) t(x)
+
+# An order key only needs to be comparable for a free range frame, whose bounds
+# are all UNBOUNDED or CURRENT ROW. A finite offset such as `1 PRECEDING` instead
+# has to be computed as `current_value - 1`, so it additionally requires
+# arithmetic on the order key type. Reject such frames during planning for every
+# order key type that has no arithmetic, rather than silently widening the frame
+# to the whole partition.
+query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Binary
+SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM (VALUES (arrow_cast('a', 'Binary'))) t(x)
+
+# Dictionary-wrapped binary is unwrapped before checking offset arithmetic.
+query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Binary
+SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM (VALUES (arrow_cast(arrow_cast('a', 'Binary'), 'Dictionary(Int32, Binary)'))) t(x)
+
+query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Utf8
+SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM (VALUES ('a')) t(x)
+
+# The end bound is rejected on the same grounds as the start bound.
+query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Boolean
+SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)
+FROM (VALUES (true)) t(x)
+
+query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type List\(Int64\)
+SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM (VALUES ([1,2])) t(x)
+
+query error DataFusion error: type_coercion\ncaused by\nError during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Null
+SELECT COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM (VALUES (NULL)) t(x)
+
+# Numeric order keys keep their finite offsets, decimals included.
+query RI
+SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM (VALUES (arrow_cast(1.5, 'Decimal128(10, 2)')),
+ (arrow_cast(2.0, 'Decimal128(10, 2)'))) t(x)
+ORDER BY x
+----
+1.5 1
+2 2
+
+# Free range frames over those same types keep working, so the rejection above is
+# narrowed to the offset form rather than to the order key type as a whole.
+query TI
+SELECT x, COUNT(*) OVER (ORDER BY x)
+FROM (VALUES ('a'), ('b'), ('b')) t(x)
+ORDER BY x
+----
+a 1
+b 3
+b 3
+
+query BI
+SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
+FROM (VALUES (false), (true), (true)) t(x)
+ORDER BY x
+----
+false 3
+true 2
+true 2
diff --git a/dev/update_function_docs.sh b/dev/update_function_docs.sh
index 04266b8cbc..2ead1253ac 100755
--- a/dev/update_function_docs.sh
+++ b/dev/update_function_docs.sh
@@ -317,6 +317,8 @@ where **offset** is an non-negative integer.
RANGE and GROUPS modes require an ORDER BY clause (with RANGE the ORDER BY must specify exactly one column).
+In RANGE mode an **offset** is measured in ORDER BY values rather than in rows, so the bound is computed by adding it to or subtracting it from the current row's ORDER BY value. That restricts `offset PRECEDING` and `offset FOLLOWING` to ORDER BY types supporting such arithmetic, namely the numeric, date, and timestamp types. Other orderable types, such as strings and binaries, can still be used with `UNBOUNDED PRECEDING`, `CURRENT ROW` and `UNBOUNDED FOLLOWING`, which are located by comparing ORDER BY values.
+
## Filter clause for aggregate window functions
Aggregate window functions support the SQL `FILTER (WHERE ...)` clause to include only rows that satisfy the predicate from the window frame in the aggregation.
diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md
index ef2785edb7..c87dbc3b8a 100644
--- a/docs/source/library-user-guide/upgrading/55.0.0.md
+++ b/docs/source/library-user-guide/upgrading/55.0.0.md
@@ -1408,3 +1408,47 @@ let table_opts = TableParquetOptions::try_from(&proto_table_opts)?;
```
See [issue #24019](https://github.com/apache/datafusion/issues/24019) for details.
+
+### RANGE window frames with offsets now reject ORDER BY types without arithmetic
+
+A `RANGE` frame offset such as `1 PRECEDING` is measured in `ORDER BY` values
+rather than in rows, so the frame bound is computed as `current_value - 1`. That
+requires arithmetic on the `ORDER BY` type. For types that have no such
+arithmetic -- `Utf8`, `Binary`, `Boolean`, `List` and `Null` -- such frames are
+now rejected when the query is planned:
+
+```sql
+SELECT x, COUNT(*) OVER (ORDER BY x RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM (VALUES ('a'), ('b'), ('c')) t(x)
+ORDER BY x;
+-- Error during planning: RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type Utf8
+```
+
+What this replaces depends on the `ORDER BY` type. Over `Utf8`, `Boolean` and
+`List` keys such queries planned successfully, and in DataFusion 54 the failing
+frame-bound arithmetic was silently treated as an overflow
+([#22140](https://github.com/apache/datafusion/pull/22140)): the bound collapsed
+to the partition edge, and the query returned results computed over that wrongly
+widened frame instead of an offset window. DataFusion 53 and earlier failed
+during execution with an arithmetic error. Rows whose order key value was `NULL`
+never reached the arithmetic, so queries where every order key value was `NULL`
+returned correct results throughout; those also now fail at planning. Over a
+`Null`-typed key the offset itself failed to coerce, erroring at planning time
+with `Cast error: Casting from Utf8 to Null not supported`, and binary keys were
+rejected for every `RANGE` frame with the internal error quoted below -- for
+those two, only the error message changes.
+
+Such an `ORDER BY` type is still usable with the `UNBOUNDED PRECEDING`,
+`CURRENT ROW` and `UNBOUNDED FOLLOWING` bounds, which are located by comparing
+`ORDER BY` values rather than by computing them. Binary order keys additionally
+gained support for those bounds in this release, having previously failed with
+`Internal error: Cannot run range queries on datatype: Binary`.
+
+**Who is affected:**
+
+- Queries using `offset PRECEDING` or `offset FOLLOWING` in a `RANGE` frame over
+ a non-numeric, non-temporal `ORDER BY` key. To count rows rather than compare
+ values, use a `ROWS` frame; to include everything up to the partition edge,
+ use `UNBOUNDED PRECEDING` or `UNBOUNDED FOLLOWING`.
+
+See [issue #24327](https://github.com/apache/datafusion/issues/24327) for details.
diff --git a/docs/source/user-guide/sql/window_functions.md b/docs/source/user-guide/sql/window_functions.md
index 2c8050ce1f..6496da1102 100644
--- a/docs/source/user-guide/sql/window_functions.md
+++ b/docs/source/user-guide/sql/window_functions.md
@@ -145,6 +145,8 @@ where **offset** is an non-negative integer.
RANGE and GROUPS modes require an ORDER BY clause (with RANGE the ORDER BY must specify exactly one column).
+In RANGE mode an **offset** is measured in ORDER BY values rather than in rows, so the bound is computed by adding it to or subtracting it from the current row's ORDER BY value. That restricts `offset PRECEDING` and `offset FOLLOWING` to ORDER BY types supporting such arithmetic, namely the numeric, date, and timestamp types. Other orderable types, such as strings and binaries, can still be used with `UNBOUNDED PRECEDING`, `CURRENT ROW` and `UNBOUNDED FOLLOWING`, which are located by comparing ORDER BY values.
+
## Filter clause for aggregate window functions
Aggregate window functions support the SQL `FILTER (WHERE ...)` clause to include only rows that satisfy the predicate from the window frame in the aggregation.