Skip to content

Apache DataFusion SQL Query Engine

fix: support RunEndEncoded arrays in function and window frame type coercion (#24565)

Thor3b63300849b5
diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs
index 781559ddd0..8e86cb3685 100644
--- a/datafusion/expr/src/type_coercion/functions.rs
+++ b/datafusion/expr/src/type_coercion/functions.rs
@@ -1158,6 +1158,16 @@ fn coerced_from<'a>(
{
Some(type_into.clone())
}
+ (_, RunEndEncoded(_, value_type))
+ if coerced_from(type_into, value_type.data_type()).is_some() =>
+ {
+ Some(type_into.clone())
+ }
+ (RunEndEncoded(_, value_type), _)
+ if coerced_from(value_type.data_type(), type_from).is_some() =>
+ {
+ Some(type_into.clone())
+ }
// coerced into type_into
(Int8, Null | Int8) => Some(type_into.clone()),
(Int16, Null | Int8 | Int16 | UInt8) => Some(type_into.clone()),
@@ -1617,6 +1627,50 @@ mod tests {
);
}
+ #[test]
+ fn test_coerced_from_run_end_encoded() {
+ let run_end_encoded_of = |value_type: DataType| {
+ DataType::RunEndEncoded(
+ Field::new("run_ends", DataType::Int32, false).into(),
+ Field::new("values", value_type, true).into(),
+ )
+ };
+
+ let type_into = run_end_encoded_of(DataType::UInt32);
+ let type_from = DataType::Int64;
+ assert_eq!(coerced_from(&type_into, &type_from), None);
+
+ let type_from = run_end_encoded_of(DataType::UInt32);
+ let type_into = DataType::Int64;
+ assert_eq!(
+ coerced_from(&type_into, &type_from),
+ Some(type_into.clone())
+ );
+
+ // Signature candidates for functions like `date_bin` are plain
+ // Timestamp, but a REE-encoded column (e.g. a segment written with
+ // REE-dict encoding for that field) should still coerce against
+ // them via the wrapped value type.
+ let type_from =
+ run_end_encoded_of(DataType::Timestamp(TimeUnit::Nanosecond, None));
+ let type_into = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ assert_eq!(
+ coerced_from(&type_into, &type_from),
+ Some(type_into.clone())
+ );
+
+ // The reverse direction: a plain type coercing into an REE target
+ // (e.g. a signature that happens to require RunEndEncoded) should
+ // succeed whenever the plain type coerces into the wrapped value
+ // type.
+ let type_into = run_end_encoded_of(DataType::Int64);
+ let type_from = DataType::Int32;
+ assert_eq!(
+ coerced_from(&type_into, &type_from),
+ Some(type_into.clone())
+ );
+ }
+
#[test]
fn test_get_valid_types_array_and_array() -> Result<()> {
let function = "array_and_array";
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index 0f82f0b0df..13b9541b6a 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -1088,6 +1088,8 @@ fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {
Ok(DataType::Interval(IntervalUnit::MonthDayNano))
} else if let DataType::Dictionary(_, value_type) = col_type {
extract_window_frame_target_type(value_type)
+ } else if let DataType::RunEndEncoded(_, value_type) = col_type {
+ extract_window_frame_target_type(value_type.data_type())
} else {
internal_err!("Cannot run range queries on datatype: {col_type}")
}
@@ -1113,8 +1115,11 @@ fn coerce_window_frame(
// `current_value ± offset`, so it is only meaningful for target
// types that support arithmetic. Other orderable target types can
// still use free range frames, whose bounds require comparison only.
+ // REE arrays are not supported by arrow's numeric kernesl.
+ // Tracked at https://github.com/apache/arrow-rs/issues/10891).
let supports_offset_arithmetic =
- target_type.is_numeric() || is_interval(&target_type);
+ !matches!(col_type, DataType::RunEndEncoded(_, _))
+ && (target_type.is_numeric() || is_interval(&target_type));
if !supports_offset_arithmetic && !window_frame.free_range() {
return plan_err!(
"RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type {target_type}"
diff --git a/datafusion/sqllogictest/test_files/run_end_encoded.slt b/datafusion/sqllogictest/test_files/run_end_encoded.slt
index b5909bc7c4..094f6ede1b 100644
--- a/datafusion/sqllogictest/test_files/run_end_encoded.slt
+++ b/datafusion/sqllogictest/test_files/run_end_encoded.slt
@@ -85,3 +85,65 @@ false false
true true
true true
true true
+
+# date_bin's signature is a plain Timestamp, not RunEndEncoded, so this only
+# plans if coerced_from() unwraps the REE value type the same way it does for
+# Dictionary (see test_coerced_from_run_end_encoded)
+statement ok
+CREATE TABLE ree_timestamps AS
+SELECT
+ arrow_cast(
+ arrow_cast(ts, 'Timestamp(Nanosecond, None)'),
+ 'RunEndEncoded("run_ends": non-null Int32, "values": Timestamp(Nanosecond, None))'
+ ) AS ts
+FROM (VALUES
+ ('2023-12-04T00:00:00'),
+ ('2023-12-04T00:05:00'),
+ ('2023-12-04T00:35:00')
+) AS t(ts);
+
+query P rowsort
+SELECT date_bin('30 minutes', ts) FROM ree_timestamps;
+----
+2023-12-04T00:00:00
+2023-12-04T00:00:00
+2023-12-04T00:30:00
+
+# RANGE window frame ordered by an REE-encoded column: extract_window_frame_target_type()
+# unwraps RunEndEncoded the same way it does for Dictionary, so a frame that
+# doesn't need offset arithmetic.
+query II
+SELECT
+ temperature,
+ SUM(temperature) OVER (
+ ORDER BY arrow_cast(
+ temperature,
+ 'RunEndEncoded("run_ends": non-null Int32, "values": Int64)'
+ )
+ RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS windowed_sum
+FROM sensor_readings
+ORDER BY temperature;
+----
+20 20
+22 42
+23 65
+24 89
+
+# An offset RANGE bound (`k PRECEDING`/`k FOLLOWING`, unlike CURRENT ROW or
+# UNBOUNDED above) is computed by adding/subtracting the offset from the
+# current row's value, and arrow's numeric kernels don't support arithmetic
+# on a still-RunEndEncoded-wrapped value (tracked at
+# https://github.com/apache/arrow-rs/issues/10891).
+query error RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type
+SELECT
+ temperature,
+ SUM(temperature) OVER (
+ ORDER BY arrow_cast(
+ temperature,
+ 'RunEndEncoded("run_ends": non-null Int32, "values": Int64)'
+ )
+ RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING
+ ) AS windowed_sum
+FROM sensor_readings
+ORDER BY temperature;