Apache DataFusion SQL Query Engine
fix: support RunEndEncoded arrays in function and window frame type coercion (#24565)
Thor3b63300849b5
diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rsindex 781559ddd0..8e86cb3685 100644--- a/datafusion/expr/src/type_coercion/functions.rs+++ b/datafusion/expr/src/type_coercion/functions.rs@@ -1158,6 +1158,16 @@ fn coerced_from<'a>({Some(type_into.clone())}+ (_, RunEndEncoded(_, value_type))+ if coerced_from(type_into, value_type.data_type()).is_some() =>+ {+ Some(type_into.clone())+ }+ (RunEndEncoded(_, value_type), _)+ if coerced_from(value_type.data_type(), type_from).is_some() =>+ {+ Some(type_into.clone())+ }// coerced into type_into(Int8, Null | Int8) => Some(type_into.clone()),(Int16, Null | Int8 | Int16 | UInt8) => Some(type_into.clone()),@@ -1617,6 +1627,50 @@ mod tests {);}+ #[test]+ fn test_coerced_from_run_end_encoded() {+ let run_end_encoded_of = |value_type: DataType| {+ DataType::RunEndEncoded(+ Field::new("run_ends", DataType::Int32, false).into(),+ Field::new("values", value_type, true).into(),+ )+ };++ let type_into = run_end_encoded_of(DataType::UInt32);+ let type_from = DataType::Int64;+ assert_eq!(coerced_from(&type_into, &type_from), None);++ let type_from = run_end_encoded_of(DataType::UInt32);+ let type_into = DataType::Int64;+ assert_eq!(+ coerced_from(&type_into, &type_from),+ Some(type_into.clone())+ );++ // Signature candidates for functions like `date_bin` are plain+ // Timestamp, but a REE-encoded column (e.g. a segment written with+ // REE-dict encoding for that field) should still coerce against+ // them via the wrapped value type.+ let type_from =+ run_end_encoded_of(DataType::Timestamp(TimeUnit::Nanosecond, None));+ let type_into = DataType::Timestamp(TimeUnit::Nanosecond, None);+ assert_eq!(+ coerced_from(&type_into, &type_from),+ Some(type_into.clone())+ );++ // The reverse direction: a plain type coercing into an REE target+ // (e.g. a signature that happens to require RunEndEncoded) should+ // succeed whenever the plain type coerces into the wrapped value+ // type.+ let type_into = run_end_encoded_of(DataType::Int64);+ let type_from = DataType::Int32;+ assert_eq!(+ coerced_from(&type_into, &type_from),+ Some(type_into.clone())+ );+ }+#[test]fn test_get_valid_types_array_and_array() -> Result<()> {let function = "array_and_array";diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 0f82f0b0df..13b9541b6a 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -1088,6 +1088,8 @@ fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {Ok(DataType::Interval(IntervalUnit::MonthDayNano))} else if let DataType::Dictionary(_, value_type) = col_type {extract_window_frame_target_type(value_type)+ } else if let DataType::RunEndEncoded(_, value_type) = col_type {+ extract_window_frame_target_type(value_type.data_type())} else {internal_err!("Cannot run range queries on datatype: {col_type}")}@@ -1113,8 +1115,11 @@ fn coerce_window_frame(// `current_value ± offset`, so it is only meaningful for target// types that support arithmetic. Other orderable target types can// still use free range frames, whose bounds require comparison only.+ // REE arrays are not supported by arrow's numeric kernesl.+ // Tracked at https://github.com/apache/arrow-rs/issues/10891).let supports_offset_arithmetic =- target_type.is_numeric() || is_interval(&target_type);+ !matches!(col_type, DataType::RunEndEncoded(_, _))+ && (target_type.is_numeric() || is_interval(&target_type));if !supports_offset_arithmetic && !window_frame.free_range() {return plan_err!("RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type {target_type}"diff --git a/datafusion/sqllogictest/test_files/run_end_encoded.slt b/datafusion/sqllogictest/test_files/run_end_encoded.sltindex b5909bc7c4..094f6ede1b 100644--- a/datafusion/sqllogictest/test_files/run_end_encoded.slt+++ b/datafusion/sqllogictest/test_files/run_end_encoded.slt@@ -85,3 +85,65 @@ false falsetrue truetrue truetrue true++# date_bin's signature is a plain Timestamp, not RunEndEncoded, so this only+# plans if coerced_from() unwraps the REE value type the same way it does for+# Dictionary (see test_coerced_from_run_end_encoded)+statement ok+CREATE TABLE ree_timestamps AS+SELECT+ arrow_cast(+ arrow_cast(ts, 'Timestamp(Nanosecond, None)'),+ 'RunEndEncoded("run_ends": non-null Int32, "values": Timestamp(Nanosecond, None))'+ ) AS ts+FROM (VALUES+ ('2023-12-04T00:00:00'),+ ('2023-12-04T00:05:00'),+ ('2023-12-04T00:35:00')+) AS t(ts);++query P rowsort+SELECT date_bin('30 minutes', ts) FROM ree_timestamps;+----+2023-12-04T00:00:00+2023-12-04T00:00:00+2023-12-04T00:30:00++# RANGE window frame ordered by an REE-encoded column: extract_window_frame_target_type()+# unwraps RunEndEncoded the same way it does for Dictionary, so a frame that+# doesn't need offset arithmetic.+query II+SELECT+ temperature,+ SUM(temperature) OVER (+ ORDER BY arrow_cast(+ temperature,+ 'RunEndEncoded("run_ends": non-null Int32, "values": Int64)'+ )+ RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW+ ) AS windowed_sum+FROM sensor_readings+ORDER BY temperature;+----+20 20+22 42+23 65+24 89++# An offset RANGE bound (`k PRECEDING`/`k FOLLOWING`, unlike CURRENT ROW or+# UNBOUNDED above) is computed by adding/subtracting the offset from the+# current row's value, and arrow's numeric kernels don't support arithmetic+# on a still-RunEndEncoded-wrapped value (tracked at+# https://github.com/apache/arrow-rs/issues/10891).+query error RANGE with offset PRECEDING/FOLLOWING is not supported for ORDER BY type+SELECT+ temperature,+ SUM(temperature) OVER (+ ORDER BY arrow_cast(+ temperature,+ 'RunEndEncoded("run_ends": non-null Int32, "values": Int64)'+ )+ RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING+ ) AS windowed_sum+FROM sensor_readings+ORDER BY temperature;