Skip to content

Apache DataFusion SQL Query Engine

Add RunEndEncoded type coercion (#18561)

Vegard Stikbakkecb1f01966b12
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs
index 122e0f987b..44f1a42c54 100644
--- a/datafusion/expr-common/src/type_coercion/binary.rs
+++ b/datafusion/expr-common/src/type_coercion/binary.rs
@@ -838,6 +838,7 @@ pub fn comparison_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<D
}
binary_numeric_coercion(lhs_type, rhs_type)
.or_else(|| dictionary_comparison_coercion(lhs_type, rhs_type, true))
+ .or_else(|| ree_comparison_coercion(lhs_type, rhs_type, true))
.or_else(|| temporal_coercion_nonstrict_timezone(lhs_type, rhs_type))
.or_else(|| string_coercion(lhs_type, rhs_type))
.or_else(|| list_coercion(lhs_type, rhs_type))
@@ -867,6 +868,7 @@ pub fn comparison_coercion_numeric(
}
binary_numeric_coercion(lhs_type, rhs_type)
.or_else(|| dictionary_comparison_coercion_numeric(lhs_type, rhs_type, true))
+ .or_else(|| ree_comparison_coercion_numeric(lhs_type, rhs_type, true))
.or_else(|| string_coercion(lhs_type, rhs_type))
.or_else(|| null_coercion(lhs_type, rhs_type))
.or_else(|| string_numeric_coercion_as_numeric(lhs_type, rhs_type))
@@ -1423,6 +1425,73 @@ fn dictionary_comparison_coercion_numeric(
)
}
+/// Coercion rules for RunEndEncoded: the type that both lhs and rhs
+/// can be casted to for the purpose of a computation.
+///
+/// Not all operators support REE, if `preserve_ree` is true
+/// REE will be preserved if possible
+///
+/// The `coerce_fn` parameter determines which comparison coercion function to use
+/// for comparing the REE value types.
+fn ree_comparison_coercion_generic(
+ lhs_type: &DataType,
+ rhs_type: &DataType,
+ preserve_ree: bool,
+ coerce_fn: fn(&DataType, &DataType) -> Option<DataType>,
+) -> Option<DataType> {
+ use arrow::datatypes::DataType::*;
+ match (lhs_type, rhs_type) {
+ (RunEndEncoded(_, lhs_values_field), RunEndEncoded(_, rhs_values_field)) => {
+ coerce_fn(lhs_values_field.data_type(), rhs_values_field.data_type())
+ }
+ (ree @ RunEndEncoded(_, values_field), other_type)
+ | (other_type, ree @ RunEndEncoded(_, values_field))
+ if preserve_ree && values_field.data_type() == other_type =>
+ {
+ Some(ree.clone())
+ }
+ (RunEndEncoded(_, values_field), _) => {
+ coerce_fn(values_field.data_type(), rhs_type)
+ }
+ (_, RunEndEncoded(_, values_field)) => {
+ coerce_fn(lhs_type, values_field.data_type())
+ }
+ _ => None,
+ }
+}
+
+/// Coercion rules for RunEndEncoded: the type that both lhs and rhs
+/// can be casted to for the purpose of a computation.
+///
+/// Not all operators support REE, if `preserve_ree` is true
+/// REE will be preserved if possible
+fn ree_comparison_coercion(
+ lhs_type: &DataType,
+ rhs_type: &DataType,
+ preserve_ree: bool,
+) -> Option<DataType> {
+ ree_comparison_coercion_generic(lhs_type, rhs_type, preserve_ree, comparison_coercion)
+}
+
+/// Coercion rules for RunEndEncoded with numeric preference: similar to
+/// [`ree_comparison_coercion`] but uses [`comparison_coercion_numeric`]
+/// which prefers numeric types over strings when both are present.
+///
+/// This is used by [`comparison_coercion_numeric`] to maintain consistent
+/// numeric-preferring semantics when dealing with REE types.
+fn ree_comparison_coercion_numeric(
+ lhs_type: &DataType,
+ rhs_type: &DataType,
+ preserve_ree: bool,
+) -> Option<DataType> {
+ ree_comparison_coercion_generic(
+ lhs_type,
+ rhs_type,
+ preserve_ree,
+ comparison_coercion_numeric,
+ )
+}
+
/// Coercion rules for string concat.
/// This is a union of string coercion rules and specified rules:
/// 1. At least one side of lhs and rhs should be string type (Utf8 / LargeUtf8)
@@ -1601,12 +1670,13 @@ fn binary_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType>
}
/// Coercion rules for like operations.
-/// This is a union of string coercion rules and dictionary coercion rules
+/// This is a union of string coercion rules, dictionary coercion rules, and REE coercion rules
pub fn like_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
string_coercion(lhs_type, rhs_type)
.or_else(|| list_coercion(lhs_type, rhs_type))
.or_else(|| binary_to_string_coercion(lhs_type, rhs_type))
.or_else(|| dictionary_comparison_coercion(lhs_type, rhs_type, false))
+ .or_else(|| ree_comparison_coercion(lhs_type, rhs_type, false))
.or_else(|| regex_null_coercion(lhs_type, rhs_type))
.or_else(|| null_coercion(lhs_type, rhs_type))
}
diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/mod.rs b/datafusion/expr-common/src/type_coercion/binary/tests/mod.rs
index 6d21d795e4..e4653d4955 100644
--- a/datafusion/expr-common/src/type_coercion/binary/tests/mod.rs
+++ b/datafusion/expr-common/src/type_coercion/binary/tests/mod.rs
@@ -77,3 +77,4 @@ mod arithmetic;
mod comparison;
mod dictionary;
mod null_coercion;
+mod run_end_encoded;
diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs b/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs
new file mode 100644
index 0000000000..9997db7a82
--- /dev/null
+++ b/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::*;
+
+#[test]
+fn test_ree_type_coercion() {
+ use DataType::*;
+
+ let lhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Int32, false)),
+ );
+ let rhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Int16, false)),
+ );
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, true),
+ Some(Int32)
+ );
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, false),
+ Some(Int32)
+ );
+
+ // Since we can coerce values of Int16 to Utf8 can support this: Coercion of Int16 to Utf8
+ let lhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Utf8, false)),
+ );
+ let rhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Int16, false)),
+ );
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, true),
+ Some(Utf8)
+ );
+
+ // Since we can coerce values of Utf8 to Binary can support this
+ let lhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Utf8, false)),
+ );
+ let rhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Binary, false)),
+ );
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, true),
+ Some(Binary)
+ );
+ let lhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Utf8, false)),
+ );
+ let rhs_type = Utf8;
+ // Don't preserve REE
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, false),
+ Some(Utf8)
+ );
+ // Preserve REE
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, true),
+ Some(lhs_type.clone())
+ );
+
+ let lhs_type = Utf8;
+ let rhs_type = RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int8, false)),
+ Arc::new(Field::new("values", Utf8, false)),
+ );
+ // Don't preserve REE
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, false),
+ Some(Utf8)
+ );
+ // Preserve REE
+ assert_eq!(
+ ree_comparison_coercion(&lhs_type, &rhs_type, true),
+ Some(rhs_type.clone())
+ );
+}