Apache DataFusion SQL Query Engine
Add RunEndEncoded type coercion (#18561)
Vegard Stikbakkecb1f01966b12
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rsindex 122e0f987b..44f1a42c54 100644--- a/datafusion/expr-common/src/type_coercion/binary.rs+++ b/datafusion/expr-common/src/type_coercion/binary.rs@@ -838,6 +838,7 @@ pub fn comparison_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<D}binary_numeric_coercion(lhs_type, rhs_type).or_else(|| dictionary_comparison_coercion(lhs_type, rhs_type, true))+ .or_else(|| ree_comparison_coercion(lhs_type, rhs_type, true)).or_else(|| temporal_coercion_nonstrict_timezone(lhs_type, rhs_type)).or_else(|| string_coercion(lhs_type, rhs_type)).or_else(|| list_coercion(lhs_type, rhs_type))@@ -867,6 +868,7 @@ pub fn comparison_coercion_numeric(}binary_numeric_coercion(lhs_type, rhs_type).or_else(|| dictionary_comparison_coercion_numeric(lhs_type, rhs_type, true))+ .or_else(|| ree_comparison_coercion_numeric(lhs_type, rhs_type, true)).or_else(|| string_coercion(lhs_type, rhs_type)).or_else(|| null_coercion(lhs_type, rhs_type)).or_else(|| string_numeric_coercion_as_numeric(lhs_type, rhs_type))@@ -1423,6 +1425,73 @@ fn dictionary_comparison_coercion_numeric()}+/// Coercion rules for RunEndEncoded: the type that both lhs and rhs+/// can be casted to for the purpose of a computation.+///+/// Not all operators support REE, if `preserve_ree` is true+/// REE will be preserved if possible+///+/// The `coerce_fn` parameter determines which comparison coercion function to use+/// for comparing the REE value types.+fn ree_comparison_coercion_generic(+ lhs_type: &DataType,+ rhs_type: &DataType,+ preserve_ree: bool,+ coerce_fn: fn(&DataType, &DataType) -> Option<DataType>,+) -> Option<DataType> {+ use arrow::datatypes::DataType::*;+ match (lhs_type, rhs_type) {+ (RunEndEncoded(_, lhs_values_field), RunEndEncoded(_, rhs_values_field)) => {+ coerce_fn(lhs_values_field.data_type(), rhs_values_field.data_type())+ }+ (ree @ RunEndEncoded(_, values_field), other_type)+ | (other_type, ree @ RunEndEncoded(_, values_field))+ if preserve_ree && values_field.data_type() == other_type =>+ {+ Some(ree.clone())+ }+ (RunEndEncoded(_, values_field), _) => {+ coerce_fn(values_field.data_type(), rhs_type)+ }+ (_, RunEndEncoded(_, values_field)) => {+ coerce_fn(lhs_type, values_field.data_type())+ }+ _ => None,+ }+}++/// Coercion rules for RunEndEncoded: the type that both lhs and rhs+/// can be casted to for the purpose of a computation.+///+/// Not all operators support REE, if `preserve_ree` is true+/// REE will be preserved if possible+fn ree_comparison_coercion(+ lhs_type: &DataType,+ rhs_type: &DataType,+ preserve_ree: bool,+) -> Option<DataType> {+ ree_comparison_coercion_generic(lhs_type, rhs_type, preserve_ree, comparison_coercion)+}++/// Coercion rules for RunEndEncoded with numeric preference: similar to+/// [`ree_comparison_coercion`] but uses [`comparison_coercion_numeric`]+/// which prefers numeric types over strings when both are present.+///+/// This is used by [`comparison_coercion_numeric`] to maintain consistent+/// numeric-preferring semantics when dealing with REE types.+fn ree_comparison_coercion_numeric(+ lhs_type: &DataType,+ rhs_type: &DataType,+ preserve_ree: bool,+) -> Option<DataType> {+ ree_comparison_coercion_generic(+ lhs_type,+ rhs_type,+ preserve_ree,+ comparison_coercion_numeric,+ )+}+/// Coercion rules for string concat./// This is a union of string coercion rules and specified rules:/// 1. At least one side of lhs and rhs should be string type (Utf8 / LargeUtf8)@@ -1601,12 +1670,13 @@ fn binary_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType>}/// Coercion rules for like operations.-/// This is a union of string coercion rules and dictionary coercion rules+/// This is a union of string coercion rules, dictionary coercion rules, and REE coercion rulespub fn like_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {string_coercion(lhs_type, rhs_type).or_else(|| list_coercion(lhs_type, rhs_type)).or_else(|| binary_to_string_coercion(lhs_type, rhs_type)).or_else(|| dictionary_comparison_coercion(lhs_type, rhs_type, false))+ .or_else(|| ree_comparison_coercion(lhs_type, rhs_type, false)).or_else(|| regex_null_coercion(lhs_type, rhs_type)).or_else(|| null_coercion(lhs_type, rhs_type))}diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/mod.rs b/datafusion/expr-common/src/type_coercion/binary/tests/mod.rsindex 6d21d795e4..e4653d4955 100644--- a/datafusion/expr-common/src/type_coercion/binary/tests/mod.rs+++ b/datafusion/expr-common/src/type_coercion/binary/tests/mod.rs@@ -77,3 +77,4 @@ mod arithmetic;mod comparison;mod dictionary;mod null_coercion;+mod run_end_encoded;diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs b/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rsnew file mode 100644index 0000000000..9997db7a82--- /dev/null+++ b/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs@@ -0,0 +1,99 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use super::*;++#[test]+fn test_ree_type_coercion() {+ use DataType::*;++ let lhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Int32, false)),+ );+ let rhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Int16, false)),+ );+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, true),+ Some(Int32)+ );+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, false),+ Some(Int32)+ );++ // Since we can coerce values of Int16 to Utf8 can support this: Coercion of Int16 to Utf8+ let lhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Utf8, false)),+ );+ let rhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Int16, false)),+ );+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, true),+ Some(Utf8)+ );++ // Since we can coerce values of Utf8 to Binary can support this+ let lhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Utf8, false)),+ );+ let rhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Binary, false)),+ );+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, true),+ Some(Binary)+ );+ let lhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Utf8, false)),+ );+ let rhs_type = Utf8;+ // Don't preserve REE+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, false),+ Some(Utf8)+ );+ // Preserve REE+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, true),+ Some(lhs_type.clone())+ );++ let lhs_type = Utf8;+ let rhs_type = RunEndEncoded(+ Arc::new(Field::new("run_ends", Int8, false)),+ Arc::new(Field::new("values", Utf8, false)),+ );+ // Don't preserve REE+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, false),+ Some(Utf8)+ );+ // Preserve REE+ assert_eq!(+ ree_comparison_coercion(&lhs_type, &rhs_type, true),+ Some(rhs_type.clone())+ );+}