Skip to content

Apache DataFusion SQL Query Engine

feat[expr-common]: add REE arithmetic coercion for numeric and decimal (#21179)

Alfonso Subiotto Marquésfab3b7172804
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs
index d5776d254c..1f02d6322a 100644
--- a/datafusion/expr-common/src/type_coercion/binary.rs
+++ b/datafusion/expr-common/src/type_coercion/binary.rs
@@ -378,6 +378,16 @@ fn math_decimal_coercion(
let (lhs_type, value_type) = math_decimal_coercion(lhs_type, value_type)?;
Some((lhs_type, value_type))
}
+ (RunEndEncoded(_, field), _) => {
+ let (value_type, rhs_type) =
+ math_decimal_coercion(field.data_type(), rhs_type)?;
+ Some((value_type, rhs_type))
+ }
+ (_, RunEndEncoded(_, field)) => {
+ let (lhs_type, value_type) =
+ math_decimal_coercion(lhs_type, field.data_type())?;
+ Some((lhs_type, value_type))
+ }
(
Null,
Decimal32(_, _) | Decimal64(_, _) | Decimal128(_, _) | Decimal256(_, _),
@@ -1414,6 +1424,15 @@ fn mathematics_numerical_coercion(
(_, Dictionary(_, value_type)) => {
mathematics_numerical_coercion(lhs_type, value_type)
}
+ (RunEndEncoded(_, lhs_field), RunEndEncoded(_, rhs_field)) => {
+ mathematics_numerical_coercion(lhs_field.data_type(), rhs_field.data_type())
+ }
+ (RunEndEncoded(_, field), _) => {
+ mathematics_numerical_coercion(field.data_type(), rhs_type)
+ }
+ (_, RunEndEncoded(_, field)) => {
+ mathematics_numerical_coercion(lhs_type, field.data_type())
+ }
_ => numerical_coercion(lhs_type, rhs_type),
}
}
@@ -1493,6 +1512,15 @@ fn both_numeric_or_null_and_numeric(lhs_type: &DataType, rhs_type: &DataType) ->
(_, Dictionary(_, value_type)) => {
lhs_type.is_numeric() && value_type.is_numeric()
}
+ (RunEndEncoded(_, lhs_field), RunEndEncoded(_, rhs_field)) => {
+ lhs_field.data_type().is_numeric() && rhs_field.data_type().is_numeric()
+ }
+ (RunEndEncoded(_, field), _) => {
+ field.data_type().is_numeric() && rhs_type.is_numeric()
+ }
+ (_, RunEndEncoded(_, field)) => {
+ lhs_type.is_numeric() && field.data_type().is_numeric()
+ }
_ => lhs_type.is_numeric() && rhs_type.is_numeric(),
}
}
diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs b/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs
index dab42214d7..38e9fb3908 100644
--- a/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs
+++ b/datafusion/expr-common/src/type_coercion/binary/tests/run_end_encoded.rs
@@ -16,11 +16,17 @@
// under the License.
use super::*;
+use DataType::*;
+
+fn ree(value_type: DataType) -> DataType {
+ RunEndEncoded(
+ Arc::new(Field::new("run_ends", Int32, false)),
+ Arc::new(Field::new("values", value_type, false)),
+ )
+}
#[test]
fn test_ree_type_coercion() {
- use DataType::*;
-
let lhs_type = RunEndEncoded(
Arc::new(Field::new("run_ends", Int8, false)),
Arc::new(Field::new("values", Int32, false)),
@@ -97,3 +103,29 @@ fn test_ree_type_coercion() {
Some(rhs_type.clone())
);
}
+
+#[test]
+fn test_ree_arithmetic_coercion() -> Result<()> {
+ test_coercion_binary_rule!(ree(Int64), Int64, Operator::Plus, Int64);
+ test_coercion_binary_rule!(Int64, ree(Int64), Operator::Multiply, Int64);
+ test_coercion_binary_rule!(ree(Int32), ree(Int64), Operator::Plus, Int64);
+
+ // Decimal unwrapping through math_decimal_coercion
+ let (lhs, rhs) =
+ BinaryTypeCoercer::new(&ree(Decimal128(10, 2)), &Operator::Plus, &Int32)
+ .get_input_types()?;
+ assert_eq!(lhs, Decimal128(10, 2));
+ assert_eq!(rhs, Decimal128(10, 0));
+
+ let (lhs, rhs) =
+ BinaryTypeCoercer::new(&Int32, &Operator::Plus, &ree(Decimal128(10, 2)))
+ .get_input_types()?;
+ assert_eq!(lhs, Decimal128(10, 0));
+ assert_eq!(rhs, Decimal128(10, 2));
+
+ let result =
+ BinaryTypeCoercer::new(&ree(Utf8), &Operator::Plus, &Int32).get_input_types();
+ assert!(result.is_err());
+
+ Ok(())
+}
diff --git a/datafusion/sqllogictest/test_files/run_end_encoded.slt b/datafusion/sqllogictest/test_files/run_end_encoded.slt
index 1f0a9b4eb3..48932d2caa 100644
--- a/datafusion/sqllogictest/test_files/run_end_encoded.slt
+++ b/datafusion/sqllogictest/test_files/run_end_encoded.slt
@@ -55,3 +55,15 @@ FROM sensor_readings;
----
sensor_A
sensor_B
+
+# Test arithmetic on REE columns (numeric and decimal value types)
+query IR rowsort
+SELECT
+ arrow_cast(temperature, 'RunEndEncoded("run_ends": non-null Int32, "values": Int64)') + 1 AS t_plus_one,
+ arrow_cast(arrow_cast(temperature, 'Decimal128(10, 2)'), 'RunEndEncoded("run_ends": non-null Int32, "values": Decimal128(10, 2))') + 1 AS t_dec_plus_one
+FROM sensor_readings;
+----
+21 21
+23 23
+24 24
+25 25