Skip to content

Apache DataFusion SQL Query Engine

feat[expr-common]: support regex and LIKE coercion on REE and Dict value types that require an extra coercion step (#21924)

Alfonso Subiotto Marquésf3cebc5196d9
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs
index 1f02d6322a..c701eccb39 100644
--- a/datafusion/expr-common/src/type_coercion/binary.rs
+++ b/datafusion/expr-common/src/type_coercion/binary.rs
@@ -1766,8 +1766,8 @@ fn binary_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType>
pub fn like_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
string_coercion(lhs_type, rhs_type)
.or_else(|| binary_to_string_coercion(lhs_type, rhs_type))
- .or_else(|| dictionary_coercion(lhs_type, rhs_type, false, string_coercion))
- .or_else(|| ree_coercion(lhs_type, rhs_type, false, string_coercion))
+ .or_else(|| dictionary_coercion(lhs_type, rhs_type, false, like_coercion))
+ .or_else(|| ree_coercion(lhs_type, rhs_type, false, like_coercion))
.or_else(|| regex_null_coercion(lhs_type, rhs_type))
.or_else(|| null_coercion(lhs_type, rhs_type))
}
@@ -1787,8 +1787,8 @@ fn regex_null_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataT
/// This is a union of string coercion rules, dictionary coercion rules, and REE coercion rules.
pub fn regex_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
string_coercion(lhs_type, rhs_type)
- .or_else(|| dictionary_coercion(lhs_type, rhs_type, false, string_coercion))
- .or_else(|| ree_coercion(lhs_type, rhs_type, false, string_coercion))
+ .or_else(|| dictionary_coercion(lhs_type, rhs_type, false, regex_coercion))
+ .or_else(|| ree_coercion(lhs_type, rhs_type, false, regex_coercion))
.or_else(|| regex_null_coercion(lhs_type, rhs_type))
}
diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
index 2b304c8de1..22d5066d5f 100644
--- a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
+++ b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
@@ -277,6 +277,23 @@ drop table strings
statement ok
drop table dict_table
+# Dict value types that themselves need further coercion against the literal
+statement ok
+create table dict_inner as
+select arrow_cast(arrow_cast(c, 'Binary'), 'Dictionary(UInt32, Binary)') as bin_col,
+ arrow_cast(arrow_cast(c, 'Dictionary(UInt32, Utf8)'),
+ 'Dictionary(Int32, Dictionary(UInt32, Utf8))') as nested_col
+from (values ('foo'), ('bar')) as t(c);
+
+query BB
+select bin_col LIKE 'foo', nested_col ~ 'foo' from dict_inner;
+----
+true true
+false false
+
+statement ok
+drop table dict_inner
+
# Ensure that regexp_like is rewritten to use the (more optimized) regex operators
statement ok
create table regexp_test as values
diff --git a/datafusion/sqllogictest/test_files/run_end_encoded.slt b/datafusion/sqllogictest/test_files/run_end_encoded.slt
index 48932d2caa..b5909bc7c4 100644
--- a/datafusion/sqllogictest/test_files/run_end_encoded.slt
+++ b/datafusion/sqllogictest/test_files/run_end_encoded.slt
@@ -67,3 +67,21 @@ FROM sensor_readings;
23 23
24 24
25 25
+
+# Regex / LIKE match against an REE column whose values are themselves Dictionary-encoded
+query BB rowsort
+SELECT
+ arrow_cast(
+ arrow_cast(sensor_id, 'Dictionary(UInt32, Utf8)'),
+ 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(UInt32, Utf8))'
+ ) ~ 'sensor_A',
+ arrow_cast(
+ arrow_cast(sensor_id, 'Dictionary(UInt32, Utf8)'),
+ 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(UInt32, Utf8))'
+ ) LIKE 'sensor_A'
+FROM sensor_readings;
+----
+false false
+true true
+true true
+true true