Skip to content

Apache DataFusion SQL Query Engine

chore: add runendencoded & listview types to dfschema equality methods (#24138)

Jeffrey Voc8b4aeaad90d
diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs
index 2f28cf99cd..262f1dcf61 100644
--- a/datafusion/common/src/dfschema.rs
+++ b/datafusion/common/src/dfschema.rs
@@ -671,6 +671,8 @@ impl DFSchema {
/// logically equivalent. For example:
/// - a Dictionary<K,V> type is logically equal to a plain V type
/// - a Dictionary<K1, V1> is also logically equal to Dictionary<K2, V1>
+ /// - a RunEndEncoded<K,V> type is logically equal to a plain V type
+ /// - a RunEndEncoded<K1, V1> is also logically equal to RunEndEncoded<K2, V1>
/// - Utf8 and Utf8View are logically equal
pub fn datatype_is_logically_equal(dt1: &DataType, dt2: &DataType) -> bool {
// check nested fields
@@ -682,8 +684,17 @@ impl DFSchema {
| (othertype, DataType::Dictionary(_, v1)) => {
Self::datatype_is_logically_equal(v1.as_ref(), othertype)
}
+ (DataType::RunEndEncoded(_, v1), DataType::RunEndEncoded(_, v2)) => {
+ Self::datatype_is_logically_equal(v1.data_type(), v2.data_type())
+ }
+ (DataType::RunEndEncoded(_, v1), othertype)
+ | (othertype, DataType::RunEndEncoded(_, v1)) => {
+ Self::datatype_is_logically_equal(v1.data_type(), othertype)
+ }
(DataType::List(f1), DataType::List(f2))
| (DataType::LargeList(f1), DataType::LargeList(f2))
+ | (DataType::ListView(f1), DataType::ListView(f2))
+ | (DataType::LargeListView(f1), DataType::LargeListView(f2))
| (DataType::FixedSizeList(f1, _), DataType::FixedSizeList(f2, _)) => {
// Don't compare the names of the technical inner field
// Usually "item" but that's not mandated
@@ -742,8 +753,17 @@ impl DFSchema {
Self::datatype_is_semantically_equal(k1.as_ref(), k2.as_ref())
&& Self::datatype_is_semantically_equal(v1.as_ref(), v2.as_ref())
}
+ (DataType::RunEndEncoded(k1, v1), DataType::RunEndEncoded(k2, v2)) => {
+ Self::datatype_is_semantically_equal(k1.data_type(), k2.data_type())
+ && Self::datatype_is_semantically_equal(
+ v1.data_type(),
+ v2.data_type(),
+ )
+ }
(DataType::List(f1), DataType::List(f2))
| (DataType::LargeList(f1), DataType::LargeList(f2))
+ | (DataType::ListView(f1), DataType::ListView(f2))
+ | (DataType::LargeListView(f1), DataType::LargeListView(f2))
| (DataType::FixedSizeList(f1, _), DataType::FixedSizeList(f2, _)) => {
// Don't compare the names of the technical inner field
// Usually "item" but that's not mandated
@@ -1754,12 +1774,20 @@ mod tests {
&DataType::List(Field::new_list_field(DataType::Int8, true).into()),
&DataType::List(Field::new("element", DataType::Int8, false).into())
));
+ assert!(DFSchema::datatype_is_logically_equal(
+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),
+ &DataType::ListView(Field::new("element", DataType::Int8, false).into())
+ ));
// Fails if element type is different
assert!(!DFSchema::datatype_is_logically_equal(
&DataType::List(Field::new_list_field(DataType::Int8, true).into()),
&DataType::List(Field::new_list_field(DataType::Int16, true).into())
));
+ assert!(!DFSchema::datatype_is_logically_equal(
+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),
+ &DataType::ListView(Field::new_list_field(DataType::Int16, true).into())
+ ));
// Test maps
let map_field = DataType::Map(
@@ -1896,6 +1924,50 @@ mod tests {
));
}
+ #[test]
+ fn test_datatype_is_logically_equivalent_to_ree() {
+ // RunEndEncoded is logically equal to its value type
+ assert!(DFSchema::datatype_is_logically_equal(
+ &DataType::Utf8,
+ &DataType::RunEndEncoded(
+ Field::new("run", DataType::Int32, false).into(),
+ Field::new("val", DataType::Utf8, true).into(),
+ )
+ ));
+
+ // Dictionary is logically equal to the logically equivalent value type
+ assert!(DFSchema::datatype_is_logically_equal(
+ &DataType::Utf8View,
+ &DataType::RunEndEncoded(
+ Field::new("run", DataType::Int32, false).into(),
+ Field::new("val", DataType::Utf8, true).into(),
+ )
+ ));
+
+ assert!(DFSchema::datatype_is_logically_equal(
+ &DataType::RunEndEncoded(
+ Field::new("run", DataType::Int32, false).into(),
+ Field::new(
+ "val",
+ DataType::List(Field::new("element", DataType::Utf8, false).into()),
+ true
+ )
+ .into(),
+ ),
+ &DataType::RunEndEncoded(
+ Field::new("run", DataType::Int64, false).into(),
+ Field::new(
+ "val",
+ DataType::List(
+ Field::new("element", DataType::Utf8View, false).into()
+ ),
+ true
+ )
+ .into(),
+ ),
+ ));
+ }
+
#[test]
fn test_datatype_is_semantically_equal() {
assert!(DFSchema::datatype_is_semantically_equal(
@@ -1945,12 +2017,20 @@ mod tests {
&DataType::List(Field::new_list_field(DataType::Int8, true).into()),
&DataType::List(Field::new("element", DataType::Int8, false).into())
));
+ assert!(DFSchema::datatype_is_semantically_equal(
+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),
+ &DataType::ListView(Field::new("element", DataType::Int8, false).into())
+ ));
// Fails if element type is different
assert!(!DFSchema::datatype_is_semantically_equal(
&DataType::List(Field::new_list_field(DataType::Int8, true).into()),
&DataType::List(Field::new_list_field(DataType::Int16, true).into())
));
+ assert!(!DFSchema::datatype_is_semantically_equal(
+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),
+ &DataType::ListView(Field::new_list_field(DataType::Int16, true).into())
+ ));
// Test maps
let map_field = DataType::Map(
@@ -2066,6 +2146,18 @@ mod tests {
));
}
+ #[test]
+ fn test_datatype_is_not_semantically_equivalent_to_ree() {
+ // RunEndEncoded is not semantically equal to its value type
+ assert!(!DFSchema::datatype_is_semantically_equal(
+ &DataType::Utf8,
+ &DataType::RunEndEncoded(
+ Field::new("run", DataType::Int32, false).into(),
+ Field::new("val", DataType::Utf8, true).into(),
+ )
+ ));
+ }
+
fn test_schema_2() -> Schema {
Schema::new(vec![
Field::new("c100", DataType::Boolean, true),