Apache DataFusion SQL Query Engine
chore: add runendencoded & listview types to dfschema equality methods (#24138)
Jeffrey Voc8b4aeaad90d
diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rsindex 2f28cf99cd..262f1dcf61 100644--- a/datafusion/common/src/dfschema.rs+++ b/datafusion/common/src/dfschema.rs@@ -671,6 +671,8 @@ impl DFSchema {/// logically equivalent. For example:/// - a Dictionary<K,V> type is logically equal to a plain V type/// - a Dictionary<K1, V1> is also logically equal to Dictionary<K2, V1>+ /// - a RunEndEncoded<K,V> type is logically equal to a plain V type+ /// - a RunEndEncoded<K1, V1> is also logically equal to RunEndEncoded<K2, V1>/// - Utf8 and Utf8View are logically equalpub fn datatype_is_logically_equal(dt1: &DataType, dt2: &DataType) -> bool {// check nested fields@@ -682,8 +684,17 @@ impl DFSchema {| (othertype, DataType::Dictionary(_, v1)) => {Self::datatype_is_logically_equal(v1.as_ref(), othertype)}+ (DataType::RunEndEncoded(_, v1), DataType::RunEndEncoded(_, v2)) => {+ Self::datatype_is_logically_equal(v1.data_type(), v2.data_type())+ }+ (DataType::RunEndEncoded(_, v1), othertype)+ | (othertype, DataType::RunEndEncoded(_, v1)) => {+ Self::datatype_is_logically_equal(v1.data_type(), othertype)+ }(DataType::List(f1), DataType::List(f2))| (DataType::LargeList(f1), DataType::LargeList(f2))+ | (DataType::ListView(f1), DataType::ListView(f2))+ | (DataType::LargeListView(f1), DataType::LargeListView(f2))| (DataType::FixedSizeList(f1, _), DataType::FixedSizeList(f2, _)) => {// Don't compare the names of the technical inner field// Usually "item" but that's not mandated@@ -742,8 +753,17 @@ impl DFSchema {Self::datatype_is_semantically_equal(k1.as_ref(), k2.as_ref())&& Self::datatype_is_semantically_equal(v1.as_ref(), v2.as_ref())}+ (DataType::RunEndEncoded(k1, v1), DataType::RunEndEncoded(k2, v2)) => {+ Self::datatype_is_semantically_equal(k1.data_type(), k2.data_type())+ && Self::datatype_is_semantically_equal(+ v1.data_type(),+ v2.data_type(),+ )+ }(DataType::List(f1), DataType::List(f2))| (DataType::LargeList(f1), DataType::LargeList(f2))+ | (DataType::ListView(f1), DataType::ListView(f2))+ | (DataType::LargeListView(f1), DataType::LargeListView(f2))| (DataType::FixedSizeList(f1, _), DataType::FixedSizeList(f2, _)) => {// Don't compare the names of the technical inner field// Usually "item" but that's not mandated@@ -1754,12 +1774,20 @@ mod tests {&DataType::List(Field::new_list_field(DataType::Int8, true).into()),&DataType::List(Field::new("element", DataType::Int8, false).into())));+ assert!(DFSchema::datatype_is_logically_equal(+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),+ &DataType::ListView(Field::new("element", DataType::Int8, false).into())+ ));// Fails if element type is differentassert!(!DFSchema::datatype_is_logically_equal(&DataType::List(Field::new_list_field(DataType::Int8, true).into()),&DataType::List(Field::new_list_field(DataType::Int16, true).into())));+ assert!(!DFSchema::datatype_is_logically_equal(+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),+ &DataType::ListView(Field::new_list_field(DataType::Int16, true).into())+ ));// Test mapslet map_field = DataType::Map(@@ -1896,6 +1924,50 @@ mod tests {));}+ #[test]+ fn test_datatype_is_logically_equivalent_to_ree() {+ // RunEndEncoded is logically equal to its value type+ assert!(DFSchema::datatype_is_logically_equal(+ &DataType::Utf8,+ &DataType::RunEndEncoded(+ Field::new("run", DataType::Int32, false).into(),+ Field::new("val", DataType::Utf8, true).into(),+ )+ ));++ // Dictionary is logically equal to the logically equivalent value type+ assert!(DFSchema::datatype_is_logically_equal(+ &DataType::Utf8View,+ &DataType::RunEndEncoded(+ Field::new("run", DataType::Int32, false).into(),+ Field::new("val", DataType::Utf8, true).into(),+ )+ ));++ assert!(DFSchema::datatype_is_logically_equal(+ &DataType::RunEndEncoded(+ Field::new("run", DataType::Int32, false).into(),+ Field::new(+ "val",+ DataType::List(Field::new("element", DataType::Utf8, false).into()),+ true+ )+ .into(),+ ),+ &DataType::RunEndEncoded(+ Field::new("run", DataType::Int64, false).into(),+ Field::new(+ "val",+ DataType::List(+ Field::new("element", DataType::Utf8View, false).into()+ ),+ true+ )+ .into(),+ ),+ ));+ }+#[test]fn test_datatype_is_semantically_equal() {assert!(DFSchema::datatype_is_semantically_equal(@@ -1945,12 +2017,20 @@ mod tests {&DataType::List(Field::new_list_field(DataType::Int8, true).into()),&DataType::List(Field::new("element", DataType::Int8, false).into())));+ assert!(DFSchema::datatype_is_semantically_equal(+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),+ &DataType::ListView(Field::new("element", DataType::Int8, false).into())+ ));// Fails if element type is differentassert!(!DFSchema::datatype_is_semantically_equal(&DataType::List(Field::new_list_field(DataType::Int8, true).into()),&DataType::List(Field::new_list_field(DataType::Int16, true).into())));+ assert!(!DFSchema::datatype_is_semantically_equal(+ &DataType::ListView(Field::new_list_field(DataType::Int8, true).into()),+ &DataType::ListView(Field::new_list_field(DataType::Int16, true).into())+ ));// Test mapslet map_field = DataType::Map(@@ -2066,6 +2146,18 @@ mod tests {));}+ #[test]+ fn test_datatype_is_not_semantically_equivalent_to_ree() {+ // RunEndEncoded is not semantically equal to its value type+ assert!(!DFSchema::datatype_is_semantically_equal(+ &DataType::Utf8,+ &DataType::RunEndEncoded(+ Field::new("run", DataType::Int32, false).into(),+ Field::new("val", DataType::Utf8, true).into(),+ )+ ));+ }+fn test_schema_2() -> Schema {Schema::new(vec![Field::new("c100", DataType::Boolean, true),