Apache DataFusion SQL Query Engine
fix: compare ScalarValue maps using Arrow ordering (#25555)
Liang-Chi Hsieh0576a0b40043
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex 34c9f9e03d..37ae21d90d 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -74,7 +74,7 @@ use arrow::array::{Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,- downcast_run_array, new_empty_array, new_null_array,+ downcast_run_array, make_comparator, new_empty_array, new_null_array,};use arrow::buffer::{BooleanBuffer, ScalarBuffer};use arrow::compute::kernels::cast::{CastOptions, cast_with_options};@@ -920,23 +920,11 @@ fn partial_cmp_map(m1: &Arc<MapArray>, m2: &Arc<MapArray>) -> Option<Ordering> {return None;}- for col_index in 0..m1.len() {- let arr1 = m1.entries().column(col_index);- let arr2 = m2.entries().column(col_index);-- let lt_res = arrow::compute::kernels::cmp::lt(arr1, arr2).ok()?;- let eq_res = arrow::compute::kernels::cmp::eq(arr1, arr2).ok()?;-- for j in 0..lt_res.len() {- if lt_res.is_valid(j) && lt_res.value(j) {- return Some(Ordering::Less);- }- if eq_res.is_valid(j) && !eq_res.value(j) {- return Some(Ordering::Greater);- }- }- }- Some(Ordering::Equal)+ // Keep ScalarValue ordering consistent with Arrow sorting and nested comparison.+ // Maps compare lexicographically by entry (key, value), including offsets and nulls.+ let comparator =+ make_comparator(m1.as_ref(), m2.as_ref(), Default::default()).ok()?;+ Some(comparator(0, 0))}impl Eq for ScalarValue {}@@ -7424,6 +7412,41 @@ mod tests {assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));}+ #[test]+ fn test_map_partial_cmp() {+ fn map(entries: Option<Vec<(&str, Option<i32>)>>) -> ScalarValue {+ ScalarValue::Map(Arc::new(MapArray::from_vec_of_maps::<+ StringArray,+ Int32Array,+ _,+ _,+ >(vec![entries], false)))+ }++ let a1 = map(Some(vec![("a", Some(1))]));+ let a2 = map(Some(vec![("a", Some(2))]));+ assert_eq!(a1.partial_cmp(&a2), Some(Ordering::Less));+ assert_eq!(a2.partial_cmp(&a1), Some(Ordering::Greater));++ // Map entries compare as (key, value) pairs, rather than comparing all keys+ // before all values. The first pair therefore determines this ordering.+ let high_first_value = map(Some(vec![("a", Some(100)), ("b", Some(1))]));+ let low_first_value = map(Some(vec![("a", Some(1)), ("c", Some(999))]));+ assert_eq!(+ high_first_value.partial_cmp(&low_first_value),+ Some(Ordering::Greater)+ );++ let prefix = map(Some(vec![("a", Some(1))]));+ let longer = map(Some(vec![("a", Some(1)), ("b", Some(2))]));+ assert_eq!(prefix.partial_cmp(&longer), Some(Ordering::Less));++ let null = map(None);+ let empty = map(Some(vec![]));+ assert_eq!(null.partial_cmp(&empty), Some(Ordering::Less));+ assert_eq!(empty.partial_cmp(&empty), Some(Ordering::Equal));+ }+#[test]fn scalar_value_to_array_u64() -> Result<()> {let value = ScalarValue::UInt64(Some(13u64));