Apache DataFusion SQL Query Engine
fix: `map[key]` panics for nested map keys (#25130)
Neil Conway301235e7a847
diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rsindex 388daa32bf..a0f024bbc7 100644--- a/datafusion/functions/src/core/getfield.rs+++ b/datafusion/functions/src/core/getfield.rs@@ -18,12 +18,11 @@use std::sync::{Arc, OnceLock};use arrow::array::{- Array, BooleanArray, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array,+ Array, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array,make_comparator,};use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, FieldRef};-use arrow_buffer::NullBuffer;use datafusion_common::cast::{as_map_array, as_struct_array};use datafusion_common::{@@ -100,29 +99,19 @@ impl Default for GetFieldFunc {}}-/// Process a map array by finding matching keys and extracting corresponding values.+/// Process a map array with a non-nested key type by comparing the single+/// lookup key against every map key with the `eq` kernel, then scanning the+/// result for each row.///-/// This function handles both simple (scalar) and nested key types by using-/// appropriate comparison strategies.+/// `eq` does not support nested types, so list, struct, and map keys go+/// through [`process_map_with_nested_key`] instead.fn process_map_array(array: &dyn Array,key_array: Arc<dyn Array>,) -> Result<ColumnarValue> {let map_array = as_map_array(array)?;- let keys = if key_array.data_type().is_nested() {- let comparator = make_comparator(- map_array.keys().as_ref(),- key_array.as_ref(),- SortOptions::default(),- )?;- let len = map_array.keys().len().min(key_array.len());- let values = (0..len).map(|i| comparator(i, i).is_eq()).collect();- let nulls = NullBuffer::union(map_array.keys().nulls(), key_array.nulls());- BooleanArray::new(values, nulls)- } else {- let be_compared = Scalar::new(key_array);- arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?- };+ let be_compared = Scalar::new(key_array);+ let keys = arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?;let original_data = map_array.entries().column(1).to_data();let capacity = Capacities::Array(original_data.len());@@ -219,19 +208,14 @@ fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<Columndict.with_values(Arc::clone(field_col)),))}- (DataType::Map(_, _), ScalarValue::List(arr), _) => {- let key_array: Arc<dyn Array> = arr;- process_map_array(&array, key_array)- }- (DataType::Map(_, _), ScalarValue::Struct(arr), _) => {- process_map_array(&array, arr as Arc<dyn Array>)- }- (DataType::Map(_, _), other, _) => {- let data_type = other.data_type();- if data_type.is_nested() {- process_map_with_nested_key(&array, &other.to_array()?)+ (DataType::Map(_, _), key, _) => {+ // The lookup key is a single scalar. `eq` does not support nested+ // key types, so those are matched with a comparator instead.+ let key_array = key.to_array()?;+ if key_array.data_type().is_nested() {+ process_map_with_nested_key(&array, key_array.as_ref())} else {- process_map_array(&array, other.to_array()?)+ process_map_array(&array, key_array)}}(DataType::Struct(_), _, Some(k)) => {@@ -672,8 +656,11 @@ impl ScalarUDFImpl for GetFieldFunc {#[cfg(test)]mod tests {use super::*;- use arrow::array::{ArrayRef, Int32Array, StructArray};- use arrow::datatypes::Fields;+ use arrow::array::{+ ArrayRef, Int32Array, Int32Builder, ListArray, ListBuilder, MapBuilder,+ StructArray,+ };+ use arrow::datatypes::{Fields, Int32Type};#[test]fn test_get_field_utf8view_key() -> Result<()> {@@ -711,6 +698,41 @@ mod tests {Ok(())}+ #[test]+ fn test_get_field_map_list_key() -> Result<()> {+ // One map row with two list keys. The lookup key matches the second+ // entry, so the match is not at the first entry of the row.+ let mut builder = MapBuilder::new(+ None,+ ListBuilder::new(Int32Builder::new()),+ Int32Builder::new(),+ );+ builder.keys().append_value([Some(1), Some(2)]);+ builder.values().append_value(1);+ builder.keys().append_value([Some(3), Some(4)]);+ builder.values().append_value(2);+ builder.append(true)?;+ let base = ColumnarValue::Array(Arc::new(builder.finish()));++ let list_key = |values: Vec<i32>| {+ ScalarValue::List(Arc::new(+ ListArray::from_iter_primitive::<Int32Type, _, _>([Some(+ values.into_iter().map(Some),+ )]),+ ))+ };++ let result = extract_single_field(base.clone(), list_key(vec![3, 4]))?;+ let expected = Int32Array::from(vec![Some(2)]);+ assert_eq!(result.into_array(1)?.as_ref(), &expected as &dyn Array);++ let result = extract_single_field(base, list_key(vec![9, 9]))?;+ let expected = Int32Array::from(vec![None]);+ assert_eq!(result.into_array(1)?.as_ref(), &expected as &dyn Array);++ Ok(())+ }+#[test]fn test_get_field_dict_encoded_struct() -> Result<()> {use arrow::array::{DictionaryArray, StringArray, UInt32Array};diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.sltindex 32fdeaac1d..debaf7f8b3 100644--- a/datafusion/sqllogictest/test_files/map.slt+++ b/datafusion/sqllogictest/test_files/map.slt@@ -597,6 +597,36 @@ NULL# TODO(https://github.com/apache/datafusion/pull/18394): Test accessing map with empty map as key# TODO(https://github.com/apache/datafusion/pull/18394): Test accessing map with null map as key+# accessing map with list key: match on a later entry, absent key, match on the first entry+query III+SELECT MAP {[1,2]:1, [3,4]:2}[[3,4]], MAP {[1,2]:1, [3,4]:2}[[9,9]], MAP {[1,2]:1, [3,4]:2}[[1,2]];+----+2 NULL 1++# accessing map with struct key+query II+SELECT MAP {named_struct('a',1):1, named_struct('a',2):2}[named_struct('a',2)],+ MAP {named_struct('a',1):1, named_struct('a',2):2}[named_struct('a',3)];+----+2 NULL++# accessing map with list key from a column, so the lookup runs on a batch rather than at planning time+statement ok+CREATE TABLE map_list_keys AS+SELECT MAP {[1,2]:1, [3,4]:2} AS m+UNION ALL SELECT MAP {[3,4]:5, [1,2]:6}+UNION ALL SELECT NULL;++query II rowsort+SELECT m[[3,4]], m[[9,9]] FROM map_list_keys;+----+2 NULL+5 NULL+NULL NULL++statement ok+DROP TABLE map_list_keys;+# accessing map with non-string keyquery ISELECT MAKE_MAP(1, null, 2, 33, 3, null)[2];