Apache DataFusion SQL Query Engine
fix: Spark `map_from_arrays`, `map_from_entries` for sliced lists (#25431)
Neil Conway6e4d9bae3c63
diff --git a/datafusion/spark/src/function/map/utils.rs b/datafusion/spark/src/function/map/utils.rsindex fa6b2a960d..cd8fa9ccdd 100644--- a/datafusion/spark/src/function/map/utils.rs+++ b/datafusion/spark/src/function/map/utils.rs@@ -164,10 +164,14 @@ fn map_deduplicate_keys(let offsets_len = keys_offsets.len();let mut new_offsets = Vec::with_capacity(offsets_len);- let mut cur_keys_offset = keys_offsets+ // For a sliced list the offsets do not start at 0: `flat_keys` and+ // `flat_values` are the full child arrays, and only the elements in+ // `offsets[0]..offsets[len]` belong to the input rows.+ let first_keys_offset = keys_offsets.first().map(|offset| *offset as usize).unwrap_or(0);+ let mut cur_keys_offset = first_keys_offset;let mut cur_values_offset = values_offsets.first().map(|offset| *offset as usize)@@ -178,9 +182,11 @@ fn map_deduplicate_keys(// Mirror Spark's `ArrayBasedMapBuilder`: the first occurrence of a key// fixes its position in the output; under LAST_WIN a later duplicate- // overwrites that slot's value. `keys_mask` selects the first-seen keys,- // `value_indices` records the source index in `flat_values` to materialize- // for each output slot (updated in place on overwrite).+ // overwrites that slot's value. `keys_mask` selects the first-seen keys+ // and has one bit per key entry of the input rows, so bit 0 corresponds to+ // `flat_keys[first_keys_offset]`. `value_indices` records the source index+ // in `flat_values` to materialize for each output slot (updated in place+ // on overwrite).let mut keys_mask_builder = BooleanBuilder::new();let mut value_indices: Vec<i32> = Vec::new();let mut key_to_output_idx: HashMap<ScalarValue, usize> = HashMap::new();@@ -231,7 +237,7 @@ fn map_deduplicate_keys(}} else {// The result entry is NULL — no keys/values emitted. Still pad the- // mask so it stays aligned with `flat_keys`.+ // mask so it stays aligned with the key entries of later rows.keys_mask_builder.append_n(num_keys_entries, false);}new_offsets.push(new_last_offset);@@ -239,7 +245,11 @@ fn map_deduplicate_keys(cur_values_offset += num_values_entries;}let keys_mask = keys_mask_builder.finish();- let needed_keys = filter(&flat_keys, &keys_mask)?;+ // Every key entry of the input rows must have exactly one mask bit;+ // otherwise the mask selects the wrong keys for all later rows.+ debug_assert_eq!(keys_mask.len(), cur_keys_offset - first_keys_offset);+ let input_keys = flat_keys.slice(first_keys_offset, keys_mask.len());+ let needed_keys = filter(&input_keys, &keys_mask)?;let value_indices_array = Int32Array::from(value_indices);let needed_values = take(&flat_values, &value_indices_array, None)?;let offsets = OffsetBuffer::new(new_offsets.into());@@ -250,6 +260,7 @@ fn map_deduplicate_keys(mod tests {use super::*;use arrow::array::{Int32Array, StringArray};+ use arrow::datatypes::Int32Type;fn int32_utf8_inputs(keys: Vec<i32>,@@ -372,4 +383,112 @@ mod tests {assert!(map.is_null(0));assert!(!map.is_null(1));}++ /// Returns the flattened `(key, value)` entries of an Int32 -> Utf8 map.+ fn int32_utf8_entries(map: &MapArray) -> Vec<(i32, Option<String>)> {+ let keys = map.keys().as_primitive::<Int32Type>();+ let values = map.values().as_string::<i32>();+ keys.values()+ .iter()+ .zip(values.iter())+ .map(|(k, v)| (*k, v.map(str::to_string)))+ .collect()+ }++ #[test]+ fn sliced_input_reads_keys_and_values_from_offset_range() {+ let (keys, values) = int32_utf8_inputs(+ vec![99, 98, 97, 1, 2, 3, 96],+ vec![Some("z"), Some("a"), Some("b"), Some("c"), Some("y")],+ );+ let keys_offsets = [3i32, 5, 6];+ let values_offsets = [1i32, 3, 4];++ let result = map_from_keys_values_offsets_nulls(+ &keys,+ &values,+ &keys_offsets,+ &values_offsets,+ None,+ None,+ false,+ )+ .unwrap();++ let map = result.as_map();+ assert_eq!(map.value_offsets(), &[0, 2, 3]);+ assert_eq!(+ int32_utf8_entries(map),+ vec![+ (1, Some("a".to_string())),+ (2, Some("b".to_string())),+ (3, Some("c".to_string())),+ ]+ );+ }++ #[test]+ fn sliced_input_last_win() {+ let (keys, values) = int32_utf8_inputs(+ vec![99, 1, 2, 1, 96],+ vec![Some("z"), Some("y"), Some("a"), Some("b"), Some("c")],+ );+ let keys_offsets = [1i32, 4];+ let values_offsets = [2i32, 5];++ let result = map_from_keys_values_offsets_nulls(+ &keys,+ &values,+ &keys_offsets,+ &values_offsets,+ None,+ None,+ true,+ )+ .unwrap();++ let map = result.as_map();+ assert_eq!(map.value_offsets(), &[0, 2]);+ assert_eq!(+ int32_utf8_entries(map),+ vec![(1, Some("c".to_string())), (2, Some("b".to_string()))]+ );+ }++ #[test]+ fn sliced_input_null_row() {+ // Row 0 is NULL; its (duplicate) keys are skipped but still occupy+ // positions in the child array ahead of row 1's keys.+ let (keys, values) = int32_utf8_inputs(+ vec![99, 1, 1, 2, 3],+ vec![+ Some("z"),+ Some("dup-a"),+ Some("dup-b"),+ Some("x"),+ Some("y"),+ ],+ );+ let offsets = [1i32, 3, 5];+ let keys_nulls = NullBuffer::from(vec![false, true]);++ let result = map_from_keys_values_offsets_nulls(+ &keys,+ &values,+ &offsets,+ &offsets,+ Some(&keys_nulls),+ None,+ false,+ )+ .unwrap();++ let map = result.as_map();+ assert_eq!(map.value_offsets(), &[0, 0, 2]);+ assert!(map.is_null(0));+ assert_eq!(+ int32_utf8_entries(map),+ vec![(2, Some("x".to_string())), (3, Some("y".to_string()))]+ );+ }}diff --git a/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.slt b/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.sltindex 8bcb705f4c..22612836c8 100644--- a/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.slt+++ b/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.slt@@ -185,3 +185,17 @@ NULLstatement okset datafusion.spark.map_key_dedup_policy = 'EXCEPTION';++# Sliced input: OFFSET makes the function see list arrays whose first offset is+# non-zero, so the child arrays still contain the skipped row's elements.+query ?+SELECT map_from_arrays(a, b)+FROM (+ SELECT * FROM values+ (array[99, 98], array['x', 'y'], 1),+ (array[1, 2], array['a', 'b'], 2)+ AS tab(a, b, id)+ ORDER BY id LIMIT 1 OFFSET 1+);+----+{1: a, 2: b}diff --git a/datafusion/sqllogictest/test_files/spark/map/map_from_entries.slt b/datafusion/sqllogictest/test_files/spark/map/map_from_entries.sltindex f3fd907372..1cfbcd1a01 100644--- a/datafusion/sqllogictest/test_files/spark/map/map_from_entries.slt+++ b/datafusion/sqllogictest/test_files/spark/map/map_from_entries.slt@@ -215,3 +215,17 @@ AS tab(data);statement okset datafusion.spark.map_key_dedup_policy = 'EXCEPTION';++# Sliced input: OFFSET makes the function see a list array whose first offset is+# non-zero, so the child array still contains the skipped row's entries.+query ?+SELECT map_from_entries(data)+FROM (+ SELECT * FROM values+ (array[struct(99, 'x'), struct(98, 'y')], 1),+ (array[struct(1, 'a'), struct(2, 'b')], 2)+ AS tab(data, id)+ ORDER BY id LIMIT 1 OFFSET 1+);+----+{1: a, 2: b}