Skip to content

Apache DataFusion SQL Query Engine

fix: Spark `map_from_arrays`, `map_from_entries` for sliced lists (#25431)

Neil Conway6e4d9bae3c63
diff --git a/datafusion/spark/src/function/map/utils.rs b/datafusion/spark/src/function/map/utils.rs
index fa6b2a960d..cd8fa9ccdd 100644
--- a/datafusion/spark/src/function/map/utils.rs
+++ b/datafusion/spark/src/function/map/utils.rs
@@ -164,10 +164,14 @@ fn map_deduplicate_keys(
let offsets_len = keys_offsets.len();
let mut new_offsets = Vec::with_capacity(offsets_len);
- let mut cur_keys_offset = keys_offsets
+ // For a sliced list the offsets do not start at 0: `flat_keys` and
+ // `flat_values` are the full child arrays, and only the elements in
+ // `offsets[0]..offsets[len]` belong to the input rows.
+ let first_keys_offset = keys_offsets
.first()
.map(|offset| *offset as usize)
.unwrap_or(0);
+ let mut cur_keys_offset = first_keys_offset;
let mut cur_values_offset = values_offsets
.first()
.map(|offset| *offset as usize)
@@ -178,9 +182,11 @@ fn map_deduplicate_keys(
// Mirror Spark's `ArrayBasedMapBuilder`: the first occurrence of a key
// fixes its position in the output; under LAST_WIN a later duplicate
- // overwrites that slot's value. `keys_mask` selects the first-seen keys,
- // `value_indices` records the source index in `flat_values` to materialize
- // for each output slot (updated in place on overwrite).
+ // overwrites that slot's value. `keys_mask` selects the first-seen keys
+ // and has one bit per key entry of the input rows, so bit 0 corresponds to
+ // `flat_keys[first_keys_offset]`. `value_indices` records the source index
+ // in `flat_values` to materialize for each output slot (updated in place
+ // on overwrite).
let mut keys_mask_builder = BooleanBuilder::new();
let mut value_indices: Vec<i32> = Vec::new();
let mut key_to_output_idx: HashMap<ScalarValue, usize> = HashMap::new();
@@ -231,7 +237,7 @@ fn map_deduplicate_keys(
}
} else {
// The result entry is NULL — no keys/values emitted. Still pad the
- // mask so it stays aligned with `flat_keys`.
+ // mask so it stays aligned with the key entries of later rows.
keys_mask_builder.append_n(num_keys_entries, false);
}
new_offsets.push(new_last_offset);
@@ -239,7 +245,11 @@ fn map_deduplicate_keys(
cur_values_offset += num_values_entries;
}
let keys_mask = keys_mask_builder.finish();
- let needed_keys = filter(&flat_keys, &keys_mask)?;
+ // Every key entry of the input rows must have exactly one mask bit;
+ // otherwise the mask selects the wrong keys for all later rows.
+ debug_assert_eq!(keys_mask.len(), cur_keys_offset - first_keys_offset);
+ let input_keys = flat_keys.slice(first_keys_offset, keys_mask.len());
+ let needed_keys = filter(&input_keys, &keys_mask)?;
let value_indices_array = Int32Array::from(value_indices);
let needed_values = take(&flat_values, &value_indices_array, None)?;
let offsets = OffsetBuffer::new(new_offsets.into());
@@ -250,6 +260,7 @@ fn map_deduplicate_keys(
mod tests {
use super::*;
use arrow::array::{Int32Array, StringArray};
+ use arrow::datatypes::Int32Type;
fn int32_utf8_inputs(
keys: Vec<i32>,
@@ -372,4 +383,112 @@ mod tests {
assert!(map.is_null(0));
assert!(!map.is_null(1));
}
+
+ /// Returns the flattened `(key, value)` entries of an Int32 -> Utf8 map.
+ fn int32_utf8_entries(map: &MapArray) -> Vec<(i32, Option<String>)> {
+ let keys = map.keys().as_primitive::<Int32Type>();
+ let values = map.values().as_string::<i32>();
+ keys.values()
+ .iter()
+ .zip(values.iter())
+ .map(|(k, v)| (*k, v.map(str::to_string)))
+ .collect()
+ }
+
+ #[test]
+ fn sliced_input_reads_keys_and_values_from_offset_range() {
+ let (keys, values) = int32_utf8_inputs(
+ vec![99, 98, 97, 1, 2, 3, 96],
+ vec![Some("z"), Some("a"), Some("b"), Some("c"), Some("y")],
+ );
+ let keys_offsets = [3i32, 5, 6];
+ let values_offsets = [1i32, 3, 4];
+
+ let result = map_from_keys_values_offsets_nulls(
+ &keys,
+ &values,
+ &keys_offsets,
+ &values_offsets,
+ None,
+ None,
+ false,
+ )
+ .unwrap();
+
+ let map = result.as_map();
+ assert_eq!(map.value_offsets(), &[0, 2, 3]);
+ assert_eq!(
+ int32_utf8_entries(map),
+ vec![
+ (1, Some("a".to_string())),
+ (2, Some("b".to_string())),
+ (3, Some("c".to_string())),
+ ]
+ );
+ }
+
+ #[test]
+ fn sliced_input_last_win() {
+ let (keys, values) = int32_utf8_inputs(
+ vec![99, 1, 2, 1, 96],
+ vec![Some("z"), Some("y"), Some("a"), Some("b"), Some("c")],
+ );
+ let keys_offsets = [1i32, 4];
+ let values_offsets = [2i32, 5];
+
+ let result = map_from_keys_values_offsets_nulls(
+ &keys,
+ &values,
+ &keys_offsets,
+ &values_offsets,
+ None,
+ None,
+ true,
+ )
+ .unwrap();
+
+ let map = result.as_map();
+ assert_eq!(map.value_offsets(), &[0, 2]);
+ assert_eq!(
+ int32_utf8_entries(map),
+ vec![(1, Some("c".to_string())), (2, Some("b".to_string()))]
+ );
+ }
+
+ #[test]
+ fn sliced_input_null_row() {
+ // Row 0 is NULL; its (duplicate) keys are skipped but still occupy
+ // positions in the child array ahead of row 1's keys.
+ let (keys, values) = int32_utf8_inputs(
+ vec![99, 1, 1, 2, 3],
+ vec![
+ Some("z"),
+ Some("dup-a"),
+ Some("dup-b"),
+ Some("x"),
+ Some("y"),
+ ],
+ );
+ let offsets = [1i32, 3, 5];
+ let keys_nulls = NullBuffer::from(vec![false, true]);
+
+ let result = map_from_keys_values_offsets_nulls(
+ &keys,
+ &values,
+ &offsets,
+ &offsets,
+ Some(&keys_nulls),
+ None,
+ false,
+ )
+ .unwrap();
+
+ let map = result.as_map();
+ assert_eq!(map.value_offsets(), &[0, 0, 2]);
+ assert!(map.is_null(0));
+ assert_eq!(
+ int32_utf8_entries(map),
+ vec![(2, Some("x".to_string())), (3, Some("y".to_string()))]
+ );
+ }
}
diff --git a/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.slt b/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.slt
index 8bcb705f4c..22612836c8 100644
--- a/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.slt
+++ b/datafusion/sqllogictest/test_files/spark/map/map_from_arrays.slt
@@ -185,3 +185,17 @@ NULL
statement ok
set datafusion.spark.map_key_dedup_policy = 'EXCEPTION';
+
+# Sliced input: OFFSET makes the function see list arrays whose first offset is
+# non-zero, so the child arrays still contain the skipped row's elements.
+query ?
+SELECT map_from_arrays(a, b)
+FROM (
+ SELECT * FROM values
+ (array[99, 98], array['x', 'y'], 1),
+ (array[1, 2], array['a', 'b'], 2)
+ AS tab(a, b, id)
+ ORDER BY id LIMIT 1 OFFSET 1
+);
+----
+{1: a, 2: b}
diff --git a/datafusion/sqllogictest/test_files/spark/map/map_from_entries.slt b/datafusion/sqllogictest/test_files/spark/map/map_from_entries.slt
index f3fd907372..1cfbcd1a01 100644
--- a/datafusion/sqllogictest/test_files/spark/map/map_from_entries.slt
+++ b/datafusion/sqllogictest/test_files/spark/map/map_from_entries.slt
@@ -215,3 +215,17 @@ AS tab(data);
statement ok
set datafusion.spark.map_key_dedup_policy = 'EXCEPTION';
+
+# Sliced input: OFFSET makes the function see a list array whose first offset is
+# non-zero, so the child array still contains the skipped row's entries.
+query ?
+SELECT map_from_entries(data)
+FROM (
+ SELECT * FROM values
+ (array[struct(99, 'x'), struct(98, 'y')], 1),
+ (array[struct(1, 'a'), struct(2, 'b')], 2)
+ AS tab(data, id)
+ ORDER BY id LIMIT 1 OFFSET 1
+);
+----
+{1: a, 2: b}