Apache DataFusion SQL Query Engine
fix, perf: Use Arrow comparator for key comparison in `map_extract` (#24999)
Neil Conwayc29363fc05d0
diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rsindex 9cc4289ca1..c65696aefe 100644--- a/datafusion/functions-nested/benches/map.rs+++ b/datafusion/functions-nested/benches/map.rs@@ -16,17 +16,18 @@// under the License.use arrow::array::{- ArrayRef, BinaryArray, BinaryViewArray, Int32Array, ListArray, StringArray,- StringViewArray,+ Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, ListArray, MapArray,+ StringArray, StringViewArray, StructArray,};use arrow::buffer::{OffsetBuffer, ScalarBuffer};-use arrow::datatypes::Field;-use criterion::{Criterion, criterion_group, criterion_main};+use arrow::datatypes::{DataType, Field};+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};use datafusion_common::ScalarValue;use datafusion_common::config::ConfigOptions;use datafusion_expr::planner::ExprPlanner;use datafusion_expr::{ColumnarValue, Expr, ScalarFunctionArgs};use datafusion_functions_nested::map::map_udf;+use datafusion_functions_nested::map_extract::map_extract_udf;use datafusion_functions_nested::planner::NestedFunctionPlanner;use rand::prelude::*;use std::collections::HashSet;@@ -208,5 +209,108 @@ fn criterion_benchmark(c: &mut Criterion) {}}-criterion_group!(benches, criterion_benchmark);+fn bench_map_extract(c: &mut Criterion) {+ let udf = map_extract_udf();+ let config_options = Arc::new(ConfigOptions::default());+ let mut group = c.benchmark_group("map_extract");++ for (rows, width) in [(1, 0), (1, 1), (1024, 1), (1024, 32)] {+ for key_type in ["int32", "utf8_view", "struct"] {+ let make_keys = |keys: Vec<i32>| -> ArrayRef {+ match key_type {+ "int32" => Arc::new(Int32Array::from(keys)),+ "utf8_view" => Arc::new(StringViewArray::from_iter_values(+ keys.iter().map(|key| format!("key_{key:016}")),+ )),+ "struct" => Arc::new(StructArray::from(vec![(+ Arc::new(Field::new("key", DataType::Int32, false)),+ Arc::new(Int32Array::from(keys)) as ArrayRef,+ )])),+ _ => unreachable!(),+ }+ };+ let keys = make_keys((0..rows).flat_map(|_| 0..width as i32).collect());+ let entries = StructArray::from(vec![+ (+ Arc::new(Field::new("key", keys.data_type().clone(), false)),+ keys,+ ),+ (+ Arc::new(Field::new("value", DataType::Int32, false)),+ Arc::new(Int32Array::from_iter_values(0..(rows * width) as i32))+ as ArrayRef,+ ),+ ]);+ let map: ArrayRef = Arc::new(MapArray::new(+ Arc::new(Field::new("entries", entries.data_type().clone(), false)),+ OffsetBuffer::from_lengths(std::iter::repeat_n(width, rows)),+ entries,+ None,+ false,+ ));+ let lookups: &[&str] = if width <= 1 {+ &["last"]+ } else {+ &["first", "last", "missing", "varying"]+ };+ for &lookup in lookups {+ let query_keys = match lookup {+ "first" => vec![0],+ "last" => vec![width.saturating_sub(1) as i32],+ "missing" => vec![width as i32],+ // Mix matches and misses with a different lookup key per row.+ "varying" => {+ (0..rows).map(|row| (row % (width + 1)) as i32).collect()+ }+ _ => unreachable!(),+ };+ let query_keys = make_keys(query_keys);+ let query_keys = if lookup == "varying" {+ ColumnarValue::Array(query_keys)+ } else {+ ColumnarValue::Scalar(+ ScalarValue::try_from_array(&query_keys, 0).unwrap(),+ )+ };+ let args = vec![ColumnarValue::Array(Arc::clone(&map)), query_keys];+ let arg_fields = args+ .iter()+ .map(|arg| Field::new("arg", arg.data_type(), true).into())+ .collect::<Vec<_>>();+ let return_type = udf+ .return_type(+ &args+ .iter()+ .map(ColumnarValue::data_type)+ .collect::<Vec<_>>(),+ )+ .unwrap();+ let return_field = Arc::new(Field::new("result", return_type, true));+ group.bench_function(+ BenchmarkId::new(+ format!("{key_type}/{lookup}"),+ format!("{rows}x{width}"),+ ),+ |b| {+ b.iter(|| {+ black_box(+ udf.invoke_with_args(ScalarFunctionArgs {+ args: args.clone(),+ arg_fields: arg_fields.clone(),+ number_rows: rows,+ return_field: Arc::clone(&return_field),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ });+ },+ );+ }+ }+ }+ group.finish();+}++criterion_group!(benches, criterion_benchmark, bench_map_extract);criterion_main!(benches);diff --git a/datafusion/functions-nested/src/map_extract.rs b/datafusion/functions-nested/src/map_extract.rsindex 40340ec2cf..a9ef716514 100644--- a/datafusion/functions-nested/src/map_extract.rs+++ b/datafusion/functions-nested/src/map_extract.rs@@ -19,10 +19,12 @@use crate::utils::{get_map_entry_field, make_scalar_function};use arrow::array::{- Array, ArrayRef, Capacities, ListArray, MapArray, MutableArrayData, make_array,+ Array, ArrayRef, ListArray, MapArray, MutableArrayData, make_array, new_empty_array,};use arrow::buffer::OffsetBuffer;+use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field};+use arrow_ord::ord::make_comparator;use datafusion_common::utils::take_function_args;use datafusion_common::{Result, cast::as_map_array, exec_err};use datafusion_expr::{@@ -31,7 +33,6 @@ use datafusion_expr::{};use datafusion_macros::user_doc;use std::sync::Arc;-use std::vec;// Create static instances of ScalarUDFs for each functionmake_udf_expr_and_func!(@@ -149,43 +150,55 @@ fn general_map_extract_inner(query_keys_array: &dyn Array,) -> Result<ArrayRef> {let keys = map_array.keys();- let mut offsets = vec![0_i32];-let values = map_array.values();- let original_data = values.to_data();- let capacity = Capacities::Array(original_data.len());+ let field = Arc::new(Field::new_list_field(map_array.value_type().clone(), true));+ let map_offsets = map_array.value_offsets();+ if map_offsets.first() == map_offsets.last() {+ return Ok(Arc::new(ListArray::new(+ field,+ OffsetBuffer::new_zeroed(map_array.len()),+ new_empty_array(values.data_type()),+ map_array.nulls().cloned(),+ )));+ }- let mut mutable =- MutableArrayData::with_capacities(vec![&original_data], true, capacity);+ // Compare keys by index using a single comparator for the batch.+ let compare =+ make_comparator(keys.as_ref(), query_keys_array, SortOptions::default())?;+ let mut offsets = Vec::with_capacity(map_array.len() + 1);+ offsets.push(0_i32);- for (row_index, offset_window) in map_array.value_offsets().windows(2).enumerate() {+ let original_data = values.to_data();+ // There is at most one output value per map row.+ let mut mutable = MutableArrayData::new(+ vec![&original_data],+ false,+ map_array.len().min(values.len()),+ );++ for (row_index, offset_window) in map_offsets.windows(2).enumerate() {let start = offset_window[0] as usize;let end = offset_window[1] as usize;- let len = end - start;-- let query_key = query_keys_array.slice(row_index, 1);-- let value_index =- (0..len).find(|&i| keys.slice(start + i, 1).as_ref() == query_key.as_ref());+ let mut offset = offsets[row_index];- match value_index {- Some(index) => {- mutable.try_extend(0, start + index, start + index + 1)?;- }- None => {- mutable.try_extend_nulls(1)?;- }+ if map_array.is_valid(row_index)+ && let Some(index) = (start..end).find(|&i| compare(i, row_index).is_eq())+ {+ mutable.try_extend(0, index, index + 1)?;+ offset += 1;}- offsets.push(offsets[row_index] + 1);++ // A missing key results in an empty list.+ offsets.push(offset);}let data = mutable.freeze();Ok(Arc::new(ListArray::new(- Arc::new(Field::new_list_field(map_array.value_type().clone(), true)),+ field,OffsetBuffer::<i32>::new(offsets.into()),- Arc::new(make_array(data)),- None,+ make_array(data),+ map_array.nulls().cloned(),)))}@@ -210,3 +223,102 @@ fn map_extract_inner(args: &[ArrayRef]) -> Result<ArrayRef> {general_map_extract_inner(map_array, key_arg)}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::{Float64Array, Int32Array, StructArray};+ use arrow::buffer::NullBuffer;+ use arrow::datatypes::Int32Type;++ fn make_map(+ keys: ArrayRef,+ values: Vec<i32>,+ offsets: Vec<i32>,+ nulls: Option<NullBuffer>,+ ) -> MapArray {+ let entries = StructArray::from(vec![+ (+ Arc::new(Field::new("key", keys.data_type().clone(), false)),+ keys,+ ),+ (+ Arc::new(Field::new("value", DataType::Int32, true)),+ Arc::new(Int32Array::from(values)) as ArrayRef,+ ),+ ]);+ MapArray::new(+ Arc::new(Field::new("entries", entries.data_type().clone(), false)),+ OffsetBuffer::new(offsets.into()),+ entries,+ nulls,+ false,+ )+ }++ #[test]+ fn map_extract_sliced_maps() -> Result<()> {+ let map = make_map(+ Arc::new(Int32Array::from(vec![0, 1, 2, 3])),+ vec![0, 10, 20, 30],+ vec![0, 1, 3, 4],+ None,+ );+ let query_keys = Int32Array::from(vec![0, 2, 9]);++ // Map offsets address the original entries; query indices address the slice.+ let result =+ general_map_extract_inner(&map.slice(1, 2), &query_keys.slice(1, 2))?;+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>([+ Some(vec![Some(20)]),+ Some(vec![]),+ ]);+ assert_eq!(result.as_ref(), &expected);++ // Empty slices may retain the original nonempty keys and values buffers.+ let result =+ general_map_extract_inner(&map.slice(1, 0), &query_keys.slice(1, 0))?;+ assert_eq!(result.len(), 0);+ Ok(())+ }++ #[test]+ fn map_extract_all_empty_maps_with_nulls() -> Result<()> {+ // No entries exist to scan, but the null map must still produce NULL+ // rather than an empty list.+ let map = make_map(+ Arc::new(Int32Array::from(Vec::<i32>::new())),+ vec![],+ vec![0, 0, 0],+ Some(NullBuffer::from(vec![true, false])),+ );+ let result = general_map_extract_inner(&map, &Int32Array::from(vec![1, 1]))?;+ let expected =+ ListArray::from_iter_primitive::<Int32Type, _, _>([Some(vec![]), None]);+ assert_eq!(result.as_ref(), &expected);+ Ok(())+ }++ #[test]+ fn map_extract_float_keys() -> Result<()> {+ let nan = f64::NAN;+ let other_nan = f64::from_bits(nan.to_bits() + 1);+ let map = make_map(+ Arc::new(Float64Array::from(vec![-0.0, 0.0, nan, other_nan])),+ vec![1, 2, 3, 4],+ vec![0, 4],+ None,+ );++ // Signed zeros and distinct NaN payloads identify different keys.+ for (query, expected) in [(-0.0, 1), (0.0, 2), (nan, 3), (other_nan, 4)] {+ let result =+ general_map_extract_inner(&map, &Float64Array::from(vec![query]))?;+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>([Some(+ vec![Some(expected)],+ )]);+ assert_eq!(result.as_ref(), &expected);+ }+ Ok(())+ }+}diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.sltindex 59f340a083..32fdeaac1d 100644--- a/datafusion/sqllogictest/test_files/map.slt+++ b/datafusion/sqllogictest/test_files/map.slt@@ -618,21 +618,21 @@ query ????select map_extract(MAP {'a': 1, 'b': NULL, 'c': 3}, 'a'), map_extract(MAP {'a': 1, 'b': NULL, 'c': 3}, 'b'),map_extract(MAP {'a': 1, 'b': NULL, 'c': 3}, 'c'), map_extract(MAP {'a': 1, 'b': NULL, 'c': 3}, 'd');-----[1] [NULL] [3] [NULL]+[1] [NULL] [3] []# key is integerquery ????select map_extract(MAP {1: 1, 2: NULL, 3:3}, 1), map_extract(MAP {1: 1, 2: NULL, 3:3}, 2),map_extract(MAP {1: 1, 2: NULL, 3:3}, 3), map_extract(MAP {1: 1, 2: NULL, 3:3}, 4);-----[1] [NULL] [3] [NULL]+[1] [NULL] [3] []# value is listquery ????select map_extract(MAP {1: [1, 2], 2: NULL, 3:[3]}, 1), map_extract(MAP {1: [1, 2], 2: NULL, 3:[3]}, 2),map_extract(MAP {1: [1, 2], 2: NULL, 3:[3]}, 3), map_extract(MAP {1: [1, 2], 2: NULL, 3:[3]}, 4);-----[[1, 2]] [NULL] [[3]] [NULL]+[[1, 2]] [NULL] [[3]] []# key in map and query key are different typesquery ?????@@ -640,7 +640,7 @@ select map_extract(MAP {1: 1, 2: 2, 3:3}, '1'), map_extract(MAP {1: 1, 2: 2, 3:3map_extract(MAP {1.0: 1, 2: 2, 3:3}, '1'), map_extract(MAP {'1': 1, '2': 2, '3':3}, 1.0),map_extract(MAP {arrow_cast('1', 'Utf8View'): 1, arrow_cast('2', 'Utf8View'): 2, arrow_cast('3', 'Utf8View'):3}, '1');-----[1] [1] [1] [NULL] [1]+[1] [1] [1] [] [1]# null argquery ?@@ -648,14 +648,47 @@ select map_extract(NULL, 'a');----NULL+# Empty maps, null lookup keys, and the element_at alias+query ????+select map_extract(map(CAST([] AS VARCHAR[]), CAST([] AS INT[])), 'a'), map_extract(MAP {'a': 1}, NULL),+ element_at(MAP {'a': 1, 'b': NULL}, 'missing'), element_at(MAP {'a': 1, 'b': NULL}, 'b');+----+[] [] [] [NULL]++# Materialize a dictionary of structs so extracting d preserves a valid dictionary+# index pointing to a null value. A typed null instead has a null dictionary index.+statement ok+CREATE TABLE map_extract_dictionary_structs AS+SELECT arrow_cast(+ named_struct('d', CAST(NULL AS VARCHAR)),+ 'Dictionary(Int8, Struct("d": Utf8))'+) AS s;++# These non-null struct keys compare equal despite their different null encodings.+query B??+WITH keys AS (+ SELECT named_struct('d', s['d']) AS stored_key,+ named_struct('d', arrow_cast(NULL, 'Dictionary(Int8, Utf8)')) AS query_key+ FROM map_extract_dictionary_structs+)+SELECT stored_key = query_key,+ map_extract(MAP {stored_key: 42}, query_key),+ map_extract(MAP {query_key: 42}, stored_key)+FROM keys;+----+true [42] [42]++statement ok+DROP TABLE map_extract_dictionary_structs;+# map_extract with columnsquery ???select map_extract(column1, 1), map_extract(column1, 5), map_extract(column1, 7) from map_array_table_1;-----[[1, NULL, 3]] [NULL] [NULL]-[NULL] [[4, NULL, 6]] [NULL]-[NULL] [NULL] [[1, NULL, 3]]-[NULL] [NULL] [NULL]+[[1, NULL, 3]] [] []+[] [[4, NULL, 6]] []+[] [] [[1, NULL, 3]]+NULL NULL NULLquery ?select column1[1] from map_array_table_1;@@ -699,22 +732,22 @@ select map_extract(column1, column2), map_extract(column1, column3), map_extract----[[1, NULL, 3]] [[1, NULL, 3]] [[1, NULL, 3]][[4, NULL, 6]] [[4, NULL, 6]] [[4, NULL, 6]]-[NULL] [NULL] [NULL]-[NULL] [NULL] [NULL]+[] [] []+NULL NULL NULLquery ???select map_extract(column1, column2), map_extract(column1, column3), map_extract(column1, column4) from map_array_table_2;-----[[1, NULL, 3]] [NULL] [[1, NULL, 3]]-[[4, NULL, 6]] [NULL] [[4, NULL, 6]]-[NULL] [NULL] [NULL]+[[1, NULL, 3]] [] [[1, NULL, 3]]+[[4, NULL, 6]] [] [[4, NULL, 6]]+[] [] []query ???select map_extract(column1, 1), map_extract(column1, 5), map_extract(column1, 7) from map_array_table_2;-----[[1, NULL, 3]] [NULL] [NULL]-[NULL] [[4, NULL, 6]] [NULL]-[NULL] [NULL] [[1, NULL, 3]]+[[1, NULL, 3]] [] []+[] [[4, NULL, 6]] []+[] [] [[1, NULL, 3]]# Tests for map_entriesdiff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 61940bf726..8de2abc5a0 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -225,6 +225,36 @@ The output type of the `floor` and `ceil` UDFs has been changed from the exact iChange the expected type or wrap the expression in `CAST`. It's recommended to avoid relying on decimal's exact precision and scale.+### `map_extract` / `element_at` return an empty list for absent keys++`map_extract` (and its alias `element_at`) previously returned a single-element+list containing `NULL` when the key was not present in the map. It now returns+an empty list, matching the documented behavior and DuckDB. Two related cases+changed at the same time, also matching DuckDB:++- A `NULL` map input now yields `NULL` instead of `[NULL]`.+- A `NULL` lookup key now yields `[]` instead of `[NULL]`.++A key that is present with a `NULL` value still returns `[NULL]`, so absent+keys and `NULL` values are now distinguishable.++**Migration guide:**++```sql+-- Before+SELECT map_extract(MAP {'a': 1}, 'missing'); -- [NULL]++-- After+SELECT map_extract(MAP {'a': 1}, 'missing'); -- []+```++Expressions that assumed the result always has exactly one element, for+example by checking its length, unnesting it, or comparing it to `[NULL]`,+should treat an empty list as the absent-key case instead.++See [issue #24981](https://github.com/apache/datafusion/issues/24981) and+[issue #24983](https://github.com/apache/datafusion/issues/24983) for details.+### `DataFrame::from_columns` accepts `IntoIterator``DataFrame::from_columns` now accepts any `IntoIterator<Item = (&str, ArrayRef)>`