Skip to content

Apache DataFusion SQL Query Engine

perf: optimize map validation for common key types (#20805)

lyneaa9d8195ac35
diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rs
index e50c4659b1..67e7f314d2 100644
--- a/datafusion/functions-nested/benches/map.rs
+++ b/datafusion/functions-nested/benches/map.rs
@@ -15,9 +15,12 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{Int32Array, ListArray, StringArray};
+use arrow::array::{
+ ArrayRef, BinaryArray, BinaryViewArray, Int32Array, ListArray, StringArray,
+ StringViewArray,
+};
use arrow::buffer::{OffsetBuffer, ScalarBuffer};
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::Field;
use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
@@ -28,33 +31,100 @@ use datafusion_functions_nested::planner::NestedFunctionPlanner;
use rand::Rng;
use rand::prelude::ThreadRng;
use std::collections::HashSet;
+use std::hash::Hash;
use std::hint::black_box;
use std::sync::Arc;
-fn keys(rng: &mut ThreadRng) -> Vec<String> {
- let mut keys = HashSet::with_capacity(1000);
+const MAP_ROWS: usize = 1000;
+const MAP_KEYS_PER_ROW: usize = 1000;
- while keys.len() < 1000 {
- keys.insert(rng.random_range(0..10000).to_string());
+fn gen_unique_values<T>(
+ rng: &mut ThreadRng,
+ mut make_value: impl FnMut(i32) -> T,
+) -> Vec<T>
+where
+ T: Eq + Hash,
+{
+ let mut values = HashSet::with_capacity(MAP_KEYS_PER_ROW);
+
+ while values.len() < MAP_KEYS_PER_ROW {
+ values.insert(make_value(rng.random_range(0..10000)));
}
- keys.into_iter().collect()
+ values.into_iter().collect()
}
-fn values(rng: &mut ThreadRng) -> Vec<i32> {
- let mut values = HashSet::with_capacity(1000);
+fn gen_repeat_values<T: Clone>(values: &[T], repeats: usize) -> Vec<T> {
+ let mut repeated = Vec::with_capacity(values.len() * repeats);
- while values.len() < 1000 {
- values.insert(rng.random_range(0..10000));
+ for _ in 0..repeats {
+ repeated.extend_from_slice(values);
}
- values.into_iter().collect()
+
+ repeated
+}
+
+fn gen_utf8_values(rng: &mut ThreadRng) -> Vec<String> {
+ gen_unique_values(rng, |value| value.to_string())
+}
+
+fn gen_binary_values(rng: &mut ThreadRng) -> Vec<Vec<u8>> {
+ gen_unique_values(rng, |value| value.to_le_bytes().to_vec())
+}
+
+fn gen_primitive_values(rng: &mut ThreadRng) -> Vec<i32> {
+ gen_unique_values(rng, |value| value)
+}
+
+fn list_array(values: ArrayRef, row_count: usize, values_per_row: usize) -> ArrayRef {
+ let offsets = (0..=row_count)
+ .map(|index| (index * values_per_row) as i32)
+ .collect::<Vec<_>>();
+ Arc::new(ListArray::new(
+ Arc::new(Field::new_list_field(values.data_type().clone(), true)),
+ OffsetBuffer::new(ScalarBuffer::from(offsets)),
+ values,
+ None,
+ ))
+}
+
+fn bench_map_case(c: &mut Criterion, name: &str, keys: ArrayRef, values: ArrayRef) {
+ let number_rows = keys.len();
+ let keys = ColumnarValue::Array(keys);
+ let values = ColumnarValue::Array(values);
+
+ let return_type = map_udf()
+ .return_type(&[keys.data_type(), values.data_type()])
+ .expect("should get return type");
+ let arg_fields = vec![
+ Field::new("a", keys.data_type(), true).into(),
+ Field::new("a", values.data_type(), true).into(),
+ ];
+ let return_field = Field::new("f", return_type, true).into();
+ let config_options = Arc::new(ConfigOptions::default());
+
+ c.bench_function(name, |b| {
+ b.iter(|| {
+ black_box(
+ map_udf()
+ .invoke_with_args(ScalarFunctionArgs {
+ args: vec![keys.clone(), values.clone()],
+ arg_fields: arg_fields.clone(),
+ number_rows,
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .expect("map should work on valid values"),
+ );
+ });
+ });
}
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("make_map_1000", |b| {
let mut rng = rand::rng();
- let keys = keys(&mut rng);
- let values = values(&mut rng);
+ let keys = gen_utf8_values(&mut rng);
+ let values = gen_primitive_values(&mut rng);
let mut buffer = Vec::new();
for i in 0..1000 {
buffer.push(Expr::Literal(
@@ -63,9 +133,7 @@ fn criterion_benchmark(c: &mut Criterion) {
));
buffer.push(Expr::Literal(ScalarValue::Int32(Some(values[i])), None));
}
-
let planner = NestedFunctionPlanner {};
-
b.iter(|| {
black_box(
planner
@@ -75,51 +143,73 @@ fn criterion_benchmark(c: &mut Criterion) {
});
});
- c.bench_function("map_1000", |b| {
- let mut rng = rand::rng();
- let field = Arc::new(Field::new_list_field(DataType::Utf8, true));
- let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1000]));
- let key_list = ListArray::new(
- field,
- offsets,
- Arc::new(StringArray::from(keys(&mut rng))),
- None,
- );
- let field = Arc::new(Field::new_list_field(DataType::Int32, true));
- let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1000]));
- let value_list = ListArray::new(
- field,
- offsets,
- Arc::new(Int32Array::from(values(&mut rng))),
- None,
- );
- let keys = ColumnarValue::Scalar(ScalarValue::List(Arc::new(key_list)));
- let values = ColumnarValue::Scalar(ScalarValue::List(Arc::new(value_list)));
-
- let return_type = map_udf()
- .return_type(&[keys.data_type(), values.data_type()])
- .expect("should get return type");
- let arg_fields = vec![
- Field::new("a", keys.data_type(), true).into(),
- Field::new("a", values.data_type(), true).into(),
- ];
- let return_field = Field::new("f", return_type, true).into();
- let config_options = Arc::new(ConfigOptions::default());
+ let mut rng = rand::rng();
+ let values = Arc::new(Int32Array::from(gen_repeat_values(
+ &gen_primitive_values(&mut rng),
+ MAP_ROWS,
+ ))) as ArrayRef;
+ let values = list_array(values, MAP_ROWS, MAP_KEYS_PER_ROW);
+ let map_cases = [
+ (
+ "map_1000_utf8",
+ list_array(
+ Arc::new(StringArray::from(gen_repeat_values(
+ &gen_utf8_values(&mut rng),
+ MAP_ROWS,
+ ))) as ArrayRef,
+ MAP_ROWS,
+ MAP_KEYS_PER_ROW,
+ ),
+ ),
+ (
+ "map_1000_binary",
+ list_array(
+ Arc::new(BinaryArray::from_iter_values(gen_repeat_values(
+ &gen_binary_values(&mut rng),
+ MAP_ROWS,
+ ))) as ArrayRef,
+ MAP_ROWS,
+ MAP_KEYS_PER_ROW,
+ ),
+ ),
+ (
+ "map_1000_utf8_view",
+ list_array(
+ Arc::new(StringViewArray::from(gen_repeat_values(
+ &gen_utf8_values(&mut rng),
+ MAP_ROWS,
+ ))) as ArrayRef,
+ MAP_ROWS,
+ MAP_KEYS_PER_ROW,
+ ),
+ ),
+ (
+ "map_1000_binary_view",
+ list_array(
+ Arc::new(BinaryViewArray::from_iter_values(gen_repeat_values(
+ &gen_binary_values(&mut rng),
+ MAP_ROWS,
+ ))) as ArrayRef,
+ MAP_ROWS,
+ MAP_KEYS_PER_ROW,
+ ),
+ ),
+ (
+ "map_1000_int32",
+ list_array(
+ Arc::new(Int32Array::from(gen_repeat_values(
+ &gen_primitive_values(&mut rng),
+ MAP_ROWS,
+ ))) as ArrayRef,
+ MAP_ROWS,
+ MAP_KEYS_PER_ROW,
+ ),
+ ),
+ ];
- b.iter(|| {
- black_box(
- map_udf()
- .invoke_with_args(ScalarFunctionArgs {
- args: vec![keys.clone(), values.clone()],
- arg_fields: arg_fields.clone(),
- number_rows: 1,
- return_field: Arc::clone(&return_field),
- config_options: Arc::clone(&config_options),
- })
- .expect("map should work on valid values"),
- );
- });
- });
+ for (name, keys) in map_cases {
+ bench_map_case(c, name, keys, Arc::clone(&values));
+ }
}
criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/functions-nested/src/map.rs b/datafusion/functions-nested/src/map.rs
index dad63ee014..e097cbda55 100644
--- a/datafusion/functions-nested/src/map.rs
+++ b/datafusion/functions-nested/src/map.rs
@@ -17,11 +17,18 @@
use std::any::Any;
use std::collections::VecDeque;
+use std::hash::Hash;
use std::sync::Arc;
-use arrow::array::{Array, ArrayData, ArrayRef, MapArray, OffsetSizeTrait, StructArray};
+use arrow::array::{
+ Array, ArrayData, ArrayRef, ArrowPrimitiveType, MapArray, OffsetSizeTrait,
+ StructArray, cast::AsArray,
+};
use arrow::buffer::Buffer;
-use arrow::datatypes::{DataType, Field, SchemaBuilder, ToByteSlice};
+use arrow::datatypes::{
+ DataType, Date32Type, Date64Type, Field, Int8Type, Int16Type, Int32Type, Int64Type,
+ SchemaBuilder, ToByteSlice, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+};
use datafusion_common::utils::{fixed_size_list_to_arrays, list_to_arrays};
use datafusion_common::{
@@ -66,23 +73,30 @@ fn make_map_batch(args: &[ColumnarValue]) -> Result<ColumnarValue> {
let key_array = keys.as_ref();
match keys_arg {
- ColumnarValue::Array(_) => {
- let row_keys = match key_array.data_type() {
- DataType::List(_) => list_to_arrays::<i32>(&keys),
- DataType::LargeList(_) => list_to_arrays::<i64>(&keys),
- DataType::FixedSizeList(_, _) => fixed_size_list_to_arrays(&keys),
- data_type => {
- return exec_err!(
- "Expected list, large_list or fixed_size_list, got {:?}",
- data_type
- );
- }
- };
-
- row_keys
+ ColumnarValue::Array(_) => match key_array.data_type() {
+ DataType::List(_) => keys
+ .as_list::<i32>()
.iter()
- .try_for_each(|key| validate_map_keys(key.as_ref()))?;
- }
+ .flatten()
+ .try_for_each(|row| validate_map_keys(row.as_ref()))?,
+ DataType::LargeList(_) => keys
+ .as_list::<i64>()
+ .iter()
+ .flatten()
+ .try_for_each(|row| validate_map_keys(row.as_ref()))?,
+ DataType::FixedSizeList(_, _) => {
+ keys.as_fixed_size_list()
+ .iter()
+ .flatten()
+ .try_for_each(|row| validate_map_keys(row.as_ref()))?
+ }
+ data_type => {
+ return exec_err!(
+ "Expected list, large_list or fixed_size_list, got {:?}",
+ data_type
+ );
+ }
+ },
ColumnarValue::Scalar(_) => {
validate_map_keys(key_array)?;
}
@@ -93,8 +107,67 @@ fn make_map_batch(args: &[ColumnarValue]) -> Result<ColumnarValue> {
make_map_batch_internal(&keys, &values, can_evaluate_to_const, &keys_arg.data_type())
}
-/// Validates that map keys are non-null and unique.
-fn validate_map_keys(array: &dyn Array) -> Result<()> {
+fn validate_unique_primitive_keys<T: ArrowPrimitiveType>(array: &dyn Array) -> Result<()>
+where
+ T::Native: Copy + Eq + Hash + std::fmt::Display,
+{
+ let primitive_array = array.as_primitive::<T>();
+ if primitive_array.null_count() > 0 {
+ return exec_err!("map key cannot be null");
+ }
+
+ if let Some(value) = find_duplicate_value(
+ primitive_array.len(),
+ primitive_array.values().iter().copied(),
+ ) {
+ return exec_err!("map key must be unique, duplicate key found: {}", value);
+ }
+
+ Ok(())
+}
+
+fn validate_unique_str_keys<'a>(
+ null_count: usize,
+ len: usize,
+ values: impl IntoIterator<Item = &'a str>,
+) -> Result<()> {
+ if null_count > 0 {
+ return exec_err!("map key cannot be null");
+ }
+
+ if let Some(value) = find_duplicate_value(len, values) {
+ return exec_err!("map key must be unique, duplicate key found: {}", value);
+ }
+
+ Ok(())
+}
+
+fn validate_unique_binary_keys<'a>(
+ null_count: usize,
+ len: usize,
+ values: impl IntoIterator<Item = &'a [u8]>,
+) -> Result<()> {
+ if null_count > 0 {
+ return exec_err!("map key cannot be null");
+ }
+
+ if let Some(value) = find_duplicate_value(len, values) {
+ return exec_err!("map key must be unique, duplicate key found: {:?}", value);
+ }
+
+ Ok(())
+}
+
+fn find_duplicate_value<T, I>(len: usize, values: I) -> Option<T>
+where
+ T: Copy + Eq + Hash,
+ I: IntoIterator<Item = T>,
+{
+ let mut seen_keys = HashSet::with_capacity(len);
+ values.into_iter().find(|value| !seen_keys.insert(*value))
+}
+
+fn validate_unique_keys_generic(array: &dyn Array) -> Result<()> {
let mut seen_keys = HashSet::with_capacity(array.len());
for i in 0..array.len() {
@@ -114,6 +187,47 @@ fn validate_map_keys(array: &dyn Array) -> Result<()> {
Ok(())
}
+/// Validates that map keys are non-null and unique.
+fn validate_map_keys(array: &dyn Array) -> Result<()> {
+ match array.data_type() {
+ DataType::Int8 => validate_unique_primitive_keys::<Int8Type>(array),
+ DataType::Int16 => validate_unique_primitive_keys::<Int16Type>(array),
+ DataType::Int32 => validate_unique_primitive_keys::<Int32Type>(array),
+ DataType::Int64 => validate_unique_primitive_keys::<Int64Type>(array),
+ DataType::UInt8 => validate_unique_primitive_keys::<UInt8Type>(array),
+ DataType::UInt16 => validate_unique_primitive_keys::<UInt16Type>(array),
+ DataType::UInt32 => validate_unique_primitive_keys::<UInt32Type>(array),
+ DataType::UInt64 => validate_unique_primitive_keys::<UInt64Type>(array),
+ DataType::Date32 => validate_unique_primitive_keys::<Date32Type>(array),
+ DataType::Date64 => validate_unique_primitive_keys::<Date64Type>(array),
+ DataType::Utf8 => {
+ let arr = array.as_string::<i32>();
+ validate_unique_str_keys(arr.null_count(), arr.len(), arr.iter().flatten())
+ }
+ DataType::LargeUtf8 => {
+ let arr = array.as_string::<i64>();
+ validate_unique_str_keys(arr.null_count(), arr.len(), arr.iter().flatten())
+ }
+ DataType::Utf8View => {
+ let arr = array.as_string_view();
+ validate_unique_str_keys(arr.null_count(), arr.len(), arr.iter().flatten())
+ }
+ DataType::Binary => {
+ let arr = array.as_binary::<i32>();
+ validate_unique_binary_keys(arr.null_count(), arr.len(), arr.iter().flatten())
+ }
+ DataType::LargeBinary => {
+ let arr = array.as_binary::<i64>();
+ validate_unique_binary_keys(arr.null_count(), arr.len(), arr.iter().flatten())
+ }
+ DataType::BinaryView => {
+ let arr = array.as_binary_view();
+ validate_unique_binary_keys(arr.null_count(), arr.len(), arr.iter().flatten())
+ }
+ _ => validate_unique_keys_generic(array),
+ }
+}
+
fn get_first_array_ref(columnar_value: &ColumnarValue) -> Result<ArrayRef> {
match columnar_value {
ColumnarValue::Scalar(value) => match value {
@@ -379,7 +493,7 @@ fn make_map_array_internal<O: OffsetSizeTrait>(
let nulls_bitmap = keys.nulls().cloned();
let keys = list_to_arrays::<O>(keys);
- let values = list_to_arrays::<O>(values);
+ let values = list_to_arrays_skipping_null_rows::<O>(values, nulls_bitmap.as_ref());
build_map_array(
&keys,
@@ -406,7 +520,8 @@ fn make_map_array_from_fixed_size_list(
let nulls_bitmap = keys.nulls().cloned();
let keys = fixed_size_list_to_arrays(keys);
- let values = fixed_size_list_to_arrays(values);
+ let values =
+ fixed_size_list_to_arrays_skipping_null_rows(values, nulls_bitmap.as_ref());
build_map_array(
&keys,
@@ -417,6 +532,41 @@ fn make_map_array_from_fixed_size_list(
nulls_bitmap,
)
}
+fn list_to_arrays_skipping_null_rows<O: OffsetSizeTrait>(
+ array: &ArrayRef,
+ null_rows: Option<&arrow::buffer::NullBuffer>,
+) -> Vec<ArrayRef> {
+ array
+ .as_list::<O>()
+ .iter()
+ .enumerate()
+ .filter_map(|(i, row)| {
+ if null_rows.is_some_and(|nulls| nulls.is_null(i)) {
+ None
+ } else {
+ row
+ }
+ })
+ .collect()
+}
+
+fn fixed_size_list_to_arrays_skipping_null_rows(
+ array: &ArrayRef,
+ null_rows: Option<&arrow::buffer::NullBuffer>,
+) -> Vec<ArrayRef> {
+ array
+ .as_fixed_size_list()
+ .iter()
+ .enumerate()
+ .filter_map(|(i, row)| {
+ if null_rows.is_some_and(|nulls| nulls.is_null(i)) {
+ None
+ } else {
+ row
+ }
+ })
+ .collect()
+}
/// Common logic to build a MapArray from decomposed list arrays
fn build_map_array(
@@ -427,6 +577,10 @@ fn build_map_array(
original_len: usize,
nulls_bitmap: Option<arrow::buffer::NullBuffer>,
) -> Result<ColumnarValue> {
+ if keys.len() != values.len() {
+ return exec_err!("map requires key and value lists to have the same length");
+ }
+
let mut key_array_vec = vec![];
let mut value_array_vec = vec![];
for (k, v) in keys.iter().zip(values.iter()) {
@@ -692,7 +846,7 @@ mod tests {
use arrow::array::FixedSizeListBuilder;
- // Build keys array as FixedSizeList(2): [['a', 'b'], ['c', 'd']]
+ // Build keys array as FixedSizeList(2): [['a', 'b'], NULL, ['c', 'd']]
let key_values_builder = arrow::array::StringBuilder::new();
let mut key_builder = FixedSizeListBuilder::new(key_values_builder, 2);
@@ -701,6 +855,11 @@ mod tests {
key_builder.values().append_value("b");
key_builder.append(true);
+ // Second map: NULL (entire map is NULL)
+ key_builder.values().append_null();
+ key_builder.values().append_null();
+ key_builder.append(false);
+
// Second map: ['c', 'd']
key_builder.values().append_value("c");
key_builder.values().append_value("d");
@@ -708,7 +867,8 @@ mod tests {
let keys_array = Arc::new(key_builder.finish());
- // Build values array as FixedSizeList(2): [[1, 2], [3, 4]]
+ // Build values array as FixedSizeList(2): [[1, 2], [99, 100], [3, 4]]
+ // The middle row should be ignored because the corresponding key row is NULL.
let value_values_builder = arrow::array::Int32Builder::new();
let mut value_builder = FixedSizeListBuilder::new(value_values_builder, 2);
@@ -716,6 +876,10 @@ mod tests {
value_builder.values().append_value(2);
value_builder.append(true);
+ value_builder.values().append_value(99);
+ value_builder.values().append_value(100);
+ value_builder.append(true);
+
value_builder.values().append_value(3);
value_builder.values().append_value(4);
value_builder.append(true);
@@ -740,8 +904,9 @@ mod tests {
_ => panic!("Expected Array result"),
};
- assert_eq!(map_array.len(), 2, "Should have 2 maps");
+ assert_eq!(map_array.len(), 3, "Should have 3 maps");
assert!(!map_array.is_null(0), "First map should not be NULL");
- assert!(!map_array.is_null(1), "Second map should not be NULL");
+ assert!(map_array.is_null(1), "Second map should be NULL");
+ assert!(!map_array.is_null(2), "Third map should not be NULL");
}
}
diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt
index 7ea54464d3..62e70e6080 100644
--- a/datafusion/sqllogictest/test_files/map.slt
+++ b/datafusion/sqllogictest/test_files/map.slt
@@ -263,6 +263,32 @@ SELECT MAP(arrow_cast(make_array('POST', 'HEAD', 'PATCH'), 'LargeList(Utf8)'), a
----
{POST: 41, HEAD: 33, PATCH: 30}
+query ?
+SELECT MAP(make_array(X'6162', X'6364'), make_array(1, 2));
+----
+{6162: 1, 6364: 2}
+
+query error DataFusion error: Execution error: map key must be unique, duplicate key found: \[100, 117, 112\]
+SELECT MAP(make_array(X'647570', X'647570'), make_array(1, 2));
+
+# StringView test: map with StringView keys
+query ?
+SELECT MAP(arrow_cast(make_array('POST', 'HEAD', 'PATCH'), 'List(Utf8View)'), make_array(41, 33, 30));
+----
+{POST: 41, HEAD: 33, PATCH: 30}
+
+query error DataFusion error: Execution error: map key cannot be null
+SELECT MAP(arrow_cast(make_array('POST', null, 'PATCH'), 'List(Utf8View)'), make_array(41, 33, 30));
+
+# BinaryView test: map with BinaryView keys
+query ?
+SELECT MAP(arrow_cast(make_array(X'6162', X'6364'), 'List(BinaryView)'), make_array(1, 2));
+----
+{6162: 1, 6364: 2}
+
+query error DataFusion error: Execution error: map key must be unique, duplicate key found: \[100, 117, 112\]
+SELECT MAP(arrow_cast(make_array(X'647570', X'647570'), 'List(BinaryView)'), make_array(1, 2));
+
statement ok
create table t as values
('a', 1, 'k1', 10, ['k1', 'k2'], [1, 2], 'POST', [[1,2,3]], ['a']),