Apache DataFusion SQL Query Engine
array_sort / array_has: monomorphize per native type instead of per Arrow type (reduce binary size) (#24823)
Jay Zhanda89c7c85b1c
diff --git a/datafusion/functions-nested/benches/array_sort.rs b/datafusion/functions-nested/benches/array_sort.rsindex 940c0396cb..2ceee91f4c 100644--- a/datafusion/functions-nested/benches/array_sort.rs+++ b/datafusion/functions-nested/benches/array_sort.rs@@ -22,6 +22,7 @@ use arrow::array::{ArrayRef, BooleanBufferBuilder, Int32Array, ListArray, Stringuse arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::datatypes::{DataType, Field};use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};+use datafusion_common::ScalarValue;use datafusion_common::config::ConfigOptions;use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};use datafusion_functions_nested::sort::ArraySort;@@ -119,9 +120,25 @@ fn create_string_list_array(num_rows: usize, elements_per_row: usize) -> ArrayRe}fn invoke_array_sort(udf: &ArraySort, array: &ArrayRef) -> ColumnarValue {+ invoke_array_sort_with_order(udf, array, None)+}++/// `order` is the optional second argument (`"ASC"` / `"DESC"`); `None` leaves+/// `array_sort` with its default (ascending) ordering.+fn invoke_array_sort_with_order(+ udf: &ArraySort,+ array: &ArrayRef,+ order: Option<&str>,+) -> ColumnarValue {+ let mut args = vec![ColumnarValue::Array(Arc::clone(array))];+ let mut arg_fields = vec![Field::new("arr", array.data_type().clone(), true).into()];+ if let Some(order) = order {+ args.push(ColumnarValue::Scalar(ScalarValue::from(order)));+ arg_fields.push(Field::new("order", DataType::Utf8, true).into());+ }udf.invoke_with_args(ScalarFunctionArgs {- args: vec![ColumnarValue::Array(Arc::clone(array))],- arg_fields: vec![Field::new("arr", array.data_type().clone(), true).into()],+ args,+ arg_fields,number_rows: array.len(),return_field: Field::new("result", array.data_type().clone(), true).into(),config_options: Arc::new(ConfigOptions::default()),@@ -149,6 +166,21 @@ fn bench_array_sort(c: &mut Criterion) {);}+ // Same shapes, descending: `sort_row` sorts ascending and reverses, so this+ // measures the cost of that extra pass.+ for &elements_per_row in &[5, 20, 100, 1000] {+ let array = create_int32_list_array(NUM_ROWS, elements_per_row, false);+ group.bench_with_input(+ BenchmarkId::new("int32_desc", elements_per_row),+ &elements_per_row,+ |b, _| {+ b.iter(|| {+ black_box(invoke_array_sort_with_order(&udf, &array, Some("DESC")));+ });+ },+ );+ }+// Int32 with nulls in the outer list (10% null rows), single size{let array = create_int32_list_array(NUM_ROWS, 50, true);@@ -174,6 +206,20 @@ fn bench_array_sort(c: &mut Criterion) {);}+ for &elements_per_row in &[5, 20, 100, 1000] {+ let array =+ create_int32_list_array_with_null_elements(NUM_ROWS, elements_per_row);+ group.bench_with_input(+ BenchmarkId::new("int32_null_elements_desc", elements_per_row),+ &elements_per_row,+ |b, _| {+ b.iter(|| {+ black_box(invoke_array_sort_with_order(&udf, &array, Some("DESC")));+ });+ },+ );+ }+// String arraysfor &elements_per_row in &[5, 20, 100, 1000] {let array = create_string_list_array(NUM_ROWS, elements_per_row);diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rsindex 24a9913d2c..95023aedd6 100644--- a/datafusion/functions-nested/src/array_has.rs+++ b/datafusion/functions-nested/src/array_has.rs@@ -18,9 +18,8 @@//! [`ScalarUDFImpl`] definitions for array_has, array_has_all and array_has_any functions.use arrow::array::{- Array, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, AsArray, BooleanArray,- BooleanBufferBuilder, Datum, MAX_INLINE_VIEW_LEN, PrimitiveArray, Scalar,- StringArrayType, StringViewArray,+ Array, ArrayRef, ArrowNativeTypeOp, AsArray, BooleanArray, BooleanBufferBuilder,+ Datum, MAX_INLINE_VIEW_LEN, Scalar, StringArrayType, StringViewArray,};use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};use arrow::datatypes::DataType;@@ -359,7 +358,7 @@ fn array_has_dispatch_for_array(None} else {downcast_primitive_array! {- visible_values => {+ visible_values, needle => {// The element-null path makes several passes over the values, so// past a large average list length the per-row `eq` kernel is// faster -- bail to it. The single-pass all-valid path has no such@@ -371,25 +370,28 @@ fn array_has_dispatch_for_array({None} else {- Some(array_has_array_primitive(- visible_values, needle, &offsets,+ Some(array_has_array_native(+ visible_values.values(),+ visible_values.nulls(),+ needle.values(),+ &offsets,combined_nulls.as_ref(),))}- },- DataType::Utf8 => Some(array_has_array_string(+ }+ (DataType::Utf8, _) => Some(array_has_array_string(visible_values.as_string::<i32>(),needle.as_string::<i32>(),&offsets,combined_nulls.as_ref(),)),- DataType::LargeUtf8 => Some(array_has_array_string(+ (DataType::LargeUtf8, _) => Some(array_has_array_string(visible_values.as_string::<i64>(),needle.as_string::<i64>(),&offsets,combined_nulls.as_ref(),)),- DataType::Utf8View => Some(array_has_array_string_view(+ (DataType::Utf8View, _) => Some(array_has_array_string_view(visible_values.as_string_view(),needle.as_string_view(),&offsets,@@ -430,21 +432,20 @@ const NULL_FAST_PATH_MAX_LEN: usize = 512;/// 2. Nulls: AND the equality bitmap with validity (a null slot's value is/// arbitrary), then reduce each row to "any bit set". Chunked to bound the/// expanded needle.-fn array_has_array_primitive<T: ArrowPrimitiveType>(- values: &PrimitiveArray<T>,- needle: &dyn Array,+///+/// Generic over the native type alone -- the caller's `downcast_primitive_array!`+/// peels off the Arrow type -- so `Int32`, `Date32` and `Time32` share a single+/// instantiation instead of getting one each.+fn array_has_array_native<N: ArrowNativeTypeOp>(+ value_slice: &[N],+ element_nulls: Option<&NullBuffer>,+ needle_slice: &[N],offsets: &[usize],combined_nulls: Option<&NullBuffer>,-) -> BooleanBuffer-where- T::Native: ArrowNativeTypeOp,-{- let needle = needle.as_primitive::<T>();+) -> BooleanBuffer {let num_rows = offsets.len() - 1;- let value_slice = values.values();- let needle_slice = needle.values();- let Some(element_nulls) = values.nulls() else {+ let Some(element_nulls) = element_nulls else {return BooleanBuffer::collect_bool(num_rows, |i| {if combined_nulls.is_some_and(|n| n.is_null(i)) {return false;@@ -461,7 +462,7 @@ where// Case 2 (see fn doc), chunked like the all/any kernels.let mut result = BooleanBufferBuilder::new(num_rows);- let mut needle_expanded: Vec<T::Native> = Vec::new();+ let mut needle_expanded: Vec<N> = Vec::new();for chunk_start in (0..num_rows).step_by(ROW_CONVERSION_CHUNK_SIZE) {let chunk_end = (chunk_start + ROW_CONVERSION_CHUNK_SIZE).min(num_rows);let elem_start = offsets[chunk_start];diff --git a/datafusion/functions-nested/src/min_max.rs b/datafusion/functions-nested/src/min_max.rsindex ba9e5e7a07..2e7a32ba31 100644--- a/datafusion/functions-nested/src/min_max.rs+++ b/datafusion/functions-nested/src/min_max.rs@@ -21,6 +21,7 @@ use arrow::array::{Array, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, AsArray, GenericListArray,OffsetSizeTrait, PrimitiveBuilder, downcast_primitive,};+use arrow::buffer::NullBuffer;use arrow::datatypes::DataType;use arrow::datatypes::DataType::{LargeList, List};use datafusion_common::Result;@@ -263,7 +264,7 @@ fn primitive_array_min_max<O: OffsetSizeTrait, T: ArrowPrimitiveType>(match len {0 => None,_ if len < ARROW_COMPUTE_THRESHOLD => {- scalar_min_max::<T>(values_slice, values_nulls, start, end, is_min)+ scalar_min_max(values_slice, values_nulls, start, end, is_min)}_ => {let slice = values_array.slice(start, len);@@ -285,14 +286,14 @@ fn primitive_array_min_max<O: OffsetSizeTrait, T: ArrowPrimitiveType>(/// Computes min or max for a single list row by directly scanning a slice of/// the flat values buffer.#[inline]-fn scalar_min_max<T: ArrowPrimitiveType>(- values_slice: &[T::Native],- values_nulls: Option<&arrow::buffer::NullBuffer>,+fn scalar_min_max<N: ArrowNativeTypeOp>(+ values_slice: &[N],+ values_nulls: Option<&NullBuffer>,start: usize,end: usize,is_min: bool,-) -> Option<T::Native> {- let mut best: Option<T::Native> = None;+) -> Option<N> {+ let mut best: Option<N> = None;for (i, &val) in values_slice[start..end].iter().enumerate() {if let Some(nulls) = values_nulls&& !nulls.is_valid(start + i)diff --git a/datafusion/functions-nested/src/sort.rs b/datafusion/functions-nested/src/sort.rsindex f4f9148f76..fadc5da4b4 100644--- a/datafusion/functions-nested/src/sort.rs+++ b/datafusion/functions-nested/src/sort.rs@@ -25,11 +25,11 @@ use arrow::array::{};use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::datatypes::{ArrowNativeTypeOp, DataType, FieldRef};-use arrow::row::{RowConverter, SortField};+use arrow::row::{RowConverter, Rows, SortField};use arrow::{compute, compute::SortOptions, downcast_primitive_array};use datafusion_common::cast::{as_large_list_array, as_list_array, as_string_array};use datafusion_common::utils::ListCoercion;-use datafusion_common::{Result, exec_err, internal_datafusion_err};+use datafusion_common::{Result, exec_err};use datafusion_expr::{ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation,ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,@@ -238,99 +238,104 @@ fn sort_primitive_list<T: ArrowPrimitiveType, OffsetSize: OffsetSizeTrait>(whereT::Native: ArrowNativeTypeOp,{- if prim_values.null_count() > 0 {- sort_list_with_nulls(prim_values, list_array, field, sort_options)- } else {- sort_list_no_nulls(prim_values, list_array, field, sort_options)+ let descending = sort_options.is_some_and(|o| o.descending);+ let nulls_first = sort_options.is_none_or(|o| o.nulls_first);+ let list_nulls = list_array.nulls();+ let offsets = list_array.offsets();++ let (values, validity) = match prim_values.nulls() {+ Some(element_nulls) if element_nulls.null_count() > 0 => {+ let (values, validity) = sort_rows_with_nulls(+ prim_values.values(),+ element_nulls,+ offsets,+ list_nulls,+ descending,+ nulls_first,+ );+ (values, Some(validity))+ }+ _ => (+ sort_rows_no_nulls(prim_values.values(), offsets, list_nulls, descending),+ None,+ ),+ };++ let sorted_values = Arc::new(+ PrimitiveArray::<T>::new(values.into(), validity)+ .with_data_type(prim_values.data_type().clone()),+ );++ Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(+ field,+ rebase_offsets(offsets),+ sorted_values,+ list_nulls.cloned(),+ )?))+}++/// Sorts one row's values in place.+///+/// [`ArrowNativeTypeOp::compare`] is a total order, so a descending sort is the+/// reverse of the ascending one. Sorting one way and reversing keeps a single+/// instantiation of the standard library's sort per native type, instead of one+/// per comparator closure.+#[inline]+fn sort_row<N: ArrowNativeTypeOp>(row: &mut [N], descending: bool) {+ row.sort_unstable_by(|a, b| a.compare(*b));+ if descending {+ row.reverse();}}/// Fast path for primitive values with no element-level nulls. Copies all/// values into a single `Vec` and sorts each row's slice in-place.-fn sort_list_no_nulls<T: ArrowPrimitiveType, OffsetSize: OffsetSizeTrait>(- prim_values: &PrimitiveArray<T>,- list_array: &GenericListArray<OffsetSize>,- field: FieldRef,- sort_options: Option<SortOptions>,-) -> Result<ArrayRef>-where- T::Native: ArrowNativeTypeOp,-{- let row_count = list_array.len();- let offsets = list_array.offsets();+fn sort_rows_no_nulls<N: ArrowNativeTypeOp, O: OffsetSizeTrait>(+ src_values: &[N],+ offsets: &[O],+ list_nulls: Option<&NullBuffer>,+ descending: bool,+) -> Vec<N> {let values_start = offsets[0].as_usize();- let values_end = offsets[row_count].as_usize();-- let descending = sort_options.is_some_and(|o| o.descending);+ let values_end = offsets[offsets.len() - 1].as_usize();// Copy all values into a mutable buffer- let mut values: Vec<T::Native> =- prim_values.values()[values_start..values_end].to_vec();+ let mut values = src_values[values_start..values_end].to_vec();for (row_index, window) in offsets.windows(2).enumerate() {- if list_array.is_null(row_index) {+ if list_nulls.is_some_and(|n| n.is_null(row_index)) {continue;}let start = window[0].as_usize() - values_start;let end = window[1].as_usize() - values_start;- let slice = &mut values[start..end];- if descending {- slice.sort_unstable_by(|a, b| b.compare(*a));- } else {- slice.sort_unstable_by(|a, b| a.compare(*b));- }+ sort_row(&mut values[start..end], descending);}- let new_offsets = rebase_offsets(offsets);- let sorted_values = Arc::new(- PrimitiveArray::<T>::new(values.into(), None)- .with_data_type(prim_values.data_type().clone()),- );-- Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(- field,- new_offsets,- sorted_values,- list_array.nulls().cloned(),- )?))+ values}/// Slow path for primitive values with element-level nulls.-fn sort_list_with_nulls<T: ArrowPrimitiveType, OffsetSize: OffsetSizeTrait>(- prim_values: &PrimitiveArray<T>,- list_array: &GenericListArray<OffsetSize>,- field: FieldRef,- sort_options: Option<SortOptions>,-) -> Result<ArrayRef>-where- T::Native: ArrowNativeTypeOp,-{- let row_count = list_array.len();- let offsets = list_array.offsets();+fn sort_rows_with_nulls<N: ArrowNativeTypeOp, O: OffsetSizeTrait>(+ src_values: &[N],+ src_nulls: &NullBuffer,+ offsets: &[O],+ list_nulls: Option<&NullBuffer>,+ descending: bool,+ nulls_first: bool,+) -> (Vec<N>, NullBuffer) {let values_start = offsets[0].as_usize();- let values_end = offsets[row_count].as_usize();- let total_values = values_end - values_start;+ let total_values = offsets[offsets.len() - 1].as_usize() - values_start;- let descending = sort_options.is_some_and(|o| o.descending);- let nulls_first = sort_options.is_none_or(|o| o.nulls_first);-- let mut out_values: Vec<T::Native> = vec![T::Native::default(); total_values];+ let mut out_values: Vec<N> = vec![N::default(); total_values];let mut validity = BooleanBufferBuilder::new(total_values);- let src_nulls = prim_values.nulls().ok_or_else(|| {- internal_datafusion_err!(- "sort_list_with_nulls called but values have no null buffer"- )- })?;- let src_values = prim_values.values();-for (row_index, window) in offsets.windows(2).enumerate() {let start = window[0].as_usize();let end = window[1].as_usize();let row_len = end - start;let out_start = start - values_start;- if list_array.is_null(row_index) || row_len == 0 {+ if list_nulls.is_some_and(|n| n.is_null(row_index)) || row_len == 0 {validity.append_n(row_len, false);continue;}@@ -342,20 +347,18 @@ where// buffer: after nulls (if nulls_first) or at the start (if nulls_last).let valid_offset = if nulls_first { null_count } else { 0 };let mut write_pos = out_start + valid_offset;- for i in start..end {- if src_nulls.is_valid(i) {- out_values[write_pos] = src_values[i];+ for (i, &value) in src_values[start..end].iter().enumerate() {+ if src_nulls.is_valid(start + i) {+ out_values[write_pos] = value;write_pos += 1;}}- let valid_slice = &mut out_values- [out_start + valid_offset..out_start + valid_offset + valid_count];- if descending {- valid_slice.sort_unstable_by(|a, b| b.compare(*a));- } else {- valid_slice.sort_unstable_by(|a, b| a.compare(*b));- }+ sort_row(+ &mut out_values+ [out_start + valid_offset..out_start + valid_offset + valid_count],+ descending,+ );// Build validity bitsif nulls_first {@@ -367,20 +370,7 @@ where}}- let new_offsets = rebase_offsets(offsets);-- let null_buffer = NullBuffer::from(validity.finish());- let sorted_values = Arc::new(- PrimitiveArray::<T>::new(out_values.into(), Some(null_buffer))- .with_data_type(prim_values.data_type().clone()),- );-- Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(- field,- new_offsets,- sorted_values,- list_array.nulls().cloned(),- )?))+ (out_values, NullBuffer::from(validity.finish()))}/// Sort a non-pritive-typed ListArray by converting all rows at once using@@ -426,9 +416,7 @@ fn array_sort_non_primitive<OffsetSize: OffsetSizeTrait>(if len <= 1 {indices.extend((local_start..local_start + len).map(OffsetSize::usize_as));} else {- sort_scratch.clear();- sort_scratch.extend(local_start..local_start + len);- sort_scratch.sort_unstable_by(|&a, &b| rows.row(a).cmp(&rows.row(b)));+ sort_row_indices(&mut sort_scratch, &rows, local_start, len);indices.extend(sort_scratch.iter().map(|&i| OffsetSize::usize_as(i)));}@@ -449,6 +437,15 @@ fn array_sort_non_primitive<OffsetSize: OffsetSizeTrait>()?))}+/// Sorts `scratch` to hold the indices `start..start + len` ordered by their+/// encoded rows. Kept free of the offset width so the standard library's sort+/// is instantiated once rather than once per list type.+fn sort_row_indices(scratch: &mut Vec<usize>, rows: &Rows, start: usize, len: usize) {+ scratch.clear();+ scratch.extend(start..start + len);+ scratch.sort_unstable_by(|&a, &b| rows.row(a).cmp(&rows.row(b)));+}+/// Select elements from `values` at the given `indices` using `compute::take`./// We consume `indices` in order to avoid an intermediate copy.fn take_by_indices<OffsetSize: OffsetSizeTrait>(