Apache DataFusion SQL Query Engine
feat(spark): implement array slice function (#19811)
cht42154ab70040ad
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex e3c11dde76..dc5875e5a1 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -70,3 +70,7 @@ name = "space"[[bench]]harness = falsename = "hex"++[[bench]]+harness = false+name = "slice"diff --git a/datafusion/spark/benches/slice.rs b/datafusion/spark/benches/slice.rsnew file mode 100644index 0000000000..0061e9be5c--- /dev/null+++ b/datafusion/spark/benches/slice.rs@@ -0,0 +1,187 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++extern crate criterion;++use arrow::array::{+ Int64Array, ListArray, ListViewArray, NullBufferBuilder, PrimitiveArray,+};+use arrow::buffer::{OffsetBuffer, ScalarBuffer};+use arrow::datatypes::{DataType, Field, Int64Type};+use criterion::{Criterion, criterion_group, criterion_main};+use datafusion_common::ScalarValue;+use datafusion_common::config::ConfigOptions;+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};+use datafusion_spark::function::array::slice;+use rand::rngs::StdRng;+use rand::{Rng, SeedableRng};+use std::hint::black_box;+use std::sync::Arc;++fn create_inputs(+ rng: &mut StdRng,+ size: usize,+ child_array_size: usize,+ null_density: f32,+) -> (ListArray, ListViewArray) {+ let mut nulls_builder = NullBufferBuilder::new(size);+ let mut sizes = Vec::with_capacity(size);++ for _ in 0..size {+ if rng.random::<f32>() < null_density {+ nulls_builder.append_null();+ } else {+ nulls_builder.append_non_null();+ }+ sizes.push(rng.random_range(1..child_array_size));+ }+ let nulls = nulls_builder.finish();++ let length = sizes.iter().sum();+ let values: PrimitiveArray<Int64Type> =+ (0..length).map(|_| Some(rng.random())).collect();+ let values = Arc::new(values);++ let offsets = OffsetBuffer::from_lengths(sizes.clone());+ let list_array = ListArray::new(+ Arc::new(Field::new_list_field(DataType::Int64, true)),+ offsets.clone(),+ values.clone(),+ nulls.clone(),+ );++ let offsets = ScalarBuffer::from(offsets.slice(0, size - 1));+ let sizes = ScalarBuffer::from_iter(sizes.into_iter().map(|v| v as i32));+ let list_view_array = ListViewArray::new(+ Arc::new(Field::new_list_field(DataType::Int64, true)),+ offsets,+ sizes,+ values,+ nulls,+ );++ (list_array, list_view_array)+}++fn random_from_to(+ rng: &mut StdRng,+ size: i64,+ null_density: f32,+) -> (Option<i64>, Option<i64>) {+ let from = if rng.random::<f32>() < null_density {+ None+ } else {+ Some(rng.random_range(1..=size))+ };++ let to = if rng.random::<f32>() < null_density {+ None+ } else {+ match from {+ Some(from) => Some(rng.random_range(from..=size)),+ None => Some(rng.random_range(1..=size)),+ }+ };++ (from, to)+}++fn array_slice_benchmark(+ name: &str,+ input: ColumnarValue,+ mut args: Vec<ColumnarValue>,+ c: &mut Criterion,+ size: usize,+) {+ args.insert(0, input);++ let array_slice = slice();+ let arg_fields = args+ .iter()+ .enumerate()+ .map(|(idx, arg)| {+ <Arc<Field>>::from(Field::new(format!("arg_{idx}"), arg.data_type(), true))+ })+ .collect::<Vec<_>>();+ c.bench_function(name, |b| {+ b.iter(|| {+ black_box(+ array_slice+ .invoke_with_args(ScalarFunctionArgs {+ args: args.clone(),+ arg_fields: arg_fields.clone(),+ number_rows: size,+ return_field: Field::new_list_field(args[0].data_type(), true)+ .into(),+ config_options: Arc::new(ConfigOptions::default()),+ })+ .unwrap(),+ )+ })+ });+}++fn criterion_benchmark(c: &mut Criterion) {+ let rng = &mut StdRng::seed_from_u64(42);+ let size = 1_000_000;+ let child_array_size = 100;+ let null_density = 0.1;++ let (list_array, list_view_array) =+ create_inputs(rng, size, child_array_size, null_density);++ let mut array_from = Vec::with_capacity(size);+ let mut array_to = Vec::with_capacity(size);+ for child_array_size in list_array.offsets().lengths() {+ let (from, to) = random_from_to(rng, child_array_size as i64, null_density);+ array_from.push(from);+ array_to.push(to);+ }++ // input+ let list_array = ColumnarValue::Array(Arc::new(list_array));+ let list_view_array = ColumnarValue::Array(Arc::new(list_view_array));++ // args+ let array_from = ColumnarValue::Array(Arc::new(Int64Array::from(array_from)));+ let array_to = ColumnarValue::Array(Arc::new(Int64Array::from(array_to)));+ let scalar_from = ColumnarValue::Scalar(ScalarValue::from(1i64));+ let scalar_to = ColumnarValue::Scalar(ScalarValue::from(child_array_size as i64 / 2));++ for input in [list_array, list_view_array] {+ let input_type = input.data_type().to_string();++ array_slice_benchmark(+ &format!("slice: input {input_type}, array args, no stride"),+ input.clone(),+ vec![array_from.clone(), array_to.clone()],+ c,+ size,+ );++ array_slice_benchmark(+ &format!("slice: input {input_type}, scalar args, no stride"),+ input.clone(),+ vec![scalar_from.clone(), scalar_to.clone()],+ c,+ size,+ );+ }+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/datafusion/spark/src/function/array/mod.rs b/datafusion/spark/src/function/array/mod.rsindex 7140653510..0d4cd40d99 100644--- a/datafusion/spark/src/function/array/mod.rs+++ b/datafusion/spark/src/function/array/mod.rs@@ -17,6 +17,7 @@pub mod repeat;pub mod shuffle;+pub mod slice;pub mod spark_array;use datafusion_expr::ScalarUDF;@@ -26,6 +27,7 @@ use std::sync::Arc;make_udf_function!(spark_array::SparkArray, array);make_udf_function!(shuffle::SparkShuffle, shuffle);make_udf_function!(repeat::SparkArrayRepeat, array_repeat);+make_udf_function!(slice::SparkSlice, slice);pub mod expr_fn {use datafusion_functions::export_functions;@@ -41,8 +43,13 @@ pub mod expr_fn {"returns an array containing element count times.",element count));+ export_functions!((+ slice,+ "Returns a slice of the array from the start index with the given length.",+ array start length+ ));}pub fn functions() -> Vec<Arc<ScalarUDF>> {- vec![array(), shuffle(), array_repeat()]+ vec![array(), shuffle(), array_repeat(), slice()]}diff --git a/datafusion/spark/src/function/array/slice.rs b/datafusion/spark/src/function/array/slice.rsnew file mode 100644index 0000000000..6c168a4f49--- /dev/null+++ b/datafusion/spark/src/function/array/slice.rs@@ -0,0 +1,172 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::{Array, ArrayRef, Int64Builder};+use arrow::datatypes::{DataType, Field, FieldRef};+use datafusion_common::cast::{as_int64_array, as_list_array};+use datafusion_common::utils::ListCoercion;+use datafusion_common::{Result, exec_err, internal_err, utils::take_function_args};+use datafusion_expr::{+ ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ReturnFieldArgs,+ ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,+};+use datafusion_functions_nested::extract::array_slice_udf;+use std::any::Any;+use std::sync::Arc;++/// Spark slice function implementation+/// Main difference from DataFusion's array_slice is that the third argument is the length of the slice and not the end index.+/// <https://spark.apache.org/docs/latest/api/sql/index.html#slice>+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct SparkSlice {+ signature: Signature,+}++impl Default for SparkSlice {+ fn default() -> Self {+ Self::new()+ }+}++impl SparkSlice {+ pub fn new() -> Self {+ Self {+ signature: Signature {+ type_signature: TypeSignature::ArraySignature(+ ArrayFunctionSignature::Array {+ arguments: vec![+ ArrayFunctionArgument::Array,+ ArrayFunctionArgument::Index,+ ArrayFunctionArgument::Index,+ ],+ array_coercion: Some(ListCoercion::FixedSizedListToList),+ },+ ),+ volatility: Volatility::Immutable,+ parameter_names: None,+ },+ }+ }+}++impl ScalarUDFImpl for SparkSlice {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn name(&self) -> &str {+ "slice"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());++ Ok(Arc::new(Field::new(+ "slice",+ args.arg_fields[0].data_type().clone(),+ nullable,+ )))+ }++ fn invoke_with_args(+ &self,+ mut func_args: ScalarFunctionArgs,+ ) -> Result<ColumnarValue> {+ let array_len = func_args+ .args+ .iter()+ .find_map(|arg| match arg {+ ColumnarValue::Array(array) => Some(array.len()),+ _ => None,+ })+ .unwrap_or(func_args.number_rows);++ let arrays = func_args+ .args+ .iter()+ .map(|arg| match arg {+ ColumnarValue::Array(array) => Ok(Arc::clone(array)),+ ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(array_len),+ })+ .collect::<Result<Vec<_>>>()?;++ let (start, end) = calculate_start_end(&arrays)?;++ array_slice_udf().invoke_with_args(ScalarFunctionArgs {+ args: vec![+ func_args.args.swap_remove(0),+ ColumnarValue::Array(start),+ ColumnarValue::Array(end),+ ],+ arg_fields: func_args.arg_fields,+ number_rows: func_args.number_rows,+ return_field: func_args.return_field,+ config_options: func_args.config_options,+ })+ }+}++fn calculate_start_end(args: &[ArrayRef]) -> Result<(ArrayRef, ArrayRef)> {+ let [values, start, length] = take_function_args("slice", args)?;++ let values_len = values.len();++ let start = as_int64_array(&start)?;+ let length = as_int64_array(&length)?;++ let values = as_list_array(values)?;++ let mut adjusted_start = Int64Builder::with_capacity(values_len);+ let mut end = Int64Builder::with_capacity(values_len);++ for row in 0..values_len {+ if values.is_null(row) || start.is_null(row) || length.is_null(row) {+ adjusted_start.append_null();+ end.append_null();+ continue;+ }+ let start = start.value(row);+ let length = length.value(row);+ let value_length = values.value(row).len() as i64;++ if start == 0 {+ return exec_err!("Start index must not be zero");+ }+ if length < 0 {+ return exec_err!("Length must be non-negative, but got {}", length);+ }++ let adjusted_start_value = if start < 0 {+ start + value_length + 1+ } else {+ start+ };++ adjusted_start.append_value(adjusted_start_value);+ end.append_value(adjusted_start_value + (length - 1));+ }++ Ok((Arc::new(adjusted_start.finish()), Arc::new(end.finish())))+}diff --git a/datafusion/sqllogictest/test_files/spark/array/slice.slt b/datafusion/sqllogictest/test_files/spark/array/slice.sltnew file mode 100644index 0000000000..21f321033b--- /dev/null+++ b/datafusion/sqllogictest/test_files/spark/array/slice.slt@@ -0,0 +1,101 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++query ?+SELECT slice([], 2, 2);+----+[]++query ?+SELECT slice([1, 2, 3, 4], 2, 2);+----+[2, 3]++query ?+SELECT slice([1, 2, 3, 4], 1, 100);+----+[1, 2, 3, 4]++query ?+SELECT slice([1, 2, 3, 4], -2, 2);+----+[3, 4]++query ?+SELECT slice([1, 2, 3, 4], 100, 2);+----+[]++query ?+SELECT slice([1, 2, 3, 4], -200, 2);+----+[]++query error DataFusion error: Execution error: Length must be non-negative, but got -2+SELECT slice([1, 2, 3, 4], 2, -2);++query error DataFusion error: Execution error: Length must be non-negative, but got -2+SELECT slice([1, 2, 3, 4], -2, -2);++query error DataFusion error: Execution error: Start index must not be zero+SELECT slice([1, 2, 3, 4], 0, -2);++query ?+SELECT slice([NULL, NULL, NULL, NULL, NULL], 2, 2);+----+[NULL, NULL]++query ?+SELECT slice(arrow_cast(NULL, 'FixedSizeList(1, Int64)'), 2, 2);+----+NULL++query ?+SELECT slice([1, 2, 3, 4], NULL, 2);+----+NULL++query ?+SELECT slice([1, 2, 3, 4], 2, NULL);+----+NULL+++query ?+SELECT slice(column1, column2, column3)+FROM VALUES+([1, 2, 3, 4], 2, 2),+([1, 2, 3, 4], 1, 100),+([1, 2, 3, 4], -2, 2),+([], 2, 2),+([1, 2, 3, 4], 100, 2),+([1, 2, 3, 4], -200, 2),+([NULL, NULL, NULL, NULL, NULL], 2, 2),+(arrow_cast(NULL, 'FixedSizeList(1, Int64)'), 2, 2),+([1, 2, 3, 4], NULL, 2),+([1, 2, 3, 4], 2, NULL);+----+[2, 3]+[1, 2, 3, 4]+[3, 4]+[]+[]+[]+[NULL, NULL]+NULL+NULL+NULL