Skip to content

Apache DataFusion SQL Query Engine

feat(spark): implement array slice function (#19811)

cht42154ab70040ad
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index e3c11dde76..dc5875e5a1 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -70,3 +70,7 @@ name = "space"
[[bench]]
harness = false
name = "hex"
+
+[[bench]]
+harness = false
+name = "slice"
diff --git a/datafusion/spark/benches/slice.rs b/datafusion/spark/benches/slice.rs
new file mode 100644
index 0000000000..0061e9be5c
--- /dev/null
+++ b/datafusion/spark/benches/slice.rs
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+extern crate criterion;
+
+use arrow::array::{
+ Int64Array, ListArray, ListViewArray, NullBufferBuilder, PrimitiveArray,
+};
+use arrow::buffer::{OffsetBuffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, Int64Type};
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+use datafusion_spark::function::array::slice;
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use std::hint::black_box;
+use std::sync::Arc;
+
+fn create_inputs(
+ rng: &mut StdRng,
+ size: usize,
+ child_array_size: usize,
+ null_density: f32,
+) -> (ListArray, ListViewArray) {
+ let mut nulls_builder = NullBufferBuilder::new(size);
+ let mut sizes = Vec::with_capacity(size);
+
+ for _ in 0..size {
+ if rng.random::<f32>() < null_density {
+ nulls_builder.append_null();
+ } else {
+ nulls_builder.append_non_null();
+ }
+ sizes.push(rng.random_range(1..child_array_size));
+ }
+ let nulls = nulls_builder.finish();
+
+ let length = sizes.iter().sum();
+ let values: PrimitiveArray<Int64Type> =
+ (0..length).map(|_| Some(rng.random())).collect();
+ let values = Arc::new(values);
+
+ let offsets = OffsetBuffer::from_lengths(sizes.clone());
+ let list_array = ListArray::new(
+ Arc::new(Field::new_list_field(DataType::Int64, true)),
+ offsets.clone(),
+ values.clone(),
+ nulls.clone(),
+ );
+
+ let offsets = ScalarBuffer::from(offsets.slice(0, size - 1));
+ let sizes = ScalarBuffer::from_iter(sizes.into_iter().map(|v| v as i32));
+ let list_view_array = ListViewArray::new(
+ Arc::new(Field::new_list_field(DataType::Int64, true)),
+ offsets,
+ sizes,
+ values,
+ nulls,
+ );
+
+ (list_array, list_view_array)
+}
+
+fn random_from_to(
+ rng: &mut StdRng,
+ size: i64,
+ null_density: f32,
+) -> (Option<i64>, Option<i64>) {
+ let from = if rng.random::<f32>() < null_density {
+ None
+ } else {
+ Some(rng.random_range(1..=size))
+ };
+
+ let to = if rng.random::<f32>() < null_density {
+ None
+ } else {
+ match from {
+ Some(from) => Some(rng.random_range(from..=size)),
+ None => Some(rng.random_range(1..=size)),
+ }
+ };
+
+ (from, to)
+}
+
+fn array_slice_benchmark(
+ name: &str,
+ input: ColumnarValue,
+ mut args: Vec<ColumnarValue>,
+ c: &mut Criterion,
+ size: usize,
+) {
+ args.insert(0, input);
+
+ let array_slice = slice();
+ let arg_fields = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| {
+ <Arc<Field>>::from(Field::new(format!("arg_{idx}"), arg.data_type(), true))
+ })
+ .collect::<Vec<_>>();
+ c.bench_function(name, |b| {
+ b.iter(|| {
+ black_box(
+ array_slice
+ .invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Field::new_list_field(args[0].data_type(), true)
+ .into(),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ });
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let rng = &mut StdRng::seed_from_u64(42);
+ let size = 1_000_000;
+ let child_array_size = 100;
+ let null_density = 0.1;
+
+ let (list_array, list_view_array) =
+ create_inputs(rng, size, child_array_size, null_density);
+
+ let mut array_from = Vec::with_capacity(size);
+ let mut array_to = Vec::with_capacity(size);
+ for child_array_size in list_array.offsets().lengths() {
+ let (from, to) = random_from_to(rng, child_array_size as i64, null_density);
+ array_from.push(from);
+ array_to.push(to);
+ }
+
+ // input
+ let list_array = ColumnarValue::Array(Arc::new(list_array));
+ let list_view_array = ColumnarValue::Array(Arc::new(list_view_array));
+
+ // args
+ let array_from = ColumnarValue::Array(Arc::new(Int64Array::from(array_from)));
+ let array_to = ColumnarValue::Array(Arc::new(Int64Array::from(array_to)));
+ let scalar_from = ColumnarValue::Scalar(ScalarValue::from(1i64));
+ let scalar_to = ColumnarValue::Scalar(ScalarValue::from(child_array_size as i64 / 2));
+
+ for input in [list_array, list_view_array] {
+ let input_type = input.data_type().to_string();
+
+ array_slice_benchmark(
+ &format!("slice: input {input_type}, array args, no stride"),
+ input.clone(),
+ vec![array_from.clone(), array_to.clone()],
+ c,
+ size,
+ );
+
+ array_slice_benchmark(
+ &format!("slice: input {input_type}, scalar args, no stride"),
+ input.clone(),
+ vec![scalar_from.clone(), scalar_to.clone()],
+ c,
+ size,
+ );
+ }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/spark/src/function/array/mod.rs b/datafusion/spark/src/function/array/mod.rs
index 7140653510..0d4cd40d99 100644
--- a/datafusion/spark/src/function/array/mod.rs
+++ b/datafusion/spark/src/function/array/mod.rs
@@ -17,6 +17,7 @@
pub mod repeat;
pub mod shuffle;
+pub mod slice;
pub mod spark_array;
use datafusion_expr::ScalarUDF;
@@ -26,6 +27,7 @@ use std::sync::Arc;
make_udf_function!(spark_array::SparkArray, array);
make_udf_function!(shuffle::SparkShuffle, shuffle);
make_udf_function!(repeat::SparkArrayRepeat, array_repeat);
+make_udf_function!(slice::SparkSlice, slice);
pub mod expr_fn {
use datafusion_functions::export_functions;
@@ -41,8 +43,13 @@ pub mod expr_fn {
"returns an array containing element count times.",
element count
));
+ export_functions!((
+ slice,
+ "Returns a slice of the array from the start index with the given length.",
+ array start length
+ ));
}
pub fn functions() -> Vec<Arc<ScalarUDF>> {
- vec![array(), shuffle(), array_repeat()]
+ vec![array(), shuffle(), array_repeat(), slice()]
}
diff --git a/datafusion/spark/src/function/array/slice.rs b/datafusion/spark/src/function/array/slice.rs
new file mode 100644
index 0000000000..6c168a4f49
--- /dev/null
+++ b/datafusion/spark/src/function/array/slice.rs
@@ -0,0 +1,172 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{Array, ArrayRef, Int64Builder};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::cast::{as_int64_array, as_list_array};
+use datafusion_common::utils::ListCoercion;
+use datafusion_common::{Result, exec_err, internal_err, utils::take_function_args};
+use datafusion_expr::{
+ ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ReturnFieldArgs,
+ ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,
+};
+use datafusion_functions_nested::extract::array_slice_udf;
+use std::any::Any;
+use std::sync::Arc;
+
+/// Spark slice function implementation
+/// Main difference from DataFusion's array_slice is that the third argument is the length of the slice and not the end index.
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#slice>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkSlice {
+ signature: Signature,
+}
+
+impl Default for SparkSlice {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkSlice {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature {
+ type_signature: TypeSignature::ArraySignature(
+ ArrayFunctionSignature::Array {
+ arguments: vec![
+ ArrayFunctionArgument::Array,
+ ArrayFunctionArgument::Index,
+ ArrayFunctionArgument::Index,
+ ],
+ array_coercion: Some(ListCoercion::FixedSizedListToList),
+ },
+ ),
+ volatility: Volatility::Immutable,
+ parameter_names: None,
+ },
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkSlice {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn name(&self) -> &str {
+ "slice"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+
+ Ok(Arc::new(Field::new(
+ "slice",
+ args.arg_fields[0].data_type().clone(),
+ nullable,
+ )))
+ }
+
+ fn invoke_with_args(
+ &self,
+ mut func_args: ScalarFunctionArgs,
+ ) -> Result<ColumnarValue> {
+ let array_len = func_args
+ .args
+ .iter()
+ .find_map(|arg| match arg {
+ ColumnarValue::Array(array) => Some(array.len()),
+ _ => None,
+ })
+ .unwrap_or(func_args.number_rows);
+
+ let arrays = func_args
+ .args
+ .iter()
+ .map(|arg| match arg {
+ ColumnarValue::Array(array) => Ok(Arc::clone(array)),
+ ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(array_len),
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ let (start, end) = calculate_start_end(&arrays)?;
+
+ array_slice_udf().invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ func_args.args.swap_remove(0),
+ ColumnarValue::Array(start),
+ ColumnarValue::Array(end),
+ ],
+ arg_fields: func_args.arg_fields,
+ number_rows: func_args.number_rows,
+ return_field: func_args.return_field,
+ config_options: func_args.config_options,
+ })
+ }
+}
+
+fn calculate_start_end(args: &[ArrayRef]) -> Result<(ArrayRef, ArrayRef)> {
+ let [values, start, length] = take_function_args("slice", args)?;
+
+ let values_len = values.len();
+
+ let start = as_int64_array(&start)?;
+ let length = as_int64_array(&length)?;
+
+ let values = as_list_array(values)?;
+
+ let mut adjusted_start = Int64Builder::with_capacity(values_len);
+ let mut end = Int64Builder::with_capacity(values_len);
+
+ for row in 0..values_len {
+ if values.is_null(row) || start.is_null(row) || length.is_null(row) {
+ adjusted_start.append_null();
+ end.append_null();
+ continue;
+ }
+ let start = start.value(row);
+ let length = length.value(row);
+ let value_length = values.value(row).len() as i64;
+
+ if start == 0 {
+ return exec_err!("Start index must not be zero");
+ }
+ if length < 0 {
+ return exec_err!("Length must be non-negative, but got {}", length);
+ }
+
+ let adjusted_start_value = if start < 0 {
+ start + value_length + 1
+ } else {
+ start
+ };
+
+ adjusted_start.append_value(adjusted_start_value);
+ end.append_value(adjusted_start_value + (length - 1));
+ }
+
+ Ok((Arc::new(adjusted_start.finish()), Arc::new(end.finish())))
+}
diff --git a/datafusion/sqllogictest/test_files/spark/array/slice.slt b/datafusion/sqllogictest/test_files/spark/array/slice.slt
new file mode 100644
index 0000000000..21f321033b
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/spark/array/slice.slt
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+query ?
+SELECT slice([], 2, 2);
+----
+[]
+
+query ?
+SELECT slice([1, 2, 3, 4], 2, 2);
+----
+[2, 3]
+
+query ?
+SELECT slice([1, 2, 3, 4], 1, 100);
+----
+[1, 2, 3, 4]
+
+query ?
+SELECT slice([1, 2, 3, 4], -2, 2);
+----
+[3, 4]
+
+query ?
+SELECT slice([1, 2, 3, 4], 100, 2);
+----
+[]
+
+query ?
+SELECT slice([1, 2, 3, 4], -200, 2);
+----
+[]
+
+query error DataFusion error: Execution error: Length must be non-negative, but got -2
+SELECT slice([1, 2, 3, 4], 2, -2);
+
+query error DataFusion error: Execution error: Length must be non-negative, but got -2
+SELECT slice([1, 2, 3, 4], -2, -2);
+
+query error DataFusion error: Execution error: Start index must not be zero
+SELECT slice([1, 2, 3, 4], 0, -2);
+
+query ?
+SELECT slice([NULL, NULL, NULL, NULL, NULL], 2, 2);
+----
+[NULL, NULL]
+
+query ?
+SELECT slice(arrow_cast(NULL, 'FixedSizeList(1, Int64)'), 2, 2);
+----
+NULL
+
+query ?
+SELECT slice([1, 2, 3, 4], NULL, 2);
+----
+NULL
+
+query ?
+SELECT slice([1, 2, 3, 4], 2, NULL);
+----
+NULL
+
+
+query ?
+SELECT slice(column1, column2, column3)
+FROM VALUES
+([1, 2, 3, 4], 2, 2),
+([1, 2, 3, 4], 1, 100),
+([1, 2, 3, 4], -2, 2),
+([], 2, 2),
+([1, 2, 3, 4], 100, 2),
+([1, 2, 3, 4], -200, 2),
+([NULL, NULL, NULL, NULL, NULL], 2, 2),
+(arrow_cast(NULL, 'FixedSizeList(1, Int64)'), 2, 2),
+([1, 2, 3, 4], NULL, 2),
+([1, 2, 3, 4], 2, NULL);
+----
+[2, 3]
+[1, 2, 3, 4]
+[3, 4]
+[]
+[]
+[]
+[NULL, NULL]
+NULL
+NULL
+NULL