Skip to content

Apache DataFusion SQL Query Engine

feat: support Spark-compatible `stack` function (#26266)

Oleks V6489d1510ce1
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index 08309a931d..6fa172136a 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -106,3 +106,7 @@ name = "sha2"
[[bench]]
harness = false
name = "floor"
+
+[[bench]]
+harness = false
+name = "stack"
diff --git a/datafusion/spark/benches/stack.rs b/datafusion/spark/benches/stack.rs
new file mode 100644
index 0000000000..a1e6e53210
--- /dev/null
+++ b/datafusion/spark/benches/stack.rs
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::datatypes::{Field, Int64Type};
+use arrow::util::bench_util::{
+ create_boolean_array_with_seed, create_primitive_array_with_seed,
+ create_string_array_with_len_range_and_prefix_and_seed,
+};
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs};
+use datafusion_spark::function::generator::stack;
+use std::hint::black_box;
+use std::sync::Arc;
+
+const SIZE: usize = 8192;
+
+fn bench_stack(c: &mut Criterion, name: &str, n: i32, values: Vec<ColumnarValue>) {
+ let stack = stack();
+ let n = ScalarValue::Int32(Some(n));
+ let args = std::iter::once(ColumnarValue::Scalar(n.clone()))
+ .chain(values)
+ .collect::<Vec<_>>();
+ let arg_fields = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| Field::new(format!("arg_{idx}"), arg.data_type(), true).into())
+ .collect::<Vec<_>>();
+ let scalar_arguments = std::iter::once(Some(&n))
+ .chain(std::iter::repeat_n(None, args.len() - 1))
+ .collect::<Vec<_>>();
+ let return_field = stack
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &arg_fields,
+ scalar_arguments: &scalar_arguments,
+ })
+ .unwrap();
+ let config_options = Arc::new(ConfigOptions::default());
+ c.bench_function(name, |b| {
+ b.iter(|| {
+ black_box(
+ stack
+ .invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: SIZE,
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .unwrap(),
+ )
+ })
+ });
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ // One seed per column, so the columns differ
+ let ints = (0..4)
+ .map(|seed| create_primitive_array_with_seed::<Int64Type>(SIZE, 0.1, seed))
+ .map(|array| ColumnarValue::Array(Arc::new(array)))
+ .collect::<Vec<_>>();
+ let strings = (0..2)
+ .map(|seed| {
+ create_string_array_with_len_range_and_prefix_and_seed::<i32>(
+ SIZE, 0.1, 12, 12, "", seed,
+ )
+ })
+ .map(|array| ColumnarValue::Array(Arc::new(array)))
+ .collect::<Vec<_>>();
+ let booleans = (0..2)
+ .map(|seed| create_boolean_array_with_seed(SIZE, 0.1, 0.5, seed))
+ .map(|array| ColumnarValue::Array(Arc::new(array)))
+ .collect::<Vec<_>>();
+ let key = |k: &str| ColumnarValue::Scalar(ScalarValue::from(k));
+
+ // stack(4, 'k1', c1, 'k2', c2, 'k3', c3, 'k4', c4): unpivot 4 columns
+ let unpivot = ["k1", "k2", "k3", "k4"]
+ .into_iter()
+ .zip(&ints)
+ .flat_map(|(k, column)| [key(k), column.clone()])
+ .collect();
+ bench_stack(c, "stack: unpivot 4 int64 columns", 4, unpivot);
+ bench_stack(c, "stack: 4 int64 columns into 2 rows", 2, ints.clone());
+ bench_stack(c, "stack: 4 int64 columns into 4 rows", 4, ints.clone());
+ bench_stack(c, "stack: 4 int64 columns into 1 row", 1, ints);
+ bench_stack(c, "stack: 2 utf8 columns into 2 rows", 2, strings);
+ bench_stack(c, "stack: 2 boolean columns into 2 rows", 2, booleans);
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/spark/src/function/generator/mod.rs b/datafusion/spark/src/function/generator/mod.rs
index a87df9a2c8..cc480a2170 100644
--- a/datafusion/spark/src/function/generator/mod.rs
+++ b/datafusion/spark/src/function/generator/mod.rs
@@ -15,11 +15,24 @@
// specific language governing permissions and limitations
// under the License.
+pub mod stack;
+
use datafusion_expr::ScalarUDF;
+use datafusion_functions::make_udf_function;
use std::sync::Arc;
-pub mod expr_fn {}
+make_udf_function!(stack::SparkStack, stack);
+
+pub mod expr_fn {
+ use datafusion_functions::export_functions;
+
+ export_functions!((
+ stack,
+ "Separates the values into `n` rows, where `n` is an integer literal, and returns them as a list of structs with fields `col0`, `col1`, and so on, which `unnest` expands.",
+ args,
+ ));
+}
pub fn functions() -> Vec<Arc<ScalarUDF>> {
- vec![]
+ vec![stack()]
}
diff --git a/datafusion/spark/src/function/generator/stack.rs b/datafusion/spark/src/function/generator/stack.rs
new file mode 100644
index 0000000000..7bdb4909f5
--- /dev/null
+++ b/datafusion/spark/src/function/generator/stack.rs
@@ -0,0 +1,579 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::array::{
+ Array, ArrayRef, AsArray, ListArray, NullBufferBuilder, PrimitiveArray, StructArray,
+ UInt32Array, downcast_primitive, new_empty_array, new_null_array,
+};
+use arrow::buffer::OffsetBuffer;
+use arrow::compute::{concat, take};
+use arrow::datatypes::{ArrowPrimitiveType, DataType, Field, FieldRef, Fields};
+use datafusion_common::types::NativeType;
+use datafusion_common::{
+ Result, ScalarValue, internal_err, plan_err, resources_datafusion_err,
+};
+use datafusion_expr::{
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+
+/// The most rows a call generates per batch. Spark streams the rows, but they are
+/// materialized here, so a large `n` such as `stack(2000000000, 1)` fails instead of
+/// exhausting memory.
+const MAX_GENERATED_ROWS: usize = 1 << 24;
+
+/// Spark-compatible `stack` expression
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#stack>
+///
+/// `stack(n, expr1, ..., exprk)` separates `expr1`, ..., `exprk` into `n` rows of
+/// `ceil(k / n)` columns named `col0`, `col1`, ... The values fill the rows in
+/// order, and the last rows are padded with NULLs.
+///
+/// Note: In Spark, `stack` is a Generator that produces rows. In DataFusion, a
+/// ScalarUDF can only return one value per row, so the `n` rows are returned as a
+/// `List<Struct<col0, col1, ...>>`, which is the data type of Spark's Generator. As
+/// in Spark, the list is not nullable while its elements and fields are. The
+/// caller (e.g. Comet, or `unnest` in SQL) is expected to unnest the list.
+///
+/// As in Spark, `n` is an `INT` in `[1, 2147483647]`, and the values of a column
+/// must have the same type, which is the type of its first non-NULL value. NULLs,
+/// Arrow types of the same Spark type (such as `Utf8` and `Utf8View`), and integer
+/// literals that fit an `INT` column are cast to it, wherever they are in the
+/// column. Unlike Spark, which accepts any foldable `n`, `n` must be an integer
+/// literal, because the return type is planned before expressions are folded.
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkStack {
+ signature: Signature,
+}
+
+impl Default for SparkStack {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkStack {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkStack {
+ fn name(&self) -> &str {
+ "stack"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ /// Validates the arguments in the order Spark's `Stack.checkInputDataTypes` does.
+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
+ let (n, values) = match args.arg_fields {
+ [n, values @ ..] if !values.is_empty() => (n, values),
+ fields => {
+ return plan_err!(
+ "The `stack` requires > 1 parameters but the actual number is {}.",
+ fields.len()
+ );
+ }
+ };
+ if n.data_type() != &DataType::Int32 {
+ return plan_err!(
+ "The first parameter requires the \"INT\" type, however \"{}\" has the type \"{}\".",
+ n.name(),
+ n.data_type()
+ );
+ }
+ let Some(n_value) = args.scalar_arguments.first().copied().flatten() else {
+ return plan_err!(
+ "The first parameter of `stack` must be an integer literal, however \"{}\" is not one.",
+ n.name()
+ );
+ };
+ let num_fields = values.len().div_ceil(parse_n(n_value)?);
+ let scalar_arguments = args.scalar_arguments.get(1..).unwrap_or_default();
+ let fields = column_types(values, scalar_arguments, num_fields)?
+ .into_iter()
+ .enumerate()
+ .map(|(i, data_type)| Field::new(format!("col{i}"), data_type, true))
+ .collect::<Fields>();
+ Ok(Arc::new(Field::new(
+ self.name(),
+ DataType::new_list(DataType::Struct(fields), true),
+ false,
+ )))
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ let ScalarFunctionArgs {
+ args,
+ number_rows,
+ return_field,
+ ..
+ } = args;
+ let DataType::List(item) = return_field.data_type() else {
+ return internal_err!("stack: unexpected return type {return_field}");
+ };
+ let [ColumnarValue::Scalar(n), values @ ..] = args.as_slice() else {
+ return internal_err!("stack: `n` must be a scalar");
+ };
+ let n = parse_n(n)?;
+ // All-scalar values return one list, which the caller expands to
+ // `number_rows` copies
+ check_generated_rows(n, number_rows)?;
+ if values.iter().all(|v| matches!(v, ColumnarValue::Scalar(_))) {
+ let list = stack(values, item, n, 1)?;
+ return Ok(ColumnarValue::Scalar(ScalarValue::List(Arc::new(list))));
+ }
+ Ok(ColumnarValue::Array(Arc::new(stack(
+ values,
+ item,
+ n,
+ number_rows,
+ )?)))
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ // `n` is an `INT` as in Spark, while integer literals are BIGINT here. The
+ // values are cast to their column's type at execution, since Spark does no
+ // implicit casts.
+ let mut arg_types = arg_types.to_vec();
+ if let Some(n) = arg_types.first_mut().filter(|n| n.is_integer()) {
+ *n = DataType::Int32;
+ }
+ Ok(arg_types)
+ }
+}
+
+/// Returns the number of rows `n` to generate per input row.
+fn parse_n(n: &ScalarValue) -> Result<usize> {
+ match n {
+ ScalarValue::Int32(Some(v)) if *v > 0 => Ok(*v as usize),
+ _ => plan_err!("The `n` must be between (0, 2147483647] (current value = {n})."),
+ }
+}
+
+/// Returns the type of each output column, which is the type of its first
+/// non-NULL value as in Spark's `StackCoercion`. Errors on the first value, in
+/// argument order, whose type differs from its column's.
+fn column_types(
+ arg_fields: &[FieldRef],
+ scalar_arguments: &[Option<&ScalarValue>],
+ num_fields: usize,
+) -> Result<Vec<DataType>> {
+ // Integer literals are BIGINT here but INT in Spark, so one that fits is an INT
+ // in a column that has an INT value, wherever it is. Being a literal never
+ // changes the column type otherwise, which must not change when constant
+ // folding turns an expression into a literal.
+ let mut has_int32 = vec![false; num_fields];
+ for (i, arg_field) in arg_fields.iter().enumerate() {
+ has_int32[i % num_fields] |= arg_field.data_type() == &DataType::Int32;
+ }
+ let mut types = vec![DataType::Null; num_fields];
+ for (i, arg_field) in arg_fields.iter().enumerate() {
+ let col = i % num_fields;
+ let literal = scalar_arguments.get(i).copied().flatten();
+ let actual = if has_int32[col]
+ && arg_field.data_type() == &DataType::Int64
+ && literal.is_some_and(|literal| literal.cast_to(&DataType::Int32).is_ok())
+ {
+ &DataType::Int32
+ } else {
+ arg_field.data_type()
+ };
+ let expected = &mut types[col];
+ if expected.is_null() {
+ *expected = actual.clone();
+ } else if !actual.is_null() && !same_spark_type(actual, expected) {
+ return plan_err!(
+ "The data type of the column ({col}) do not have the same type: \"{expected}\" ({}) <> \"{actual}\" ({}).",
+ col + 1,
+ i + 1
+ );
+ }
+ }
+ Ok(types)
+}
+
+/// Whether two Arrow types are the same Spark type. Types of one logical type, such
+/// as `Utf8` and `Utf8View`, match, and list elements and map entries match
+/// regardless of their field names, which Spark types do not have. Nullability
+/// must match, as in Spark.
+fn same_spark_type(left: &DataType, right: &DataType) -> bool {
+ use DataType::*;
+ fn same_field(left: &FieldRef, right: &FieldRef) -> bool {
+ left.is_nullable() == right.is_nullable()
+ && same_spark_type(left.data_type(), right.data_type())
+ }
+ left == right
+ || match (left, right) {
+ (
+ List(left) | LargeList(left) | ListView(left) | LargeListView(left),
+ List(right) | LargeList(right) | ListView(right) | LargeListView(right),
+ ) => same_field(left, right),
+ // Map casts match the key and value fields by position
+ (Map(left, _), Map(right, _)) => {
+ match (left.data_type(), right.data_type()) {
+ (Struct(left), Struct(right)) => {
+ left.len() == right.len()
+ && left
+ .iter()
+ .zip(right.iter())
+ .all(|(l, r)| same_field(l, r))
+ }
+ _ => false,
+ }
+ }
+ (Struct(left), Struct(right)) => {
+ left.len() == right.len()
+ && left.iter().zip(right.iter()).all(|(left, right)| {
+ left.name() == right.name() && same_field(left, right)
+ })
+ }
+ _ => NativeType::from(left) == NativeType::from(right),
+ }
+}
+
+/// Errors if generating `n` rows for each of `number_rows` input rows exceeds
+/// `MAX_GENERATED_ROWS`.
+fn check_generated_rows(n: usize, number_rows: usize) -> Result<()> {
+ if number_rows.saturating_mul(n) > MAX_GENERATED_ROWS {
+ return Err(resources_datafusion_err!(
+ "stack: generating {n} rows for each of {number_rows} input rows exceeds the limit of {MAX_GENERATED_ROWS} rows per batch"
+ ));
+ }
+ Ok(())
+}
+
+/// Generates `n` rows of `item`'s struct type for each of `number_rows` input
+/// rows. Field `j` of generated row `i` is `values[i * num_fields + j]`, or NULL
+/// past the last value.
+fn stack(
+ values: &[ColumnarValue],
+ item: &FieldRef,
+ n: usize,
+ number_rows: usize,
+) -> Result<ListArray> {
+ let DataType::Struct(fields) = item.data_type() else {
+ return internal_err!("stack: unexpected element type {item}");
+ };
+ if fields.len() != values.len().div_ceil(n) {
+ return internal_err!(
+ "stack: {} values in {n} rows need {} fields, got {item}",
+ values.len(),
+ values.len().div_ceil(n)
+ );
+ }
+ check_generated_rows(n, number_rows)?;
+ // Every array is one value per input row, which the kernels rely on
+ if let Some(len) = values.iter().find_map(|value| match value {
+ ColumnarValue::Array(array) if array.len() != number_rows => Some(array.len()),
+ _ => None,
+ }) {
+ return internal_err!("stack: array of {len} rows in a batch of {number_rows}");
+ }
+ // The limit keeps the offsets within `i32`
+ let offsets = OffsetBuffer::from_repeated_length(n, number_rows);
+ let columns = if number_rows == 0 {
+ // The column builders below need at least one row
+ fields
+ .iter()
+ .map(|field| new_empty_array(field.data_type()))
+ .collect()
+ } else {
+ // Cast every value to its column's type, as a caller such as Comet may plan
+ // the return type itself
+ let values = values
+ .iter()
+ .zip(fields.iter().cycle())
+ .map(|(value, field)| value.cast_to(field.data_type(), None))
+ .collect::<Result<Vec<_>>>()?;
+ if n == 1 {
+ // Every value is a column as is, without copying arrays
+ ColumnarValue::values_to_arrays(&values)?
+ } else {
+ stack_columns(&values, fields, n, number_rows)?
+ }
+ };
+ let rows = StructArray::try_new(fields.clone(), columns, None)?;
+ Ok(ListArray::try_new(
+ Arc::clone(item),
+ offsets,
+ Arc::new(rows),
+ None,
+ )?)
+}
+
+/// Builds each output column from its values, one per generated row, with the
+/// rows past the last value as NULL padding. An array value is read at the input
+/// row. A scalar is a single value array read at index 0, so it is never
+/// broadcast, and with a single input row both readings agree.
+fn stack_columns(
+ values: &[ColumnarValue],
+ fields: &Fields,
+ n: usize,
+ number_rows: usize,
+) -> Result<Vec<ArrayRef>> {
+ let num_fields = fields.len();
+ let mut sources = Vec::with_capacity(values.len().div_ceil(num_fields));
+ fields
+ .iter()
+ .enumerate()
+ .map(|(col, field)| {
+ let data_type = field.data_type();
+ sources.clear();
+ for value in values.iter().skip(col).step_by(num_fields) {
+ sources.push(value.to_array(1)?);
+ }
+ macro_rules! primitive_helper {
+ ($t:ty) => {
+ Ok(stack_primitive::<$t>(&sources, n, number_rows, data_type))
+ };
+ }
+ downcast_primitive! {
+ data_type => (primitive_helper),
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary
+ if sources.iter().all(|source| source.len() == 1) =>
+ {
+ // The same `n` values for every input row, such as unpivot
+ // keys, which `tile` copies more than 10x faster than `take`
+ tile(stack_take(&sources, n, 1, data_type)?, number_rows)
+ }
+ _ => stack_take(&sources, n, number_rows, data_type),
+ }
+ })
+ .collect()
+}
+
+/// Writes source `i` to every `n`-th slot starting at `i`, and leaves the slots of
+/// padding rows NULL. This avoids the index lookups of `take`, and only visits the
+/// NULL slots of the validity. It is about 2-3x faster for primitive types.
+fn stack_primitive<T: ArrowPrimitiveType>(
+ sources: &[ArrayRef],
+ n: usize,
+ number_rows: usize,
+ data_type: &DataType,
+) -> ArrayRef {
+ let len = number_rows * n;
+ let mut buffer = vec![T::Native::default(); len];
+ let mut nulls = NullBufferBuilder::new_with_len(len);
+ for (i, source) in sources.iter().enumerate() {
+ let source = source.as_primitive::<T>();
+ let per_row = source.len() > 1;
+ let slots = buffer[i..].iter_mut().step_by(n);
+ // `for` loops, as `for_each` on these iterators is 2x slower
+ if per_row {
+ for (slot, value) in slots.zip(source.values().iter()) {
+ *slot = *value;
+ }
+ } else if source.is_valid(0) {
+ let value = source.value(0);
+ for slot in slots {
+ *slot = value;
+ }
+ }
+ if let Some(source_nulls) = source.nulls().filter(|nulls| nulls.null_count() > 0)
+ {
+ if per_row {
+ for input_row in (!source_nulls.inner()).set_indices() {
+ nulls.set_bit(input_row * n + i, false);
+ }
+ } else {
+ for input_row in 0..number_rows {
+ nulls.set_bit(input_row * n + i, false);
+ }
+ }
+ }
+ }
+ for i in sources.len()..n {
+ for input_row in 0..number_rows {
+ nulls.set_bit(input_row * n + i, false);
+ }
+ }
+ Arc::new(
+ PrimitiveArray::<T>::new(buffer.into(), nulls.build())
+ .with_data_type(data_type.clone()),
+ )
+}
+
+/// Takes every output value from the concatenated sources, plus one NULL at the
+/// end when there are padding rows. `take` has dedicated kernels for every type,
+/// such as `Boolean`, and keeps the data buffers of view types shared.
+fn stack_take(
+ sources: &[ArrayRef],
+ n: usize,
+ number_rows: usize,
+ data_type: &DataType,
+) -> Result<ArrayRef> {
+ let padding = n - sources.len();
+ // A NULL only when there is padding, as any NULL makes `take` track validity
+ let null = (padding > 0).then(|| new_null_array(data_type, 1));
+ let mut arrays = Vec::with_capacity(sources.len() + 1);
+ // Source `i` is read at `start + input_row * step`
+ let mut steps = Vec::with_capacity(sources.len());
+ let mut start = 0;
+ for source in sources {
+ arrays.push(source.as_ref());
+ steps.push((start, u32::from(source.len() > 1)));
+ start += source.len() as u32;
+ }
+ arrays.extend(null.as_deref());
+ let null_index = start;
+ let concatenated = concat(&arrays)?;
+ // Padding slots keep the NULL's index
+ let mut indices = vec![null_index; number_rows * n];
+ for (i, &(start, step)) in steps.iter().enumerate() {
+ for (slot, input_row) in indices[i..].iter_mut().step_by(n).zip(0u32..) {
+ *slot = start + input_row * step;
+ }
+ }
+ Ok(take(&concatenated, &UInt32Array::from(indices), None)?)
+}
+
+/// Concatenates `count` copies of `block`. It doubles the block up to 256 copies
+/// and then concatenates references to it, so the output is written about once.
+fn tile(mut block: ArrayRef, count: usize) -> Result<ArrayRef> {
+ let len = block.len();
+ let mut copies = 1;
+ while copies * 2 <= count.min(256) {
+ block = concat(&[block.as_ref(), block.as_ref()])?;
+ copies *= 2;
+ }
+ let tail = block.slice(0, (count % copies) * len);
+ let mut parts = vec![block.as_ref(); count / copies];
+ parts.push(tail.as_ref());
+ Ok(concat(&parts)?)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::Int64Array;
+
+ fn item(fields: Vec<Field>) -> FieldRef {
+ Arc::new(Field::new_list_field(DataType::Struct(fields.into()), true))
+ }
+
+ #[test]
+ fn test_stack_empty_batch() -> Result<()> {
+ let empty: ArrayRef = Arc::new(Int64Array::from(Vec::<i64>::new()));
+ let values = [
+ ColumnarValue::Array(Arc::clone(&empty)),
+ ColumnarValue::Array(empty),
+ ];
+ let item = item(vec![Field::new("col0", DataType::Int64, true)]);
+ let list = stack(&values, &item, 2, 0)?;
+ assert!(list.is_empty() && list.values().is_empty());
+ Ok(())
+ }
+
+ #[test]
+ fn test_stack_generated_rows_limit() {
+ let item = item(vec![Field::new("col0", DataType::Int64, true)]);
+ let values = [ColumnarValue::Scalar(ScalarValue::Int64(Some(1)))];
+ let err = stack(&values, &item, i32::MAX as usize, 1).unwrap_err();
+ assert!(
+ err.to_string().contains(
+ "generating 2147483647 rows for each of 1 input rows exceeds the limit"
+ ),
+ "{err}"
+ );
+ assert!(stack(&values, &item, MAX_GENERATED_ROWS, 1).is_ok());
+ }
+
+ #[test]
+ fn test_stack_all_scalar_limit_uses_batch_size() {
+ let item = item(vec![Field::new("col0", DataType::Int64, true)]);
+ let return_field = Arc::new(Field::new(
+ "stack",
+ DataType::List(Arc::clone(&item)),
+ false,
+ ));
+ let args = ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Scalar(ScalarValue::Int32(Some(1 << 20))),
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
+ ],
+ arg_fields: vec![],
+ number_rows: 8192,
+ return_field,
+ config_options: Arc::default(),
+ };
+ let err = SparkStack::new().invoke_with_args(args).unwrap_err();
+ assert!(
+ err.to_string().contains(
+ "generating 1048576 rows for each of 8192 input rows exceeds the limit"
+ ),
+ "{err}"
+ );
+ }
+
+ #[test]
+ fn test_column_types_widen_only_bigint_literals() {
+ let fields = |types: [DataType; 2]| -> Vec<FieldRef> {
+ types.map(|t| Arc::new(Field::new("a", t, true))).to_vec()
+ };
+ let small = ScalarValue::Int16(Some(0));
+ let err = column_types(
+ &fields([DataType::Int32, DataType::Int16]),
+ &[None, Some(&small)],
+ 1,
+ )
+ .unwrap_err();
+ assert!(err.to_string().contains("\"Int32\" (1) <> \"Int16\" (2)"));
+ let big = ScalarValue::Int64(Some(0));
+ let types = column_types(
+ &fields([DataType::Int32, DataType::Int64]),
+ &[None, Some(&big)],
+ 1,
+ )
+ .unwrap();
+ assert_eq!(types, [DataType::Int32]);
+ }
+
+ #[test]
+ fn test_stack_array_length_mismatch() {
+ let item = item(vec![Field::new("col0", DataType::Int64, true)]);
+ let array: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3]));
+ let err = stack(&[ColumnarValue::Array(array)], &item, 1, 2).unwrap_err();
+ assert!(err.to_string().contains("array of 3 rows in a batch of 2"));
+ }
+
+ #[test]
+ fn test_tile_block_boundaries() -> Result<()> {
+ let block: ArrayRef = Arc::new(Int64Array::from(vec![1, 2]));
+ for count in [0, 1, 2, 255, 256, 257, 511, 512, 513, 1000] {
+ let tiled = tile(Arc::clone(&block), count)?;
+ let expected = (0..count).flat_map(|_| [1, 2]).collect::<Vec<i64>>();
+ assert_eq!(
+ tiled.as_primitive::<arrow::datatypes::Int64Type>().values(),
+ &expected
+ );
+ }
+ Ok(())
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/spark/generator/stack.slt b/datafusion/sqllogictest/test_files/spark/generator/stack.slt
new file mode 100644
index 0000000000..a69b6edae9
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/spark/generator/stack.slt
@@ -0,0 +1,196 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Tests for Spark-compatible stack function
+# https://spark.apache.org/docs/latest/api/sql/index.html#stack
+#
+# Spark's stack generates rows. Here it returns them as a list of structs, which
+# unnest expands. `n` must be an integer literal, so it is written without a cast.
+
+# Values fill n rows of ceil(k / n) columns in order, padded with NULLs
+query ?
+SELECT stack(1, 1, 2, 3);
+----
+[{col0: 1, col1: 2, col2: 3}]
+
+query ?
+SELECT stack(2, 1, 2, 3);
+----
+[{col0: 1, col1: 2}, {col0: 3, col1: NULL}]
+
+query ?
+SELECT stack(4, 1, 2, 3);
+----
+[{col0: 1}, {col0: 2}, {col0: 3}, {col0: NULL}]
+
+query TB
+SELECT arrow_typeof(stack(2, a, b, c)), arrow_field(stack(2, a, b, c))['nullable'] FROM (VALUES (1, 2, 3)) AS t(a, b, c);
+----
+List(Struct("col0": Int64, "col1": Int64)) false
+
+# Various column types
+query ?
+SELECT stack(2, 1.5::DECIMAL(10, 2), '2024-01-01'::DATE, true, [1, 2], named_struct('a', 1), NULL, NULL, false, [3], named_struct('a', 2));
+----
+[{col0: 1.50, col1: 2024-01-01, col2: true, col3: [1, 2], col4: {a: 1}}, {col0: NULL, col1: NULL, col2: false, col3: [3], col4: {a: 2}}]
+
+# NULLs take the type of the first non-NULL value in their column
+query ?T
+SELECT stack(3, 1, 1.1, NULL, 2, NULL, 'b', NULL, 3.3, 'c'), arrow_typeof(stack(3, 1, 1.1, NULL, 2, NULL, 'b', NULL, 3.3, 'c'));
+----
+[{col0: 1, col1: 1.1, col2: NULL}, {col0: 2, col1: NULL, col2: b}, {col0: NULL, col1: 3.3, col2: c}] List(Struct("col0": Int64, "col1": Float64, "col2": Utf8))
+
+# A column of only NULLs has the NULL type, and the list is never NULL
+query ?T
+SELECT stack(2, NULL, NULL), arrow_typeof(stack(2, NULL, NULL));
+----
+[{col0: NULL}, {col0: NULL}] List(Struct("col0": Null))
+
+# A NULL before an array, and a column of NULL type
+query ??
+SELECT stack(2, NULL, a), stack(2, a, b) FROM (VALUES (1, NULL), (NULL, NULL)) AS t(a, b);
+----
+[{col0: NULL}, {col0: 1}] [{col0: 1}, {col0: NULL}]
+[{col0: NULL}, {col0: NULL}] [{col0: NULL}, {col0: NULL}]
+
+# NULLs in a boolean column
+query ?
+SELECT stack(2, b, NOT b) FROM (VALUES (true), (NULL)) AS t(b);
+----
+[{col0: true}, {col0: false}]
+[{col0: NULL}, {col0: NULL}]
+
+# NULL inside a list and struct, and NULL in their place
+query ??
+SELECT stack(2, [1, NULL], NULL), stack(2, named_struct('a', NULL::INT), NULL);
+----
+[{col0: [1, NULL]}, {col0: NULL}] [{col0: {a: NULL}}, {col0: NULL}]
+
+# Array inputs, also with one row per input row
+query ??
+SELECT stack(2, a, b, c), stack(1, a, 'x') FROM (VALUES (1, 2, 3), (4, 5, 6)) AS t(a, b, c);
+----
+[{col0: 1, col1: 2}, {col0: 3, col1: NULL}] [{col0: 1, col1: x}]
+[{col0: 4, col1: 5}, {col0: 6, col1: NULL}] [{col0: 4, col1: x}]
+
+# Array, scalar, NULL and padding values in the same columns
+query ?
+SELECT stack(3, i, 'a', s, -1, NULL, 'b', i) FROM (VALUES (1, 's1'), (NULL, 's2'), (3, NULL)) AS t(i, s);
+----
+[{col0: 1, col1: a, col2: s1}, {col0: -1, col1: NULL, col2: b}, {col0: 1, col1: NULL, col2: NULL}]
+[{col0: NULL, col1: a, col2: s2}, {col0: -1, col1: NULL, col2: b}, {col0: NULL, col1: NULL, col2: NULL}]
+[{col0: 3, col1: a, col2: NULL}, {col0: -1, col1: NULL, col2: b}, {col0: 3, col1: NULL, col2: NULL}]
+
+# Values of one Spark type with different Arrow types are cast to the first one
+query ??T
+SELECT stack(2, 'a', v), stack(2, d, 'a'), arrow_typeof(stack(2, d, 'a')) FROM (VALUES (arrow_cast('b', 'Utf8View'), arrow_cast('b', 'Dictionary(Int32, Utf8)'))) AS t(v, d);
+----
+[{col0: a}, {col0: b}] [{col0: b}, {col0: a}] List(Struct("col0": Dictionary(Int32, Utf8)))
+
+# List elements and map entries match regardless of their field names, such as
+# `element` from Spark's array and `item` from DataFusion's
+query ???
+SELECT stack(2, array(1), [2]), stack(2, named_struct('a', array(1)), named_struct('a', [2])), stack(2, map_from_arrays(array('a'), array(1)), arrow_cast(map_from_arrays(array('b'), array(2)), 'Map("key_value": non-null Struct("key": non-null Utf8, "value": Int64), unsorted)'));
+----
+[{col0: [1]}, {col0: [2]}] [{col0: {a: [1]}}, {col0: {a: [2]}}] [{col0: {a: 1}}, {col0: {b: 2}}]
+
+# Nullability must match, as in Spark
+query error The data type of the column \(0\) do not have the same type
+SELECT stack(2, arrow_cast([1], 'List(non-null Int64)'), [2, NULL]);
+
+# Integer literals are BIGINT here but INT in Spark, so one that fits can follow an
+# INT value
+query ?T
+SELECT stack(2, a, 0), arrow_typeof(stack(2, a, 0)) FROM (VALUES (1::INT)) AS t(a);
+----
+[{col0: 1}, {col0: 0}] List(Struct("col0": Int32))
+
+query error The data type of the column \(0\) do not have the same type: "Int32" \(1\) <> "Int64" \(2\)\.
+SELECT stack(2, a, 3000000000) FROM (VALUES (1::INT)) AS t(a);
+
+query error The data type of the column \(0\) do not have the same type: "Int16" \(1\) <> "Int64" \(2\)\.
+SELECT stack(2, a, 0) FROM (VALUES (1::SMALLINT)) AS t(a);
+
+# The order does not matter, and the literal can be in a later group of values
+query ?T
+SELECT stack(2, 0, a), arrow_typeof(stack(2, 0, a)) FROM (VALUES (1::INT)) AS t(a);
+----
+[{col0: 0}, {col0: 1}] List(Struct("col0": Int32))
+
+query ?T
+SELECT stack(2, 'x', 1, 'y', a), arrow_typeof(stack(2, 'x', 1, 'y', a)) FROM (VALUES (2::INT)) AS t(a);
+----
+[{col0: x, col1: 1}, {col0: y, col1: 2}] List(Struct("col0": Utf8, "col1": Int32))
+
+query error The data type of the column \(0\) do not have the same type: "Int64" \(1\) <> "Int32" \(2\)\.
+SELECT stack(2, 3000000000, a) FROM (VALUES (1::INT)) AS t(a);
+
+# unnest expands the generated rows, keeping those with NULL values
+query ITI rowsort
+SELECT c1, unnest(unnest(stack(2, 'Key', c1, 'Value', c2))) FROM (VALUES (0, NULL), (NULL, 2)) AS t(c1, c2);
+----
+0 Key 0
+0 Value NULL
+NULL Key NULL
+NULL Value 2
+
+# The keys stay aligned with their values over more input rows than one tiled
+# block holds
+query I
+SELECT sum(CASE WHEN s['col0'] = 'k1' THEN s['col1'] ELSE -s['col1'] END) FROM (SELECT unnest(stack(2, 'k1', value, 'k2', -value)) AS s FROM generate_series(1, 1000));
+----
+1001000
+
+query error The `stack` requires > 1 parameters but the actual number is 0\.
+SELECT stack();
+
+query error The `stack` requires > 1 parameters but the actual number is 1\.
+SELECT stack(1);
+
+# n must be a positive integer literal
+query error The first parameter requires the "INT" type, however "NULL" has the type "Null"\.
+SELECT stack(NULL, 1, 2);
+
+query error The first parameter requires the "INT" type, however "Float64\(1\.1\)" has the type "Float64"\.
+SELECT stack(1.1, 1, 2, 3);
+
+query error The `n` must be between \(0, 2147483647\] \(current value = -1\)\.
+SELECT stack(-1, 1, 2, 3);
+
+# Spark types 2147483648 as BIGINT and reports the type instead
+query error The `n` must be between \(0, 2147483647\] \(current value = 2147483648\)\.
+SELECT stack(2147483648, 1);
+
+query error The first parameter of `stack` must be an integer literal, however "n" is not one\.
+SELECT stack(n, a, b, c) FROM (VALUES (2, 1, 2, 3)) AS t(n, a, b, c);
+
+# Spark accepts any foldable n, such as 3 - 1 or 2::INT. DataFusion only sees the
+# value of a literal when it plans the return type, before expressions are folded.
+query error The first parameter of `stack` must be an integer literal, however "Int64\(3\) - Int64\(1\)" is not one\.
+SELECT stack(3 - 1, 1, '2', NULL, NULL);
+
+# The generated rows are materialized, so their number is limited
+query error stack: generating 2000000000 rows for each of 1 input rows exceeds the limit
+SELECT stack(2000000000, 1);
+
+# The values of a column must have the same type. The first mismatch in
+# argument order is reported.
+query error The data type of the column \(0\) do not have the same type: "Int64" \(1\) <> "Utf8" \(2\)\.
+SELECT stack(2, 1, '2.2');
+
+query error The data type of the column \(1\) do not have the same type: "Utf8" \(2\) <> "Int64" \(4\)\.
+SELECT stack(3, 1, 'a', 2, 5, 'x', 'b');