Skip to content

Apache DataFusion SQL Query Engine

spark: support `collect_list` `collect_set` for `windows` execution (#23281)

Oleks V01c7527be0b3
diff --git a/datafusion/spark/src/function/aggregate/collect.rs b/datafusion/spark/src/function/aggregate/collect.rs
index 5af0fd39cc..310bc1c890 100644
--- a/datafusion/spark/src/function/aggregate/collect.rs
+++ b/datafusion/spark/src/function/aggregate/collect.rs
@@ -18,7 +18,7 @@
use arrow::array::ArrayRef;
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::utils::SingleRowListArrayBuilder;
-use datafusion_common::{Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::format_state_name;
use datafusion_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility};
@@ -33,6 +33,19 @@ use std::sync::Arc;
// - returns an empty list when all inputs are NULL
// - does not support ordering
+/// Build an empty list `ScalarValue` for a `List(element_type)` data type.
+/// Used as the result for empty window frames and for groups whose inputs
+/// were all NULL, matching Spark's `collect_list` / `collect_set` semantics.
+fn empty_list_scalar(list_type: &DataType) -> Result<ScalarValue> {
+ let DataType::List(field) = list_type else {
+ return internal_err!(
+ "collect_list/collect_set expected List return type, got {list_type:?}"
+ );
+ };
+ let empty = arrow::array::new_empty_array(field.data_type());
+ Ok(SingleRowListArrayBuilder::new(empty).build_list_scalar())
+}
+
// <https://spark.apache.org/docs/latest/api/sql/index.html#collect_list>
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkCollectList {
@@ -81,14 +94,17 @@ impl AggregateUDFImpl for SparkCollectList {
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
- let field = &acc_args.expr_fields[0];
- let data_type = field.data_type().clone();
+ let element_type = acc_args.expr_fields[0].data_type().clone();
let ignore_nulls = true;
Ok(Box::new(NullToEmptyListAccumulator::new(
- ArrayAggAccumulator::try_new(&data_type, ignore_nulls)?,
- data_type,
+ ArrayAggAccumulator::try_new(&element_type, ignore_nulls)?,
+ acc_args.return_type().clone(),
)))
}
+
+ fn default_value(&self, data_type: &DataType) -> Result<ScalarValue> {
+ empty_list_scalar(data_type)
+ }
}
// <https://spark.apache.org/docs/latest/api/sql/index.html#collect_set>
@@ -139,14 +155,17 @@ impl AggregateUDFImpl for SparkCollectSet {
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
- let field = &acc_args.expr_fields[0];
- let data_type = field.data_type().clone();
+ let element_type = acc_args.expr_fields[0].data_type().clone();
let ignore_nulls = true;
Ok(Box::new(NullToEmptyListAccumulator::new(
- DistinctArrayAggAccumulator::try_new(&data_type, None, ignore_nulls)?,
- data_type,
+ DistinctArrayAggAccumulator::try_new(&element_type, None, ignore_nulls)?,
+ acc_args.return_type().clone(),
)))
}
+
+ fn default_value(&self, data_type: &DataType) -> Result<ScalarValue> {
+ empty_list_scalar(data_type)
+ }
}
/// Wrapper accumulator that returns an empty list instead of NULL when all inputs are NULL.
@@ -154,12 +173,12 @@ impl AggregateUDFImpl for SparkCollectSet {
#[derive(Debug)]
struct NullToEmptyListAccumulator<T: Accumulator> {
inner: T,
- data_type: DataType,
+ list_type: DataType,
}
impl<T: Accumulator> NullToEmptyListAccumulator<T> {
- pub fn new(inner: T, data_type: DataType) -> Self {
- Self { inner, data_type }
+ pub fn new(inner: T, list_type: DataType) -> Self {
+ Self { inner, list_type }
}
}
@@ -179,14 +198,21 @@ impl<T: Accumulator> Accumulator for NullToEmptyListAccumulator<T> {
fn evaluate(&mut self) -> Result<ScalarValue> {
let result = self.inner.evaluate()?;
if result.is_null() {
- let empty_array = arrow::array::new_empty_array(&self.data_type);
- Ok(SingleRowListArrayBuilder::new(empty_array).build_list_scalar())
+ empty_list_scalar(&self.list_type)
} else {
Ok(result)
}
}
+ fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
+ self.inner.retract_batch(values)
+ }
+
+ fn supports_retract_batch(&self) -> bool {
+ self.inner.supports_retract_batch()
+ }
+
fn size(&self) -> usize {
- self.inner.size() + self.data_type.size()
+ self.inner.size() + self.list_type.size()
}
}
diff --git a/datafusion/sqllogictest/test_files/spark/aggregate/collect_window.slt b/datafusion/sqllogictest/test_files/spark/aggregate/collect_window.slt
new file mode 100644
index 0000000000..5661cb9432
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/spark/aggregate/collect_window.slt
@@ -0,0 +1,398 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#######
+# Tests for Spark-compat collect_list / collect_set as WINDOW functions.
+# Spark semantics:
+# - NULL inputs are skipped (Hive collect_list/collect_set behavior).
+# - An empty frame (or one where all inputs were NULL) evaluates to []
+# rather than NULL (nullable = false in Spark's Collect aggregate).
+# - collect_list preserves frame order; collect_set deduplicates.
+# Validates that NullToEmptyListAccumulator forwards retract_batch
+# so the wrapped ArrayAggAccumulator / DistinctArrayAggAccumulator can
+# drive sliding window frames.
+#######
+
+statement ok
+CREATE TABLE t(ts INT, val TEXT) AS VALUES
+ (1, 'A'), (2, 'B'), (3, 'C'), (4, 'D'), (5, 'E');
+
+# Unbounded preceding frame — accumulator only sees update_batch.
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+FROM t;
+----
+[A]
+[A, B]
+[A, B, C]
+[A, B, C, D]
+[A, B, C, D, E]
+
+# Bounded sliding ROWS frame — requires retract_batch.
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM t;
+----
+[A]
+[A, B]
+[B, C]
+[C, D]
+[D, E]
+
+# Wider sliding window.
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)
+FROM t;
+----
+[A]
+[A, B]
+[A, B, C]
+[B, C, D]
+[C, D, E]
+
+# Centered sliding window with PRECEDING + FOLLOWING.
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)
+FROM t;
+----
+[A, B]
+[A, B, C]
+[B, C, D]
+[C, D, E]
+[D, E]
+
+# Unbounded both sides — every row sees the full input.
+query ?
+SELECT collect_list(val)
+ OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+FROM t;
+----
+[A, B, C, D, E]
+[A, B, C, D, E]
+[A, B, C, D, E]
+[A, B, C, D, E]
+[A, B, C, D, E]
+
+# Single-row frame.
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN CURRENT ROW AND CURRENT ROW)
+FROM t;
+----
+[A]
+[B]
+[C]
+[D]
+[E]
+
+# Empty leading frame on the first row — Spark returns [].
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND 1 PRECEDING)
+FROM t;
+----
+[]
+[A]
+[A, B]
+[B, C]
+[C, D]
+
+# Empty trailing frame on the last row — Spark returns [].
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING)
+FROM t;
+----
+[B, C]
+[C, D]
+[D, E]
+[E]
+[]
+
+#######
+# NULL handling — Spark's collect_list skips NULL inputs.
+#######
+
+statement ok
+CREATE TABLE t_nulls(ts INT, val TEXT) AS VALUES
+ (1, 'A'), (2, NULL), (3, 'C'), (4, NULL), (5, 'E');
+
+# NULLs filtered out of the materialized list, but the row still emits one entry.
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM t_nulls;
+----
+[A]
+[A]
+[C]
+[C]
+[E]
+
+# Wider frame.
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)
+FROM t_nulls;
+----
+[A]
+[A]
+[A, C]
+[C]
+[C, E]
+
+# All-NULL frame collapses to [].
+statement ok
+CREATE TABLE t_allnull(ts INT, val TEXT) AS VALUES
+ (1, NULL), (2, NULL), (3, NULL);
+
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+FROM t_allnull;
+----
+[]
+[]
+[]
+
+#######
+# PARTITION BY — each partition starts with fresh accumulator state.
+#######
+
+statement ok
+CREATE TABLE t_parts(grp INT, ts INT, val TEXT) AS VALUES
+ (1, 1, 'A'), (1, 2, 'B'), (1, 3, 'C'),
+ (2, 1, 'X'), (2, 2, 'Y'), (2, 3, 'Z');
+
+query I?
+SELECT grp, collect_list(val)
+ OVER (PARTITION BY grp ORDER BY ts
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM t_parts
+ORDER BY grp, ts;
+----
+1 [A]
+1 [A, B]
+1 [B, C]
+2 [X]
+2 [X, Y]
+2 [Y, Z]
+
+#######
+# RANGE frame with value gaps — exercises multi-row retract.
+#######
+
+statement ok
+CREATE TABLE t_range(ts INT, val TEXT) AS VALUES
+ (1, 'A'), (2, 'B'), (3, 'C'), (4, 'D'), (100, 'E');
+
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING)
+FROM t_range;
+----
+[A, B, C]
+[A, B, C, D]
+[A, B, C, D]
+[B, C, D]
+[E]
+
+#######
+# GROUPS frame — rows tied on ORDER BY are processed together.
+#######
+
+statement ok
+CREATE TABLE t_groups(ts INT, val TEXT) AS VALUES
+ (1, 'A'), (1, 'B'), (2, 'C'), (2, 'D'), (3, 'E');
+
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM t_groups;
+----
+[A, B]
+[A, B]
+[A, B, C, D]
+[A, B, C, D]
+[C, D, E]
+
+#######
+# Integer-typed input — guards against type-specific regressions.
+#######
+
+statement ok
+CREATE TABLE t_int(ts INT, val INT) AS VALUES
+ (1, 10), (2, 20), (3, 30), (4, 40), (5, 50);
+
+query ?
+SELECT collect_list(val)
+ OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM t_int;
+----
+[10]
+[10, 20]
+[20, 30]
+[30, 40]
+[40, 50]
+
+#######
+# collect_set as a WINDOW function.
+# array_sort wraps the result because the underlying HashMap iteration
+# order is not deterministic.
+#######
+
+statement ok
+CREATE TABLE t_set(ts INT, val TEXT) AS VALUES
+ (1,'A'),(2,'A'),(3,'B'),(4,'C'),(5,'B');
+
+# Sliding ROWS frame, 2 PRECEDING.
+# Frame contents per row:
+# [A] -> {A}
+# [A,A] -> {A}
+# [A,A,B] -> {A,B}
+# [A,B,C] -> {A,B,C}
+# [B,C,B] -> {B,C}
+query ?
+SELECT array_sort(collect_set(val)
+ OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW))
+FROM t_set;
+----
+[A]
+[A]
+[A, B]
+[A, B, C]
+[B, C]
+
+# Narrower frame.
+query ?
+SELECT array_sort(collect_set(val)
+ OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW))
+FROM t_set;
+----
+[A]
+[A]
+[A, B]
+[B, C]
+[B, C]
+
+# Unbounded preceding — every distinct seen so far stays in.
+query ?
+SELECT array_sort(collect_set(val)
+ OVER (ORDER BY ts ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW))
+FROM t_set;
+----
+[A]
+[A]
+[A, B]
+[A, B, C]
+[A, B, C]
+
+# collect_set with NULLs — NULL never enters the set.
+statement ok
+CREATE TABLE t_set_nulls(ts INT, val TEXT) AS VALUES
+ (1,'A'),(2,NULL),(3,'A'),(4,NULL),(5,'B');
+
+query ?
+SELECT array_sort(collect_set(val)
+ OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW))
+FROM t_set_nulls;
+----
+[A]
+[A]
+[A]
+[A]
+[B]
+
+# collect_set with PARTITION BY — partition isolation.
+statement ok
+CREATE TABLE t_set_parts(grp INT, ts INT, val TEXT) AS VALUES
+ (1, 1, 'A'), (1, 2, 'A'), (1, 3, 'B'),
+ (2, 1, 'B'), (2, 2, 'C'), (2, 3, 'C');
+
+query I?
+SELECT grp, array_sort(collect_set(val)
+ OVER (PARTITION BY grp ORDER BY ts
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW))
+FROM t_set_parts
+ORDER BY grp, ts;
+----
+1 [A]
+1 [A]
+1 [A, B]
+2 [B]
+2 [B, C]
+2 [C]
+
+# Empty leading frame on the first row — Spark returns [].
+query ?
+SELECT array_sort(collect_set(val)
+ OVER (ORDER BY ts ROWS BETWEEN 2 PRECEDING AND 1 PRECEDING))
+FROM t_set;
+----
+[]
+[A]
+[A]
+[A, B]
+[B, C]
+
+# All-NULL window — set is empty.
+query ?
+SELECT array_sort(collect_set(val)
+ OVER (ORDER BY ts ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW))
+FROM t_allnull;
+----
+[]
+[]
+[]
+
+#######
+# Cleanup
+#######
+
+statement ok
+DROP TABLE t;
+
+statement ok
+DROP TABLE t_nulls;
+
+statement ok
+DROP TABLE t_allnull;
+
+statement ok
+DROP TABLE t_parts;
+
+statement ok
+DROP TABLE t_range;
+
+statement ok
+DROP TABLE t_groups;
+
+statement ok
+DROP TABLE t_int;
+
+statement ok
+DROP TABLE t_set;
+
+statement ok
+DROP TABLE t_set_nulls;
+
+statement ok
+DROP TABLE t_set_parts;