Skip to content

Apache DataFusion SQL Query Engine

fix: Ensure `array_append`, `array_prepend` handle NULLs correctly (#25229)

Neil Conway00c58dbf06cc
diff --git a/datafusion/functions-nested/src/concat.rs b/datafusion/functions-nested/src/concat.rs
index 1f03a0b170..8b8d39574e 100644
--- a/datafusion/functions-nested/src/concat.rs
+++ b/datafusion/functions-nested/src/concat.rs
@@ -572,7 +572,8 @@ where
///
/// This function takes a ListArray, an ArrayRef, a FieldRef, and a boolean flag
/// indicating whether to append or prepend the elements. It returns a `Result<ArrayRef>`
-/// representing the resulting ListArray after the operation.
+/// representing the resulting ListArray after the operation. A NULL list is
+/// treated as an empty list, so its result holds only the element.
///
/// # Arguments
///
@@ -583,10 +584,13 @@ where
///
/// # Examples
///
+/// ```text
/// generic_append_and_prepend(
/// [1, 2, 3], 4, append => [1, 2, 3, 4]
/// 5, [6, 7, 8], prepend => [5, 6, 7, 8]
+/// NULL, 4, append => [4]
/// )
+/// ```
fn generic_append_and_prepend<O: OffsetSizeTrait>(
list_array: &GenericListArray<O>,
element_array: &ArrayRef,
@@ -612,8 +616,14 @@ where
let element_index = 1;
for (row_index, offset_window) in list_array.offsets().windows(2).enumerate() {
- let start = offset_window[0].to_usize().unwrap();
- let end = offset_window[1].to_usize().unwrap();
+ let (start, end) = if list_array.is_null(row_index) {
+ (0, 0)
+ } else {
+ (
+ offset_window[0].to_usize().unwrap(),
+ offset_window[1].to_usize().unwrap(),
+ )
+ };
if is_append {
mutable.try_extend(values_index, start, end)?;
mutable.try_extend(element_index, row_index, row_index + 1)?;
diff --git a/datafusion/sqllogictest/test_files/array/array_append.slt b/datafusion/sqllogictest/test_files/array/array_append.slt
index 0758a09a49..5a13a5827b 100644
--- a/datafusion/sqllogictest/test_files/array/array_append.slt
+++ b/datafusion/sqllogictest/test_files/array/array_append.slt
@@ -192,7 +192,7 @@ select array_append(column1, column2) from fixed_arrays_values;
[11, 12, 13, 14, 15, 16, 17, 18, NULL, 20, 12]
[21, 22, 23, NULL, 25, 26, 27, 28, 29, 30, 23]
[31, 32, 33, 34, 35, NULL, 37, 38, 39, 40, 34]
-[NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 44]
+[44]
[41, 42, 43, 44, 45, 46, 47, 48, 49, 50, NULL]
[51, 52, NULL, 54, 55, 56, 57, 58, 59, 60, 55]
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 66]
@@ -247,8 +247,8 @@ select array_append(column2, 100.1), array_append(column3, '.') from fixed_size_
[7.7, 8.8, 9.9, 100.1] [d, NULL, l, o, r, .]
[10.1, NULL, 12.2, 100.1] [s, i, t, a, b, .]
[13.3, 14.4, 15.5, 100.1] [a, m, e, t, x, .]
-[NULL, NULL, NULL, 100.1] [,, a, b, c, d, .]
-[16.6, 17.7, 18.8, 100.1] [NULL, NULL, NULL, NULL, NULL, .]
+[100.1] [,, a, b, c, d, .]
+[16.6, 17.7, 18.8, 100.1] [.]
# array_append with columns and scalars #2
query ??
@@ -331,5 +331,35 @@ from values (3), (NULL);
[1, 2, 3] List(Int64)
[1, 2, NULL] List(Int64)
+# https://github.com/apache/datafusion/issues/25222
+query I??
+select
+ column3,
+ nullif(column1, column2),
+ array_append(nullif(column1, column2), 9)
+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
+order by column3;
+----
+1 NULL [9]
+2 [4, 5, 6] [4, 5, 6, 9]
+
+query I?
+select
+ column3,
+ array_append(arrow_cast(nullif(column1, column2), 'LargeList(Int64)'), 9)
+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
+order by column3;
+----
+1 [9]
+2 [4, 5, 6, 9]
+
+query I?
+select column2, array_append(column1, 9)
+from values (arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)'), 1), (NULL, 2)
+order by column2;
+----
+1 [1, 2, 3, 9]
+2 [9]
+
include ./cleanup.slt.part
diff --git a/datafusion/sqllogictest/test_files/array/array_prepend.slt b/datafusion/sqllogictest/test_files/array/array_prepend.slt
index bfb61ab4f9..d0e2cb7a8a 100644
--- a/datafusion/sqllogictest/test_files/array/array_prepend.slt
+++ b/datafusion/sqllogictest/test_files/array/array_prepend.slt
@@ -196,7 +196,7 @@ select array_prepend(column2, column1) from fixed_arrays_values;
[12, 11, 12, 13, 14, 15, 16, 17, 18, NULL, 20]
[23, 21, 22, 23, NULL, 25, 26, 27, 28, 29, 30]
[34, 31, 32, 33, 34, 35, NULL, 37, 38, 39, 40]
-[44, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL]
+[44]
[NULL, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50]
[55, 51, 52, NULL, 54, 55, 56, 57, 58, 59, 60]
[66, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70]
@@ -251,8 +251,8 @@ select array_prepend(100.1, column2), array_prepend('.', column3) from fixed_siz
[100.1, 7.7, 8.8, 9.9] [., d, NULL, l, o, r]
[100.1, 10.1, NULL, 12.2] [., s, i, t, a, b]
[100.1, 13.3, 14.4, 15.5] [., a, m, e, t, x]
-[100.1, NULL, NULL, NULL] [., ,, a, b, c, d]
-[100.1, 16.6, 17.7, 18.8] [., NULL, NULL, NULL, NULL, NULL]
+[100.1] [., ,, a, b, c, d]
+[100.1, 16.6, 17.7, 18.8] [.]
# array_prepend with columns and scalars #2 (element is list)
query ??
@@ -336,4 +336,35 @@ from values (0), (NULL);
[NULL, 1, 2] List(Int64)
+# https://github.com/apache/datafusion/issues/25222
+query I??
+select
+ column3,
+ nullif(column1, column2),
+ array_prepend(9, nullif(column1, column2))
+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
+order by column3;
+----
+1 NULL [9]
+2 [4, 5, 6] [9, 4, 5, 6]
+
+query I?
+select
+ column3,
+ array_prepend(9, arrow_cast(nullif(column1, column2), 'LargeList(Int64)'))
+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
+order by column3;
+----
+1 [9]
+2 [9, 4, 5, 6]
+
+query I?
+select column2, array_prepend(9, column1)
+from values (arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)'), 1), (NULL, 2)
+order by column2;
+----
+1 [9, 1, 2, 3]
+2 [9]
+
+
include ./cleanup.slt.part