Apache DataFusion SQL Query Engine
fix: Ensure `array_append`, `array_prepend` handle NULLs correctly (#25229)
Neil Conway00c58dbf06cc
diff --git a/datafusion/functions-nested/src/concat.rs b/datafusion/functions-nested/src/concat.rsindex 1f03a0b170..8b8d39574e 100644--- a/datafusion/functions-nested/src/concat.rs+++ b/datafusion/functions-nested/src/concat.rs@@ -572,7 +572,8 @@ where////// This function takes a ListArray, an ArrayRef, a FieldRef, and a boolean flag/// indicating whether to append or prepend the elements. It returns a `Result<ArrayRef>`-/// representing the resulting ListArray after the operation.+/// representing the resulting ListArray after the operation. A NULL list is+/// treated as an empty list, so its result holds only the element.////// # Arguments///@@ -583,10 +584,13 @@ where////// # Examples///+/// ```text/// generic_append_and_prepend(/// [1, 2, 3], 4, append => [1, 2, 3, 4]/// 5, [6, 7, 8], prepend => [5, 6, 7, 8]+/// NULL, 4, append => [4]/// )+/// ```fn generic_append_and_prepend<O: OffsetSizeTrait>(list_array: &GenericListArray<O>,element_array: &ArrayRef,@@ -612,8 +616,14 @@ wherelet element_index = 1;for (row_index, offset_window) in list_array.offsets().windows(2).enumerate() {- let start = offset_window[0].to_usize().unwrap();- let end = offset_window[1].to_usize().unwrap();+ let (start, end) = if list_array.is_null(row_index) {+ (0, 0)+ } else {+ (+ offset_window[0].to_usize().unwrap(),+ offset_window[1].to_usize().unwrap(),+ )+ };if is_append {mutable.try_extend(values_index, start, end)?;mutable.try_extend(element_index, row_index, row_index + 1)?;diff --git a/datafusion/sqllogictest/test_files/array/array_append.slt b/datafusion/sqllogictest/test_files/array/array_append.sltindex 0758a09a49..5a13a5827b 100644--- a/datafusion/sqllogictest/test_files/array/array_append.slt+++ b/datafusion/sqllogictest/test_files/array/array_append.slt@@ -192,7 +192,7 @@ select array_append(column1, column2) from fixed_arrays_values;[11, 12, 13, 14, 15, 16, 17, 18, NULL, 20, 12][21, 22, 23, NULL, 25, 26, 27, 28, 29, 30, 23][31, 32, 33, 34, 35, NULL, 37, 38, 39, 40, 34]-[NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 44]+[44][41, 42, 43, 44, 45, 46, 47, 48, 49, 50, NULL][51, 52, NULL, 54, 55, 56, 57, 58, 59, 60, 55][61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 66]@@ -247,8 +247,8 @@ select array_append(column2, 100.1), array_append(column3, '.') from fixed_size_[7.7, 8.8, 9.9, 100.1] [d, NULL, l, o, r, .][10.1, NULL, 12.2, 100.1] [s, i, t, a, b, .][13.3, 14.4, 15.5, 100.1] [a, m, e, t, x, .]-[NULL, NULL, NULL, 100.1] [,, a, b, c, d, .]-[16.6, 17.7, 18.8, 100.1] [NULL, NULL, NULL, NULL, NULL, .]+[100.1] [,, a, b, c, d, .]+[16.6, 17.7, 18.8, 100.1] [.]# array_append with columns and scalars #2query ??@@ -331,5 +331,35 @@ from values (3), (NULL);[1, 2, 3] List(Int64)[1, 2, NULL] List(Int64)+# https://github.com/apache/datafusion/issues/25222+query I??+select+ column3,+ nullif(column1, column2),+ array_append(nullif(column1, column2), 9)+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)+order by column3;+----+1 NULL [9]+2 [4, 5, 6] [4, 5, 6, 9]++query I?+select+ column3,+ array_append(arrow_cast(nullif(column1, column2), 'LargeList(Int64)'), 9)+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)+order by column3;+----+1 [9]+2 [4, 5, 6, 9]++query I?+select column2, array_append(column1, 9)+from values (arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)'), 1), (NULL, 2)+order by column2;+----+1 [1, 2, 3, 9]+2 [9]+include ./cleanup.slt.partdiff --git a/datafusion/sqllogictest/test_files/array/array_prepend.slt b/datafusion/sqllogictest/test_files/array/array_prepend.sltindex bfb61ab4f9..d0e2cb7a8a 100644--- a/datafusion/sqllogictest/test_files/array/array_prepend.slt+++ b/datafusion/sqllogictest/test_files/array/array_prepend.slt@@ -196,7 +196,7 @@ select array_prepend(column2, column1) from fixed_arrays_values;[12, 11, 12, 13, 14, 15, 16, 17, 18, NULL, 20][23, 21, 22, 23, NULL, 25, 26, 27, 28, 29, 30][34, 31, 32, 33, 34, 35, NULL, 37, 38, 39, 40]-[44, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL]+[44][NULL, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50][55, 51, 52, NULL, 54, 55, 56, 57, 58, 59, 60][66, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70]@@ -251,8 +251,8 @@ select array_prepend(100.1, column2), array_prepend('.', column3) from fixed_siz[100.1, 7.7, 8.8, 9.9] [., d, NULL, l, o, r][100.1, 10.1, NULL, 12.2] [., s, i, t, a, b][100.1, 13.3, 14.4, 15.5] [., a, m, e, t, x]-[100.1, NULL, NULL, NULL] [., ,, a, b, c, d]-[100.1, 16.6, 17.7, 18.8] [., NULL, NULL, NULL, NULL, NULL]+[100.1] [., ,, a, b, c, d]+[100.1, 16.6, 17.7, 18.8] [.]# array_prepend with columns and scalars #2 (element is list)query ??@@ -336,4 +336,35 @@ from values (0), (NULL);[NULL, 1, 2] List(Int64)+# https://github.com/apache/datafusion/issues/25222+query I??+select+ column3,+ nullif(column1, column2),+ array_prepend(9, nullif(column1, column2))+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)+order by column3;+----+1 NULL [9]+2 [4, 5, 6] [9, 4, 5, 6]++query I?+select+ column3,+ array_prepend(9, arrow_cast(nullif(column1, column2), 'LargeList(Int64)'))+from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)+order by column3;+----+1 [9]+2 [9, 4, 5, 6]++query I?+select column2, array_prepend(9, column1)+from values (arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)'), 1), (NULL, 2)+order by column2;+----+1 [9, 1, 2, 3]+2 [9]++include ./cleanup.slt.part