Apache DataFusion SQL Query Engine
bug: fix `array_remove_*` with NULLS (#21013)
Oleks V6ab16cc7fd97
diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rsindex 9e957c93e1..3d4076800e 100644--- a/datafusion/functions-nested/src/remove.rs+++ b/datafusion/functions-nested/src/remove.rs@@ -390,7 +390,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(let mut valid = NullBufferBuilder::new(list_array.len());for (row_index, offset_window) in list_array.offsets().windows(2).enumerate() {- if list_array.is_null(row_index) {+ if list_array.is_null(row_index) || element_array.is_null(row_index) {offsets.push(offsets[row_index]);valid.append_null();continue;diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.sltindex 83e9c9cc9c..573938af0c 100644--- a/datafusion/sqllogictest/test_files/array.slt+++ b/datafusion/sqllogictest/test_files/array.slt@@ -5528,21 +5528,47 @@ selectarray_remove(make_array(1, null, 2), null),array_remove(make_array(1, null, 2, null), null);-----[1, 2] [1, 2, NULL]+NULL NULLquery ??selectarray_remove(arrow_cast(make_array(1, null, 2), 'LargeList(Int64)'), null),array_remove(arrow_cast(make_array(1, null, 2, null), 'LargeList(Int64)'), null);-----[1, 2] [1, 2, NULL]+NULL NULLquery ??selectarray_remove(arrow_cast(make_array(1, null, 2), 'FixedSizeList(3, Int64)'), null),array_remove(arrow_cast(make_array(1, null, 2, null), 'FixedSizeList(4, Int64)'), null);-----[1, 2] [1, 2, NULL]+NULL NULL++# array_remove with null element from column+query ?+select array_remove(column1, column2) from (values+ (make_array(1, 2, 3), 2),+ (make_array(4, 5, 6), null),+ (make_array(7, 8, 9), 8),+ (null, 1)+) as t(column1, column2);+----+[1, 3]+NULL+[7, 9]+NULL++# array_remove with null element from column (LargeList)+query ?+select array_remove(column1, column2) from (values+ (arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 2),+ (arrow_cast(make_array(4, 5, 6), 'LargeList(Int64)'), null),+ (arrow_cast(make_array(7, 8, 9), 'LargeList(Int64)'), 8)+) as t(column1, column2);+----+[1, 3]+NULL+[7, 9]# array_remove scalar function #2 (element is list)query ??@@ -5685,6 +5711,46 @@ select array_remove(make_array([1, 2, 3], [4, 5, 6], [4, 5, 6], [10, 11, 12], [1## array_remove_n (aliases: `list_remove_n`)+# array_remove_n with null element scalar+query ??+select array_remove_n(make_array(1, 2, 2, 1, 1), NULL, 2),+ array_remove_n(make_array(1, 2, 2, 1, 1), 2, 2);+----+NULL [1, 1, 1]++# array_remove_n with null element scalar (LargeList)+query ??+select array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), NULL, 2),+ array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, 2);+----+NULL [1, 1, 1]++# array_remove_n with null element from column+query ?+select array_remove_n(column1, column2, column3) from (values+ (make_array(1, 2, 2, 1, 1), 2, 2),+ (make_array(3, 4, 4, 3, 3), null, 2),+ (make_array(5, 6, 6, 5, 5), 6, 1),+ (null, 1, 1)+) as t(column1, column2, column3);+----+[1, 1, 1]+NULL+[5, 6, 5, 5]+NULL++# array_remove_n with null element from column (LargeList)+query ?+select array_remove_n(column1, column2, column3) from (values+ (arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, 2),+ (arrow_cast(make_array(3, 4, 4, 3, 3), 'LargeList(Int64)'), null, 2),+ (arrow_cast(make_array(5, 6, 6, 5, 5), 'LargeList(Int64)'), 6, 1)+) as t(column1, column2, column3);+----+[1, 1, 1]+NULL+[5, 6, 5, 5]+# array_remove_n scalar function #1query ???select array_remove_n(make_array(1, 2, 2, 1, 1), 2, 2), array_remove_n(make_array(1.0, 2.0, 2.0, 1.0, 1.0), 1.0, 2), array_remove_n(make_array('h', 'e', 'l', 'l', 'o'), 'l', 3);@@ -5777,7 +5843,33 @@ select array_remove_n(make_array([1, 2, 3], [4, 5, 6], [4, 5, 6], [10, 11, 12],query ?select array_remove_all(make_array(1, 2, 2, 1, 1), NULL);-----[1, 2, 2, 1, 1]+NULL++# array_remove_all with null element from column+query ?+select array_remove_all(column1, column2) from (values+ (make_array(1, 2, 2, 1, 1), 2),+ (make_array(3, 4, 4, 3, 3), null),+ (make_array(5, 6, 6, 5, 5), 6),+ (null, 1)+) as t(column1, column2);+----+[1, 1, 1]+NULL+[5, 5, 5]+NULL++# array_remove_all with null element from column (LargeList)+query ?+select array_remove_all(column1, column2) from (values+ (arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2),+ (arrow_cast(make_array(3, 4, 4, 3, 3), 'LargeList(Int64)'), null),+ (arrow_cast(make_array(5, 6, 6, 5, 5), 'LargeList(Int64)'), 6)+) as t(column1, column2);+----+[1, 1, 1]+NULL+[5, 5, 5]# array_remove_all scalar function #1query ???