Apache DataFusion SQL Query Engine
Fix `array_repeat` handling of null count values (#20102)
lyne2f90194129d8
diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rsindex 28ec827cc5..5e78a4d0f6 100644--- a/datafusion/functions-nested/src/repeat.rs+++ b/datafusion/functions-nested/src/repeat.rs@@ -19,21 +19,23 @@use crate::utils::make_scalar_function;use arrow::array::{- Array, ArrayRef, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, UInt64Array,+ Array, ArrayRef, BooleanBufferBuilder, GenericListArray, Int64Array, OffsetSizeTrait,+ UInt64Array,};use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::compute;-use arrow::compute::cast;use arrow::datatypes::DataType;use arrow::datatypes::{DataType::{LargeList, List},Field,};-use datafusion_common::cast::{as_large_list_array, as_list_array, as_uint64_array};-use datafusion_common::{Result, exec_err, utils::take_function_args};+use datafusion_common::cast::{as_int64_array, as_large_list_array, as_list_array};+use datafusion_common::types::{NativeType, logical_int64};+use datafusion_common::{DataFusionError, Result};use datafusion_expr::{ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,};+use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};use datafusion_macros::user_doc;use std::any::Any;use std::sync::Arc;@@ -88,7 +90,17 @@ impl Default for ArrayRepeat {impl ArrayRepeat {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::coercible(+ vec![+ Coercion::new_exact(TypeSignatureClass::Any),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int64()),+ vec![TypeSignatureClass::Integer],+ NativeType::Int64,+ ),+ ],+ Volatility::Immutable,+ ),aliases: vec![String::from("list_repeat")],}}@@ -132,23 +144,6 @@ impl ScalarUDFImpl for ArrayRepeat {&self.aliases}- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- let [first_type, second_type] = take_function_args(self.name(), arg_types)?;-- // Coerce the second argument to Int64/UInt64 if it's a numeric type- let second = match second_type {- DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => {- DataType::Int64- }- DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => {- DataType::UInt64- }- _ => return exec_err!("count must be an integer type"),- };-- Ok(vec![first_type.clone(), second])- }-fn documentation(&self) -> Option<&Documentation> {self.doc()}@@ -156,15 +151,7 @@ impl ScalarUDFImpl for ArrayRepeat {fn array_repeat_inner(args: &[ArrayRef]) -> Result<ArrayRef> {let element = &args[0];- let count_array = &args[1];-- let count_array = match count_array.data_type() {- DataType::Int64 => &cast(count_array, &DataType::UInt64)?,- DataType::UInt64 => count_array,- _ => return exec_err!("count must be an integer type"),- };-- let count_array = as_uint64_array(count_array)?;+ let count_array = as_int64_array(&args[1])?;match element.data_type() {List(_) => {@@ -193,21 +180,31 @@ fn array_repeat_inner(args: &[ArrayRef]) -> Result<ArrayRef> {/// ```fn general_repeat<O: OffsetSizeTrait>(array: &ArrayRef,- count_array: &UInt64Array,+ count_array: &Int64Array,) -> Result<ArrayRef> {- // Build offsets and take_indices- let total_repeated_values: usize =- count_array.values().iter().map(|&c| c as usize).sum();+ let total_repeated_values: usize = (0..count_array.len())+ .map(|i| get_count_with_validity(count_array, i))+ .sum();+let mut take_indices = Vec::with_capacity(total_repeated_values);let mut offsets = Vec::with_capacity(count_array.len() + 1);offsets.push(O::zero());let mut running_offset = 0usize;- for (idx, &count) in count_array.values().iter().enumerate() {- let count = count as usize;- running_offset += count;- offsets.push(O::from_usize(running_offset).unwrap());- take_indices.extend(std::iter::repeat_n(idx as u64, count))+ for idx in 0..count_array.len() {+ let count = get_count_with_validity(count_array, idx);+ running_offset = running_offset.checked_add(count).ok_or_else(|| {+ DataFusionError::Execution(+ "array_repeat: running_offset overflowed usize".to_string(),+ )+ })?;+ let offset = O::from_usize(running_offset).ok_or_else(|| {+ DataFusionError::Execution(format!(+ "array_repeat: offset {running_offset} exceeds the maximum value for offset type"+ ))+ })?;+ offsets.push(offset);+ take_indices.extend(std::iter::repeat_n(idx as u64, count));}// Build the flattened values@@ -222,7 +219,7 @@ fn general_repeat<O: OffsetSizeTrait>(Arc::new(Field::new_list_field(array.data_type().to_owned(), true)),OffsetBuffer::new(offsets.into()),repeated_values,- None,+ count_array.nulls().cloned(),)?))}@@ -238,23 +235,24 @@ fn general_repeat<O: OffsetSizeTrait>(/// ```fn general_list_repeat<O: OffsetSizeTrait>(list_array: &GenericListArray<O>,- count_array: &UInt64Array,+ count_array: &Int64Array,) -> Result<ArrayRef> {- let counts = count_array.values();let list_offsets = list_array.value_offsets();// calculate capacities for pre-allocation- let outer_total = counts.iter().map(|&c| c as usize).sum();- let inner_total = counts- .iter()- .enumerate()- .filter(|&(i, _)| !list_array.is_null(i))- .map(|(i, &c)| {- let len = list_offsets[i + 1].to_usize().unwrap()- - list_offsets[i].to_usize().unwrap();- len * (c as usize)- })- .sum();+ let mut outer_total = 0usize;+ let mut inner_total = 0usize;+ for i in 0..count_array.len() {+ let count = get_count_with_validity(count_array, i);+ if count > 0 {+ outer_total += count;+ if list_array.is_valid(i) {+ let len = list_offsets[i + 1].to_usize().unwrap()+ - list_offsets[i].to_usize().unwrap();+ inner_total += len * count;+ }+ }+ }// Build inner structureslet mut inner_offsets = Vec::with_capacity(outer_total + 1);@@ -263,17 +261,27 @@ fn general_list_repeat<O: OffsetSizeTrait>(let mut inner_running = 0usize;inner_offsets.push(O::zero());- for (row_idx, &count) in counts.iter().enumerate() {- let is_valid = !list_array.is_null(row_idx);+ for row_idx in 0..count_array.len() {+ let count = get_count_with_validity(count_array, row_idx);+ let list_is_valid = list_array.is_valid(row_idx);let start = list_offsets[row_idx].to_usize().unwrap();let end = list_offsets[row_idx + 1].to_usize().unwrap();let row_len = end - start;for _ in 0..count {- inner_running += row_len;- inner_offsets.push(O::from_usize(inner_running).unwrap());- inner_nulls.append(is_valid);- if is_valid {+ inner_running = inner_running.checked_add(row_len).ok_or_else(|| {+ DataFusionError::Execution(+ "array_repeat: inner offset overflowed usize".to_string(),+ )+ })?;+ let offset = O::from_usize(inner_running).ok_or_else(|| {+ DataFusionError::Execution(format!(+ "array_repeat: offset {inner_running} exceeds the maximum value for offset type"+ ))+ })?;+ inner_offsets.push(offset);+ inner_nulls.append(list_is_valid);+ if list_is_valid {take_indices.extend(start as u64..end as u64);}}@@ -298,8 +306,24 @@ fn general_list_repeat<O: OffsetSizeTrait>(list_array.data_type().to_owned(),true,)),- OffsetBuffer::<O>::from_lengths(counts.iter().map(|&c| c as usize)),+ OffsetBuffer::<O>::from_lengths(+ count_array+ .iter()+ .map(|c| c.map(|v| if v > 0 { v as usize } else { 0 }).unwrap_or(0)),+ ),Arc::new(inner_list),- None,+ count_array.nulls().cloned(),)?))}++/// Helper function to get count from count_array at given index+/// Return 0 for null values or non-positive count.+#[inline]+fn get_count_with_validity(count_array: &Int64Array, idx: usize) -> usize {+ if count_array.is_null(idx) {+ 0+ } else {+ let c = count_array.value(idx);+ if c > 0 { c as usize } else { 0 }+ }+}diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.sltindex c27433e7ef..2b98ae14d2 100644--- a/datafusion/sqllogictest/test_files/array.slt+++ b/datafusion/sqllogictest/test_files/array.slt@@ -3256,24 +3256,99 @@ drop table array_repeat_table;statement okdrop table large_array_repeat_table;-+# array_repeat: arrays with NULL countsstatement okcreate table array_repeat_null_count_tableas values(1, 2),(2, null),-(3, 1);+(3, 1),+(4, -1),+(null, null);query I?select column1, array_repeat(column1, column2) from array_repeat_null_count_table;----1 [1, 1]-2 []+2 NULL3 [3]+4 []+NULL NULLstatement okdrop table array_repeat_null_count_table+# array_repeat: nested arrays with NULL counts+statement ok+create table array_repeat_nested_null_count_table+as values+([[1, 2], [3, 4]], 2),+([[5, 6], [7, 8]], null),+([[null, null], [9, 10]], 1),+(null, 3),+([[11, 12]], -1);++query ??+select column1, array_repeat(column1, column2) from array_repeat_nested_null_count_table;+----+[[1, 2], [3, 4]] [[[1, 2], [3, 4]], [[1, 2], [3, 4]]]+[[5, 6], [7, 8]] NULL+[[NULL, NULL], [9, 10]] [[[NULL, NULL], [9, 10]]]+NULL [NULL, NULL, NULL]+[[11, 12]] []++statement ok+drop table array_repeat_nested_null_count_table++# array_repeat edge cases: empty arrays+query ???+select array_repeat([], 3), array_repeat([], 0), array_repeat([], null);+----+[[], [], []] [] NULL++query ??+select array_repeat(null::int, 0), array_repeat(null::int, null);+----+[] NULL++# array_repeat LargeList with NULL count+statement ok+create table array_repeat_large_list_null_table+as values+(arrow_cast([1, 2, 3], 'LargeList(Int64)'), 2),+(arrow_cast([4, 5], 'LargeList(Int64)'), null),+(arrow_cast(null, 'LargeList(Int64)'), 3);++query ??+select column1, array_repeat(column1, column2) from array_repeat_large_list_null_table;+----+[1, 2, 3] [[1, 2, 3], [1, 2, 3]]+[4, 5] NULL+NULL [NULL, NULL, NULL]++statement ok+drop table array_repeat_large_list_null_table++# array_repeat edge cases: LargeList nested with NULL count+statement ok+create table array_repeat_large_nested_null_table+as values+(arrow_cast([[1, 2], [3, 4]], 'LargeList(List(Int64))'), 2),+(arrow_cast([[5, 6], [7, 8]], 'LargeList(List(Int64))'), null),+(arrow_cast([[null, null]], 'LargeList(List(Int64))'), 1),+(null, 3);++query ??+select column1, array_repeat(column1, column2) from array_repeat_large_nested_null_table;+----+[[1, 2], [3, 4]] [[[1, 2], [3, 4]], [[1, 2], [3, 4]]]+[[5, 6], [7, 8]] NULL+[[NULL, NULL]] [[[NULL, NULL]]]+NULL [NULL, NULL, NULL]++statement ok+drop table array_repeat_large_nested_null_table+## array_concat (aliases: `array_cat`, `list_concat`, `list_cat`)# test with empty array