Apache DataFusion SQL Query Engine
fix: return execution error instead of capacity overflow panic in array_resize (#23306)
Burak Şen8272b4aa2057
diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rsindex d11064bf7e..e4fd8421fe 100644--- a/datafusion/functions-nested/src/resize.rs+++ b/datafusion/functions-nested/src/resize.rs@@ -219,6 +219,14 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(}}+ if output_values_len > max_resize_values(&data_type)+ || O::from_usize(output_values_len).is_none()+ {+ return exec_err!(+ "array_resize: resulting array of {output_values_len} elements exceeds the maximum array size"+ );+ }+// The fast path is valid when at least one row grows and every row would// use the same fill value.let use_bulk_fill = max_extra > 0@@ -342,12 +350,26 @@ where)?))}+/// Largest element count whose eager value buffer stays within `isize::MAX`+/// bytes, so `array_resize` rejects oversized results instead of panicking.+/// Only primitive and `FixedSizeBinary` leaves are byte-exact.+fn max_resize_values(value_type: &DataType) -> usize {+ let element_width = match value_type {+ DataType::FixedSizeBinary(size) if *size > 0 => *size as usize,+ _ => value_type.primitive_width().unwrap_or(size_of::<u128>()),+ };++ (isize::MAX as usize) / element_width.max(1)+}+#[cfg(test)]mod tests {use super::array_resize_inner;- use arrow::array::{ArrayRef, AsArray, Int64Array, ListArray};- use arrow::buffer::{NullBuffer, ScalarBuffer};- use arrow::datatypes::Int32Type;+ use arrow::array::{+ ArrayRef, AsArray, FixedSizeBinaryArray, Int64Array, LargeListArray, ListArray,+ };+ use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};+ use arrow::datatypes::{DataType, Field, Int32Type, Int64Type};use datafusion_common::Result;use std::sync::Arc;@@ -373,4 +395,73 @@ mod tests {Ok(())}++ #[test]+ fn test_array_resize_large_size_errors_without_panicking() {+ let array: ArrayRef =+ Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(vec![+ Some(vec![Some(1)]),+ ]));+ let size: ArrayRef = Arc::new(Int64Array::from(vec![i64::MAX]));+ let fill: ArrayRef = Arc::new(Int64Array::from(vec![0]));++ let err = array_resize_inner(&[array, size, fill]).unwrap_err();+ assert!(+ err.to_string().contains("exceeds the maximum array size"),+ "unexpected error: {err}"+ );+ }++ #[test]+ fn test_array_resize_fixed_size_binary_large_size_errors_without_panicking() {+ let values =+ FixedSizeBinaryArray::try_from_iter(vec![vec![0u8; 32]].into_iter()).unwrap();+ let elem_field =+ Arc::new(Field::new_list_field(DataType::FixedSizeBinary(32), true));+ let offsets = OffsetBuffer::<i64>::new(vec![0i64, 1].into());+ let array: ArrayRef = Arc::new(LargeListArray::new(+ elem_field,+ offsets,+ Arc::new(values) as ArrayRef,+ None,+ ));+ // Passes the width-16 bound (isize::MAX / 16) but overflows at width 32.+ let size: ArrayRef = Arc::new(Int64Array::from(vec![400_000_000_000_000_000i64]));++ let err = array_resize_inner(&[array, size]).unwrap_err();+ assert!(+ err.to_string().contains("exceeds the maximum array size"),+ "unexpected error: {err}"+ );+ }++ #[test]+ fn test_array_resize_accumulates_values_across_rows() {+ // Each row's target (6e17) is individually under the width-8 cap+ // (isize::MAX / 8), but their sum (1.2e18) exceeds it, so the guard+ // must reject based on the accumulated total rather than per row.+ let values = Int64Array::from(vec![1, 2]);+ let offsets = OffsetBuffer::<i64>::new(vec![0i64, 1, 2].into());+ let elem_field = Arc::new(Field::new_list_field(DataType::Int64, true));+ let array: ArrayRef = Arc::new(LargeListArray::new(+ elem_field,+ offsets,+ Arc::new(values) as ArrayRef,+ None,+ ));+ let size: ArrayRef = Arc::new(Int64Array::from(vec![+ 600_000_000_000_000_000i64,+ 600_000_000_000_000_000i64,+ ]));++ let err = array_resize_inner(&[array, size]).unwrap_err();+ assert!(+ err.to_string().contains("1200000000000000000"),+ "expected accumulated total in error: {err}"+ );+ assert!(+ err.to_string().contains("exceeds the maximum array size"),+ "unexpected error: {err}"+ );+ }}diff --git a/datafusion/sqllogictest/test_files/array/array_resize.slt b/datafusion/sqllogictest/test_files/array/array_resize.sltindex cb2ffa3a7e..37f8f2c693 100644--- a/datafusion/sqllogictest/test_files/array/array_resize.slt+++ b/datafusion/sqllogictest/test_files/array/array_resize.slt@@ -64,6 +64,23 @@ select array_resize(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 5, 4);query errorselect array_resize(make_array(1, 2, 3), -5, 2);+# array_resize with a very large size should error instead of panicking (capacity overflow)+query error DataFusion error: Execution error: array_resize: resulting array of 9223372036854775807 elements exceeds the maximum array size+select array_resize(make_array(1), 9223372036854775807, 0);++query error DataFusion error: Execution error: array_resize: resulting array of 9223372036854775807 elements exceeds the maximum array size+select array_resize(arrow_cast(make_array(1), 'LargeList(Int64)'), 9223372036854775807, 0);++# List size above i32::MAX must error via the offset-type guard instead of+# silently truncating the offsets or attempting a multi-GB allocation+query error DataFusion error: Execution error: array_resize: resulting array of 3000000000 elements exceeds the maximum array size+select array_resize(make_array(1), 3000000000, 0);++# Non-primitive element types (e.g. Utf8) fall back to a conservative byte+# width; a size above that cap must error instead of attempting a huge allocation.+query error DataFusion error: Execution error: array_resize: resulting array of 600000000000000000 elements exceeds the maximum array size+select array_resize(arrow_cast(make_array('a'), 'LargeList(Utf8)'), 600000000000000000);+# array_resize scalar function #5query ?select array_resize(make_array(1.1, 2.2, 3.3), 10, 9.9);