Apache DataFusion SQL Query Engine
fix: guard repeat array length overflow (#22293)
Sean Kenneth Doherty32a1fe5498c0
diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rsindex b551d2ac70..a53f1e2e4f 100644--- a/datafusion/functions/src/string/repeat.rs+++ b/datafusion/functions/src/string/repeat.rs@@ -26,7 +26,9 @@ use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};use datafusion_common::cast::as_int64_array;use datafusion_common::types::{NativeType, logical_int64, logical_string};use datafusion_common::utils::take_function_args;-use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err, internal_err};+use datafusion_common::{+ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,+};use datafusion_expr::{ColumnarValue, Documentation, Volatility};use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};@@ -166,7 +168,21 @@ fn compute_repeat(s: &str, count: i64, max_size: usize) -> Result<String> {if count <= 0 {return Ok(String::new());}- let result_len = s.len().saturating_mul(count as usize);+ let result_len = repeat_len(s.len(), count, max_size)?;+ debug_assert!(result_len <= max_size);+ let count = repeat_count(count, max_size)?;+ Ok(s.repeat(count))+}++fn repeat_len(string_len: usize, count: i64, max_size: usize) -> Result<usize> {+ let count = repeat_count(count, max_size)?;+ let result_len = string_len.checked_mul(count).ok_or_else(|| {+ exec_datafusion_err!(+ "string size overflow on repeat, max size is {}, but got {}",+ max_size,+ usize::MAX+ )+ })?;if result_len > max_size {return exec_err!("string size overflow on repeat, max size is {}, but got {}",@@ -174,7 +190,18 @@ fn compute_repeat(s: &str, count: i64, max_size: usize) -> Result<String> {result_len);}- Ok(s.repeat(count as usize))+ Ok(result_len)+}++fn repeat_count(count: i64, max_size: usize) -> Result<usize> {+ match usize::try_from(count) {+ Ok(count) => Ok(count),+ Err(_) => exec_err!(+ "string size overflow on repeat, max size is {}, but got {}",+ max_size,+ usize::MAX+ ),+ }}/// Repeats string the specified number of times.@@ -227,22 +254,22 @@ fn calculate_capacities<'a, S>(whereS: StringArrayType<'a>,{- let mut total_capacity = 0;- let mut max_item_capacity = 0;+ let mut total_capacity = 0usize;+ let mut max_item_capacity = 0usize;string_array.iter().zip(number_array.iter()).try_for_each(|(string, number)| -> Result<(), DataFusionError> {match (string, number) {(Some(string), Some(number)) if number >= 0 => {- let item_capacity = string.len() * number as usize;- if item_capacity > max_str_len {- return exec_err!(- "string size overflow on repeat, max size is {}, but got {}",- max_str_len,- number as usize * string.len()- );- }- total_capacity += item_capacity;+ let item_capacity = repeat_len(string.len(), number, max_str_len)?;+ total_capacity =+ total_capacity.checked_add(item_capacity).ok_or_else(|| {+ exec_datafusion_err!(+ "string size overflow on repeat, max size is {}, but got {}",+ max_str_len,+ usize::MAX+ )+ })?;max_item_capacity = max_item_capacity.max(item_capacity);}_ => (),@@ -487,6 +514,18 @@ mod tests {assert_sliced_offset_output::<StringArray>(result);}+ #[test]+ fn test_repeat_string_array_overflow() {+ let strings: ArrayRef = Arc::new(StringArray::from(vec![Some("abc")]));+ let counts: ArrayRef = Arc::new(Int64Array::from(vec![Some(i64::MAX)]));++ let err = super::repeat(&strings, &counts).unwrap_err().to_string();+ assert!(+ err.contains("string size overflow on repeat"),+ "unexpected error: {err}"+ );+ }+#[test]fn test_repeat_sliced_large_string_with_null_offset() {let (strings, counts) =diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.sltindex 97f2a40c13..d7547bf145 100644--- a/datafusion/sqllogictest/test_files/string/string_literal.slt+++ b/datafusion/sqllogictest/test_files/string/string_literal.slt@@ -391,6 +391,10 @@ SELECT repeat(arrow_cast('foo', 'Dictionary(Int32, Utf8)'), 3)----foofoofoo+query error DataFusion error: Execution error: string size overflow on repeat, max size is 2147483647, but got \d++SELECT repeat(x, 9223372036854775807)+FROM (VALUES ('abc')) AS t(x);+query TSELECT arrow_typeof(repeat('foo', 3))----