Apache DataFusion SQL Query Engine
fix: Avoid integer overflow in substr() (#20199)
Neil Conway0401a4732f90
diff --git a/datafusion/functions/src/unicode/substr.rs b/datafusion/functions/src/unicode/substr.rsindex 9517aac8e0..505388089f 100644--- a/datafusion/functions/src/unicode/substr.rs+++ b/datafusion/functions/src/unicode/substr.rs@@ -185,7 +185,10 @@ pub fn get_true_start_end(let start = start.checked_sub(1).unwrap_or(start);let end = match count {- Some(count) => start + count as i64,+ Some(count) => {+ let count_i64 = i64::try_from(count).unwrap_or(i64::MAX);+ start.saturating_add(count_i64)+ }None => input.len() as i64,};let count_to_end = count.is_some();@@ -247,7 +250,7 @@ pub fn enable_ascii_fast_path<'a, V: StringArrayType<'a>>(// HACK: can be simplified if function has specialized// implementation for `ScalarValue` (implement without `make_scalar_function()`)- let avg_prefix_len = start+ let total_prefix_len = start.iter().zip(count.iter()).take(n_sample)@@ -255,11 +258,11 @@ pub fn enable_ascii_fast_path<'a, V: StringArrayType<'a>>(let start = start.unwrap_or(0);let count = count.unwrap_or(0);// To get substring, need to decode from 0 to start+count instead of start to start+count- start + count+ start.saturating_add(count)})- .sum::<i64>();+ .fold(0i64, |acc, val| acc.saturating_add(val));- avg_prefix_len as f64 / n_sample as f64 <= short_prefix_threshold+ (total_prefix_len as f64 / n_sample as f64) <= short_prefix_threshold}None => false,};@@ -810,7 +813,7 @@ mod tests {SubstrFunc::new(),vec![ColumnarValue::Scalar(ScalarValue::from("abc")),- ColumnarValue::Scalar(ScalarValue::from(-9223372036854775808i64)),+ ColumnarValue::Scalar(ScalarValue::from(i64::MIN)),],Ok(Some("abc")),&str,@@ -821,7 +824,7 @@ mod tests {SubstrFunc::new(),vec![ColumnarValue::Scalar(ScalarValue::from("overflow")),- ColumnarValue::Scalar(ScalarValue::from(-9223372036854775808i64)),+ ColumnarValue::Scalar(ScalarValue::from(i64::MIN)),ColumnarValue::Scalar(ScalarValue::from(1i64)),],exec_err!("negative overflow when calculating skip value"),@@ -829,6 +832,18 @@ mod tests {Utf8View,StringViewArray);+ test_function!(+ SubstrFunc::new(),+ vec![+ ColumnarValue::Scalar(ScalarValue::from("large count")),+ ColumnarValue::Scalar(ScalarValue::from(2i64)),+ ColumnarValue::Scalar(ScalarValue::from(i64::MAX)),+ ],+ Ok(Some("arge count")),+ &str,+ Utf8View,+ StringViewArray+ );Ok(())}