Skip to content

Apache DataFusion SQL Query Engine

fix: Avoid integer overflow in substr() (#20199)

Neil Conway0401a4732f90
diff --git a/datafusion/functions/src/unicode/substr.rs b/datafusion/functions/src/unicode/substr.rs
index 9517aac8e0..505388089f 100644
--- a/datafusion/functions/src/unicode/substr.rs
+++ b/datafusion/functions/src/unicode/substr.rs
@@ -185,7 +185,10 @@ pub fn get_true_start_end(
let start = start.checked_sub(1).unwrap_or(start);
let end = match count {
- Some(count) => start + count as i64,
+ Some(count) => {
+ let count_i64 = i64::try_from(count).unwrap_or(i64::MAX);
+ start.saturating_add(count_i64)
+ }
None => input.len() as i64,
};
let count_to_end = count.is_some();
@@ -247,7 +250,7 @@ pub fn enable_ascii_fast_path<'a, V: StringArrayType<'a>>(
// HACK: can be simplified if function has specialized
// implementation for `ScalarValue` (implement without `make_scalar_function()`)
- let avg_prefix_len = start
+ let total_prefix_len = start
.iter()
.zip(count.iter())
.take(n_sample)
@@ -255,11 +258,11 @@ pub fn enable_ascii_fast_path<'a, V: StringArrayType<'a>>(
let start = start.unwrap_or(0);
let count = count.unwrap_or(0);
// To get substring, need to decode from 0 to start+count instead of start to start+count
- start + count
+ start.saturating_add(count)
})
- .sum::<i64>();
+ .fold(0i64, |acc, val| acc.saturating_add(val));
- avg_prefix_len as f64 / n_sample as f64 <= short_prefix_threshold
+ (total_prefix_len as f64 / n_sample as f64) <= short_prefix_threshold
}
None => false,
};
@@ -810,7 +813,7 @@ mod tests {
SubstrFunc::new(),
vec![
ColumnarValue::Scalar(ScalarValue::from("abc")),
- ColumnarValue::Scalar(ScalarValue::from(-9223372036854775808i64)),
+ ColumnarValue::Scalar(ScalarValue::from(i64::MIN)),
],
Ok(Some("abc")),
&str,
@@ -821,7 +824,7 @@ mod tests {
SubstrFunc::new(),
vec![
ColumnarValue::Scalar(ScalarValue::from("overflow")),
- ColumnarValue::Scalar(ScalarValue::from(-9223372036854775808i64)),
+ ColumnarValue::Scalar(ScalarValue::from(i64::MIN)),
ColumnarValue::Scalar(ScalarValue::from(1i64)),
],
exec_err!("negative overflow when calculating skip value"),
@@ -829,6 +832,18 @@ mod tests {
Utf8View,
StringViewArray
);
+ test_function!(
+ SubstrFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::from("large count")),
+ ColumnarValue::Scalar(ScalarValue::from(2i64)),
+ ColumnarValue::Scalar(ScalarValue::from(i64::MAX)),
+ ],
+ Ok(Some("arge count")),
+ &str,
+ Utf8View,
+ StringViewArray
+ );
Ok(())
}