Skip to content

Apache DataFusion SQL Query Engine

fix: prevent date_trunc array underflow (#25131)

Ben Bellickb239d041b543
diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs
index 22c8af262a..0ed61a29b3 100644
--- a/datafusion/functions/src/datetime/date_trunc.rs
+++ b/datafusion/functions/src/datetime/date_trunc.rs
@@ -778,13 +778,30 @@ fn general_date_trunc_array_fine_granularity<T: ArrowTimestampType>(
if let Some(unit) = unit {
let unit = unit.get();
- let array = PrimitiveArray::<T>::from_iter_values_with_nulls(
- array
- .values()
- .iter()
- .map(|v| *v - i64::rem_euclid(*v, unit)),
- array.nulls().cloned(),
- )
+ // Truncation can only underflow within one `unit` of `i64::MIN`.
+ // Track that possibility while computing the common case so the loop
+ // remains infallible and can be vectorized.
+ let underflow_bound = i64::MIN + unit;
+ let mut maybe_underflow = false;
+ let values: Vec<i64> = array
+ .values()
+ .iter()
+ .map(|value| {
+ maybe_underflow |= *value < underflow_bound;
+ value.wrapping_sub(value.rem_euclid(unit))
+ })
+ .collect();
+ let array: PrimitiveArray<T> = if maybe_underflow {
+ array.try_unary(|value| {
+ value.checked_sub(value.rem_euclid(unit)).ok_or_else(|| {
+ exec_datafusion_err!(
+ "Timestamp {value} out of range after truncating to {granularity}"
+ )
+ })
+ })?
+ } else {
+ PrimitiveArray::new(values.into(), array.nulls().cloned())
+ }
.with_timezone_opt(tz_opt);
Ok(Arc::new(array))
} else {
@@ -856,12 +873,17 @@ mod tests {
use std::sync::Arc;
use crate::datetime::date_trunc::{
- DateTruncFunc, date_trunc_coarse, parse_granularity,
+ DateTruncFunc, NANOS_PER_MICROSECOND, date_trunc_coarse,
+ general_date_trunc_array_fine_granularity, parse_granularity,
};
use arrow::array::cast::as_primitive_array;
- use arrow::array::types::TimestampNanosecondType;
- use arrow::array::{Array, TimestampNanosecondArray};
+ use arrow::array::types::{ArrowTimestampType, TimestampNanosecondType};
+ use arrow::array::{
+ Array, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray,
+ TimestampNanosecondArray, TimestampSecondArray,
+ };
+ use arrow::buffer::NullBuffer;
use arrow::compute::DatePart;
use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos;
use arrow::datatypes::{DataType, Field, TimeUnit};
@@ -1339,6 +1361,101 @@ mod tests {
}
}
+ fn assert_fine_granularity_underflow<T: ArrowTimestampType>(
+ array: PrimitiveArray<T>,
+ granularity: DatePart,
+ ) {
+ let error =
+ general_date_trunc_array_fine_granularity(T::UNIT, &array, granularity, None)
+ .unwrap_err();
+ assert!(
+ error
+ .strip_backtrace()
+ .contains("Timestamp -9223372036854775808 out of range"),
+ "unexpected error: {error}"
+ );
+ }
+
+ #[test]
+ fn test_date_trunc_fine_granularity_underflow_for_each_timestamp_unit() {
+ assert_fine_granularity_underflow(
+ TimestampSecondArray::from(vec![i64::MIN]),
+ DatePart::Minute,
+ );
+ assert_fine_granularity_underflow(
+ TimestampMillisecondArray::from(vec![i64::MIN]),
+ DatePart::Second,
+ );
+ assert_fine_granularity_underflow(
+ TimestampMicrosecondArray::from(vec![i64::MIN]),
+ DatePart::Millisecond,
+ );
+ assert_fine_granularity_underflow(
+ TimestampNanosecondArray::from(vec![i64::MIN]),
+ DatePart::Microsecond,
+ );
+ }
+
+ #[test]
+ fn test_date_trunc_fine_granularity_minimum_safe_input() {
+ // The boundary below i64::MIN is unrepresentable. Inputs become safe
+ // at the next microsecond boundary.
+ let distance_to_next_boundary =
+ NANOS_PER_MICROSECOND - i64::MIN.rem_euclid(NANOS_PER_MICROSECOND);
+ let minimum_safe_input = i64::MIN + distance_to_next_boundary;
+ let unsafe_input = TimestampNanosecondArray::from(vec![minimum_safe_input - 1]);
+ assert!(
+ general_date_trunc_array_fine_granularity(
+ TimeUnit::Nanosecond,
+ &unsafe_input,
+ DatePart::Microsecond,
+ None,
+ )
+ .is_err()
+ );
+
+ let input = TimestampNanosecondArray::from(vec![
+ Some(minimum_safe_input),
+ Some(minimum_safe_input + 1),
+ None,
+ ]);
+ let result = general_date_trunc_array_fine_granularity(
+ TimeUnit::Nanosecond,
+ &input,
+ DatePart::Microsecond,
+ None,
+ )
+ .unwrap();
+ let result = as_primitive_array::<TimestampNanosecondType>(&result);
+ let expected = TimestampNanosecondArray::from(vec![
+ Some(minimum_safe_input),
+ Some(minimum_safe_input),
+ None,
+ ]);
+ assert_eq!(result, &expected);
+ }
+
+ #[test]
+ fn test_date_trunc_fine_granularity_ignores_null_slot_values() {
+ // Physical values behind null slots are arbitrary. An unsafe value in
+ // a null slot may trigger the checked fallback, but must not cause an
+ // error or alter the null bitmap.
+ let input = TimestampNanosecondArray::new(
+ vec![i64::MIN, 1].into(),
+ Some(NullBuffer::from(vec![false, true])),
+ );
+ let result = general_date_trunc_array_fine_granularity(
+ TimeUnit::Nanosecond,
+ &input,
+ DatePart::Microsecond,
+ None,
+ )
+ .unwrap();
+ let result = as_primitive_array::<TimestampNanosecondType>(&result);
+ let expected = TimestampNanosecondArray::from(vec![None, Some(0)]);
+ assert_eq!(result, &expected);
+ }
+
#[test]
fn test_date_trunc_fine_granularity_timezones() {
let cases = [