Apache DataFusion SQL Query Engine
Prevent overflow and panics when casting DATE to TIMESTAMP by validating bounds (#18761)
kosiew794660482bb2
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex 787bd78b1d..3fd5a37224 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -92,6 +92,103 @@ use chrono::{Duration, NaiveDate};use half::f16;pub use struct_builder::ScalarStructBuilder;+const SECONDS_PER_DAY: i64 = 86_400;+const MILLIS_PER_DAY: i64 = SECONDS_PER_DAY * 1_000;+const MICROS_PER_DAY: i64 = MILLIS_PER_DAY * 1_000;+const NANOS_PER_DAY: i64 = MICROS_PER_DAY * 1_000;+const MICROS_PER_MILLISECOND: i64 = 1_000;+const NANOS_PER_MILLISECOND: i64 = 1_000_000;++/// Returns the multiplier that converts the input date representation into the+/// desired timestamp unit, if the conversion requires a multiplication that can+/// overflow an `i64`.+pub fn date_to_timestamp_multiplier(+ source_type: &DataType,+ target_type: &DataType,+) -> Option<i64> {+ let DataType::Timestamp(target_unit, _) = target_type else {+ return None;+ };++ // Only `Timestamp` target types have a time unit; otherwise no+ // multiplier applies (handled above). The function returns `Some(m)`+ // when converting the `source_type` to `target_type` requires a+ // multiplication that could overflow `i64`. It returns `None` when+ // the conversion is a division or otherwise doesn't require a+ // multiplication (e.g. Date64 -> Second).+ match source_type {+ // Date32 stores days since epoch. Converting to any timestamp+ // unit requires multiplying by the per-day factor (seconds,+ // milliseconds, microseconds, nanoseconds).+ DataType::Date32 => Some(match target_unit {+ TimeUnit::Second => SECONDS_PER_DAY,+ TimeUnit::Millisecond => MILLIS_PER_DAY,+ TimeUnit::Microsecond => MICROS_PER_DAY,+ TimeUnit::Nanosecond => NANOS_PER_DAY,+ }),++ // Date64 stores milliseconds since epoch. Converting to+ // seconds is a division (no multiplication), so return `None`.+ // Converting to milliseconds is 1:1 (multiplier 1). Converting+ // to micro/nano requires multiplying by 1_000 / 1_000_000.+ DataType::Date64 => match target_unit {+ TimeUnit::Second => None,+ // Converting Date64 (ms since epoch) to millisecond timestamps+ // is an identity conversion and does not require multiplication.+ // Returning `None` indicates no multiplication-based overflow+ // check is necessary.+ TimeUnit::Millisecond => None,+ TimeUnit::Microsecond => Some(MICROS_PER_MILLISECOND),+ TimeUnit::Nanosecond => Some(NANOS_PER_MILLISECOND),+ },++ _ => None,+ }+}++/// Ensures the provided value can be represented as a timestamp with the given+/// multiplier. Returns an [`DataFusionError::Execution`] when the converted+/// value would overflow the timestamp range.+pub fn ensure_timestamp_in_bounds(+ value: i64,+ multiplier: i64,+ source_type: &DataType,+ target_type: &DataType,+) -> Result<()> {+ if multiplier <= 1 {+ return Ok(());+ }++ if value.checked_mul(multiplier).is_none() {+ let target = format_timestamp_type_for_error(target_type);+ _exec_err!(+ "Cannot cast {} value {} to {}: converted value exceeds the representable i64 range",+ source_type,+ value,+ target+ )+ } else {+ Ok(())+ }+}++/// Format a `DataType::Timestamp` into a short, stable string used in+/// user-facing error messages.+pub(crate) fn format_timestamp_type_for_error(target_type: &DataType) -> String {+ match target_type {+ DataType::Timestamp(unit, _) => {+ let s = match unit {+ TimeUnit::Second => "s",+ TimeUnit::Millisecond => "ms",+ TimeUnit::Microsecond => "us",+ TimeUnit::Nanosecond => "ns",+ };+ format!("Timestamp({s})")+ }+ other => format!("{other}"),+ }+}+/// A dynamically typed, nullable single value.////// While an arrow [`Array`]) stores one or more values of the same type, in a@@ -3619,11 +3716,27 @@ impl ScalarValue {target_type: &DataType,cast_options: &CastOptions<'static>,) -> Result<Self> {+ let source_type = self.data_type();+ if let Some(multiplier) = date_to_timestamp_multiplier(&source_type, target_type)+ {+ if let Some(value) = self.date_scalar_value_as_i64() {+ ensure_timestamp_in_bounds(value, multiplier, &source_type, target_type)?;+ }+ }+let scalar_array = self.to_array()?;let cast_arr = cast_with_options(&scalar_array, target_type, cast_options)?;ScalarValue::try_from_array(&cast_arr, 0)}+ fn date_scalar_value_as_i64(&self) -> Option<i64> {+ match self {+ ScalarValue::Date32(Some(value)) => Some(i64::from(*value)),+ ScalarValue::Date64(Some(value)) => Some(*value),+ _ => None,+ }+ }+fn eq_array_decimal32(array: &ArrayRef,index: usize,@@ -4991,7 +5104,7 @@ mod tests {use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};use arrow::compute::{is_null, kernels};use arrow::datatypes::{- ArrowNumericType, Fields, Float64Type, DECIMAL256_MAX_PRECISION,+ ArrowNumericType, Fields, Float64Type, TimeUnit, DECIMAL256_MAX_PRECISION,};use arrow::error::ArrowError;use arrow::util::pretty::pretty_format_columns;@@ -5024,6 +5137,52 @@ mod tests {assert_eq!(actual, &expected);}+ #[test]+ fn test_format_timestamp_type_for_error_and_bounds() {+ // format helper+ let ts_ns = format_timestamp_type_for_error(&DataType::Timestamp(+ TimeUnit::Nanosecond,+ None,+ ));+ assert_eq!(ts_ns, "Timestamp(ns)");++ let ts_us = format_timestamp_type_for_error(&DataType::Timestamp(+ TimeUnit::Microsecond,+ None,+ ));+ assert_eq!(ts_us, "Timestamp(us)");++ // ensure_timestamp_in_bounds: Date32 non-overflow+ let ok = ensure_timestamp_in_bounds(+ 1000,+ NANOS_PER_DAY,+ &DataType::Date32,+ &DataType::Timestamp(TimeUnit::Nanosecond, None),+ );+ assert!(ok.is_ok());++ // Date32 overflow -- known large day value (9999-12-31 -> 2932896)+ let err = ensure_timestamp_in_bounds(+ 2932896,+ NANOS_PER_DAY,+ &DataType::Date32,+ &DataType::Timestamp(TimeUnit::Nanosecond, None),+ );+ assert!(err.is_err());+ let msg = err.unwrap_err().to_string();+ assert!(msg.contains("Cannot cast Date32 value 2932896 to Timestamp(ns): converted value exceeds the representable i64 range"));++ // Date64 overflow for ns (millis * 1_000_000)+ let overflow_millis: i64 = (i64::MAX / NANOS_PER_MILLISECOND) + 1;+ let err2 = ensure_timestamp_in_bounds(+ overflow_millis,+ NANOS_PER_MILLISECOND,+ &DataType::Date64,+ &DataType::Timestamp(TimeUnit::Nanosecond, None),+ );+ assert!(err2.is_err());+ }+#[test]fn test_scalar_value_from_for_struct() {let boolean = Arc::new(BooleanArray::from(vec![false]));@@ -8605,6 +8764,19 @@ mod tests {assert!(dense_scalar.is_null());}+ #[test]+ fn cast_date_to_timestamp_overflow_returns_error() {+ let scalar = ScalarValue::Date32(Some(i32::MAX));+ let err = scalar+ .cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None))+ .expect_err("expected cast to fail");+ assert!(+ err.to_string()+ .contains("converted value exceeds the representable i64 range"),+ "unexpected error: {err}"+ );+ }+#[test]fn null_dictionary_scalar_produces_null_dictionary_array() {let dictionary_scalar = ScalarValue::Dictionary(diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rsindex 28f0dcb8bd..84899137e5 100644--- a/datafusion/core/tests/sql/select.rs+++ b/datafusion/core/tests/sql/select.rs@@ -414,3 +414,20 @@ async fn test_select_no_projection() -> Result<()> {");Ok(())}++#[tokio::test]+async fn test_select_cast_date_literal_to_timestamp_overflow() -> Result<()> {+ let ctx = SessionContext::new();+ let err = ctx+ .sql("SELECT CAST(DATE '9999-12-31' AS TIMESTAMP)")+ .await?+ .collect()+ .await+ .unwrap_err();++ assert_contains!(+ err.to_string(),+ "Cannot cast Date32 value 2932896 to Timestamp(ns): converted value exceeds the representable i64 range"+ );+ Ok(())+}diff --git a/datafusion/expr-common/src/columnar_value.rs b/datafusion/expr-common/src/columnar_value.rsindex d508164b6b..585b47a980 100644--- a/datafusion/expr-common/src/columnar_value.rs+++ b/datafusion/expr-common/src/columnar_value.rs@@ -17,12 +17,19 @@//! [`ColumnarValue`] represents the result of evaluating an expression.-use arrow::array::{Array, ArrayRef, NullArray};-use arrow::compute::{kernels, CastOptions};-use arrow::datatypes::DataType;-use arrow::util::pretty::pretty_format_columns;-use datafusion_common::format::DEFAULT_CAST_OPTIONS;-use datafusion_common::{internal_err, Result, ScalarValue};+use arrow::{+ array::{Array, ArrayRef, Date32Array, Date64Array, NullArray},+ compute::{kernels, max, min, CastOptions},+ datatypes::DataType,+ util::pretty::pretty_format_columns,+};+use datafusion_common::internal_datafusion_err;+use datafusion_common::{+ format::DEFAULT_CAST_OPTIONS,+ internal_err,+ scalar::{date_to_timestamp_multiplier, ensure_timestamp_in_bounds},+ Result, ScalarValue,+};use std::fmt;use std::sync::Arc;@@ -275,9 +282,14 @@ impl ColumnarValue {) -> Result<ColumnarValue> {let cast_options = cast_options.cloned().unwrap_or(DEFAULT_CAST_OPTIONS);match self {- ColumnarValue::Array(array) => Ok(ColumnarValue::Array(- kernels::cast::cast_with_options(array, cast_type, &cast_options)?,- )),+ ColumnarValue::Array(array) => {+ ensure_date_array_timestamp_bounds(array, cast_type)?;+ Ok(ColumnarValue::Array(kernels::cast::cast_with_options(+ array,+ cast_type,+ &cast_options,+ )?))+ }ColumnarValue::Scalar(scalar) => Ok(ColumnarValue::Scalar(scalar.cast_to_with_options(cast_type, &cast_options)?,)),@@ -285,6 +297,59 @@ impl ColumnarValue {}}+fn ensure_date_array_timestamp_bounds(+ array: &ArrayRef,+ cast_type: &DataType,+) -> Result<()> {+ let source_type = array.data_type().clone();+ let Some(multiplier) = date_to_timestamp_multiplier(&source_type, cast_type) else {+ return Ok(());+ };++ if multiplier <= 1 {+ return Ok(());+ }++ // Use compute kernels to find min/max instead of iterating all elements+ let (min_val, max_val): (Option<i64>, Option<i64>) = match &source_type {+ DataType::Date32 => {+ let arr = array+ .as_any()+ .downcast_ref::<Date32Array>()+ .ok_or_else(|| {+ internal_datafusion_err!(+ "Expected Date32Array but found {}",+ array.data_type()+ )+ })?;+ (min(arr).map(|v| v as i64), max(arr).map(|v| v as i64))+ }+ DataType::Date64 => {+ let arr = array+ .as_any()+ .downcast_ref::<Date64Array>()+ .ok_or_else(|| {+ internal_datafusion_err!(+ "Expected Date64Array but found {}",+ array.data_type()+ )+ })?;+ (min(arr), max(arr))+ }+ _ => return Ok(()), // Not a date type, nothing to do+ };++ // Only validate the min and max values instead of all elements+ if let Some(min) = min_val {+ ensure_timestamp_in_bounds(min, multiplier, &source_type, cast_type)?;+ }+ if let Some(max) = max_val {+ ensure_timestamp_in_bounds(max, multiplier, &source_type, cast_type)?;+ }++ Ok(())+}+// Implement Display trait for ColumnarValueimpl fmt::Display for ColumnarValue {fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {@@ -312,7 +377,10 @@ impl fmt::Display for ColumnarValue {#[cfg(test)]mod tests {use super::*;- use arrow::array::Int32Array;+ use arrow::{+ array::{Date64Array, Int32Array},+ datatypes::TimeUnit,+ };#[test]fn into_array_of_size() {@@ -484,4 +552,19 @@ mod tests {));}++ #[test]+ fn cast_date64_array_to_timestamp_overflow() {+ let overflow_value = i64::MAX / 1_000_000 + 1;+ let array: ArrayRef = Arc::new(Date64Array::from(vec![Some(overflow_value)]));+ let value = ColumnarValue::Array(array);+ let result =+ value.cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None), None);+ let err = result.expect_err("expected overflow to be detected");+ assert!(+ err.to_string()+ .contains("converted value exceeds the representable i64 range"),+ "unexpected error: {err}"+ );+ }}