Apache DataFusion SQL Query Engine
fix: preserve timestamp precision when coercing mixed time units (#22759)
fys51673d07f894
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex c2f2c0e00e..c9013af726 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -148,6 +148,30 @@ pub fn date_to_timestamp_multiplier(}}+/// Returns the multiplier that converts the input timestamp representation into+/// the desired timestamp unit, if the conversion requires a multiplication that+/// can overflow an `i64`.+pub fn timestamp_to_timestamp_multiplier(+ source_type: &DataType,+ target_type: &DataType,+) -> Option<i64> {+ let (DataType::Timestamp(source_unit, _), DataType::Timestamp(target_unit, _)) =+ (source_type, target_type)+ else {+ return None;+ };++ match (source_unit, target_unit) {+ (TimeUnit::Second, TimeUnit::Millisecond) => Some(1_000),+ (TimeUnit::Second, TimeUnit::Microsecond) => Some(1_000_000),+ (TimeUnit::Second, TimeUnit::Nanosecond) => Some(1_000_000_000),+ (TimeUnit::Millisecond, TimeUnit::Microsecond) => Some(1_000),+ (TimeUnit::Millisecond, TimeUnit::Nanosecond) => Some(1_000_000),+ (TimeUnit::Microsecond, TimeUnit::Nanosecond) => Some(1_000),+ _ => None,+ }+}+/// Ensures the provided value can be represented as a timestamp with the given/// multiplier. Returns an [`DataFusionError::Execution`] when the converted/// value would overflow the timestamp range.@@ -4265,7 +4289,8 @@ impl ScalarValue {}if let Some(multiplier) = date_to_timestamp_multiplier(&source_type, target_type)- && let Some(value) = self.date_scalar_value_as_i64()+ .or_else(|| timestamp_to_timestamp_multiplier(&source_type, target_type))+ && let Some(value) = self.temporal_scalar_value_as_i64(){ensure_timestamp_in_bounds(value, multiplier, &source_type, target_type)?;}@@ -4287,10 +4312,14 @@ impl ScalarValue {ScalarValue::try_from_array(&cast_arr, 0)}- fn date_scalar_value_as_i64(&self) -> Option<i64> {+ fn temporal_scalar_value_as_i64(&self) -> Option<i64> {match self {ScalarValue::Date32(Some(value)) => Some(i64::from(*value)),ScalarValue::Date64(Some(value)) => Some(*value),+ ScalarValue::TimestampSecond(Some(value), _)+ | ScalarValue::TimestampMillisecond(Some(value), _)+ | ScalarValue::TimestampMicrosecond(Some(value), _)+ | ScalarValue::TimestampNanosecond(Some(value), _) => Some(*value),_ => None,}}@@ -10161,6 +10190,19 @@ mod tests {);}+ #[test]+ fn cast_timestamp_to_timestamp_overflow_returns_error() {+ let scalar = ScalarValue::TimestampSecond(Some(i64::MAX), None);+ let err = scalar+ .cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None))+ .expect_err("expected cast to fail");+ assert!(+ err.to_string()+ .contains("converted value exceeds the representable i64 range"),+ "unexpected error: {err}"+ );+ }+#[test]fn null_dictionary_scalar_produces_null_dictionary_array() {let dictionary_scalar = ScalarValue::Dictionary(diff --git a/datafusion/expr-common/src/columnar_value.rs b/datafusion/expr-common/src/columnar_value.rsindex bc6b8177ab..caeb3f10da 100644--- a/datafusion/expr-common/src/columnar_value.rs+++ b/datafusion/expr-common/src/columnar_value.rs@@ -18,9 +18,12 @@//! [`ColumnarValue`] represents the result of evaluating an expression.use arrow::{- array::{Array, ArrayRef, Date32Array, Date64Array, NullArray},+ array::{+ Array, ArrayRef, Date32Array, Date64Array, NullArray, TimestampMicrosecondArray,+ TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,+ },compute::{CastOptions, kernels, max, min},- datatypes::DataType,+ datatypes::{DataType, TimeUnit},util::pretty::pretty_format_columns,};use datafusion_common::internal_datafusion_err;@@ -28,7 +31,10 @@ use datafusion_common::{Result, ScalarValue,format::DEFAULT_CAST_OPTIONS,internal_err,- scalar::{date_to_timestamp_multiplier, ensure_timestamp_in_bounds},+ scalar::{+ date_to_timestamp_multiplier, ensure_timestamp_in_bounds,+ timestamp_to_timestamp_multiplier,+ },};use std::fmt;use std::sync::Arc;@@ -319,7 +325,7 @@ fn cast_array_by_name() {datafusion_common::nested_struct::cast_column(array, cast_type, cast_options)} else {- ensure_date_array_timestamp_bounds(array, cast_type)?;+ ensure_temporal_array_timestamp_bounds(array, cast_type)?;Ok(kernels::cast::cast_with_options(array,cast_type,@@ -328,12 +334,14 @@ fn cast_array_by_name(}}-fn ensure_date_array_timestamp_bounds(+fn ensure_temporal_array_timestamp_bounds(array: &ArrayRef,cast_type: &DataType,) -> Result<()> {let source_type = array.data_type().clone();- let Some(multiplier) = date_to_timestamp_multiplier(&source_type, cast_type) else {+ let Some(multiplier) = date_to_timestamp_multiplier(&source_type, cast_type)+ .or_else(|| timestamp_to_timestamp_multiplier(&source_type, cast_type))+ else {return Ok(());};@@ -367,7 +375,55 @@ fn ensure_date_array_timestamp_bounds(})?;(min(arr), max(arr))}- _ => return Ok(()), // Not a date type, nothing to do+ DataType::Timestamp(TimeUnit::Second, _) => {+ let arr = array+ .as_any()+ .downcast_ref::<TimestampSecondArray>()+ .ok_or_else(|| {+ internal_datafusion_err!(+ "Expected TimestampSecondArray but found {}",+ array.data_type()+ )+ })?;+ (min(arr), max(arr))+ }+ DataType::Timestamp(TimeUnit::Millisecond, _) => {+ let arr = array+ .as_any()+ .downcast_ref::<TimestampMillisecondArray>()+ .ok_or_else(|| {+ internal_datafusion_err!(+ "Expected TimestampMillisecondArray but found {}",+ array.data_type()+ )+ })?;+ (min(arr), max(arr))+ }+ DataType::Timestamp(TimeUnit::Microsecond, _) => {+ let arr = array+ .as_any()+ .downcast_ref::<TimestampMicrosecondArray>()+ .ok_or_else(|| {+ internal_datafusion_err!(+ "Expected TimestampMicrosecondArray but found {}",+ array.data_type()+ )+ })?;+ (min(arr), max(arr))+ }+ DataType::Timestamp(TimeUnit::Nanosecond, _) => {+ let arr = array+ .as_any()+ .downcast_ref::<TimestampNanosecondArray>()+ .ok_or_else(|| {+ internal_datafusion_err!(+ "Expected TimestampNanosecondArray but found {}",+ array.data_type()+ )+ })?;+ (min(arr), max(arr))+ }+ _ => return Ok(()), // Not a temporal type that needs checking.};// Only validate the min and max values instead of all elements@@ -694,4 +750,20 @@ mod tests {"unexpected error: {err}");}++ #[test]+ fn cast_timestamp_array_to_timestamp_overflow() {+ let overflow_value = i64::MAX / 1_000_000_000 + 1;+ let array: ArrayRef =+ Arc::new(TimestampSecondArray::from(vec![Some(overflow_value)]));+ let value = ColumnarValue::Array(array);+ let result =+ value.cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None), None);+ let err = result.expect_err("expected overflow to be detected");+ assert!(+ err.to_string()+ .contains("converted value exceeds the representable i64 range"),+ "unexpected error: {err}"+ );+ }}diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rsindex aec87ec5ff..4581745ccb 100644--- a/datafusion/expr-common/src/type_coercion/binary.rs+++ b/datafusion/expr-common/src/type_coercion/binary.rs@@ -2048,22 +2048,10 @@ fn temporal_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataTypfn timeunit_coercion(lhs_unit: &TimeUnit, rhs_unit: &TimeUnit) -> TimeUnit {use arrow::datatypes::TimeUnit::*;match (lhs_unit, rhs_unit) {- (Second, Millisecond) => Second,- (Second, Microsecond) => Second,- (Second, Nanosecond) => Second,- (Millisecond, Second) => Second,- (Millisecond, Microsecond) => Millisecond,- (Millisecond, Nanosecond) => Millisecond,- (Microsecond, Second) => Second,- (Microsecond, Millisecond) => Millisecond,- (Microsecond, Nanosecond) => Microsecond,- (Nanosecond, Second) => Second,- (Nanosecond, Millisecond) => Millisecond,- (Nanosecond, Microsecond) => Microsecond,- (l, r) => {- assert_eq!(l, r);- *l- }+ (Second, Second) => Second,+ (Nanosecond, _) | (_, Nanosecond) => Nanosecond,+ (Microsecond, _) | (_, Microsecond) => Microsecond,+ (Millisecond, _) | (_, Millisecond) => Millisecond,}}diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rs b/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rsindex eb5622fedb..70a8fc0e35 100644--- a/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rs+++ b/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rs@@ -40,8 +40,8 @@ fn test_date_timestamp_arithmetic_error() -> Result<()> {&DataType::Timestamp(Millisecond, None),).get_input_types()?;- assert_eq!(lhs, DataType::Timestamp(Millisecond, None));- assert_eq!(rhs, DataType::Timestamp(Millisecond, None));+ assert_eq!(lhs, DataType::Timestamp(Nanosecond, None));+ assert_eq!(rhs, DataType::Timestamp(Nanosecond, None));let err =BinaryTypeCoercer::new(&DataType::Date32, &Operator::Plus, &DataType::Date64)diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rsindex f8bff3ca90..5f6b7dfcc1 100644--- a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs+++ b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs@@ -575,6 +575,24 @@ fn test_type_coercion_compare() -> Result<()> {Operator::Eq,DataType::Timestamp(Second, Some("Europe/Brussels".into())));+ test_coercion_binary_rule!(+ DataType::Timestamp(Second, None),+ DataType::Timestamp(Millisecond, None),+ Operator::Eq,+ DataType::Timestamp(Millisecond, None)+ );+ test_coercion_binary_rule!(+ DataType::Timestamp(Second, Some("America/New_York".into())),+ DataType::Timestamp(Nanosecond, Some("Europe/Brussels".into())),+ Operator::Lt,+ DataType::Timestamp(Nanosecond, Some("America/New_York".into()))+ );+ test_coercion_binary_rule!(+ DataType::Timestamp(Microsecond, None),+ DataType::Timestamp(Nanosecond, None),+ Operator::GtEq,+ DataType::Timestamp(Nanosecond, None)+ );// listlet inner_field = Arc::new(Field::new_list_field(DataType::Int64, true));@@ -872,6 +890,24 @@ fn test_type_union_coercion_prefers_string() {);}+#[test]+fn test_type_union_coercion_prefers_finer_timestamp_unit() {+ assert_eq!(+ type_union_coercion(+ &DataType::Timestamp(Second, None),+ &DataType::Timestamp(Millisecond, None),+ ),+ Some(DataType::Timestamp(Millisecond, None))+ );+ assert_eq!(+ type_union_resolution(&[+ DataType::Timestamp(Second, None),+ DataType::Timestamp(Nanosecond, None),+ ]),+ Some(DataType::Timestamp(Nanosecond, None))+ );+}+/// Tests that comparison operators coerce to numeric when comparing/// numeric and string types.#[test]diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex 958ff86b4f..89c6f0a121 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -2480,6 +2480,51 @@ SELECT TIMESTAMPTZ '2020-01-01 00:00:00Z' = TIMESTAMP '2020-01-01'----true+query BBB+SELECT+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00', 'Timestamp(Second, None)') =+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00.123', 'Timestamp(Millisecond, None)'),+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00', 'Timestamp(Second, None)') =+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00.000', 'Timestamp(Millisecond, None)'),+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00', 'Timestamp(Second, None)') <+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00.123', 'Timestamp(Millisecond, None)')+----+false true true++query ?+SELECT+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00.123', 'Timestamp(Millisecond, None)') -+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00', 'Timestamp(Second, None)')+----+0 days 0 hours 0 mins 0.123 secs++query TP+SELECT arrow_typeof(ts), ts+FROM (+ SELECT arrow_cast(TIMESTAMP '2024-01-01 00:00:00', 'Timestamp(Second, None)') AS ts+ UNION ALL+ SELECT arrow_cast(TIMESTAMP '2024-01-01 00:00:00.123', 'Timestamp(Millisecond, None)') AS ts+)+ORDER BY ts+----+Timestamp(ms) 2024-01-01T00:00:00+Timestamp(ms) 2024-01-01T00:00:00.123++query TP+SELECT+ arrow_typeof(+ coalesce(+ arrow_cast(NULL, 'Timestamp(Second, None)'),+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00.123', 'Timestamp(Millisecond, None)')+ )+ ),+ coalesce(+ arrow_cast(NULL, 'Timestamp(Second, None)'),+ arrow_cast(TIMESTAMP '2024-01-01 00:00:00.123', 'Timestamp(Millisecond, None)')+ )+----+Timestamp(ms) 2024-01-01T00:00:00.123+# verify timestamp cast with integer inputquery PPPPPPSELECT to_timestamp(null), to_timestamp(0), to_timestamp(1926632005), to_timestamp(1), to_timestamp(-1), to_timestamp(0-1)@@ -3959,17 +4004,17 @@ truequery ?select arrow_cast('2024-06-17T11:00:00', 'Timestamp(Nanosecond, Some("UTC"))') - arrow_cast('2024-06-17T12:00:00', 'Timestamp(Microsecond, Some("UTC"))');-----0 days -1 hours 0 mins 0.000000 secs+0 days -1 hours 0 mins 0.000000000 secsquery ?select arrow_cast('2024-06-17T13:00:00', 'Timestamp(Nanosecond, Some("+00:00"))') - arrow_cast('2024-06-17T12:00:00', 'Timestamp(Microsecond, Some("UTC"))');-----0 days 1 hours 0 mins 0.000000 secs+0 days 1 hours 0 mins 0.000000000 secsquery ?select arrow_cast('2024-06-17T13:00:00', 'Timestamp(Nanosecond, Some("UTC"))') - arrow_cast('2024-06-17T12:00:00', 'Timestamp(Microsecond, Some("+00:00"))');-----0 days 1 hours 0 mins 0.000000 secs+0 days 1 hours 0 mins 0.000000000 secs# not supported: coercion across timezonesquery error@@ -5331,7 +5376,7 @@ SELECT to_timestamp(arrow_cast(-9223372036, 'Int64'));1677-09-21T00:12:44# Overflow error when value exceeds valid range-query error Arithmetic overflow+query error converted value exceeds the representable i64 rangeSELECT to_timestamp(arrow_cast(9223372037, 'Int64'));# Float truncation behavior