Skip to content

Apache DataFusion SQL Query Engine

fix: correct date_trunc for times before the epoch (#18356)

Martin Hilton52894dbc7b52
diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs
index 405aabfde9..543ed8038b 100644
--- a/datafusion/functions/src/datetime/date_trunc.rs
+++ b/datafusion/functions/src/datetime/date_trunc.rs
@@ -16,6 +16,7 @@
// under the License.
use std::any::Any;
+use std::num::NonZeroI64;
use std::ops::{Add, Sub};
use std::str::FromStr;
use std::sync::Arc;
@@ -28,7 +29,7 @@ use arrow::array::types::{
ArrowTimestampType, TimestampMicrosecondType, TimestampMillisecondType,
TimestampNanosecondType, TimestampSecondType,
};
-use arrow::array::{Array, ArrayRef, Int64Array, PrimitiveArray};
+use arrow::array::{Array, ArrayRef, PrimitiveArray};
use arrow::datatypes::DataType::{self, Null, Timestamp, Utf8, Utf8View};
use arrow::datatypes::TimeUnit::{self, Microsecond, Millisecond, Nanosecond, Second};
use datafusion_common::cast::as_primitive_array;
@@ -456,37 +457,40 @@ fn general_date_trunc_array_fine_granularity<T: ArrowTimestampType>(
granularity: &str,
) -> Result<ArrayRef> {
let unit = match (tu, granularity) {
- (Second, "minute") => Some(Int64Array::new_scalar(60)),
- (Second, "hour") => Some(Int64Array::new_scalar(3600)),
- (Second, "day") => Some(Int64Array::new_scalar(86400)),
-
- (Millisecond, "second") => Some(Int64Array::new_scalar(1_000)),
- (Millisecond, "minute") => Some(Int64Array::new_scalar(60_000)),
- (Millisecond, "hour") => Some(Int64Array::new_scalar(3_600_000)),
- (Millisecond, "day") => Some(Int64Array::new_scalar(86_400_000)),
-
- (Microsecond, "millisecond") => Some(Int64Array::new_scalar(1_000)),
- (Microsecond, "second") => Some(Int64Array::new_scalar(1_000_000)),
- (Microsecond, "minute") => Some(Int64Array::new_scalar(60_000_000)),
- (Microsecond, "hour") => Some(Int64Array::new_scalar(3_600_000_000)),
- (Microsecond, "day") => Some(Int64Array::new_scalar(86_400_000_000)),
-
- (Nanosecond, "microsecond") => Some(Int64Array::new_scalar(1_000)),
- (Nanosecond, "millisecond") => Some(Int64Array::new_scalar(1_000_000)),
- (Nanosecond, "second") => Some(Int64Array::new_scalar(1_000_000_000)),
- (Nanosecond, "minute") => Some(Int64Array::new_scalar(60_000_000_000)),
- (Nanosecond, "hour") => Some(Int64Array::new_scalar(3_600_000_000_000)),
- (Nanosecond, "day") => Some(Int64Array::new_scalar(86_400_000_000_000)),
+ (Second, "minute") => NonZeroI64::new(60),
+ (Second, "hour") => NonZeroI64::new(3600),
+ (Second, "day") => NonZeroI64::new(86400),
+
+ (Millisecond, "second") => NonZeroI64::new(1_000),
+ (Millisecond, "minute") => NonZeroI64::new(60_000),
+ (Millisecond, "hour") => NonZeroI64::new(3_600_000),
+ (Millisecond, "day") => NonZeroI64::new(86_400_000),
+
+ (Microsecond, "millisecond") => NonZeroI64::new(1_000),
+ (Microsecond, "second") => NonZeroI64::new(1_000_000),
+ (Microsecond, "minute") => NonZeroI64::new(60_000_000),
+ (Microsecond, "hour") => NonZeroI64::new(3_600_000_000),
+ (Microsecond, "day") => NonZeroI64::new(86_400_000_000),
+
+ (Nanosecond, "microsecond") => NonZeroI64::new(1_000),
+ (Nanosecond, "millisecond") => NonZeroI64::new(1_000_000),
+ (Nanosecond, "second") => NonZeroI64::new(1_000_000_000),
+ (Nanosecond, "minute") => NonZeroI64::new(60_000_000_000),
+ (Nanosecond, "hour") => NonZeroI64::new(3_600_000_000_000),
+ (Nanosecond, "day") => NonZeroI64::new(86_400_000_000_000),
_ => None,
};
if let Some(unit) = unit {
- let original_type = array.data_type();
- let array = arrow::compute::cast(array, &DataType::Int64)?;
- let array = arrow::compute::kernels::numeric::div(&array, &unit)?;
- let array = arrow::compute::kernels::numeric::mul(&array, &unit)?;
- let array = arrow::compute::cast(&array, original_type)?;
- Ok(array)
+ let unit = unit.get();
+ let array = PrimitiveArray::<T>::from_iter_values_with_nulls(
+ array
+ .values()
+ .iter()
+ .map(|v| *v - i64::rem_euclid(*v, unit)),
+ array.nulls().cloned(),
+ );
+ Ok(Arc::new(array))
} else {
// truncate to the same or smaller unit
Ok(Arc::new(array.clone()))
diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt
index 84dd7098a2..250d4e9830 100644
--- a/datafusion/sqllogictest/test_files/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/timestamps.slt
@@ -1687,6 +1687,30 @@ SELECT DATE_TRUNC('second', '2022-08-03 14:38:50Z');
----
2022-08-03T14:38:50
+# DATE_TRUNC handling of times before the unix epoch (issue 18334)
+query PPPPPPPPPPP
+SELECT
+ d,
+ DATE_TRUNC('year', d),
+ DATE_TRUNC('quarter', d),
+ DATE_TRUNC('month', d),
+ DATE_TRUNC('week', d),
+ DATE_TRUNC('day', d),
+ DATE_TRUNC('hour', d),
+ DATE_TRUNC('minute', d),
+ DATE_TRUNC('second', d),
+ DATE_TRUNC('millisecond', d),
+ DATE_TRUNC('microsecond', d),
+FROM (VALUES
+ (TIMESTAMP '1900-06-15 07:09:00'),
+ (TIMESTAMP '1970-01-01 00:00:00'),
+ (TIMESTAMP '2024-12-31 23:39:01.123456789')
+) AS t(d);
+----
+1900-06-15T07:09:00 1900-01-01T00:00:00 1900-04-01T00:00:00 1900-06-01T00:00:00 1900-06-11T00:00:00 1900-06-15T00:00:00 1900-06-15T07:00:00 1900-06-15T07:09:00 1900-06-15T07:09:00 1900-06-15T07:09:00 1900-06-15T07:09:00
+1970-01-01T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00 1969-12-29T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00
+2024-12-31T23:39:01.123456789 2024-01-01T00:00:00 2024-10-01T00:00:00 2024-12-01T00:00:00 2024-12-30T00:00:00 2024-12-31T00:00:00 2024-12-31T23:00:00 2024-12-31T23:39:00 2024-12-31T23:39:01 2024-12-31T23:39:01.123 2024-12-31T23:39:01.123456
+
# Test that interval can add a timestamp
query P
SELECT timestamp '2013-07-01 12:00:00' + INTERVAL '8' DAY;