Skip to content

Apache DataFusion SQL Query Engine

fix: keep fractional seconds in EXTRACT(EPOCH) for Duration (#26676)

Matt Butrovich702803d0220c
diff --git a/datafusion/functions/benches/datetime_expressions/date_part.rs b/datafusion/functions/benches/datetime_expressions/date_part.rs
index aa4b3e1dba..219f75f45e 100644
--- a/datafusion/functions/benches/datetime_expressions/date_part.rs
+++ b/datafusion/functions/benches/datetime_expressions/date_part.rs
@@ -39,6 +39,7 @@ const BATCH_SIZE: usize = 1000;
const TS_BOUND: i64 = 2_006_463_600;
const SEC_DAY: i64 = 86_400;
const DAYS_SINCE_EPOCH: i64 = TS_BOUND / SEC_DAY;
+const NULL_DENSITY: f64 = 0.1;
fn generate_timestamp_ns_array(rng: &mut StdRng) -> TimestampNanosecondArray {
TimestampNanosecondArray::from(
@@ -162,6 +163,31 @@ fn generate_duration_nanosecond_array(rng: &mut StdRng) -> DurationNanosecondArr
)
}
+// Durations under `i32::MAX` nanoseconds (about 2.1 seconds)
+fn generate_duration_nanosecond_small_array(rng: &mut StdRng) -> DurationNanosecondArray {
+ DurationNanosecondArray::from(
+ (0..BATCH_SIZE)
+ .map(|_| rng.random_range(0..i32::MAX as i64))
+ .collect::<Vec<_>>(),
+ )
+}
+
+fn generate_duration_nanosecond_array_with_nulls(
+ rng: &mut StdRng,
+) -> DurationNanosecondArray {
+ DurationNanosecondArray::from(
+ (0..BATCH_SIZE)
+ .map(|_| {
+ if rng.random_bool(NULL_DENSITY) {
+ None
+ } else {
+ Some(rng.random_range(0..TS_BOUND * 1_000_000_000))
+ }
+ })
+ .collect::<Vec<_>>(),
+ )
+}
+
fn bench_date_part(
c: &mut Criterion,
udf: &Arc<ScalarUDF>,
@@ -213,6 +239,10 @@ fn criterion_benchmark(c: &mut Criterion) {
let duration_ns = Arc::new(generate_duration_nanosecond_array(&mut rng)) as ArrayRef;
let date32 = Arc::new(generate_date32_array(&mut rng)) as ArrayRef;
let date64 = Arc::new(generate_date64_array(&mut rng)) as ArrayRef;
+ let duration_ns_small =
+ Arc::new(generate_duration_nanosecond_small_array(&mut rng)) as ArrayRef;
+ let duration_ns_nulls =
+ Arc::new(generate_duration_nanosecond_array_with_nulls(&mut rng)) as ArrayRef;
let udf = date_part();
@@ -322,6 +352,8 @@ fn criterion_benchmark(c: &mut Criterion) {
("interval_dt", &interval_dt),
("interval_mdn", &interval_mdn),
("duration_ns", &duration_ns),
+ ("duration_ns_small", &duration_ns_small),
+ ("duration_ns_nulls", &duration_ns_nulls),
] {
bench_date_part(
c,
diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs
index 5b8d32062b..6dcb948a0e 100644
--- a/datafusion/functions/src/datetime/date_part.rs
+++ b/datafusion/functions/src/datetime/date_part.rs
@@ -39,7 +39,9 @@ use crate::datetime::common::parse_tz;
use datafusion_common::{
Result, ScalarValue,
cast::{
- as_date32_array, as_date64_array, as_int32_array, as_interval_dt_array,
+ as_date32_array, as_date64_array, as_duration_microsecond_array,
+ as_duration_millisecond_array, as_duration_nanosecond_array,
+ as_duration_second_array, as_int32_array, as_interval_dt_array,
as_interval_mdn_array, as_interval_ym_array, as_primitive_array,
as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array,
as_time64_nanosecond_array, as_timestamp_microsecond_array,
@@ -504,48 +506,6 @@ fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {
}
}
-/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the
-/// result to a total number of seconds, milliseconds, microseconds or
-/// nanoseconds
-///
-/// Given epoch return f64, this is a duplicated function to optimize for f64 type
-fn seconds(array: &dyn Array, unit: TimeUnit) -> Result<ArrayRef> {
- let sf = match unit {
- Second => 1_f64,
- Millisecond => 1_000_f64,
- Microsecond => 1_000_000_f64,
- Nanosecond => 1_000_000_000_f64,
- };
- let secs = date_part(array, DatePart::Second)?;
- // This assumes array is primitive and not a dictionary
- let secs = as_int32_array(secs.as_ref())?;
- let subsecs = date_part(array, DatePart::Nanosecond)?;
- let subsecs = as_int32_array(subsecs.as_ref())?;
-
- // Special case where there are no nulls.
- if subsecs.null_count() == 0 {
- let r: Float64Array = binary(secs, subsecs, |secs, subsecs| {
- (secs as f64 + ((subsecs % 1_000_000_000) as f64 / 1_000_000_000_f64)) * sf
- })?;
- Ok(Arc::new(r))
- } else {
- // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case
- // where the number of nanoseconds overflows.
- let r: Float64Array = secs
- .iter()
- .zip(subsecs)
- .map(|(secs, subsecs)| {
- secs.map(|secs| {
- let subsecs = subsecs.unwrap_or(0);
- (secs as f64 + ((subsecs % 1_000_000_000) as f64 / 1_000_000_000_f64))
- * sf
- })
- })
- .collect();
- Ok(Arc::new(r))
- }
-}
-
fn epoch(array: &dyn Array) -> Result<ArrayRef> {
const SECONDS_IN_A_DAY: f64 = 86400_f64;
// Note: Month-to-second conversion uses 30 days as an approximation.
@@ -589,7 +549,16 @@ fn epoch(array: &dyn Array) -> Result<ArrayRef> {
+ x.nanoseconds as f64 / 1_000_000_000_f64
})
}
- Duration(_) => return seconds(array, Second),
+ Duration(Second) => as_duration_second_array(array)?.unary(|x| x as f64),
+ Duration(Millisecond) => {
+ as_duration_millisecond_array(array)?.unary(|x| x as f64 / 1_000_f64)
+ }
+ Duration(Microsecond) => {
+ as_duration_microsecond_array(array)?.unary(|x| x as f64 / 1_000_000_f64)
+ }
+ Duration(Nanosecond) => {
+ as_duration_nanosecond_array(array)?.unary(|x| x as f64 / 1_000_000_000_f64)
+ }
d => return exec_err!("Cannot convert {d:?} to epoch"),
};
Ok(Arc::new(f))
diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt
index 03269772d9..96cbe78d49 100644
--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt
+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt
@@ -1178,6 +1178,78 @@ SELECT extract(epoch from arrow_cast(2, 'Duration(Second)'))
----
2
+# epoch keeps the fraction of a duration whose total sub-second units exceed i32::MAX
+query RR
+SELECT extract(epoch from arrow_cast(2147483647, 'Duration(Nanosecond)')),
+ extract(epoch from arrow_cast(2147483648, 'Duration(Nanosecond)'))
+----
+2.147483647 2.147483648
+
+# Reproducer from https://github.com/apache/datafusion/issues/26673: every row keeps its fraction.
+# Before the fix, the rows at or above 2^31 ns (2.147483648 s) lost their fraction.
+query TTR
+SELECT description,
+ arrow_typeof(finished_at - started_at),
+ extract(epoch from finished_at - started_at)
+FROM (VALUES
+ ('1.25 seconds', timestamp '2025-01-01 00:00:00', timestamp '2025-01-01 00:00:01.250000'),
+ ('2.10 seconds', timestamp '2025-01-01 00:00:00', timestamp '2025-01-01 00:00:02.100000'),
+ ('2.50 seconds', timestamp '2025-01-01 00:00:00', timestamp '2025-01-01 00:00:02.500000'),
+ ('4.763 seconds', timestamp '2025-01-01 00:00:00', timestamp '2025-01-01 00:00:04.763000'),
+ ('12.923350 seconds', timestamp '2025-01-01 00:00:00', timestamp '2025-01-01 00:00:12.923350')
+) AS t(description, started_at, finished_at)
+----
+1.25 seconds Duration(ns) 1.25
+2.10 seconds Duration(ns) 2.1
+2.50 seconds Duration(ns) 2.5
+4.763 seconds Duration(ns) 4.763
+12.923350 seconds Duration(ns) 12.92335
+
+query R
+SELECT extract(epoch from arrow_cast(2147483648, 'Duration(Microsecond)'))
+----
+2147.483648
+
+query R
+SELECT extract(epoch from arrow_cast(2147483648, 'Duration(Millisecond)'))
+----
+2147483.648
+
+query R
+SELECT extract(epoch from arrow_cast(2147483648, 'Duration(Second)'))
+----
+2147483648
+
+query R
+SELECT extract(epoch from arrow_cast(-2500000000, 'Duration(Nanosecond)'))
+----
+-2.5
+
+query R
+SELECT extract(epoch from arrow_cast(NULL, 'Duration(Nanosecond)'))
+----
+NULL
+
+# NULLs stay NULL and don't change the other rows of the batch
+query R
+SELECT extract(epoch from d) FROM (VALUES
+ (arrow_cast(2500000000, 'Duration(Nanosecond)')),
+ (NULL),
+ (arrow_cast(1500000000, 'Duration(Nanosecond)'))
+) AS t(d)
+----
+2.5
+NULL
+1.5
+
+query RRRR
+SELECT extract(epoch from arrow_cast(9223372036854775807, 'Duration(Nanosecond)')),
+ extract(epoch from arrow_cast(-9223372036854775807, 'Duration(Nanosecond)')),
+ extract(epoch from arrow_cast(9223372036854775807, 'Duration(Second)')),
+ extract(epoch from arrow_cast(-9223372036854775807, 'Duration(Second)'))
+----
+9223372036.854776 -9223372036.854776 9223372036854776000 -9223372036854776000
+
query I
SELECT extract(millisecond from arrow_cast(2, 'Duration(Second)'))
----