Skip to content

Apache DataFusion SQL Query Engine

fix(spark): floor instead of truncate in unix_millis/unix_seconds for pre-epoch timestamp (#26129)

lazzypanda74ec76ade4e38
diff --git a/datafusion/spark/src/function/datetime/unix.rs b/datafusion/spark/src/function/datetime/unix.rs
index e82ff3b6e4..158648b97f 100644
--- a/datafusion/spark/src/function/datetime/unix.rs
+++ b/datafusion/spark/src/function/datetime/unix.rs
@@ -4,10 +4,11 @@
use std::sync::Arc;
-use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, TimeUnit};
+use datafusion_common::cast::as_int64_array;
use datafusion_common::types::logical_date;
use datafusion_common::utils::take_function_args;
-use datafusion_common::{Result, internal_err};
+use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
use datafusion_expr::{
Coercion, ColumnarValue, Expr, ExprSchemable, ReturnFieldArgs, ScalarFunctionArgs,
@@ -111,6 +112,24 @@ impl SparkUnixTimestamp {
pub fn seconds() -> Self {
Self::new("unix_seconds", TimeUnit::Second)
}
+
+ fn downscale_divisor(&self, input_type: &DataType) -> Option<i64> {
+ match (input_type, self.time_unit) {
+ (DataType::Timestamp(TimeUnit::Millisecond, _), TimeUnit::Second)
+ | (DataType::Timestamp(TimeUnit::Microsecond, _), TimeUnit::Millisecond)
+ | (DataType::Timestamp(TimeUnit::Nanosecond, _), TimeUnit::Microsecond) => {
+ Some(1_000)
+ }
+ (DataType::Timestamp(TimeUnit::Microsecond, _), TimeUnit::Second)
+ | (DataType::Timestamp(TimeUnit::Nanosecond, _), TimeUnit::Millisecond) => {
+ Some(1_000_000)
+ }
+ (DataType::Timestamp(TimeUnit::Nanosecond, _), TimeUnit::Second) => {
+ Some(1_000_000_000)
+ }
+ _ => None,
+ }
+ }
}
impl ScalarUDFImpl for SparkUnixTimestamp {
@@ -131,8 +150,30 @@ impl ScalarUDFImpl for SparkUnixTimestamp {
Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable)))
}
- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- internal_err!("invoke_with_args should not be called on `{}`", self.name())
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ let [ts] = take_function_args(self.name(), args.args)?;
+ let Some(divisor) = self.downscale_divisor(&ts.data_type()) else {
+ return ts
+ .cast_to(
+ &DataType::Timestamp(self.time_unit, Some("UTC".into())),
+ None,
+ )?
+ .cast_to(&DataType::Int64, None);
+ };
+ // Spark floors negative fractional timestamps, unlike Arrow's
+ // truncating timestamp casts. The argument has already been evaluated once.
+ match ts.cast_to(&DataType::Int64, None)? {
+ ColumnarValue::Scalar(ScalarValue::Int64(ticks)) => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(
+ ticks.map(|ticks| ticks.div_euclid(divisor)),
+ )))
+ }
+ ColumnarValue::Array(ticks) => Ok(ColumnarValue::Array(Arc::new(
+ as_int64_array(&ticks)?
+ .unary::<_, Int64Type>(|ticks| ticks.div_euclid(divisor)),
+ ))),
+ other => internal_err!("Expected Int64 ticks, got {other:?}"),
+ }
}
fn simplify(
@@ -141,6 +182,12 @@ impl ScalarUDFImpl for SparkUnixTimestamp {
info: &SimplifyContext,
) -> Result<ExprSimplifyResult> {
let [ts] = take_function_args(self.name(), args)?;
+ if self
+ .downscale_divisor(&ts.get_type(info.schema())?)
+ .is_some()
+ {
+ return Ok(ExprSimplifyResult::Original(vec![ts]));
+ }
Ok(ExprSimplifyResult::Simplified(
ts.cast_to(
&DataType::Timestamp(self.time_unit, Some("UTC".into())),
@@ -150,3 +197,80 @@ impl ScalarUDFImpl for SparkUnixTimestamp {
))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::Int64Array;
+ use datafusion::prelude::SessionContext;
+ use datafusion_expr::{ScalarUDF, create_udf};
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ #[tokio::test]
+ async fn test_unix_timestamp_evaluates_volatile_input_once() -> Result<()> {
+ for (function, unit, divisor) in [
+ (SparkUnixTimestamp::seconds(), "Millisecond", 1_000),
+ (SparkUnixTimestamp::milliseconds(), "Microsecond", 1_000),
+ (SparkUnixTimestamp::microseconds(), "Nanosecond", 1_000),
+ (SparkUnixTimestamp::seconds(), "Microsecond", 1_000_000),
+ (SparkUnixTimestamp::milliseconds(), "Nanosecond", 1_000_000),
+ (SparkUnixTimestamp::seconds(), "Nanosecond", 1_000_000_000),
+ ] {
+ let function = ScalarUDF::from(function);
+ for array_input in [false, true] {
+ let ctx = SessionContext::new();
+ let calls = Arc::new(AtomicUsize::new(0));
+ let udf_calls = Arc::clone(&calls);
+ ctx.register_udf(create_udf(
+ "volatile_ticks",
+ vec![DataType::Int64],
+ DataType::Int64,
+ Volatility::Volatile,
+ Arc::new(move |args| {
+ let ticks =
+ if udf_calls.fetch_add(1, Ordering::SeqCst).is_multiple_of(2)
+ {
+ -1
+ } else {
+ divisor
+ };
+ Ok(match &args[0] {
+ ColumnarValue::Scalar(_) => {
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(ticks)))
+ }
+ ColumnarValue::Array(array) => ColumnarValue::Array(
+ Arc::new(Int64Array::from(vec![ticks; array.len()])),
+ ),
+ })
+ }),
+ ));
+ ctx.register_udf(function.clone());
+ let (input, from, expected) = if array_input {
+ (
+ "column1",
+ " FROM (VALUES (0), (1))",
+ vec![Some(-1), Some(-1)],
+ )
+ } else {
+ ("0", "", vec![Some(-1)])
+ };
+ let query = format!(
+ "SELECT {}(arrow_cast(volatile_ticks({input}), \
+ 'Timestamp({unit}, None)')) AS result{from}",
+ function.name()
+ );
+ let batches = ctx.sql(&query).await?.collect().await?;
+ assert_eq!(calls.load(Ordering::SeqCst), 1, "{query}");
+ let actual = batches
+ .iter()
+ .map(|batch| as_int64_array(batch.column(0)))
+ .collect::<Result<Vec<_>>>()?
+ .into_iter()
+ .flat_map(|array| array.iter())
+ .collect::<Vec<_>>();
+ assert_eq!(actual, expected, "{query}");
+ }
+ }
+ Ok(())
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/unix.slt b/datafusion/sqllogictest/test_files/spark/datetime/unix.slt
index e3109a1b9f..0c93db63f1 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/unix.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/unix.slt
@@ -119,3 +119,67 @@ NULL
query error Function 'unix_seconds' requires Timestamp, but received String \(DataType: Utf8View\)
SELECT unix_seconds('1970-01-01 00:00:01Z'::string);
+
+# Spark floors (Math.floorDiv) when converting to a coarser unit, so pre-epoch
+# timestamps with a sub-unit remainder round toward negative infinity.
+statement ok
+CREATE TABLE unix_floor_us AS SELECT arrow_cast(column1, 'Timestamp(Microsecond, None)') AS ts FROM (VALUES
+ (-1), (-1000), (-1001), (-1000000), (-1000001), (1500), (NULL)
+);
+
+query III
+SELECT arrow_cast(ts, 'Int64'), unix_millis(ts), unix_seconds(ts) FROM unix_floor_us;
+----
+-1 -1 -1
+-1000 -1 -1
+-1001 -2 -1
+-1000000 -1000 -1
+-1000001 -1001 -2
+1500 1 0
+NULL NULL NULL
+
+statement ok
+DROP TABLE unix_floor_us;
+
+# Same results when the expressions are constant-folded
+query II
+SELECT
+ unix_millis(arrow_cast(-1001, 'Timestamp(Microsecond, None)')),
+ unix_seconds(arrow_cast(-1000001, 'Timestamp(Microsecond, None)'));
+----
+-2 -2
+
+# Other input units: nanoseconds floor too, milliseconds to seconds floors,
+# and conversions to a finer unit are unchanged
+query IIII
+SELECT
+ unix_millis(arrow_cast(-1, 'Timestamp(Nanosecond, None)')),
+ unix_micros(arrow_cast(-1, 'Timestamp(Nanosecond, None)')),
+ unix_seconds(arrow_cast(-1, 'Timestamp(Millisecond, None)')),
+ unix_micros(arrow_cast(-1, 'Timestamp(Millisecond, None)'));
+----
+-1 -1 -1 -1000
+
+# Nanoseconds to seconds floor negative remainders but preserve exact multiples.
+query II
+SELECT
+ unix_seconds(arrow_cast(-1000000001, 'Timestamp(Nanosecond, None)')),
+ unix_seconds(arrow_cast(-1000000000, 'Timestamp(Nanosecond, None)'));
+----
+-2 -1
+
+# Timezone metadata does not change the epoch-based result.
+query II
+SELECT
+ unix_millis(arrow_cast(-1, 'Timestamp(Microsecond, Some("+02:00"))')),
+ unix_seconds(arrow_cast(-1, 'Timestamp(Microsecond, Some("+02:00"))'));
+----
+-1 -1
+
+# Extreme values do not overflow
+query II
+SELECT
+ unix_seconds(arrow_cast(-9223372036854775808, 'Timestamp(Microsecond, None)')),
+ unix_seconds(arrow_cast(9223372036854775, 'Timestamp(Millisecond, None)'));
+----
+-9223372036855 9223372036854