Apache DataFusion SQL Query Engine
fix(spark): floor instead of truncate in unix_millis/unix_seconds for pre-epoch timestamp (#26129)
lazzypanda74ec76ade4e38
diff --git a/datafusion/spark/src/function/datetime/unix.rs b/datafusion/spark/src/function/datetime/unix.rsindex e82ff3b6e4..158648b97f 100644--- a/datafusion/spark/src/function/datetime/unix.rs+++ b/datafusion/spark/src/function/datetime/unix.rs@@ -4,10 +4,11 @@use std::sync::Arc;-use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};+use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, TimeUnit};+use datafusion_common::cast::as_int64_array;use datafusion_common::types::logical_date;use datafusion_common::utils::take_function_args;-use datafusion_common::{Result, internal_err};+use datafusion_common::{Result, ScalarValue, internal_err};use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};use datafusion_expr::{Coercion, ColumnarValue, Expr, ExprSchemable, ReturnFieldArgs, ScalarFunctionArgs,@@ -111,6 +112,24 @@ impl SparkUnixTimestamp {pub fn seconds() -> Self {Self::new("unix_seconds", TimeUnit::Second)}++ fn downscale_divisor(&self, input_type: &DataType) -> Option<i64> {+ match (input_type, self.time_unit) {+ (DataType::Timestamp(TimeUnit::Millisecond, _), TimeUnit::Second)+ | (DataType::Timestamp(TimeUnit::Microsecond, _), TimeUnit::Millisecond)+ | (DataType::Timestamp(TimeUnit::Nanosecond, _), TimeUnit::Microsecond) => {+ Some(1_000)+ }+ (DataType::Timestamp(TimeUnit::Microsecond, _), TimeUnit::Second)+ | (DataType::Timestamp(TimeUnit::Nanosecond, _), TimeUnit::Millisecond) => {+ Some(1_000_000)+ }+ (DataType::Timestamp(TimeUnit::Nanosecond, _), TimeUnit::Second) => {+ Some(1_000_000_000)+ }+ _ => None,+ }+ }}impl ScalarUDFImpl for SparkUnixTimestamp {@@ -131,8 +150,30 @@ impl ScalarUDFImpl for SparkUnixTimestamp {Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable)))}- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {- internal_err!("invoke_with_args should not be called on `{}`", self.name())+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ let [ts] = take_function_args(self.name(), args.args)?;+ let Some(divisor) = self.downscale_divisor(&ts.data_type()) else {+ return ts+ .cast_to(+ &DataType::Timestamp(self.time_unit, Some("UTC".into())),+ None,+ )?+ .cast_to(&DataType::Int64, None);+ };+ // Spark floors negative fractional timestamps, unlike Arrow's+ // truncating timestamp casts. The argument has already been evaluated once.+ match ts.cast_to(&DataType::Int64, None)? {+ ColumnarValue::Scalar(ScalarValue::Int64(ticks)) => {+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(+ ticks.map(|ticks| ticks.div_euclid(divisor)),+ )))+ }+ ColumnarValue::Array(ticks) => Ok(ColumnarValue::Array(Arc::new(+ as_int64_array(&ticks)?+ .unary::<_, Int64Type>(|ticks| ticks.div_euclid(divisor)),+ ))),+ other => internal_err!("Expected Int64 ticks, got {other:?}"),+ }}fn simplify(@@ -141,6 +182,12 @@ impl ScalarUDFImpl for SparkUnixTimestamp {info: &SimplifyContext,) -> Result<ExprSimplifyResult> {let [ts] = take_function_args(self.name(), args)?;+ if self+ .downscale_divisor(&ts.get_type(info.schema())?)+ .is_some()+ {+ return Ok(ExprSimplifyResult::Original(vec![ts]));+ }Ok(ExprSimplifyResult::Simplified(ts.cast_to(&DataType::Timestamp(self.time_unit, Some("UTC".into())),@@ -150,3 +197,80 @@ impl ScalarUDFImpl for SparkUnixTimestamp {))}}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::Int64Array;+ use datafusion::prelude::SessionContext;+ use datafusion_expr::{ScalarUDF, create_udf};+ use std::sync::atomic::{AtomicUsize, Ordering};++ #[tokio::test]+ async fn test_unix_timestamp_evaluates_volatile_input_once() -> Result<()> {+ for (function, unit, divisor) in [+ (SparkUnixTimestamp::seconds(), "Millisecond", 1_000),+ (SparkUnixTimestamp::milliseconds(), "Microsecond", 1_000),+ (SparkUnixTimestamp::microseconds(), "Nanosecond", 1_000),+ (SparkUnixTimestamp::seconds(), "Microsecond", 1_000_000),+ (SparkUnixTimestamp::milliseconds(), "Nanosecond", 1_000_000),+ (SparkUnixTimestamp::seconds(), "Nanosecond", 1_000_000_000),+ ] {+ let function = ScalarUDF::from(function);+ for array_input in [false, true] {+ let ctx = SessionContext::new();+ let calls = Arc::new(AtomicUsize::new(0));+ let udf_calls = Arc::clone(&calls);+ ctx.register_udf(create_udf(+ "volatile_ticks",+ vec![DataType::Int64],+ DataType::Int64,+ Volatility::Volatile,+ Arc::new(move |args| {+ let ticks =+ if udf_calls.fetch_add(1, Ordering::SeqCst).is_multiple_of(2)+ {+ -1+ } else {+ divisor+ };+ Ok(match &args[0] {+ ColumnarValue::Scalar(_) => {+ ColumnarValue::Scalar(ScalarValue::Int64(Some(ticks)))+ }+ ColumnarValue::Array(array) => ColumnarValue::Array(+ Arc::new(Int64Array::from(vec![ticks; array.len()])),+ ),+ })+ }),+ ));+ ctx.register_udf(function.clone());+ let (input, from, expected) = if array_input {+ (+ "column1",+ " FROM (VALUES (0), (1))",+ vec![Some(-1), Some(-1)],+ )+ } else {+ ("0", "", vec![Some(-1)])+ };+ let query = format!(+ "SELECT {}(arrow_cast(volatile_ticks({input}), \+ 'Timestamp({unit}, None)')) AS result{from}",+ function.name()+ );+ let batches = ctx.sql(&query).await?.collect().await?;+ assert_eq!(calls.load(Ordering::SeqCst), 1, "{query}");+ let actual = batches+ .iter()+ .map(|batch| as_int64_array(batch.column(0)))+ .collect::<Result<Vec<_>>>()?+ .into_iter()+ .flat_map(|array| array.iter())+ .collect::<Vec<_>>();+ assert_eq!(actual, expected, "{query}");+ }+ }+ Ok(())+ }+}diff --git a/datafusion/sqllogictest/test_files/spark/datetime/unix.slt b/datafusion/sqllogictest/test_files/spark/datetime/unix.sltindex e3109a1b9f..0c93db63f1 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/unix.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/unix.slt@@ -119,3 +119,67 @@ NULLquery error Function 'unix_seconds' requires Timestamp, but received String \(DataType: Utf8View\)SELECT unix_seconds('1970-01-01 00:00:01Z'::string);++# Spark floors (Math.floorDiv) when converting to a coarser unit, so pre-epoch+# timestamps with a sub-unit remainder round toward negative infinity.+statement ok+CREATE TABLE unix_floor_us AS SELECT arrow_cast(column1, 'Timestamp(Microsecond, None)') AS ts FROM (VALUES+ (-1), (-1000), (-1001), (-1000000), (-1000001), (1500), (NULL)+);++query III+SELECT arrow_cast(ts, 'Int64'), unix_millis(ts), unix_seconds(ts) FROM unix_floor_us;+----+-1 -1 -1+-1000 -1 -1+-1001 -2 -1+-1000000 -1000 -1+-1000001 -1001 -2+1500 1 0+NULL NULL NULL++statement ok+DROP TABLE unix_floor_us;++# Same results when the expressions are constant-folded+query II+SELECT+ unix_millis(arrow_cast(-1001, 'Timestamp(Microsecond, None)')),+ unix_seconds(arrow_cast(-1000001, 'Timestamp(Microsecond, None)'));+----+-2 -2++# Other input units: nanoseconds floor too, milliseconds to seconds floors,+# and conversions to a finer unit are unchanged+query IIII+SELECT+ unix_millis(arrow_cast(-1, 'Timestamp(Nanosecond, None)')),+ unix_micros(arrow_cast(-1, 'Timestamp(Nanosecond, None)')),+ unix_seconds(arrow_cast(-1, 'Timestamp(Millisecond, None)')),+ unix_micros(arrow_cast(-1, 'Timestamp(Millisecond, None)'));+----+-1 -1 -1 -1000++# Nanoseconds to seconds floor negative remainders but preserve exact multiples.+query II+SELECT+ unix_seconds(arrow_cast(-1000000001, 'Timestamp(Nanosecond, None)')),+ unix_seconds(arrow_cast(-1000000000, 'Timestamp(Nanosecond, None)'));+----+-2 -1++# Timezone metadata does not change the epoch-based result.+query II+SELECT+ unix_millis(arrow_cast(-1, 'Timestamp(Microsecond, Some("+02:00"))')),+ unix_seconds(arrow_cast(-1, 'Timestamp(Microsecond, Some("+02:00"))'));+----+-1 -1++# Extreme values do not overflow+query II+SELECT+ unix_seconds(arrow_cast(-9223372036854775808, 'Timestamp(Microsecond, None)')),+ unix_seconds(arrow_cast(9223372036854775, 'Timestamp(Millisecond, None)'));+----+-9223372036855 9223372036854