Apache DataFusion SQL Query Engine
feat: improve Spark from_utc_timestamp compatibility (#25979)
nat-openai36c031242f06
diff --git a/datafusion/functions/src/datetime/to_local_time.rs b/datafusion/functions/src/datetime/to_local_time.rsindex 10e61a3b57..091510c792 100644--- a/datafusion/functions/src/datetime/to_local_time.rs+++ b/datafusion/functions/src/datetime/to_local_time.rs@@ -15,7 +15,6 @@// specific language governing permissions and limitations// under the License.-use std::ops::Add;use std::sync::Arc;use arrow::array::timezone::Tz;@@ -30,8 +29,8 @@ use chrono::{DateTime, MappedLocalTime, Offset, TimeDelta, TimeZone, Utc};use datafusion_common::cast::as_primitive_array;use datafusion_common::{- Result, ScalarValue, exec_err, internal_datafusion_err, internal_err,- utils::take_function_args,+ Result, ScalarValue, exec_datafusion_err, exec_err, internal_datafusion_err,+ internal_err, utils::take_function_args,};use datafusion_expr::{Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -311,7 +310,10 @@ pub(crate) fn to_local_time(time_value: &ColumnarValue) -> Result<ColumnarValue>/// ```////// See `test_adjust_to_local_time()` for example-pub fn adjust_to_local_time<T: ArrowTimestampType>(ts: i64, tz: Tz) -> Result<i64> {+pub fn adjust_to_local_time<T: ArrowTimestampType>(+ ts: i64,+ tz: impl TimeZone + Copy,+) -> Result<i64> {fn convert_timestamp<F>(ts: i64, converter: F) -> Result<DateTime<Utc>>whereF: Fn(i64) -> MappedLocalTime<DateTime<Utc>>,@@ -341,12 +343,18 @@ pub fn adjust_to_local_time<T: ArrowTimestampType>(ts: i64, tz: Tz) -> Result<i6.fix().local_minus_utc() as i64;- let adjusted_date_time = date_time.add(- // This should not fail under normal circumstances as the- // maximum possible offset is 26 hours (93,600 seconds)- TimeDelta::try_seconds(offset_seconds)- .ok_or_else(|| internal_datafusion_err!("Offset seconds should be less than i64::MAX / 1_000 or greater than -i64::MAX / 1_000"))?,- );+ let adjusted_date_time = date_time+ .checked_add_signed(+ // This should not fail under normal circumstances as the+ // maximum possible offset is 26 hours (93,600 seconds)+ TimeDelta::try_seconds(offset_seconds)+ .ok_or_else(|| internal_datafusion_err!("Offset seconds should be less than i64::MAX / 1_000 or greater than -i64::MAX / 1_000"))?,+ )+ .ok_or_else(|| {+ exec_datafusion_err!(+ "Timezone adjustment of timestamp {ts} by {offset_seconds} seconds exceeds Chrono's calendar range"+ )+ })?;// convert the naive datetime back to i64match T::UNIT {diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex 6fa172136a..c7aee8691f 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -107,6 +107,10 @@ name = "sha2"harness = falsename = "floor"+[[bench]]+harness = false+name = "from_utc_timestamp"+[[bench]]harness = falsename = "stack"diff --git a/datafusion/spark/benches/from_utc_timestamp.rs b/datafusion/spark/benches/from_utc_timestamp.rsnew file mode 100644index 0000000000..44acd9a376--- /dev/null+++ b/datafusion/spark/benches/from_utc_timestamp.rs@@ -0,0 +1,117 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Measure column, computed, and literal timezone evaluation across batch widths and null+//! densities. Computed timezones exercise selection and copying of unrelated columns, while+//! literal timezones exercise scalar broadcasting. A physical expression benchmark is needed+//! because ordinary SQL uses the eager UDF instead of the conditional evaluator.++use std::hint::black_box;+use std::sync::Arc;++use arrow::array::{ArrayRef, Int64Array, StringArray, TimestampMicrosecondArray};+use arrow::datatypes::{DataType, Field, Schema, TimeUnit};+use arrow::record_batch::RecordBatch;+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};+use datafusion::prelude::{SessionContext, col, lit};+use datafusion_common::DFSchema;+use datafusion_expr::when;+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+use datafusion_spark::function::datetime::from_utc_timestamp::SparkFromUtcTimestampExpr;++fn batch(rows: usize, columns: usize, null_every: Option<usize>) -> RecordBatch {+ let mut fields = vec![+ Field::new(+ "timestamp",+ DataType::Timestamp(TimeUnit::Microsecond, None),+ true,+ ),+ Field::new("timezone", DataType::Utf8, false),+ ];+ let mut arrays: Vec<ArrayRef> = vec![+ Arc::new(TimestampMicrosecondArray::from_iter((0..rows).map(|row| {+ null_every+ .is_none_or(|every| row % every != 0)+ .then_some(row as i64 * 86_400_000_000)+ }))),+ Arc::new(StringArray::from_iter_values(+ (0..rows).map(|row| if (row / 2) % 2 == 0 { "PST" } else { "EST" }),+ )),+ ];++ for column in 2..columns {+ fields.push(Field::new(+ format!("unused_{column}"),+ DataType::Int64,+ false,+ ));+ arrays.push(Arc::new(Int64Array::from_iter_values(+ (0..rows).map(|row| (column * rows + row) as i64),+ )));+ }++ RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).unwrap()+}++fn criterion_benchmark(c: &mut Criterion) {+ let rows = 8192;+ let ctx = SessionContext::new();+ let timezones = [+ ("column", col("timezone")),+ (+ "computed",+ when(col("timezone").eq(lit("PST")), lit("PST"))+ .otherwise(lit("EST"))+ .unwrap(),+ ),+ ("literal", lit("PST")),+ ];++ for (source, timezone) in timezones {+ let mut group = c.benchmark_group(format!("from_utc_timestamp_{source}"));+ group.throughput(Throughput::Elements(rows as u64));+ for columns in [2, 64, 256] {+ for (null_every, name) in [+ (None, "no_nulls"),+ (Some(100), "sparse_nulls"),+ (Some(2), "half_nulls"),+ ] {+ let batch = batch(rows, columns, null_every);+ let schema = DFSchema::try_from(batch.schema().as_ref().clone()).unwrap();+ let expr = SparkFromUtcTimestampExpr::new(+ ctx.create_physical_expr(col("timestamp"), &schema).unwrap(),+ ctx.create_physical_expr(timezone.clone(), &schema).unwrap(),+ );+ let result = expr.evaluate(&batch).unwrap().into_array(rows).unwrap();+ assert_eq!(result.len(), rows);+ assert_eq!(result.null_count(), batch.column(0).null_count());++ group.bench_with_input(+ BenchmarkId::new(format!("{columns}_columns"), name),+ &batch,+ |b, batch| {+ b.iter(|| black_box(expr.evaluate(black_box(batch)).unwrap()));+ },+ );+ }+ }+ group.finish();+ }+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/datafusion/spark/src/function/datetime/from_utc_timestamp.rs b/datafusion/spark/src/function/datetime/from_utc_timestamp.rsindex bfca677c1d..3fe73ecd96 100644--- a/datafusion/spark/src/function/datetime/from_utc_timestamp.rs+++ b/datafusion/spark/src/function/datetime/from_utc_timestamp.rs@@ -15,24 +15,30 @@// specific language governing permissions and limitations// under the License.+use std::fmt::{Display, Formatter};+use std::hash::{Hash, Hasher};use std::sync::Arc;-use arrow::array::timezone::Tz;-use arrow::array::{Array, ArrayRef, AsArray, PrimitiveBuilder, StringArrayType};+use super::timezone::SparkTimeZone;+use arrow::array::{+ Array, ArrayRef, AsArray, BooleanArray, PrimitiveBuilder, StringArrayType,+ new_empty_array,+};use arrow::datatypes::TimeUnit;use arrow::datatypes::{- ArrowTimestampType, DataType, Field, FieldRef, TimestampMicrosecondType,+ ArrowTimestampType, DataType, Field, FieldRef, Schema, TimestampMicrosecondType,TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,};+use arrow::record_batch::RecordBatch;use datafusion_common::types::{NativeType, logical_string};use datafusion_common::utils::take_function_args;use datafusion_common::{Result, exec_datafusion_err, exec_err, internal_err};use datafusion_expr::{- Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,- Signature, TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs,+ ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,};-use datafusion_functions::datetime::to_local_time::adjust_to_local_time;use datafusion_functions::utils::make_scalar_function;+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;/// Apache Spark `from_utc_timestamp` function.///@@ -42,6 +48,21 @@ use datafusion_functions::utils::make_scalar_function;/// timezone-agnostic. So in Apache Spark this function just shift the timestamp value from UTC timezone to/// the given timezone.///+/// # Compatibility+///+/// This function accepts Spark's additional offset spellings, Java short IDs, and+/// legacy SystemV zone IDs. They are parsed only when both arguments are non-null.+/// Spark's generated execution may validate a foldable timezone before processing+/// rows, so this implementation can return nulls where Spark raises an error.+///+/// Timezone conversion uses Chrono's calendar range, which is narrower than Spark's+/// implemented timestamp range, although it includes Spark's documented range.+/// Region timezone rules from `chrono-tz` may also differ after 2099 because it does+/// not extrapolate later daylight saving time transitions.+///+/// This UDF operates on already-evaluated arguments. Callers that need conditional+/// evaluation of computed timezone expressions should construct [`SparkFromUtcTimestampExpr`].+////// See <https://spark.apache.org/docs/latest/api/sql/index.html#from_utc_timestamp>#[derive(Debug, PartialEq, Eq, Hash)]pub struct SparkFromUtcTimestamp {@@ -169,16 +190,21 @@ where{let ts_primitive = ts_array.as_primitive::<T>();let mut builder = PrimitiveBuilder::<T>::with_capacity(ts_array.len());+ // Parse a scalar zone once per batch and avoid re-parsing consecutive column values+ let mut last_timezone = None;for (ts_opt, tz_opt) in ts_primitive.iter().zip(tz_array.iter()) {match (ts_opt, tz_opt) {(Some(ts), Some(tz_str)) => {- let tz: Tz = tz_str.parse().map_err(|e| {- exec_datafusion_err!(- "`from_utc_timestamp`: invalid timezone '{tz_str}': {e}"- )- })?;- let val = adjust_to_local_time::<T>(ts, tz)?;+ let timezone = match last_timezone {+ Some((previous, timezone)) if previous == tz_str => timezone,+ _ => {+ let timezone = parse_timezone(tz_str)?;+ last_timezone = Some((tz_str, timezone));+ timezone+ }+ };+ let val = timezone.adjust_to_local_time::<T>(ts)?;builder.append_value(val);}_ => builder.append_null(),@@ -188,3 +214,653 @@ wherebuilder = builder.with_timezone_opt(return_tz_opt);Ok(Arc::new(builder.finish()))}++fn parse_timezone(timezone: &str) -> Result<SparkTimeZone> {+ SparkTimeZone::parse(timezone).map_err(|e| {+ exec_datafusion_err!("`from_utc_timestamp`: invalid timezone '{timezone}': {e}")+ })+}++/// Physical `from_utc_timestamp` expression with conditional timezone evaluation.+///+/// The timestamp child is evaluated once for each non-empty batch. If every timestamp is null,+/// the timezone child is skipped. Otherwise, literal and column timezones are read directly, and+/// their values are ignored on null timestamp rows. Computed timezone expressions are evaluated+/// only for rows with non-null timestamps. Empty batches evaluate neither child.+///+/// Registering [`SparkFromUtcTimestamp`] as a UDF does not use this expression: ordinary SQL+/// evaluates the UDF's arguments eagerly. Consumers must explicitly construct this expression+/// after planning and coercing its children to obtain conditional timezone evaluation.+///+/// This intentionally differs from Spark's generated evaluation of foldable timezones, which+/// evaluates the timezone and validates a non-null value before processing timestamp rows. This+/// expression can therefore skip an invalid timezone or an error in the timezone expression when+/// no timestamp is non-null. It also evaluates the timestamp first when the timezone is null, so a+/// timestamp error can be raised even though Spark's generated path would return null. Planners+/// using this expression should expose these differences as an incompatibility.+#[derive(Debug, Clone)]+pub struct SparkFromUtcTimestampExpr {+ timestamp: Arc<dyn PhysicalExpr>,+ timezone: Arc<dyn PhysicalExpr>,+}++impl SparkFromUtcTimestampExpr {+ /// Construct an expression that skips computed timezone evaluation for null timestamps.+ ///+ /// Both children must already have supported types: a timestamp (any time unit, with or+ /// without timezone metadata) and a string (`Utf8`, `LargeUtf8`, or `Utf8View`). This+ /// constructor does not validate or coerce their types. Insert required casts during+ /// planning, keeping timezone casts inside the timezone child so they are also skipped+ /// for null timestamps.+ ///+ /// # Example+ ///+ /// A consumer can plan typed children before constructing the physical expression.+ /// The invalid integer cast in the timezone child is skipped for the null timestamp.+ ///+ /// ```+ /// use std::sync::Arc;+ /// use arrow::array::{ArrayRef, StringArray, TimestampMicrosecondArray};+ /// use arrow::datatypes::{DataType, TimeUnit};+ /// use arrow::record_batch::RecordBatch;+ /// use datafusion::prelude::{SessionContext, col};+ /// use datafusion_common::DFSchema;+ /// use datafusion_expr::cast;+ /// use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+ /// use datafusion_spark::function::datetime::from_utc_timestamp::SparkFromUtcTimestampExpr;+ ///+ /// let batch = RecordBatch::try_from_iter([+ /// ("ts", Arc::new(StringArray::from(vec![Some("1970-01-01T00:00:00"), None])) as ArrayRef),+ /// ("value", Arc::new(StringArray::from(vec!["1", "invalid"])) as ArrayRef),+ /// ])?;+ /// let schema = DFSchema::try_from(batch.schema().as_ref().clone())?;+ /// let ctx = SessionContext::new();+ /// let timestamp = ctx.create_physical_expr(+ /// cast(col("ts"), DataType::Timestamp(TimeUnit::Microsecond, None)),+ /// &schema,+ /// )?;+ /// let timezone = ctx.parse_sql_expr(+ /// "CASE WHEN CAST(value AS INT) = 1 THEN 'GMT+1' ELSE 'UTC' END",+ /// &schema,+ /// )?;+ /// let timezone = ctx.create_physical_expr(cast(timezone, DataType::Utf8), &schema)?;+ /// let expr = SparkFromUtcTimestampExpr::new(timestamp, timezone);+ /// let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ /// let expected = TimestampMicrosecondArray::from(vec![Some(3_600_000_000), None]);+ /// assert_eq!(result.as_ref(), &expected);+ /// # Ok::<(), datafusion_common::DataFusionError>(())+ /// ```+ pub fn new(+ timestamp: Arc<dyn PhysicalExpr>,+ timezone: Arc<dyn PhysicalExpr>,+ ) -> Self {+ Self {+ timestamp,+ timezone,+ }+ }+}++impl Display for SparkFromUtcTimestampExpr {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(+ f,+ "from_utc_timestamp({}, {})",+ self.timestamp, self.timezone+ )+ }+}++impl PartialEq for SparkFromUtcTimestampExpr {+ fn eq(&self, other: &Self) -> bool {+ self.timestamp.eq(&other.timestamp) && self.timezone.eq(&other.timezone)+ }+}++impl Eq for SparkFromUtcTimestampExpr {}++impl Hash for SparkFromUtcTimestampExpr {+ fn hash<H: Hasher>(&self, state: &mut H) {+ self.timestamp.hash(state);+ self.timezone.hash(state);+ }+}++impl PhysicalExpr for SparkFromUtcTimestampExpr {+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "from_utc_timestamp(")?;+ self.timestamp.fmt_sql(f)?;+ write!(f, ", ")?;+ self.timezone.fmt_sql(f)?;+ write!(f, ")")+ }++ fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {+ let timestamp = self.timestamp.return_field(input_schema)?;+ Ok(Arc::new(Field::new(+ "from_utc_timestamp",+ timestamp.data_type().clone(),+ timestamp.is_nullable() || self.timezone.nullable(input_schema)?,+ )))+ }++ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {+ // There are no rows on which either child should run.+ if batch.num_rows() == 0 {+ return Ok(ColumnarValue::Array(new_empty_array(+ &self.data_type(batch.schema_ref())?,+ )));+ }++ let timestamp = self.timestamp.evaluate(batch)?;+ let timezone = match ×tamp {+ ColumnarValue::Scalar(value) if value.is_null() => {+ return Ok(timestamp);+ }+ ColumnarValue::Scalar(_) => self.timezone.evaluate(batch)?,+ ColumnarValue::Array(array) if array.null_count() == array.len() => {+ return Ok(timestamp);+ }+ ColumnarValue::Array(_)+ if matches!(+ self.timezone.placement(),+ ExpressionPlacement::Literal | ExpressionPlacement::Column+ ) =>+ {+ // Literals and columns only read existing values. The conversion loop skips null+ // timestamps, so avoid filtering and copying unrelated batch columns.+ self.timezone.evaluate(batch)?+ }+ ColumnarValue::Array(array) => match array.nulls() {+ Some(nulls) => {+ let selection = BooleanArray::new(nulls.inner().clone(), None);+ self.timezone.evaluate_selection(batch, &selection)?+ }+ None => self.timezone.evaluate(batch)?,+ },+ };+ make_scalar_function(spark_from_utc_timestamp, vec![])(&[timestamp, timezone])+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![&self.timestamp, &self.timezone]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ let [timestamp, timezone] = children.as_slice() else {+ return internal_err!(+ "SparkFromUtcTimestampExpr expected 2 children, got {}",+ children.len()+ );+ };+ Ok(Arc::new(Self::new(+ Arc::clone(timestamp),+ Arc::clone(timezone),+ )))+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::{+ LargeStringArray, StringArray, StringViewArray, TimestampMicrosecondArray,+ };+ use arrow::compute::CastOptions;+ use chrono::{DateTime, Utc};+ use datafusion::physical_expr::expressions::{+ BinaryExpr, CaseExpr, CastExpr, Column, Literal,+ };+ use datafusion::prelude::SessionContext;+ use datafusion_common::{DFSchema, ScalarValue, config::ConfigOptions};+ use datafusion_expr::Operator;+ use datafusion_physical_expr_common::physical_expr::fmt_sql;++ fn expression_batch(timestamps: Vec<Option<i64>>, values: Vec<&str>) -> RecordBatch {+ RecordBatch::try_new(+ Arc::new(Schema::new(vec![+ Field::new(+ "timestamp",+ DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("UTC"))),+ true,+ ),+ Field::new("value", DataType::Utf8, false),+ ])),+ vec![+ Arc::new(+ TimestampMicrosecondArray::from(timestamps).with_timezone("UTC"),+ ),+ Arc::new(StringArray::from(values)),+ ],+ )+ .unwrap()+ }++ fn throwing_timestamp() -> Arc<dyn PhysicalExpr> {+ Arc::new(CastExpr::new(+ Arc::new(Literal::new(ScalarValue::Utf8(Some("invalid".to_owned())))),+ DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("UTC"))),+ Some(CastOptions {+ safe: false,+ ..Default::default()+ }),+ ))+ }++ fn computed_timezone() -> Arc<dyn PhysicalExpr> {+ let value = Arc::new(CastExpr::new(+ Arc::new(Column::new("value", 1)),+ DataType::Int32,+ Some(CastOptions {+ safe: false,+ ..Default::default()+ }),+ ));+ let is_zero = Arc::new(BinaryExpr::new(+ value,+ Operator::Eq,+ Arc::new(Literal::new(ScalarValue::Int32(Some(0)))),+ ));+ Arc::new(+ CaseExpr::try_new(+ None,+ vec![(+ is_zero,+ Arc::new(Literal::new(ScalarValue::Utf8(Some("UTC".to_owned())))),+ )],+ Some(Arc::new(Literal::new(ScalarValue::Utf8(Some(+ "PST".to_owned(),+ ))))),+ )+ .unwrap(),+ )+ }++ fn invoke(+ timestamp: ColumnarValue,+ timezone: ColumnarValue,+ ) -> Result<ColumnarValue> {+ let return_field = Arc::new(Field::new("result", timestamp.data_type(), true));+ SparkFromUtcTimestamp::new().invoke_with_args(ScalarFunctionArgs {+ args: vec![timestamp, timezone],+ arg_fields: vec![],+ number_rows: 0,+ return_field,+ config_options: Arc::new(ConfigOptions::default()),+ })+ }++ fn timestamps(values: Vec<Option<i64>>) -> ColumnarValue {+ ColumnarValue::Array(Arc::new(+ TimestampMicrosecondArray::from(values).with_timezone("UTC"),+ ))+ }++ fn scalar_zone(value: Option<&str>) -> ColumnarValue {+ ColumnarValue::Scalar(ScalarValue::Utf8(value.map(str::to_owned)))+ }++ fn assert_timestamps(result: ColumnarValue, expected: &[Option<i64>]) {+ let ColumnarValue::Array(array) = result else {+ panic!("expected timestamp array");+ };+ let array = array.as_primitive::<TimestampMicrosecondType>();+ assert_eq!(array.iter().collect::<Vec<_>>(), expected);+ assert_eq!(array.timezone(), Some("UTC"));+ }++ #[test]+ fn column_zones_preserve_values_nulls_and_timestamp_metadata() {+ let summer = 1_719_792_000_123_456;+ let zones = vec![+ Some("+00:00:01"),+ Some("GMT+1"),+ Some("PST"),+ Some("invalid"),+ None,+ ];+ let zone_arrays: Vec<ArrayRef> = vec![+ Arc::new(StringArray::from(zones.clone())),+ Arc::new(LargeStringArray::from(zones.clone())),+ Arc::new(StringViewArray::from(zones)),+ ];+ for zones in zone_arrays {+ let result = invoke(+ timestamps(vec![Some(-1), Some(0), Some(summer), None, Some(summer)]),+ ColumnarValue::Array(zones),+ )+ .unwrap();+ assert_timestamps(+ result,+ &[+ Some(999_999),+ Some(3_600_000_000),+ Some(summer - 25_200_000_000),+ None,+ None,+ ],+ );+ }+ }++ #[test]+ fn scalar_and_array_arguments_work_across_batches() {+ for values in [vec![Some(-1), None], vec![Some(123_456), Some(-1_000_001)]] {+ let expected = values+ .iter()+ .map(|value| value.map(|value| value + 3_723_000_000))+ .collect::<Vec<_>>();+ let result =+ invoke(timestamps(values), scalar_zone(Some("UTC+1:02:03"))).unwrap();+ assert_timestamps(result, &expected);+ }++ let timestamp = 1_719_792_000_123_456;+ let result = invoke(+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(+ Some(timestamp),+ Some(Arc::from("UTC")),+ )),+ ColumnarValue::Array(Arc::new(StringArray::from(vec![+ Some("GMT+1"),+ Some("PST"),+ None,+ Some("GMT+1"),+ ]))),+ )+ .unwrap();+ assert_timestamps(+ result,+ &[+ Some(timestamp + 3_600_000_000),+ Some(timestamp - 25_200_000_000),+ None,+ Some(timestamp + 3_600_000_000),+ ],+ );+ }++ #[test]+ fn fixed_offset_seconds_preserve_each_timestamp_unit() {+ for (input, expected) in [+ (+ ScalarValue::TimestampSecond(Some(-1), None),+ ScalarValue::TimestampSecond(Some(0), None),+ ),+ (+ ScalarValue::TimestampMillisecond(Some(-1), None),+ ScalarValue::TimestampMillisecond(Some(999), None),+ ),+ (+ ScalarValue::TimestampMicrosecond(Some(-1), None),+ ScalarValue::TimestampMicrosecond(Some(999_999), None),+ ),+ (+ ScalarValue::TimestampNanosecond(Some(-1), None),+ ScalarValue::TimestampNanosecond(Some(999_999_999), None),+ ),+ ] {+ let result = invoke(+ ColumnarValue::Scalar(input.clone()),+ scalar_zone(Some("+00:00:01")),+ )+ .unwrap();+ let ColumnarValue::Scalar(actual) = result else {+ panic!("expected scalar timestamp");+ };+ assert_eq!(actual, expected);++ let expr = SparkFromUtcTimestampExpr::new(+ Arc::new(Literal::new(input)),+ Arc::new(Literal::new(ScalarValue::Utf8(Some(+ "+00:00:01".to_owned(),+ )))),+ );+ let result = expr+ .evaluate(&expression_batch(vec![Some(0)], vec!["0"]))+ .unwrap();+ let ColumnarValue::Scalar(actual) = result else {+ panic!("expected scalar timestamp");+ };+ assert_eq!(actual, expected);+ }+ }++ #[test]+ fn invalid_literal_timezone_is_validated_lazily() {+ let expr = SparkFromUtcTimestampExpr::new(+ Arc::new(Column::new("timestamp", 0)),+ Arc::new(Literal::new(ScalarValue::Utf8(Some("+19:00".to_owned())))),+ );+ assert_timestamps(+ expr.evaluate(&expression_batch(vec![None, None], vec!["0", "0"]))+ .unwrap(),+ &[None, None],+ );+ let error = expr+ .evaluate(&expression_batch(vec![Some(0)], vec!["0"]))+ .unwrap_err();+ assert!(error.to_string().contains("invalid timezone '+19:00'"));+ }++ #[test]+ fn null_timezone_does_not_skip_timestamp_evaluation() {+ let expr = SparkFromUtcTimestampExpr::new(+ throwing_timestamp(),+ Arc::new(Literal::new(ScalarValue::Utf8(None))),+ );+ let error = expr+ .evaluate(&expression_batch(vec![Some(0)], vec!["0"]))+ .unwrap_err();+ assert!(error.to_string().contains("invalid"));+ }++ #[test]+ fn timezone_evaluates_only_non_null_timestamp_rows() {+ let summer = 1_719_792_000_123_456;+ let expr = SparkFromUtcTimestampExpr::new(+ Arc::new(Column::new("timestamp", 0)),+ computed_timezone(),+ );+ let batch = expression_batch(+ vec![None, Some(0), None, Some(summer)],+ vec!["invalid", "0", "invalid", "1"],+ );+ assert_timestamps(+ expr.evaluate(&batch).unwrap(),+ &[None, Some(0), None, Some(summer - 25_200_000_000)],+ );++ // The same timezone child must still throw when its timestamp is non-null.+ let error = expr+ .evaluate(&expression_batch(vec![Some(0)], vec!["invalid"]))+ .unwrap_err();+ assert!(error.to_string().contains("invalid"));+ }++ #[tokio::test]+ async fn planned_udf_requires_explicit_conditional_expression() -> Result<()> {+ let ctx = SessionContext::new();+ ctx.register_udf(+ crate::function::datetime::from_utc_timestamp()+ .as_ref()+ .clone(),+ );+ let batch = RecordBatch::try_from_iter([+ (+ "timestamp_text",+ Arc::new(StringArray::from(vec![+ None,+ Some("1970-01-01T00:00:00"),+ Some("1970-01-01T00:00:00"),+ ])) as ArrayRef,+ ),+ (+ "value",+ Arc::new(StringArray::from(vec!["invalid", "0", "1"])) as ArrayRef,+ ),+ ])?;+ ctx.register_batch("input", batch.clone())?;+ let sql_expr = "from_utc_timestamp(timestamp_text, \+ CASE WHEN CAST(value AS INT) = 0 THEN 'UTC' ELSE 'PST' END)";++ // Registering the UDF leaves ordinary SQL argument evaluation eager.+ let error = ctx+ .sql(&format!("SELECT {sql_expr} FROM input"))+ .await?+ .collect()+ .await+ .unwrap_err();+ assert!(error.to_string().contains("invalid"));++ // Reuse the planner's children, including its implicit string-to-timestamp cast.+ let schema = DFSchema::try_from(batch.schema().as_ref().clone())?;+ let planned =+ ctx.create_physical_expr(ctx.parse_sql_expr(sql_expr, &schema)?, &schema)?;+ let children = planned.children();+ let conditional = SparkFromUtcTimestampExpr::new(+ Arc::clone(children[0]),+ Arc::clone(children[1]),+ );+ assert_eq!(+ conditional.data_type(batch.schema_ref())?,+ DataType::Timestamp(TimeUnit::Microsecond, None)+ );+ assert_eq!(+ fmt_sql(&conditional).to_string(),+ fmt_sql(planned.as_ref()).to_string()+ );+ let result = conditional.evaluate(&batch)?.into_array(batch.num_rows())?;+ let expected =+ TimestampMicrosecondArray::from(vec![None, Some(0), Some(-28_800_000_000)]);+ assert_eq!(result.as_ref(), &expected);++ // The same computed timezone must fail when its timestamp is required.+ let non_null_batch = RecordBatch::try_new(+ batch.schema(),+ vec![+ Arc::new(StringArray::from(vec!["1970-01-01T00:00:00"; 3])),+ Arc::clone(batch.column(1)),+ ],+ )?;+ let error = conditional.evaluate(&non_null_batch).unwrap_err();+ assert!(error.to_string().contains("invalid"));+ Ok(())+ }++ #[test]+ fn timezone_column_ignores_values_on_null_timestamp_rows() {+ let summer = 1_719_792_000_123_456;+ let expr = SparkFromUtcTimestampExpr::new(+ Arc::new(Column::new("timestamp", 0)),+ Arc::new(Column::new("value", 1)),+ );+ let batch = expression_batch(+ vec![None, Some(0), None, Some(summer)],+ vec!["invalid", "UTC", "+19:00", "PST"],+ );+ assert_timestamps(+ expr.evaluate(&batch).unwrap(),+ &[None, Some(0), None, Some(summer - 25_200_000_000)],+ );++ let error = expr+ .evaluate(&expression_batch(vec![Some(0)], vec!["+19:00"]))+ .unwrap_err();+ assert!(error.to_string().contains("invalid timezone '+19:00'"));+ }++ #[test]+ fn timezone_skips_all_null_and_scalar_null_timestamps() {+ let batch = expression_batch(vec![None, None], vec!["invalid", "invalid"]);+ let expr = SparkFromUtcTimestampExpr::new(+ Arc::new(Column::new("timestamp", 0)),+ computed_timezone(),+ );+ assert_timestamps(expr.evaluate(&batch).unwrap(), &[None, None]);++ let expr = SparkFromUtcTimestampExpr::new(+ Arc::new(Literal::new(ScalarValue::TimestampMicrosecond(None, None))),+ computed_timezone(),+ );+ assert!(matches!(+ expr.evaluate(&batch).unwrap(),+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None))+ ));+ }++ #[test]+ fn empty_batch_skips_children() {+ let batch = expression_batch(vec![], vec![]);+ let expr =+ SparkFromUtcTimestampExpr::new(throwing_timestamp(), computed_timezone());+ assert_timestamps(expr.evaluate(&batch).unwrap(), &[]);+ }++ #[test]+ fn fixed_offset_outside_calendar_range_returns_error() {+ for (timestamp, timezone) in [+ (DateTime::<Utc>::MAX_UTC.timestamp_micros(), "+00:00:01"),+ (DateTime::<Utc>::MIN_UTC.timestamp_micros(), "-00:00:01"),+ ] {+ let error = invoke(+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(+ Some(timestamp),+ None,+ )),+ scalar_zone(Some(timezone)),+ )+ .unwrap_err();+ assert!(+ error+ .to_string()+ .contains("exceeds Chrono's calendar range")+ );+ }+ }++ #[test]+ fn invalid_zones_are_parsed_only_for_non_null_pairs() {+ let zones =+ ColumnarValue::Array(Arc::new(StringArray::from(vec!["invalid", "+19:00"])));+ assert_timestamps(+ invoke(timestamps(vec![None, None]), zones).unwrap(),+ &[None, None],+ );++ let error = invoke(timestamps(vec![None, Some(0)]), scalar_zone(Some("+19:00")))+ .unwrap_err();+ assert!(error.to_string().contains("invalid timezone '+19:00'"));++ let result = invoke(+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None)),+ scalar_zone(Some("invalid")),+ )+ .unwrap();+ assert!(matches!(+ result,+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None))+ ));++ let result = invoke(+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(+ Some(i64::MAX),+ None,+ )),+ scalar_zone(None),+ )+ .unwrap();+ assert!(matches!(+ result,+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None))+ ));++ assert_timestamps(+ invoke(timestamps(vec![]), scalar_zone(Some("invalid"))).unwrap(),+ &[],+ );+ }+}diff --git a/datafusion/spark/src/function/datetime/mod.rs b/datafusion/spark/src/function/datetime/mod.rsindex 70ab024c32..a365fbe4a9 100644--- a/datafusion/spark/src/function/datetime/mod.rs+++ b/datafusion/spark/src/function/datetime/mod.rs@@ -29,6 +29,7 @@ pub mod make_interval;pub mod monthname;pub mod next_day;pub mod time_trunc;+mod timezone;pub mod to_utc_timestamp;pub mod trunc;pub mod unix;diff --git a/datafusion/spark/src/function/datetime/timezone.rs b/datafusion/spark/src/function/datetime/timezone.rsnew file mode 100644index 0000000000..17971c85ea--- /dev/null+++ b/datafusion/spark/src/function/datetime/timezone.rs@@ -0,0 +1,442 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::timezone::Tz;+use arrow::datatypes::{ArrowTimestampType, TimeUnit};+use chrono::{DateTime, Datelike, FixedOffset, NaiveDate, Utc};+use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err};+use datafusion_functions::datetime::to_local_time::adjust_to_local_time;++/// A timezone resolved using Spark's `getZoneId` spelling rules.+///+/// Fixed offsets are kept as seconds to preserve Spark's second-resolution offsets.+/// Region IDs use Arrow's Chrono-backed timezone database.+/// Java's legacy SystemV zones require their own rules because that database omits them.+#[derive(Debug, Clone, Copy)]+pub(super) enum SparkTimeZone {+ FixedOffset(i32),+ Region(Tz),+ SystemV(i32),+}++impl SparkTimeZone {+ pub(super) fn parse(timezone: &str) -> Result<Self> {+ // These are the exact aliases in java.time.ZoneId.SHORT_IDS. The regional+ // aliases retain their daylight saving rules; EST, HST, and MST are fixed.+ let timezone = match timezone {+ "Z" => return Ok(Self::FixedOffset(0)),+ // The JDK explicitly excludes this legacy IANA alias.+ "ROC" => return Err(exec_datafusion_err!("Unknown timezone '{timezone}'")),+ "EST" => return Ok(Self::FixedOffset(-5 * 3600)),+ "HST" => return Ok(Self::FixedOffset(-10 * 3600)),+ "MST" => return Ok(Self::FixedOffset(-7 * 3600)),+ "ACT" => "Australia/Darwin",+ "AET" => "Australia/Sydney",+ "AGT" => "America/Argentina/Buenos_Aires",+ "ART" => "Africa/Cairo",+ "AST" => "America/Anchorage",+ "BET" => "America/Sao_Paulo",+ "BST" => "Asia/Dhaka",+ "CAT" => "Africa/Harare",+ "CNT" => "America/St_Johns",+ "CST" => "America/Chicago",+ "CTT" => "Asia/Shanghai",+ "EAT" => "Africa/Addis_Ababa",+ "ECT" => "Europe/Paris",+ "IET" => "America/Indiana/Indianapolis",+ "IST" => "Asia/Kolkata",+ "JST" => "Asia/Tokyo",+ "MIT" => "Pacific/Apia",+ "NET" => "Asia/Yerevan",+ "NST" => "Pacific/Auckland",+ "PLT" => "Asia/Karachi",+ "PNT" => "America/Phoenix",+ "PRT" => "America/Puerto_Rico",+ "PST" => "America/Los_Angeles",+ "SST" => "Pacific/Guadalcanal",+ "VST" => "Asia/Ho_Chi_Minh",+ "SystemV/AST4" => return Ok(Self::FixedOffset(-4 * 3600)),+ "SystemV/EST5" => return Ok(Self::FixedOffset(-5 * 3600)),+ "SystemV/CST6" => return Ok(Self::FixedOffset(-6 * 3600)),+ "SystemV/MST7" => return Ok(Self::FixedOffset(-7 * 3600)),+ "SystemV/PST8" => return Ok(Self::FixedOffset(-8 * 3600)),+ "SystemV/YST9" => return Ok(Self::FixedOffset(-9 * 3600)),+ "SystemV/HST10" => return Ok(Self::FixedOffset(-10 * 3600)),+ "SystemV/AST4ADT" => return Ok(Self::SystemV(-4 * 3600)),+ "SystemV/EST5EDT" => return Ok(Self::SystemV(-5 * 3600)),+ "SystemV/CST6CDT" => return Ok(Self::SystemV(-6 * 3600)),+ "SystemV/MST7MDT" => return Ok(Self::SystemV(-7 * 3600)),+ "SystemV/PST8PDT" => return Ok(Self::SystemV(-8 * 3600)),+ "SystemV/YST9YDT" => return Ok(Self::SystemV(-9 * 3600)),+ _ => timezone,+ };++ let offset = if timezone.starts_with(['+', '-']) {+ Some(timezone)+ } else {+ ["UTC", "GMT", "UT"].into_iter().find_map(|prefix| {+ timezone+ .strip_prefix(prefix)+ .filter(|suffix| suffix.is_empty() || suffix.starts_with(['+', '-']))+ })+ };+ if let Some(offset) = offset {+ if offset.is_empty() {+ return Ok(Self::FixedOffset(0));+ }+ return parse_offset(offset).map(Self::FixedOffset).ok_or_else(|| {+ exec_datafusion_err!("Invalid timezone offset '{offset}'")+ });+ }++ Ok(Self::Region(timezone.parse()?))+ }++ pub(super) fn adjust_to_local_time<T: ArrowTimestampType>(+ self,+ ts: i64,+ ) -> Result<i64> {+ match self {+ Self::Region(timezone) => adjust_to_local_time::<T>(ts, timezone),+ Self::FixedOffset(seconds) => adjust_by_offset::<T>(ts, seconds),+ Self::SystemV(standard_offset) => {+ adjust_by_offset::<T>(ts, system_v_offset::<T>(ts, standard_offset)?)+ }+ }+ }+}++/// Parses Java's signed offsets after Spark's single-digit hour/minute normalization.+///+/// Spark permits one-digit hours before a colon, and one-digit minutes only when+/// minutes are the final component. Java also accepts compact HHMM and HHMMSS forms.+fn parse_offset(offset: &str) -> Option<i32> {+ let sign = match offset.as_bytes().first()? {+ b'+' => 1,+ b'-' => -1,+ _ => return None,+ };+ let value = &offset[1..];+ let (hours, minutes, seconds) = if let Some((hours, rest)) = value.split_once(':') {+ let hours = parse_digits(hours, 1, 2)?;+ if let Some((minutes, seconds)) = rest.split_once(':') {+ (+ hours,+ parse_digits(minutes, 2, 2)?,+ parse_digits(seconds, 2, 2)?,+ )+ } else {+ (hours, parse_digits(rest, 1, 2)?, 0)+ }+ } else {+ // Check ASCII before byte slicing so malformed Unicode inputs cannot panic.+ if !value.bytes().all(|byte| byte.is_ascii_digit()) {+ return None;+ }+ match value.len() {+ 1 | 2 => (parse_digits(value, 1, 2)?, 0, 0),+ 4 => (+ parse_digits(&value[..2], 2, 2)?,+ parse_digits(&value[2..], 2, 2)?,+ 0,+ ),+ 6 => (+ parse_digits(&value[..2], 2, 2)?,+ parse_digits(&value[2..4], 2, 2)?,+ parse_digits(&value[4..], 2, 2)?,+ ),+ _ => return None,+ }+ };+ if hours > 18+ || minutes > 59+ || seconds > 59+ || (hours == 18 && (minutes != 0 || seconds != 0))+ {+ return None;+ }+ Some(sign * (hours * 3600 + minutes * 60 + seconds))+}++fn parse_digits(value: &str, min_len: usize, max_len: usize) -> Option<i32> {+ if !(min_len..=max_len).contains(&value.len())+ || !value.bytes().all(|byte| byte.is_ascii_digit())+ {+ return None;+ }+ value.parse().ok()+}++fn units_per_second<T: ArrowTimestampType>() -> i64 {+ match T::UNIT {+ TimeUnit::Second => 1,+ TimeUnit::Millisecond => 1_000,+ TimeUnit::Microsecond => 1_000_000,+ TimeUnit::Nanosecond => 1_000_000_000,+ }+}++fn adjust_by_offset<T: ArrowTimestampType>(ts: i64, offset_seconds: i32) -> Result<i64> {+ let timezone = FixedOffset::east_opt(offset_seconds).ok_or_else(|| {+ internal_datafusion_err!("Spark timezone offsets must be within 18 hours")+ })?;+ adjust_to_local_time::<T>(ts, timezone)+}++/// Java's SystemV daylight zones use the same calendar with different standard offsets.+/// The JDK starts these transitions in 1900. Most years change on the last Sundays of+/// April and October; 1974 and 1975 have exceptional start/end dates.+/// The rule definitions are in OpenJDK's `make/data/tzdata/jdk11_backward`.+fn system_v_offset<T: ArrowTimestampType>(ts: i64, standard_offset: i32) -> Result<i32> {+ let seconds = ts.div_euclid(units_per_second::<T>());+ let year = DateTime::<Utc>::from_timestamp(seconds, 0)+ .ok_or_else(|| {+ exec_datafusion_err!("Timestamp is outside Chrono's calendar range")+ })?+ .year();+ if year < 1900 {+ return Ok(standard_offset);+ }++ let start = match year {+ 1974 => NaiveDate::from_ymd_opt(year, 1, 6),+ 1975 => NaiveDate::from_ymd_opt(year, 2, 23),+ _ => last_sunday(year, 4, 30),+ };+ let end = if year == 1974 {+ last_sunday(year, 11, 30)+ } else {+ last_sunday(year, 10, 31)+ };+ let transition = |date: Option<NaiveDate>, offset_before: i32| {+ date.and_then(|date| date.and_hms_opt(2, 0, 0))+ .map(|local| local.and_utc().timestamp() - i64::from(offset_before))+ .ok_or_else(|| {+ exec_datafusion_err!("Timestamp is outside Chrono's calendar range")+ })+ };+ let start = transition(start, standard_offset)?;+ let end = transition(end, standard_offset + 3600)?;+ Ok(if (start..end).contains(&seconds) {+ standard_offset + 3600+ } else {+ standard_offset+ })+}++fn last_sunday(year: i32, month: u32, last_day: u32) -> Option<NaiveDate> {+ let date = NaiveDate::from_ymd_opt(year, month, last_day)?;+ date.with_day(last_day - date.weekday().num_days_from_sunday())+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::datatypes::TimestampMicrosecondType;++ fn shift(timezone: &str, timestamp: i64) -> i64 {+ SparkTimeZone::parse(timezone)+ .unwrap()+ .adjust_to_local_time::<TimestampMicrosecondType>(timestamp)+ .unwrap()+ }++ #[test]+ fn spark_offset_spellings_preserve_sign_and_seconds() {+ let offsets = [+ ("+0", 0),+ ("-00", 0),+ ("+1", 3600),+ ("-01", -3600),+ ("+0102", 3720),+ ("-01:02", -3720),+ ("+1:02", 3720),+ ("+01:2", 3720),+ ("+1:2", 3720),+ ("+010203", 3723),+ ("-01:02:03", -3723),+ ("+1:02:03", 3723),+ ("+00:00:01", 1),+ ("-000001", -1),+ ("+18:00", 64800),+ ("-180000", -64800),+ ];+ for prefix in ["", "UTC", "GMT", "UT"] {+ for (offset, seconds) in offsets {+ let timezone = format!("{prefix}{offset}");+ assert_eq!(+ shift(&timezone, 123_456),+ 123_456 + i64::from(seconds) * 1_000_000,+ "{timezone}"+ );+ }+ }+ for timezone in ["Z", "UTC", "GMT", "UT", "GMT0", "Etc/UTC"] {+ assert_eq!(shift(timezone, -123_456), -123_456, "{timezone}");+ }+ assert_eq!(shift("GMT+1", 0), 3_600_000_000);+ assert_eq!(shift("Etc/GMT+1", 0), -3_600_000_000);+ }++ #[test]+ fn malformed_offsets_and_unknown_ids_are_rejected() {+ for prefix in ["", "UTC", "GMT", "UT"] {+ for offset in [+ "+",+ "-",+ "+19",+ "-18:00:01",+ "+18:01",+ "+01:60",+ "+01:99",+ "+01:00:60",+ "+1:2:03",+ "+01:02:3",+ "+123",+ "+12345",+ "+001:00",+ "+01:",+ "+:01",+ "+01::00",+ "+01:00:00:00",+ "+01:00 ",+ "+01",+ "+01:٠0",+ "++01",+ "--01",+ ] {+ let timezone = format!("{prefix}{offset}");+ assert!(SparkTimeZone::parse(&timezone).is_err(), "{timezone}");+ }+ }+ for timezone in [+ "",+ " ",+ "utc",+ "pst",+ "PDT",+ "ROC",+ "America/Unknown",+ "SystemV/PST9PDT",+ " UTC",+ ] {+ assert!(SparkTimeZone::parse(timezone).is_err(), "{timezone}");+ }+ }++ #[test]+ fn java_short_ids_match_their_regions_in_winter_and_summer() {+ let aliases = [+ ("ACT", "Australia/Darwin"),+ ("AET", "Australia/Sydney"),+ ("AGT", "America/Argentina/Buenos_Aires"),+ ("ART", "Africa/Cairo"),+ ("AST", "America/Anchorage"),+ ("BET", "America/Sao_Paulo"),+ ("BST", "Asia/Dhaka"),+ ("CAT", "Africa/Harare"),+ ("CNT", "America/St_Johns"),+ ("CST", "America/Chicago"),+ ("CTT", "Asia/Shanghai"),+ ("EAT", "Africa/Addis_Ababa"),+ ("ECT", "Europe/Paris"),+ ("IET", "America/Indiana/Indianapolis"),+ ("IST", "Asia/Kolkata"),+ ("JST", "Asia/Tokyo"),+ ("MIT", "Pacific/Apia"),+ ("NET", "Asia/Yerevan"),+ ("NST", "Pacific/Auckland"),+ ("PLT", "Asia/Karachi"),+ ("PNT", "America/Phoenix"),+ ("PRT", "America/Puerto_Rico"),+ ("PST", "America/Los_Angeles"),+ ("SST", "Pacific/Guadalcanal"),+ ("VST", "Asia/Ho_Chi_Minh"),+ ("EST", "-05:00"),+ ("HST", "-10:00"),+ ("MST", "-07:00"),+ ];+ for (alias, canonical) in aliases {+ for timestamp in [1_704_067_200_123_456, 1_719_792_000_123_456] {+ assert_eq!(+ shift(alias, timestamp),+ shift(canonical, timestamp),+ "{alias}"+ );+ }+ }+ }++ #[test]+ fn system_v_zones_match_java_transition_boundaries() {+ let zones = [+ ("SystemV/AST4ADT", -4),+ ("SystemV/EST5EDT", -5),+ ("SystemV/CST6CDT", -6),+ ("SystemV/MST7MDT", -7),+ ("SystemV/PST8PDT", -8),+ ("SystemV/YST9YDT", -9),+ ];+ for (timezone, hours) in zones {+ let standard = hours * 3600;+ for (local, offset_before, offset_after) in [+ ("1900-04-29T02:00:00Z", standard, standard + 3600),+ ("1974-01-06T02:00:00Z", standard, standard + 3600),+ ("1974-11-24T02:00:00Z", standard + 3600, standard),+ ("1975-02-23T02:00:00Z", standard, standard + 3600),+ ("2024-04-28T02:00:00Z", standard, standard + 3600),+ ("2024-10-27T02:00:00Z", standard + 3600, standard),+ ] {+ let transition = DateTime::parse_from_rfc3339(local)+ .unwrap()+ .timestamp_micros()+ - i64::from(offset_before) * 1_000_000;+ assert_eq!(+ shift(timezone, transition - 1),+ transition - 1 + i64::from(offset_before) * 1_000_000,+ "{timezone} before {local}"+ );+ assert_eq!(+ shift(timezone, transition),+ transition + i64::from(offset_after) * 1_000_000,+ "{timezone} at {local}"+ );+ }+ let before_transitions = DateTime::parse_from_rfc3339("1899-07-01T00:00:00Z")+ .unwrap()+ .timestamp_micros();+ assert_eq!(+ shift(timezone, before_transitions),+ before_transitions + i64::from(standard) * 1_000_000+ );+ }+ for (timezone, hours) in [+ ("SystemV/AST4", -4),+ ("SystemV/EST5", -5),+ ("SystemV/CST6", -6),+ ("SystemV/MST7", -7),+ ("SystemV/PST8", -8),+ ("SystemV/YST9", -9),+ ("SystemV/HST10", -10),+ ] {+ assert_eq!(+ shift(timezone, 1_719_792_000_000_000),+ 1_719_792_000_000_000 + i64::from(hours) * 3_600_000_000+ );+ }+ }+}diff --git a/datafusion/sqllogictest/test_files/spark/datetime/from_utc_timestamp.slt b/datafusion/sqllogictest/test_files/spark/datetime/from_utc_timestamp.sltindex 8200fccfc1..8b04956cb3 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/from_utc_timestamp.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/from_utc_timestamp.slt@@ -154,3 +154,28 @@ query PSELECT from_utc_timestamp('2020-11-04T14:06:40'::timestamp, 'America/New_York'::string);----2020-11-04T09:06:40++# Spark timezone spellings supplied as column values, including second-resolution offsets.+query P+SELECT from_utc_timestamp(column1, column2)+FROM VALUES+('2024-07-01T00:00:00.123456'::timestamp, 'PST'::string),+('2024-07-01T00:00:00.123456'::timestamp, 'GMT+1'::string),+('2024-07-01T00:00:00.123456'::timestamp, 'UTC+1:2'::string),+('2024-07-01T00:00:00.123456'::timestamp, 'UT-010203'::string),+('2024-04-15T00:00:00.123456'::timestamp, 'SystemV/PST8PDT'::string),+('2024-07-01T00:00:00.123456'::timestamp, 'SystemV/HST10'::string),+(NULL::timestamp, '+19:00'::string);+----+2024-06-30T17:00:00.123456+2024-07-01T01:00:00.123456+2024-07-01T01:02:00.123456+2024-06-30T22:57:57.123456+2024-04-14T16:00:00.123456+2024-06-30T14:00:00.123456+NULL++# Arrow accepts this offset, but Spark limits offsets to 18 hours.+query error invalid timezone '\+19:00'+SELECT from_utc_timestamp(column1, column2)+FROM VALUES ('2024-07-01'::timestamp, '+19:00'::string);diff --git a/typos.toml b/typos.tomlindex 196766f12f..6a958616a9 100644--- a/typos.toml+++ b/typos.toml@@ -17,6 +17,7 @@ Ois = "Ois"alo = "alo"# abbreviations, common words, etc.+IST = "IST"typ = "typ"datas = "datas"YOUY = "YOUY"