Apache DataFusion SQL Query Engine
fix: `from_unixtime` should respect `datafusion.execution.time_zone` (#25161)
Adrian Garcia Badaraccof8cc678272c5
diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rsindex 85494f3abf..8af4553f5c 100644--- a/datafusion/functions/src/datetime/from_unixtime.rs+++ b/datafusion/functions/src/datetime/from_unixtime.rs@@ -17,21 +17,34 @@use std::sync::Arc;+use arrow::array::AsArray;+use arrow::array::timezone::Tz;+use arrow::compute::kernels::aggregate::{max, min};use arrow::datatypes::DataType::{Int64, Timestamp, Utf8};+use arrow::datatypes::Int64Type;use arrow::datatypes::TimeUnit::Second;use arrow::datatypes::{DataType, Field, FieldRef};+use chrono::{DateTime, NaiveDateTime, Offset, TimeDelta, TimeZone};+use datafusion_common::config::ConfigOptions;use datafusion_common::{Result, ScalarValue, exec_err, internal_err};use datafusion_expr::TypeSignature::Exact;use datafusion_expr::sort_properties::{ExprProperties, SortProperties};use datafusion_expr::{- ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,- Signature, Volatility,+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,+ ScalarUDFImpl, Signature, Volatility,};use datafusion_macros::user_doc;#[user_doc(doc_section(label = "Time and Date Functions"),- description = "Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`) return the corresponding timestamp.",+ description = r#"+Converts an integer to a timestamp with second precision (`Timestamp(Second)`).+The integer is interpreted as the number of seconds since the unix epoch+(`1970-01-01T00:00:00Z`).++If the optional `timezone` argument is omitted, the timestamp is returned in the+session time zone (`datafusion.execution.time_zone`), which is unset (i.e.+timezone-naive) by default."#,syntax_example = "from_unixtime(expression[, timezone])",sql_example = r#"```sql> select from_unixtime(1599572549, 'America/New_York');@@ -40,31 +53,59 @@ use datafusion_macros::user_doc;+-----------------------------------------------------------+| 2020-09-08T09:42:29-04:00 |+-----------------------------------------------------------+++-- Without an explicit timezone the session time zone is used+> SET datafusion.execution.time_zone = 'America/New_York';+> select from_unixtime(1599572549);++----------------------------------++| from_unixtime(Int64(1599572549)) |++----------------------------------++| 2020-09-08T09:42:29-04:00 |++----------------------------------+```"#,standard_argument(name = "expression",),argument(name = "timezone",- description = "Optional timezone to use when converting the integer to a timestamp. If not provided, the default timezone is UTC."+ description = "Optional timezone to use when converting the integer to a timestamp. If not provided, the session time zone (`datafusion.execution.time_zone`) is used, which is unset (timezone-naive) by default."))]#[derive(Debug, PartialEq, Eq, Hash)]pub struct FromUnixtimeFunc {signature: Signature,+ /// Timezone from `datafusion.execution.time_zone`, used by the+ /// single-argument form. The two-argument form always uses the timezone+ /// given explicitly as the second argument.+ timezone: Option<Arc<str>>,}impl Default for FromUnixtimeFunc {fn default() -> Self {- Self::new()+ Self::new_with_config(&ConfigOptions::default())}}impl FromUnixtimeFunc {+ #[deprecated(since = "56.0.0", note = "use `new_with_config` instead")]+ /// Deprecated constructor retained for backwards compatibility.+ ///+ /// Prefer [`FromUnixtimeFunc::new_with_config`], which picks up the session+ /// time zone from [`ConfigOptions`]. This helper mirrors the canonical+ /// default (no timezone) provided by `ConfigOptions::default()`.pub fn new() -> Self {+ Self::new_with_config(&ConfigOptions::default())+ }++ pub fn new_with_config(config: &ConfigOptions) -> Self {Self {signature: Signature::one_of(vec![Exact(vec![Int64, Utf8]), Exact(vec![Int64])],Volatility::Immutable,),+ timezone: config+ .execution+ .time_zone+ .as_ref()+ .map(|tz| Arc::from(tz.as_str())),}}}@@ -78,12 +119,19 @@ impl ScalarUDFImpl for FromUnixtimeFunc {&self.signature}+ fn with_updated_config(&self, config: &ConfigOptions) -> Option<ScalarUDF> {+ Some(Self::new_with_config(config).into())+ }+fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {// Length check handled in the signaturedebug_assert!(matches!(args.scalar_arguments.len(), 1 | 2));if args.scalar_arguments.len() == 1 {- Ok(Field::new(self.name(), Timestamp(Second, None), true).into())+ Ok(+ Field::new(self.name(), Timestamp(Second, self.timezone.clone()), true)+ .into(),+ )} else {args.scalar_arguments[1].and_then(|sv| {@@ -132,20 +180,25 @@ impl ScalarUDFImpl for FromUnixtimeFunc {);}- match len {- 1 => args[0].cast_to(&Timestamp(Second, None), None),+ let timezone = match len {+ 1 => self.timezone.clone(),2 => match &args[1] {- ColumnarValue::Scalar(ScalarValue::Utf8(Some(tz))) => args[0]- .cast_to(&Timestamp(Second, Some(Arc::from(tz.to_string()))), None),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(tz))) => {+ Some(Arc::<str>::from(tz.as_str()))+ }_ => {- exec_err!(+ return exec_err!("Unsupported data type {} for function from_unixtime",args[1].data_type()- )+ );}},_ => unreachable!(),- }+ };++ validate_local_datetime_range(&args[0], timezone.as_deref())?;++ args[0].cast_to(&Timestamp(Second, timezone), None)}fn output_ordering(&self, inputs: &[ExprProperties]) -> Result<SortProperties> {@@ -171,17 +224,317 @@ impl ScalarUDFImpl for FromUnixtimeFunc {}}+/// An upper bound on the magnitude of any time zone's UTC offset, rounded up to+/// a whole day. Real offsets never exceed 15 hours (including the historical+/// local mean time offsets used before a zone's first transition).+const MAX_TIMEZONE_OFFSET_SECONDS: i64 = 24 * 60 * 60;++/// The range of `Timestamp(Second)` values whose local date and time is+/// representable in *every* time zone.+///+/// Values inside this range never need a per-value check: shifting them by any+/// possible UTC offset keeps them inside [`NaiveDateTime`]'s range.+fn always_representable_range() -> (i64, i64) {+ (+ NaiveDateTime::MIN.and_utc().timestamp() + MAX_TIMEZONE_OFFSET_SECONDS,+ NaiveDateTime::MAX.and_utc().timestamp() - MAX_TIMEZONE_OFFSET_SECONDS,+ )+}++/// Returns an error if any value in `values` has no representable local date+/// and time in `timezone`.+///+/// A `Timestamp(Second, Some(tz))` is *rendered* by shifting the UTC instant by+/// the zone's offset. Arrow does that with `DateTime::naive_local`, which+/// **panics** when the shifted value falls outside [`NaiveDateTime`]'s range,+/// so such values have to be rejected before they are produced: the panic would+/// otherwise happen far away from `from_unixtime`, when the value is formatted.+///+/// See <https://github.com/apache/datafusion/issues/16594>.+fn validate_local_datetime_range(+ values: &ColumnarValue,+ timezone: Option<&str>,+) -> Result<()> {+ // A timezone naive timestamp is never shifted, so it can only be out of+ // range if the UTC instant itself is, which the cast already rejects.+ let Some(timezone) = timezone else {+ return Ok(());+ };++ let (safe_min, safe_max) = always_representable_range();++ let (value_min, value_max) = match values {+ ColumnarValue::Scalar(ScalarValue::Int64(Some(value))) => (*value, *value),+ // NULL (and anything else the cast will reject) needs no check.+ ColumnarValue::Scalar(_) => return Ok(()),+ ColumnarValue::Array(array) => {+ let array = array.as_primitive::<Int64Type>();+ match (min(array), max(array)) {+ (Some(value_min), Some(value_max)) => (value_min, value_max),+ // All null.+ _ => return Ok(()),+ }+ }+ };++ // Fast path: no value can overflow, whatever the zone's offset is.+ if value_min >= safe_min && value_max <= safe_max {+ return Ok(());+ }++ let Ok(tz) = timezone.parse::<Tz>() else {+ // An unparsable time zone is reported by the cast below.+ return Ok(());+ };++ let check = |seconds: i64| -> Result<()> {+ if seconds >= safe_min && seconds <= safe_max {+ return Ok(());+ }+ let Some(utc) = DateTime::from_timestamp(seconds, 0) else {+ // Not representable as an instant at all: the cast reports this.+ return Ok(());+ };+ let utc = utc.naive_utc();+ let offset = tz.offset_from_utc_datetime(&utc).fix().local_minus_utc();+ if utc+ .checked_add_signed(TimeDelta::seconds(i64::from(offset)))+ .is_none()+ {+ return exec_err!(+ "Cannot convert {seconds} to a timestamp in timezone \"{timezone}\" \+ for function from_unixtime: the local date and time is outside the \+ supported range"+ );+ }+ Ok(())+ };++ match values {+ ColumnarValue::Scalar(_) => check(value_min),+ ColumnarValue::Array(array) => {+ // Only the values near the limits do any real work; `check` returns+ // immediately for everything inside the always representable range.+ for value in array.as_primitive::<Int64Type>().iter().flatten() {+ check(value)?;+ }+ Ok(())+ }+ }+}+#[cfg(test)]mod test {use crate::datetime::from_unixtime::FromUnixtimeFunc;+ use arrow::array::{ArrayRef, Int64Array};use arrow::datatypes::TimeUnit::Second;- use arrow::datatypes::{DataType, Field};+ use arrow::datatypes::{DataType, Field, FieldRef};+ use arrow::util::display::{ArrayFormatter, FormatOptions};+ use datafusion_common::Result;use datafusion_common::ScalarValue;use datafusion_common::ScalarValue::Int64;use datafusion_common::config::ConfigOptions;- use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};+ use datafusion_expr::{+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,+ };use std::sync::Arc;+ /// The last second since the epoch that `chrono::NaiveDateTime` can+ /// represent (`+262142-12-31T23:59:59`).+ const MAX_UTC_SECONDS: i64 = 8210266876799;+ /// The first second since the epoch that `chrono::NaiveDateTime` can+ /// represent (`-262143-01-01T00:00:00`).+ const MIN_UTC_SECONDS: i64 = -8334601228800;++ /// Invoke the single argument form with `datafusion.execution.time_zone`+ /// set to `timezone`.+ fn from_unixtime_session_tz(+ seconds: i64,+ timezone: Option<&str>,+ ) -> Result<ColumnarValue> {+ let mut options = ConfigOptions::default();+ options.execution.time_zone = timezone.map(str::to_string);+ let func = FromUnixtimeFunc::new_with_config(&options);++ let arg_field: FieldRef = Field::new("a", DataType::Int64, true).into();+ let return_field = Field::new(+ "f",+ DataType::Timestamp(Second, timezone.map(Arc::from)),+ true,+ )+ .into();+ func.invoke_with_args(ScalarFunctionArgs {+ args: vec![ColumnarValue::Scalar(Int64(Some(seconds)))],+ arg_fields: vec![arg_field],+ number_rows: 1,+ return_field,+ config_options: Arc::new(options),+ })+ }++ /// Invoke the two argument form with an explicit `timezone`.+ fn from_unixtime_explicit_tz(+ values: ColumnarValue,+ timezone: &str,+ ) -> Result<ColumnarValue> {+ let number_rows = match &values {+ ColumnarValue::Array(array) => array.len(),+ ColumnarValue::Scalar(_) => 1,+ };+ let arg_fields: Vec<FieldRef> = vec![+ Field::new("a", DataType::Int64, true).into(),+ Field::new("b", DataType::Utf8, true).into(),+ ];+ let return_field = Field::new(+ "f",+ DataType::Timestamp(Second, Some(Arc::from(timezone))),+ true,+ )+ .into();+ FromUnixtimeFunc::default().invoke_with_args(ScalarFunctionArgs {+ args: vec![+ values,+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(timezone.to_string()))),+ ],+ arg_fields,+ number_rows,+ return_field,+ config_options: Arc::new(ConfigOptions::default()),+ })+ }++ /// Render the result the way a client would.+ ///+ /// The out of range local date and time panics when the value is+ /// *formatted*, not when it is produced, so every bounds test has to go+ /// through a formatter to be meaningful.+ fn display(value: ColumnarValue) -> String {+ let array = value.to_array(1).unwrap();+ let options = FormatOptions::default();+ let formatter = ArrayFormatter::try_new(array.as_ref(), &options).unwrap();+ formatter.value(0).to_string()+ }++ /// The single argument form used to silently produce values that panicked+ /// downstream once a session time zone was set.+ ///+ /// `America/New_York` is at -04:56:02 (local mean time) that far in the+ /// past, so the local date and time runs out one offset earlier than the+ /// UTC instant does.+ #[test]+ fn test_named_timezone_lower_bound() {+ let last_ok = MIN_UTC_SECONDS + 17762;+ assert_eq!(last_ok, -8334601211038);++ let value = from_unixtime_session_tz(last_ok, Some("America/New_York")).unwrap();+ assert_eq!(display(value), "-262143-01-01T00:00:00-04:56");++ let err = from_unixtime_session_tz(last_ok - 1, Some("America/New_York"))+ .expect_err("expected an out of range error, not a value that panics");+ assert!(+ err.message().contains("outside the supported range"),+ "unexpected error: {err}"+ );+ }++ /// The same bound in the other direction: a zone that is *ahead* of UTC+ /// runs out of local date and time before the UTC instant does.+ ///+ /// This is the two argument form, which panicked on `main` as well+ /// (<https://github.com/apache/datafusion/issues/16594>).+ #[test]+ fn test_named_timezone_upper_bound() {+ // +09:00, with no daylight saving.+ let last_ok = MAX_UTC_SECONDS - 32400;++ let value = from_unixtime_explicit_tz(+ ColumnarValue::Scalar(Int64(Some(last_ok))),+ "Asia/Tokyo",+ )+ .unwrap();+ assert_eq!(display(value), "+262142-12-31T23:59:59+09:00");++ let err = from_unixtime_explicit_tz(+ ColumnarValue::Scalar(Int64(Some(last_ok + 1))),+ "Asia/Tokyo",+ )+ .expect_err("expected an out of range error, not a value that panics");+ assert!(+ err.message().contains("outside the supported range"),+ "unexpected error: {err}"+ );+ }++ /// Fixed offset zones hit exactly the same bound.+ #[test]+ fn test_fixed_offset_upper_bound() {+ let last_ok = MAX_UTC_SECONDS - 28800;++ let value = from_unixtime_session_tz(last_ok, Some("+08:00")).unwrap();+ assert_eq!(display(value), "+262142-12-31T23:59:59+08:00");++ let err = from_unixtime_session_tz(last_ok + 1, Some("+08:00"))+ .expect_err("expected an out of range error, not a value that panics");+ assert!(+ err.message().contains("outside the supported range"),+ "unexpected error: {err}"+ );+ }++ #[test]+ fn test_fixed_offset_lower_bound() {+ let last_ok = MIN_UTC_SECONDS + 28800;++ let value = from_unixtime_explicit_tz(+ ColumnarValue::Scalar(Int64(Some(last_ok))),+ "-08:00",+ )+ .unwrap();+ assert_eq!(display(value), "-262143-01-01T00:00:00-08:00");++ let err = from_unixtime_explicit_tz(+ ColumnarValue::Scalar(Int64(Some(last_ok - 1))),+ "-08:00",+ )+ .expect_err("expected an out of range error, not a value that panics");+ assert!(+ err.message().contains("outside the supported range"),+ "unexpected error: {err}"+ );+ }++ /// Array input is checked too, not just the constant folded scalar case.+ #[test]+ fn test_array_input_is_bounds_checked() {+ let array: ArrayRef =+ Arc::new(Int64Array::from(vec![Some(0), None, Some(MIN_UTC_SECONDS)]));+ let err =+ from_unixtime_explicit_tz(ColumnarValue::Array(array), "America/New_York")+ .expect_err("expected an out of range error, not a value that panics");+ assert!(+ err.message().contains("outside the supported range"),+ "unexpected error: {err}"+ );++ // In range values still work.+ let array: ArrayRef = Arc::new(Int64Array::from(vec![Some(0), None]));+ let value =+ from_unixtime_explicit_tz(ColumnarValue::Array(array), "America/New_York")+ .unwrap();+ assert_eq!(display(value), "1969-12-31T19:00:00-05:00");+ }++ /// Without a time zone the value is never shifted, so the full+ /// `NaiveDateTime` range stays usable, exactly as before this bound check.+ #[test]+ fn test_timezone_naive_extremes_are_still_accepted() {+ let value = from_unixtime_session_tz(MIN_UTC_SECONDS, None).unwrap();+ assert_eq!(display(value), "-262143-01-01T00:00:00");++ let value = from_unixtime_session_tz(MAX_UTC_SECONDS, None).unwrap();+ assert_eq!(display(value), "+262142-12-31T23:59:59");+ }+#[test]fn test_without_timezone() {let arg_field = Arc::new(Field::new("a", DataType::Int64, true));@@ -192,7 +545,7 @@ mod test {return_field: Field::new("f", DataType::Timestamp(Second, None), true).into(),config_options: Arc::new(ConfigOptions::default()),};- let result = FromUnixtimeFunc::new().invoke_with_args(args).unwrap();+ let result = FromUnixtimeFunc::default().invoke_with_args(args).unwrap();match result {ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), None)) => {@@ -202,6 +555,92 @@ mod test {}}+ /// The single argument form reports (and produces) the session time zone+ /// configured via `datafusion.execution.time_zone`.+ #[test]+ fn test_session_timezone_is_used_without_explicit_timezone() {+ let mut options = ConfigOptions::default();+ options.execution.time_zone = Some("America/Denver".to_string());++ let func = FromUnixtimeFunc::new_with_config(&options);++ let arg_field: FieldRef = Field::new("a", DataType::Int64, true).into();+ let scalar_arguments = vec![None];+ let return_field = func+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: std::slice::from_ref(&arg_field),+ scalar_arguments: &scalar_arguments,+ })+ .unwrap();+ assert_eq!(+ return_field.data_type(),+ &DataType::Timestamp(Second, Some(Arc::from("America/Denver")))+ );++ let args = ScalarFunctionArgs {+ args: vec![ColumnarValue::Scalar(Int64(Some(1729900800)))],+ arg_fields: vec![arg_field],+ number_rows: 1,+ return_field,+ config_options: Arc::new(options),+ };+ let result = func.invoke_with_args(args).unwrap();++ match result {+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), Some(tz))) => {+ assert_eq!(sec, 1729900800);+ assert_eq!(tz.as_ref(), "America/Denver");+ }+ other => panic!("Expected timezone aware scalar value, got {other:?}"),+ }+ }++ /// An explicit second argument wins over the session time zone.+ #[test]+ fn test_explicit_timezone_overrides_session_timezone() {+ let mut options = ConfigOptions::default();+ options.execution.time_zone = Some("America/Denver".to_string());++ let func = FromUnixtimeFunc::new_with_config(&options);++ let arg_fields: Vec<FieldRef> = vec![+ Field::new("a", DataType::Int64, true).into(),+ Field::new("b", DataType::Utf8, true).into(),+ ];+ let tz_arg = ScalarValue::Utf8(Some("+08:00".to_string()));+ let scalar_arguments = vec![None, Some(&tz_arg)];+ let return_field = func+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &arg_fields,+ scalar_arguments: &scalar_arguments,+ })+ .unwrap();+ assert_eq!(+ return_field.data_type(),+ &DataType::Timestamp(Second, Some(Arc::from("+08:00")))+ );++ let args = ScalarFunctionArgs {+ args: vec![+ ColumnarValue::Scalar(Int64(Some(1729900800))),+ ColumnarValue::Scalar(tz_arg.clone()),+ ],+ arg_fields,+ number_rows: 1,+ return_field,+ config_options: Arc::new(options),+ };+ let result = func.invoke_with_args(args).unwrap();++ match result {+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), Some(tz))) => {+ assert_eq!(sec, 1729900800);+ assert_eq!(tz.as_ref(), "+08:00");+ }+ other => panic!("Expected timezone aware scalar value, got {other:?}"),+ }+ }+#[test]fn test_with_timezone() {let arg_fields = vec![@@ -225,7 +664,7 @@ mod test {.into(),config_options: Arc::new(ConfigOptions::default()),};- let result = FromUnixtimeFunc::new().invoke_with_args(args).unwrap();+ let result = FromUnixtimeFunc::default().invoke_with_args(args).unwrap();match result {ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(sec), Some(tz))) => {diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rsindex 39b9453295..9246d1d25d 100644--- a/datafusion/functions/src/datetime/mod.rs+++ b/datafusion/functions/src/datetime/mod.rs@@ -47,12 +47,12 @@ make_udf_function!(date_part::DatePartFunc, date_part);make_udf_function!(date_trunc::DateTruncFunc, date_trunc);make_udf_function!(make_date::MakeDateFunc, make_date);make_udf_function!(make_time::MakeTimeFunc, make_time);-make_udf_function!(from_unixtime::FromUnixtimeFunc, from_unixtime);make_udf_function!(to_char::ToCharFunc, to_char);make_udf_function!(to_date::ToDateFunc, to_date);make_udf_function!(to_local_time::ToLocalTimeFunc, to_local_time);make_udf_function!(to_time::ToTimeFunc, to_time);make_udf_function!(to_unixtime::ToUnixtimeFunc, to_unixtime);+make_udf_function_with_config!(from_unixtime::FromUnixtimeFunc, from_unixtime);make_udf_function_with_config!(to_timestamp::ToTimestampFunc, to_timestamp);make_udf_function_with_config!(to_timestamp::ToTimestampSecondsFunc,@@ -79,8 +79,8 @@ pub mod expr_fn {"returns current UTC time as a Time64 value",),(from_unixtime,- "converts an integer to RFC3339 timestamp format string",- unixtime+ "converts an integer of epoch seconds to a second-precision timestamp",+ @config unixtime),(date_bin,"coerces an arbitrary timestamp to the start of the nearest specified interval",@@ -281,7 +281,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {date_bin(),date_part(),date_trunc(),- from_unixtime(),+ from_unixtime(&config),make_date(),make_time(),now(&config),diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 108be43d44..92c9101afb 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -3480,7 +3480,7 @@ mod tests {] {let unparser = Unparser::new(dialect.as_ref());let expr = Expr::ScalarFunction(ScalarFunction {- func: Arc::new(ScalarUDF::from(FromUnixtimeFunc::new())),+ func: Arc::new(ScalarUDF::from(FromUnixtimeFunc::default())),args: vec![col("date_col")],});diff --git a/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt b/datafusion/sqllogictest/test_files/from_unixtime_timezone.sltnew file mode 100644index 0000000000..a295fe48ef--- /dev/null+++ b/datafusion/sqllogictest/test_files/from_unixtime_timezone.slt@@ -0,0 +1,168 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++##########+## from_unixtime timezone tests+##+## https://github.com/apache/datafusion/issues/12892+##########++## Make sure we start from the default (unset) session time zone+statement ok+RESET datafusion.execution.time_zone++## Test 1: session time zone unset -> timezone naive, as before+query TP+SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400);+----+Timestamp(s) 2024-01-01T12:00:00++## Test 2: fixed offset session time zone+statement ok+SET datafusion.execution.time_zone = '+08:00';++query TP+SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400);+----+Timestamp(s, "+08:00") 2024-01-01T20:00:00+08:00++## The reproducer from the issue: to_unixtime/from_unixtime must round trip+query I+SELECT to_unixtime(from_unixtime(1704110400));+----+1704110400++## Test 3: named IANA session time zone+statement ok+SET datafusion.execution.time_zone = 'America/Denver';++query TP+SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400);+----+Timestamp(s, "America/Denver") 2024-01-01T05:00:00-07:00++# January alone (`-07:00`) cannot tell a DST-aware zone from a fixed `-07:00`+# offset. A July instant must render `-06:00`, which only a DST-aware zone does.+query TP+SELECT arrow_typeof(from_unixtime(1719835200)), from_unixtime(1719835200);+----+Timestamp(s, "America/Denver") 2024-07-01T06:00:00-06:00++## Test 4: an explicit timezone argument wins over the session time zone+query TP+SELECT+ arrow_typeof(from_unixtime(1704110400, 'America/New_York')),+ from_unixtime(1704110400, 'America/New_York');+----+Timestamp(s, "America/New_York") 2024-01-01T07:00:00-05:00++## Test 5: array (not constant folded) input carries the session time zone too+query TP+SELECT arrow_typeof(from_unixtime(v)), from_unixtime(v) FROM (VALUES (1704110400), (0)) AS t(v);+----+Timestamp(s, "America/Denver") 2024-01-01T05:00:00-07:00+Timestamp(s, "America/Denver") 1969-12-31T17:00:00-07:00++## Test 6: NULL input stays NULL, but keeps the session time zone in its type+query TP+SELECT arrow_typeof(from_unixtime(NULL)), from_unixtime(NULL);+----+Timestamp(s, "America/Denver") NULL++query TP+SELECT arrow_typeof(from_unixtime(arrow_cast(NULL, 'Int64'), 'America/New_York')), from_unixtime(arrow_cast(NULL, 'Int64'), 'America/New_York');+----+Timestamp(s, "America/New_York") NULL++## Test 7: the underlying instant is unchanged - only the displayed time zone differs+query B+SELECT from_unixtime(1704110400) = from_unixtime(1704110400, 'America/New_York');+----+true++## Test 8: RESET restores the timezone naive behaviour+statement ok+RESET datafusion.execution.time_zone++query TP+SELECT arrow_typeof(from_unixtime(1704110400)), from_unixtime(1704110400);+----+Timestamp(s) 2024-01-01T12:00:00++##########+## Out of range values+##+## A `Timestamp(Second, Some(tz))` is rendered by shifting the UTC instant by+## the zone's offset, and that shifted value has to stay inside the range+## chrono's `NaiveDateTime` can represent, otherwise formatting the value+## panics.+##+## https://github.com/apache/datafusion/issues/16594+##########++## The last value `America/New_York` can represent (its offset is -04:56 that+## far in the past, so the local time runs out before the UTC instant does)+query P+SELECT from_unixtime(-8334601211038, 'America/New_York');+----+-262143-01-01T00:00:00-04:56++## One second earlier is an error, not a panic+query error DataFusion error: Execution error: Cannot convert \-8334601211039 to a timestamp in timezone "America/New_York" for function from_unixtime: the local date and time is outside the supported range+SELECT from_unixtime(-8334601211039, 'America/New_York');++## The same bound applies to the session time zone form+statement ok+SET datafusion.execution.time_zone = 'America/New_York';++query P+SELECT from_unixtime(-8334601211038);+----+-262143-01-01T00:00:00-04:56++query error DataFusion error: Execution error: Cannot convert \-8334601211039 to a timestamp in timezone "America/New_York" for function from_unixtime: the local date and time is outside the supported range+SELECT from_unixtime(-8334601211039);++## ... and to array (not constant folded) input+query error DataFusion error: Execution error: Cannot convert \-8334601211039 to a timestamp in timezone "America/New_York" for function from_unixtime: the local date and time is outside the supported range+SELECT from_unixtime(v) FROM (VALUES (0), (-8334601211039)) AS t(v);++## A zone that is ahead of UTC runs out at the other end+statement ok+SET datafusion.execution.time_zone = '+08:00';++query P+SELECT from_unixtime(8210266847999);+----++262142-12-31T23:59:59+08:00++query error DataFusion error: Execution error: Cannot convert 8210266848000 to a timestamp in timezone "\+08:00" for function from_unixtime: the local date and time is outside the supported range+SELECT from_unixtime(8210266848000);++statement ok+RESET datafusion.execution.time_zone++## Without a time zone the full range stays usable+query P+SELECT from_unixtime(-8334601228800);+----+-262143-01-01T00:00:00++query P+SELECT from_unixtime(8210266876799);+----++262142-12-31T23:59:59diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex 6726b7e145..11f50b84e3 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -2659,7 +2659,13 @@ _Alias of [date_trunc](#date_trunc)._### `from_unixtime`-Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`) return the corresponding timestamp.+Converts an integer to a timestamp with second precision (`Timestamp(Second)`).+The integer is interpreted as the number of seconds since the unix epoch+(`1970-01-01T00:00:00Z`).++If the optional `timezone` argument is omitted, the timestamp is returned in the+session time zone (`datafusion.execution.time_zone`), which is unset (i.e.+timezone-naive) by default.```sqlfrom_unixtime(expression[, timezone])@@ -2668,7 +2674,7 @@ from_unixtime(expression[, timezone])#### Arguments- **expression**: The expression to operate on. Can be a constant, column, or function, and any combination of operators.-- **timezone**: Optional timezone to use when converting the integer to a timestamp. If not provided, the default timezone is UTC.+- **timezone**: Optional timezone to use when converting the integer to a timestamp. If not provided, the session time zone (`datafusion.execution.time_zone`) is used, which is unset (timezone-naive) by default.#### Example@@ -2679,6 +2685,15 @@ from_unixtime(expression[, timezone])+-----------------------------------------------------------+| 2020-09-08T09:42:29-04:00 |+-----------------------------------------------------------+++-- Without an explicit timezone the session time zone is used+> SET datafusion.execution.time_zone = 'America/New_York';+> select from_unixtime(1599572549);++----------------------------------++| from_unixtime(Int64(1599572549)) |++----------------------------------++| 2020-09-08T09:42:29-04:00 |++----------------------------------+```### `make_date`