Apache DataFusion SQL Query Engine
Change default `time_zone` to `None` (was `"+00:00"`) (#18359)
Bruce Ritchie9ea67f538c06
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex bc321b227e..f4afdf7002 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -467,9 +467,8 @@ config_namespace! {/// The default time zone///- /// Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime- /// according to this time zone, and then extract the hour- pub time_zone: String, default = "+00:00".into()+ /// Some functions, e.g. `now` return timestamps in this time zone+ pub time_zone: Option<String>, default = None/// Parquet optionspub parquet: ParquetOptions, default = Default::default()diff --git a/datafusion/core/tests/expr_api/simplification.rs b/datafusion/core/tests/expr_api/simplification.rsindex 572a7e2b33..46c36c6abd 100644--- a/datafusion/core/tests/expr_api/simplification.rs+++ b/datafusion/core/tests/expr_api/simplification.rs@@ -514,7 +514,7 @@ fn multiple_now() -> Result<()> {// expect the same timestamp appears in both exprslet actual = get_optimized_plan_formatted(plan, &time);let expected = format!(- "Projection: TimestampNanosecond({}, Some(\"+00:00\")) AS now(), TimestampNanosecond({}, Some(\"+00:00\")) AS t2\n TableScan: test",+ "Projection: TimestampNanosecond({}, None) AS now(), TimestampNanosecond({}, None) AS t2\n TableScan: test",time.timestamp_nanos_opt().unwrap(),time.timestamp_nanos_opt().unwrap());diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rsindex fb1371da6c..3ca8f846aa 100644--- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs@@ -1812,7 +1812,7 @@ async fn test_config_options_work_for_scalar_func() -> Result<()> {});let mut config = SessionConfig::new();- config.options_mut().execution.time_zone = "AEST".into();+ config.options_mut().execution.time_zone = Some("AEST".into());let ctx = SessionContext::new_with_config(config);diff --git a/datafusion/functions/src/datetime/current_date.rs b/datafusion/functions/src/datetime/current_date.rsindex 18b99bca86..da690b4e6b 100644--- a/datafusion/functions/src/datetime/current_date.rs+++ b/datafusion/functions/src/datetime/current_date.rs@@ -108,7 +108,14 @@ impl ScalarUDFImpl for CurrentDateFunc {let days = info.execution_props().config_options()- .and_then(|config| config.execution.time_zone.parse::<Tz>().ok())+ .and_then(|config| {+ config+ .execution+ .time_zone+ .as_ref()+ .map(|tz| tz.parse::<Tz>().ok())+ })+ .flatten().map_or_else(|| datetime_to_days(&now_ts),|tz| {diff --git a/datafusion/functions/src/datetime/current_time.rs b/datafusion/functions/src/datetime/current_time.rsindex 4f5b199cce..9f3456b877 100644--- a/datafusion/functions/src/datetime/current_time.rs+++ b/datafusion/functions/src/datetime/current_time.rs@@ -104,7 +104,14 @@ impl ScalarUDFImpl for CurrentTimeFunc {let nano = info.execution_props().config_options()- .and_then(|config| config.execution.time_zone.parse::<Tz>().ok())+ .and_then(|config| {+ config+ .execution+ .time_zone+ .as_ref()+ .map(|tz| tz.parse::<Tz>().ok())+ })+ .flatten().map_or_else(|| datetime_to_time_nanos(&now_ts),|tz| {@@ -167,7 +174,11 @@ mod tests {fn set_session_timezone_env(tz: &str, start_time: DateTime<Utc>) -> MockSimplifyInfo {let mut config = datafusion_common::config::ConfigOptions::default();- config.execution.time_zone = tz.to_string();+ config.execution.time_zone = if tz.is_empty() {+ None+ } else {+ Some(tz.to_string())+ };let mut execution_props =ExecutionProps::new().with_query_execution_start_time(start_time);execution_props.config_options = Some(Arc::new(config));diff --git a/datafusion/functions/src/datetime/now.rs b/datafusion/functions/src/datetime/now.rsindex fe317d0a16..4723548a45 100644--- a/datafusion/functions/src/datetime/now.rs+++ b/datafusion/functions/src/datetime/now.rs@@ -33,7 +33,7 @@ use datafusion_macros::user_doc;#[user_doc(doc_section(label = "Time and Date Functions"),description = r#"-Returns the current UTC timestamp.+Returns the current timestamp in the system configured timezone (None by default).The `now()` return value is determined at query time and will return the same timestamp, no matter when in the query plan the function executes."#,@@ -58,8 +58,7 @@ impl NowFunc {////// Prefer [`NowFunc::new_with_config`] which allows specifying the/// timezone via [`ConfigOptions`]. This helper now mirrors the- /// canonical default offset (`"+00:00"`) provided by- /// `ConfigOptions::default()`.+ /// canonical default offset (None) provided by `ConfigOptions::default()`.pub fn new() -> Self {Self::new_with_config(&ConfigOptions::default())}@@ -68,7 +67,11 @@ impl NowFunc {Self {signature: Signature::nullary(Volatility::Stable),aliases: vec!["current_timestamp".to_string()],- timezone: Some(Arc::from(config.execution.time_zone.as_str())),+ timezone: config+ .execution+ .time_zone+ .as_ref()+ .map(|tz| Arc::from(tz.as_str())),}}}@@ -178,6 +181,6 @@ mod tests {ScalarValue::TimestampNanosecond(None, configured_now.timezone.clone());assert_eq!(legacy_scalar, configured_scalar);- assert_eq!(Some("+00:00"), legacy_now.timezone.as_deref());+ assert_eq!(None, legacy_now.timezone.as_deref());}}diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rsindex 99138e1b00..7bac033767 100644--- a/datafusion/sql/src/planner.rs+++ b/datafusion/sql/src/planner.rs@@ -693,7 +693,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {// Timestamp With Time Zone// INPUT : [SQLDataType] TimestampTz + [Config] Time Zone// OUTPUT: [ArrowDataType] Timestamp<TimeUnit, Some(Time Zone)>- Some(self.context_provider.options().execution.time_zone.clone())+ self.context_provider.options().execution.time_zone.clone()} else {// Timestamp Without Time zoneNonediff --git a/datafusion/sqllogictest/test_files/arrow_typeof.slt b/datafusion/sqllogictest/test_files/arrow_typeof.sltindex cbb20acb2d..5ba62be687 100644--- a/datafusion/sqllogictest/test_files/arrow_typeof.slt+++ b/datafusion/sqllogictest/test_files/arrow_typeof.slt@@ -67,7 +67,7 @@ Timestamp(ns)query TSELECT arrow_typeof(now())-----Timestamp(ns, "+00:00")+Timestamp(ns)# arrow_typeof_timestamp_date32(query Tdiff --git a/datafusion/sqllogictest/test_files/dates.slt b/datafusion/sqllogictest/test_files/dates.sltindex 32315eec20..abf64675e9 100644--- a/datafusion/sqllogictest/test_files/dates.slt+++ b/datafusion/sqllogictest/test_files/dates.slt@@ -91,7 +91,7 @@ where d3_date > now() + '5 days';----DataFusion error: type_coercioncaused by-Error during planning: Cannot coerce arithmetic expression Timestamp(ns, "+00:00") + Utf8 to valid types+Error during planning: Cannot coerce arithmetic expression Timestamp(ns) + Utf8 to valid types# DATE minus DATEdiff --git a/datafusion/sqllogictest/test_files/ddl.slt b/datafusion/sqllogictest/test_files/ddl.sltindex 64c7828459..0579659832 100644--- a/datafusion/sqllogictest/test_files/ddl.slt+++ b/datafusion/sqllogictest/test_files/ddl.slt@@ -867,7 +867,7 @@ query TTTTTTshow columns FROM table_with_pk;----datafusion public table_with_pk sn Int32 NO-datafusion public table_with_pk ts Timestamp(ns, "+00:00") NO+datafusion public table_with_pk ts Timestamp(ns) NOdatafusion public table_with_pk currency Utf8View NOdatafusion public table_with_pk amount Float32 YESdiff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.sltindex 08636b482e..fe7871c22b 100644--- a/datafusion/sqllogictest/test_files/group_by.slt+++ b/datafusion/sqllogictest/test_files/group_by.slt@@ -5556,7 +5556,7 @@ SELECTarrow_cast('2024-01-01T00:00:00Z'::timestamptz, 'Timestamp(Second, Some("+08:00"))') AS tsGROUP BY ts, text-----foo 2024-01-01T08:00:00+08:00+foo 2024-01-01T00:00:00+08:00# Test multi group by int + Decimal128statement okdiff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex c674057151..7009d976d6 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -267,7 +267,7 @@ datafusion.execution.sort_spill_reservation_bytes 10485760datafusion.execution.spill_compression uncompresseddatafusion.execution.split_file_groups_by_statistics falsedatafusion.execution.target_partitions 7-datafusion.execution.time_zone +00:00+datafusion.execution.time_zone NULLdatafusion.execution.use_row_number_estimates_to_optimize_partitioning falsedatafusion.explain.analyze_level devdatafusion.explain.format indent@@ -387,7 +387,7 @@ datafusion.execution.sort_spill_reservation_bytes 10485760 Specifies the reservedatafusion.execution.spill_compression uncompressed Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed.datafusion.execution.split_file_groups_by_statistics false Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimentaldatafusion.execution.target_partitions 7 Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system-datafusion.execution.time_zone +00:00 The default time zone Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this time zone, and then extract the hour+datafusion.execution.time_zone NULL The default time zone Some functions, e.g. `now` return timestamps in this time zonedatafusion.execution.use_row_number_estimates_to_optimize_partitioning false Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future.datafusion.explain.analyze_level dev Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers.datafusion.explain.format indent Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format.@@ -459,14 +459,14 @@ datafusion.execution.batch_size 8192 Default batch size while creating new batchquery TTSHOW TIME ZONE-----datafusion.execution.time_zone +00:00+datafusion.execution.time_zone NULL# show_timezone_default_utc# https://github.com/apache/datafusion/issues/3255query TTSHOW TIMEZONE-----datafusion.execution.time_zone +00:00+datafusion.execution.time_zone NULL# show_time_zone_default_utc_verbose@@ -474,14 +474,14 @@ datafusion.execution.time_zone +00:00query TTTSHOW TIME ZONE VERBOSE-----datafusion.execution.time_zone +00:00 The default time zone Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this time zone, and then extract the hour+datafusion.execution.time_zone NULL The default time zone Some functions, e.g. `now` return timestamps in this time zone# show_timezone_default_utc# https://github.com/apache/datafusion/issues/3255query TTTSHOW TIMEZONE VERBOSE-----datafusion.execution.time_zone +00:00 The default time zone Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this time zone, and then extract the hour+datafusion.execution.time_zone NULL The default time zone Some functions, e.g. `now` return timestamps in this time zone# show empty verbosediff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex 7252c84caf..8753d39cb7 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -165,7 +165,7 @@ GROUP BY tsORDER BY tsLIMIT 1;-----2020-09-08T13:42:29.190855123Z+2020-09-08T13:42:29.190855123diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.sltindex e4676ae533..0166cd2572 100644--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt@@ -570,4 +570,4 @@ WHERE trace_id = '00000000000000000000000000000002' AND deployment_environment =ORDER BY start_timestamp, trace_idLIMIT 1;-----2024-10-01T00:00:00Z+2024-10-01T00:00:00diff --git a/datafusion/sqllogictest/test_files/table_functions.slt b/datafusion/sqllogictest/test_files/table_functions.sltindex 1bffbc3b3a..c843400efc 100644--- a/datafusion/sqllogictest/test_files/table_functions.slt+++ b/datafusion/sqllogictest/test_files/table_functions.slt@@ -414,7 +414,7 @@ SELECT * FROM range(TIMESTAMP '2023-01-01T00:00:00', TIMESTAMP '2023-01-03T00:00query PSELECT * FROM range(TIMESTAMPTZ '2023-02-01T00:00:00-07:00', TIMESTAMPTZ '2023-02-01T09:00:00+01:00', INTERVAL '1' HOUR);-----2023-02-01T07:00:00Z+2023-02-01T07:00:00# Basic date range with hour intervalquery Pdiff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex d9b4a818f9..c2fabb5e6e 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -942,22 +942,22 @@ CREATE TABLE table1 (statement okINSERT INTO table1 (bar, foo, time) VALUES-(200.0, 'me', '1970-01-01T00:00:00.000000010Z'),-(1.0, 'me', '1970-01-01T00:00:00.000000030Z'),-(1.0, 'me', '1970-01-01T00:00:00.000000040Z'),-(2.0, 'you', '1970-01-01T00:00:00.000000020Z');+(200.0, 'me', '1970-01-01T00:00:00.000000010'),+(1.0, 'me', '1970-01-01T00:00:00.000000030'),+(1.0, 'me', '1970-01-01T00:00:00.000000040'),+(2.0, 'you', '1970-01-01T00:00:00.000000020');query TPSELECT foo, first_value(time ORDER BY time DESC NULLS LAST) AS time FROM table1 GROUP BY foo ORDER BY foo;-----me 1970-01-01T00:00:00.000000040Z-you 1970-01-01T00:00:00.000000020Z+me 1970-01-01T00:00:00.000000040+you 1970-01-01T00:00:00.000000020query TPSELECT foo, last_value(time ORDER BY time DESC NULLS LAST) AS time FROM table1 GROUP BY foo ORDER BY foo;-----me 1970-01-01T00:00:00.000000010Z-you 1970-01-01T00:00:00.000000020Z+me 1970-01-01T00:00:00.000000010+you 1970-01-01T00:00:00.000000020statement okdrop table table1;@@ -5766,15 +5766,15 @@ CREATE TABLE table_test_distinct_count (statement okINSERT INTO table_test_distinct_count (k, v, time) VALUES- ('a', 1, '1970-01-01T00:01:00.00Z'),- ('a', 1, '1970-01-01T00:02:00.00Z'),- ('a', 1, '1970-01-01T00:03:00.00Z'),- ('a', 2, '1970-01-01T00:03:00.00Z'),- ('a', 1, '1970-01-01T00:04:00.00Z'),- ('b', 3, '1970-01-01T00:01:00.00Z'),- ('b', 3, '1970-01-01T00:02:00.00Z'),- ('b', 4, '1970-01-01T00:03:00.00Z'),- ('b', 4, '1970-01-01T00:03:00.00Z');+ ('a', 1, '1970-01-01T00:01:00.00'),+ ('a', 1, '1970-01-01T00:02:00.00'),+ ('a', 1, '1970-01-01T00:03:00.00'),+ ('a', 2, '1970-01-01T00:03:00.00'),+ ('a', 1, '1970-01-01T00:04:00.00'),+ ('b', 3, '1970-01-01T00:01:00.00'),+ ('b', 3, '1970-01-01T00:02:00.00'),+ ('b', 4, '1970-01-01T00:03:00.00'),+ ('b', 4, '1970-01-01T00:03:00.00');query TPIISELECT@@ -5793,15 +5793,15 @@ SELECTFROM table_test_distinct_countORDER BY k, time;-----a 1970-01-01T00:01:00Z 1 1-a 1970-01-01T00:02:00Z 2 1-a 1970-01-01T00:03:00Z 4 2-a 1970-01-01T00:03:00Z 4 2-a 1970-01-01T00:04:00Z 4 2-b 1970-01-01T00:01:00Z 1 1-b 1970-01-01T00:02:00Z 2 1-b 1970-01-01T00:03:00Z 4 2-b 1970-01-01T00:03:00Z 4 2+a 1970-01-01T00:01:00 1 1+a 1970-01-01T00:02:00 2 1+a 1970-01-01T00:03:00 4 2+a 1970-01-01T00:03:00 4 2+a 1970-01-01T00:04:00 4 2+b 1970-01-01T00:01:00 1 1+b 1970-01-01T00:02:00 2 1+b 1970-01-01T00:03:00 4 2+b 1970-01-01T00:03:00 4 2query TT@@ -5854,15 +5854,15 @@ SELECTFROM table_test_distinct_countORDER BY k, time;-----a 1970-01-01T00:01:00Z 1 1-a 1970-01-01T00:02:00Z 2 1-a 1970-01-01T00:03:00Z 5 3-a 1970-01-01T00:03:00Z 5 3-a 1970-01-01T00:04:00Z 5 3-b 1970-01-01T00:01:00Z 3 3-b 1970-01-01T00:02:00Z 6 3-b 1970-01-01T00:03:00Z 14 7-b 1970-01-01T00:03:00Z 14 7+a 1970-01-01T00:01:00 1 1+a 1970-01-01T00:02:00 2 1+a 1970-01-01T00:03:00 5 3+a 1970-01-01T00:03:00 5 3+a 1970-01-01T00:04:00 5 3+b 1970-01-01T00:01:00 3 3+b 1970-01-01T00:02:00 6 3+b 1970-01-01T00:03:00 14 7+b 1970-01-01T00:03:00 14 7diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex af5c603404..0b227000f7 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -176,6 +176,20 @@ let indices = projection_exprs.column_indices();_execution plan_ of the query. With this release, `DESCRIBE query` now outputsthe computed _schema_ of the query, consistent with the behavior of `DESCRIBE table_name`.+### `datafusion.execution.time_zone` default configuration changed++The default value for `datafusion.execution.time_zone` previously was a string value of `+00:00` (GMT/Zulu time).+This was changed to be an `Option<String>` with a default of `None`. If you want to change the timezone back+to the previous value you can execute the sql:++```sql+SET+TIMEZONE = '+00:00';+```++This change was made to better support using the default timezone in scalar UDF functions such as+`now`, `current_date`, `current_time`, and `to_timestamp` among others.+### Introduction of `TableSchema` and changes to `FileSource::with_schema()` methodA new `TableSchema` struct has been introduced in the `datafusion-datasource` crate to better manage table schemas with partition columns. This struct helps distinguish between:diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex 7ca5eb8f7b..5950a4fa9a 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -77,7 +77,7 @@ The following configuration settings are available:| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting || datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true. || datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system |-| datafusion.execution.time_zone | +00:00 | The default time zone Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this time zone, and then extract the hour |+| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now` return timestamps in this time zone || datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. || datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file || datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata |diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex d090b5b70c..7c88d1fd9c 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -2629,7 +2629,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo### `now`-Returns the current UTC timestamp.+Returns the current timestamp in the system configured timezone (None by default).The `now()` return value is determined at query time and will return the same timestamp, no matter when in the query plan the function executes.