Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[QP] Fix `convertTimezone` exprs with no `src` TZ and SQL source (#64710)

Braden Shepherdsondd5117bdb17c
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index 3b2ea883a58..3e9fc00c913 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -12,6 +12,7 @@
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
@@ -559,7 +560,8 @@
(let [datetime (fn [x target-timezone]
[:datetime x target-timezone])
hsql-form (sql.qp/->honeysql driver arg)
- timestamptz? (h2x/is-of-type? hsql-form "timestamp")]
+ timestamptz? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? hsql-form "timestamp"))]
(sql.u/validate-convert-timezone-args timestamptz? target-timezone source-timezone)
(-> (if timestamptz?
hsql-form
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
index 73f24e5e03a..49d405c67dc 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
@@ -10,6 +10,7 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]]
+ [metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
@@ -199,7 +200,8 @@
(defmethod sql.qp/->honeysql [:clickhouse :convert-timezone]
[driver [_ arg target-timezone source-timezone]]
(let [expr (sql.qp/->honeysql driver (cond-> arg (string? arg) u.date/parse))
- with-tz-info? (h2x/is-of-type? expr #"(?:nullable\(|lowcardinality\()?(datetime64\(\d, {0,1}'.*|datetime\(.*)")
+ with-tz-info? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? expr #"(?:nullable\(|lowcardinality\()?(datetime64\(\d, {0,1}'.*|datetime\(.*)"))
_ (sql.u/validate-convert-timezone-args with-tz-info? target-timezone source-timezone)]
(if (not with-tz-info?)
[:'plus
@@ -616,4 +618,3 @@
(defmethod sql.params.substitution/->replacement-snippet-info [:clickhouse UUID]
[_driver this]
{:replacement-snippet (format "CAST('%s' AS UUID)" (str this))})
-
diff --git a/modules/drivers/oracle/src/metabase/driver/oracle.clj b/modules/drivers/oracle/src/metabase/driver/oracle.clj
index 6e117c40583..6b34285b501 100644
--- a/modules/drivers/oracle/src/metabase/driver/oracle.clj
+++ b/modules/drivers/oracle/src/metabase/driver/oracle.clj
@@ -20,6 +20,7 @@
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison]
[metabase.driver.sql.query-processor.empty-string-is-null :as sql.qp.empty-string-is-null]
+ [metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
@@ -293,7 +294,8 @@
(defmethod sql.qp/->honeysql [:oracle :convert-timezone]
[driver [_ arg target-timezone source-timezone]]
(let [expr (sql.qp/->honeysql driver arg)
- has-timezone? (h2x/is-of-type? expr #"timestamp(\(\d\))? with time zone")]
+ has-timezone? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? expr #"timestamp(\(\d\))? with time zone"))]
(sql.u/validate-convert-timezone-args has-timezone? target-timezone source-timezone)
(-> (if has-timezone?
expr
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index c8eb9ff3311..446d7a15aed 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -23,6 +23,7 @@
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
[metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.driver.sync :as driver.s]
[metabase.util :as u]
@@ -571,7 +572,8 @@
(defmethod sql.qp/->honeysql [:snowflake :convert-timezone]
[driver [_ arg target-timezone source-timezone]]
(let [hsql-form (sql.qp/->honeysql driver arg)
- timestamptz? (h2x/is-of-type? hsql-form "timestamptz")]
+ timestamptz? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? hsql-form "timestamptz"))]
(sql.u/validate-convert-timezone-args timestamptz? target-timezone source-timezone)
(-> (if timestamptz?
[:convert_timezone target-timezone hsql-form]
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index da7f473c54b..2d82356f440 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -21,6 +21,7 @@
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison]
+ [metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.util :as u]
@@ -430,7 +431,8 @@
(defmethod sql.qp/->honeysql [:sqlserver :convert-timezone]
[driver [_ arg target-timezone source-timezone]]
(let [expr (sql.qp/->honeysql driver arg)
- datetimeoffset? (h2x/is-of-type? expr "datetimeoffset")]
+ datetimeoffset? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? expr "datetimeoffset"))]
(sql.u/validate-convert-timezone-args datetimeoffset? target-timezone source-timezone)
(-> (if datetimeoffset?
expr
diff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.clj
index 2166aef6328..c688d1a3154 100644
--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj
+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj
@@ -19,6 +19,7 @@
[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync.interface]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
@@ -361,7 +362,8 @@
[driver [_ arg target-timezone source-timezone]]
(let [expr (sql.qp/->honeysql driver (cond-> arg
(string? arg) u.date/parse))
- with_timezone? (h2x/is-of-type? expr #"(?i)^timestamp(?:\(\d+\))? with time zone$")
+ with_timezone? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? expr #"(?i)^timestamp(?:\(\d+\))? with time zone$"))
_ (sql.u/validate-convert-timezone-args with_timezone? target-timezone source-timezone)
expr [:at_timezone
(if with_timezone?
diff --git a/modules/drivers/vertica/src/metabase/driver/vertica.clj b/modules/drivers/vertica/src/metabase/driver/vertica.clj
index 6eff0b7f2b1..f9ca33586c6 100644
--- a/modules/drivers/vertica/src/metabase/driver/vertica.clj
+++ b/modules/drivers/vertica/src/metabase/driver/vertica.clj
@@ -15,6 +15,7 @@
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.empty-string-is-null
:as sql.qp.empty-string-is-null]
+ [metabase.driver.sql.query-processor.util :as sql.qp.u]
[metabase.driver.sql.util :as sql.u]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
@@ -139,7 +140,8 @@
(defmethod sql.qp/->honeysql [:vertica :convert-timezone]
[driver [_ arg target-timezone source-timezone]]
(let [expr (cast-timestamp (sql.qp/->honeysql driver arg))
- timestamptz? (h2x/is-of-type? expr "timestamptz")]
+ timestamptz? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? expr "timestamptz"))]
(sql.u/validate-convert-timezone-args timestamptz? target-timezone source-timezone)
(-> (if timestamptz?
expr
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index 3d85154331c..961cce9227b 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -550,7 +550,8 @@
(defmethod sql.qp/->honeysql [:mysql :convert-timezone]
[driver [_ arg target-timezone source-timezone]]
(let [expr (sql.qp/->honeysql driver arg)
- timestamp? (h2x/is-of-type? expr "timestamp")]
+ timestamp? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? expr "timestamp"))]
(sql.u/validate-convert-timezone-args timestamp? target-timezone source-timezone)
(h2x/with-database-type-info
[:convert_tz expr (or source-timezone (driver-api/results-timezone-id)) target-timezone]
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index fa810d77a53..fee07cb8118 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -592,7 +592,8 @@
[driver [_ arg target-timezone source-timezone]]
(let [expr (sql.qp/->honeysql driver (cond-> arg
(string? arg) u.date/parse))
- timestamptz? (or (h2x/is-of-type? expr "timestamptz")
+ timestamptz? (or (sql.qp.u/field-with-tz? arg)
+ (h2x/is-of-type? expr "timestamptz")
(h2x/is-of-type? expr "timestamp with time zone"))
_ (sql.u/validate-convert-timezone-args timestamptz? target-timezone source-timezone)
expr [:timezone target-timezone (if (not timestamptz?)
diff --git a/src/metabase/driver/sql/query_processor/util.clj b/src/metabase/driver/sql/query_processor/util.clj
index 85382085a61..52d666ab401 100644
--- a/src/metabase/driver/sql/query_processor/util.clj
+++ b/src/metabase/driver/sql/query_processor/util.clj
@@ -20,3 +20,13 @@
(pop)
(conj (first nfc-path)))]
(apply h2x/identifier (cons :field parent-components))))
+
+(defn field-with-tz?
+ "Given a clause, possibly a `:field` ref, return true iff its type is known to have timezone information."
+ [arg]
+ (if-let [[kind _id-or-name opts] (and (vector? arg) arg)]
+ (and (= kind :field)
+ (when-let [type ((some-fn :effective-type :base-type) opts)]
+ (or (isa? type :type/DateTimeWithTZ)
+ (isa? type :type/TimeWithTZ))))
+ false))
diff --git a/test/metabase/query_processor_test/date_time_zone_functions_test.clj b/test/metabase/query_processor_test/date_time_zone_functions_test.clj
index 637fb48215f..b352b365d57 100644
--- a/test/metabase/query_processor_test/date_time_zone_functions_test.clj
+++ b/test/metabase/query_processor_test/date_time_zone_functions_test.clj
@@ -4,10 +4,12 @@
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.driver :as driver]
+ [metabase.driver.sql.util :as sql.u]
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.result-metadata :as lib.metadata.result-metadata]
+ [metabase.lib.options :as lib.options]
[metabase.lib.test-util :as lib.tu]
[metabase.query-processor :as qp]
[metabase.query-processor.middleware.annotate :as annotate]
@@ -878,6 +880,35 @@
mt/process-query
mt/rows))))))))))
+(deftest convert-timezone-native-query-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :convert-timezone)
+ (mt/dataset times-mixed
+ (testing "MBQL query with native first stage and MBQL second stage can convert timezones without a src (#64705)"
+ ;; We allow `:convert-timezone` with only the destination timezone, when the underlying column has a known
+ ;; timezone. When the first stage is a native query (or when SQL-based sandboxing makes it so, in #64705)
+ ;; the driver needs to know if the input column has a timezone.
+ (when (driver.u/supports? driver/*driver* :set-timezone (mt/db))
+ (mt/with-report-timezone-id! "Europe/Rome"
+ (let [mp (mt/metadata-provider)
+ table (lib.metadata/table mp (mt/id :times))
+ col-fn #(lib.options/ensure-uuid [:field {:base-type :type/DateTimeWithLocalTZ} "dt_tz"])
+ sql (format "SELECT %s FROM %s WHERE 1 = 1"
+ (sql.u/quote-name driver/*driver* :field "dt_tz")
+ (sql.u/quote-name driver/*driver* :table (:schema table) (:name table)))
+ query (-> (lib/native-query mp sql)
+ lib/append-stage
+ (lib/with-fields [(col-fn)])
+ ;; TODO: Can't build this with `lib/convert-timezone` because it requires both
+ ;; src and dest timezones, while the UI only requires dest.
+ (lib/expression "expr" (lib.options/ensure-uuid
+ [:convert-timezone {} (col-fn) "America/Toronto"]))
+ (lib/order-by (col-fn))
+ (lib/limit 1))]
+ (is (= [["2004-03-19T03:19:09+01:00" "2004-03-18T21:19:09-05:00"]]
+ (-> query
+ mt/process-query
+ mt/rows))))))))))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Datetime diff tests |
;;; +----------------------------------------------------------------------------------------------------------------+