Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fall back on effective-type for time fields (#75442)

metamben9ab9b085d420
diff --git a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj b/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
index 4eec9acb901..b3dd68137a3 100644
--- a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
+++ b/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
@@ -167,7 +167,7 @@
(defmethod sql.qp/date [:sqlite :second]
[_driver _unit expr]
- (if (= (h2x/database-type expr) "time")
+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)
(->time (strftime "%H:%M:%S" expr))
(->datetime (strftime "%Y-%m-%d %H:%M:%S" expr))))
@@ -177,7 +177,7 @@
(defmethod sql.qp/date [:sqlite :minute]
[_driver _unit expr]
- (if (= (h2x/database-type expr) "time")
+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)
(->time (strftime "%H:%M" expr))
(->datetime (strftime "%Y-%m-%d %H:%M" expr))))
@@ -187,7 +187,7 @@
(defmethod sql.qp/date [:sqlite :hour]
[_driver _unit expr]
- (if (= (h2x/database-type expr) "time")
+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)
(->time (strftime "%H:00" expr))
(->datetime (strftime "%Y-%m-%d %H:00" expr))))
diff --git a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
index 5c72b21bc9b..6bb864972ac 100644
--- a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
+++ b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
@@ -8,6 +8,7 @@
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
+ [metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -15,6 +16,7 @@
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.util.honey-sql-2 :as h2x]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -24,6 +26,14 @@
(is (= "UTC"
(driver/db-default-timezone :sqlite (mt/db))))))
+(deftest ^:parallel hour-bucketing-time-without-database-type-test
+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "
+ "fields referenced by name from a source query, #75193) should use the time-aware path "
+ "and not produce a DATETIME-format result")
+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]
+ (is (= ["TIME(STRFTIME('%H:00', \"test_col\"))"]
+ (sql.qp/format-honeysql :sqlite (sql.qp/date :sqlite :hour expr)))))))
+
(deftest current-user-table-privileges-test
(testing "SQLite table privileges normalization"
(mt/test-driver :sqlite
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index 28d2bfebe99..9958478c592 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -254,7 +254,7 @@
(defmethod sql.qp/date [:sqlserver :minute]
[_driver _unit expr]
- (if (= (h2x/database-type expr) "time")
+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)
(time-from-parts (date-part :hour expr) (date-part :minute expr) 0 0 0)
(h2x/maybe-cast :smalldatetime expr)))
@@ -282,7 +282,7 @@
(defmethod sql.qp/date [:sqlserver :hour]
[_driver _unit expr]
- (if (= (h2x/database-type expr) "time")
+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)
(time-from-parts (date-part :hour expr) 0 0 0 0)
(date-time-2-from-parts (h2x/year expr) (h2x/month expr) (h2x/day expr) (date-part :hour expr) 0 0 0 0)))
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index 50e84e39d60..56c518befa1 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -37,6 +37,14 @@
(set! *warn-on-reflection* true)
+(deftest ^:parallel hour-bucketing-time-without-database-type-test
+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "
+ "fields referenced by name from a source query, #75193) should use TIMEFROMPARTS and "
+ "not produce a DATETIME2FROMPARTS result that requires a date component")
+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]
+ (is (= ["TIMEFROMPARTS(DATEPART(hour, \"test_col\"), 0, 0, 0, 0)"]
+ (sql.qp/format-honeysql :sqlserver (sql.qp/date :sqlserver :hour expr)))))))
+
(deftest ^:parallel fix-order-bys-test
(testing "Remove order-by from joins"
(let [original {:joins [{:alias "C3"
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index 42873ac1e2e..5286509d20c 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -533,14 +533,14 @@
(defmethod sql.qp/date [:mysql :minute]
[_driver _unit expr]
- (let [format-str (if (= (h2x/database-type expr) "time")
+ (let [format-str (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)
"%H:%i"
"%Y-%m-%d %H:%i")]
(trunc-with-format format-str expr)))
(defmethod sql.qp/date [:mysql :hour]
[_driver _unit expr]
- (let [format-str (if (= (h2x/database-type expr) "time")
+ (let [format-str (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)
"%H"
"%Y-%m-%d %H")]
(trunc-with-format format-str expr)))
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index c732f5f5b80..58dc3ccaf00 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -543,20 +543,23 @@
(mu/defn- date-trunc
[unit :- driver-api/schema.temporal-bucketing.unit.date-time.truncate
expr]
- (condp = (h2x/database-type expr)
+ ;; Branches are ordered most-specific-first because `database-or-effective-type-isa?` checks `isa?` on the effective
+ ;; type fallback: `:type/TimeWithTZ` is a descendant of `:type/Time`, so the timetz branch must run first to avoid a
+ ;; nested-source-query `timetz` column being routed to the plain-time path (#75193, #68065).
+ (cond
+ (h2x/database-or-effective-type-isa? expr "timetz" :type/TimeWithTZ)
+ (h2x/cast "timetz" (time-trunc unit expr))
+
;; apparently there is no convenient way to truncate a TIME column in Postgres, you can try to use `date_trunc`
;; but it returns an interval (??) and other insane things. This seems to be slightly less insane.
- "time"
+ (h2x/database-or-effective-type-isa? expr "time" :type/Time)
(time-trunc unit expr)
- "timetz"
- (h2x/cast "timetz" (time-trunc unit expr))
-
;; postgres returns timestamp or timestamptz from `date_trunc`, so cast back if we've got a date column
- "date"
+ (h2x/database-or-effective-type-isa? expr "date" :type/Date)
(h2x/cast "date" [:date_trunc (h2x/literal unit) expr])
- #_else
+ :else
(let [expr' (h2x/->pg-timestamp expr)]
(-> [:date_trunc (h2x/literal unit) expr']
(h2x/with-database-type-info (h2x/database-type expr'))))))
diff --git a/src/metabase/util/honey_sql_2.clj b/src/metabase/util/honey_sql_2.clj
index 436f4083a16..5f6a907ce63 100644
--- a/src/metabase/util/honey_sql_2.clj
+++ b/src/metabase/util/honey_sql_2.clj
@@ -302,6 +302,18 @@
(let [info (type-info honeysql-form)]
(or (:effective-type info) (:base-type info))))
+(defn database-or-effective-type-isa?
+ "Returns true if `honeysql-form`'s known `database-type` (case-insensitive) equals `db-type`, OR — when no
+ `database-type` is attached — if its [[effective-type]] descends from `effective-type-supertype`. Useful in driver
+ bucketing code that special-cases columns by their warehouse type and needs to fall back when the column reached
+ the driver from a nested query (so the database-type was lost) but its Metabase effective type is still known."
+ [honeysql-form db-type effective-type-supertype]
+ (let [dbt (database-type honeysql-form)]
+ (if dbt
+ (= (u/lower-case-en dbt) (u/lower-case-en (name db-type)))
+ (when effective-type-supertype
+ (isa? (effective-type honeysql-form) effective-type-supertype)))))
+
(defn is-of-type?
"Is `honeysql-form` a typed form with `db-type`?
Where `db-type` could be a string or a regex.
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index 3ec0140cedb..01d4c3f4209 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -127,6 +127,14 @@
(is (= #{"dbone_a" "dbone_b" "dbone_c"}
(into #{} (map :name) (driver/describe-fields :mysql database)))))))))
+(deftest ^:parallel hour-bucketing-time-without-database-type-test
+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "
+ "fields referenced by name from a source query, #75193) should use the TIME-only format "
+ "string and not produce a DATETIME-with-date format")
+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]
+ (is (= ["STR_TO_DATE(DATE_FORMAT(CAST(`test_col` AS datetime), '%H'), '%H')"]
+ (sql.qp/format-honeysql :mysql (sql.qp/date :mysql :hour expr)))))))
+
(deftest date-test
;; make sure stuff at least compiles. Even if the result probably isn't as concise as it could be.
;; See [[metabase.query-processor.date-time-zone-functions-test/extract-week-tests]] for something that tests
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 6ea2990c9cb..4dc654c4a9c 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -85,6 +85,14 @@
(is (= ["extract(month from NOW())"]
(sql.qp/format-honeysql :postgres (#'postgres/extract :month :%now)))))
+(deftest ^:parallel hour-bucketing-time-without-database-type-test
+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "
+ "fields referenced by name from a source query, #75193) should use the time-aware path "
+ "and not cast to `timestamp`")
+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]
+ (is (= ["MAKE_TIME(extract(hour from \"test_col\")::integer, 0, 0.0)"]
+ (sql.qp/format-honeysql :postgres (sql.qp/date :postgres :hour expr)))))))
+
(deftest ^:parallel datetime-diff-test
(is (= [["CAST("
" extract("
diff --git a/test/metabase/query_processor/time_field_test.clj b/test/metabase/query_processor/time_field_test.clj
index 1ed62deb2d1..c5f9aad79e1 100644
--- a/test/metabase/query_processor/time_field_test.clj
+++ b/test/metabase/query_processor/time_field_test.clj
@@ -6,6 +6,7 @@
[metabase.driver :as driver]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-util.notebook-helpers :as notebook-helpers]
[metabase.query-processor.test :as qp]
[metabase.query-processor.test-util :as qp.test-util]
[metabase.test :as mt]
@@ -191,3 +192,39 @@
(is (=? [[1 #"07:23:00Z?"]
[2 #"07:14:00Z?"]]
results)))))))
+
+(deftest hour-bucketing-time-field-in-source-query-test
+ (testing "Hour group-by on a time field from a source query should work (#75193, #68065)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :test/time-type)
+ (mt/dataset time-test-data
+ (let [mp (mt/metadata-provider)
+ users (lib.metadata/table mp (mt/id :users))
+ time-field (lib.metadata/field mp (mt/id :users :last_login_time))
+ ;; Two-stage query: stage 1 selects the time field; stage 2 groups it by :hour
+ ;; with a count. In stage 2 the field is referenced by name (string), not by
+ ;; integer ID — `field-metadata` is nil and `database-type` is lost. Drivers
+ ;; that dispatch their `:hour` bucketing on `(h2x/database-type expr)` fall
+ ;; through to a timestamp/datetime cast, which errors (Postgres, SQL Server)
+ ;; or silently misgroups (MySQL, SQLite).
+ query (-> (lib/query mp users)
+ (lib/with-fields [time-field])
+ lib/append-stage
+ (notebook-helpers/add-breakout
+ "Summaries" "Last Login Time"
+ {:col-fn #(lib/with-temporal-bucket % :hour)})
+ (as-> q (lib/aggregate q (lib/count))))]
+ (mt/with-native-query-testing-context query
+ (is (=? [[#"01:00:00Z?" 1]
+ [#"07:00:00Z?" 1]
+ [#"08:00:00Z?" 2]
+ [#"09:00:00Z?" 2]
+ [#"10:00:00Z?" 2]
+ [#"12:00:00Z?" 2]
+ [#"13:00:00Z?" 1]
+ [#"15:00:00Z?" 1]
+ [#"16:00:00Z?" 1]
+ [#"17:00:00Z?" 1]
+ [#"19:00:00Z?" 1]]
+ (mt/formatted-rows
+ [str int]
+ (qp/process-query query))))))))))