The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fall back on effective-type for time fields (#75442)
metamben9ab9b085d420
diff --git a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj b/modules/drivers/sqlite/src/metabase/driver/sqlite.cljindex 4eec9acb901..b3dd68137a3 100644--- a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj+++ b/modules/drivers/sqlite/src/metabase/driver/sqlite.clj@@ -167,7 +167,7 @@(defmethod sql.qp/date [:sqlite :second][_driver _unit expr]- (if (= (h2x/database-type expr) "time")+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)(->time (strftime "%H:%M:%S" expr))(->datetime (strftime "%Y-%m-%d %H:%M:%S" expr))))@@ -177,7 +177,7 @@(defmethod sql.qp/date [:sqlite :minute][_driver _unit expr]- (if (= (h2x/database-type expr) "time")+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)(->time (strftime "%H:%M" expr))(->datetime (strftime "%Y-%m-%d %H:%M" expr))))@@ -187,7 +187,7 @@(defmethod sql.qp/date [:sqlite :hour][_driver _unit expr]- (if (= (h2x/database-type expr) "time")+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)(->time (strftime "%H:00" expr))(->datetime (strftime "%Y-%m-%d %H:00" expr))))diff --git a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.cljindex 5c72b21bc9b..6bb864972ac 100644--- a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj+++ b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj@@ -8,6 +8,7 @@[metabase.driver :as driver][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]+ [metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor-test-util :as sql.qp-test-util][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -15,6 +16,7 @@[metabase.sync.core :as sync][metabase.test :as mt][metabase.util :as u]+ [metabase.util.honey-sql-2 :as h2x][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -24,6 +26,14 @@(is (= "UTC"(driver/db-default-timezone :sqlite (mt/db))))))+(deftest ^:parallel hour-bucketing-time-without-database-type-test+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "+ "fields referenced by name from a source query, #75193) should use the time-aware path "+ "and not produce a DATETIME-format result")+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]+ (is (= ["TIME(STRFTIME('%H:00', \"test_col\"))"]+ (sql.qp/format-honeysql :sqlite (sql.qp/date :sqlite :hour expr)))))))+(deftest current-user-table-privileges-test(testing "SQLite table privileges normalization"(mt/test-driver :sqlitediff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex 28d2bfebe99..9958478c592 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -254,7 +254,7 @@(defmethod sql.qp/date [:sqlserver :minute][_driver _unit expr]- (if (= (h2x/database-type expr) "time")+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)(time-from-parts (date-part :hour expr) (date-part :minute expr) 0 0 0)(h2x/maybe-cast :smalldatetime expr)))@@ -282,7 +282,7 @@(defmethod sql.qp/date [:sqlserver :hour][_driver _unit expr]- (if (= (h2x/database-type expr) "time")+ (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)(time-from-parts (date-part :hour expr) 0 0 0 0)(date-time-2-from-parts (h2x/year expr) (h2x/month expr) (h2x/day expr) (date-part :hour expr) 0 0 0 0)))diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 50e84e39d60..56c518befa1 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -37,6 +37,14 @@(set! *warn-on-reflection* true)+(deftest ^:parallel hour-bucketing-time-without-database-type-test+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "+ "fields referenced by name from a source query, #75193) should use TIMEFROMPARTS and "+ "not produce a DATETIME2FROMPARTS result that requires a date component")+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]+ (is (= ["TIMEFROMPARTS(DATEPART(hour, \"test_col\"), 0, 0, 0, 0)"]+ (sql.qp/format-honeysql :sqlserver (sql.qp/date :sqlserver :hour expr)))))))+(deftest ^:parallel fix-order-bys-test(testing "Remove order-by from joins"(let [original {:joins [{:alias "C3"diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex 42873ac1e2e..5286509d20c 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -533,14 +533,14 @@(defmethod sql.qp/date [:mysql :minute][_driver _unit expr]- (let [format-str (if (= (h2x/database-type expr) "time")+ (let [format-str (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)"%H:%i""%Y-%m-%d %H:%i")](trunc-with-format format-str expr)))(defmethod sql.qp/date [:mysql :hour][_driver _unit expr]- (let [format-str (if (= (h2x/database-type expr) "time")+ (let [format-str (if (h2x/database-or-effective-type-isa? expr "time" :type/Time)"%H""%Y-%m-%d %H")](trunc-with-format format-str expr)))diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex c732f5f5b80..58dc3ccaf00 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -543,20 +543,23 @@(mu/defn- date-trunc[unit :- driver-api/schema.temporal-bucketing.unit.date-time.truncateexpr]- (condp = (h2x/database-type expr)+ ;; Branches are ordered most-specific-first because `database-or-effective-type-isa?` checks `isa?` on the effective+ ;; type fallback: `:type/TimeWithTZ` is a descendant of `:type/Time`, so the timetz branch must run first to avoid a+ ;; nested-source-query `timetz` column being routed to the plain-time path (#75193, #68065).+ (cond+ (h2x/database-or-effective-type-isa? expr "timetz" :type/TimeWithTZ)+ (h2x/cast "timetz" (time-trunc unit expr))+;; apparently there is no convenient way to truncate a TIME column in Postgres, you can try to use `date_trunc`;; but it returns an interval (??) and other insane things. This seems to be slightly less insane.- "time"+ (h2x/database-or-effective-type-isa? expr "time" :type/Time)(time-trunc unit expr)- "timetz"- (h2x/cast "timetz" (time-trunc unit expr))-;; postgres returns timestamp or timestamptz from `date_trunc`, so cast back if we've got a date column- "date"+ (h2x/database-or-effective-type-isa? expr "date" :type/Date)(h2x/cast "date" [:date_trunc (h2x/literal unit) expr])- #_else+ :else(let [expr' (h2x/->pg-timestamp expr)](-> [:date_trunc (h2x/literal unit) expr'](h2x/with-database-type-info (h2x/database-type expr'))))))diff --git a/src/metabase/util/honey_sql_2.clj b/src/metabase/util/honey_sql_2.cljindex 436f4083a16..5f6a907ce63 100644--- a/src/metabase/util/honey_sql_2.clj+++ b/src/metabase/util/honey_sql_2.clj@@ -302,6 +302,18 @@(let [info (type-info honeysql-form)](or (:effective-type info) (:base-type info))))+(defn database-or-effective-type-isa?+ "Returns true if `honeysql-form`'s known `database-type` (case-insensitive) equals `db-type`, OR — when no+ `database-type` is attached — if its [[effective-type]] descends from `effective-type-supertype`. Useful in driver+ bucketing code that special-cases columns by their warehouse type and needs to fall back when the column reached+ the driver from a nested query (so the database-type was lost) but its Metabase effective type is still known."+ [honeysql-form db-type effective-type-supertype]+ (let [dbt (database-type honeysql-form)]+ (if dbt+ (= (u/lower-case-en dbt) (u/lower-case-en (name db-type)))+ (when effective-type-supertype+ (isa? (effective-type honeysql-form) effective-type-supertype)))))+(defn is-of-type?"Is `honeysql-form` a typed form with `db-type`?Where `db-type` could be a string or a regex.diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.cljindex 3ec0140cedb..01d4c3f4209 100644--- a/test/metabase/driver/mysql_test.clj+++ b/test/metabase/driver/mysql_test.clj@@ -127,6 +127,14 @@(is (= #{"dbone_a" "dbone_b" "dbone_c"}(into #{} (map :name) (driver/describe-fields :mysql database)))))))))+(deftest ^:parallel hour-bucketing-time-without-database-type-test+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "+ "fields referenced by name from a source query, #75193) should use the TIME-only format "+ "string and not produce a DATETIME-with-date format")+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]+ (is (= ["STR_TO_DATE(DATE_FORMAT(CAST(`test_col` AS datetime), '%H'), '%H')"]+ (sql.qp/format-honeysql :mysql (sql.qp/date :mysql :hour expr)))))))+(deftest date-test;; make sure stuff at least compiles. Even if the result probably isn't as concise as it could be.;; See [[metabase.query-processor.date-time-zone-functions-test/extract-week-tests]] for something that testsdiff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 6ea2990c9cb..4dc654c4a9c 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -85,6 +85,14 @@(is (= ["extract(month from NOW())"](sql.qp/format-honeysql :postgres (#'postgres/extract :month :%now)))))+(deftest ^:parallel hour-bucketing-time-without-database-type-test+ (testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "+ "fields referenced by name from a source query, #75193) should use the time-aware path "+ "and not cast to `timestamp`")+ (let [expr (h2x/with-type-info :test_col {:effective-type :type/Time})]+ (is (= ["MAKE_TIME(extract(hour from \"test_col\")::integer, 0, 0.0)"]+ (sql.qp/format-honeysql :postgres (sql.qp/date :postgres :hour expr)))))))+(deftest ^:parallel datetime-diff-test(is (= [["CAST("" extract("diff --git a/test/metabase/query_processor/time_field_test.clj b/test/metabase/query_processor/time_field_test.cljindex 1ed62deb2d1..c5f9aad79e1 100644--- a/test/metabase/query_processor/time_field_test.clj+++ b/test/metabase/query_processor/time_field_test.clj@@ -6,6 +6,7 @@[metabase.driver :as driver][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-util.notebook-helpers :as notebook-helpers][metabase.query-processor.test :as qp][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt]@@ -191,3 +192,39 @@(is (=? [[1 #"07:23:00Z?"][2 #"07:14:00Z?"]]results)))))))++(deftest hour-bucketing-time-field-in-source-query-test+ (testing "Hour group-by on a time field from a source query should work (#75193, #68065)"+ (mt/test-drivers (mt/normal-drivers-with-feature :test/time-type)+ (mt/dataset time-test-data+ (let [mp (mt/metadata-provider)+ users (lib.metadata/table mp (mt/id :users))+ time-field (lib.metadata/field mp (mt/id :users :last_login_time))+ ;; Two-stage query: stage 1 selects the time field; stage 2 groups it by :hour+ ;; with a count. In stage 2 the field is referenced by name (string), not by+ ;; integer ID — `field-metadata` is nil and `database-type` is lost. Drivers+ ;; that dispatch their `:hour` bucketing on `(h2x/database-type expr)` fall+ ;; through to a timestamp/datetime cast, which errors (Postgres, SQL Server)+ ;; or silently misgroups (MySQL, SQLite).+ query (-> (lib/query mp users)+ (lib/with-fields [time-field])+ lib/append-stage+ (notebook-helpers/add-breakout+ "Summaries" "Last Login Time"+ {:col-fn #(lib/with-temporal-bucket % :hour)})+ (as-> q (lib/aggregate q (lib/count))))]+ (mt/with-native-query-testing-context query+ (is (=? [[#"01:00:00Z?" 1]+ [#"07:00:00Z?" 1]+ [#"08:00:00Z?" 2]+ [#"09:00:00Z?" 2]+ [#"10:00:00Z?" 2]+ [#"12:00:00Z?" 2]+ [#"13:00:00Z?" 1]+ [#"15:00:00Z?" 1]+ [#"16:00:00Z?" 1]+ [#"17:00:00Z?" 1]+ [#"19:00:00Z?" 1]]+ (mt/formatted-rows+ [str int]+ (qp/process-query query))))))))))