Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use effective/base type info for casts if database-type is missing (#71771)

metambencdcf995f5f4e
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index 13ae8c932c4..35b77725641 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -404,10 +404,11 @@
(defn- date-trunc
[unit expr]
- (let [acceptable-types (case unit
- (:millisecond :second :minute :hour) #{"time" "timestampltz" "timestampntz" "timestamptz"}
- (:day :week :month :quarter :year) #{"date" "timestampltz" "timestampntz" "timestamptz"})
- expr (h2x/cast-unless-type-in "timestampntz" acceptable-types expr)]
+ (let [[acceptable-types effective-supertype]
+ (case unit
+ (:millisecond :second :minute :hour) [#{"time" "timestampltz" "timestampntz" "timestamptz"} :type/Temporal]
+ (:day :week :month :quarter :year) [#{"date" "timestampltz" "timestampntz" "timestamptz"} :type/HasDate])
+ expr (h2x/cast-unless-type-in "timestampntz" acceptable-types effective-supertype expr)]
(-> [:date_trunc (h2x/literal unit) (in-report-timezone expr)]
(h2x/with-database-type-info (h2x/database-type expr)))))
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index e854f24cfcc..9250a6b8671 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -25,6 +25,7 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
+ [metabase.lib.test-util.notebook-helpers :as notebook-helpers]
^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
[metabase.secrets.core :as secret]
@@ -1474,3 +1475,25 @@
table (:table-row (first rows))]
(is (= "users" (:name table)) "Should be plain name, not 'transform_test_users'")
(is (= "PUBLIC" (:schema table)))))))
+
+(deftest hour-bucketing-time-field-in-source-query-test
+ (testing "Hour group-by on time fields from a source query should work (#68065)"
+ (mt/test-driver :snowflake
+ (mt/dataset time-test-data
+ (let [mp (mt/metadata-provider)
+ users (lib.metadata/table mp (mt/id :users))
+ time-field (lib.metadata/field mp (mt/id :users :last_login_time))
+ ;; Build a two-stage query: the first stage selects the time field,
+ ;; the second stage groups it by :hour. In the second stage the field is
+ ;; referenced by name (string), not by integer ID — this is where the bug
+ ;; manifests: field-metadata is nil and database-type info is lost, causing
+ ;; Snowflake to incorrectly cast to TIMESTAMPNTZ.
+ query (-> (lib/query mp users)
+ (lib/with-fields [time-field])
+ lib/append-stage
+ (notebook-helpers/add-breakout
+ "Summaries" "Last Login Time"
+ {:col-fn #(lib/with-temporal-bucket % :hour)}))]
+ (mt/with-native-query-testing-context query
+ (is (some? (mt/rows (qp/process-query query)))
+ "Hour bucketing on a time field from a source query should not error")))))))
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index e1d94ee3a45..81a97c01716 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -923,14 +923,20 @@
casted-field (cast-field-if-needed driver field-metadata identifier)
database-type (or (h2x/database-type casted-field)
(:database-type field-metadata))
- maybe-add-db-type (fn [expr]
+ effective-type (when-not database-type
+ (let [et (or (:effective-type options) (:base-type options))]
+ (when (isa? et :type/Temporal)
+ et)))
+ maybe-add-type-info (fn [expr]
(if (h2x/type-info->db-type (h2x/type-info expr))
expr
- (h2x/with-database-type-info expr database-type)))]
+ (if database-type
+ (h2x/with-database-type-info expr database-type)
+ (h2x/with-type-info expr {:effective-type effective-type}))))]
(u/prog1
(cond->> (if allow-casting? casted-field identifier)
;; only add type info if it wasn't added by [[cast-field-if-needed]]
- database-type maybe-add-db-type
+ (or database-type effective-type) maybe-add-type-info
(:temporal-unit options) (apply-temporal-bucketing driver options)
(:binning options) (apply-binning options))
(log/trace (binding [*print-meta* true]
diff --git a/src/metabase/util/honey_sql_2.clj b/src/metabase/util/honey_sql_2.clj
index 180794a15aa..fcfecf85b8e 100644
--- a/src/metabase/util/honey_sql_2.clj
+++ b/src/metabase/util/honey_sql_2.clj
@@ -295,6 +295,13 @@
[honeysql-form]
(some-> honeysql-form type-info type-info->db-type))
+(defn effective-type
+ "Returns the Metabase effective type from the type-info of `honeysql-form` if present,
+ falling back to `:base-type`. Returns `nil` if neither is set."
+ [honeysql-form]
+ (let [info (type-info honeysql-form)]
+ (or (:effective-type info) (:base-type info))))
+
(defn is-of-type?
"Is `honeysql-form` a typed form with `db-type`?
Where `db-type` could be a string or a regex.
@@ -351,15 +358,25 @@
(defn cast-unless-type-in
"Cast `expr` to `desired-type` unless `expr` is of one of the `acceptable-types`. Returns a typed HoneySQL form.
+ When `database-type` is not available on `expr` but `effective-type` is, and `effective-type` is a descendant of
+ `effective-type-supertype`, the cast is also skipped. This handles card-sourced fields that lack `database-type`
+ metadata but have Metabase type information.
+
;; cast to TIMESTAMP unless form is already a TIMESTAMP, TIMESTAMPTZ, or DATE
- (cast-unless-type-in \"timestamp\" #{\"timestamp\" \"timestamptz\" \"date\"} form)"
+ (cast-unless-type-in \"timestamp\" #{\"timestamp\" \"timestamptz\" \"date\"} form)
+
+ ;; same, but also skip the cast if effective-type isa? :type/Temporal
+ (cast-unless-type-in \"timestamp\" #{\"timestamp\" \"timestamptz\" \"date\"} :type/Temporal form)"
{:added "0.42.0"}
- [desired-type acceptable-types expr]
- {:pre [(string? desired-type) (set? acceptable-types)]}
- (if (some (partial is-of-type? expr)
- acceptable-types)
- expr
- (cast desired-type expr)))
+ ([desired-type acceptable-types expr]
+ (cast-unless-type-in desired-type acceptable-types nil expr))
+ ([desired-type acceptable-types effective-type-supertype expr]
+ {:pre [(string? desired-type) (set? acceptable-types)]}
+ (if (or (some (partial is-of-type? expr) acceptable-types)
+ (when (and effective-type-supertype (not (database-type expr)))
+ (isa? (effective-type expr) effective-type-supertype)))
+ expr
+ (cast desired-type expr))))
(defn- math-operator [operator]
(fn [& args]
@@ -444,7 +461,7 @@
(defn ->pg-timestamp
"Cast to timestamp, preserving timestamptz if present."
[honeysql-form]
- (cast-unless-type-in "timestamp" #{"timestamp" "timestamptz" "timestamp with time zone" "date"} honeysql-form))
+ (cast-unless-type-in "timestamp" #{"timestamp" "timestamptz" "timestamp with time zone" "date"} :type/HasDate honeysql-form))
(defmulti add-interval-honeysql-form
"Return a HoneySQL form that represents addition of some temporal interval to the original `hsql-form`.
diff --git a/test/metabase/query_processor/explicit_joins_test.clj b/test/metabase/query_processor/explicit_joins_test.clj
index c081465880a..0428a010f1d 100644
--- a/test/metabase/query_processor/explicit_joins_test.clj
+++ b/test/metabase/query_processor/explicit_joins_test.clj
@@ -13,6 +13,7 @@
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.mocks-31769 :as lib.tu.mocks-31769]
+ [metabase.lib.test-util.notebook-helpers :as notebook-helpers]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.preprocess :as qp.preprocess]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
@@ -1524,6 +1525,56 @@
[:asc &o.orders.id]]
:limit 3})))))))
+(deftest datetime-diff-with-card-join-test
+ (testing "datetime-diff between a table field and a joined card field should produce correct results (#71551)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join :expressions :datetime-diff)
+ (mt/with-report-timezone-id! "US/Pacific"
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ orders-created (lib.metadata/field mp (mt/id :orders :created_at))
+ orders-product (lib.metadata/field mp (mt/id :orders :product_id))
+ build-query (fn [joined-q join-group-spec]
+ (let [joined-created (notebook-helpers/find-col-with-spec
+ joined-q
+ (lib/filterable-columns joined-q)
+ join-group-spec
+ {:semantic-type :type/CreationTimestamp})]
+ (-> joined-q
+ (lib/expression "diff" (lib/expression-clause
+ :datetime-diff
+ [orders-created joined-created :hour]
+ nil))
+ (as-> $q (lib/with-fields $q [(lib/expression-ref $q "diff")]))
+ (lib/order-by orders-created :asc)
+ (lib/limit 5))))
+ tbl-query (build-query
+ (-> (lib/query mp orders)
+ (lib/join (-> (lib/join-clause
+ (lib.metadata/table mp (mt/id :products))
+ [(lib/= orders-product
+ (lib.metadata/field mp (mt/id :products :id)))])
+ (lib/with-join-alias "P"))))
+ {:name "P"})
+ mp2 (qp.test-util/metadata-provider-with-cards-with-metadata-for-queries
+ [(mt/mbql-query products)])
+ card (lib.metadata/card mp2 1)
+ card-query (build-query
+ (-> (lib/query mp2 orders)
+ (as-> $q
+ (lib/join $q (-> (lib/join-clause card)
+ (lib/with-join-alias "Card")
+ (lib/with-join-conditions
+ [(let [rhs-cols (lib/join-condition-rhs-columns
+ $q card (lib/ref orders-product) nil)]
+ (lib/= orders-product
+ (-> (notebook-helpers/find-col-with-spec
+ $q rhs-cols "Card 1" "ID")
+ (lib/with-join-alias "Card"))))])))))
+ "Card 1")]
+ (mt/with-native-query-testing-context card-query
+ (is (= (mt/rows (qp/process-query tbl-query))
+ (mt/rows (qp/process-query card-query))))))))))
+
(deftest ^:parallel self-join-in-source-card-test
(testing "When query uses a source card with a self-join, query should work (#27521)"
(let [mp (mt/metadata-provider)