The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Use effective/base type info for casts if database-type is missing (#71771)
metambencdcf995f5f4e
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex 13ae8c932c4..35b77725641 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -404,10 +404,11 @@(defn- date-trunc[unit expr]- (let [acceptable-types (case unit- (:millisecond :second :minute :hour) #{"time" "timestampltz" "timestampntz" "timestamptz"}- (:day :week :month :quarter :year) #{"date" "timestampltz" "timestampntz" "timestamptz"})- expr (h2x/cast-unless-type-in "timestampntz" acceptable-types expr)]+ (let [[acceptable-types effective-supertype]+ (case unit+ (:millisecond :second :minute :hour) [#{"time" "timestampltz" "timestampntz" "timestamptz"} :type/Temporal]+ (:day :week :month :quarter :year) [#{"date" "timestampltz" "timestampntz" "timestamptz"} :type/HasDate])+ expr (h2x/cast-unless-type-in "timestampntz" acceptable-types effective-supertype expr)](-> [:date_trunc (h2x/literal unit) (in-report-timezone expr)](h2x/with-database-type-info (h2x/database-type expr)))))diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex e854f24cfcc..9250a6b8671 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -25,6 +25,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu]+ [metabase.lib.test-util.notebook-helpers :as notebook-helpers]^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp][metabase.secrets.core :as secret]@@ -1474,3 +1475,25 @@table (:table-row (first rows))](is (= "users" (:name table)) "Should be plain name, not 'transform_test_users'")(is (= "PUBLIC" (:schema table)))))))++(deftest hour-bucketing-time-field-in-source-query-test+ (testing "Hour group-by on time fields from a source query should work (#68065)"+ (mt/test-driver :snowflake+ (mt/dataset time-test-data+ (let [mp (mt/metadata-provider)+ users (lib.metadata/table mp (mt/id :users))+ time-field (lib.metadata/field mp (mt/id :users :last_login_time))+ ;; Build a two-stage query: the first stage selects the time field,+ ;; the second stage groups it by :hour. In the second stage the field is+ ;; referenced by name (string), not by integer ID — this is where the bug+ ;; manifests: field-metadata is nil and database-type info is lost, causing+ ;; Snowflake to incorrectly cast to TIMESTAMPNTZ.+ query (-> (lib/query mp users)+ (lib/with-fields [time-field])+ lib/append-stage+ (notebook-helpers/add-breakout+ "Summaries" "Last Login Time"+ {:col-fn #(lib/with-temporal-bucket % :hour)}))]+ (mt/with-native-query-testing-context query+ (is (some? (mt/rows (qp/process-query query)))+ "Hour bucketing on a time field from a source query should not error")))))))diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex e1d94ee3a45..81a97c01716 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -923,14 +923,20 @@casted-field (cast-field-if-needed driver field-metadata identifier)database-type (or (h2x/database-type casted-field)(:database-type field-metadata))- maybe-add-db-type (fn [expr]+ effective-type (when-not database-type+ (let [et (or (:effective-type options) (:base-type options))]+ (when (isa? et :type/Temporal)+ et)))+ maybe-add-type-info (fn [expr](if (h2x/type-info->db-type (h2x/type-info expr))expr- (h2x/with-database-type-info expr database-type)))]+ (if database-type+ (h2x/with-database-type-info expr database-type)+ (h2x/with-type-info expr {:effective-type effective-type}))))](u/prog1(cond->> (if allow-casting? casted-field identifier);; only add type info if it wasn't added by [[cast-field-if-needed]]- database-type maybe-add-db-type+ (or database-type effective-type) maybe-add-type-info(:temporal-unit options) (apply-temporal-bucketing driver options)(:binning options) (apply-binning options))(log/trace (binding [*print-meta* true]diff --git a/src/metabase/util/honey_sql_2.clj b/src/metabase/util/honey_sql_2.cljindex 180794a15aa..fcfecf85b8e 100644--- a/src/metabase/util/honey_sql_2.clj+++ b/src/metabase/util/honey_sql_2.clj@@ -295,6 +295,13 @@[honeysql-form](some-> honeysql-form type-info type-info->db-type))+(defn effective-type+ "Returns the Metabase effective type from the type-info of `honeysql-form` if present,+ falling back to `:base-type`. Returns `nil` if neither is set."+ [honeysql-form]+ (let [info (type-info honeysql-form)]+ (or (:effective-type info) (:base-type info))))+(defn is-of-type?"Is `honeysql-form` a typed form with `db-type`?Where `db-type` could be a string or a regex.@@ -351,15 +358,25 @@(defn cast-unless-type-in"Cast `expr` to `desired-type` unless `expr` is of one of the `acceptable-types`. Returns a typed HoneySQL form.+ When `database-type` is not available on `expr` but `effective-type` is, and `effective-type` is a descendant of+ `effective-type-supertype`, the cast is also skipped. This handles card-sourced fields that lack `database-type`+ metadata but have Metabase type information.+;; cast to TIMESTAMP unless form is already a TIMESTAMP, TIMESTAMPTZ, or DATE- (cast-unless-type-in \"timestamp\" #{\"timestamp\" \"timestamptz\" \"date\"} form)"+ (cast-unless-type-in \"timestamp\" #{\"timestamp\" \"timestamptz\" \"date\"} form)++ ;; same, but also skip the cast if effective-type isa? :type/Temporal+ (cast-unless-type-in \"timestamp\" #{\"timestamp\" \"timestamptz\" \"date\"} :type/Temporal form)"{:added "0.42.0"}- [desired-type acceptable-types expr]- {:pre [(string? desired-type) (set? acceptable-types)]}- (if (some (partial is-of-type? expr)- acceptable-types)- expr- (cast desired-type expr)))+ ([desired-type acceptable-types expr]+ (cast-unless-type-in desired-type acceptable-types nil expr))+ ([desired-type acceptable-types effective-type-supertype expr]+ {:pre [(string? desired-type) (set? acceptable-types)]}+ (if (or (some (partial is-of-type? expr) acceptable-types)+ (when (and effective-type-supertype (not (database-type expr)))+ (isa? (effective-type expr) effective-type-supertype)))+ expr+ (cast desired-type expr))))(defn- math-operator [operator](fn [& args]@@ -444,7 +461,7 @@(defn ->pg-timestamp"Cast to timestamp, preserving timestamptz if present."[honeysql-form]- (cast-unless-type-in "timestamp" #{"timestamp" "timestamptz" "timestamp with time zone" "date"} honeysql-form))+ (cast-unless-type-in "timestamp" #{"timestamp" "timestamptz" "timestamp with time zone" "date"} :type/HasDate honeysql-form))(defmulti add-interval-honeysql-form"Return a HoneySQL form that represents addition of some temporal interval to the original `hsql-form`.diff --git a/test/metabase/query_processor/explicit_joins_test.clj b/test/metabase/query_processor/explicit_joins_test.cljindex c081465880a..0428a010f1d 100644--- a/test/metabase/query_processor/explicit_joins_test.clj+++ b/test/metabase/query_processor/explicit_joins_test.clj@@ -13,6 +13,7 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.mocks-31769 :as lib.tu.mocks-31769]+ [metabase.lib.test-util.notebook-helpers :as notebook-helpers][metabase.query-processor.compile :as qp.compile][metabase.query-processor.preprocess :as qp.preprocess]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]@@ -1524,6 +1525,56 @@[:asc &o.orders.id]]:limit 3})))))))+(deftest datetime-diff-with-card-join-test+ (testing "datetime-diff between a table field and a joined card field should produce correct results (#71551)"+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join :expressions :datetime-diff)+ (mt/with-report-timezone-id! "US/Pacific"+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ orders-created (lib.metadata/field mp (mt/id :orders :created_at))+ orders-product (lib.metadata/field mp (mt/id :orders :product_id))+ build-query (fn [joined-q join-group-spec]+ (let [joined-created (notebook-helpers/find-col-with-spec+ joined-q+ (lib/filterable-columns joined-q)+ join-group-spec+ {:semantic-type :type/CreationTimestamp})]+ (-> joined-q+ (lib/expression "diff" (lib/expression-clause+ :datetime-diff+ [orders-created joined-created :hour]+ nil))+ (as-> $q (lib/with-fields $q [(lib/expression-ref $q "diff")]))+ (lib/order-by orders-created :asc)+ (lib/limit 5))))+ tbl-query (build-query+ (-> (lib/query mp orders)+ (lib/join (-> (lib/join-clause+ (lib.metadata/table mp (mt/id :products))+ [(lib/= orders-product+ (lib.metadata/field mp (mt/id :products :id)))])+ (lib/with-join-alias "P"))))+ {:name "P"})+ mp2 (qp.test-util/metadata-provider-with-cards-with-metadata-for-queries+ [(mt/mbql-query products)])+ card (lib.metadata/card mp2 1)+ card-query (build-query+ (-> (lib/query mp2 orders)+ (as-> $q+ (lib/join $q (-> (lib/join-clause card)+ (lib/with-join-alias "Card")+ (lib/with-join-conditions+ [(let [rhs-cols (lib/join-condition-rhs-columns+ $q card (lib/ref orders-product) nil)]+ (lib/= orders-product+ (-> (notebook-helpers/find-col-with-spec+ $q rhs-cols "Card 1" "ID")+ (lib/with-join-alias "Card"))))])))))+ "Card 1")]+ (mt/with-native-query-testing-context card-query+ (is (= (mt/rows (qp/process-query tbl-query))+ (mt/rows (qp/process-query card-query))))))))))+(deftest ^:parallel self-join-in-source-card-test(testing "When query uses a source card with a self-join, query should work (#27521)"(let [mp (mt/metadata-provider)