The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix coercion for UNIX timestamp fields via implicit joins (#68164)
Timothy S. Deane694056f0fc7
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.cljindex 8b10e09a92f..d54f8628820 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj@@ -448,7 +448,8 @@ function(bin) {(->rvalue (assoc (driver-api/field (driver-api/metadata-provider) id-or-name)::source-alias source-alias::join-field join-field- ::inherited? (not (pos-int? (driver-api/qp.add.source-table opts))))))+ ::inherited? (not (or (pos-int? (driver-api/qp.add.source-table opts))+ (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table opts))))))(if-let [mapped (find-mapped-field-name field)](str \$ mapped)(str \$ (scope-with-join-field (name id-or-name) join-field source-alias))))diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex 99864e2a79d..fdd5dbb129d 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -280,6 +280,7 @@add-fieldfieldable-columnsfields+ find-column-for-namefind-visible-column-for-refremove-fieldwith-fields]diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljcindex 2553fae7e50..0159cbb211e 100644--- a/src/metabase/lib/field.cljc+++ b/src/metabase/lib/field.cljc@@ -731,6 +731,25 @@query stage-number)](lib.equality/find-matching-column query stage-number field-ref columns))))+(mu/defn find-column-for-name :- [:maybe ::lib.schema.metadata/column]+ "Find the first column in `columns` whose `:name` matches `column-name`.+ Matching is case-insensitive.++ Optionally pass a `pred` function for additional filtering; only columns+ satisfying `(pred column)` will be considered."+ ([columns :- [:sequential ::lib.schema.metadata/column]+ column-name :- :string]+ (find-column-for-name columns column-name nil))++ ([columns :- [:sequential ::lib.schema.metadata/column]+ column-name :- :string+ pred :- [:maybe ifn?]]+ (let [lower-name (u/lower-case-en column-name)]+ (m/find-first (fn [col]+ (and (= (u/lower-case-en (:name col)) lower-name)+ (or (nil? pred) (pred col))))+ columns))))+(defn json-field?"Return true if field is a JSON field, false if not."[field]diff --git a/src/metabase/query_processor/util/add_alias_info.clj b/src/metabase/query_processor/util/add_alias_info.cljindex cb8a73750d6..8aec783cf06 100644--- a/src/metabase/query_processor/util/add_alias_info.clj+++ b/src/metabase/query_processor/util/add_alias_info.clj@@ -222,6 +222,12 @@field-ref #(-> %(assoc ::source-table (source-table query path col)::source-alias (escaped-source-alias query path join-alias source-col-alias))+ ;; For implicit joins, ::source-table will be a string (join alias) rather than an+ ;; integer table ID. We need to explicitly enable coercion since the SQL QP checks+ ;; for (pos-int? ::source-table) to determine if coercion should be applied. This+ ;; follows the same pattern used by the SparkSQL driver. See #67704.+ (cond-> implicit-join?+ (assoc :qp/allow-coercion-for-columns-without-integer-qp.add.source-table true))(m/assoc-some ::nfc-path (not-empty (:nfc-path col)))))))(defn- fix-field-ref-if-it-should-actually-be-an-expression-refdiff --git a/test/metabase/query_processor/implicit_joins_test.clj b/test/metabase/query_processor/implicit_joins_test.cljindex e2a4b37fa47..b158686127d 100644--- a/test/metabase/query_processor/implicit_joins_test.clj+++ b/test/metabase/query_processor/implicit_joins_test.clj@@ -286,3 +286,97 @@(lib/breakout big-col))](is (= [["b1"] ["b2"]](mt/rows (qp/process-query query))))))))))++(mt/defdataset unix-timestamp-fk-dataset+ [["events" [{:field-name "event_timestamp"+ :base-type :type/Integer+ :effective-type :type/DateTime+ :coercion-strategy :Coercion/UNIXSeconds->DateTime}+ {:field-name "name"+ :base-type :type/Text}]+ ;; UNIX timestamps for dates in 2024+ [[1704067200 "New Year 2024"] ; 2024-01-01 00:00:00 UTC+ [1706745600 "Groundhog Day 2024"] ; 2024-02-01 00:00:00 UTC+ [1709251200 "March 2024"]]] ; 2024-03-01 00:00:00 UTC+ ["logs" [{:field-name "event_id"+ :base-type :type/Integer+ :fk :events}+ {:field-name "message"+ :base-type :type/Text}]+ [[1 "Log entry 1"]+ [1 "Log entry 2"]+ [2 "Log entry 3"]+ [3 "Log entry 4"]+ [3 "Log entry 5"]]]])++(deftest coerced-field-via-implicit-join-test+ (testing "Filtering/aggregating on a coerced UNIX field via implicit join should apply coercion (#67704)"+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (mt/dataset unix-timestamp-fk-dataset+ (let [mp (mt/metadata-provider)+ query (lib/query mp (lib.metadata/table mp (mt/id :logs)))+ ;; Find the event_timestamp column that's implicitly joinable via FK.+ ;; Columns with :fk-field-id are from implicit joins.+ filterable-cols (lib/filterable-columns query)+ event-timestamp (lib/find-column-for-name filterable-cols "event_timestamp" :fk-field-id)+ _ (is (some? event-timestamp) "Should find event_timestamp via implicit join")]+ (testing "Filter on coerced field via implicit join"+ ;; This would fail with "operator does not exist: integer >= timestamp"+ ;; if coercion is not applied to the implicitly joined field.+ (let [query (-> query+ (lib/aggregate (lib/count))+ (lib/filter (lib/> event-timestamp+ (lib/absolute-datetime #t "2024-01-15T00:00:00Z" :default))))]+ (mt/with-native-query-testing-context query+ ;; 3 logs for events after Jan 15 (Groundhog Day + March)+ (is (= [[3]]+ (mt/formatted-rows [int] (qp/process-query query)))))))+ (testing "Breakout on coerced field via implicit join"+ ;; This would fail with "function date_trunc(unknown, integer) does not exist"+ ;; if coercion is not applied to the implicitly joined field.+ (let [query (-> query+ (lib/aggregate (lib/count))+ (lib/breakout (lib/with-temporal-bucket event-timestamp :month)))]+ (mt/with-native-query-testing-context query+ ;; 3 months with log counts: Jan=2, Feb=1, Mar=2+ (is (= 3+ (count (mt/rows (qp/process-query query)))))))))))))++(deftest coerced-field-via-explicit-join-test+ (testing "Coerced fields via explicit joins should also work correctly (sanity check for #67704 fix)"+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)+ (mt/dataset unix-timestamp-fk-dataset+ (let [mp (mt/metadata-provider)+ logs-table (lib.metadata/table mp (mt/id :logs))+ events-table (lib.metadata/table mp (mt/id :events))+ event-id-col (lib.metadata/field mp (mt/id :logs :event_id))+ events-id-col (lib.metadata/field mp (mt/id :events :id))+ ;; Create explicit join+ join-clause (-> (lib/join-clause events-table)+ (lib/with-join-conditions [(lib/= event-id-col+ (lib/with-join-alias events-id-col "Events"))])+ (lib/with-join-fields :all))+ query (-> (lib/query mp logs-table)+ (lib/join join-clause))+ ;; Find the event_timestamp column from the explicit join.+ ;; Columns from explicit joins have :metabase.lib.join/join-alias.+ filterable-cols (lib/filterable-columns query)+ event-timestamp (lib/find-column-for-name filterable-cols "event_timestamp" :metabase.lib.join/join-alias)+ _ (is (some? event-timestamp) "Should find event_timestamp via explicit join")]+ (testing "Filter on coerced field via explicit join"+ (let [query (-> query+ (lib/aggregate (lib/count))+ (lib/filter (lib/> event-timestamp+ (lib/absolute-datetime #t "2024-01-15T00:00:00Z" :default))))]+ (mt/with-native-query-testing-context query+ ;; 3 logs for events after Jan 15 (Groundhog Day + March)+ (is (= [[3]]+ (mt/formatted-rows [int] (qp/process-query query)))))))+ (testing "Breakout on coerced field via explicit join"+ (let [query (-> query+ (lib/aggregate (lib/count))+ (lib/breakout (lib/with-temporal-bucket event-timestamp :month)))]+ (mt/with-native-query-testing-context query+ ;; 3 months with log counts+ (is (= 3+ (count (mt/rows (qp/process-query query)))))))))))))