Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix coercion for UNIX timestamp fields via implicit joins (#68164)

Timothy S. Deane694056f0fc7
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index 8b10e09a92f..d54f8628820 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -448,7 +448,8 @@ function(bin) {
(->rvalue (assoc (driver-api/field (driver-api/metadata-provider) id-or-name)
::source-alias source-alias
::join-field join-field
- ::inherited? (not (pos-int? (driver-api/qp.add.source-table opts))))))
+ ::inherited? (not (or (pos-int? (driver-api/qp.add.source-table opts))
+ (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table opts))))))
(if-let [mapped (find-mapped-field-name field)]
(str \$ mapped)
(str \$ (scope-with-join-field (name id-or-name) join-field source-alias))))
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index 99864e2a79d..fdd5dbb129d 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -280,6 +280,7 @@
add-field
fieldable-columns
fields
+ find-column-for-name
find-visible-column-for-ref
remove-field
with-fields]
diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljc
index 2553fae7e50..0159cbb211e 100644
--- a/src/metabase/lib/field.cljc
+++ b/src/metabase/lib/field.cljc
@@ -731,6 +731,25 @@
query stage-number)]
(lib.equality/find-matching-column query stage-number field-ref columns))))
+(mu/defn find-column-for-name :- [:maybe ::lib.schema.metadata/column]
+ "Find the first column in `columns` whose `:name` matches `column-name`.
+ Matching is case-insensitive.
+
+ Optionally pass a `pred` function for additional filtering; only columns
+ satisfying `(pred column)` will be considered."
+ ([columns :- [:sequential ::lib.schema.metadata/column]
+ column-name :- :string]
+ (find-column-for-name columns column-name nil))
+
+ ([columns :- [:sequential ::lib.schema.metadata/column]
+ column-name :- :string
+ pred :- [:maybe ifn?]]
+ (let [lower-name (u/lower-case-en column-name)]
+ (m/find-first (fn [col]
+ (and (= (u/lower-case-en (:name col)) lower-name)
+ (or (nil? pred) (pred col))))
+ columns))))
+
(defn json-field?
"Return true if field is a JSON field, false if not."
[field]
diff --git a/src/metabase/query_processor/util/add_alias_info.clj b/src/metabase/query_processor/util/add_alias_info.clj
index cb8a73750d6..8aec783cf06 100644
--- a/src/metabase/query_processor/util/add_alias_info.clj
+++ b/src/metabase/query_processor/util/add_alias_info.clj
@@ -222,6 +222,12 @@
field-ref #(-> %
(assoc ::source-table (source-table query path col)
::source-alias (escaped-source-alias query path join-alias source-col-alias))
+ ;; For implicit joins, ::source-table will be a string (join alias) rather than an
+ ;; integer table ID. We need to explicitly enable coercion since the SQL QP checks
+ ;; for (pos-int? ::source-table) to determine if coercion should be applied. This
+ ;; follows the same pattern used by the SparkSQL driver. See #67704.
+ (cond-> implicit-join?
+ (assoc :qp/allow-coercion-for-columns-without-integer-qp.add.source-table true))
(m/assoc-some ::nfc-path (not-empty (:nfc-path col)))))))
(defn- fix-field-ref-if-it-should-actually-be-an-expression-ref
diff --git a/test/metabase/query_processor/implicit_joins_test.clj b/test/metabase/query_processor/implicit_joins_test.clj
index e2a4b37fa47..b158686127d 100644
--- a/test/metabase/query_processor/implicit_joins_test.clj
+++ b/test/metabase/query_processor/implicit_joins_test.clj
@@ -286,3 +286,97 @@
(lib/breakout big-col))]
(is (= [["b1"] ["b2"]]
(mt/rows (qp/process-query query))))))))))
+
+(mt/defdataset unix-timestamp-fk-dataset
+ [["events" [{:field-name "event_timestamp"
+ :base-type :type/Integer
+ :effective-type :type/DateTime
+ :coercion-strategy :Coercion/UNIXSeconds->DateTime}
+ {:field-name "name"
+ :base-type :type/Text}]
+ ;; UNIX timestamps for dates in 2024
+ [[1704067200 "New Year 2024"] ; 2024-01-01 00:00:00 UTC
+ [1706745600 "Groundhog Day 2024"] ; 2024-02-01 00:00:00 UTC
+ [1709251200 "March 2024"]]] ; 2024-03-01 00:00:00 UTC
+ ["logs" [{:field-name "event_id"
+ :base-type :type/Integer
+ :fk :events}
+ {:field-name "message"
+ :base-type :type/Text}]
+ [[1 "Log entry 1"]
+ [1 "Log entry 2"]
+ [2 "Log entry 3"]
+ [3 "Log entry 4"]
+ [3 "Log entry 5"]]]])
+
+(deftest coerced-field-via-implicit-join-test
+ (testing "Filtering/aggregating on a coerced UNIX field via implicit join should apply coercion (#67704)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)
+ (mt/dataset unix-timestamp-fk-dataset
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/table mp (mt/id :logs)))
+ ;; Find the event_timestamp column that's implicitly joinable via FK.
+ ;; Columns with :fk-field-id are from implicit joins.
+ filterable-cols (lib/filterable-columns query)
+ event-timestamp (lib/find-column-for-name filterable-cols "event_timestamp" :fk-field-id)
+ _ (is (some? event-timestamp) "Should find event_timestamp via implicit join")]
+ (testing "Filter on coerced field via implicit join"
+ ;; This would fail with "operator does not exist: integer >= timestamp"
+ ;; if coercion is not applied to the implicitly joined field.
+ (let [query (-> query
+ (lib/aggregate (lib/count))
+ (lib/filter (lib/> event-timestamp
+ (lib/absolute-datetime #t "2024-01-15T00:00:00Z" :default))))]
+ (mt/with-native-query-testing-context query
+ ;; 3 logs for events after Jan 15 (Groundhog Day + March)
+ (is (= [[3]]
+ (mt/formatted-rows [int] (qp/process-query query)))))))
+ (testing "Breakout on coerced field via implicit join"
+ ;; This would fail with "function date_trunc(unknown, integer) does not exist"
+ ;; if coercion is not applied to the implicitly joined field.
+ (let [query (-> query
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib/with-temporal-bucket event-timestamp :month)))]
+ (mt/with-native-query-testing-context query
+ ;; 3 months with log counts: Jan=2, Feb=1, Mar=2
+ (is (= 3
+ (count (mt/rows (qp/process-query query)))))))))))))
+
+(deftest coerced-field-via-explicit-join-test
+ (testing "Coerced fields via explicit joins should also work correctly (sanity check for #67704 fix)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :left-join)
+ (mt/dataset unix-timestamp-fk-dataset
+ (let [mp (mt/metadata-provider)
+ logs-table (lib.metadata/table mp (mt/id :logs))
+ events-table (lib.metadata/table mp (mt/id :events))
+ event-id-col (lib.metadata/field mp (mt/id :logs :event_id))
+ events-id-col (lib.metadata/field mp (mt/id :events :id))
+ ;; Create explicit join
+ join-clause (-> (lib/join-clause events-table)
+ (lib/with-join-conditions [(lib/= event-id-col
+ (lib/with-join-alias events-id-col "Events"))])
+ (lib/with-join-fields :all))
+ query (-> (lib/query mp logs-table)
+ (lib/join join-clause))
+ ;; Find the event_timestamp column from the explicit join.
+ ;; Columns from explicit joins have :metabase.lib.join/join-alias.
+ filterable-cols (lib/filterable-columns query)
+ event-timestamp (lib/find-column-for-name filterable-cols "event_timestamp" :metabase.lib.join/join-alias)
+ _ (is (some? event-timestamp) "Should find event_timestamp via explicit join")]
+ (testing "Filter on coerced field via explicit join"
+ (let [query (-> query
+ (lib/aggregate (lib/count))
+ (lib/filter (lib/> event-timestamp
+ (lib/absolute-datetime #t "2024-01-15T00:00:00Z" :default))))]
+ (mt/with-native-query-testing-context query
+ ;; 3 logs for events after Jan 15 (Groundhog Day + March)
+ (is (= [[3]]
+ (mt/formatted-rows [int] (qp/process-query query)))))))
+ (testing "Breakout on coerced field via explicit join"
+ (let [query (-> query
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib/with-temporal-bucket event-timestamp :month)))]
+ (mt/with-native-query-testing-context query
+ ;; 3 months with log counts
+ (is (= 3
+ (count (mt/rows (qp/process-query query)))))))))))))