Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[QP] Do coercions (eg. Unix timestamps) with native sandboxing (RCLS) (#70885)

Braden Shepherdson3559ec98b08c
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
index 58e4c8d2de9..5c8e16285da 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
@@ -284,7 +284,12 @@
(keep (fn [table-col]
(when-let [native-col (m/find-first #(= (:name %) (:name table-col))
native-cols)]
- (merge native-col table-col))))
+ (cond-> (merge native-col table-col)
+ ;; If the table column would have had a coercion strategy applied, add some metadata keys to
+ ;; ensure the strategy is propagated to the first MBQL stage.
+ (:coercion-strategy table-col)
+ (assoc :qp/native-sandbox-column.force-coercion-strategy (:coercion-strategy table-col)
+ :qp/native-sandbox-column.propagate-coercion? true)))))
original-table-cols))
(mu/defn- apply-sandbox-to-stage :- [:and
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index 083bc5f06a8..63835999d15 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -1824,3 +1824,78 @@
(testing "text field → :string/="
(is (= :string/=
(:type (attr-remapping->parameter mp {"cat" "foo"} ["cat" [:variable [:field (mt/id :venues :name) nil]]]))))))))
+
+(deftest unix-timestamp-coercion-with-mbql-sandbox-test
+ (testing "UNIX timestamp coercion should be applied when querying through an MBQL sandbox (#69867)"
+ (mt/test-drivers (e2e-test-drivers)
+ ;; Use venues.price (an integer column) and temporarily give it a UNIX timestamp coercion strategy.
+ ;; With coercion applied, price values (1-4) become timestamps near the Unix epoch.
+ ;; Without coercion, they come back as raw integers.
+ (met/with-gtaps! {:gtaps {:venues {:query (mt/mbql-query venues)}}
+ :attributes {}}
+ (tu/with-temp-vals-in-db :model/Field (mt/id :venues :price)
+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime
+ :effective_type :type/Instant}
+ (let [result (mt/run-mbql-query venues {:limit 1, :order-by [[:asc $id]]})]
+ ;; venues.price for venue 1 is 3; with coercion this becomes 1970-01-01T00:00:03Z
+ ;; The native_form SQL should contain a timestamp coercion expression
+ (is (string? (-> result mt/rows first last))
+ "Price column should be coerced to a timestamp string, not returned as a raw integer")))))))
+
+(deftest unix-timestamp-coercion-with-native-sandbox-test
+ (testing "UNIX timestamp coercion should be applied when querying through a native SQL sandbox (#69867)"
+ (mt/test-drivers (e2e-test-drivers)
+ ;; Use venues.price (an integer column) and temporarily give it a UNIX timestamp coercion strategy.
+ ;; With coercion applied, price values (1-4) become timestamps near the Unix epoch.
+ ;; Without coercion, they come back as raw integers.
+ (met/with-gtaps! {:gtaps {:venues {:query (mt/native-query
+ {:query (format-honeysql
+ {:select [:*]
+ :from [[(identifier :venues)]]
+ :order-by [[(identifier :venues :id) :asc]]})})}}
+ :attributes {}}
+ (tu/with-temp-vals-in-db :model/Field (mt/id :venues :price)
+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime
+ :effective_type :type/Instant}
+ (let [result (mt/run-mbql-query venues {:limit 1, :order-by [[:asc $id]]})]
+ ;; BUG (#69867): With a native SQL sandbox, the coercion is NOT applied.
+ (is (string? (-> result mt/rows first last))
+ "Price column should be coerced to a timestamp string, not returned as a raw integer")))))))
+
+(deftest unix-timestamp-coercion-with-native-sandbox-and-multiple-stages-test
+ (testing "UNIX timestamp coercion should be applied when querying through a native SQL sandbox (#69867)"
+ (mt/test-drivers (e2e-test-drivers)
+ ;; Use venues.price (an integer column) and temporarily give it a UNIX timestamp coercion strategy.
+ ;; With coercion applied, price values (1-4) become timestamps near the Unix epoch.
+ ;; Without coercion, they come back as raw integers.
+ (testing "and a multi-stage query should apply coercion exactly once (no double-coercion)"
+ (met/with-gtaps! {:gtaps {:venues {:query (mt/native-query
+ {:query (format-honeysql
+ {:select [:*]
+ :from [[(identifier :venues)]]
+ :order-by [[(identifier :venues :id) :asc]]})})}}
+ :attributes {}}
+ (tu/with-temp-vals-in-db :model/Field (mt/id :venues :price)
+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime
+ :effective_type :type/Instant}
+ ;; A 2-stage query: inner stage selects from venues, outer stage wraps it.
+ ;; With sandboxing this becomes 3 levels deep. Coercion must be applied only in the
+ ;; innermost stage (closest to the table), not re-applied in outer stages.
+ ;; If double-coerced, the value would be wildly wrong (treating a timestamp as seconds
+ ;; and adding it to epoch again).
+ (let [mp (mt/metadata-provider)
+ base (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/order-by (lib.metadata/field mp (mt/id :venues :id)))
+ (lib/limit 1)
+ lib/append-stage)
+ field (lib.metadata/field mp (mt/id :venues :price))
+ price (m/find-first #(= (:name field) (:name %))
+ (lib/filterable-columns base))
+ query (lib/filter base (lib/< price "2037-12-31"))
+ [row] (-> query qp/process-query mt/rows)]
+ ;; price=3 → coerced once → "1970-01-01T00:00:03Z"
+ ;; price=3 → coerced twice → would be an enormous date or an error
+ (is (string? (last row))
+ "Price column should be coerced to a timestamp string")
+ (is (str/starts-with? (last row) "1970-01-01")
+ "Price should be coerced exactly once, producing a date near the Unix epoch"))))))))
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index d3622c2e08a..2e8645a0d3e 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -902,15 +902,24 @@
;; https://linear.app/metabase-inc/issue/ENG-8766/[epic]-field-refs-overhaul
field-metadata (when (integer? id-or-name)
(driver-api/field (driver-api/metadata-provider) id-or-name))
- allow-casting? (and field-metadata
- (or (pos-int? (driver-api/qp.add.source-table options))
- (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table options))
+ allow-casting? (and (or (:qp/native-sandbox-column.force-coercion-strategy options)
+ (and field-metadata
+ (or (pos-int? (driver-api/qp.add.source-table options))
+ (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table options))))
(not (:qp/ignore-coercion options)))
;; preserve metadata attached to the original field clause, for example BigQuery temporal type information.
identifier (-> (apply h2x/identifier :field
(concat source-table-aliases (->honeysql driver [::nfc-path source-nfc-path]) [source-alias]))
(with-meta (meta field-clause)))
identifier (->honeysql driver identifier)
+ ;; If no field-metadata is available, but a coercion strategy must be applied, synthesize enough
+ ;; `field-metadata` to be able to cast it correctly.
+ field-metadata (or field-metadata
+ (when allow-casting?
+ (-> options
+ (select-keys [:base-type :effetive-type])
+ (assoc :coercion-strategy
+ (:qp/native-sandbox-column.force-coercion-strategy options)))))
casted-field (cast-field-if-needed driver field-metadata identifier)
database-type (or (h2x/database-type casted-field)
(:database-type field-metadata))
diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljc
index 8645d2ffa44..74a24eaa9d8 100644
--- a/src/metabase/lib/field.cljc
+++ b/src/metabase/lib/field.cljc
@@ -427,7 +427,8 @@
[:base-type
:inherited-temporal-unit
:lib/original-binning
- :lib/original-effective-type])
+ :lib/original-effective-type
+ :qp/native-sandbox-column.force-coercion-strategy])
{:lib/binning :binning
:lib/temporal-unit :temporal-unit
:lib/ref-name :name
diff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljc
index 62a28fd24cb..b805f7e4300 100644
--- a/src/metabase/lib/field/util.cljc
+++ b/src/metabase/lib/field/util.cljc
@@ -124,6 +124,18 @@
;; desired-column-alias is previous stage => source column alias in next stage
:lib/source-column-alias ((some-fn :lib/desired-column-alias :lib/source-column-alias :name) col)
:lib/source :source/previous-stage)
+
+ ;; Native sandboxes need special handling for any type coercions set on the sandboxed table's fields.
+ ;; The columns first appear in the native stage, but we need to propagate the coercion metadata to the next stage
+ ;; so it gets applied in MBQL. After that first propagation, it should always be removed to prevent
+ ;; double-coercion. So sandboxing middleware sets two fields on the metadata: a flag, and the coercion strategy.
+ ;; If the input column has both set, we propagate the strategy into the next stage. The flag is never propagated,
+ ;; so the coercion strategy only reaches one stage after the native sandbox, as desired! See QUE2-376.
+ (dissoc :qp/native-sandbox-column.propagate-coercion?)
+ (cond-> #_col
+ (not (:qp/native-sandbox-column.propagate-coercion? col))
+ (dissoc :qp/native-sandbox-column.force-coercion-strategy))
+
;;
;; Remove `:lib/desired-column-alias`, which needs to be recalculated in the context
;; of what is returned by the current stage, to prevent any confusion; its value is likely wrong now and we
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index 2b4bcab8894..ac4c4bf8f25 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -516,7 +516,23 @@
;; stage. [[metabase.lib.metadata.result-metadata/super-broken-legacy-field-ref]] uses this to know to force Field
;; ID refs for QP `:field_ref` in results metadata to preserve historic behavior to avoid breaking legacy viz
;; settings that use it as a key.
- [:qp/implicit-field? {:optional true} [:maybe :boolean]]]
+ [:qp/implicit-field? {:optional true} [:maybe :boolean]]
+ ;;
+ ;; Coercion strategies (eg. UNIX seconds -> :type/DateTime) are configured on :model/Fields in the Admin UI, and
+ ;; reflected on refs to that field on the *first* stage of an MBQL query or a join, where the column first appears
+ ;; in the query. After the column passes through a stage boundary, it's no longer marked with the coercion strategy
+ ;; to avoid double-coercion.
+ ;;
+ ;; *However*, when a table is sandboxed by a native query, and it has fields which need coercion, the SQL subquery
+ ;; does not do the coercion (it's supposed to be a drop-in replacement for the table) but then the MBQL refs to its
+ ;; columns are not in the first stage anymore! So the sandboxing middleware sets this flag to the
+ ;; `:coercion-strategy`, along with `:qp/native-sandbox-column.propagate-coercion? true` (see below). Lib will
+ ;; propagate the coercion strategy through *exactly one* stage boundary, so it can get from the SQL first stage to
+ ;; the earliest MBQL stage, where the coercion will get applied correctly. See QUE2-376 or #69867 for more details.
+ [:qp/native-sandbox-column.force-coercion-strategy {:optional true} :keyword]
+ ;;
+ ;; See above about `:qp/native-sandbox-column.force-coercion-strategy`.
+ [:qp/native-sandbox-column.propagate-coercion? {:optional true} :boolean]]
;;
;; Additional constraints
;;