The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[QP] Do coercions (eg. Unix timestamps) with native sandboxing (RCLS) (#70885)
Braden Shepherdson3559ec98b08c
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.cljindex 58e4c8d2de9..5c8e16285da 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj@@ -284,7 +284,12 @@(keep (fn [table-col](when-let [native-col (m/find-first #(= (:name %) (:name table-col))native-cols)]- (merge native-col table-col))))+ (cond-> (merge native-col table-col)+ ;; If the table column would have had a coercion strategy applied, add some metadata keys to+ ;; ensure the strategy is propagated to the first MBQL stage.+ (:coercion-strategy table-col)+ (assoc :qp/native-sandbox-column.force-coercion-strategy (:coercion-strategy table-col)+ :qp/native-sandbox-column.propagate-coercion? true)))))original-table-cols))(mu/defn- apply-sandbox-to-stage :- [:anddiff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 083bc5f06a8..63835999d15 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -1824,3 +1824,78 @@(testing "text field → :string/="(is (= :string/=(:type (attr-remapping->parameter mp {"cat" "foo"} ["cat" [:variable [:field (mt/id :venues :name) nil]]]))))))))++(deftest unix-timestamp-coercion-with-mbql-sandbox-test+ (testing "UNIX timestamp coercion should be applied when querying through an MBQL sandbox (#69867)"+ (mt/test-drivers (e2e-test-drivers)+ ;; Use venues.price (an integer column) and temporarily give it a UNIX timestamp coercion strategy.+ ;; With coercion applied, price values (1-4) become timestamps near the Unix epoch.+ ;; Without coercion, they come back as raw integers.+ (met/with-gtaps! {:gtaps {:venues {:query (mt/mbql-query venues)}}+ :attributes {}}+ (tu/with-temp-vals-in-db :model/Field (mt/id :venues :price)+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime+ :effective_type :type/Instant}+ (let [result (mt/run-mbql-query venues {:limit 1, :order-by [[:asc $id]]})]+ ;; venues.price for venue 1 is 3; with coercion this becomes 1970-01-01T00:00:03Z+ ;; The native_form SQL should contain a timestamp coercion expression+ (is (string? (-> result mt/rows first last))+ "Price column should be coerced to a timestamp string, not returned as a raw integer")))))))++(deftest unix-timestamp-coercion-with-native-sandbox-test+ (testing "UNIX timestamp coercion should be applied when querying through a native SQL sandbox (#69867)"+ (mt/test-drivers (e2e-test-drivers)+ ;; Use venues.price (an integer column) and temporarily give it a UNIX timestamp coercion strategy.+ ;; With coercion applied, price values (1-4) become timestamps near the Unix epoch.+ ;; Without coercion, they come back as raw integers.+ (met/with-gtaps! {:gtaps {:venues {:query (mt/native-query+ {:query (format-honeysql+ {:select [:*]+ :from [[(identifier :venues)]]+ :order-by [[(identifier :venues :id) :asc]]})})}}+ :attributes {}}+ (tu/with-temp-vals-in-db :model/Field (mt/id :venues :price)+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime+ :effective_type :type/Instant}+ (let [result (mt/run-mbql-query venues {:limit 1, :order-by [[:asc $id]]})]+ ;; BUG (#69867): With a native SQL sandbox, the coercion is NOT applied.+ (is (string? (-> result mt/rows first last))+ "Price column should be coerced to a timestamp string, not returned as a raw integer")))))))++(deftest unix-timestamp-coercion-with-native-sandbox-and-multiple-stages-test+ (testing "UNIX timestamp coercion should be applied when querying through a native SQL sandbox (#69867)"+ (mt/test-drivers (e2e-test-drivers)+ ;; Use venues.price (an integer column) and temporarily give it a UNIX timestamp coercion strategy.+ ;; With coercion applied, price values (1-4) become timestamps near the Unix epoch.+ ;; Without coercion, they come back as raw integers.+ (testing "and a multi-stage query should apply coercion exactly once (no double-coercion)"+ (met/with-gtaps! {:gtaps {:venues {:query (mt/native-query+ {:query (format-honeysql+ {:select [:*]+ :from [[(identifier :venues)]]+ :order-by [[(identifier :venues :id) :asc]]})})}}+ :attributes {}}+ (tu/with-temp-vals-in-db :model/Field (mt/id :venues :price)+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime+ :effective_type :type/Instant}+ ;; A 2-stage query: inner stage selects from venues, outer stage wraps it.+ ;; With sandboxing this becomes 3 levels deep. Coercion must be applied only in the+ ;; innermost stage (closest to the table), not re-applied in outer stages.+ ;; If double-coerced, the value would be wildly wrong (treating a timestamp as seconds+ ;; and adding it to epoch again).+ (let [mp (mt/metadata-provider)+ base (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (lib/order-by (lib.metadata/field mp (mt/id :venues :id)))+ (lib/limit 1)+ lib/append-stage)+ field (lib.metadata/field mp (mt/id :venues :price))+ price (m/find-first #(= (:name field) (:name %))+ (lib/filterable-columns base))+ query (lib/filter base (lib/< price "2037-12-31"))+ [row] (-> query qp/process-query mt/rows)]+ ;; price=3 → coerced once → "1970-01-01T00:00:03Z"+ ;; price=3 → coerced twice → would be an enormous date or an error+ (is (string? (last row))+ "Price column should be coerced to a timestamp string")+ (is (str/starts-with? (last row) "1970-01-01")+ "Price should be coerced exactly once, producing a date near the Unix epoch"))))))))diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex d3622c2e08a..2e8645a0d3e 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -902,15 +902,24 @@;; https://linear.app/metabase-inc/issue/ENG-8766/[epic]-field-refs-overhaulfield-metadata (when (integer? id-or-name)(driver-api/field (driver-api/metadata-provider) id-or-name))- allow-casting? (and field-metadata- (or (pos-int? (driver-api/qp.add.source-table options))- (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table options))+ allow-casting? (and (or (:qp/native-sandbox-column.force-coercion-strategy options)+ (and field-metadata+ (or (pos-int? (driver-api/qp.add.source-table options))+ (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table options))))(not (:qp/ignore-coercion options)));; preserve metadata attached to the original field clause, for example BigQuery temporal type information.identifier (-> (apply h2x/identifier :field(concat source-table-aliases (->honeysql driver [::nfc-path source-nfc-path]) [source-alias]))(with-meta (meta field-clause)))identifier (->honeysql driver identifier)+ ;; If no field-metadata is available, but a coercion strategy must be applied, synthesize enough+ ;; `field-metadata` to be able to cast it correctly.+ field-metadata (or field-metadata+ (when allow-casting?+ (-> options+ (select-keys [:base-type :effetive-type])+ (assoc :coercion-strategy+ (:qp/native-sandbox-column.force-coercion-strategy options)))))casted-field (cast-field-if-needed driver field-metadata identifier)database-type (or (h2x/database-type casted-field)(:database-type field-metadata))diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljcindex 8645d2ffa44..74a24eaa9d8 100644--- a/src/metabase/lib/field.cljc+++ b/src/metabase/lib/field.cljc@@ -427,7 +427,8 @@[:base-type:inherited-temporal-unit:lib/original-binning- :lib/original-effective-type])+ :lib/original-effective-type+ :qp/native-sandbox-column.force-coercion-strategy]){:lib/binning :binning:lib/temporal-unit :temporal-unit:lib/ref-name :namediff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljcindex 62a28fd24cb..b805f7e4300 100644--- a/src/metabase/lib/field/util.cljc+++ b/src/metabase/lib/field/util.cljc@@ -124,6 +124,18 @@;; desired-column-alias is previous stage => source column alias in next stage:lib/source-column-alias ((some-fn :lib/desired-column-alias :lib/source-column-alias :name) col):lib/source :source/previous-stage)++ ;; Native sandboxes need special handling for any type coercions set on the sandboxed table's fields.+ ;; The columns first appear in the native stage, but we need to propagate the coercion metadata to the next stage+ ;; so it gets applied in MBQL. After that first propagation, it should always be removed to prevent+ ;; double-coercion. So sandboxing middleware sets two fields on the metadata: a flag, and the coercion strategy.+ ;; If the input column has both set, we propagate the strategy into the next stage. The flag is never propagated,+ ;; so the coercion strategy only reaches one stage after the native sandbox, as desired! See QUE2-376.+ (dissoc :qp/native-sandbox-column.propagate-coercion?)+ (cond-> #_col+ (not (:qp/native-sandbox-column.propagate-coercion? col))+ (dissoc :qp/native-sandbox-column.force-coercion-strategy))+;;;; Remove `:lib/desired-column-alias`, which needs to be recalculated in the context;; of what is returned by the current stage, to prevent any confusion; its value is likely wrong now and wediff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex 2b4bcab8894..ac4c4bf8f25 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -516,7 +516,23 @@;; stage. [[metabase.lib.metadata.result-metadata/super-broken-legacy-field-ref]] uses this to know to force Field;; ID refs for QP `:field_ref` in results metadata to preserve historic behavior to avoid breaking legacy viz;; settings that use it as a key.- [:qp/implicit-field? {:optional true} [:maybe :boolean]]]+ [:qp/implicit-field? {:optional true} [:maybe :boolean]]+ ;;+ ;; Coercion strategies (eg. UNIX seconds -> :type/DateTime) are configured on :model/Fields in the Admin UI, and+ ;; reflected on refs to that field on the *first* stage of an MBQL query or a join, where the column first appears+ ;; in the query. After the column passes through a stage boundary, it's no longer marked with the coercion strategy+ ;; to avoid double-coercion.+ ;;+ ;; *However*, when a table is sandboxed by a native query, and it has fields which need coercion, the SQL subquery+ ;; does not do the coercion (it's supposed to be a drop-in replacement for the table) but then the MBQL refs to its+ ;; columns are not in the first stage anymore! So the sandboxing middleware sets this flag to the+ ;; `:coercion-strategy`, along with `:qp/native-sandbox-column.propagate-coercion? true` (see below). Lib will+ ;; propagate the coercion strategy through *exactly one* stage boundary, so it can get from the SQL first stage to+ ;; the earliest MBQL stage, where the coercion will get applied correctly. See QUE2-376 or #69867 for more details.+ [:qp/native-sandbox-column.force-coercion-strategy {:optional true} :keyword]+ ;;+ ;; See above about `:qp/native-sandbox-column.force-coercion-strategy`.+ [:qp/native-sandbox-column.propagate-coercion? {:optional true} :boolean]];;;; Additional constraints;;