The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-2539] enforce run transform result schema (#75376)
YB1142b7e95fc5
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljindex e8434dd510b..ef47624f43f 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -222,8 +222,10 @@(with-open [rdr (io/reader file)](->> (line-seq rdr) (remove str/blank?) count)))-(defn- run-python-transform-impl!- "Core Python transform execution. Returns {:status :result :logs :events}.+(mu/defn- run-python-transform-impl! :- ::driver/run-transform-result+ "Core Python transform execution. Returns an open map carrying an integer `:rows-affected` (the JSONL row count),+ plus the HTTP runner response's `:status`/`:body`. Conforms to [[::driver/run-transform-result]] so the Python+ path is held to the same contract as the SQL `run-transform!` methods. Every non-success path throws.Options:- `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 553567e9a48..de10cffa0c9 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -1331,11 +1331,20 @@dispatch-on-initialized-driver:hierarchy #'hierarchy)+(mr/def ::run-transform-result+ "Result map returned by every `run-transform!` implementation. Must carry an integer `:rows-affected` (the row count+ the transform wrote). Open map — implementations may attach extra keys."+ [:map+ [:rows-affected :int]])+(defmulti run-transform!"Runs a transform.Drivers that support any of the `:transforms/...` features must implement this method for the appropriate transform- types."+ types.++ Implementations must return a map conforming to [[::run-transform-result]] — i.e. containing an integer+ `:rows-affected`. Define them with `mu/defmethod ... :- ::run-transform-result` so the contract is validated."{:added "0.57.0",:arglists '([driver{:keys [transform-type conn-spec query output-table] :as _transform-details}diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex bcb8867874d..4a1c7ca3d9e 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -216,7 +216,7 @@(throw e))));; Follows similar logic to `transfer-file-to-db :table`-(defmethod driver/run-transform! [:sql :table]+(mu/defmethod driver/run-transform! [:sql :table] :- ::driver/run-transform-result[driver {:keys [conn-spec output-table database] :as transform-details} _opts](let [table-exists? (driver/table-exists? driver database{:schema (namespace output-table)@@ -238,7 +238,7 @@:else(run-with-drop-create-fallback-strategy! driver database output-table transform-details conn-spec))))-(defmethod driver/run-transform! [:sql :table-incremental]+(mu/defmethod driver/run-transform! [:sql :table-incremental] :- ::driver/run-transform-result[driver {:keys [conn-spec database output-table] :as transform-details} _opts](let [queries (if (driver/table-exists? driver database {:schema (namespace output-table):name (name output-table)})diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.cljindex e6b8501f4bd..da7c6107072 100644--- a/test/metabase/driver/sql/transforms_test.clj+++ b/test/metabase/driver/sql/transforms_test.clj@@ -10,6 +10,7 @@[metabase.test :as mt][metabase.transforms-base.util :as transforms-base.u][metabase.transforms.test-util :as transforms.tu]+ [metabase.util.malli.registry :as mr][toucan2.core :as t2]))(deftest compile-transform-contract-test@@ -189,6 +190,22 @@(is (= 3 (:rows-affected append-result))"the INSERT...SELECT path reports the flat, true insert count"))))))))+(deftest ^:parallel run-transform-result-schema-test+ ;; Pins the contract that every `run-transform!` implementation's return schema+ ;; (`::driver/run-transform-result`) enforces: a map carrying an integer `:rows-affected`. The+ ;; SQL `[:sql :table]` / `[:sql :table-incremental]` methods declare this via `mu/defmethod`, and+ ;; the Python path (`run-python-transform-impl!`) reuses the same open schema. No DB needed.+ (testing "valid: a bare integer :rows-affected"+ (is (true? (mr/validate ::driver/run-transform-result {:rows-affected 0})))+ (is (true? (mr/validate ::driver/run-transform-result {:rows-affected 42}))))+ (testing "valid: open map tolerates extra keys (e.g. the Python path's :status/:body)"+ (is (true? (mr/validate ::driver/run-transform-result {:rows-affected 3, :status 200, :body {}}))))+ (testing "invalid: missing, nil, or non-integer :rows-affected"+ (is (false? (mr/validate ::driver/run-transform-result {})))+ (is (false? (mr/validate ::driver/run-transform-result {:rows-affected nil})))+ (is (false? (mr/validate ::driver/run-transform-result {:rows-affected "5"})))+ (is (false? (mr/validate ::driver/run-transform-result {:rows-affected 1.5})))))+(defn- venues-source-query"Lib query projecting [name, price] from venues — the source shape both characterization testsuse. Non-identity columns only, so SQL Server's `SELECT INTO` doesn't carry an `IDENTITY`diff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.cljindex 6ec040738bd..88b798ec854 100644--- a/test/metabase/transforms/execute_test.clj+++ b/test/metabase/transforms/execute_test.clj@@ -367,7 +367,9 @@(let [calls (atom [])record-call! (fn [call-name] (swap! calls conj call-name))mock-query "CREATE TABLE test_schema.test_table AS (SELECT 1 AS x);"- mock-rows [[1]]+ ;; `execute-raw-queries!` yields one `{:rows-affected N}` map per statement; `run-transform!`+ ;; returns `(last …)`, which must satisfy `::driver/run-transform-result`.+ mock-rows [{:rows-affected 1}]transform-details {:output-table :test_schema/test_table:database {:id 1}:transform-type :table}]