Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GDGT-2539] enforce run transform result schema (#75376)

YB1142b7e95fc5
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index e8434dd510b..ef47624f43f 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -222,8 +222,10 @@
(with-open [rdr (io/reader file)]
(->> (line-seq rdr) (remove str/blank?) count)))
-(defn- run-python-transform-impl!
- "Core Python transform execution. Returns {:status :result :logs :events}.
+(mu/defn- run-python-transform-impl! :- ::driver/run-transform-result
+ "Core Python transform execution. Returns an open map carrying an integer `:rows-affected` (the JSONL row count),
+ plus the HTTP runner response's `:status`/`:body`. Conforms to [[::driver/run-transform-result]] so the Python
+ path is held to the same contract as the SQL `run-transform!` methods. Every non-success path throws.
Options:
- `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 553567e9a48..de10cffa0c9 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -1331,11 +1331,20 @@
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
+(mr/def ::run-transform-result
+ "Result map returned by every `run-transform!` implementation. Must carry an integer `:rows-affected` (the row count
+ the transform wrote). Open map — implementations may attach extra keys."
+ [:map
+ [:rows-affected :int]])
+
(defmulti run-transform!
"Runs a transform.
Drivers that support any of the `:transforms/...` features must implement this method for the appropriate transform
- types."
+ types.
+
+ Implementations must return a map conforming to [[::run-transform-result]] — i.e. containing an integer
+ `:rows-affected`. Define them with `mu/defmethod ... :- ::run-transform-result` so the contract is validated."
{:added "0.57.0",
:arglists '([driver
{:keys [transform-type conn-spec query output-table] :as _transform-details}
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index bcb8867874d..4a1c7ca3d9e 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -216,7 +216,7 @@
(throw e))))
;; Follows similar logic to `transfer-file-to-db :table`
-(defmethod driver/run-transform! [:sql :table]
+(mu/defmethod driver/run-transform! [:sql :table] :- ::driver/run-transform-result
[driver {:keys [conn-spec output-table database] :as transform-details} _opts]
(let [table-exists? (driver/table-exists? driver database
{:schema (namespace output-table)
@@ -238,7 +238,7 @@
:else
(run-with-drop-create-fallback-strategy! driver database output-table transform-details conn-spec))))
-(defmethod driver/run-transform! [:sql :table-incremental]
+(mu/defmethod driver/run-transform! [:sql :table-incremental] :- ::driver/run-transform-result
[driver {:keys [conn-spec database output-table] :as transform-details} _opts]
(let [queries (if (driver/table-exists? driver database {:schema (namespace output-table)
:name (name output-table)})
diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.clj
index e6b8501f4bd..da7c6107072 100644
--- a/test/metabase/driver/sql/transforms_test.clj
+++ b/test/metabase/driver/sql/transforms_test.clj
@@ -10,6 +10,7 @@
[metabase.test :as mt]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.test-util :as transforms.tu]
+ [metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
(deftest compile-transform-contract-test
@@ -189,6 +190,22 @@
(is (= 3 (:rows-affected append-result))
"the INSERT...SELECT path reports the flat, true insert count"))))))))
+(deftest ^:parallel run-transform-result-schema-test
+ ;; Pins the contract that every `run-transform!` implementation's return schema
+ ;; (`::driver/run-transform-result`) enforces: a map carrying an integer `:rows-affected`. The
+ ;; SQL `[:sql :table]` / `[:sql :table-incremental]` methods declare this via `mu/defmethod`, and
+ ;; the Python path (`run-python-transform-impl!`) reuses the same open schema. No DB needed.
+ (testing "valid: a bare integer :rows-affected"
+ (is (true? (mr/validate ::driver/run-transform-result {:rows-affected 0})))
+ (is (true? (mr/validate ::driver/run-transform-result {:rows-affected 42}))))
+ (testing "valid: open map tolerates extra keys (e.g. the Python path's :status/:body)"
+ (is (true? (mr/validate ::driver/run-transform-result {:rows-affected 3, :status 200, :body {}}))))
+ (testing "invalid: missing, nil, or non-integer :rows-affected"
+ (is (false? (mr/validate ::driver/run-transform-result {})))
+ (is (false? (mr/validate ::driver/run-transform-result {:rows-affected nil})))
+ (is (false? (mr/validate ::driver/run-transform-result {:rows-affected "5"})))
+ (is (false? (mr/validate ::driver/run-transform-result {:rows-affected 1.5})))))
+
(defn- venues-source-query
"Lib query projecting [name, price] from venues — the source shape both characterization tests
use. Non-identity columns only, so SQL Server's `SELECT INTO` doesn't carry an `IDENTITY`
diff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.clj
index 6ec040738bd..88b798ec854 100644
--- a/test/metabase/transforms/execute_test.clj
+++ b/test/metabase/transforms/execute_test.clj
@@ -367,7 +367,9 @@
(let [calls (atom [])
record-call! (fn [call-name] (swap! calls conj call-name))
mock-query "CREATE TABLE test_schema.test_table AS (SELECT 1 AS x);"
- mock-rows [[1]]
+ ;; `execute-raw-queries!` yields one `{:rows-affected N}` map per statement; `run-transform!`
+ ;; returns `(last …)`, which must satisfy `::driver/run-transform-result`.
+ mock-rows [{:rows-affected 1}]
transform-details {:output-table :test_schema/test_table
:database {:id 1}
:transform-type :table}]