Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

wraps transforms execute! multimethod with a function (#66555)

Ngoc Khuat9db0f4c5e896
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index d7029878b78..eae7d122576 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -296,6 +296,7 @@
metabase.lib.convert/->legacy-MBQL {:message "Conversion to legacy MBQL is discouraged, please use Lib to manipulate MBQL queries"}
metabase.lib.core/->legacy-MBQL {:message "Conversion to legacy MBQL is discouraged, please use Lib to manipulate MBQL queries"}
metabase.lib.equality/find-column-indexes-for-refs {:message "Use lib.equality/closest-matches-in-metadata or lib.equality/find-closest-matches-for-refs instead of lib.equality/find-column-indexes-for-refs"}
+ metabase-enterprise.transforms.interface/execute! {:message "Use metabase-enterprise.transforms.execute/execute! instead of metabase-enterprise.transforms.interface/execute!"}
next.jdbc/active-tx? {:message "Use metabase.app-db.core/in-transaction? to see if we are in a transaction"}
toucan2.core/debug {:message "Don't commit code using t2/debug"}
toucan2.tools.debug/debug {:message "Don't commit code using t2/debug"}}
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
index 3e324a8432d..7c9f598b5d2 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
@@ -4,6 +4,7 @@
[metabase-enterprise.transforms.api.transform-job]
[metabase-enterprise.transforms.api.transform-tag]
[metabase-enterprise.transforms.canceling :as transforms.canceling]
+ [metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.transform :as transform.model]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -306,8 +307,8 @@
_ (check-feature-enabled! transform)
start-promise (promise)]
(u.jvm/in-virtual-thread*
- (transforms.i/execute! transform {:start-promise start-promise
- :run-method :manual}))
+ (transforms.execute/execute! transform {:start-promise start-promise
+ :run-method :manual}))
(when (instance? Throwable @start-promise)
(throw @start-promise))
(let [result @start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
index e05ac25b569..b1f201dd99e 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
@@ -1,81 +1,17 @@
(ns metabase-enterprise.transforms.execute
(:require
- [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
- [metabase-enterprise.transforms.util :as transforms.util]
- [metabase.driver :as driver]
- [metabase.driver.util :as driver.u]
- [metabase.events.core :as events]
- [metabase.lib.schema.common :as schema.common]
- [metabase.query-processor.compile :as qp.compile]
- [metabase.util.log :as log]
- [metabase.util.malli.registry :as mr]
- [toucan2.core :as t2]))
+ [metabase-enterprise.transforms.interface :as transforms.i]))
(set! *warn-on-reflection* true)
-(mr/def ::transform-details
- [:map
- [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
- [:conn-spec :any]
- [:query ::qp.compile/compiled]
- [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
-
-(mr/def ::transform-opts
- [:map
- [:overwrite? :boolean]])
-
-(defn- transform-opts [{:keys [transform-type]}]
- (case transform-type
- :table {:overwrite? true}
-
- ;; once we have more than just append, dispatch on :target-incremental-strategy
- :table-incremental {}))
-
-(defn run-mbql-transform!
+(defn execute!
"Run `transform` and sync its target table.
This is executing synchronously, but supports being kicked off in the background
by delivering the `start-promise` just before the start when the beginning of the execution has been booked
in the database."
- ([transform] (run-mbql-transform! transform nil))
- ([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
- (try
- (let [db (get-in source [:query :database])
- {driver :engine :as database} (t2/select-one :model/Database db)
- transform-details {:db-id db
- :database database
- :transform-id id
- :transform-type (keyword (:type target))
- :conn-spec (driver/connection-spec driver database)
- :query (transforms.util/compile-source transform)
- :output-schema (:schema target)
- :output-table (transforms.util/qualified-table-name driver target)}
- opts (transform-opts transform-details)
- features (transforms.util/required-database-features transform)]
-
- (when (transforms.util/db-routing-enabled? database)
- (throw (ex-info "Transforms are not supported on databases with DB routing enabled."
- {:driver driver, :database database})))
- (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
- (throw (ex-info "The database does not support the requested transform target type."
- {:driver driver, :database database, :features features})))
- ;; mark the execution as started and notify any observers
- (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
- (when start-promise
- (deliver start-promise [:started run-id]))
- (log/info "Executing transform" id "with target" (pr-str target))
- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
- (transforms.util/run-cancelable-transform!
- run-id driver transform-details
- (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
- (transforms.util/sync-target! target database)
- ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
- (events/publish-event! :event/transform-run-complete {:object transform-details}))))
- (catch Throwable t
- (log/error t "Error executing transform")
- (when start-promise
- ;; if the start-promise has been delivered, this is a no-op,
- ;; but we assume nobody would catch the exception anyway
- (deliver start-promise t))
- (throw t)))))
+ ([transform]
+ (execute! transform nil))
+ ([transform opts]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (transforms.i/execute! transform opts)))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/interface.clj b/enterprise/backend/src/metabase_enterprise/transforms/interface.clj
index 11309db3b92..735d41567bf 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/interface.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/interface.clj
@@ -35,7 +35,9 @@
- `:run-method`
Expected to be either `:cron` (for scheduled runs) or `:manual` (for ad-hoc or test runs)
- Used for instrumentation / metadata purposes."
+ Used for instrumentation / metadata purposes.
+
+ Do not use this directly. Use [[metabase-enterprise.transforms.execute/execute!]] instead."
{:added "0.57.0" :arglists '([transform options])}
(fn [transform _options]
(transform->transform-type transform)))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
index f6798cf4844..22474699032 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
@@ -5,8 +5,8 @@
[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
[clojurewerkz.quartzite.triggers :as triggers]
[flatland.ordered.set :as ordered-set]
+ [metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
- [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.job-run :as transforms.job-run]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.ordering :as transforms.ordering]
@@ -76,7 +76,7 @@
(when (transform-run/running-run-for-transform-id transform-id)
(recur))))
(log/info "Executing job transform" (pr-str transform-id))
- (transforms.i/execute! transform {:run-method run-method})
+ (transforms.execute/execute! transform {:run-method run-method})
(transforms.job-run/add-run-activity! run-id))))
(defn run-transforms!
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
index 8bc4a697d8a..17ce1b442ef 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
@@ -1,7 +1,16 @@
(ns metabase-enterprise.transforms.query-impl
(:require
- [metabase-enterprise.transforms.execute :as transforms.execute]
- [metabase-enterprise.transforms.interface :as transforms.i]))
+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+ [metabase-enterprise.transforms.interface :as transforms.i]
+ [metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.driver :as driver]
+ [metabase.driver.util :as driver.u]
+ [metabase.events.core :as events]
+ [metabase.lib.schema.common :as schema.common]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.util.log :as log]
+ [metabase.util.malli.registry :as mr]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -13,5 +22,68 @@
[transform]
(-> transform :source :query :database))
+(mr/def ::transform-details
+ [:map
+ [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
+ [:conn-spec :any]
+ [:query ::qp.compile/compiled]
+ [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
+
+(mr/def ::transform-opts
+ [:map
+ [:overwrite? :boolean]])
+
+(defn- transform-opts [{:keys [transform-type]}]
+ (case transform-type
+ :table {:overwrite? true}
+
+ ;; once we have more than just append, dispatch on :target-incremental-strategy
+ :table-incremental {}))
+
+(defn- run-mbql-transform!
+ ([transform] (run-mbql-transform! transform nil))
+ ([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
+ (try
+ (let [db (get-in source [:query :database])
+ {driver :engine :as database} (t2/select-one :model/Database db)
+ transform-details {:db-id db
+ :database database
+ :transform-id id
+ :transform-type (keyword (:type target))
+ :conn-spec (driver/connection-spec driver database)
+ :query (transforms.util/compile-source transform)
+ :output-schema (:schema target)
+ :output-table (transforms.util/qualified-table-name driver target)}
+ opts (transform-opts transform-details)
+ features (transforms.util/required-database-features transform)]
+
+ (when (transforms.util/db-routing-enabled? database)
+ (throw (ex-info "Transforms are not supported on databases with DB routing enabled."
+ {:driver driver, :database database})))
+ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
+ (throw (ex-info "The database does not support the requested transform target type."
+ {:driver driver, :database database, :features features})))
+ ;; mark the execution as started and notify any observers
+ (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
+ (when start-promise
+ (deliver start-promise [:started run-id]))
+ (log/info "Executing transform" id "with target" (pr-str target))
+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
+ (transforms.util/run-cancelable-transform!
+ run-id driver transform-details
+ (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
+ (transforms.util/sync-target! target database)
+ ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
+ (events/publish-event! :event/transform-run-complete {:object transform-details}))))
+ (catch Throwable t
+ (log/error t "Error executing transform")
+ (when start-promise
+ ;; if the start-promise has been delivered, this is a no-op,
+ ;; but we assume nobody would catch the exception anyway
+ (deliver start-promise t))
+ (throw t)))))
+
+#_{:clj-kondo/ignore [:discouraged-var]}
(defmethod transforms.i/execute! :query [transform opts]
- (transforms.execute/run-mbql-transform! transform opts))
+ (run-mbql-transform! transform opts))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
index 869127ed003..4f12c7cb12a 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
@@ -11,6 +11,7 @@
[transform]
(-> transform :source :database))
+#_{:clj-kondo/ignore [:discouraged-var]}
(defmethod transforms.i/execute! :python
[transform options]
(transforms-python.execute/execute-python-transform! transform options))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
index 583779ed178..4742c062100 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
@@ -1,7 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test
(:require
[clojure.test :refer :all]
- [metabase-enterprise.transforms.interface :as transforms.i]
+ [metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! delete-schema!]]
@@ -52,14 +52,14 @@
:source {:type :query
:query t1-query}
:target target1}]
- (transforms.i/execute! t1 {:run-method :manual})
+ (transforms.execute/execute! t1 {:run-method :manual})
(let [table1 (transforms.tu/wait-for-table table1-name 10000)
t2-query (make-query (->> table1 :name (table-name->qp-table (mt/metadata-provider))) "category" lib/= "Gizmo")]
(mt/with-temp [:model/Transform t2 {:name "transform2"
:source {:type :query
:query t2-query}
:target target2}]
- (transforms.i/execute! t2 {:run-method :cron})
+ (transforms.execute/execute! t2 {:run-method :cron})
(let [table2 (transforms.tu/wait-for-table table2-name 10000)
check-query (lib/aggregate (make-query (->> table2 :name (table-name->qp-table (mt/metadata-provider)))) (lib/count))
query-result (qp/process-query check-query)]
@@ -109,7 +109,7 @@
:query (lib/limit query-no-limit 5)}
:target limit-table}]
(doseq [transform [transform-no-limit transform-limit]]
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [table-name (-> transform :target :name)
_ (transforms.tu/wait-for-table table-name 10000)
table-result (lib.metadata/table mp (mt/id (keyword table-name)))
@@ -162,7 +162,7 @@
:source {:type :query
:query query}
:target target-table}]
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
query-result (->> (lib/query mp table-result)
@@ -213,7 +213,7 @@
:source {:type :query
:query query}
:target target-table}]
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
query-result (->> (lib/query mp table-result)
@@ -238,7 +238,7 @@
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"ERROR: permission denied for database transforms-test"
- (transforms.i/execute! transform {:run-method :manual})))))))))
+ (transforms.execute/execute! transform {:run-method :manual})))))))))
(finally
(driver/execute-raw-queries! driver/*driver* spec
[[(format "DROP OWNED BY %s;" no-schema-user)]
@@ -266,4 +266,4 @@
clojure.lang.ExceptionInfo
#"Transforms are not supported on databases with DB routing enabled."
(mt/with-current-user (mt/user->id :crowberto)
- (transforms.i/execute! transform {:run-method :manual}))))))))))
+ (transforms.execute/execute! transform {:run-method :manual}))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index 9cdef5a6fbc..75bf835cef1 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -5,7 +5,7 @@
[clojure.test :refer :all]
[honey.sql :as sql]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
- [metabase-enterprise.transforms.interface :as transforms.i]
+ [metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
[metabase-enterprise.transforms.util :as transforms.u]
@@ -71,8 +71,8 @@
[transform transform-type checkpoint-field run-opts]
(if (= transform-type :python)
(with-python-order-by! checkpoint-field
- (transforms.i/execute! transform run-opts))
- (transforms.i/execute! transform run-opts)))
+ (transforms.execute/execute! transform run-opts))
+ (transforms.execute/execute! transform run-opts)))
(defn- make-incremental-source-query
"Create a native query with optional checkpoint template tag."
@@ -447,7 +447,7 @@
:target-incremental-strategy {:type "append"}}}]
(mt/with-temp [:model/Transform transform transform-payload]
(testing "First run processes all existing data"
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(transforms.tu/wait-for-table target-table 10000)
(let [row-count (get-table-row-count target-table)
checkpoint (get-checkpoint-value (:id transform))]
@@ -456,7 +456,7 @@
(format "Checkpoint should be MAX(%s) = %s" (:field-name checkpoint-config) expected-second-checkpoint))))
(testing "Second run without new data adds nothing"
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [row-count (get-table-row-count target-table)]
(is (= 16 row-count) "Should still have 16 rows, no new data")))
@@ -472,7 +472,7 @@
:price 319.99
:created-at "2024-01-21T11:00:00"}]
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [row-count (get-table-row-count target-table)
checkpoint (get-checkpoint-value (:id transform))]
(is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")
@@ -515,7 +515,7 @@
:target-incremental-strategy {:type "append"}}}]
(mt/with-temp [:model/Transform transform transform-payload]
(testing "First run processes first batch"
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(transforms.tu/wait-for-table target-table 10000)
(let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)
@@ -526,7 +526,7 @@
(format "Checkpoint should be MAX(%s) from first 10 rows" field-name))))
(testing "Second run processes remaining data"
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)
checkpoint (get-checkpoint-value (:id transform))]
@@ -536,7 +536,7 @@
(format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))
(testing "Third run without new data adds nothing"
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [row-count (get-table-row-count target-table)]
(is (= 16 row-count) "Should still have 16 rows, no new data")))
@@ -547,7 +547,7 @@
:price 299.99
:created-at "2024-01-21T10:00:00"}]
- (transforms.i/execute! transform {:run-method :manual})
+ (transforms.execute/execute! transform {:run-method :manual})
(let [row-count (get-table-row-count target-table)
checkpoint (get-checkpoint-value (:id transform))]
(is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
index 86074a5a57d..8f335f7b5c4 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
@@ -3,7 +3,7 @@
(:require
[clojure.test :refer :all]
[clojure.tools.logging :as log]
- [metabase-enterprise.transforms.interface :as transforms.i]
+ [metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.jobs :as jobs]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase.test :as mt]))
@@ -134,8 +134,8 @@
(with-redefs [log/log* (fn [_ level _ message]
(swap! logged-messages conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)
- transforms.i/execute! (fn [_ _]
- (reset! run-called? true))]
+ transforms.execute/execute! (fn [_ _]
+ (reset! run-called? true))]
(#'jobs/run-transform! run-id :scheduled query-transform)
(is (empty? (filter (comp #{:warn} :level) @logged-messages))
"Should not log warnings when feature is enabled")
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
index 85e6cce2c51..138cd9598c6 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
@@ -1,6 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test
(:require
[clojure.test :refer :all]
+ [metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.ordering :as ordering]
[metabase.driver :as driver]
@@ -111,7 +112,7 @@
LIMIT 100"}}
"venues_transform_2")]
(try
- (transforms.i/execute! transform1 {:run-method :manual})
+ (transforms.execute/execute! transform1 {:run-method :manual})
(let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")]
(is (= #{{:table table1} {:transform t2}}
(transform-deps-for-db (t2/select-one :model/Transform t3)))))