The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
wraps transforms execute! multimethod with a function (#66555)
Ngoc Khuat9db0f4c5e896
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex d7029878b78..eae7d122576 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -296,6 +296,7 @@metabase.lib.convert/->legacy-MBQL {:message "Conversion to legacy MBQL is discouraged, please use Lib to manipulate MBQL queries"}metabase.lib.core/->legacy-MBQL {:message "Conversion to legacy MBQL is discouraged, please use Lib to manipulate MBQL queries"}metabase.lib.equality/find-column-indexes-for-refs {:message "Use lib.equality/closest-matches-in-metadata or lib.equality/find-closest-matches-for-refs instead of lib.equality/find-column-indexes-for-refs"}+ metabase-enterprise.transforms.interface/execute! {:message "Use metabase-enterprise.transforms.execute/execute! instead of metabase-enterprise.transforms.interface/execute!"}next.jdbc/active-tx? {:message "Use metabase.app-db.core/in-transaction? to see if we are in a transaction"}toucan2.core/debug {:message "Don't commit code using t2/debug"}toucan2.tools.debug/debug {:message "Don't commit code using t2/debug"}}diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.cljindex 3e324a8432d..7c9f598b5d2 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj@@ -4,6 +4,7 @@[metabase-enterprise.transforms.api.transform-job][metabase-enterprise.transforms.api.transform-tag][metabase-enterprise.transforms.canceling :as transforms.canceling]+ [metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.models.transform :as transform.model][metabase-enterprise.transforms.models.transform-run :as transform-run]@@ -306,8 +307,8 @@_ (check-feature-enabled! transform)start-promise (promise)](u.jvm/in-virtual-thread*- (transforms.i/execute! transform {:start-promise start-promise- :run-method :manual}))+ (transforms.execute/execute! transform {:start-promise start-promise+ :run-method :manual}))(when (instance? Throwable @start-promise)(throw @start-promise))(let [result @start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.cljindex e05ac25b569..b1f201dd99e 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj@@ -1,81 +1,17 @@(ns metabase-enterprise.transforms.execute(:require- [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]- [metabase-enterprise.transforms.util :as transforms.util]- [metabase.driver :as driver]- [metabase.driver.util :as driver.u]- [metabase.events.core :as events]- [metabase.lib.schema.common :as schema.common]- [metabase.query-processor.compile :as qp.compile]- [metabase.util.log :as log]- [metabase.util.malli.registry :as mr]- [toucan2.core :as t2]))+ [metabase-enterprise.transforms.interface :as transforms.i]))(set! *warn-on-reflection* true)-(mr/def ::transform-details- [:map- [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]- [:conn-spec :any]- [:query ::qp.compile/compiled]- [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])--(mr/def ::transform-opts- [:map- [:overwrite? :boolean]])--(defn- transform-opts [{:keys [transform-type]}]- (case transform-type- :table {:overwrite? true}-- ;; once we have more than just append, dispatch on :target-incremental-strategy- :table-incremental {}))--(defn run-mbql-transform!+(defn execute!"Run `transform` and sync its target table.This is executing synchronously, but supports being kicked off in the backgroundby delivering the `start-promise` just before the start when the beginning of the execution has been bookedin the database."- ([transform] (run-mbql-transform! transform nil))- ([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]- (try- (let [db (get-in source [:query :database])- {driver :engine :as database} (t2/select-one :model/Database db)- transform-details {:db-id db- :database database- :transform-id id- :transform-type (keyword (:type target))- :conn-spec (driver/connection-spec driver database)- :query (transforms.util/compile-source transform)- :output-schema (:schema target)- :output-table (transforms.util/qualified-table-name driver target)}- opts (transform-opts transform-details)- features (transforms.util/required-database-features transform)]-- (when (transforms.util/db-routing-enabled? database)- (throw (ex-info "Transforms are not supported on databases with DB routing enabled."- {:driver driver, :database database})))- (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)- (throw (ex-info "The database does not support the requested transform target type."- {:driver driver, :database database, :features features})))- ;; mark the execution as started and notify any observers- (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]- (when start-promise- (deliver start-promise [:started run-id]))- (log/info "Executing transform" id "with target" (pr-str target))- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]- (transforms.util/run-cancelable-transform!- run-id driver transform-details- (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]- (transforms.util/sync-target! target database)- ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.- (events/publish-event! :event/transform-run-complete {:object transform-details}))))- (catch Throwable t- (log/error t "Error executing transform")- (when start-promise- ;; if the start-promise has been delivered, this is a no-op,- ;; but we assume nobody would catch the exception anyway- (deliver start-promise t))- (throw t)))))+ ([transform]+ (execute! transform nil))+ ([transform opts]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (transforms.i/execute! transform opts)))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/interface.clj b/enterprise/backend/src/metabase_enterprise/transforms/interface.cljindex 11309db3b92..735d41567bf 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/interface.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/interface.clj@@ -35,7 +35,9 @@- `:run-method`Expected to be either `:cron` (for scheduled runs) or `:manual` (for ad-hoc or test runs)- Used for instrumentation / metadata purposes."+ Used for instrumentation / metadata purposes.++ Do not use this directly. Use [[metabase-enterprise.transforms.execute/execute!]] instead."{:added "0.57.0" :arglists '([transform options])}(fn [transform _options](transform->transform-type transform)))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.cljindex f6798cf4844..22474699032 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj@@ -5,8 +5,8 @@[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval][clojurewerkz.quartzite.triggers :as triggers][flatland.ordered.set :as ordered-set]+ [metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]- [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.models.job-run :as transforms.job-run][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.ordering :as transforms.ordering]@@ -76,7 +76,7 @@(when (transform-run/running-run-for-transform-id transform-id)(recur))))(log/info "Executing job transform" (pr-str transform-id))- (transforms.i/execute! transform {:run-method run-method})+ (transforms.execute/execute! transform {:run-method run-method})(transforms.job-run/add-run-activity! run-id))))(defn run-transforms!diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.cljindex 8bc4a697d8a..17ce1b442ef 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj@@ -1,7 +1,16 @@(ns metabase-enterprise.transforms.query-impl(:require- [metabase-enterprise.transforms.execute :as transforms.execute]- [metabase-enterprise.transforms.interface :as transforms.i]))+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]+ [metabase-enterprise.transforms.interface :as transforms.i]+ [metabase-enterprise.transforms.util :as transforms.util]+ [metabase.driver :as driver]+ [metabase.driver.util :as driver.u]+ [metabase.events.core :as events]+ [metabase.lib.schema.common :as schema.common]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.util.log :as log]+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -13,5 +22,68 @@[transform](-> transform :source :query :database))+(mr/def ::transform-details+ [:map+ [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]+ [:conn-spec :any]+ [:query ::qp.compile/compiled]+ [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])++(mr/def ::transform-opts+ [:map+ [:overwrite? :boolean]])++(defn- transform-opts [{:keys [transform-type]}]+ (case transform-type+ :table {:overwrite? true}++ ;; once we have more than just append, dispatch on :target-incremental-strategy+ :table-incremental {}))++(defn- run-mbql-transform!+ ([transform] (run-mbql-transform! transform nil))+ ([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]+ (try+ (let [db (get-in source [:query :database])+ {driver :engine :as database} (t2/select-one :model/Database db)+ transform-details {:db-id db+ :database database+ :transform-id id+ :transform-type (keyword (:type target))+ :conn-spec (driver/connection-spec driver database)+ :query (transforms.util/compile-source transform)+ :output-schema (:schema target)+ :output-table (transforms.util/qualified-table-name driver target)}+ opts (transform-opts transform-details)+ features (transforms.util/required-database-features transform)]++ (when (transforms.util/db-routing-enabled? database)+ (throw (ex-info "Transforms are not supported on databases with DB routing enabled."+ {:driver driver, :database database})))+ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)+ (throw (ex-info "The database does not support the requested transform target type."+ {:driver driver, :database database, :features features})))+ ;; mark the execution as started and notify any observers+ (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]+ (when start-promise+ (deliver start-promise [:started run-id]))+ (log/info "Executing transform" id "with target" (pr-str target))+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]+ (transforms.util/run-cancelable-transform!+ run-id driver transform-details+ (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]+ (transforms.util/sync-target! target database)+ ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.+ (events/publish-event! :event/transform-run-complete {:object transform-details}))))+ (catch Throwable t+ (log/error t "Error executing transform")+ (when start-promise+ ;; if the start-promise has been delivered, this is a no-op,+ ;; but we assume nobody would catch the exception anyway+ (deliver start-promise t))+ (throw t)))))++#_{:clj-kondo/ignore [:discouraged-var]}(defmethod transforms.i/execute! :query [transform opts]- (transforms.execute/run-mbql-transform! transform opts))+ (run-mbql-transform! transform opts))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.cljindex 869127ed003..4f12c7cb12a 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj@@ -11,6 +11,7 @@[transform](-> transform :source :database))+#_{:clj-kondo/ignore [:discouraged-var]}(defmethod transforms.i/execute! :python[transform options](transforms-python.execute/execute-python-transform! transform options))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.cljindex 583779ed178..4742c062100 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj@@ -1,7 +1,7 @@(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test(:require[clojure.test :refer :all]- [metabase-enterprise.transforms.interface :as transforms.i]+ [metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.query-test-util :as query-test-util][metabase-enterprise.transforms.test-dataset :as transforms-dataset][metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! delete-schema!]]@@ -52,14 +52,14 @@:source {:type :query:query t1-query}:target target1}]- (transforms.i/execute! t1 {:run-method :manual})+ (transforms.execute/execute! t1 {:run-method :manual})(let [table1 (transforms.tu/wait-for-table table1-name 10000)t2-query (make-query (->> table1 :name (table-name->qp-table (mt/metadata-provider))) "category" lib/= "Gizmo")](mt/with-temp [:model/Transform t2 {:name "transform2":source {:type :query:query t2-query}:target target2}]- (transforms.i/execute! t2 {:run-method :cron})+ (transforms.execute/execute! t2 {:run-method :cron})(let [table2 (transforms.tu/wait-for-table table2-name 10000)check-query (lib/aggregate (make-query (->> table2 :name (table-name->qp-table (mt/metadata-provider)))) (lib/count))query-result (qp/process-query check-query)]@@ -109,7 +109,7 @@:query (lib/limit query-no-limit 5)}:target limit-table}](doseq [transform [transform-no-limit transform-limit]]- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [table-name (-> transform :target :name)_ (transforms.tu/wait-for-table table-name 10000)table-result (lib.metadata/table mp (mt/id (keyword table-name)))@@ -162,7 +162,7 @@:source {:type :query:query query}:target target-table}]- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))query-result (->> (lib/query mp table-result)@@ -213,7 +213,7 @@:source {:type :query:query query}:target target-table}]- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))query-result (->> (lib/query mp table-result)@@ -238,7 +238,7 @@(is (thrown-with-msg?clojure.lang.ExceptionInfo#"ERROR: permission denied for database transforms-test"- (transforms.i/execute! transform {:run-method :manual})))))))))+ (transforms.execute/execute! transform {:run-method :manual})))))))))(finally(driver/execute-raw-queries! driver/*driver* spec[[(format "DROP OWNED BY %s;" no-schema-user)]@@ -266,4 +266,4 @@clojure.lang.ExceptionInfo#"Transforms are not supported on databases with DB routing enabled."(mt/with-current-user (mt/user->id :crowberto)- (transforms.i/execute! transform {:run-method :manual}))))))))))+ (transforms.execute/execute! transform {:run-method :manual}))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex 9cdef5a6fbc..75bf835cef1 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -5,7 +5,7 @@[clojure.test :refer :all][honey.sql :as sql][metabase-enterprise.transforms-python.python-runner :as python-runner]- [metabase-enterprise.transforms.interface :as transforms.i]+ [metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.test-dataset :as transforms-dataset][metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]][metabase-enterprise.transforms.util :as transforms.u]@@ -71,8 +71,8 @@[transform transform-type checkpoint-field run-opts](if (= transform-type :python)(with-python-order-by! checkpoint-field- (transforms.i/execute! transform run-opts))- (transforms.i/execute! transform run-opts)))+ (transforms.execute/execute! transform run-opts))+ (transforms.execute/execute! transform run-opts)))(defn- make-incremental-source-query"Create a native query with optional checkpoint template tag."@@ -447,7 +447,7 @@:target-incremental-strategy {:type "append"}}}](mt/with-temp [:model/Transform transform transform-payload](testing "First run processes all existing data"- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(transforms.tu/wait-for-table target-table 10000)(let [row-count (get-table-row-count target-table)checkpoint (get-checkpoint-value (:id transform))]@@ -456,7 +456,7 @@(format "Checkpoint should be MAX(%s) = %s" (:field-name checkpoint-config) expected-second-checkpoint))))(testing "Second run without new data adds nothing"- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)](is (= 16 row-count) "Should still have 16 rows, no new data")))@@ -472,7 +472,7 @@:price 319.99:created-at "2024-01-21T11:00:00"}]- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)checkpoint (get-checkpoint-value (:id transform))](is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")@@ -515,7 +515,7 @@:target-incremental-strategy {:type "append"}}}](mt/with-temp [:model/Transform transform transform-payload](testing "First run processes first batch"- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(transforms.tu/wait-for-table target-table 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)@@ -526,7 +526,7 @@(format "Checkpoint should be MAX(%s) from first 10 rows" field-name))))(testing "Second run processes remaining data"- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)checkpoint (get-checkpoint-value (:id transform))]@@ -536,7 +536,7 @@(format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))(testing "Third run without new data adds nothing"- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)](is (= 16 row-count) "Should still have 16 rows, no new data")))@@ -547,7 +547,7 @@:price 299.99:created-at "2024-01-21T10:00:00"}]- (transforms.i/execute! transform {:run-method :manual})+ (transforms.execute/execute! transform {:run-method :manual})(let [row-count (get-table-row-count target-table)checkpoint (get-checkpoint-value (:id transform))](is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")diff --git a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.cljindex 86074a5a57d..8f335f7b5c4 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj@@ -3,7 +3,7 @@(:require[clojure.test :refer :all][clojure.tools.logging :as log]- [metabase-enterprise.transforms.interface :as transforms.i]+ [metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.jobs :as jobs][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase.test :as mt]))@@ -134,8 +134,8 @@(with-redefs [log/log* (fn [_ level _ message](swap! logged-messages conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)- transforms.i/execute! (fn [_ _]- (reset! run-called? true))]+ transforms.execute/execute! (fn [_ _]+ (reset! run-called? true))](#'jobs/run-transform! run-id :scheduled query-transform)(is (empty? (filter (comp #{:warn} :level) @logged-messages))"Should not log warnings when feature is enabled")diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex 85e6cce2c51..138cd9598c6 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -1,6 +1,7 @@(ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test(:require[clojure.test :refer :all]+ [metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.ordering :as ordering][metabase.driver :as driver]@@ -111,7 +112,7 @@LIMIT 100"}}"venues_transform_2")](try- (transforms.i/execute! transform1 {:run-method :manual})+ (transforms.execute/execute! transform1 {:run-method :manual})(let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")](is (= #{{:table table1} {:transform t2}}(transform-deps-for-db (t2/select-one :model/Transform t3)))))