The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Polymorphic transforms (#64084)
Ngoc Khuat65d95d943885
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex d129a0edb8e..96be9193836 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2004,6 +2004,7 @@metabase-enterprise.transforms.coremetabase-enterprise.transforms.initmetabase-enterprise.transforms.instrumentation+ metabase-enterprise.transforms.interfacemetabase-enterprise.transforms.util}:uses #{analyticsapi@@ -2020,15 +2021,14 @@synctasktask-history- util- enterprise/transforms-python}}+ util}}enterprise/transforms-python{:team "Querying":api #{metabase-enterprise.transforms-python.api- metabase-enterprise.transforms-python.executemetabase-enterprise.transforms-python.init}- :uses #{api+ :uses #{analytics+ apiapp-dbconfigdriverdiff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.cljindex 4e839be4a9a..f6a3c81a326 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj@@ -1,10 +1,9 @@(ns metabase-enterprise.transforms.api(:require- [metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase-enterprise.transforms.api.transform-job][metabase-enterprise.transforms.api.transform-tag][metabase-enterprise.transforms.canceling :as transforms.canceling]- [metabase-enterprise.transforms.execute :as transforms.execute]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.models.transform :as transform.model][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.models.transform-run-cancelation :as transform-run-cancelation]@@ -69,7 +68,7 @@(defn- check-database-feature[transform]- (let [database (api/check-400 (t2/select-one :model/Database (transforms.util/target-database-id transform))+ (let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))(deferred-tru "The target database cannot be found."))feature (transforms.util/required-database-feature transform)](api/check-400 (not (:is_sample database))@@ -139,7 +138,7 @@(log/info "get transform" id)(api/check-superuser)(let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))- target-table (transforms.util/target-table (transforms.util/target-database-id transform) target :active true)]+ target-table (transforms.util/target-table (transforms.i/target-db-id transform) target :active true)](-> transform(t2/hydrate :last_run :transform_tag_ids)(u/update-some :last_run transforms.util/localize-run-timestamps)@@ -255,13 +254,9 @@(let [transform (api/check-404 (t2/select-one :model/Transform id))_ (check-feature-enabled! transform)start-promise (promise)]- (if (transforms.util/python-transform? transform)- (u.jvm/in-virtual-thread*- (transforms-python.execute/execute-python-transform! transform {:start-promise start-promise- :run-method :manual}))- (u.jvm/in-virtual-thread*- (transforms.execute/run-mbql-transform! transform {:start-promise start-promise- :run-method :manual})))+ (u.jvm/in-virtual-thread*+ (transforms.i/execute! transform {:start-promise start-promise+ :run-method :manual}))(when (instance? Throwable @start-promise)(throw @start-promise))(let [result @start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.cljindex fa920c6bbae..5409450fa00 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj@@ -54,10 +54,11 @@(when start-promise(deliver start-promise [:started run-id]))(log/info "Executing transform" id "with target" (pr-str target))- (transforms.util/run-cancelable-transform!- run-id driver transform-details- (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))- (transforms.instrumentation/with-stage-timing [run-id :table-sync]+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]+ (transforms.util/run-cancelable-transform!+ run-id driver transform-details+ (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]](transforms.util/sync-target! target database run-id);; This event must be published only after the sync is complete - the new table needs to be in AppDB.(events/publish-event! :event/transform-run-complete {:object transform-details}))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/init.clj b/enterprise/backend/src/metabase_enterprise/transforms/init.cljindex 6cb332dbd24..c0ca2cd1de7 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/init.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/init.clj@@ -1,7 +1,6 @@(ns metabase-enterprise.transforms.init(:require[metabase-enterprise.transforms.canceling]- [metabase-enterprise.transforms.execute][metabase-enterprise.transforms.jobs][metabase-enterprise.transforms.models.job-run][metabase-enterprise.transforms.models.transform]@@ -11,6 +10,7 @@[metabase-enterprise.transforms.models.transform-run-cancelation][metabase-enterprise.transforms.models.transform-tag][metabase-enterprise.transforms.models.transform-transform-tag]+ [metabase-enterprise.transforms.query-impl][metabase-enterprise.transforms.schedule][metabase-enterprise.transforms.settings][metabase-enterprise.transforms.timeout]))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.cljindex c991d0ede58..e194fb652ac 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj@@ -8,61 +8,39 @@(set! *warn-on-reflection* true)-(mr/def ::stage-type- [:enum :export :computation :import])--(def ^:private label-to-stage- {:dwh-to-file :export- :file-to-s3 :export- :file-to-dwh :import- :python-execution :computation- :mbql-query :computation- :table-sync :import})--(mr/def ::stage-label- [:enum- :dwh-to-file- :file-to-s3- :file-to-dwh-- :python-execution- :mbql-query- :table-sync])+(mr/def ::stage [:tuple simple-keyword? simple-keyword?])(mu/defn record-stage-start!"Record the start of a transform stage."- [job-run-id :- [:maybe pos-int?]- stage-label :- ::stage-label]- (let [stage-type (label-to-stage stage-label)]- (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))- (prometheus/inc! :metabase-transforms/stage-started- {:stage-type (name stage-type)- :stage-label (name stage-label)})))+ [job-run-id :- [:maybe pos-int?]+ [stage-type stage-label] :- ::stage]+ (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))+ (prometheus/inc! :metabase-transforms/stage-started+ {:stage-type (name stage-type)+ :stage-label (name stage-label)}))(mu/defn record-stage-completion!"Record the successful completion of a transform stage."- [job-run-id :- [:maybe pos-int?]- stage-label :- ::stage-label- duration-ms :- int?]- (let [stage-type (label-to-stage stage-label)]- (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)- (let [labels {:stage-type (name stage-type)- :stage-label (name stage-label)}]- (prometheus/inc! :metabase-transforms/stage-completed labels)- (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))+ [job-run-id :- [:maybe pos-int?]+ [stage-type stage-label] :- ::stage+ duration-ms :- int?]+ (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)+ (let [labels {:stage-type (name stage-type)+ :stage-label (name stage-label)}]+ (prometheus/inc! :metabase-transforms/stage-completed labels)+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))(mu/defn record-stage-failure!"Record the failure of a transform stage."- [job-run-id :- [:maybe pos-int?]- stage-label :- ::stage-label- duration-ms :- int?]- (let [stage-type (label-to-stage stage-label)]- (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))- (let [labels {:stage-type (name stage-type)- :stage-label (name stage-label)}]- (prometheus/inc! :metabase-transforms/stage-failed labels)- (when duration-ms- (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))))+ [job-run-id :- [:maybe pos-int?]+ [stage-type stage-label] :- ::stage+ duration-ms :- int?]+ (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))+ (let [labels {:stage-type (name stage-type)+ :stage-label (name stage-label)}]+ (prometheus/inc! :metabase-transforms/stage-failed labels)+ (when duration-ms+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))(defn with-timing"Generic timing function that executes body while recording metrics.@@ -93,10 +71,10 @@(throw t)))))(defmacro with-stage-timing- "Execute body while timing a transform stage. Automatically records start, completion or failure.+ "Execute body while timing a transform stage ([stage-type stage-label]). Automatically records start, completion or failure.Usage:- (with-stage-timing [job-run-id :dwh-to-file]+ (with-stage-timing [job-run-id [:export :dwh-to-file]];; stage implementationresult)"[args & body]@@ -108,10 +86,10 @@(mu/defn record-data-transfer!"Record metrics about data transfer (size and rows)."- [job-run-id :- [:maybe pos-int?]- stage-label :- ::stage-label- bytes :- [:maybe int?]- rows :- [:maybe int?]]+ [job-run-id :- [:maybe pos-int?]+ stage-label :- simple-keyword?+ bytes :- [:maybe int?]+ rows :- [:maybe int?]](if (and (nil? bytes) (nil? rows))(log/warnf "Data transfer recorded: run-id=%d stage=%s but no bytes or rows provided"job-run-id@@ -163,22 +141,3 @@:error-fn record-job-failure!}[~job-id ~run-method](^:once fn* [] ~@body)))--(mu/defn record-python-api-call!- "Record metrics about Python API calls."- [job-run-id :- [:maybe pos-int?]- duration-ms :- int?- status :- [:enum :success :error :timeout]]- (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)- (prometheus/inc! :metabase-transforms/python-api-calls-total {:status (name status)})- (prometheus/observe! :metabase-transforms/python-api-call-duration-ms {} duration-ms))--(defmacro with-python-api-timing- "Execute body while timing a Python API call."- [[job-run-id] & body]- `(with-timing {:success-fn (fn [job-run-id# duration-ms#]- (record-python-api-call! job-run-id# duration-ms# :success))- :error-fn (fn [job-run-id# duration-ms#]- (record-python-api-call! job-run-id# duration-ms# :error))}- [~job-run-id]- (^:once fn* [] ~@body)))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/interface.clj b/enterprise/backend/src/metabase_enterprise/transforms/interface.cljnew file mode 100644index 00000000000..11309db3b92--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms/interface.clj@@ -0,0 +1,57 @@+(ns metabase-enterprise.transforms.interface)++(defn- transform->transform-type+ [transform]+ (-> transform :source :type keyword))++(defmulti source-db-id+ "Return the ID of the source database for a given `transform`. The source database is where the data originates from+ before being transformed and written to the target destination.++ NOTE: Currently returns a single database ID, but we may want to change this to return multiple database IDs in the+ future since transforms can have multiple input sources."+ {:added "0.57.0" :arglists '([transform])}+ transform->transform-type)++(defmulti target-db-id+ "Return the ID of the target database for a given `transform`. The target database is the destination where the+ transformed data will be written. Often this is the same database as the source database."+ {:added "0.57.0" :arglists '([transform])}+ transform->transform-type)++(defmulti execute!+ "Execute a transform operation. Runs the actual transformation process, which might involve running SQL queries,+ Python scripts, or other transformation logic depending on the transform type.++ This method blocks and may take significant time depending on the data volume. Implementations+ should handle errors gracefully and provide appropriate logging.++ Returns nil (or a result that can be discarded).++ Options:+ - `:start-promise`+ Will have a `transform_run.run_id` value delivered once the execution is registered with the database.+ Callers can await this promise to identify the transform_run record, which enables progress / status monitoring.++ - `:run-method`+ Expected to be either `:cron` (for scheduled runs) or `:manual` (for ad-hoc or test runs)+ Used for instrumentation / metadata purposes."+ {:added "0.57.0" :arglists '([transform options])}+ (fn [transform _options]+ (transform->transform-type transform)))++(defmulti table-dependencies+ "Return a set of logical table dependencies of the transform, including indirect dependencies via cards.+ The transform execution system uses these dependencies to determine the correct order of execution+ and to detect circular dependencies.++ Each dependency is represented as either:+ - `{:table <table-id>}`+ A dependency on a table that exists and has been synced.+ - `{:transform <transform-id>}`+ A dependency on a table that does not yet exist, but is known to be the target of another transform.+ Represents a 'placeholder' table (as we no table id / metadata) for the same purposes.++ An empty set indicates no dependencies."+ {:added "0.57.0" :arglists '([transform])}+ transform->transform-type)diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.cljindex 7740701daad..f6798cf4844 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj@@ -5,9 +5,8 @@[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval][clojurewerkz.quartzite.triggers :as triggers][flatland.ordered.set :as ordered-set]- [metabase-enterprise.transforms-python.execute :as transforms-python.execute]- [metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.models.job-run :as transforms.job-run][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.ordering :as transforms.ordering]@@ -77,11 +76,7 @@(when (transform-run/running-run-for-transform-id transform-id)(recur))))(log/info "Executing job transform" (pr-str transform-id))- (if (transforms.util/python-transform? transform)- (transforms.instrumentation/with-stage-timing [run-id :python-execution]- (transforms-python.execute/execute-python-transform! transform {:run-method run-method}))- (transforms.instrumentation/with-stage-timing [run-id :mbql-query]- (transforms.execute/run-mbql-transform! transform {:run-method run-method})))+ (transforms.i/execute! transform {:run-method run-method})(transforms.job-run/add-run-activity! run-id))))(defn run-transforms!diff --git a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj b/enterprise/backend/src/metabase_enterprise/transforms/ordering.cljindex 0b7c509f7c6..cc18d98e565 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj@@ -3,6 +3,7 @@[clojure.set :as set][clojure.string :as str][flatland.ordered.set :refer [ordered-set]]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver][metabase.lib-be.core :as lib-be]@@ -16,30 +17,24 @@(set! *warn-on-reflection* true)-(mu/defn- transform-deps :- ::driver/native-query-deps+(defmethod transforms.i/table-dependencies :query[transform]- (if (transforms.util/python-transform? transform)- ;; Python transforms have dependencies via source-tables mapping- (into #{}- (map #(hash-map :table %))- (vals (get-in transform [:source :source-tables])))- ;; Query transforms have dependencies via SQL query analysis- (let [query (-> (get-in transform [:source :query])- transforms.util/massage-sql-query- qp.preprocess/preprocess)- driver (-> query- lib.metadata/database- :engine)]- (if (lib/native-only-query? query)- (driver/native-query-deps driver query)- (into #{}- (map (fn [table-id]- {:table table-id}))- (lib/all-source-table-ids query))))))+ (let [query (-> (get-in transform [:source :query])+ transforms.util/massage-sql-query+ qp.preprocess/preprocess)+ driver (-> query+ lib.metadata/database+ :engine)]+ (if (lib/native-only-query? query)+ (driver/native-query-deps driver query)+ (into #{}+ (map (fn [table-id]+ {:table table-id}))+ (lib/all-source-table-ids query)))))(defn- dependency-map [transforms](into {}- (map (juxt :id transform-deps))+ (map (juxt :id transforms.i/table-dependencies))transforms))(mu/defn- output-table-map@@ -65,7 +60,7 @@(let [;; Group all transforms by their databasetransforms-by-db (->> transforms(map (fn [transform]- (let [db-id (transforms.util/target-database-id transform)]+ (let [db-id (transforms.i/target-db-id transform)]{db-id [transform]})))(apply merge-with into))transform-ids (into #{} (map :id) transforms)@@ -130,9 +125,9 @@db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)output-tables (output-table-map mp db-transforms)transform-ids (into #{} (map :id) db-transforms)- node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]- (or (output-tables table)- (transform-ids transform)))))+ node->children #(->> % transforms-by-id transforms.i/table-dependencies (keep (fn [{:keys [table transform]}]+ (or (output-tables table)+ (transform-ids transform)))))id->name (comp :name transforms-by-id)cycle (find-cycle node->children [transform-id])](when cyclediff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.cljnew file mode 100644index 00000000000..8bc4a697d8a--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj@@ -0,0 +1,17 @@+(ns metabase-enterprise.transforms.query-impl+ (:require+ [metabase-enterprise.transforms.execute :as transforms.execute]+ [metabase-enterprise.transforms.interface :as transforms.i]))++(set! *warn-on-reflection* true)++(defmethod transforms.i/source-db-id :query+ [transform]+ (-> transform :source :query :database))++(defmethod transforms.i/target-db-id :query+ [transform]+ (-> transform :source :query :database))++(defmethod transforms.i/execute! :query [transform opts]+ (transforms.execute/run-mbql-transform! transform opts))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex 6e5b895c1e4..d0486b9dd66 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -4,11 +4,11 @@[clojure.string :as str][java-time.api :as t][metabase-enterprise.transforms.canceling :as canceling]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.settings :as transforms.settings][metabase.driver :as driver][metabase.lib.schema.common :as lib.schema.common]- [metabase.lib.schema.id :as lib.schema.id][metabase.premium-features.core :as premium-features :refer [defenterprise]][metabase.query-processor.compile :as qp.compile][metabase.query-processor.parameters.dates :as params.dates]@@ -105,17 +105,10 @@(log/info "Syncing target" (pr-str target) "for transform")(activate-table-and-mark-computed! database target)))-(mu/defn target-database-id :- ::lib.schema.id/database- "Return the target database id of a transform"- [transform]- (if (python-transform? transform)- (-> transform :target :database)- (-> transform :source :query :database)))-(defn target-table-exists?"Test if the target table of a transform already exists."[{:keys [target] :as transform}]- (let [db-id (target-database-id transform)+ (let [db-id (transforms.i/target-db-id transform){driver :engine :as database} (t2/select-one :model/Database db-id)](driver/table-exists? driver database target)))@@ -157,7 +150,7 @@[{:keys [id target], :as transform}](when target(let [target (update target :type keyword)- database-id (target-database-id transform)+ database-id (transforms.i/target-db-id transform){driver :engine :as database} (t2/select-one :model/Database database-id)](driver/drop-transform-target! driver database target)(log/info "Deactivating target " (pr-str target) "for transform" id)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 2b667319915..9224cb7a77f 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -285,7 +285,7 @@(with-open [in (python-runner/open-output @shared-storage-ref)](io/copy in temp-file))(let [file-size (.length temp-file)]- (transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]+ (transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]](transfer-file-to-db driver db transform output-manifest temp-file))(transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))(finally@@ -332,8 +332,9 @@(log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))(save-log-to-transform-run-message! run-id message-log))ex-message-fn #(exceptional-run-message message-log %)- result (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn)]- (transforms.instrumentation/with-stage-timing [run-id :table-sync]+ result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]+ (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]](transforms.util/sync-target! target db run-id)){:run_id run-id:result result}))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.cljnew file mode 100644index 00000000000..869127ed003--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj@@ -0,0 +1,22 @@+(ns metabase-enterprise.transforms-python.impl+ (:require+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]+ [metabase-enterprise.transforms.interface :as transforms.i]))++(defmethod transforms.i/target-db-id :python+ [transform]+ (-> transform :target :database))++(defmethod transforms.i/source-db-id :python+ [transform]+ (-> transform :source :database))++(defmethod transforms.i/execute! :python+ [transform options]+ (transforms-python.execute/execute-python-transform! transform options))++(defmethod transforms.i/table-dependencies :python+ [transform]+ (into #{}+ (map #(hash-map :table %))+ (vals (get-in transform [:source :source-tables]))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/init.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/init.cljindex e33b623721d..f29d803d57d 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/init.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/init.clj@@ -1,5 +1,4 @@(ns metabase-enterprise.transforms-python.init(:require+ [metabase-enterprise.transforms-python.impl][metabase-enterprise.transforms-python.settings]))--(comment metabase-enterprise.transforms-python.settings/keepme)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex e899336415e..c12c1542741 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -7,12 +7,14 @@[metabase-enterprise.transforms-python.s3 :as s3][metabase-enterprise.transforms-python.settings :as transforms-python.settings][metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]+ [metabase.analytics.prometheus :as prometheus][metabase.config.core :as config][metabase.query-processor :as qp][metabase.query-processor.pipeline :as qp.pipeline][metabase.util.i18n :as i18n][metabase.util.json :as json][metabase.util.log :as log]+ [metabase.util.malli :as mu][toucan2.core :as t2])(:import(clojure.lang PersistentQueue)@@ -175,6 +177,25 @@(let [server-url (transforms-python.settings/python-runner-url)](python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))+(mu/defn record-python-api-call!+ "Record metrics about Python API calls."+ [job-run-id :- [:maybe pos-int?]+ duration-ms :- int?+ status :- [:enum :success :error :timeout]]+ (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)+ (prometheus/inc! :metabase-transforms/python-api-calls-total {:status (name status)})+ (prometheus/observe! :metabase-transforms/python-api-call-duration-ms {} duration-ms))++(defmacro with-python-api-timing+ "Execute body while timing a Python API call."+ [[job-run-id] & body]+ `(transforms.instrumentation/with-timing {:success-fn (fn [job-run-id# duration-ms#]+ (record-python-api-call! job-run-id# duration-ms# :success))+ :error-fn (fn [job-run-id# duration-ms#]+ (record-python-api-call! job-run-id# duration-ms# :error))}+ [~job-run-id]+ (^:once fn* [] ~@body)))+(defn execute-python-code-http-call!"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returnsthe response from the server."@@ -193,7 +214,7 @@:events_url (:url events):table_mapping table-name->url:manifest_mapping table-name->manifest-url}- response (transforms.instrumentation/with-python-api-timing [run-id]+ response (with-python-api-timing [run-id](python-runner-request server-url :post "/execute" {:body (json/encode payload)}))];; when a 500 is returned we observe a string in the body (despite the python returning json);; always try to parse the returned string as json before yielding (could tighten this up at some point)@@ -267,7 +288,7 @@driver (t2/select-one-fn :engine :model/Database db-id)fields-meta (fields-metadata driver table-id)manifest (generate-manifest table-id fields-meta)]- (transforms.instrumentation/with-stage-timing [run-id :dwh-to-file]+ (transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]](write-table-data-to-file!{:db-id db-id:driver driver@@ -281,7 +302,7 @@(let [data-size (.length tmp-data-file)meta-size (.length tmp-meta-file)](transforms.instrumentation/record-data-transfer! run-id :dwh-to-file data-size nil)- (transforms.instrumentation/with-stage-timing [run-id :file-to-s3]+ (transforms.instrumentation/with-stage-timing [run-id [:export :file-to-s3]](s3/upload-file s3-client bucket-name data-path tmp-data-file)(s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))(transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.cljindex f7796ce3825..583779ed178 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj@@ -1,7 +1,7 @@(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test(:require[clojure.test :refer :all]- [metabase-enterprise.transforms.execute :as transforms.execute]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.query-test-util :as query-test-util][metabase-enterprise.transforms.test-dataset :as transforms-dataset][metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! delete-schema!]]@@ -52,14 +52,14 @@:source {:type :query:query t1-query}:target target1}]- (transforms.execute/run-mbql-transform! t1 {:run-method :manual})+ (transforms.i/execute! t1 {:run-method :manual})(let [table1 (transforms.tu/wait-for-table table1-name 10000)t2-query (make-query (->> table1 :name (table-name->qp-table (mt/metadata-provider))) "category" lib/= "Gizmo")](mt/with-temp [:model/Transform t2 {:name "transform2":source {:type :query:query t2-query}:target target2}]- (transforms.execute/run-mbql-transform! t2 {:run-method :cron})+ (transforms.i/execute! t2 {:run-method :cron})(let [table2 (transforms.tu/wait-for-table table2-name 10000)check-query (lib/aggregate (make-query (->> table2 :name (table-name->qp-table (mt/metadata-provider)))) (lib/count))query-result (qp/process-query check-query)]@@ -109,7 +109,7 @@:query (lib/limit query-no-limit 5)}:target limit-table}](doseq [transform [transform-no-limit transform-limit]]- (transforms.execute/run-mbql-transform! transform {:run-method :manual})+ (transforms.i/execute! transform {:run-method :manual})(let [table-name (-> transform :target :name)_ (transforms.tu/wait-for-table table-name 10000)table-result (lib.metadata/table mp (mt/id (keyword table-name)))@@ -162,7 +162,7 @@:source {:type :query:query query}:target target-table}]- (transforms.execute/run-mbql-transform! transform {:run-method :manual})+ (transforms.i/execute! transform {:run-method :manual})(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))query-result (->> (lib/query mp table-result)@@ -213,7 +213,7 @@:source {:type :query:query query}:target target-table}]- (transforms.execute/run-mbql-transform! transform {:run-method :manual})+ (transforms.i/execute! transform {:run-method :manual})(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))query-result (->> (lib/query mp table-result)@@ -238,7 +238,7 @@(is (thrown-with-msg?clojure.lang.ExceptionInfo#"ERROR: permission denied for database transforms-test"- (transforms.execute/run-mbql-transform! transform {:run-method :manual})))))))))+ (transforms.i/execute! transform {:run-method :manual})))))))))(finally(driver/execute-raw-queries! driver/*driver* spec[[(format "DROP OWNED BY %s;" no-schema-user)]@@ -266,4 +266,4 @@clojure.lang.ExceptionInfo#"Transforms are not supported on databases with DB routing enabled."(mt/with-current-user (mt/user->id :crowberto)- (transforms.execute/run-mbql-transform! transform {:run-method :manual}))))))))))+ (transforms.i/execute! transform {:run-method :manual}))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.cljindex aeee55451f7..86074a5a57d 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj@@ -3,7 +3,7 @@(:require[clojure.test :refer :all][clojure.tools.logging :as log]- [metabase-enterprise.transforms.execute :as transforms.execute]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.jobs :as jobs][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase.test :as mt]))@@ -134,8 +134,8 @@(with-redefs [log/log* (fn [_ level _ message](swap! logged-messages conj {:level level :message message}))transform-run/running-run-for-transform-id (constantly nil)- transforms.execute/run-mbql-transform! (fn [_ _]- (reset! run-called? true))]+ transforms.i/execute! (fn [_ _]+ (reset! run-called? true))](#'jobs/run-transform! run-id :scheduled query-transform)(is (empty? (filter (comp #{:warn} :level) @logged-messages))"Should not log warnings when feature is enabled")diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex 9e8939460d3..85e6cce2c51 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -1,7 +1,7 @@(ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test(:require[clojure.test :refer :all]- [metabase-enterprise.transforms.execute :as transforms.execute]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.ordering :as ordering][metabase.driver :as driver][metabase.driver.sql :as driver.sql]@@ -53,7 +53,7 @@(defn- transform-deps-for-db [transform](mt/with-metadata-provider (mt/id)- (#'ordering/transform-deps transform)))+ (#'transforms.i/table-dependencies transform)))(deftest not-run-transform-dependency-ordering-test(mt/test-driver :postgres@@ -111,7 +111,7 @@LIMIT 100"}}"venues_transform_2")](try- (transforms.execute/run-mbql-transform! transform1 {:run-method :manual})+ (transforms.i/execute! transform1 {:run-method :manual})(let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")](is (= #{{:table table1} {:transform t2}}(transform-deps-for-db (t2/select-one :model/Transform t3)))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.cljindex 6e7b50ae061..b36ea5efb50 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj@@ -4,7 +4,7 @@[clojure.core.async :as a][clojure.string :as str][clojure.test :refer :all]- [metabase-enterprise.transforms-python.execute :as transforms.execute]+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase-enterprise.transforms-python.python-runner :as python-runner][metabase-enterprise.transforms-python.s3 :as s3][metabase-enterprise.transforms-python.settings :as transforms-python.settings]@@ -65,7 +65,7 @@:target target}](with-transform-cleanup! [_target target](mt/with-temp [:model/Transform transform transform-def]- (transforms.execute/execute-python-transform! transform {:run-method :manual})+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})(let [table (transforms.tu/wait-for-table table-name 10000)columns (t2/select :model/Field :table_id (:id table) {:order-by [:position]})column-names (filterv (fn [x] (not= x "_id")) ;; for mongodiff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex fabddc94569..cbe370e31ad 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -1,6 +1,7 @@(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.ordering-test(:require[clojure.test :refer :all]+ [metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.ordering :as ordering][metabase.driver :as driver][metabase.driver.sql :as driver.sql]@@ -36,7 +37,7 @@(defn- transform-deps-for-db [transform](mt/with-metadata-provider (mt/id)- (#'ordering/transform-deps transform)))+ (#'transforms.i/table-dependencies transform)))(deftest python-transform-basic-dependencies-test(testing "Python transforms with source-tables dependencies are extracted correctly"