Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Polymorphic transforms (#64084)

Ngoc Khuat65d95d943885
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index d129a0edb8e..96be9193836 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2004,6 +2004,7 @@
metabase-enterprise.transforms.core
metabase-enterprise.transforms.init
metabase-enterprise.transforms.instrumentation
+ metabase-enterprise.transforms.interface
metabase-enterprise.transforms.util}
:uses #{analytics
api
@@ -2020,15 +2021,14 @@
sync
task
task-history
- util
- enterprise/transforms-python}}
+ util}}
enterprise/transforms-python
{:team "Querying"
:api #{metabase-enterprise.transforms-python.api
- metabase-enterprise.transforms-python.execute
metabase-enterprise.transforms-python.init}
- :uses #{api
+ :uses #{analytics
+ api
app-db
config
driver
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
index 4e839be4a9a..f6a3c81a326 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
@@ -1,10 +1,9 @@
(ns metabase-enterprise.transforms.api
(:require
- [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms.api.transform-job]
[metabase-enterprise.transforms.api.transform-tag]
[metabase-enterprise.transforms.canceling :as transforms.canceling]
- [metabase-enterprise.transforms.execute :as transforms.execute]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.transform :as transform.model]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.models.transform-run-cancelation :as transform-run-cancelation]
@@ -69,7 +68,7 @@
(defn- check-database-feature
[transform]
- (let [database (api/check-400 (t2/select-one :model/Database (transforms.util/target-database-id transform))
+ (let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))
(deferred-tru "The target database cannot be found."))
feature (transforms.util/required-database-feature transform)]
(api/check-400 (not (:is_sample database))
@@ -139,7 +138,7 @@
(log/info "get transform" id)
(api/check-superuser)
(let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))
- target-table (transforms.util/target-table (transforms.util/target-database-id transform) target :active true)]
+ target-table (transforms.util/target-table (transforms.i/target-db-id transform) target :active true)]
(-> transform
(t2/hydrate :last_run :transform_tag_ids)
(u/update-some :last_run transforms.util/localize-run-timestamps)
@@ -255,13 +254,9 @@
(let [transform (api/check-404 (t2/select-one :model/Transform id))
_ (check-feature-enabled! transform)
start-promise (promise)]
- (if (transforms.util/python-transform? transform)
- (u.jvm/in-virtual-thread*
- (transforms-python.execute/execute-python-transform! transform {:start-promise start-promise
- :run-method :manual}))
- (u.jvm/in-virtual-thread*
- (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
- :run-method :manual})))
+ (u.jvm/in-virtual-thread*
+ (transforms.i/execute! transform {:start-promise start-promise
+ :run-method :manual}))
(when (instance? Throwable @start-promise)
(throw @start-promise))
(let [result @start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
index fa920c6bbae..5409450fa00 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
@@ -54,10 +54,11 @@
(when start-promise
(deliver start-promise [:started run-id]))
(log/info "Executing transform" id "with target" (pr-str target))
- (transforms.util/run-cancelable-transform!
- run-id driver transform-details
- (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
- (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
+ (transforms.util/run-cancelable-transform!
+ run-id driver transform-details
+ (fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target database run-id)
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
(events/publish-event! :event/transform-run-complete {:object transform-details}))))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/init.clj b/enterprise/backend/src/metabase_enterprise/transforms/init.clj
index 6cb332dbd24..c0ca2cd1de7 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/init.clj
@@ -1,7 +1,6 @@
(ns metabase-enterprise.transforms.init
(:require
[metabase-enterprise.transforms.canceling]
- [metabase-enterprise.transforms.execute]
[metabase-enterprise.transforms.jobs]
[metabase-enterprise.transforms.models.job-run]
[metabase-enterprise.transforms.models.transform]
@@ -11,6 +10,7 @@
[metabase-enterprise.transforms.models.transform-run-cancelation]
[metabase-enterprise.transforms.models.transform-tag]
[metabase-enterprise.transforms.models.transform-transform-tag]
+ [metabase-enterprise.transforms.query-impl]
[metabase-enterprise.transforms.schedule]
[metabase-enterprise.transforms.settings]
[metabase-enterprise.transforms.timeout]))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj
index c991d0ede58..e194fb652ac 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj
@@ -8,61 +8,39 @@
(set! *warn-on-reflection* true)
-(mr/def ::stage-type
- [:enum :export :computation :import])
-
-(def ^:private label-to-stage
- {:dwh-to-file :export
- :file-to-s3 :export
- :file-to-dwh :import
- :python-execution :computation
- :mbql-query :computation
- :table-sync :import})
-
-(mr/def ::stage-label
- [:enum
- :dwh-to-file
- :file-to-s3
- :file-to-dwh
-
- :python-execution
- :mbql-query
- :table-sync])
+(mr/def ::stage [:tuple simple-keyword? simple-keyword?])
(mu/defn record-stage-start!
"Record the start of a transform stage."
- [job-run-id :- [:maybe pos-int?]
- stage-label :- ::stage-label]
- (let [stage-type (label-to-stage stage-label)]
- (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
- (prometheus/inc! :metabase-transforms/stage-started
- {:stage-type (name stage-type)
- :stage-label (name stage-label)})))
+ [job-run-id :- [:maybe pos-int?]
+ [stage-type stage-label] :- ::stage]
+ (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+ (prometheus/inc! :metabase-transforms/stage-started
+ {:stage-type (name stage-type)
+ :stage-label (name stage-label)}))
(mu/defn record-stage-completion!
"Record the successful completion of a transform stage."
- [job-run-id :- [:maybe pos-int?]
- stage-label :- ::stage-label
- duration-ms :- int?]
- (let [stage-type (label-to-stage stage-label)]
- (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
- (let [labels {:stage-type (name stage-type)
- :stage-label (name stage-label)}]
- (prometheus/inc! :metabase-transforms/stage-completed labels)
- (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+ [job-run-id :- [:maybe pos-int?]
+ [stage-type stage-label] :- ::stage
+ duration-ms :- int?]
+ (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+ (let [labels {:stage-type (name stage-type)
+ :stage-label (name stage-label)}]
+ (prometheus/inc! :metabase-transforms/stage-completed labels)
+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))
(mu/defn record-stage-failure!
"Record the failure of a transform stage."
- [job-run-id :- [:maybe pos-int?]
- stage-label :- ::stage-label
- duration-ms :- int?]
- (let [stage-type (label-to-stage stage-label)]
- (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))
- (let [labels {:stage-type (name stage-type)
- :stage-label (name stage-label)}]
- (prometheus/inc! :metabase-transforms/stage-failed labels)
- (when duration-ms
- (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))))
+ [job-run-id :- [:maybe pos-int?]
+ [stage-type stage-label] :- ::stage
+ duration-ms :- int?]
+ (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))
+ (let [labels {:stage-type (name stage-type)
+ :stage-label (name stage-label)}]
+ (prometheus/inc! :metabase-transforms/stage-failed labels)
+ (when duration-ms
+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
(defn with-timing
"Generic timing function that executes body while recording metrics.
@@ -93,10 +71,10 @@
(throw t)))))
(defmacro with-stage-timing
- "Execute body while timing a transform stage. Automatically records start, completion or failure.
+ "Execute body while timing a transform stage ([stage-type stage-label]). Automatically records start, completion or failure.
Usage:
- (with-stage-timing [job-run-id :dwh-to-file]
+ (with-stage-timing [job-run-id [:export :dwh-to-file]]
;; stage implementation
result)"
[args & body]
@@ -108,10 +86,10 @@
(mu/defn record-data-transfer!
"Record metrics about data transfer (size and rows)."
- [job-run-id :- [:maybe pos-int?]
- stage-label :- ::stage-label
- bytes :- [:maybe int?]
- rows :- [:maybe int?]]
+ [job-run-id :- [:maybe pos-int?]
+ stage-label :- simple-keyword?
+ bytes :- [:maybe int?]
+ rows :- [:maybe int?]]
(if (and (nil? bytes) (nil? rows))
(log/warnf "Data transfer recorded: run-id=%d stage=%s but no bytes or rows provided"
job-run-id
@@ -163,22 +141,3 @@
:error-fn record-job-failure!}
[~job-id ~run-method]
(^:once fn* [] ~@body)))
-
-(mu/defn record-python-api-call!
- "Record metrics about Python API calls."
- [job-run-id :- [:maybe pos-int?]
- duration-ms :- int?
- status :- [:enum :success :error :timeout]]
- (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
- (prometheus/inc! :metabase-transforms/python-api-calls-total {:status (name status)})
- (prometheus/observe! :metabase-transforms/python-api-call-duration-ms {} duration-ms))
-
-(defmacro with-python-api-timing
- "Execute body while timing a Python API call."
- [[job-run-id] & body]
- `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
- (record-python-api-call! job-run-id# duration-ms# :success))
- :error-fn (fn [job-run-id# duration-ms#]
- (record-python-api-call! job-run-id# duration-ms# :error))}
- [~job-run-id]
- (^:once fn* [] ~@body)))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/interface.clj b/enterprise/backend/src/metabase_enterprise/transforms/interface.clj
new file mode 100644
index 00000000000..11309db3b92
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms/interface.clj
@@ -0,0 +1,57 @@
+(ns metabase-enterprise.transforms.interface)
+
+(defn- transform->transform-type
+ [transform]
+ (-> transform :source :type keyword))
+
+(defmulti source-db-id
+ "Return the ID of the source database for a given `transform`. The source database is where the data originates from
+ before being transformed and written to the target destination.
+
+ NOTE: Currently returns a single database ID, but we may want to change this to return multiple database IDs in the
+ future since transforms can have multiple input sources."
+ {:added "0.57.0" :arglists '([transform])}
+ transform->transform-type)
+
+(defmulti target-db-id
+ "Return the ID of the target database for a given `transform`. The target database is the destination where the
+ transformed data will be written. Often this is the same database as the source database."
+ {:added "0.57.0" :arglists '([transform])}
+ transform->transform-type)
+
+(defmulti execute!
+ "Execute a transform operation. Runs the actual transformation process, which might involve running SQL queries,
+ Python scripts, or other transformation logic depending on the transform type.
+
+ This method blocks and may take significant time depending on the data volume. Implementations
+ should handle errors gracefully and provide appropriate logging.
+
+ Returns nil (or a result that can be discarded).
+
+ Options:
+ - `:start-promise`
+ Will have a `transform_run.run_id` value delivered once the execution is registered with the database.
+ Callers can await this promise to identify the transform_run record, which enables progress / status monitoring.
+
+ - `:run-method`
+ Expected to be either `:cron` (for scheduled runs) or `:manual` (for ad-hoc or test runs)
+ Used for instrumentation / metadata purposes."
+ {:added "0.57.0" :arglists '([transform options])}
+ (fn [transform _options]
+ (transform->transform-type transform)))
+
+(defmulti table-dependencies
+ "Return a set of logical table dependencies of the transform, including indirect dependencies via cards.
+ The transform execution system uses these dependencies to determine the correct order of execution
+ and to detect circular dependencies.
+
+ Each dependency is represented as either:
+ - `{:table <table-id>}`
+ A dependency on a table that exists and has been synced.
+ - `{:transform <transform-id>}`
+ A dependency on a table that does not yet exist, but is known to be the target of another transform.
+ Represents a 'placeholder' table (as we no table id / metadata) for the same purposes.
+
+ An empty set indicates no dependencies."
+ {:added "0.57.0" :arglists '([transform])}
+ transform->transform-type)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
index 7740701daad..f6798cf4844 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
@@ -5,9 +5,8 @@
[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
[clojurewerkz.quartzite.triggers :as triggers]
[flatland.ordered.set :as ordered-set]
- [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
- [metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.job-run :as transforms.job-run]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.ordering :as transforms.ordering]
@@ -77,11 +76,7 @@
(when (transform-run/running-run-for-transform-id transform-id)
(recur))))
(log/info "Executing job transform" (pr-str transform-id))
- (if (transforms.util/python-transform? transform)
- (transforms.instrumentation/with-stage-timing [run-id :python-execution]
- (transforms-python.execute/execute-python-transform! transform {:run-method run-method}))
- (transforms.instrumentation/with-stage-timing [run-id :mbql-query]
- (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+ (transforms.i/execute! transform {:run-method run-method})
(transforms.job-run/add-run-activity! run-id))))
(defn run-transforms!
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
index 0b7c509f7c6..cc18d98e565 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
@@ -3,6 +3,7 @@
[clojure.set :as set]
[clojure.string :as str]
[flatland.ordered.set :refer [ordered-set]]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.lib-be.core :as lib-be]
@@ -16,30 +17,24 @@
(set! *warn-on-reflection* true)
-(mu/defn- transform-deps :- ::driver/native-query-deps
+(defmethod transforms.i/table-dependencies :query
[transform]
- (if (transforms.util/python-transform? transform)
- ;; Python transforms have dependencies via source-tables mapping
- (into #{}
- (map #(hash-map :table %))
- (vals (get-in transform [:source :source-tables])))
- ;; Query transforms have dependencies via SQL query analysis
- (let [query (-> (get-in transform [:source :query])
- transforms.util/massage-sql-query
- qp.preprocess/preprocess)
- driver (-> query
- lib.metadata/database
- :engine)]
- (if (lib/native-only-query? query)
- (driver/native-query-deps driver query)
- (into #{}
- (map (fn [table-id]
- {:table table-id}))
- (lib/all-source-table-ids query))))))
+ (let [query (-> (get-in transform [:source :query])
+ transforms.util/massage-sql-query
+ qp.preprocess/preprocess)
+ driver (-> query
+ lib.metadata/database
+ :engine)]
+ (if (lib/native-only-query? query)
+ (driver/native-query-deps driver query)
+ (into #{}
+ (map (fn [table-id]
+ {:table table-id}))
+ (lib/all-source-table-ids query)))))
(defn- dependency-map [transforms]
(into {}
- (map (juxt :id transform-deps))
+ (map (juxt :id transforms.i/table-dependencies))
transforms))
(mu/defn- output-table-map
@@ -65,7 +60,7 @@
(let [;; Group all transforms by their database
transforms-by-db (->> transforms
(map (fn [transform]
- (let [db-id (transforms.util/target-database-id transform)]
+ (let [db-id (transforms.i/target-db-id transform)]
{db-id [transform]})))
(apply merge-with into))
transform-ids (into #{} (map :id) transforms)
@@ -130,9 +125,9 @@
db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)
output-tables (output-table-map mp db-transforms)
transform-ids (into #{} (map :id) db-transforms)
- node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]
- (or (output-tables table)
- (transform-ids transform)))))
+ node->children #(->> % transforms-by-id transforms.i/table-dependencies (keep (fn [{:keys [table transform]}]
+ (or (output-tables table)
+ (transform-ids transform)))))
id->name (comp :name transforms-by-id)
cycle (find-cycle node->children [transform-id])]
(when cycle
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
new file mode 100644
index 00000000000..8bc4a697d8a
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
@@ -0,0 +1,17 @@
+(ns metabase-enterprise.transforms.query-impl
+ (:require
+ [metabase-enterprise.transforms.execute :as transforms.execute]
+ [metabase-enterprise.transforms.interface :as transforms.i]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod transforms.i/source-db-id :query
+ [transform]
+ (-> transform :source :query :database))
+
+(defmethod transforms.i/target-db-id :query
+ [transform]
+ (-> transform :source :query :database))
+
+(defmethod transforms.i/execute! :query [transform opts]
+ (transforms.execute/run-mbql-transform! transform opts))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index 6e5b895c1e4..d0486b9dd66 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -4,11 +4,11 @@
[clojure.string :as str]
[java-time.api :as t]
[metabase-enterprise.transforms.canceling :as canceling]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.settings :as transforms.settings]
[metabase.driver :as driver]
[metabase.lib.schema.common :as lib.schema.common]
- [metabase.lib.schema.id :as lib.schema.id]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.parameters.dates :as params.dates]
@@ -105,17 +105,10 @@
(log/info "Syncing target" (pr-str target) "for transform")
(activate-table-and-mark-computed! database target)))
-(mu/defn target-database-id :- ::lib.schema.id/database
- "Return the target database id of a transform"
- [transform]
- (if (python-transform? transform)
- (-> transform :target :database)
- (-> transform :source :query :database)))
-
(defn target-table-exists?
"Test if the target table of a transform already exists."
[{:keys [target] :as transform}]
- (let [db-id (target-database-id transform)
+ (let [db-id (transforms.i/target-db-id transform)
{driver :engine :as database} (t2/select-one :model/Database db-id)]
(driver/table-exists? driver database target)))
@@ -157,7 +150,7 @@
[{:keys [id target], :as transform}]
(when target
(let [target (update target :type keyword)
- database-id (target-database-id transform)
+ database-id (transforms.i/target-db-id transform)
{driver :engine :as database} (t2/select-one :model/Database database-id)]
(driver/drop-transform-target! driver database target)
(log/info "Deactivating target " (pr-str target) "for transform" id)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 2b667319915..9224cb7a77f 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -285,7 +285,7 @@
(with-open [in (python-runner/open-output @shared-storage-ref)]
(io/copy in temp-file))
(let [file-size (.length temp-file)]
- (transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]
+ (transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]]
(transfer-file-to-db driver db transform output-manifest temp-file))
(transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
(finally
@@ -332,8 +332,9 @@
(log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))
(save-log-to-transform-run-message! run-id message-log))
ex-message-fn #(exceptional-run-message message-log %)
- result (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn)]
- (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+ result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
+ (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target db run-id))
{:run_id run-id
:result result}))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
new file mode 100644
index 00000000000..869127ed003
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
@@ -0,0 +1,22 @@
+(ns metabase-enterprise.transforms-python.impl
+ (:require
+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
+ [metabase-enterprise.transforms.interface :as transforms.i]))
+
+(defmethod transforms.i/target-db-id :python
+ [transform]
+ (-> transform :target :database))
+
+(defmethod transforms.i/source-db-id :python
+ [transform]
+ (-> transform :source :database))
+
+(defmethod transforms.i/execute! :python
+ [transform options]
+ (transforms-python.execute/execute-python-transform! transform options))
+
+(defmethod transforms.i/table-dependencies :python
+ [transform]
+ (into #{}
+ (map #(hash-map :table %))
+ (vals (get-in transform [:source :source-tables]))))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/init.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/init.clj
index e33b623721d..f29d803d57d 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/init.clj
@@ -1,5 +1,4 @@
(ns metabase-enterprise.transforms-python.init
(:require
+ [metabase-enterprise.transforms-python.impl]
[metabase-enterprise.transforms-python.settings]))
-
-(comment metabase-enterprise.transforms-python.settings/keepme)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index e899336415e..c12c1542741 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -7,12 +7,14 @@
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+ [metabase.analytics.prometheus :as prometheus]
[metabase.config.core :as config]
[metabase.query-processor :as qp]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.util.i18n :as i18n]
[metabase.util.json :as json]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
[toucan2.core :as t2])
(:import
(clojure.lang PersistentQueue)
@@ -175,6 +177,25 @@
(let [server-url (transforms-python.settings/python-runner-url)]
(python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))
+(mu/defn record-python-api-call!
+ "Record metrics about Python API calls."
+ [job-run-id :- [:maybe pos-int?]
+ duration-ms :- int?
+ status :- [:enum :success :error :timeout]]
+ (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+ (prometheus/inc! :metabase-transforms/python-api-calls-total {:status (name status)})
+ (prometheus/observe! :metabase-transforms/python-api-call-duration-ms {} duration-ms))
+
+(defmacro with-python-api-timing
+ "Execute body while timing a Python API call."
+ [[job-run-id] & body]
+ `(transforms.instrumentation/with-timing {:success-fn (fn [job-run-id# duration-ms#]
+ (record-python-api-call! job-run-id# duration-ms# :success))
+ :error-fn (fn [job-run-id# duration-ms#]
+ (record-python-api-call! job-run-id# duration-ms# :error))}
+ [~job-run-id]
+ (^:once fn* [] ~@body)))
+
(defn execute-python-code-http-call!
"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
the response from the server."
@@ -193,7 +214,7 @@
:events_url (:url events)
:table_mapping table-name->url
:manifest_mapping table-name->manifest-url}
- response (transforms.instrumentation/with-python-api-timing [run-id]
+ response (with-python-api-timing [run-id]
(python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
;; when a 500 is returned we observe a string in the body (despite the python returning json)
;; always try to parse the returned string as json before yielding (could tighten this up at some point)
@@ -267,7 +288,7 @@
driver (t2/select-one-fn :engine :model/Database db-id)
fields-meta (fields-metadata driver table-id)
manifest (generate-manifest table-id fields-meta)]
- (transforms.instrumentation/with-stage-timing [run-id :dwh-to-file]
+ (transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]]
(write-table-data-to-file!
{:db-id db-id
:driver driver
@@ -281,7 +302,7 @@
(let [data-size (.length tmp-data-file)
meta-size (.length tmp-meta-file)]
(transforms.instrumentation/record-data-transfer! run-id :dwh-to-file data-size nil)
- (transforms.instrumentation/with-stage-timing [run-id :file-to-s3]
+ (transforms.instrumentation/with-stage-timing [run-id [:export :file-to-s3]]
(s3/upload-file s3-client bucket-name data-path tmp-data-file)
(s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))
(transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
index f7796ce3825..583779ed178 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
@@ -1,7 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test
(:require
[clojure.test :refer :all]
- [metabase-enterprise.transforms.execute :as transforms.execute]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! delete-schema!]]
@@ -52,14 +52,14 @@
:source {:type :query
:query t1-query}
:target target1}]
- (transforms.execute/run-mbql-transform! t1 {:run-method :manual})
+ (transforms.i/execute! t1 {:run-method :manual})
(let [table1 (transforms.tu/wait-for-table table1-name 10000)
t2-query (make-query (->> table1 :name (table-name->qp-table (mt/metadata-provider))) "category" lib/= "Gizmo")]
(mt/with-temp [:model/Transform t2 {:name "transform2"
:source {:type :query
:query t2-query}
:target target2}]
- (transforms.execute/run-mbql-transform! t2 {:run-method :cron})
+ (transforms.i/execute! t2 {:run-method :cron})
(let [table2 (transforms.tu/wait-for-table table2-name 10000)
check-query (lib/aggregate (make-query (->> table2 :name (table-name->qp-table (mt/metadata-provider)))) (lib/count))
query-result (qp/process-query check-query)]
@@ -109,7 +109,7 @@
:query (lib/limit query-no-limit 5)}
:target limit-table}]
(doseq [transform [transform-no-limit transform-limit]]
- (transforms.execute/run-mbql-transform! transform {:run-method :manual})
+ (transforms.i/execute! transform {:run-method :manual})
(let [table-name (-> transform :target :name)
_ (transforms.tu/wait-for-table table-name 10000)
table-result (lib.metadata/table mp (mt/id (keyword table-name)))
@@ -162,7 +162,7 @@
:source {:type :query
:query query}
:target target-table}]
- (transforms.execute/run-mbql-transform! transform {:run-method :manual})
+ (transforms.i/execute! transform {:run-method :manual})
(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
query-result (->> (lib/query mp table-result)
@@ -213,7 +213,7 @@
:source {:type :query
:query query}
:target target-table}]
- (transforms.execute/run-mbql-transform! transform {:run-method :manual})
+ (transforms.i/execute! transform {:run-method :manual})
(let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
query-result (->> (lib/query mp table-result)
@@ -238,7 +238,7 @@
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"ERROR: permission denied for database transforms-test"
- (transforms.execute/run-mbql-transform! transform {:run-method :manual})))))))))
+ (transforms.i/execute! transform {:run-method :manual})))))))))
(finally
(driver/execute-raw-queries! driver/*driver* spec
[[(format "DROP OWNED BY %s;" no-schema-user)]
@@ -266,4 +266,4 @@
clojure.lang.ExceptionInfo
#"Transforms are not supported on databases with DB routing enabled."
(mt/with-current-user (mt/user->id :crowberto)
- (transforms.execute/run-mbql-transform! transform {:run-method :manual}))))))))))
+ (transforms.i/execute! transform {:run-method :manual}))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
index aeee55451f7..86074a5a57d 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
@@ -3,7 +3,7 @@
(:require
[clojure.test :refer :all]
[clojure.tools.logging :as log]
- [metabase-enterprise.transforms.execute :as transforms.execute]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.jobs :as jobs]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase.test :as mt]))
@@ -134,8 +134,8 @@
(with-redefs [log/log* (fn [_ level _ message]
(swap! logged-messages conj {:level level :message message}))
transform-run/running-run-for-transform-id (constantly nil)
- transforms.execute/run-mbql-transform! (fn [_ _]
- (reset! run-called? true))]
+ transforms.i/execute! (fn [_ _]
+ (reset! run-called? true))]
(#'jobs/run-transform! run-id :scheduled query-transform)
(is (empty? (filter (comp #{:warn} :level) @logged-messages))
"Should not log warnings when feature is enabled")
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
index 9e8939460d3..85e6cce2c51 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
@@ -1,7 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test
(:require
[clojure.test :refer :all]
- [metabase-enterprise.transforms.execute :as transforms.execute]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.ordering :as ordering]
[metabase.driver :as driver]
[metabase.driver.sql :as driver.sql]
@@ -53,7 +53,7 @@
(defn- transform-deps-for-db [transform]
(mt/with-metadata-provider (mt/id)
- (#'ordering/transform-deps transform)))
+ (#'transforms.i/table-dependencies transform)))
(deftest not-run-transform-dependency-ordering-test
(mt/test-driver :postgres
@@ -111,7 +111,7 @@
LIMIT 100"}}
"venues_transform_2")]
(try
- (transforms.execute/run-mbql-transform! transform1 {:run-method :manual})
+ (transforms.i/execute! transform1 {:run-method :manual})
(let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")]
(is (= #{{:table table1} {:transform t2}}
(transform-deps-for-db (t2/select-one :model/Transform t3)))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
index 6e7b50ae061..b36ea5efb50 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -4,7 +4,7 @@
[clojure.core.async :as a]
[clojure.string :as str]
[clojure.test :refer :all]
- [metabase-enterprise.transforms-python.execute :as transforms.execute]
+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
@@ -65,7 +65,7 @@
:target target}]
(with-transform-cleanup! [_target target]
(mt/with-temp [:model/Transform transform transform-def]
- (transforms.execute/execute-python-transform! transform {:run-method :manual})
+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
(let [table (transforms.tu/wait-for-table table-name 10000)
columns (t2/select :model/Field :table_id (:id table) {:order-by [:position]})
column-names (filterv (fn [x] (not= x "_id")) ;; for mongo
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index fabddc94569..cbe370e31ad 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -1,6 +1,7 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.ordering-test
(:require
[clojure.test :refer :all]
+ [metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.ordering :as ordering]
[metabase.driver :as driver]
[metabase.driver.sql :as driver.sql]
@@ -36,7 +37,7 @@
(defn- transform-deps-for-db [transform]
(mt/with-metadata-provider (mt/id)
- (#'ordering/transform-deps transform)))
+ (#'transforms.i/table-dependencies transform)))
(deftest python-transform-basic-dependencies-test
(testing "Python transforms with source-tables dependencies are extracted correctly"