Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Transforms prepare for python (#63698)

Nicola Momettoeab4bf076d5d
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
index ea891dd76b1..bc1b82fdca0 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
@@ -1,14 +1,9 @@
(ns metabase-enterprise.transforms.execute
(:require
- [clojure.core.async :as a]
- [metabase-enterprise.transforms.canceling :as canceling]
- [metabase-enterprise.transforms.models.transform-run :as transform-run]
- [metabase-enterprise.transforms.settings :as transforms.settings]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.common :as schema.common]
- [metabase.query-processor.pipeline :as qp.pipeline]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
@@ -26,35 +21,6 @@
[:map
[:overwrite? :boolean]])
-(defn- sync-target!
- ([transform-id run-id]
- (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)
- db (get-in source [:query :database])
- database (t2/select-one :model/Database db)]
- (sync-target! target database run-id)))
- ([target database _run-id]
- ;; sync the new table (note that even a failed sync status means that the execution succeeded)
- (log/info "Syncing target" (pr-str target) "for transform")
- (transforms.util/activate-table-and-mark-computed! database target)))
-
-(defn- run-transform!
- "Run a compiled transform"
- [run-id driver {:keys [db-id conn-spec output-schema] :as transform-details} opts]
- ;; local run is responsible for status
- (try
- (when-not (driver/schema-exists? driver db-id output-schema)
- (driver/create-schema-if-needed! driver conn-spec output-schema))
- (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
- (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
- (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)
- (driver/run-transform! driver transform-details opts))
- (transform-run/succeed-started-run! run-id)
- (catch Throwable t
- (transform-run/fail-started-run! run-id {:message (.getMessage t)})
- (throw t))
- (finally
- (canceling/chan-end-run! run-id))))
-
(defn run-mbql-transform!
"Run `transform` and sync its target table.
@@ -81,20 +47,14 @@
(throw (ex-info "The database does not support the requested transform target type."
{:driver driver, :database database, :feature feature})))
;; mark the execution as started and notify any observers
- (let [{run-id :id} (try
- (transform-run/start-run! id {:run_method run-method})
- (catch java.sql.SQLException e
- (if (= (.getSQLState e) "23505")
- (throw (ex-info "Transform is already running"
- {:error :already-running
- :transform-id id}
- e))
- (throw e))))]
+ (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
(when start-promise
(deliver start-promise [:started run-id]))
(log/info "Executing transform" id "with target" (pr-str target))
- (run-transform! run-id driver transform-details opts)
- (sync-target! target database run-id)))
+ (transforms.util/run-cancelable-transform!
+ run-id driver transform-details
+ (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+ (transforms.util/sync-target! target database run-id)))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
index 41d0761da66..68ab07b40af 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
@@ -64,14 +64,17 @@
(map transforms-by-id complete)))))
(defn- run-transform! [run-id run-method {transform-id :id :as transform}]
- (when (transform-run/running-run-for-run-id transform-id)
+ (when
+ (transform-run/running-run-for-transform-id transform-id)
(log/warn "Transform" (pr-str transform-id) "already running, waiting")
(loop []
(Thread/sleep 2000)
- (when (transform-run/running-run-for-run-id transform-id)
+ (when (transform-run/running-run-for-transform-id transform-id)
(recur))))
+
(log/info "Executing job transform" (pr-str transform-id))
(transforms.execute/run-mbql-transform! transform {:run-method run-method})
+
(transforms.job-run/add-run-activity! run-id))
(defn run-transforms!
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj
index d9eb4687949..cc34f34e236 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj
@@ -147,19 +147,13 @@
:message "Canceled by user but could not guarantee run stopped."})
(cancel/delete-old-canceling-runs!)))
-(defn running-run-for-run-id
- "Return a single active transform run or nil."
- [id]
- (t2/select-one :model/TransformRun
- :id id
- :is_active true))
-
(defn running-run-for-transform-id
"Return a single active transform run or nil."
[transform-id]
(t2/select-one :model/TransformRun
:transform_id transform-id
:is_active true))
+
(defn- timestamp-constraint
[field-name date-string]
(let [{:keys [start end]}
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index d4cf6877ee3..b22684bd615 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -1,8 +1,14 @@
(ns metabase-enterprise.transforms.util
(:require
+ [clojure.core.async :as a]
+ [clojure.string :as str]
[java-time.api :as t]
+ [metabase-enterprise.transforms.canceling :as canceling]
+ [metabase-enterprise.transforms.models.transform-run :as transform-run]
+ [metabase-enterprise.transforms.settings :as transforms.settings]
[metabase.driver :as driver]
[metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor.pipeline :as qp.pipeline]
[metabase.sync.core :as sync]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
@@ -21,6 +27,55 @@
(keyword schema name)
(keyword name)))
+(defn try-start-unless-already-running
+ "Start a transform run, throwing an informative error if already running."
+ [id run-method]
+ (try
+ (transform-run/start-run! id {:run_method run-method})
+ (catch java.sql.SQLException e
+ (if (= (.getSQLState e) "23505")
+ (throw (ex-info "Transform is already running"
+ {:error :already-running
+ :transform-id id}
+ e))
+ (throw e)))))
+
+(defn run-cancelable-transform!
+ "Execute a transform with cancellation support and proper error handling."
+ [run-id driver {:keys [db-id conn-spec output-schema]} run-transform!]
+ ;; local run is responsible for status, using canceling lifecycle
+ (try
+ (when-not (driver/schema-exists? driver db-id output-schema)
+ (driver/create-schema-if-needed! driver conn-spec output-schema))
+ (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
+ (let [cancel-chan (a/promise-chan)
+ ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
+ (canceling/chan-start-run! run-id cancel-chan)
+ (run-transform! cancel-chan))]
+ (transform-run/succeed-started-run! run-id)
+ ret)
+ (catch Throwable t
+ (let [{:keys [transform-run-message]} (ex-data t)
+ message (str/join "\n" (remove str/blank? [transform-run-message (.getMessage t)]))]
+ (transform-run/fail-started-run! run-id {:message message}))
+ (throw t))
+ (finally
+ (canceling/chan-end-run! run-id))))
+
+(declare activate-table-and-mark-computed!)
+
+(defn sync-target!
+ "Sync target of a transform"
+ ([transform-id run-id]
+ (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)
+ db (get-in source [:query :database])
+ database (t2/select-one :model/Database db)]
+ (sync-target! target database run-id)))
+ ([target database _run-id]
+ ;; sync the new table (note that even a failed sync status means that the execution succeeded)
+ (log/info "Syncing target" (pr-str target) "for transform")
+ (activate-table-and-mark-computed! database target)))
+
(defn target-table-exists?
"Test if the target table of a transform already exists."
[{:keys [source target] :as _transform}]