The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Transforms prepare for python (#63698)
Nicola Momettoeab4bf076d5d
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.cljindex ea891dd76b1..bc1b82fdca0 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj@@ -1,14 +1,9 @@(ns metabase-enterprise.transforms.execute(:require- [clojure.core.async :as a]- [metabase-enterprise.transforms.canceling :as canceling]- [metabase-enterprise.transforms.models.transform-run :as transform-run]- [metabase-enterprise.transforms.settings :as transforms.settings][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.lib.schema.common :as schema.common]- [metabase.query-processor.pipeline :as qp.pipeline][metabase.util.log :as log][metabase.util.malli.registry :as mr][toucan2.core :as t2]))@@ -26,35 +21,6 @@[:map[:overwrite? :boolean]])-(defn- sync-target!- ([transform-id run-id]- (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)- db (get-in source [:query :database])- database (t2/select-one :model/Database db)]- (sync-target! target database run-id)))- ([target database _run-id]- ;; sync the new table (note that even a failed sync status means that the execution succeeded)- (log/info "Syncing target" (pr-str target) "for transform")- (transforms.util/activate-table-and-mark-computed! database target)))--(defn- run-transform!- "Run a compiled transform"- [run-id driver {:keys [db-id conn-spec output-schema] :as transform-details} opts]- ;; local run is responsible for status- (try- (when-not (driver/schema-exists? driver db-id output-schema)- (driver/create-schema-if-needed! driver conn-spec output-schema))- (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))- (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]- (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)- (driver/run-transform! driver transform-details opts))- (transform-run/succeed-started-run! run-id)- (catch Throwable t- (transform-run/fail-started-run! run-id {:message (.getMessage t)})- (throw t))- (finally- (canceling/chan-end-run! run-id))))-(defn run-mbql-transform!"Run `transform` and sync its target table.@@ -81,20 +47,14 @@(throw (ex-info "The database does not support the requested transform target type."{:driver driver, :database database, :feature feature})));; mark the execution as started and notify any observers- (let [{run-id :id} (try- (transform-run/start-run! id {:run_method run-method})- (catch java.sql.SQLException e- (if (= (.getSQLState e) "23505")- (throw (ex-info "Transform is already running"- {:error :already-running- :transform-id id}- e))- (throw e))))]+ (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)](when start-promise(deliver start-promise [:started run-id]))(log/info "Executing transform" id "with target" (pr-str target))- (run-transform! run-id driver transform-details opts)- (sync-target! target database run-id)))+ (transforms.util/run-cancelable-transform!+ run-id driver transform-details+ (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))+ (transforms.util/sync-target! target database run-id)))(catch Throwable t(log/error t "Error executing transform")(when start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.cljindex 41d0761da66..68ab07b40af 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj@@ -64,14 +64,17 @@(map transforms-by-id complete)))))(defn- run-transform! [run-id run-method {transform-id :id :as transform}]- (when (transform-run/running-run-for-run-id transform-id)+ (when+ (transform-run/running-run-for-transform-id transform-id)(log/warn "Transform" (pr-str transform-id) "already running, waiting")(loop [](Thread/sleep 2000)- (when (transform-run/running-run-for-run-id transform-id)+ (when (transform-run/running-run-for-transform-id transform-id)(recur))))+(log/info "Executing job transform" (pr-str transform-id))(transforms.execute/run-mbql-transform! transform {:run-method run-method})+(transforms.job-run/add-run-activity! run-id))(defn run-transforms!diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.cljindex d9eb4687949..cc34f34e236 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj@@ -147,19 +147,13 @@:message "Canceled by user but could not guarantee run stopped."})(cancel/delete-old-canceling-runs!)))-(defn running-run-for-run-id- "Return a single active transform run or nil."- [id]- (t2/select-one :model/TransformRun- :id id- :is_active true))-(defn running-run-for-transform-id"Return a single active transform run or nil."[transform-id](t2/select-one :model/TransformRun:transform_id transform-id:is_active true))+(defn- timestamp-constraint[field-name date-string](let [{:keys [start end]}diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex d4cf6877ee3..b22684bd615 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -1,8 +1,14 @@(ns metabase-enterprise.transforms.util(:require+ [clojure.core.async :as a]+ [clojure.string :as str][java-time.api :as t]+ [metabase-enterprise.transforms.canceling :as canceling]+ [metabase-enterprise.transforms.models.transform-run :as transform-run]+ [metabase-enterprise.transforms.settings :as transforms.settings][metabase.driver :as driver][metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.pipeline :as qp.pipeline][metabase.sync.core :as sync][metabase.util :as u][metabase.util.date-2 :as u.date]@@ -21,6 +27,55 @@(keyword schema name)(keyword name)))+(defn try-start-unless-already-running+ "Start a transform run, throwing an informative error if already running."+ [id run-method]+ (try+ (transform-run/start-run! id {:run_method run-method})+ (catch java.sql.SQLException e+ (if (= (.getSQLState e) "23505")+ (throw (ex-info "Transform is already running"+ {:error :already-running+ :transform-id id}+ e))+ (throw e)))))++(defn run-cancelable-transform!+ "Execute a transform with cancellation support and proper error handling."+ [run-id driver {:keys [db-id conn-spec output-schema]} run-transform!]+ ;; local run is responsible for status, using canceling lifecycle+ (try+ (when-not (driver/schema-exists? driver db-id output-schema)+ (driver/create-schema-if-needed! driver conn-spec output-schema))+ (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))+ (let [cancel-chan (a/promise-chan)+ ret (binding [qp.pipeline/*canceled-chan* cancel-chan]+ (canceling/chan-start-run! run-id cancel-chan)+ (run-transform! cancel-chan))]+ (transform-run/succeed-started-run! run-id)+ ret)+ (catch Throwable t+ (let [{:keys [transform-run-message]} (ex-data t)+ message (str/join "\n" (remove str/blank? [transform-run-message (.getMessage t)]))]+ (transform-run/fail-started-run! run-id {:message message}))+ (throw t))+ (finally+ (canceling/chan-end-run! run-id))))++(declare activate-table-and-mark-computed!)++(defn sync-target!+ "Sync target of a transform"+ ([transform-id run-id]+ (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)+ db (get-in source [:query :database])+ database (t2/select-one :model/Database db)]+ (sync-target! target database run-id)))+ ([target database _run-id]+ ;; sync the new table (note that even a failed sync status means that the execution succeeded)+ (log/info "Syncing target" (pr-str target) "for transform")+ (activate-table-and-mark-computed! database target)))+(defn target-table-exists?"Test if the target table of a transform already exists."[{:keys [source target] :as _transform}]