Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Dispatch transforms within a job run in parallel (#73098)

Ngoc Khuatc607c65bb769
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index c6f94ba6c80..1e4e2e85057 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -11,6 +11,7 @@
[metabase.task.core :as task]
[metabase.tracing.core :as tracing]
[metabase.transforms-base.ordering :as transforms-base.ordering]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.models.job-run :as transforms.job-run]
@@ -21,7 +22,10 @@
[metabase.util :as u]
[metabase.util.i18n :as i18n]
[metabase.util.log :as log]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.util.concurrent ArrayBlockingQueue BlockingQueue ExecutorService Executors ThreadFactory)
+ (org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))
(set! *warn-on-reflection* true)
@@ -114,32 +118,158 @@
(recur))))
(transforms.job-run/add-run-activity! run-id))))
+(defn- named-thread-factory ^ThreadFactory [pattern]
+ (.build (doto (BasicThreadFactory$Builder.)
+ (.namingPattern pattern)
+ (.daemon true))))
+
+(defn- submit-transform!
+ [^ExecutorService executor
+ ^BlockingQueue completions
+ run-id run-method user-id transform]
+ (let [task (bound-fn []
+ (let [tid (:id transform)]
+ (try
+ (run-transform! run-id run-method user-id transform)
+ (.put completions {::status :succeeded ::transform transform})
+ (catch Throwable t
+ (log/errorf t "Transform %s in run %s failed" (pr-str tid) (pr-str run-id))
+ ;; Record the failure with as much debugging context as we can; the coordinator
+ ;; preserves these fields onto :failures. We catch Throwable (not just
+ ;; Exception) so a non-Exception Error — e.g. OutOfMemoryError — is still
+ ;; reported as a transform failure and lets the coordinator unwind cleanly,
+ ;; rather than vanishing into the worker's FutureTask. `::fatal` flags that case
+ ;; so consumers can distinguish it. (We don't rethrow: under `.submit` the
+ ;; FutureTask would just swallow it, so a rethrow would be a silent no-op.)
+ (.put completions {::status :failed
+ ::transform transform
+ ::message (or (ex-message t) (str t))
+ ::ex-data (ex-data t)
+ ::throwable t
+ ::fatal (not (instance? Exception t))})))))]
+ (.submit executor ^Runnable task)))
+
+(defn- transform-target-key
+ "A coarse identity for a transform's output table, used to keep two transforms that write the
+ same target table from being dispatched concurrently. Returns nil when the transform has no
+ resolved target (in which case it isn't subject to the co-writer guard)."
+ [transform]
+ (when-let [{:keys [database schema name]} (:target transform)]
+ [database schema name]))
+
(defn run-transforms!
"Run a series of transforms and their dependencies.
+ SQL transforms are dispatched concurrently up to `transform-run-job-sql-concurrency`, respecting
+ the DAG: a transform is only started once all of its dependencies have succeeded.
+
+ Python transforms run in a separate single-slot lane regardless of the SQL pool size, because
+ the python-runner service has only one worker; oversubscribing it would just queue requests
+ against their own per-call timeouts.
+
+ If a dependency failed, dependents are recorded as failures (transitively) without being
+ executed.
+
Updates the transform-job-run specified by run-id after every completion.
Returns a map with :status and a collection of :failures if failed."
[run-id transform-ids-to-run {:keys [run-method start-promise user-id]}]
(let [{plan :order deps :deps} (get-plan transform-ids-to-run)
- successful (volatile! #{})
- failures (volatile! [])]
- (when start-promise
- (deliver start-promise :started))
-
- (doseq [transform plan]
- (if (every? @successful (get deps (:id transform)))
- (try
- (run-transform! run-id run-method user-id transform)
- (vswap! successful conj (:id transform))
- (catch Exception e
- (vswap! failures conj {::transform transform
- ::message (.getMessage e)})))
- (vswap! failures conj {::transform transform
- ::message (i18n/trs "Failed to run because one or more of the transforms it depends on failed.")})))
-
- (if (seq @failures)
- {::status :failed
- ::failures @failures}
+ n (max 1 (transforms.settings/transform-run-job-sql-concurrency))
+ sql-executor (Executors/newFixedThreadPool n (named-thread-factory "transforms-sql-worker-%d"))
+ py-executor (Executors/newSingleThreadExecutor (named-thread-factory "transforms-python-worker-%d"))
+ lanes {:sql {:executor sql-executor :capacity n}
+ :py {:executor py-executor :capacity 1}}
+ lane-for (fn [t] (if (transforms-base.u/python-transform? t) :py :sql))
+ completions (ArrayBlockingQueue. (max 2 (inc n)))
+ state (volatile! {:succeeded #{}
+ :failed #{}
+ :failures []
+ :in-flight {:sql #{} :py #{}}
+ ;; target tables currently being written by an in-flight transform
+ :in-flight-targets #{}})
+ busy? (fn [{:keys [in-flight]}] (some seq (vals in-flight)))
+ ;; Plan is topo-sorted, so a single pass propagates the full cascade of dep-failures.
+ dispatch! (fn [st]
+ (reduce
+ (fn [{:keys [succeeded failed in-flight-targets] :as st} t]
+ (let [id (:id t)
+ dep-ids (get deps id)
+ lane-key (lane-for t)
+ {:keys [^ExecutorService executor capacity]} (get lanes lane-key)
+ in-flight-now (get-in st [:in-flight lane-key])
+ tkey (transform-target-key t)]
+ (cond
+ (or (succeeded id) (failed id) (in-flight-now id))
+ st
+
+ (some failed dep-ids)
+ (-> st
+ (update :failed conj id)
+ (update :failures conj
+ {::transform t
+ ::message (i18n/trs "Failed to run because one or more of the transforms it depends on failed.")}))
+
+ ;; Defer if another in-flight transform is already writing this target
+ ;; table. `transform-ordering` builds DAG edges from table *reads*, not
+ ;; co-*writes*, so two transforms sharing a target are independent in
+ ;; the DAG and would otherwise dispatch concurrently — producing
+ ;; nondeterministic DDL. This shouldn't happen in a well-formed config;
+ ;; the guard just stops a misconfiguration from racing. Retried on a
+ ;; later pass once the in-flight writer completes (something is always
+ ;; in-flight while a target is held, so the loop won't exit early).
+ (and tkey (contains? in-flight-targets tkey))
+ st
+
+ (and (every? succeeded dep-ids)
+ (< (count in-flight-now) capacity))
+ (try
+ (submit-transform! executor completions run-id run-method user-id t)
+ (cond-> (update-in st [:in-flight lane-key] conj id)
+ tkey (update :in-flight-targets conj tkey))
+ (catch java.util.concurrent.RejectedExecutionException e
+ ;; The pool was shut down out from under us (abnormal-exit window).
+ ;; Record the failure so the run reflects the partial dispatch
+ ;; rather than silently dropping this transform.
+ (-> st
+ (update :failed conj id)
+ (update :failures conj
+ {::transform t
+ ::message (or (ex-message e) (str e))}))))
+
+ :else st)))
+ st
+ plan))]
+ (when start-promise (deliver start-promise :started))
+ (try
+ (vreset! state (dispatch! @state))
+ (while (busy? @state)
+ (let [completion (.take completions)
+ {::keys [transform status]} completion
+ id (:id transform)
+ lane-key (lane-for transform)
+ tkey (transform-target-key transform)]
+ (vswap! state
+ (fn [st]
+ (let [st' (cond-> (update-in st [:in-flight lane-key] disj id)
+ tkey (update :in-flight-targets disj tkey))]
+ (case status
+ :succeeded (update st' :succeeded conj id)
+ ;; Preserve the full failure context the worker attached (::message plus
+ ;; ::ex-data / ::throwable / ::fatal) so downstream consumers — notifications
+ ;; and logs — can surface it. dispatch!'s dep-failure path produces a subset
+ ;; of the same shape, and consumers only read ::transform / ::message.
+ :failed (-> st'
+ (update :failed conj id)
+ (update :failures conj (dissoc completion ::status)))))))
+ (vreset! state (dispatch! @state))))
+ (finally
+ ;; shutdownNow (not shutdown): on the happy path the in-flight set is already drained, so
+ ;; this is equivalent to shutdown; on an abnormal exit (e.g. an interrupted take) it also
+ ;; interrupts any workers still running against a run that has already been failed.
+ (.shutdownNow sql-executor)
+ (.shutdownNow py-executor)))
+ (if (seq (:failures @state))
+ {::status :failed ::failures (:failures @state)}
{::status :succeeded})))
(defn- job-transform-ids [job-id]
diff --git a/src/metabase/transforms/settings.clj b/src/metabase/transforms/settings.clj
index 218e9d5e1e8..3ab587732aa 100644
--- a/src/metabase/transforms/settings.clj
+++ b/src/metabase/transforms/settings.clj
@@ -21,6 +21,21 @@
:encryption :no
:audit :getter)
+;; Only governs the SQL lane. Python transforms always run one-at-a-time within a job because the
+;; python-runner service is single-threaded; dispatching them in parallel would just queue them
+;; against their own per-call timeouts.
+(setting/defsetting transform-run-job-sql-concurrency
+ (deferred-tru "Maximum number of SQL-backed transforms a single transform-job run may execute in parallel.")
+ :type :integer
+ :visibility :internal
+ :default 3
+ :feature :transforms-basic
+ :doc "This setting is only configurable on instances with the transforms add-on; OSS
+ deployments without the add-on always use the default."
+ :export? false
+ :encryption :no
+ :audit :getter)
+
(setting/defsetting transforms-enabled
(deferred-tru "Enable transforms for instances that have not explicitly purchased the transform add-on.")
:type :boolean
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index f297572207e..f2bfb4ab5a2 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -17,6 +17,7 @@
[metabase.transforms.jobs :as jobs]
[metabase.transforms.models.job-run :as transforms.job-run]
[metabase.transforms.models.transform-run :as transform-run]
+ [metabase.transforms.settings :as transforms.settings]
[metabase.transforms.test-dataset :as transforms-dataset]
[metabase.transforms.test-util :refer [with-transform-cleanup!]]
[metabase.transforms.util :as transforms.u]
@@ -624,3 +625,102 @@
(deref fut2 30000 {:error :timeout})]]
(is (every? #(= :succeeded (-> % :result ::jobs/status)) results)
"Both threads should succeed"))))))))))))))
+
+(deftest run-transforms-parallel-dispatch-test
+ (mt/with-premium-features #{:transforms-basic}
+ (testing "Independent transforms dispatch concurrently up to the configured limit"
+ (let [plan [{:id 1} {:id 2} {:id 3} {:id 4}]
+ deps {1 #{} 2 #{} 3 #{} 4 #{}}
+ barrier (CyclicBarrier. 4)]
+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
+ jobs/run-transform! (fn [_ _ _ _]
+ (.await barrier 5 TimeUnit/SECONDS))]
+ (is (= {::jobs/status :succeeded}
+ (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual}))
+ "all four independents must reach the barrier (i.e. be live simultaneously)")))))
+
+ (testing "Dependents wait for their dependencies even when concurrency > 1"
+ (let [plan [{:id 1} {:id 2}]
+ deps {1 #{} 2 #{1}}
+ order (atom [])]
+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
+ jobs/run-transform! (fn [_ _ _ transform]
+ (when (= 1 (:id transform))
+ (Thread/sleep 100))
+ (swap! order conj (:id transform)))]
+ (jobs/run-transforms! 0 #{1 2} {:run-method :manual})
+ (is (= [1 2] @order) "a must complete before b starts")))))
+
+ (testing "Dependents of a failed transform are skipped transitively"
+ (let [plan [{:id 1} {:id 2} {:id 3}]
+ deps {1 #{} 2 #{1} 3 #{2}}]
+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
+ jobs/run-transform! (fn [_ _ _ transform]
+ (when (= 1 (:id transform))
+ (throw (ex-info "boom" {}))))]
+ (let [result (jobs/run-transforms! 0 #{1 2 3} {:run-method :manual})]
+ (is (= :failed (::jobs/status result)))
+ (is (= #{1 2 3} (set (map (comp :id ::jobs/transform) (::jobs/failures result))))
+ "a failed, b & c skipped as dep-failed"))))))
+
+ (testing "Concurrency is capped by the setting"
+ (let [plan (mapv (fn [i] {:id i}) (range 1 9))
+ deps (into {} (map (fn [i] [i #{}]) (range 1 9)))
+ ;; 2-party barrier: workers must pair up before either can exit. This gives
+ ;; deterministic "two alive" windows we can observe via the live counter — no sleeps.
+ partner-up (CyclicBarrier. 2)
+ live (atom 0)
+ max-live (atom 0)]
+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 2]
+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
+ jobs/run-transform! (fn [_ _ _ _]
+ (let [n (swap! live inc)]
+ (swap! max-live max n))
+ (.await partner-up 5 TimeUnit/SECONDS)
+ (swap! live dec))]
+ (jobs/run-transforms! 0 (set (range 1 9)) {:run-method :manual})
+ (is (= 2 @max-live) "should never exceed the concurrency setting")))))
+
+ (testing "Python transforms are serialized in their own lane (n=1)"
+ (let [py (fn [id] {:id id :source {:type :python}})
+ plan [(py 1) (py 2) (py 3) (py 4)]
+ deps {1 #{} 2 #{} 3 #{} 4 #{}}
+ live (atom 0)
+ max-live (atom 0)]
+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]
+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
+ jobs/run-transform! (fn [_ _ _ _]
+ (let [n (swap! live inc)]
+ (swap! max-live max n))
+ ;; A short sleep gives any (hypothetical) parallel
+ ;; python worker a window to show up in the counter.
+ (Thread/sleep 50)
+ (swap! live dec))]
+ (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual})
+ (is (= 1 @max-live)
+ "python transforms must run one at a time even with concurrency=4")))))
+
+ (testing "SQL and Python lanes run in parallel without contending for slots"
+ (let [py (fn [id] {:id id :source {:type :python}})
+ sql (fn [id] {:id id})
+ plan [(sql 1) (sql 2) (sql 3) (py 4) (py 5)]
+ deps {1 #{} 2 #{} 3 #{} 4 #{} 5 #{}}
+ sql-barrier (CyclicBarrier. 3)
+ py-live (atom 0)
+ max-py (atom 0)]
+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 3]
+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})
+ jobs/run-transform! (fn [_ _ _ transform]
+ (if (= :python (-> transform :source :type))
+ (do (let [n (swap! py-live inc)]
+ (swap! max-py max n))
+ (Thread/sleep 50)
+ (swap! py-live dec))
+ (.await sql-barrier 5 TimeUnit/SECONDS)))]
+ (is (= {::jobs/status :succeeded}
+ (jobs/run-transforms! 0 #{1 2 3 4 5} {:run-method :manual}))
+ "all three SQL transforms reach the barrier (run in parallel)")
+ (is (= 1 @max-py) "python lane stays single-slot")))))))