The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Dispatch transforms within a job run in parallel (#73098)
Ngoc Khuatc607c65bb769
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex c6f94ba6c80..1e4e2e85057 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -11,6 +11,7 @@[metabase.task.core :as task][metabase.tracing.core :as tracing][metabase.transforms-base.ordering :as transforms-base.ordering]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.execute :as transforms.execute][metabase.transforms.instrumentation :as transforms.instrumentation][metabase.transforms.models.job-run :as transforms.job-run]@@ -21,7 +22,10 @@[metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.log :as log]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent ArrayBlockingQueue BlockingQueue ExecutorService Executors ThreadFactory)+ (org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))(set! *warn-on-reflection* true)@@ -114,32 +118,158 @@(recur))))(transforms.job-run/add-run-activity! run-id))))+(defn- named-thread-factory ^ThreadFactory [pattern]+ (.build (doto (BasicThreadFactory$Builder.)+ (.namingPattern pattern)+ (.daemon true))))++(defn- submit-transform!+ [^ExecutorService executor+ ^BlockingQueue completions+ run-id run-method user-id transform]+ (let [task (bound-fn []+ (let [tid (:id transform)]+ (try+ (run-transform! run-id run-method user-id transform)+ (.put completions {::status :succeeded ::transform transform})+ (catch Throwable t+ (log/errorf t "Transform %s in run %s failed" (pr-str tid) (pr-str run-id))+ ;; Record the failure with as much debugging context as we can; the coordinator+ ;; preserves these fields onto :failures. We catch Throwable (not just+ ;; Exception) so a non-Exception Error — e.g. OutOfMemoryError — is still+ ;; reported as a transform failure and lets the coordinator unwind cleanly,+ ;; rather than vanishing into the worker's FutureTask. `::fatal` flags that case+ ;; so consumers can distinguish it. (We don't rethrow: under `.submit` the+ ;; FutureTask would just swallow it, so a rethrow would be a silent no-op.)+ (.put completions {::status :failed+ ::transform transform+ ::message (or (ex-message t) (str t))+ ::ex-data (ex-data t)+ ::throwable t+ ::fatal (not (instance? Exception t))})))))]+ (.submit executor ^Runnable task)))++(defn- transform-target-key+ "A coarse identity for a transform's output table, used to keep two transforms that write the+ same target table from being dispatched concurrently. Returns nil when the transform has no+ resolved target (in which case it isn't subject to the co-writer guard)."+ [transform]+ (when-let [{:keys [database schema name]} (:target transform)]+ [database schema name]))+(defn run-transforms!"Run a series of transforms and their dependencies.+ SQL transforms are dispatched concurrently up to `transform-run-job-sql-concurrency`, respecting+ the DAG: a transform is only started once all of its dependencies have succeeded.++ Python transforms run in a separate single-slot lane regardless of the SQL pool size, because+ the python-runner service has only one worker; oversubscribing it would just queue requests+ against their own per-call timeouts.++ If a dependency failed, dependents are recorded as failures (transitively) without being+ executed.+Updates the transform-job-run specified by run-id after every completion.Returns a map with :status and a collection of :failures if failed."[run-id transform-ids-to-run {:keys [run-method start-promise user-id]}](let [{plan :order deps :deps} (get-plan transform-ids-to-run)- successful (volatile! #{})- failures (volatile! [])]- (when start-promise- (deliver start-promise :started))-- (doseq [transform plan]- (if (every? @successful (get deps (:id transform)))- (try- (run-transform! run-id run-method user-id transform)- (vswap! successful conj (:id transform))- (catch Exception e- (vswap! failures conj {::transform transform- ::message (.getMessage e)})))- (vswap! failures conj {::transform transform- ::message (i18n/trs "Failed to run because one or more of the transforms it depends on failed.")})))-- (if (seq @failures)- {::status :failed- ::failures @failures}+ n (max 1 (transforms.settings/transform-run-job-sql-concurrency))+ sql-executor (Executors/newFixedThreadPool n (named-thread-factory "transforms-sql-worker-%d"))+ py-executor (Executors/newSingleThreadExecutor (named-thread-factory "transforms-python-worker-%d"))+ lanes {:sql {:executor sql-executor :capacity n}+ :py {:executor py-executor :capacity 1}}+ lane-for (fn [t] (if (transforms-base.u/python-transform? t) :py :sql))+ completions (ArrayBlockingQueue. (max 2 (inc n)))+ state (volatile! {:succeeded #{}+ :failed #{}+ :failures []+ :in-flight {:sql #{} :py #{}}+ ;; target tables currently being written by an in-flight transform+ :in-flight-targets #{}})+ busy? (fn [{:keys [in-flight]}] (some seq (vals in-flight)))+ ;; Plan is topo-sorted, so a single pass propagates the full cascade of dep-failures.+ dispatch! (fn [st]+ (reduce+ (fn [{:keys [succeeded failed in-flight-targets] :as st} t]+ (let [id (:id t)+ dep-ids (get deps id)+ lane-key (lane-for t)+ {:keys [^ExecutorService executor capacity]} (get lanes lane-key)+ in-flight-now (get-in st [:in-flight lane-key])+ tkey (transform-target-key t)]+ (cond+ (or (succeeded id) (failed id) (in-flight-now id))+ st++ (some failed dep-ids)+ (-> st+ (update :failed conj id)+ (update :failures conj+ {::transform t+ ::message (i18n/trs "Failed to run because one or more of the transforms it depends on failed.")}))++ ;; Defer if another in-flight transform is already writing this target+ ;; table. `transform-ordering` builds DAG edges from table *reads*, not+ ;; co-*writes*, so two transforms sharing a target are independent in+ ;; the DAG and would otherwise dispatch concurrently — producing+ ;; nondeterministic DDL. This shouldn't happen in a well-formed config;+ ;; the guard just stops a misconfiguration from racing. Retried on a+ ;; later pass once the in-flight writer completes (something is always+ ;; in-flight while a target is held, so the loop won't exit early).+ (and tkey (contains? in-flight-targets tkey))+ st++ (and (every? succeeded dep-ids)+ (< (count in-flight-now) capacity))+ (try+ (submit-transform! executor completions run-id run-method user-id t)+ (cond-> (update-in st [:in-flight lane-key] conj id)+ tkey (update :in-flight-targets conj tkey))+ (catch java.util.concurrent.RejectedExecutionException e+ ;; The pool was shut down out from under us (abnormal-exit window).+ ;; Record the failure so the run reflects the partial dispatch+ ;; rather than silently dropping this transform.+ (-> st+ (update :failed conj id)+ (update :failures conj+ {::transform t+ ::message (or (ex-message e) (str e))}))))++ :else st)))+ st+ plan))]+ (when start-promise (deliver start-promise :started))+ (try+ (vreset! state (dispatch! @state))+ (while (busy? @state)+ (let [completion (.take completions)+ {::keys [transform status]} completion+ id (:id transform)+ lane-key (lane-for transform)+ tkey (transform-target-key transform)]+ (vswap! state+ (fn [st]+ (let [st' (cond-> (update-in st [:in-flight lane-key] disj id)+ tkey (update :in-flight-targets disj tkey))]+ (case status+ :succeeded (update st' :succeeded conj id)+ ;; Preserve the full failure context the worker attached (::message plus+ ;; ::ex-data / ::throwable / ::fatal) so downstream consumers — notifications+ ;; and logs — can surface it. dispatch!'s dep-failure path produces a subset+ ;; of the same shape, and consumers only read ::transform / ::message.+ :failed (-> st'+ (update :failed conj id)+ (update :failures conj (dissoc completion ::status)))))))+ (vreset! state (dispatch! @state))))+ (finally+ ;; shutdownNow (not shutdown): on the happy path the in-flight set is already drained, so+ ;; this is equivalent to shutdown; on an abnormal exit (e.g. an interrupted take) it also+ ;; interrupts any workers still running against a run that has already been failed.+ (.shutdownNow sql-executor)+ (.shutdownNow py-executor)))+ (if (seq (:failures @state))+ {::status :failed ::failures (:failures @state)}{::status :succeeded})))(defn- job-transform-ids [job-id]diff --git a/src/metabase/transforms/settings.clj b/src/metabase/transforms/settings.cljindex 218e9d5e1e8..3ab587732aa 100644--- a/src/metabase/transforms/settings.clj+++ b/src/metabase/transforms/settings.clj@@ -21,6 +21,21 @@:encryption :no:audit :getter)+;; Only governs the SQL lane. Python transforms always run one-at-a-time within a job because the+;; python-runner service is single-threaded; dispatching them in parallel would just queue them+;; against their own per-call timeouts.+(setting/defsetting transform-run-job-sql-concurrency+ (deferred-tru "Maximum number of SQL-backed transforms a single transform-job run may execute in parallel.")+ :type :integer+ :visibility :internal+ :default 3+ :feature :transforms-basic+ :doc "This setting is only configurable on instances with the transforms add-on; OSS+ deployments without the add-on always use the default."+ :export? false+ :encryption :no+ :audit :getter)+(setting/defsetting transforms-enabled(deferred-tru "Enable transforms for instances that have not explicitly purchased the transform add-on."):type :booleandiff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.cljindex f297572207e..f2bfb4ab5a2 100644--- a/test/metabase/transforms/jobs_test.clj+++ b/test/metabase/transforms/jobs_test.clj@@ -17,6 +17,7 @@[metabase.transforms.jobs :as jobs][metabase.transforms.models.job-run :as transforms.job-run][metabase.transforms.models.transform-run :as transform-run]+ [metabase.transforms.settings :as transforms.settings][metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :refer [with-transform-cleanup!]][metabase.transforms.util :as transforms.u]@@ -624,3 +625,102 @@(deref fut2 30000 {:error :timeout})]](is (every? #(= :succeeded (-> % :result ::jobs/status)) results)"Both threads should succeed"))))))))))))))++(deftest run-transforms-parallel-dispatch-test+ (mt/with-premium-features #{:transforms-basic}+ (testing "Independent transforms dispatch concurrently up to the configured limit"+ (let [plan [{:id 1} {:id 2} {:id 3} {:id 4}]+ deps {1 #{} 2 #{} 3 #{} 4 #{}}+ barrier (CyclicBarrier. 4)]+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})+ jobs/run-transform! (fn [_ _ _ _]+ (.await barrier 5 TimeUnit/SECONDS))]+ (is (= {::jobs/status :succeeded}+ (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual}))+ "all four independents must reach the barrier (i.e. be live simultaneously)")))))++ (testing "Dependents wait for their dependencies even when concurrency > 1"+ (let [plan [{:id 1} {:id 2}]+ deps {1 #{} 2 #{1}}+ order (atom [])]+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})+ jobs/run-transform! (fn [_ _ _ transform]+ (when (= 1 (:id transform))+ (Thread/sleep 100))+ (swap! order conj (:id transform)))]+ (jobs/run-transforms! 0 #{1 2} {:run-method :manual})+ (is (= [1 2] @order) "a must complete before b starts")))))++ (testing "Dependents of a failed transform are skipped transitively"+ (let [plan [{:id 1} {:id 2} {:id 3}]+ deps {1 #{} 2 #{1} 3 #{2}}]+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})+ jobs/run-transform! (fn [_ _ _ transform]+ (when (= 1 (:id transform))+ (throw (ex-info "boom" {}))))]+ (let [result (jobs/run-transforms! 0 #{1 2 3} {:run-method :manual})]+ (is (= :failed (::jobs/status result)))+ (is (= #{1 2 3} (set (map (comp :id ::jobs/transform) (::jobs/failures result))))+ "a failed, b & c skipped as dep-failed"))))))++ (testing "Concurrency is capped by the setting"+ (let [plan (mapv (fn [i] {:id i}) (range 1 9))+ deps (into {} (map (fn [i] [i #{}]) (range 1 9)))+ ;; 2-party barrier: workers must pair up before either can exit. This gives+ ;; deterministic "two alive" windows we can observe via the live counter — no sleeps.+ partner-up (CyclicBarrier. 2)+ live (atom 0)+ max-live (atom 0)]+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 2]+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})+ jobs/run-transform! (fn [_ _ _ _]+ (let [n (swap! live inc)]+ (swap! max-live max n))+ (.await partner-up 5 TimeUnit/SECONDS)+ (swap! live dec))]+ (jobs/run-transforms! 0 (set (range 1 9)) {:run-method :manual})+ (is (= 2 @max-live) "should never exceed the concurrency setting")))))++ (testing "Python transforms are serialized in their own lane (n=1)"+ (let [py (fn [id] {:id id :source {:type :python}})+ plan [(py 1) (py 2) (py 3) (py 4)]+ deps {1 #{} 2 #{} 3 #{} 4 #{}}+ live (atom 0)+ max-live (atom 0)]+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 4]+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})+ jobs/run-transform! (fn [_ _ _ _]+ (let [n (swap! live inc)]+ (swap! max-live max n))+ ;; A short sleep gives any (hypothetical) parallel+ ;; python worker a window to show up in the counter.+ (Thread/sleep 50)+ (swap! live dec))]+ (jobs/run-transforms! 0 #{1 2 3 4} {:run-method :manual})+ (is (= 1 @max-live)+ "python transforms must run one at a time even with concurrency=4")))))++ (testing "SQL and Python lanes run in parallel without contending for slots"+ (let [py (fn [id] {:id id :source {:type :python}})+ sql (fn [id] {:id id})+ plan [(sql 1) (sql 2) (sql 3) (py 4) (py 5)]+ deps {1 #{} 2 #{} 3 #{} 4 #{} 5 #{}}+ sql-barrier (CyclicBarrier. 3)+ py-live (atom 0)+ max-py (atom 0)]+ (mt/with-temporary-setting-values [transforms.settings/transform-run-job-sql-concurrency 3]+ (with-redefs [jobs/get-plan (fn [_] {:order plan :deps deps})+ jobs/run-transform! (fn [_ _ _ transform]+ (if (= :python (-> transform :source :type))+ (do (let [n (swap! py-live inc)]+ (swap! max-py max n))+ (Thread/sleep 50)+ (swap! py-live dec))+ (.await sql-barrier 5 TimeUnit/SECONDS)))]+ (is (= {::jobs/status :succeeded}+ (jobs/run-transforms! 0 #{1 2 3 4 5} {:run-method :manual}))+ "all three SQL transforms reach the barrier (run in parallel)")+ (is (= 1 @max-py) "python lane stays single-slot")))))))