The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Guard transform runner from being poisoned by unrunable transforms (#72149)
Ngoc Khuat655825076b3b
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex 5bb0942f80d..e088c75cd39 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -44,7 +44,7 @@(testing "transform-ordering correctly resolves the dependency"(is (= {t-a #{}t-b #{t-a}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t-a t-b]]))))))))+ (:dependencies (ordering/transform-ordering #{t-a t-b} (t2/select :model/Transform :id [:in [t-a t-b]])))))))))(deftest python-transform-mixed-source-tables-test(testing "Python transform with mixed int and name-based refs"@@ -68,4 +68,4 @@(testing "transform-ordering resolves both dependencies"(is (= {t-a #{}t-b #{t-a}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t-a t-b]]))))))))+ (:dependencies (ordering/transform-ordering #{t-a t-b} (t2/select :model/Transform :id [:in [t-a t-b]])))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex 6f095a12bda..9ddc219af9d 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -55,7 +55,7 @@:model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)] "output_2")](is (= {t1 #{}t2 #{t1}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]])))))))))+ (:dependencies (ordering/transform-ordering #{t1 t2} (t2/select :model/Transform :id [:in [t1 t2]]))))))))))(deftest python-transform-multiple-dependencies-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)@@ -73,7 +73,7 @@(is (= {t1 #{}t2 #{}t3 #{t1 t2}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))+ (:dependencies (ordering/transform-ordering #{t1 t2 t3} (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))))(deftest mixed-transform-ordering-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python :transforms/table)@@ -98,4 +98,4 @@(is (= {t1 #{}t2 #{t1}t3 #{t2}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))+ (:dependencies (ordering/transform-ordering #{t1 t2 t3} (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))))diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex e0fd59ce938..14726495f94 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -24,16 +24,6 @@(set! *warn-on-reflection* true)-(defn- get-deps [ordering transform-ids]- (loop [found #{}- [current-transform & more-transforms] transform-ids]- (if current-transform- (recur (conj found current-transform)- (if (found current-transform)- more-transforms- (into more-transforms (get ordering current-transform))))- found)))-(defn- next-transform [ordering transforms-by-id complete](-> (transforms-base.ordering/available-transforms ordering #{} complete)first@@ -53,25 +43,36 @@(defn- get-plan [transform-ids](tracing/with-span :tasks "task.transform.plan" {:transform/count (count transform-ids)}- (let [all-transforms (t2/select :model/Transform)- global-ordering (transforms-base.ordering/transform-ordering all-transforms)- relevant-ids (get-deps global-ordering transform-ids)- transforms-by-id (into {}- (keep (fn [{:keys [id] :as transform}]- (when (relevant-ids id)- [id transform])))- all-transforms)- ordering (sorted-ordering (select-keys global-ordering relevant-ids) transforms-by-id)]- (when-let [cycle (transforms-base.ordering/find-cycle ordering)]- (let [id->name (into {} (map (juxt :id :name)) all-transforms)]- (throw (ex-info (str "Cyclic transform definitions detected: "- (str/join " → " (map id->name cycle)))- {:cycle cycle}))))- (loop [complete (ordered-set/ordered-set)]- (if-let [current-transform (next-transform ordering transforms-by-id complete)]- (recur (conj complete (:id current-transform)))- {:order (map transforms-by-id complete)- :deps global-ordering})))))+ (let [all-transforms (t2/select :model/Transform)+ ;; Walk only the dependency closure of the transforms we're asked to run.+ ;; `table-dependencies` (and the QP preprocessing it triggers) is therefore called+ ;; only on transforms in that closure — never on unrelated transforms elsewhere in+ ;; the system. This is what prevents a single broken transform (e.g. one on a+ ;; routing-enabled database) from poisoning the scheduler when no job has asked for it.+ {:keys [dependencies not-found failed]}+ (transforms-base.ordering/transform-ordering transform-ids all-transforms)]+ (when (seq not-found)+ (log/warnf "transform-ordering: %d scheduled id(s) not found in transforms (likely deleted between scheduling and lookup): %s"+ (count not-found) (pr-str (sort not-found))))+ (when (seq failed)+ (log/warnf "transform-ordering: %d transform(s) failed dep extraction; treated as leaves: %s"+ (count failed) (pr-str (sort failed))))+ (let [transforms-by-id (into {}+ (keep (fn [{:keys [id] :as transform}]+ (when (contains? dependencies id)+ [id transform])))+ all-transforms)+ sorted-ord (sorted-ordering dependencies transforms-by-id)]+ (when-let [cycle (transforms-base.ordering/find-cycle sorted-ord)]+ (let [id->name (into {} (map (juxt :id :name)) all-transforms)]+ (throw (ex-info (str "Cyclic transform definitions detected: "+ (str/join " → " (map id->name cycle)))+ {:cycle cycle}))))+ (loop [complete (ordered-set/ordered-set)]+ (if-let [current-transform (next-transform sorted-ord transforms-by-id complete)]+ (recur (conj complete (:id current-transform)))+ {:order (map transforms-by-id complete)+ :deps dependencies}))))))(defn- block-until-not-already-running [transform-id](when-let [active-run (transform-run/running-run-for-transform-id transform-id)]diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.cljindex efa1836664b..a442b4f6ff9 100644--- a/src/metabase/transforms_base/ordering.clj+++ b/src/metabase/transforms_base/ordering.clj@@ -9,10 +9,10 @@[metabase.driver :as driver][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.preprocess :as qp.preprocess][metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.util :as transforms-base.u]+ [metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.malli :as mu][toucan2.core :as t2])@@ -62,10 +62,15 @@;;; ------------------------------------------------- Ordering Logic --------------------------------------------------(defn- dependency-map [transforms]- (into {}- (map (juxt :id transforms-base.i/table-dependencies))- transforms))+(defn safe-table-dependencies+ "Like `table-dependencies`, but returns `#{}` if the computation throws. Used by cycle+ detection where a single broken transform must not block the whole check. Callers that need+ to know *which* transforms failed should walk the graph themselves and capture the failure+ ids — see `transform-ordering`."+ [transform]+ (try+ (transforms-base.i/table-dependencies transform)+ (catch Throwable _ #{})))(mu/defn- output-table-map[transforms]@@ -102,37 +107,65 @@(get (:by-triple target-refs) [database_id schema table])))))(defn transform-ordering- "Computes an 'ordering' of a given list of transforms.-- The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just- the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'- dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."- [transforms]- (let [;; Group all transforms by their database, skipping transforms with no target db- transforms-by-db (->> transforms- (keep (fn [transform]- (when-let [db-id (transforms-base.i/target-db-id transform)]- {db-id [transform]})))- (apply merge-with into))- transform-ids (into #{} (map :id) transforms)- target-refs (target-ref-map transforms)- {:keys [output-tables- dependencies]} (->> transforms-by-db- (map (mu/fn [[_db-id db-transforms] :- [:tuple- [:maybe ::lib.schema.id/database]- [:maybe [:sequential :any]]]]- {:output-tables (output-table-map db-transforms)- :dependencies (dependency-map db-transforms)}))- (apply merge-with merge))]- ;; Transforms without a target database are invalid and shouldn't form part of the dependency graph.- ;; Give them empty dependency sets so they don't interfere with ordering.- (into (zipmap (map :id transforms) (repeat #{}))- (update-vals dependencies- (fn [deps]- (into #{}- (keep (fn [dep]- (resolve-dependency dep output-tables transform-ids target-refs)))- deps))))))+ "Compute the execution ordering for the dependency closure of `start-ids`.++ Walks the dependency graph starting from `start-ids`, calling `table-dependencies` only on+ transforms it actually visits. `all-transforms` provides the resolution context: when a+ transform in the closure references a table produced by another transform, that producer is+ discovered by looking it up in lookup tables built from `all-transforms`. This is what makes+ cross-DB dependencies (e.g. Python transforms pulling from a database different from their+ target) resolve correctly.++ Per-transform `table-dependencies` failures are caught and treated as no dependencies, with+ the failing id captured in `:failed`. The ordering is a best-effort scheduling hint —+ execution-time checks (e.g. `throw-if-db-routing-enabled!`) are the source of truth for+ whether a transform can actually run. This means a single broken transform anywhere in the+ system can never poison the scheduler: it will simply be treated as a leaf in the closure+ (or skipped entirely if no caller depends on it), and any actual problem with running it+ will surface at execution time and be attributed to the transform that tried to run it.++ Returns a map:++ {:dependencies {transform-id -> #{transform-ids it depends on}}+ :not-found #{ids in start-ids that don't refer to any transform in all-transforms}+ :failed #{ids whose table-dependencies threw}}++ `:dependencies` is restricted to the transitive closure reachable from `start-ids`.+ Diagnostics in `:not-found` and `:failed` let the caller decide how to surface them+ (logging, metrics, error responses)."+ [start-ids all-transforms]+ (let [id->xf (u/index-by :id all-transforms)+ output-tables (output-table-map all-transforms)+ target-refs (target-ref-map all-transforms)+ all-ids (into #{} (map :id) all-transforms)]+ (loop [visited {}+ not-found #{}+ failed #{}+ queue (vec start-ids)]+ (if-let [id (first queue)]+ (cond+ (contains? visited id)+ (recur visited not-found failed (rest queue))++ (not (id->xf id))+ (recur visited (conj not-found id) failed (rest queue))++ :else+ (let [transform (id->xf id)+ [raw-deps fail?] (try+ [(transforms-base.i/table-dependencies transform) false]+ (catch Throwable _ [#{} true]))+ resolved-ids (into #{}+ (keep (fn [dep]+ (resolve-dependency dep output-tables all-ids target-refs)))+ raw-deps)]+ (recur (assoc visited id resolved-ids)+ not-found+ (cond-> failed fail? (conj id))+ (into (rest queue) resolved-ids))))+ {:dependencies visited+ :not-found not-found+ :failed failed}))))(defn find-cycle"Finds a path containing a cycle in the directed graph `node->children`.@@ -181,7 +214,7 @@output-tables (output-table-map db-transforms)transform-ids (into #{} (map :id) db-transforms)target-refs (target-ref-map transforms)- node->children #(->> % transforms-by-id transforms-base.i/table-dependencies+ node->children #(->> % transforms-by-id safe-table-dependencies(keep (fn [dep] (resolve-dependency dep output-tables transform-ids target-refs))))id->name (comp :name transforms-by-id)cycle (find-cycle node->children [transform-id])]diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex cd4e83bf1b3..c74f91f5328 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -187,8 +187,8 @@[:id ms/PositiveInt]]](api/read-check :model/Transform id)(let [id->transform (t2/select-pk->fn identity :model/Transform)- global-ordering (transforms.core/transform-ordering (vals id->transform))- dep-ids (get global-ordering id)+ {graph :dependencies} (transforms.core/transform-ordering #{id} (vals id->transform))+ dep-ids (get graph id)dependencies (map id->transform dep-ids)](->> (t2/hydrate dependencies :creator :owner)transforms.u/add-source-readable)))diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.cljindex 4f28da86354..2aad772d778 100644--- a/test/metabase/transforms/jobs_test.clj+++ b/test/metabase/transforms/jobs_test.clj@@ -24,31 +24,6 @@(set! *warn-on-reflection* true)-(deftest basic-deps-test- (let [ordering {1 #{2 3}- 2 #{3 4}- 3 #{}- 4 #{5}- 5 #{}- 6 #{7 8}- 7 #{}- 8 #{}}]- (is (= #{1 2 3 4 5}- (#'jobs/get-deps ordering [1])))- (is (= #{1 2 3 4 5 6 7 8}- (#'jobs/get-deps ordering [1 6])))- (is (= #{2 3 4 5 6 7 8}- (#'jobs/get-deps ordering [2 6])))- (is (= #{1 2 3 4 5}- (#'jobs/get-deps ordering [1 2 3])))))--(deftest cycle-deps-test- (let [ordering {1 #{2}- 2 #{3}- 3 #{1}}]- (is (= #{1 2 3}- (#'jobs/get-deps ordering [1])))))-(deftest job-transform-ids-test(testing "single tag, single transform"(mt/with-temp [:model/TransformTag tag {:name "tag-a"}@@ -450,6 +425,32 @@(is (mt/received-email-subject? :crowberto #"The job .* had failures"))(is (mt/received-email-body? :crowberto #"transform0")))))))))))))+(deftest get-plan-ignores-unrelated-routing-enabled-transforms-test+ (testing "get-plan must not scan unrelated transforms on routing-enabled databases"+ ;; Regression: a transform on a routing-enabled database is unrunnable (by design), but historically+ ;; `get-plan` would fetch *every* transform in the system and call `table-dependencies` on each to+ ;; build a global dependency graph. The routing-enabled transform would throw during that scan,+ ;; taking down the whole scheduler and sending a misleading failure email naming the zombie+ ;; transform — even when no job was asking to run it.+ (mt/with-premium-features #{:database-routing :transforms-basic}+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Database _destination {:engine :h2+ :router_database_id (mt/id)+ :details {:destination_database true}}+ :model/DatabaseRouter _ {:database_id (mt/id)+ :user_attribute "db_name"}+ ;; Zombie transform on a routing-enabled database, NOT tagged to any job.+ :model/Transform _zombie {:name "zombie-transform"+ :source {:type :query+ :query (lib/native-query mp "SELECT 1")}+ :creator_id (mt/user->id :crowberto)+ :target {:type "table"+ :database (mt/id)+ :schema "PUBLIC"+ :name "zombie_out"}}]+ (testing "get-plan with empty transform-ids must not throw on unrelated zombies"+ (is (empty? (:order (#'jobs/get-plan #{}))))))))))+(deftest run-transforms!-race-condition-test;; Previously a race would ensure one transform run got the duplicate key error and aborted.;; Because it is possible to set up transforms to run on overlapping schedules, such races are inevitable.diff --git a/test/metabase/transforms_base/ordering_test.clj b/test/metabase/transforms_base/ordering_test.cljindex c390b54bfcf..0aa51fcdc16 100644--- a/test/metabase/transforms_base/ordering_test.clj+++ b/test/metabase/transforms_base/ordering_test.clj@@ -35,7 +35,7 @@:type "query",:query {:source-table (mt/id :orders)}})](is (= {t1 #{}}- (ordering/transform-ordering (t2/select :model/Transform :id t1))))))+ (:dependencies (ordering/transform-ordering #{t1} (t2/select :model/Transform :id t1)))))))(deftest dependency-ordering-test(mt/with-temp [:model/Table {table :id} {:schema (default-schema-or-public)@@ -54,7 +54,7 @@"orders_3")](is (= {parent #{}child #{parent}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [parent child]]))))))+ (:dependencies (ordering/transform-ordering #{parent child} (t2/select :model/Transform :id [:in [parent child]])))))))(defn- transform-deps-for-db [transform](mt/with-metadata-provider (mt/id)@@ -95,7 +95,7 @@t2 #{}t3 #{t1}t4 #{t1 t2}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3 t4]]))))))+ (:dependencies (ordering/transform-ordering #{t1 t2 t3 t4} (t2/select :model/Transform :id [:in [t1 t2 t3 t4]])))))))(testing "dependencies are correctly identified when some transform have been run and some haven't"(mt/with-temp [:model/Transform {t1 :id :as transform1} (make-transform{:database (mt/id),@@ -123,7 +123,7 @@(is (= {t1 #{}t2 #{}t3 #{t1 t2}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))+ (:dependencies (ordering/transform-ordering #{t1 t2 t3} (t2/select :model/Transform :id [:in [t1 t2 t3]])))))(finally(t2/delete! :model/Table :name "checkins_transform")))))))@@ -156,7 +156,7 @@(is (= {t1 #{}t2 #{}t3 #{t1 t2}}- (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))+ (:dependencies (ordering/transform-ordering #{t1 t2 t3} (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))))(deftest ^:parallel basic-dependencies-test(mt/with-temp [:model/Transform {t1 :id} (make-transformdiff --git a/test/metabase/transforms_base/ordering_unit_test.clj b/test/metabase/transforms_base/ordering_unit_test.cljnew file mode 100644index 00000000000..10adee02d4c--- /dev/null+++ b/test/metabase/transforms_base/ordering_unit_test.clj@@ -0,0 +1,88 @@+(ns metabase.transforms-base.ordering-unit-test+ "Pure-data unit tests for `transform-ordering`'s closure-walking algorithm.++ These tests do not require a real database or query processor — they stub+ `table-dependencies` via `with-redefs` so the walk can be exercised in isolation+ on synthetic transform maps. Driver-gated integration tests live in+ `metabase.transforms-base.ordering-test`."+ (:require+ [clojure.test :refer :all]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.ordering :as ordering]))++(defn- tx+ "Build a synthetic transform map for tests. `dep-ids` is the set of transform ids this+ transform should (after resolution) depend on. Each transform gets a unique target_table_id+ so the dep extractor stub can emit `{:table <id>}` deps and `resolve-dependency` can map+ them back to transform ids via `output-tables`."+ [id dep-ids]+ {:id id+ :target_table_id (* 100 id)+ :target {:database 1 :schema "public" :name (str "t" id)}+ :test-deps (into #{} (map (fn [dep-id] {:table (* 100 dep-id)})) dep-ids)})++(deftest transform-ordering-closure-walk-test+ (testing "transform-ordering only visits transforms reachable from start-ids"+ ;; Stub table-dependencies to read the :test-deps key we put on the synthetic transforms.+ ;; This keeps the test focused on the closure-walk and dep-resolution logic without+ ;; needing a real query processor.+ (with-redefs [transforms-base.i/table-dependencies :test-deps]+ (testing "empty start-ids returns empty ordering"+ (is (= {:dependencies {} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{} [(tx 1 #{})]))))++ (testing "single transform with no deps"+ (is (= {:dependencies {1 #{}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{1} [(tx 1 #{})]))))++ (testing "direct dependency is resolved and included in the closure"+ (is (= {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})]))))++ (testing "transitive dependencies are walked outward"+ (is (= {:dependencies {1 #{} 2 #{1} 3 #{2}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{2})]))))++ (testing "multiple start ids with a shared upstream"+ (is (= {:dependencies {1 #{} 2 #{1} 3 #{1}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{2 3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{1})]))))++ (testing "unrelated transforms are never visited or included"+ (let [{:keys [dependencies]} (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{}) (tx 3 #{})])]+ (is (= {2 #{}} dependencies))+ (is (not (contains? dependencies 1)))+ (is (not (contains? dependencies 3)))))++ (testing "non-existent start ids are captured in :not-found, not in :dependencies"+ (is (= {:dependencies {} :not-found #{999} :failed #{}}+ (ordering/transform-ordering #{999} [(tx 1 #{})]))))++ (testing "a cycle in the dep graph does not infinite-loop"+ (is (= {:dependencies {1 #{2} 2 #{1}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{1})])))))))++(deftest transform-ordering-catches-per-transform-failures-test+ (testing "per-transform dep-extraction failures are caught, captured in :failed, and treated as no deps"+ (testing "failure on a start-id: the transform becomes a leaf, its supposed deps are not visited"+ (with-redefs [transforms-base.i/table-dependencies+ (fn [transform]+ (if (= (:id transform) 1)+ (throw (ex-info "simulated extraction failure" {}))+ (:test-deps transform)))]+ ;; Transform 1 is tagged but its extractor throws. The walk catches, captures 1 in :failed,+ ;; treats 1 as a leaf, and the supposed downstream dep (2) is never visited.+ (is (= {:dependencies {1 #{}} :not-found #{} :failed #{1}}+ (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{})])))))++ (testing "failure on a discovered upstream: upstream is still included (the parent's deps found it), but has no further deps of its own"+ (with-redefs [transforms-base.i/table-dependencies+ (fn [transform]+ (if (= (:id transform) 1)+ (throw (ex-info "simulated upstream failure" {}))+ (:test-deps transform)))]+ ;; Transform 2 is tagged and depends on 1. 2's extractor succeeds, so the edge 2→1 is+ ;; recorded. Then 1 is visited, its extractor throws, so 1 is captured in :failed and+ ;; treated as a leaf. The parent edge 2→1 is preserved, which is what run-transforms!+ ;; needs for skip-on-failure attribution.+ (is (= {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{1}}+ (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})])))))))