The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
reduce compute load of transform planning (#76406)
Nicola Mometto1ed83cde4bc0
diff --git a/resources/migrations/063/20260624_gdgt_2553_transform_table_dependencies.yaml b/resources/migrations/063/20260624_gdgt_2553_transform_table_dependencies.yamlnew file mode 100644index 00000000000..4d7975cc5fe--- /dev/null+++ b/resources/migrations/063/20260624_gdgt_2553_transform_table_dependencies.yaml@@ -0,0 +1,23 @@+## Precompute transform table dependencies into a column so the ordering read paths+## (job details, cycle detection) don't have to invoke the SQL parser on every read.++databaseChangeLog:++ - changeSet:+ id: v63.q8x2kf+ author: bronsa+ comment: Add table_dependencies column to transform table+ changes:+ - addColumn:+ tableName: transform+ columns:+ - column:+ name: table_dependencies+ type: ${text.type}+ remarks: Precomputed table dependencies (JSON) used for execution ordering and cycle detection. Null means not yet computed.+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: transform+ columnName: table_dependenciesdiff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex 6dd15f0a04a..7125cabfb46 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -56,13 +56,13 @@(defn- get-plan [transform-ids](tracing/with-span :tasks "task.transform.plan" {:transform/count (count transform-ids)}- (let [all-transforms (t2/select :model/Transform)+ (let [all-transforms (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies]);; Walk only the dependency closure of the transforms we're asked to run.;; `table-dependencies` (and the QP preprocessing it triggers) is therefore called;; only on transforms in that closure — never on unrelated transforms elsewhere in;; the system. This is what prevents a single broken transform (e.g. one on a;; routing-enabled database) from poisoning the scheduler when no job has asked for it.- {:keys [dependencies not-found failed]}+ {:keys [dependencies not-found failed uncached]}(transforms-base.ordering/transform-ordering transform-ids all-transforms)](when (seq not-found)(log/warnf "transform-ordering: %d scheduled id(s) not found in transforms (likely deleted between scheduling and lookup): %s"@@ -70,14 +70,15 @@(when (seq failed)(log/warnf "transform-ordering: %d transform(s) failed dep extraction; treated as leaves: %s"(count failed) (pr-str (sort failed))))- (let [transforms-by-id (into {}- (keep (fn [{:keys [id] :as transform}]- (when (contains? dependencies id)- [id transform])))- all-transforms)+ ;; Lazily backfill the table_dependencies column for any closure transform we had to compute live.+ (transforms-base.ordering/persist-table-dependencies! uncached)+ ;; Fetch full rows only for the closure, which is what callers actually consume.+ (let [transforms-by-id (if (seq dependencies)+ (u/index-by :id (t2/select :model/Transform :id [:in (keys dependencies)]))+ {})sorted-ord (sorted-ordering dependencies transforms-by-id)](when-let [cycle (transforms-base.ordering/find-cycle sorted-ord)]- (let [id->name (into {} (map (juxt :id :name)) all-transforms)]+ (let [id->name (into {} (map (juxt :id :name)) (vals transforms-by-id))](throw (ex-info (str "Cyclic transform definitions detected: "(str/join " → " (map id->name cycle))){:cycle cycle}))))diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex c6dadd8f5db..e219359ddf9 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -116,10 +116,12 @@mi/json-in))(t2/deftransforms :model/Transform- {:source_type mi/transform-keyword- :source {:out transform-source-out, :in transform-source-in}- :target mi/transform-json- :run_trigger mi/transform-keyword})+ {:source_type mi/transform-keyword+ :source {:out transform-source-out, :in transform-source-in}+ :target mi/transform-json+ ;; nil round-trips as NULL+ :table_dependencies {:in #(some-> % mi/json-in), :out mi/json-out-with-keywordization}+ :run_trigger mi/transform-keyword})(defmethod collection/allowed-namespaces :model/Transform[_]@@ -161,6 +163,10 @@(assoc :source_type (transforms-base.u/transform-source-type source):source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))+ ;; Invalidate cached deps when the source changes+ (:source (t2/changes transform))+ (assoc :table_dependencies nil)+target-changed?(assoc :target_db_id target-db-id)@@ -399,7 +405,7 @@(defmethod serdes/make-spec "Transform"[_model-name opts]{:copy [:name :description :entity_id :owner_email]- :skip [:source_type :target_db_id :target_table_id :last_checkpoint_value]+ :skip [:source_type :target_db_id :target_table_id :last_checkpoint_value :table_dependencies]:transform {:created_at (serdes/date):creator_id (serdes/fk :model/User):owner_user_id (serdes/fk :model/User)diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.cljindex a442b4f6ff9..039996296f0 100644--- a/src/metabase/transforms_base/ordering.clj+++ b/src/metabase/transforms_base/ordering.clj@@ -14,6 +14,8 @@[metabase.transforms-base.util :as transforms-base.u][metabase.util :as u][metabase.util.i18n :as i18n]+ [metabase.util.json :as json]+ [metabase.util.log :as log][metabase.util.malli :as mu][toucan2.core :as t2])(:import@@ -62,14 +64,39 @@;;; ------------------------------------------------- Ordering Logic -------------------------------------------------+(defn stored-or-live-deps+ "Dependencies of `transform`, preferring the precomputed `:table_dependencies` column and falling+ back to a live `table-dependencies` call when it is absent (`nil`). An empty (but non-nil) stored+ value is taken at face value as \"no dependencies\"."+ [transform]+ (if-some [deps (:table_dependencies transform)]+ deps+ (transforms-base.i/table-dependencies+ ;; load :source on demand on a cache miss, so callers can omit the heavy blob from their select+ (cond-> transform+ (and (not (:source transform)) (:id transform))+ (assoc :source (t2/select-one-fn :source [:model/Transform :id :source] (:id transform)))))))++(defn references-card-or-snippet?+ "True if `transform`'s source query reads through a saved card or native snippet."+ [transform]+ (let [query (:query (or (:source transform)+ (t2/select-one-fn :source [:model/Transform :id :source] (:id transform))))]+ (boolean+ (when query+ (try+ (or (seq (lib/all-source-card-ids query))+ (seq (lib/all-template-tag-snippet-ids query)))+ (catch Throwable _ false))))))+(defn safe-table-dependencies- "Like `table-dependencies`, but returns `#{}` if the computation throws. Used by cycle+ "Like `stored-or-live-deps`, but returns `#{}` if the computation throws. Used by cycledetection where a single broken transform must not block the whole check. Callers that needto know *which* transforms failed should walk the graph themselves and capture the failureids — see `transform-ordering`."[transform](try- (transforms-base.i/table-dependencies transform)+ (stored-or-live-deps transform)(catch Throwable _ #{})))(mu/defn- output-table-map@@ -128,9 +155,12 @@{:dependencies {transform-id -> #{transform-ids it depends on}}:not-found #{ids in start-ids that don't refer to any transform in all-transforms}- :failed #{ids whose table-dependencies threw}}+ :failed #{ids whose table-dependencies threw}+ :uncached {transform-id -> raw-deps} for visited transforms whose `:table_dependencies`+ column was absent, computed live, and safe to cache}- `:dependencies` is restricted to the transitive closure reachable from `start-ids`.+ `:dependencies` is restricted to the transitive closure reachable from `start-ids`. `:uncached`+ lets the caller persist freshly computed deps so later reads hit the cache instead.Diagnostics in `:not-found` and `:failed` let the caller decide how to surface them(logging, metrics, error responses)."[start-ids all-transforms]@@ -141,19 +171,21 @@(loop [visited {}not-found #{}failed #{}+ uncached {}queue (vec start-ids)](if-let [id (first queue)](cond(contains? visited id)- (recur visited not-found failed (rest queue))+ (recur visited not-found failed uncached (rest queue))(not (id->xf id))- (recur visited (conj not-found id) failed (rest queue))+ (recur visited (conj not-found id) failed uncached (rest queue)):else(let [transform (id->xf id)+ miss? (nil? (:table_dependencies transform))[raw-deps fail?] (try- [(transforms-base.i/table-dependencies transform) false]+ [(stored-or-live-deps transform) false](catch Throwable _ [#{} true]))resolved-ids (into #{}(keep (fn [dep]@@ -162,10 +194,25 @@(recur (assoc visited id resolved-ids)not-found(cond-> failed fail? (conj id))+ ;; Only cache transforms that don't read through a card/snippet; those pass through+ ;; since their deps can change without the transform doing.+ (cond-> uncached (and miss? (not fail?) (not (references-card-or-snippet? transform)))+ (assoc id raw-deps))(into (rest queue) resolved-ids)))){:dependencies visited:not-found not-found- :failed failed}))))+ :failed failed+ :uncached uncached}))))++(defn persist-table-dependencies!+ "Best-effort write-back of `:uncached` deps from `transform-ordering` into the+ `transform.table_dependencies` column, keyed by transform id."+ [id->raw-deps]+ (doseq [[id raw-deps] id->raw-deps]+ (try+ (t2/update! (t2/table-name :model/Transform) id {:table_dependencies (json/encode (vec raw-deps))})+ (catch Throwable e+ (log/warnf e "Failed to cache table-dependencies for transform %s" id)))))(defn find-cycle"Finds a path containing a cycle in the directed graph `node->children`.@@ -201,16 +248,20 @@```"[{transform-id :id :as to-check}]- (let [transforms (map (fn [{:keys [id] :as transform}]+ (let [db-id (get-in to-check [:source :query :database])+ ;; Recompute the transform under test live — its source may have just changed, so any stored+ ;; deps are stale — and pin its source db. Every other transform uses its stored deps.+ to-check (-> to-check (assoc :source_database_id db-id) (dissoc :table_dependencies))+ transforms (map (fn [{:keys [id] :as transform}](if (= id transform-id)to-checktransform))- (t2/select :model/Transform))+ (t2/select [:model/Transform :id :name :target :target_table_id+ :source_database_id :table_dependencies]))transforms-by-id (into {}(map (juxt :id identity))transforms)- db-id (get-in to-check [:source :query :database])- db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)+ db-transforms (filter #(= (:source_database_id %) db-id) transforms)output-tables (output-table-map db-transforms)transform-ids (into #{} (map :id) db-transforms)target-refs (target-ref-map transforms)diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 69a9628acc9..373b8da6ec0 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -80,6 +80,7 @@[:description [:maybe :string]][:source :any][:target :any]+ [:table_dependencies {:optional true} [:maybe [:sequential :map]]][:source_type :keyword][:source_database_id {:optional true} [:maybe pos-int?]][:source_readable {:optional true} [:maybe :boolean]]diff --git a/test/metabase/transforms/models/transform_test.clj b/test/metabase/transforms/models/transform_test.cljindex 36bf058b3f5..ce778794a1c 100644--- a/test/metabase/transforms/models/transform_test.clj+++ b/test/metabase/transforms/models/transform_test.clj@@ -111,3 +111,32 @@(t2/update! :model/Transform transform-id {:name "Deserialized Update"})(t2/delete! :model/Transform transform-id)))(is (empty? @events-published)))))))++(defn- stored-deps [transform-id]+ (t2/select-one-fn :table_dependencies :model/Transform transform-id))++(deftest table-dependencies-cache-test+ (testing "table_dependencies is not computed on insert — it is filled lazily on the first read"+ (mt/with-temp [:model/Transform {id :id}+ {:name "deps insert"+ :source {:type "query"+ :query {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}}}]+ (is (nil? (stored-deps id)))))+ (testing "changing the source invalidates any cached table_dependencies"+ (mt/with-temp [:model/Transform {id :id}+ {:name "deps update"+ :source {:type "query"+ :query {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}}}]+ ;; seed a cached value (as the lazy read path would)+ (t2/update! :model/Transform id {:table_dependencies [{:table (mt/id :orders)}]})+ (is (= [{:table (mt/id :orders)}] (stored-deps id)))+ (t2/update! :model/Transform id+ {:source {:type "query"+ :query {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :people)}}}})+ (is (nil? (stored-deps id))))))diff --git a/test/metabase/transforms_base/ordering_test.clj b/test/metabase/transforms_base/ordering_test.cljindex 228bb533c6d..714f80e19a8 100644--- a/test/metabase/transforms_base/ordering_test.clj+++ b/test/metabase/transforms_base/ordering_test.clj@@ -369,3 +369,48 @@(is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",:cycle [t1 t3 t2]}(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))++;;; --------------------------------------- Precomputed table_dependencies ---------------------------------------+;;; These exercise the read side in isolation (no driver / SQL parser): transforms carry their deps in the+;;; `:table_dependencies` key the way the appdb column supplies them.++(deftest stored-or-live-deps-test+ (testing "prefers the stored :table_dependencies value"+ (is (= [{:table 1}] (ordering/stored-or-live-deps {:table_dependencies [{:table 1}]}))))+ (testing "an empty (non-nil) stored value means \"no dependencies\""+ (is (= [] (ordering/stored-or-live-deps {:table_dependencies []}))))+ (testing "falls back to a live computation when the stored value is nil"+ ;; no :source and an unknown type -> the :default table-dependencies impl returns #{}+ (is (= #{} (ordering/stored-or-live-deps {:id 1})))))++(deftest transform-ordering-reads-stored-deps-test+ (testing "transform-ordering resolves dependencies from :table_dependencies, never parsing SQL"+ (let [producer {:id 1 :target_table_id 100 :table_dependencies []}+ consumer {:id 2 :table_dependencies [{:table 100}]}]+ (is (= {1 #{} 2 #{1}}+ (:dependencies (ordering/transform-ordering #{2} [producer consumer]))))))+ (testing "a :table-ref dependency resolves to the producing transform by (db, schema, name)"+ (let [producer {:id 1 :target {:database 10 :schema "public" :name "out"} :table_dependencies []}+ consumer {:id 2 :table_dependencies [{:table-ref {:database_id 10 :schema "public" :table "out"}}]}]+ (is (= {1 #{} 2 #{1}}+ (:dependencies (ordering/transform-ordering #{2} [producer consumer]))))))+ (testing "an empty stored value yields no dependencies"+ (is (= {1 #{}}+ (:dependencies (ordering/transform-ordering #{1} [{:id 1 :table_dependencies []}]))))))++(deftest lazy-table-dependencies-backfill-test+ (testing "a transform with no cached :table_dependencies is reported in :uncached and persisted"+ (mt/test-driver (mt/normal-driver-select {:+parent :sql-jdbc})+ (mt/with-temp [:model/Transform {id :id} (make-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}})]+ ;; simulate a row created before the column existed+ (t2/update! (t2/table-name :model/Transform) id {:table_dependencies nil})+ (mt/with-metadata-provider (mt/id)+ (let [all (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies])+ {:keys [uncached]} (ordering/transform-ordering #{id} all)]+ (is (contains? uncached id))+ (ordering/persist-table-dependencies! uncached)+ (is (= [{:table (mt/id :orders)}]+ (t2/select-one-fn :table_dependencies :model/Transform id)))))))))diff --git a/test/metabase/transforms_base/ordering_unit_test.clj b/test/metabase/transforms_base/ordering_unit_test.cljindex ce382faba8b..ee2f93c544a 100644--- a/test/metabase/transforms_base/ordering_unit_test.clj+++ b/test/metabase/transforms_base/ordering_unit_test.clj@@ -28,31 +28,31 @@;; needing a real query processor.(with-redefs [transforms-base.i/table-dependencies :test-deps](testing "empty start-ids returns empty ordering"- (is (= {:dependencies {} :not-found #{} :failed #{}}- (ordering/transform-ordering #{} [(tx 1 #{})]))))+ (is (=? {:dependencies {} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{} [(tx 1 #{})]))))(testing "single transform with no deps"- (is (= {:dependencies {1 #{}} :not-found #{} :failed #{}}- (ordering/transform-ordering #{1} [(tx 1 #{})]))))+ (is (=? {:dependencies {1 #{}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{1} [(tx 1 #{})]))))(testing "direct dependency is resolved and included in the closure"- (is (= {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{}}- (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})]))))+ (is (=? {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})]))))(testing "transitive dependencies are walked outward"- (is (= {:dependencies {1 #{} 2 #{1} 3 #{2}} :not-found #{} :failed #{}}- (ordering/transform-ordering #{3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{2})]))))+ (is (=? {:dependencies {1 #{} 2 #{1} 3 #{2}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{2})]))))(testing "multiple start ids with a shared upstream"- (is (= {:dependencies {1 #{} 2 #{1} 3 #{1}} :not-found #{} :failed #{}}- (ordering/transform-ordering #{2 3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{1})]))))+ (is (=? {:dependencies {1 #{} 2 #{1} 3 #{1}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{2 3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{1})]))))(testing "unrelated transforms are never visited or included"(let [{:keys [dependencies]} (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{}) (tx 3 #{})])](is (= {2 #{}} dependencies))(is (not (contains? dependencies 1)))(is (not (contains? dependencies 3)))))(testing "non-existent start ids are captured in :not-found, not in :dependencies"- (is (= {:dependencies {} :not-found #{999} :failed #{}}- (ordering/transform-ordering #{999} [(tx 1 #{})]))))+ (is (=? {:dependencies {} :not-found #{999} :failed #{}}+ (ordering/transform-ordering #{999} [(tx 1 #{})]))))(testing "a cycle in the dep graph does not infinite-loop"- (is (= {:dependencies {1 #{2} 2 #{1}} :not-found #{} :failed #{}}- (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{1})])))))))+ (is (=? {:dependencies {1 #{2} 2 #{1}} :not-found #{} :failed #{}}+ (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{1})])))))))(deftest transform-ordering-catches-per-transform-failures-test(testing "per-transform dep-extraction failures are caught, captured in :failed, and treated as no deps"@@ -64,8 +64,8 @@(:test-deps transform)))];; Transform 1 is tagged but its extractor throws. The walk catches, captures 1 in :failed,;; treats 1 as a leaf, and the supposed downstream dep (2) is never visited.- (is (= {:dependencies {1 #{}} :not-found #{} :failed #{1}}- (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{})])))))+ (is (=? {:dependencies {1 #{}} :not-found #{} :failed #{1}}+ (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{})])))))(testing "failure on a discovered upstream: upstream is still included (the parent's deps found it), but has no further deps of its own"(with-redefs [transforms-base.i/table-dependencies(fn [transform]@@ -76,5 +76,17 @@;; recorded. Then 1 is visited, its extractor throws, so 1 is captured in :failed and;; treated as a leaf. The parent edge 2→1 is preserved, which is what run-transforms!;; needs for skip-on-failure attribution.- (is (= {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{1}}- (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})])))))))+ (is (=? {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{1}}+ (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})])))))))++(deftest passthrough-card-or-snippet-test+ (testing "transforms that read through a card/snippet are omitted from :uncached (they pass through)"+ (with-redefs [transforms-base.i/table-dependencies :test-deps+ ;; pretend transform 1 references a card/snippet+ ordering/references-card-or-snippet? #(= 1 (:id %))]+ (let [{:keys [uncached]} (ordering/transform-ordering+ #{1 2}+ [{:id 1 :test-deps #{}}+ {:id 2 :test-deps #{}}])]+ (is (not (contains? uncached 1)))+ (is (contains? uncached 2))))))