Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

reduce compute load of transform planning (#76406)

Nicola Mometto1ed83cde4bc0
diff --git a/resources/migrations/063/20260624_gdgt_2553_transform_table_dependencies.yaml b/resources/migrations/063/20260624_gdgt_2553_transform_table_dependencies.yaml
new file mode 100644
index 00000000000..4d7975cc5fe
--- /dev/null
+++ b/resources/migrations/063/20260624_gdgt_2553_transform_table_dependencies.yaml
@@ -0,0 +1,23 @@
+## Precompute transform table dependencies into a column so the ordering read paths
+## (job details, cycle detection) don't have to invoke the SQL parser on every read.
+
+databaseChangeLog:
+
+ - changeSet:
+ id: v63.q8x2kf
+ author: bronsa
+ comment: Add table_dependencies column to transform table
+ changes:
+ - addColumn:
+ tableName: transform
+ columns:
+ - column:
+ name: table_dependencies
+ type: ${text.type}
+ remarks: Precomputed table dependencies (JSON) used for execution ordering and cycle detection. Null means not yet computed.
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: transform
+ columnName: table_dependencies
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index 6dd15f0a04a..7125cabfb46 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -56,13 +56,13 @@
(defn- get-plan [transform-ids]
(tracing/with-span :tasks "task.transform.plan" {:transform/count (count transform-ids)}
- (let [all-transforms (t2/select :model/Transform)
+ (let [all-transforms (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies])
;; Walk only the dependency closure of the transforms we're asked to run.
;; `table-dependencies` (and the QP preprocessing it triggers) is therefore called
;; only on transforms in that closure — never on unrelated transforms elsewhere in
;; the system. This is what prevents a single broken transform (e.g. one on a
;; routing-enabled database) from poisoning the scheduler when no job has asked for it.
- {:keys [dependencies not-found failed]}
+ {:keys [dependencies not-found failed uncached]}
(transforms-base.ordering/transform-ordering transform-ids all-transforms)]
(when (seq not-found)
(log/warnf "transform-ordering: %d scheduled id(s) not found in transforms (likely deleted between scheduling and lookup): %s"
@@ -70,14 +70,15 @@
(when (seq failed)
(log/warnf "transform-ordering: %d transform(s) failed dep extraction; treated as leaves: %s"
(count failed) (pr-str (sort failed))))
- (let [transforms-by-id (into {}
- (keep (fn [{:keys [id] :as transform}]
- (when (contains? dependencies id)
- [id transform])))
- all-transforms)
+ ;; Lazily backfill the table_dependencies column for any closure transform we had to compute live.
+ (transforms-base.ordering/persist-table-dependencies! uncached)
+ ;; Fetch full rows only for the closure, which is what callers actually consume.
+ (let [transforms-by-id (if (seq dependencies)
+ (u/index-by :id (t2/select :model/Transform :id [:in (keys dependencies)]))
+ {})
sorted-ord (sorted-ordering dependencies transforms-by-id)]
(when-let [cycle (transforms-base.ordering/find-cycle sorted-ord)]
- (let [id->name (into {} (map (juxt :id :name)) all-transforms)]
+ (let [id->name (into {} (map (juxt :id :name)) (vals transforms-by-id))]
(throw (ex-info (str "Cyclic transform definitions detected: "
(str/join " → " (map id->name cycle)))
{:cycle cycle}))))
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index c6dadd8f5db..e219359ddf9 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -116,10 +116,12 @@
mi/json-in))
(t2/deftransforms :model/Transform
- {:source_type mi/transform-keyword
- :source {:out transform-source-out, :in transform-source-in}
- :target mi/transform-json
- :run_trigger mi/transform-keyword})
+ {:source_type mi/transform-keyword
+ :source {:out transform-source-out, :in transform-source-in}
+ :target mi/transform-json
+ ;; nil round-trips as NULL
+ :table_dependencies {:in #(some-> % mi/json-in), :out mi/json-out-with-keywordization}
+ :run_trigger mi/transform-keyword})
(defmethod collection/allowed-namespaces :model/Transform
[_]
@@ -161,6 +163,10 @@
(assoc :source_type (transforms-base.u/transform-source-type source)
:source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
+ ;; Invalidate cached deps when the source changes
+ (:source (t2/changes transform))
+ (assoc :table_dependencies nil)
+
target-changed?
(assoc :target_db_id target-db-id)
@@ -399,7 +405,7 @@
(defmethod serdes/make-spec "Transform"
[_model-name opts]
{:copy [:name :description :entity_id :owner_email]
- :skip [:source_type :target_db_id :target_table_id :last_checkpoint_value]
+ :skip [:source_type :target_db_id :target_table_id :last_checkpoint_value :table_dependencies]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:owner_user_id (serdes/fk :model/User)
diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.clj
index a442b4f6ff9..039996296f0 100644
--- a/src/metabase/transforms_base/ordering.clj
+++ b/src/metabase/transforms_base/ordering.clj
@@ -14,6 +14,8 @@
[metabase.transforms-base.util :as transforms-base.u]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
[toucan2.core :as t2])
(:import
@@ -62,14 +64,39 @@
;;; ------------------------------------------------- Ordering Logic -------------------------------------------------
+(defn stored-or-live-deps
+ "Dependencies of `transform`, preferring the precomputed `:table_dependencies` column and falling
+ back to a live `table-dependencies` call when it is absent (`nil`). An empty (but non-nil) stored
+ value is taken at face value as \"no dependencies\"."
+ [transform]
+ (if-some [deps (:table_dependencies transform)]
+ deps
+ (transforms-base.i/table-dependencies
+ ;; load :source on demand on a cache miss, so callers can omit the heavy blob from their select
+ (cond-> transform
+ (and (not (:source transform)) (:id transform))
+ (assoc :source (t2/select-one-fn :source [:model/Transform :id :source] (:id transform)))))))
+
+(defn references-card-or-snippet?
+ "True if `transform`'s source query reads through a saved card or native snippet."
+ [transform]
+ (let [query (:query (or (:source transform)
+ (t2/select-one-fn :source [:model/Transform :id :source] (:id transform))))]
+ (boolean
+ (when query
+ (try
+ (or (seq (lib/all-source-card-ids query))
+ (seq (lib/all-template-tag-snippet-ids query)))
+ (catch Throwable _ false))))))
+
(defn safe-table-dependencies
- "Like `table-dependencies`, but returns `#{}` if the computation throws. Used by cycle
+ "Like `stored-or-live-deps`, but returns `#{}` if the computation throws. Used by cycle
detection where a single broken transform must not block the whole check. Callers that need
to know *which* transforms failed should walk the graph themselves and capture the failure
ids — see `transform-ordering`."
[transform]
(try
- (transforms-base.i/table-dependencies transform)
+ (stored-or-live-deps transform)
(catch Throwable _ #{})))
(mu/defn- output-table-map
@@ -128,9 +155,12 @@
{:dependencies {transform-id -> #{transform-ids it depends on}}
:not-found #{ids in start-ids that don't refer to any transform in all-transforms}
- :failed #{ids whose table-dependencies threw}}
+ :failed #{ids whose table-dependencies threw}
+ :uncached {transform-id -> raw-deps} for visited transforms whose `:table_dependencies`
+ column was absent, computed live, and safe to cache}
- `:dependencies` is restricted to the transitive closure reachable from `start-ids`.
+ `:dependencies` is restricted to the transitive closure reachable from `start-ids`. `:uncached`
+ lets the caller persist freshly computed deps so later reads hit the cache instead.
Diagnostics in `:not-found` and `:failed` let the caller decide how to surface them
(logging, metrics, error responses)."
[start-ids all-transforms]
@@ -141,19 +171,21 @@
(loop [visited {}
not-found #{}
failed #{}
+ uncached {}
queue (vec start-ids)]
(if-let [id (first queue)]
(cond
(contains? visited id)
- (recur visited not-found failed (rest queue))
+ (recur visited not-found failed uncached (rest queue))
(not (id->xf id))
- (recur visited (conj not-found id) failed (rest queue))
+ (recur visited (conj not-found id) failed uncached (rest queue))
:else
(let [transform (id->xf id)
+ miss? (nil? (:table_dependencies transform))
[raw-deps fail?] (try
- [(transforms-base.i/table-dependencies transform) false]
+ [(stored-or-live-deps transform) false]
(catch Throwable _ [#{} true]))
resolved-ids (into #{}
(keep (fn [dep]
@@ -162,10 +194,25 @@
(recur (assoc visited id resolved-ids)
not-found
(cond-> failed fail? (conj id))
+ ;; Only cache transforms that don't read through a card/snippet; those pass through
+ ;; since their deps can change without the transform doing.
+ (cond-> uncached (and miss? (not fail?) (not (references-card-or-snippet? transform)))
+ (assoc id raw-deps))
(into (rest queue) resolved-ids))))
{:dependencies visited
:not-found not-found
- :failed failed}))))
+ :failed failed
+ :uncached uncached}))))
+
+(defn persist-table-dependencies!
+ "Best-effort write-back of `:uncached` deps from `transform-ordering` into the
+ `transform.table_dependencies` column, keyed by transform id."
+ [id->raw-deps]
+ (doseq [[id raw-deps] id->raw-deps]
+ (try
+ (t2/update! (t2/table-name :model/Transform) id {:table_dependencies (json/encode (vec raw-deps))})
+ (catch Throwable e
+ (log/warnf e "Failed to cache table-dependencies for transform %s" id)))))
(defn find-cycle
"Finds a path containing a cycle in the directed graph `node->children`.
@@ -201,16 +248,20 @@
```
"
[{transform-id :id :as to-check}]
- (let [transforms (map (fn [{:keys [id] :as transform}]
+ (let [db-id (get-in to-check [:source :query :database])
+ ;; Recompute the transform under test live — its source may have just changed, so any stored
+ ;; deps are stale — and pin its source db. Every other transform uses its stored deps.
+ to-check (-> to-check (assoc :source_database_id db-id) (dissoc :table_dependencies))
+ transforms (map (fn [{:keys [id] :as transform}]
(if (= id transform-id)
to-check
transform))
- (t2/select :model/Transform))
+ (t2/select [:model/Transform :id :name :target :target_table_id
+ :source_database_id :table_dependencies]))
transforms-by-id (into {}
(map (juxt :id identity))
transforms)
- db-id (get-in to-check [:source :query :database])
- db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)
+ db-transforms (filter #(= (:source_database_id %) db-id) transforms)
output-tables (output-table-map db-transforms)
transform-ids (into #{} (map :id) db-transforms)
target-refs (target-ref-map transforms)
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 69a9628acc9..373b8da6ec0 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -80,6 +80,7 @@
[:description [:maybe :string]]
[:source :any]
[:target :any]
+ [:table_dependencies {:optional true} [:maybe [:sequential :map]]]
[:source_type :keyword]
[:source_database_id {:optional true} [:maybe pos-int?]]
[:source_readable {:optional true} [:maybe :boolean]]
diff --git a/test/metabase/transforms/models/transform_test.clj b/test/metabase/transforms/models/transform_test.clj
index 36bf058b3f5..ce778794a1c 100644
--- a/test/metabase/transforms/models/transform_test.clj
+++ b/test/metabase/transforms/models/transform_test.clj
@@ -111,3 +111,32 @@
(t2/update! :model/Transform transform-id {:name "Deserialized Update"})
(t2/delete! :model/Transform transform-id)))
(is (empty? @events-published)))))))
+
+(defn- stored-deps [transform-id]
+ (t2/select-one-fn :table_dependencies :model/Transform transform-id))
+
+(deftest table-dependencies-cache-test
+ (testing "table_dependencies is not computed on insert — it is filled lazily on the first read"
+ (mt/with-temp [:model/Transform {id :id}
+ {:name "deps insert"
+ :source {:type "query"
+ :query {:database (mt/id)
+ :type "query"
+ :query {:source-table (mt/id :orders)}}}}]
+ (is (nil? (stored-deps id)))))
+ (testing "changing the source invalidates any cached table_dependencies"
+ (mt/with-temp [:model/Transform {id :id}
+ {:name "deps update"
+ :source {:type "query"
+ :query {:database (mt/id)
+ :type "query"
+ :query {:source-table (mt/id :orders)}}}}]
+ ;; seed a cached value (as the lazy read path would)
+ (t2/update! :model/Transform id {:table_dependencies [{:table (mt/id :orders)}]})
+ (is (= [{:table (mt/id :orders)}] (stored-deps id)))
+ (t2/update! :model/Transform id
+ {:source {:type "query"
+ :query {:database (mt/id)
+ :type "query"
+ :query {:source-table (mt/id :people)}}}})
+ (is (nil? (stored-deps id))))))
diff --git a/test/metabase/transforms_base/ordering_test.clj b/test/metabase/transforms_base/ordering_test.clj
index 228bb533c6d..714f80e19a8 100644
--- a/test/metabase/transforms_base/ordering_test.clj
+++ b/test/metabase/transforms_base/ordering_test.clj
@@ -369,3 +369,48 @@
(is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
:cycle [t1 t3 t2]}
(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
+
+;;; --------------------------------------- Precomputed table_dependencies ---------------------------------------
+;;; These exercise the read side in isolation (no driver / SQL parser): transforms carry their deps in the
+;;; `:table_dependencies` key the way the appdb column supplies them.
+
+(deftest stored-or-live-deps-test
+ (testing "prefers the stored :table_dependencies value"
+ (is (= [{:table 1}] (ordering/stored-or-live-deps {:table_dependencies [{:table 1}]}))))
+ (testing "an empty (non-nil) stored value means \"no dependencies\""
+ (is (= [] (ordering/stored-or-live-deps {:table_dependencies []}))))
+ (testing "falls back to a live computation when the stored value is nil"
+ ;; no :source and an unknown type -> the :default table-dependencies impl returns #{}
+ (is (= #{} (ordering/stored-or-live-deps {:id 1})))))
+
+(deftest transform-ordering-reads-stored-deps-test
+ (testing "transform-ordering resolves dependencies from :table_dependencies, never parsing SQL"
+ (let [producer {:id 1 :target_table_id 100 :table_dependencies []}
+ consumer {:id 2 :table_dependencies [{:table 100}]}]
+ (is (= {1 #{} 2 #{1}}
+ (:dependencies (ordering/transform-ordering #{2} [producer consumer]))))))
+ (testing "a :table-ref dependency resolves to the producing transform by (db, schema, name)"
+ (let [producer {:id 1 :target {:database 10 :schema "public" :name "out"} :table_dependencies []}
+ consumer {:id 2 :table_dependencies [{:table-ref {:database_id 10 :schema "public" :table "out"}}]}]
+ (is (= {1 #{} 2 #{1}}
+ (:dependencies (ordering/transform-ordering #{2} [producer consumer]))))))
+ (testing "an empty stored value yields no dependencies"
+ (is (= {1 #{}}
+ (:dependencies (ordering/transform-ordering #{1} [{:id 1 :table_dependencies []}]))))))
+
+(deftest lazy-table-dependencies-backfill-test
+ (testing "a transform with no cached :table_dependencies is reported in :uncached and persisted"
+ (mt/test-driver (mt/normal-driver-select {:+parent :sql-jdbc})
+ (mt/with-temp [:model/Transform {id :id} (make-transform
+ {:database (mt/id)
+ :type "query"
+ :query {:source-table (mt/id :orders)}})]
+ ;; simulate a row created before the column existed
+ (t2/update! (t2/table-name :model/Transform) id {:table_dependencies nil})
+ (mt/with-metadata-provider (mt/id)
+ (let [all (t2/select [:model/Transform :id :target :target_table_id :created_at :table_dependencies])
+ {:keys [uncached]} (ordering/transform-ordering #{id} all)]
+ (is (contains? uncached id))
+ (ordering/persist-table-dependencies! uncached)
+ (is (= [{:table (mt/id :orders)}]
+ (t2/select-one-fn :table_dependencies :model/Transform id)))))))))
diff --git a/test/metabase/transforms_base/ordering_unit_test.clj b/test/metabase/transforms_base/ordering_unit_test.clj
index ce382faba8b..ee2f93c544a 100644
--- a/test/metabase/transforms_base/ordering_unit_test.clj
+++ b/test/metabase/transforms_base/ordering_unit_test.clj
@@ -28,31 +28,31 @@
;; needing a real query processor.
(with-redefs [transforms-base.i/table-dependencies :test-deps]
(testing "empty start-ids returns empty ordering"
- (is (= {:dependencies {} :not-found #{} :failed #{}}
- (ordering/transform-ordering #{} [(tx 1 #{})]))))
+ (is (=? {:dependencies {} :not-found #{} :failed #{}}
+ (ordering/transform-ordering #{} [(tx 1 #{})]))))
(testing "single transform with no deps"
- (is (= {:dependencies {1 #{}} :not-found #{} :failed #{}}
- (ordering/transform-ordering #{1} [(tx 1 #{})]))))
+ (is (=? {:dependencies {1 #{}} :not-found #{} :failed #{}}
+ (ordering/transform-ordering #{1} [(tx 1 #{})]))))
(testing "direct dependency is resolved and included in the closure"
- (is (= {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{}}
- (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})]))))
+ (is (=? {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{}}
+ (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})]))))
(testing "transitive dependencies are walked outward"
- (is (= {:dependencies {1 #{} 2 #{1} 3 #{2}} :not-found #{} :failed #{}}
- (ordering/transform-ordering #{3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{2})]))))
+ (is (=? {:dependencies {1 #{} 2 #{1} 3 #{2}} :not-found #{} :failed #{}}
+ (ordering/transform-ordering #{3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{2})]))))
(testing "multiple start ids with a shared upstream"
- (is (= {:dependencies {1 #{} 2 #{1} 3 #{1}} :not-found #{} :failed #{}}
- (ordering/transform-ordering #{2 3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{1})]))))
+ (is (=? {:dependencies {1 #{} 2 #{1} 3 #{1}} :not-found #{} :failed #{}}
+ (ordering/transform-ordering #{2 3} [(tx 1 #{}) (tx 2 #{1}) (tx 3 #{1})]))))
(testing "unrelated transforms are never visited or included"
(let [{:keys [dependencies]} (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{}) (tx 3 #{})])]
(is (= {2 #{}} dependencies))
(is (not (contains? dependencies 1)))
(is (not (contains? dependencies 3)))))
(testing "non-existent start ids are captured in :not-found, not in :dependencies"
- (is (= {:dependencies {} :not-found #{999} :failed #{}}
- (ordering/transform-ordering #{999} [(tx 1 #{})]))))
+ (is (=? {:dependencies {} :not-found #{999} :failed #{}}
+ (ordering/transform-ordering #{999} [(tx 1 #{})]))))
(testing "a cycle in the dep graph does not infinite-loop"
- (is (= {:dependencies {1 #{2} 2 #{1}} :not-found #{} :failed #{}}
- (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{1})])))))))
+ (is (=? {:dependencies {1 #{2} 2 #{1}} :not-found #{} :failed #{}}
+ (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{1})])))))))
(deftest transform-ordering-catches-per-transform-failures-test
(testing "per-transform dep-extraction failures are caught, captured in :failed, and treated as no deps"
@@ -64,8 +64,8 @@
(:test-deps transform)))]
;; Transform 1 is tagged but its extractor throws. The walk catches, captures 1 in :failed,
;; treats 1 as a leaf, and the supposed downstream dep (2) is never visited.
- (is (= {:dependencies {1 #{}} :not-found #{} :failed #{1}}
- (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{})])))))
+ (is (=? {:dependencies {1 #{}} :not-found #{} :failed #{1}}
+ (ordering/transform-ordering #{1} [(tx 1 #{2}) (tx 2 #{})])))))
(testing "failure on a discovered upstream: upstream is still included (the parent's deps found it), but has no further deps of its own"
(with-redefs [transforms-base.i/table-dependencies
(fn [transform]
@@ -76,5 +76,17 @@
;; recorded. Then 1 is visited, its extractor throws, so 1 is captured in :failed and
;; treated as a leaf. The parent edge 2→1 is preserved, which is what run-transforms!
;; needs for skip-on-failure attribution.
- (is (= {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{1}}
- (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})])))))))
+ (is (=? {:dependencies {1 #{} 2 #{1}} :not-found #{} :failed #{1}}
+ (ordering/transform-ordering #{2} [(tx 1 #{}) (tx 2 #{1})])))))))
+
+(deftest passthrough-card-or-snippet-test
+ (testing "transforms that read through a card/snippet are omitted from :uncached (they pass through)"
+ (with-redefs [transforms-base.i/table-dependencies :test-deps
+ ;; pretend transform 1 references a card/snippet
+ ordering/references-card-or-snippet? #(= 1 (:id %))]
+ (let [{:keys [uncached]} (ordering/transform-ordering
+ #{1 2}
+ [{:id 1 :test-deps #{}}
+ {:id 2 :test-deps #{}}])]
+ (is (not (contains? uncached 1)))
+ (is (contains? uncached 2))))))