Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Create inactive tables and fields when deserializing non-existent references (#73741)

Alexander Polyankinff8dacfc60a0
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 2ffe8d5049c..c4a8198bacd 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2278,8 +2278,7 @@
task-history
tracing
transforms-base
- util
- warehouse-schema}
+ util}
:friends #{models
enterprise/transforms-python}
:model-exports #{:model/Transform
@@ -2312,8 +2311,7 @@
lib-be
query-processor
sync
- util
- warehouse-schema}
+ util}
:friends #{transforms
enterprise/transforms-python}}
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index e4ee8ed6128..2c42a85e106 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -1368,7 +1368,11 @@
(mt/with-premium-features #{:dependencies}
(let [mp (mt/metadata-provider)
products (lib.metadata/table mp (mt/id :products))]
- (mt/with-temp [:model/Transform {unreffed-transform-id :id} {:name "Unreferenced Transform - unreftest"
+ (mt/with-temp [:model/Table {referenced-table-id :id} {:db_id (mt/id)
+ :schema "PUBLIC"
+ :name "referenced_transform_table"
+ :active true}
+ :model/Transform {unreffed-transform-id :id} {:name "Unreferenced Transform - unreftest"
:source {:type :query
:query (lib/query mp products)}
:target {:schema "PUBLIC"
@@ -1378,24 +1382,20 @@
:query (lib/query mp products)}
:target {:schema "PUBLIC"
:name "referenced_transform_table"}}]
- ;; Simulate the referenced transform having been run: activate its provisional target table
+ ;; Simulate the referenced transform having been run: link it to its target table
;; so the table→transform dep is visible in the dependency graph (which filters active=true).
- (t2/update! :model/Table {:db_id (mt/id) :schema "PUBLIC" :name "referenced_transform_table"} {:active true})
+ (t2/update! :model/Transform referenced-transform-id {:target_table_id referenced-table-id})
(events/publish-event! :event/transform-run-complete
{:object {:db-id (mt/id)
:output-schema "PUBLIC"
:output-table "referenced_transform_table"
:transform-id referenced-transform-id}})
(deps.test/synchronously-run-backfill!)
- (try
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]
- (is (=? {:data [{:id unreffed-transform-id
- :type "transform"
- :data {:name "Unreferenced Transform - unreftest"}}]}
- response)))
- (finally
- ;; Clean up provisional table rows created by define-before-insert
- (t2/delete! :model/Table :db_id (mt/id) :name [:in ["referenced_transform_table" "unreferenced_transform_table"]]))))))))
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]
+ (is (=? {:data [{:id unreffed-transform-id
+ :type "transform"
+ :data {:name "Unreferenced Transform - unreftest"}}]}
+ response))))))))
(deftest ^:sequential unreferenced-snippets-test
(testing "GET /api/ee/dependencies/unreferenced - only unreferenced snippets are returned"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index 27fc7016d87..62c7cbe7ae4 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -339,20 +339,19 @@
(run-with-dependencies-setup!
(fn [_]
(let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
- (mt/with-temp [:model/Transform {transform-id :id} {:target target}]
- ;; Transform after-insert creates the target table row via upsert-transform-target-table!
- (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
+ (mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
+ :model/Transform {transform-id :id} {:target target}]
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))
(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test
(testing "python transform update events handles downstream dependencies"
@@ -368,42 +367,41 @@
:body
"import pandas as pd\n\ndef transform(orders):\n return orders"}
target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target
- :source source}
- ;; test_table is created by the Transform after-insert via upsert-transform-target-table!
- :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]
- (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]
- (testing "initial run"
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "keeping target"
+ (mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}
+ :model/Transform {transform-id :id :as transform} {:target target
+ :source source}]
+ (testing "initial run"
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "keeping target"
+ (events/publish-event! :event/update-transform
+ {:object transform
+ :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "changing target update"
+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
+ (let [updated (t2/select-one :model/Transform :id transform-id)]
(events/publish-event! :event/update-transform
- {:object transform
+ {:object updated
:user-id api/*current-user-id*})
(deps.test/synchronously-run-backfill!)
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "changing target update"
- (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
- (let [updated (t2/select-one :model/Transform :id transform-id)]
- (events/publish-event! :event/update-transform
- {:object updated
- :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test
(testing "query transform update events handles downstream dependencies"
@@ -412,41 +410,40 @@
(let [source {:query (lib/native-query mp "select * from orders")
:type :query}
target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target :source source}
- ;; test_table is created by the Transform after-insert via upsert-transform-target-table!
- :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]
- (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]
- (testing "initial run"
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "keeping target"
+ (mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}
+ :model/Transform {transform-id :id :as transform} {:target target :source source}]
+ (testing "initial run"
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "keeping target"
+ (events/publish-event! :event/update-transform
+ {:object transform
+ :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "changing target update"
+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
+ (let [updated (t2/select-one :model/Transform :id transform-id)]
(events/publish-event! :event/update-transform
- {:object transform
+ {:object updated
:user-id api/*current-user-id*})
(deps.test/synchronously-run-backfill!)
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "changing target update"
- (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
- (let [updated (t2/select-one :model/Transform :id transform-id)]
- (events/publish-event! :event/update-transform
- {:object updated
- :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
(deftest segment-update-sets-correct-dependencies
(run-with-dependencies-setup!
diff --git a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj
index dc137d77170..9fc89743636 100644
--- a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj
@@ -5,8 +5,7 @@
[metabase.permissions.core :as perms]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
- [metabase.util.i18n :refer [tru]]
- [metabase.warehouse-schema.models.table :as table]))
+ [metabase.util.i18n :refer [tru]]))
(use-fixtures :once (fixtures/initialize :db))
@@ -209,8 +208,6 @@
:dataset_query (mt/native-query {:query "SELECT * FROM checkins WHERE date > '2014-01-01'"})}]
(let [regular-user-id (mt/user->id :rasta)]
(perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)
- ;; GC leaked provisional table rows so they don't appear in blocked-tables
- (table/gc-transform-target-tables!)
(let [result (permission-debug.impl/debug-permissions
{:user-id regular-user-id
:model-id (str (:id card))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index 3b86b60115a..13fe2c43c82 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -11,6 +11,9 @@
[metabase-enterprise.serialization.v2.load :as serdes.load]
[metabase-enterprise.serialization.v2.storage :as storage]
[metabase-enterprise.serialization.v2.storage.files :as storage.files]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
[metabase.settings.core :as setting]
@@ -1015,3 +1018,23 @@
"ingestion should succeed")
(is (t2/exists? :model/Collection :entity_id coll-eid)
"collection should have been imported from old-format path"))))))))
+
+(deftest query-with-missing-table-and-field-test
+ (testing "An exported query whose table/field have been deleted re-imports by synthesizing inactive rows"
+ (mt/with-temp
+ [:model/Database {db-id :id} {}
+ :model/Table {table-id :id, table-name :name, table-schema :schema} {:db_id db-id}
+ :model/Field {field-id :id, field-name :name} {:table_id table-id :base_type :type/Text}]
+ (let [mp (lib-be/application-database-metadata-provider db-id)
+ query (-> (lib/query mp (lib.metadata/table mp table-id))
+ (lib/with-fields [(lib.metadata/field mp field-id)]))
+ exported (serdes/export-mbql query)
+ _ (t2/delete! :model/Field :id field-id)
+ _ (t2/delete! :model/Table :id table-id)
+ imported (serdes/import-mbql exported)
+ table (t2/select-one :model/Table :db_id db-id :schema table-schema :name table-name)
+ field (t2/select-one :model/Field :table_id (:id table) :name field-name)]
+ (is (=? {:db_id db-id :schema table-schema :name table-name :active false} table))
+ (is (=? {:table_id (:id table) :name field-name :active false} field))
+ (is (= (:id table) (lib/primary-source-table-id imported)))
+ (is (=? [[:field {} (:id field)]] (lib/fields imported)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index 9ddc219af9d..bf762a82f91 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -5,7 +5,6 @@
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.ordering :as ordering]
[metabase.transforms.test-util :as transforms.tu]
- [metabase.warehouse-schema.models.table :as ws.table]
[toucan2.core :as t2]))
(defn- sql-tx [query & [name schema]]
@@ -48,11 +47,11 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(testing "Python transforms are ordered correctly based on source-tables dependencies"
(let [schema (transforms.tu/default-schema-or-public)]
- (ws.table/gc-transform-target-tables!)
(mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}
:model/Field _ {:table_id table1 :name "foo"}
:model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
:model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)] "output_2")]
+ (t2/update! :model/Transform t1 {:target_table_id table1})
(is (= {t1 #{}
t2 #{t1}}
(:dependencies (ordering/transform-ordering #{t1 t2} (t2/select :model/Transform :id [:in [t1 t2]]))))))))))
@@ -61,7 +60,6 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(testing "Python transforms with multiple table dependencies"
(let [schema (transforms.tu/default-schema-or-public)]
- (ws.table/gc-transform-target-tables!)
(mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}
:model/Field _ {:table_id table1 :name "foo"}
:model/Table {table2 :id} {:schema schema :name "output_2"}
@@ -69,7 +67,10 @@
:model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
:model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2")
:model/Transform {t3 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)
- (transforms.tu/source-table-entry "output_2" table2)] "final_output")]
+ (transforms.tu/source-table-entry "output_2" table2)]
+ "final_output")]
+ (t2/update! :model/Transform t1 {:target_table_id table1})
+ (t2/update! :model/Transform t2 {:target_table_id table2})
(is (= {t1 #{}
t2 #{}
t3 #{t1 t2}}
@@ -79,7 +80,6 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python :transforms/table)
(testing "Python and query transforms are ordered together correctly"
(let [schema (transforms.tu/default-schema-or-public)]
- (ws.table/gc-transform-target-tables!)
(mt/with-temp [:model/Table {table1 :id} {:schema schema :name "sql_output"}
:model/Field _ {:table_id table1 :name "foo"}
:model/Table {table2 :id} {:schema schema :name "python_output"}
@@ -95,6 +95,8 @@
:model/Transform {t3 :id} (sql-tx {:database (mt/id)
:type "query"
:query {:source-table table2}} "final_output")]
+ (t2/update! :model/Transform t1 {:target_table_id table1})
+ (t2/update! :model/Transform t2 {:target_table_id table2})
(is (= {t1 #{}
t2 #{t1}
t3 #{t2}}
diff --git a/src/metabase/models/serialization/resolve/db.clj b/src/metabase/models/serialization/resolve/db.clj
index 89f52e575fd..9908dd9328d 100644
--- a/src/metabase/models/serialization/resolve/db.clj
+++ b/src/metabase/models/serialization/resolve/db.clj
@@ -84,25 +84,59 @@
;; Need to break a circular dependency here.
(:id ((clojure.core/resolve 'metabase.users.models.user/serdes-synthesize-user!) {:email email :is_active false})))))
+(defn- synthesize-table!
+ "Creates a new inactive Table for a deserialized reference whose `[db-name schema table-name]`
+ triple doesn't match any existing row. Returns the new table id."
+ [db-id schema table-name]
+ (:id (t2/insert-returning-instance! :model/Table
+ {:db_id db-id
+ :schema schema
+ :name table-name
+ :active false})))
+
+(defn- synthesize-field!
+ "Walks a field path from top-level to deepest, returning each existing field id and creating any
+ missing ones as inactive Fields. Returns the leaf field id."
+ [table-id field-names]
+ (loop [parent-id nil
+ remaining field-names]
+ (if-let [field-name (first remaining)]
+ (let [field-id (or (t2/select-one-pk :model/Field
+ :table_id table-id
+ :name field-name
+ :parent_id parent-id)
+ (t2/insert-returning-pk! :model/Field
+ {:table_id table-id
+ :parent_id parent-id
+ :name field-name
+ :active false
+ :base_type :type/*
+ :database_type "NULL"}))]
+ (recur field-id (rest remaining)))
+ parent-id)))
+
(defn import-table-fk
- "Given [db-name schema table-name], return numeric table_id."
+ "Given [db-name schema table-name], return numeric table_id. If the database exists but the table
+ doesn't, synthesize an inactive Table from the path so we can still resolve the reference."
[[db-name schema table-name :as table-id]]
(when table-id
(if-let [db-id (t2/select-one-fn :id :model/Database :name db-name)]
(or (t2/select-one-fn :id :model/Table :name table-name :schema schema :db_id db-id)
- (throw (ex-info (format "table id present, but no table found: %s" table-id)
- {:table-id table-id})))
+ (synthesize-table! db-id schema table-name))
(throw (ex-info (format "table id present, but database not found: %s" table-id)
{:table-id table-id
:database-names (sort (t2/select-fn-vec :name :model/Table))})))))
(defn import-field-fk
- "Given [db-name schema table-name field-name ...], return numeric field_id."
+ "Given [db-name schema table-name field-name ...], return numeric field_id. If part of the parent
+ chain is missing, synthesize inactive Fields for the missing nodes so we can still resolve the
+ reference."
[resolver [db-name schema table-name & fields :as field-id]]
(when field-id
(let [table-id (resolve/import-table-fk resolver [db-name schema table-name])
field-q (serdes/recursively-find-field-q table-id (reverse fields))]
- (t2/select-one-pk :model/Field field-q))))
+ (or (t2/select-one-pk :model/Field field-q)
+ (synthesize-field! table-id fields)))))
;;; ============================================================
;;; Resolver constructors
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index e1aa1b271c3..87da36fd72a 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -20,7 +20,6 @@
[metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.log :as log]
- [metabase.warehouse-schema.models.table :as ws.table]
[methodical.core :as methodical]
[toucan2.core :as t2]
[toucan2.instance :as t2.instance]))
@@ -104,22 +103,11 @@
(when collection_id
(collection/check-allowed-content :model/Transform collection_id))
(let [target-db-id (transforms-base.i/target-db-id transform)
- ;; This is defensive code to cope with some tests for remote sync, where we deserialize a transform
- ;; with a concrete database id within it, for a potentially non-existent database.
- ;; In practice, our serialized representation should not contain any database ids, and this should
- ;; not be required.
- ;; TODO (Chris 2026-02-02) -- Update tests so this workaround is unnecessary.
- valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))
- target-name (get-in transform [:target :name])
- table-id (when (and valid-db-id? target-name)
- (transforms-base.u/upsert-target-table!
- target-db-id (get-in transform [:target :schema]) target-name))]
+ valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))]
(when-not valid-db-id?
(log/warnf "Invalid target database id (%s) ignored for new transform (%s)" target-db-id (:name transform)))
(-> transform
(assoc-in [:target :database] target-db-id)
- (cond-> table-id (-> (assoc-in [:target :table_id] table-id)
- (assoc :target_table_id table-id)))
(assoc
:source_type (transforms-base.u/transform-source-type source)
:target_db_id (when valid-db-id? target-db-id)
@@ -136,11 +124,7 @@
target-db-id (when target-changed?
;; No database existence check added here, unlike for insert.
;; Just allow updates for an invalid target to fail.
- (transforms-base.i/target-db-id transform))
- target-name (get-in transform [:target :name])
- table-id (when (and target-db-id target-name)
- (transforms-base.u/upsert-target-table!
- target-db-id (get-in transform [:target :schema]) target-name))]
+ (transforms-base.i/target-db-id transform))]
(cond-> transform
source
(assoc :source_type (transforms-base.u/transform-source-type source)
@@ -149,10 +133,6 @@
target-changed?
(assoc :target_db_id target-db-id)
- table-id
- (-> (assoc-in [:target :table_id] table-id)
- (assoc :target_table_id table-id))
-
;; Reset checkpoint when the incremental filter field changes
(let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])
new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])]
@@ -260,7 +240,6 @@
transform)
(t2/define-before-delete :model/Transform [transform]
- (ws.table/delete-orphaned-provisional-table! (:target_table_id transform) (:id transform))
(when-not mi/*deserializing?*
(events/publish-event! :event/delete-transform {:id (:id transform)}))
(search.core/delete! :model/Transform [(str (:id transform))])
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 2d594622490..6590415d4e6 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -30,7 +30,6 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.warehouse-schema.models.table :as table]
[toucan2.core :as t2])
(:import
(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -588,12 +587,10 @@
(and (:table_id entry) (not (:table entry)))
(merge (int-id->metadata (:table_id entry)) entry)
- ;; Has table metadata but no table_id — look it up, upsert transform target if not found
+ ;; Has table metadata but no table_id — look it up. Leaves :table_id nil if the
+ ;; referenced table doesn't exist yet; resolved later at execute time.
(missing-table-id? entry)
- (assoc entry :table_id (or (ref-lookup (source-table-ref->key entry))
- (when (and (:database_id entry) (:table entry))
- (table/upsert-transform-target-table!
- (:database_id entry) (:schema entry) (:table entry)))))
+ (assoc entry :table_id (ref-lookup (source-table-ref->key entry)))
;; Already fully populated
:else entry))
@@ -708,16 +705,6 @@
;;; ------------------------------------------------- Misc -------------------------------------------------
-(defn upsert-target-table!
- "Upsert a provisional table entry for a transform's target, creating it if it doesn't exist.
- Returns the table ID.
-
- Thin wrapper around [[metabase.warehouse-schema.models.table/upsert-transform-target-table!]] —
- exists because the `models` module cannot depend on `warehouse-schema` directly, but can
- depend on `transforms-base` (which is allowed to use `warehouse-schema`)."
- [db-id schema table-name]
- (table/upsert-transform-target-table! db-id schema table-name))
-
(defn is-temp-transform-table?
"Return true when `table` matches the transform temporary table naming pattern."
[table]
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 5703139b539..1abdbb08607 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -399,79 +399,6 @@
[_table]
[:schema :name (serdes/hydrated-hash :db :db_id)])
-;;; ----------------------------------------- Transform Target Tables -----------------------------------------------
-
-(defn upsert-transform-target-table!
- "Ensure a metabase_table row exists for the given (db_id, schema, name) triple.
- If the table doesn't exist, creates it as a transform target (inactive, not yet physically materialized).
- If it already exists (active or inactive), returns its id without modification.
- Returns the table id.
-
- The `transform_target` column is **conservative**: it is set eagerly to `true` when a transform is
- configured to target this table (synchronous, never stale), and cleared asynchronously when no
- transforms reference it anymore. This means the column may have false positives (a table marked as
- a target when no transform actually targets it) but **never false negatives** (a targeted table
- will always have `transform_target = true`)."
- [db-id schema table-name]
- (app-db/update-or-insert!
- :model/Table
- {:db_id db-id :schema schema :name table-name}
- (fn [existing]
- (when-not existing
- {:display_name (humanization/name->human-readable-name table-name)
- :active false
- :transform_target true
- :data_source :metabase-transform
- :data_authority :computed
- :initial_sync_status "complete"}))))
-
-(defn delete-orphaned-provisional-table!
- "If `table-id` points at an inactive provisional table that is not referenced by any other
- transform (excluding `exclude-transform-id`), delete it."
- [table-id exclude-transform-id]
- (when table-id
- (when-let [table (t2/select-one :model/Table :id table-id
- :active false :transform_target true :deactivated_at nil
- :data_source :metabase-transform)]
- (let [referenced?
- (seq
- (t2/query {:select [[[:inline 1] :ref]]
- :from [:transform]
- :where [:and
- [:= :target_table_id (:id table)]
- [:not= :id exclude-transform-id]]}))]
- (when-not referenced?
- (t2/delete! :model/Table :id (:id table)))))))
-
-(defn gc-transform-target-tables!
- "Deletes provisional table rows (created by [[upsert-transform-target-table!]]) that are no longer
- referenced by any Transform. Safe because these rows were never active,
- so they have no child records.
-
- Note: this only handles *inactive* provisional tables. Active tables that were previously
- targeted by a transform retain `transform_target = true` even after the transform is deleted.
- A separate process to reset `transform_target` on active, unreferenced tables is not yet
- implemented.
-
- Uses FK-based NOT IN queries rather than scanning JSON columns."
- []
- (let [candidate-ids (t2/select-fn-set :id :model/Table
- :active false :transform_target true :deactivated_at nil
- :data_source :metabase-transform)]
- (when (seq candidate-ids)
- (let [referenced-ids
- (into #{}
- (map :id)
- (t2/query {:select [[:target_table_id :id]]
- :from [:transform]
- :where [:and
- [:not= :target_table_id nil]
- [:in :target_table_id candidate-ids]]}))
- dead-ids (into [] (remove referenced-ids) candidate-ids)]
- (when (seq dead-ids)
- (log/infof "Deleting %d orphaned transform target table(s)" (count dead-ids))
- (t2/delete! :model/Table :id [:in dead-ids]))))))
-
;;; ------------------------------------------------ Field ordering -------------------------------------------------
(def field-order-rule
diff --git a/test/metabase/models/serialization/resolve/db_test.clj b/test/metabase/models/serialization/resolve/db_test.clj
new file mode 100644
index 00000000000..9f278c4ea4a
--- /dev/null
+++ b/test/metabase/models/serialization/resolve/db_test.clj
@@ -0,0 +1,90 @@
+(ns metabase.models.serialization.resolve.db-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.models.serialization :as serdes]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(deftest import-table-fk-existing-test
+ (testing "returns the existing table id when found"
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:db_id db-id :name "users" :schema "public"}]
+ (is (= table-id (serdes/*import-table-fk* ["test-db" "public" "users"]))))))
+
+(deftest import-table-fk-synthesizes-inactive-table-test
+ (testing "creates an inactive table when the database exists but the table doesn't"
+ (mt/with-model-cleanup [:model/Table]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}]
+ (let [table-id (serdes/*import-table-fk* ["test-db" "public" "missing"])]
+ (is (=? {:id table-id
+ :db_id db-id
+ :schema "public"
+ :name "missing"
+ :active false}
+ (t2/select-one :model/Table :id table-id))))))))
+
+(deftest import-table-fk-no-schema-test
+ (testing "synthesizes a table when schema is nil"
+ (mt/with-model-cleanup [:model/Table]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}]
+ (let [table-id (serdes/*import-table-fk* ["test-db" nil "schemaless"])]
+ (is (=? {:id table-id
+ :db_id db-id
+ :schema nil
+ :name "schemaless"
+ :active false}
+ (t2/select-one :model/Table :id table-id))))))))
+
+(deftest import-table-fk-throws-when-database-missing-test
+ (testing "throws when the database itself does not exist"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"database not found"
+ (serdes/*import-table-fk* ["does-not-exist" "public" "users"])))))
+
+(deftest import-field-fk-existing-test
+ (testing "returns the existing field id when found"
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:db_id db-id :name "users" :schema "public"}
+ :model/Field {field-id :id} {:table_id table-id :name "email"}]
+ (is (= field-id
+ (serdes/*import-field-fk* ["test-db" "public" "users" "email"]))))))
+
+(deftest import-field-fk-synthesizes-inactive-field-test
+ (testing "creates an inactive field when the table exists but the field doesn't"
+ (mt/with-model-cleanup [:model/Field]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:db_id db-id :name "users" :schema "public"}]
+ (let [field-id (serdes/*import-field-fk* ["test-db" "public" "users" "missing"])]
+ (is (=? {:id field-id
+ :table_id table-id
+ :parent_id nil
+ :name "missing"
+ :active false}
+ (t2/select-one :model/Field :id field-id))))))))
+
+(deftest import-field-fk-synthesizes-nested-chain-test
+ (testing "creates inactive parent fields along the chain when missing"
+ (mt/with-model-cleanup [:model/Field]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:db_id db-id :name "events" :schema "public"}]
+ (let [field-id (serdes/*import-field-fk* ["test-db" "public" "events" "outer" "middle" "inner"])
+ inner (t2/select-one :model/Field :id field-id)
+ middle (t2/select-one :model/Field :id (:parent_id inner))
+ outer (t2/select-one :model/Field :id (:parent_id middle))]
+ (is (=? {:name "inner" :active false} inner))
+ (is (=? {:name "middle" :active false :parent_id (:id outer)} middle))
+ (is (=? {:name "outer" :active false :parent_id nil :table_id table-id} outer)))))))
+
+(deftest import-field-fk-reuses-existing-parent-test
+ (testing "uses an existing parent field rather than creating a duplicate"
+ (mt/with-model-cleanup [:model/Field]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Table {table-id :id} {:db_id db-id :name "events" :schema "public"}
+ :model/Field {parent-id :id} {:table_id table-id :name "outer"}]
+ (let [field-id (serdes/*import-field-fk* ["test-db" "public" "events" "outer" "inner"])]
+ (is (=? {:id field-id
+ :name "inner"
+ :parent_id parent-id
+ :active false}
+ (t2/select-one :model/Field :id field-id)))
+ (is (= 1 (t2/count :model/Field :table_id table-id :name "outer"))))))))
diff --git a/test/metabase/transforms/models_test.clj b/test/metabase/transforms/models_test.clj
index 816aafd5ab3..f9a92957b18 100644
--- a/test/metabase/transforms/models_test.clj
+++ b/test/metabase/transforms/models_test.clj
@@ -197,35 +197,39 @@
:schema nil
:name "test_table_null_schema"
:db_id db-id}}]
-
- (testing "Hydrates table with non-NULL schema"
- (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]
- (is (some? (:table hydrated))
- "Transform should have hydrated table")
- (is (= table-with-schema-id (-> hydrated :table :id))
- "Should hydrate correct table with schema")))
-
- (testing "Hydrates table with NULL schema"
- (let [hydrated (t2/hydrate transform2 :table-with-db-and-fields)]
- (is (some? (:table hydrated))
- "Transform with NULL schema should have hydrated table")
- (is (= table-null-schema-id (-> hydrated :table :id))
- "Should hydrate correct table with NULL schema")))
-
- (testing "Hydrates multiple transforms with mixed schemas in single batch"
- (let [hydrated (t2/hydrate [transform1 transform2] :table-with-db-and-fields)]
- (is (= 2 (count hydrated))
- "Should hydrate both transforms")
- (is (every? (comp some? :table) hydrated)
- "Both transforms should have hydrated tables")
- (is (= #{table-with-schema-id table-null-schema-id}
- (set (map (comp :id :table) hydrated)))
- "Should hydrate both tables correctly")))
-
- (testing "Does not hydrate unrelated tables"
- (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]
- (is (not= other-table-id (-> hydrated :table :id))
- "Should not hydrate unrelated table"))))))
+ (t2/update! :model/Transform (:id transform1) {:target_table_id table-with-schema-id})
+ (t2/update! :model/Transform (:id transform2) {:target_table_id table-null-schema-id})
+ (let [transform1 (t2/select-one :model/Transform :id (:id transform1))
+ transform2 (t2/select-one :model/Transform :id (:id transform2))]
+
+ (testing "Hydrates table with non-NULL schema"
+ (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]
+ (is (some? (:table hydrated))
+ "Transform should have hydrated table")
+ (is (= table-with-schema-id (-> hydrated :table :id))
+ "Should hydrate correct table with schema")))
+
+ (testing "Hydrates table with NULL schema"
+ (let [hydrated (t2/hydrate transform2 :table-with-db-and-fields)]
+ (is (some? (:table hydrated))
+ "Transform with NULL schema should have hydrated table")
+ (is (= table-null-schema-id (-> hydrated :table :id))
+ "Should hydrate correct table with NULL schema")))
+
+ (testing "Hydrates multiple transforms with mixed schemas in single batch"
+ (let [hydrated (t2/hydrate [transform1 transform2] :table-with-db-and-fields)]
+ (is (= 2 (count hydrated))
+ "Should hydrate both transforms")
+ (is (every? (comp some? :table) hydrated)
+ "Both transforms should have hydrated tables")
+ (is (= #{table-with-schema-id table-null-schema-id}
+ (set (map (comp :id :table) hydrated)))
+ "Should hydrate both tables correctly")))
+
+ (testing "Does not hydrate unrelated tables"
+ (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]
+ (is (not= other-table-id (-> hydrated :table :id))
+ "Should not hydrate unrelated table")))))))
(deftest creator-hydration-test
(testing "Transform creator hydration returns user details"
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index 2bd6f5a9e19..464d2802fe5 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -210,10 +210,10 @@
result (transforms-base.u/normalize-source-tables source-tables)]
(is (= 999 (:table_id (first result))))))
- (testing "creates transform target table for non-existent table ref"
+ (testing "leaves :table_id nil for non-existent table ref"
(let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]
result (transforms-base.u/normalize-source-tables source-tables)]
- (is (int? (:table_id (first result))))))
+ (is (nil? (:table_id (first result))))))
(testing "handles entries needing different kinds of enrichment"
(let [source-tables [{:alias "t1" :table_id (:id t1)}
@@ -368,26 +368,26 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms-basic}
(let [target {:type "table" :schema nil :name "test_output_table"}]
- ;; The Transform after-insert hook creates a provisional table for the target,
- ;; so we don't need to create one explicitly.
- (mt/with-temp [:model/Transform {transform-id :id :as transform}
+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id)
+ :schema nil
+ :name "test_output_table"}
+ :model/Transform {transform-id :id :as transform}
{:target target
:source {:type "query"
:query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}]
- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_output_table" :schema nil)]
- ;; Mock execute-base! to return success without actually running a query,
- ;; run-cancelable-transform! to bypass schema creation / cancellation infra,
- ;; and sync-target! to skip driver calls but still return the provisional table
- ;; so complete-execution! can set transform_id on it.
- (mt/with-dynamic-fn-redefs
- [transforms-base.i/execute-base! (constantly {:status :succeeded})
- transforms-base.u/sync-target! (fn [_target _database]
- (t2/select-one :model/Table table-id))
- transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]
- (run-fn (a/promise-chan) nil))]
- (transforms.execute/execute! transform {:run-method :manual})
- (is (= transform-id
- (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))))
+ ;; Mock execute-base! to return success without actually running a query,
+ ;; run-cancelable-transform! to bypass schema creation / cancellation infra,
+ ;; and sync-target! to skip driver calls but still return the target table
+ ;; so complete-execution! can set transform_id on it.
+ (mt/with-dynamic-fn-redefs
+ [transforms-base.i/execute-base! (constantly {:status :succeeded})
+ transforms-base.u/sync-target! (fn [_target _database]
+ (t2/select-one :model/Table table-id))
+ transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]
+ (run-fn (a/promise-chan) nil))]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (is (= transform-id
+ (t2/select-one-fn :transform_id :model/Table :id table-id))))))))))
(deftest transform-hydration-test
(testing "hydrating :transform on a table"
diff --git a/test/metabase/transforms_base/ordering_test.clj b/test/metabase/transforms_base/ordering_test.clj
index 0aa51fcdc16..e49c5c1dbe3 100644
--- a/test/metabase/transforms_base/ordering_test.clj
+++ b/test/metabase/transforms_base/ordering_test.clj
@@ -42,11 +42,12 @@
:name "orders_2"}
:model/Field _ {:table_id table
:name "foo"}
- :model/Transform {parent :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders)}}
- "orders_2")
+ :model/Transform {parent :id} (-> (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table (mt/id :orders)}}
+ "orders_2")
+ (assoc :target_table_id table))
:model/Transform {child :id} (make-transform
{:database (mt/id)
:type "query"
@@ -299,11 +300,12 @@
:name "table_1"}
:model/Field _ {:table_id table1
:name "foo"}
- :model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
- "table_1")]
+ :model/Transform {t1 :id} (-> (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table table1}}
+ "table_1")
+ (assoc :target_table_id table1))]
(is (= {:cycle-str "transform_table_1"
:cycle [t1]}
(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
@@ -314,16 +316,18 @@
:model/Table {table2 :id} {:schema (default-schema-or-public), :name "table_2"}
:model/Field _ {:table_id table2
:name "foo"}
- :model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
- "table_2")
- :model/Transform {t2 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table2}}
- "table_1")]
+ :model/Transform {t1 :id} (-> (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table table1}}
+ "table_2")
+ (assoc :target_table_id table2))
+ :model/Transform {t2 :id} (-> (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table table2}}
+ "table_1")
+ (assoc :target_table_id table1))]
(is (= {:cycle-str "transform_table_2 -> transform_table_1",
:cycle [t1 t2]}
(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
@@ -340,21 +344,24 @@
:name "table_3"}
:model/Field _ {:table_id table3
:name "foo"}
- :model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table1}}
- "table_2")
- :model/Transform {t2 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table2}}
- "table_3")
- :model/Transform {t3 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table table3}}
- "table_1")]
+ :model/Transform {t1 :id} (-> (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table table1}}
+ "table_2")
+ (assoc :target_table_id table2))
+ :model/Transform {t2 :id} (-> (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table table2}}
+ "table_3")
+ (assoc :target_table_id table3))
+ :model/Transform {t3 :id} (-> (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table table3}}
+ "table_1")
+ (assoc :target_table_id table1))]
(is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
:cycle [t1 t3 t2]}
(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index a5f1b882223..786ac947b27 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -626,7 +626,8 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-temp [:model/Table {table :id} {:schema "public", :name "orders_2"}
:model/Field _ {:table_id table, :name "foo"}
- :model/Transform parent (->transform "transform1" (mt/mbql-query orders))
+ :model/Transform parent (-> (->transform "transform1" (mt/mbql-query orders))
+ (assoc :target_table_id table))
:model/Transform child (-> (->transform "transform2" (mt/mbql-query nil {:source-table table}))
(assoc-in [:target :name] "orders_3"))]
(let [deps-resp (mt/user-http-request :lucky :get 200 (format "transform/%s/dependencies" (:id child)))]