The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Create inactive tables and fields when deserializing non-existent references (#73741)
Alexander Polyankinff8dacfc60a0
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 2ffe8d5049c..c4a8198bacd 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2278,8 +2278,7 @@task-historytracingtransforms-base- util- warehouse-schema}+ util}:friends #{modelsenterprise/transforms-python}:model-exports #{:model/Transform@@ -2312,8 +2311,7 @@lib-bequery-processorsync- util- warehouse-schema}+ util}:friends #{transformsenterprise/transforms-python}}diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.cljindex e4ee8ed6128..2c42a85e106 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj@@ -1368,7 +1368,11 @@(mt/with-premium-features #{:dependencies}(let [mp (mt/metadata-provider)products (lib.metadata/table mp (mt/id :products))]- (mt/with-temp [:model/Transform {unreffed-transform-id :id} {:name "Unreferenced Transform - unreftest"+ (mt/with-temp [:model/Table {referenced-table-id :id} {:db_id (mt/id)+ :schema "PUBLIC"+ :name "referenced_transform_table"+ :active true}+ :model/Transform {unreffed-transform-id :id} {:name "Unreferenced Transform - unreftest":source {:type :query:query (lib/query mp products)}:target {:schema "PUBLIC"@@ -1378,24 +1382,20 @@:query (lib/query mp products)}:target {:schema "PUBLIC":name "referenced_transform_table"}}]- ;; Simulate the referenced transform having been run: activate its provisional target table+ ;; Simulate the referenced transform having been run: link it to its target table;; so the table→transform dep is visible in the dependency graph (which filters active=true).- (t2/update! :model/Table {:db_id (mt/id) :schema "PUBLIC" :name "referenced_transform_table"} {:active true})+ (t2/update! :model/Transform referenced-transform-id {:target_table_id referenced-table-id})(events/publish-event! :event/transform-run-complete{:object {:db-id (mt/id):output-schema "PUBLIC":output-table "referenced_transform_table":transform-id referenced-transform-id}})(deps.test/synchronously-run-backfill!)- (try- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]- (is (=? {:data [{:id unreffed-transform-id- :type "transform"- :data {:name "Unreferenced Transform - unreftest"}}]}- response)))- (finally- ;; Clean up provisional table rows created by define-before-insert- (t2/delete! :model/Table :db_id (mt/id) :name [:in ["referenced_transform_table" "unreferenced_transform_table"]]))))))))+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]+ (is (=? {:data [{:id unreffed-transform-id+ :type "transform"+ :data {:name "Unreferenced Transform - unreftest"}}]}+ response))))))))(deftest ^:sequential unreferenced-snippets-test(testing "GET /api/ee/dependencies/unreferenced - only unreferenced snippets are returned"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex 27fc7016d87..62c7cbe7ae4 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -339,20 +339,19 @@(run-with-dependencies-setup!(fn [_](let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]- (mt/with-temp [:model/Transform {transform-id :id} {:target target}]- ;; Transform after-insert creates the target table row via upsert-transform-target-table!- (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))+ (mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}+ :model/Transform {transform-id :id} {:target target}]+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test(testing "python transform update events handles downstream dependencies"@@ -368,42 +367,41 @@:body"import pandas as pd\n\ndef transform(orders):\n return orders"}target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target- :source source}- ;; test_table is created by the Transform after-insert via upsert-transform-target-table!- :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]- (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]- (testing "initial run"- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "keeping target"+ (mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}+ :model/Transform {transform-id :id :as transform} {:target target+ :source source}]+ (testing "initial run"+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "keeping target"+ (events/publish-event! :event/update-transform+ {:object transform+ :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "changing target update"+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})+ (let [updated (t2/select-one :model/Transform :id transform-id)](events/publish-event! :event/update-transform- {:object transform+ {:object updated:user-id api/*current-user-id*})(deps.test/synchronously-run-backfill!)- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "changing target update"- (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})- (let [updated (t2/select-one :model/Transform :id transform-id)]- (events/publish-event! :event/update-transform- {:object updated- :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test(testing "query transform update events handles downstream dependencies"@@ -412,41 +410,40 @@(let [source {:query (lib/native-query mp "select * from orders"):type :query}target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target :source source}- ;; test_table is created by the Transform after-insert via upsert-transform-target-table!- :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]- (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]- (testing "initial run"- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "keeping target"+ (mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}+ :model/Transform {transform-id :id :as transform} {:target target :source source}]+ (testing "initial run"+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "keeping target"+ (events/publish-event! :event/update-transform+ {:object transform+ :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "changing target update"+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})+ (let [updated (t2/select-one :model/Transform :id transform-id)](events/publish-event! :event/update-transform- {:object transform+ {:object updated:user-id api/*current-user-id*})(deps.test/synchronously-run-backfill!)- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "changing target update"- (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})- (let [updated (t2/select-one :model/Transform :id transform-id)]- (events/publish-event! :event/update-transform- {:object updated- :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))(deftest segment-update-sets-correct-dependencies(run-with-dependencies-setup!diff --git a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.cljindex dc137d77170..9fc89743636 100644--- a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj@@ -5,8 +5,7 @@[metabase.permissions.core :as perms][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.util.i18n :refer [tru]]- [metabase.warehouse-schema.models.table :as table]))+ [metabase.util.i18n :refer [tru]]))(use-fixtures :once (fixtures/initialize :db))@@ -209,8 +208,6 @@:dataset_query (mt/native-query {:query "SELECT * FROM checkins WHERE date > '2014-01-01'"})}](let [regular-user-id (mt/user->id :rasta)](perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)- ;; GC leaked provisional table rows so they don't appear in blocked-tables- (table/gc-transform-target-tables!)(let [result (permission-debug.impl/debug-permissions{:user-id regular-user-id:model-id (str (:id card))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex 3b86b60115a..13fe2c43c82 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -11,6 +11,9 @@[metabase-enterprise.serialization.v2.load :as serdes.load][metabase-enterprise.serialization.v2.storage :as storage][metabase-enterprise.serialization.v2.storage.files :as storage.files]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.models.serialization :as serdes][metabase.search.core :as search][metabase.settings.core :as setting]@@ -1015,3 +1018,23 @@"ingestion should succeed")(is (t2/exists? :model/Collection :entity_id coll-eid)"collection should have been imported from old-format path"))))))))++(deftest query-with-missing-table-and-field-test+ (testing "An exported query whose table/field have been deleted re-imports by synthesizing inactive rows"+ (mt/with-temp+ [:model/Database {db-id :id} {}+ :model/Table {table-id :id, table-name :name, table-schema :schema} {:db_id db-id}+ :model/Field {field-id :id, field-name :name} {:table_id table-id :base_type :type/Text}]+ (let [mp (lib-be/application-database-metadata-provider db-id)+ query (-> (lib/query mp (lib.metadata/table mp table-id))+ (lib/with-fields [(lib.metadata/field mp field-id)]))+ exported (serdes/export-mbql query)+ _ (t2/delete! :model/Field :id field-id)+ _ (t2/delete! :model/Table :id table-id)+ imported (serdes/import-mbql exported)+ table (t2/select-one :model/Table :db_id db-id :schema table-schema :name table-name)+ field (t2/select-one :model/Field :table_id (:id table) :name field-name)]+ (is (=? {:db_id db-id :schema table-schema :name table-name :active false} table))+ (is (=? {:table_id (:id table) :name field-name :active false} field))+ (is (= (:id table) (lib/primary-source-table-id imported)))+ (is (=? [[:field {} (:id field)]] (lib/fields imported)))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex 9ddc219af9d..bf762a82f91 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -5,7 +5,6 @@[metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.ordering :as ordering][metabase.transforms.test-util :as transforms.tu]- [metabase.warehouse-schema.models.table :as ws.table][toucan2.core :as t2]))(defn- sql-tx [query & [name schema]]@@ -48,11 +47,11 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)(testing "Python transforms are ordered correctly based on source-tables dependencies"(let [schema (transforms.tu/default-schema-or-public)]- (ws.table/gc-transform-target-tables!)(mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}:model/Field _ {:table_id table1 :name "foo"}:model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1"):model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)] "output_2")]+ (t2/update! :model/Transform t1 {:target_table_id table1})(is (= {t1 #{}t2 #{t1}}(:dependencies (ordering/transform-ordering #{t1 t2} (t2/select :model/Transform :id [:in [t1 t2]]))))))))))@@ -61,7 +60,6 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)(testing "Python transforms with multiple table dependencies"(let [schema (transforms.tu/default-schema-or-public)]- (ws.table/gc-transform-target-tables!)(mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}:model/Field _ {:table_id table1 :name "foo"}:model/Table {table2 :id} {:schema schema :name "output_2"}@@ -69,7 +67,10 @@:model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1"):model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2"):model/Transform {t3 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)- (transforms.tu/source-table-entry "output_2" table2)] "final_output")]+ (transforms.tu/source-table-entry "output_2" table2)]+ "final_output")]+ (t2/update! :model/Transform t1 {:target_table_id table1})+ (t2/update! :model/Transform t2 {:target_table_id table2})(is (= {t1 #{}t2 #{}t3 #{t1 t2}}@@ -79,7 +80,6 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python :transforms/table)(testing "Python and query transforms are ordered together correctly"(let [schema (transforms.tu/default-schema-or-public)]- (ws.table/gc-transform-target-tables!)(mt/with-temp [:model/Table {table1 :id} {:schema schema :name "sql_output"}:model/Field _ {:table_id table1 :name "foo"}:model/Table {table2 :id} {:schema schema :name "python_output"}@@ -95,6 +95,8 @@:model/Transform {t3 :id} (sql-tx {:database (mt/id):type "query":query {:source-table table2}} "final_output")]+ (t2/update! :model/Transform t1 {:target_table_id table1})+ (t2/update! :model/Transform t2 {:target_table_id table2})(is (= {t1 #{}t2 #{t1}t3 #{t2}}diff --git a/src/metabase/models/serialization/resolve/db.clj b/src/metabase/models/serialization/resolve/db.cljindex 89f52e575fd..9908dd9328d 100644--- a/src/metabase/models/serialization/resolve/db.clj+++ b/src/metabase/models/serialization/resolve/db.clj@@ -84,25 +84,59 @@;; Need to break a circular dependency here.(:id ((clojure.core/resolve 'metabase.users.models.user/serdes-synthesize-user!) {:email email :is_active false})))))+(defn- synthesize-table!+ "Creates a new inactive Table for a deserialized reference whose `[db-name schema table-name]`+ triple doesn't match any existing row. Returns the new table id."+ [db-id schema table-name]+ (:id (t2/insert-returning-instance! :model/Table+ {:db_id db-id+ :schema schema+ :name table-name+ :active false})))++(defn- synthesize-field!+ "Walks a field path from top-level to deepest, returning each existing field id and creating any+ missing ones as inactive Fields. Returns the leaf field id."+ [table-id field-names]+ (loop [parent-id nil+ remaining field-names]+ (if-let [field-name (first remaining)]+ (let [field-id (or (t2/select-one-pk :model/Field+ :table_id table-id+ :name field-name+ :parent_id parent-id)+ (t2/insert-returning-pk! :model/Field+ {:table_id table-id+ :parent_id parent-id+ :name field-name+ :active false+ :base_type :type/*+ :database_type "NULL"}))]+ (recur field-id (rest remaining)))+ parent-id)))+(defn import-table-fk- "Given [db-name schema table-name], return numeric table_id."+ "Given [db-name schema table-name], return numeric table_id. If the database exists but the table+ doesn't, synthesize an inactive Table from the path so we can still resolve the reference."[[db-name schema table-name :as table-id]](when table-id(if-let [db-id (t2/select-one-fn :id :model/Database :name db-name)](or (t2/select-one-fn :id :model/Table :name table-name :schema schema :db_id db-id)- (throw (ex-info (format "table id present, but no table found: %s" table-id)- {:table-id table-id})))+ (synthesize-table! db-id schema table-name))(throw (ex-info (format "table id present, but database not found: %s" table-id){:table-id table-id:database-names (sort (t2/select-fn-vec :name :model/Table))})))))(defn import-field-fk- "Given [db-name schema table-name field-name ...], return numeric field_id."+ "Given [db-name schema table-name field-name ...], return numeric field_id. If part of the parent+ chain is missing, synthesize inactive Fields for the missing nodes so we can still resolve the+ reference."[resolver [db-name schema table-name & fields :as field-id]](when field-id(let [table-id (resolve/import-table-fk resolver [db-name schema table-name])field-q (serdes/recursively-find-field-q table-id (reverse fields))]- (t2/select-one-pk :model/Field field-q))))+ (or (t2/select-one-pk :model/Field field-q)+ (synthesize-field! table-id fields)))));;; ============================================================;;; Resolver constructorsdiff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex e1aa1b271c3..87da36fd72a 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -20,7 +20,6 @@[metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.log :as log]- [metabase.warehouse-schema.models.table :as ws.table][methodical.core :as methodical][toucan2.core :as t2][toucan2.instance :as t2.instance]))@@ -104,22 +103,11 @@(when collection_id(collection/check-allowed-content :model/Transform collection_id))(let [target-db-id (transforms-base.i/target-db-id transform)- ;; This is defensive code to cope with some tests for remote sync, where we deserialize a transform- ;; with a concrete database id within it, for a potentially non-existent database.- ;; In practice, our serialized representation should not contain any database ids, and this should- ;; not be required.- ;; TODO (Chris 2026-02-02) -- Update tests so this workaround is unnecessary.- valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))- target-name (get-in transform [:target :name])- table-id (when (and valid-db-id? target-name)- (transforms-base.u/upsert-target-table!- target-db-id (get-in transform [:target :schema]) target-name))]+ valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))](when-not valid-db-id?(log/warnf "Invalid target database id (%s) ignored for new transform (%s)" target-db-id (:name transform)))(-> transform(assoc-in [:target :database] target-db-id)- (cond-> table-id (-> (assoc-in [:target :table_id] table-id)- (assoc :target_table_id table-id)))(assoc:source_type (transforms-base.u/transform-source-type source):target_db_id (when valid-db-id? target-db-id)@@ -136,11 +124,7 @@target-db-id (when target-changed?;; No database existence check added here, unlike for insert.;; Just allow updates for an invalid target to fail.- (transforms-base.i/target-db-id transform))- target-name (get-in transform [:target :name])- table-id (when (and target-db-id target-name)- (transforms-base.u/upsert-target-table!- target-db-id (get-in transform [:target :schema]) target-name))]+ (transforms-base.i/target-db-id transform))](cond-> transformsource(assoc :source_type (transforms-base.u/transform-source-type source)@@ -149,10 +133,6 @@target-changed?(assoc :target_db_id target-db-id)- table-id- (-> (assoc-in [:target :table_id] table-id)- (assoc :target_table_id table-id))-;; Reset checkpoint when the incremental filter field changes(let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])]@@ -260,7 +240,6 @@transform)(t2/define-before-delete :model/Transform [transform]- (ws.table/delete-orphaned-provisional-table! (:target_table_id transform) (:id transform))(when-not mi/*deserializing?*(events/publish-event! :event/delete-transform {:id (:id transform)}))(search.core/delete! :model/Transform [(str (:id transform))])diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 2d594622490..6590415d4e6 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -30,7 +30,6 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.warehouse-schema.models.table :as table][toucan2.core :as t2])(:import(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)@@ -588,12 +587,10 @@(and (:table_id entry) (not (:table entry)))(merge (int-id->metadata (:table_id entry)) entry)- ;; Has table metadata but no table_id — look it up, upsert transform target if not found+ ;; Has table metadata but no table_id — look it up. Leaves :table_id nil if the+ ;; referenced table doesn't exist yet; resolved later at execute time.(missing-table-id? entry)- (assoc entry :table_id (or (ref-lookup (source-table-ref->key entry))- (when (and (:database_id entry) (:table entry))- (table/upsert-transform-target-table!- (:database_id entry) (:schema entry) (:table entry)))))+ (assoc entry :table_id (ref-lookup (source-table-ref->key entry)));; Already fully populated:else entry))@@ -708,16 +705,6 @@;;; ------------------------------------------------- Misc --------------------------------------------------(defn upsert-target-table!- "Upsert a provisional table entry for a transform's target, creating it if it doesn't exist.- Returns the table ID.-- Thin wrapper around [[metabase.warehouse-schema.models.table/upsert-transform-target-table!]] —- exists because the `models` module cannot depend on `warehouse-schema` directly, but can- depend on `transforms-base` (which is allowed to use `warehouse-schema`)."- [db-id schema table-name]- (table/upsert-transform-target-table! db-id schema table-name))-(defn is-temp-transform-table?"Return true when `table` matches the transform temporary table naming pattern."[table]diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 5703139b539..1abdbb08607 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -399,79 +399,6 @@[_table][:schema :name (serdes/hydrated-hash :db :db_id)])-;;; ----------------------------------------- Transform Target Tables -------------------------------------------------(defn upsert-transform-target-table!- "Ensure a metabase_table row exists for the given (db_id, schema, name) triple.- If the table doesn't exist, creates it as a transform target (inactive, not yet physically materialized).- If it already exists (active or inactive), returns its id without modification.- Returns the table id.-- The `transform_target` column is **conservative**: it is set eagerly to `true` when a transform is- configured to target this table (synchronous, never stale), and cleared asynchronously when no- transforms reference it anymore. This means the column may have false positives (a table marked as- a target when no transform actually targets it) but **never false negatives** (a targeted table- will always have `transform_target = true`)."- [db-id schema table-name]- (app-db/update-or-insert!- :model/Table- {:db_id db-id :schema schema :name table-name}- (fn [existing]- (when-not existing- {:display_name (humanization/name->human-readable-name table-name)- :active false- :transform_target true- :data_source :metabase-transform- :data_authority :computed- :initial_sync_status "complete"}))))--(defn delete-orphaned-provisional-table!- "If `table-id` points at an inactive provisional table that is not referenced by any other- transform (excluding `exclude-transform-id`), delete it."- [table-id exclude-transform-id]- (when table-id- (when-let [table (t2/select-one :model/Table :id table-id- :active false :transform_target true :deactivated_at nil- :data_source :metabase-transform)]- (let [referenced?- (seq- (t2/query {:select [[[:inline 1] :ref]]- :from [:transform]- :where [:and- [:= :target_table_id (:id table)]- [:not= :id exclude-transform-id]]}))]- (when-not referenced?- (t2/delete! :model/Table :id (:id table)))))))--(defn gc-transform-target-tables!- "Deletes provisional table rows (created by [[upsert-transform-target-table!]]) that are no longer- referenced by any Transform. Safe because these rows were never active,- so they have no child records.-- Note: this only handles *inactive* provisional tables. Active tables that were previously- targeted by a transform retain `transform_target = true` even after the transform is deleted.- A separate process to reset `transform_target` on active, unreferenced tables is not yet- implemented.-- Uses FK-based NOT IN queries rather than scanning JSON columns."- []- (let [candidate-ids (t2/select-fn-set :id :model/Table- :active false :transform_target true :deactivated_at nil- :data_source :metabase-transform)]- (when (seq candidate-ids)- (let [referenced-ids- (into #{}- (map :id)- (t2/query {:select [[:target_table_id :id]]- :from [:transform]- :where [:and- [:not= :target_table_id nil]- [:in :target_table_id candidate-ids]]}))- dead-ids (into [] (remove referenced-ids) candidate-ids)]- (when (seq dead-ids)- (log/infof "Deleting %d orphaned transform target table(s)" (count dead-ids))- (t2/delete! :model/Table :id [:in dead-ids]))))))-;;; ------------------------------------------------ Field ordering -------------------------------------------------(def field-order-rulediff --git a/test/metabase/models/serialization/resolve/db_test.clj b/test/metabase/models/serialization/resolve/db_test.cljnew file mode 100644index 00000000000..9f278c4ea4a--- /dev/null+++ b/test/metabase/models/serialization/resolve/db_test.clj@@ -0,0 +1,90 @@+(ns metabase.models.serialization.resolve.db-test+ (:require+ [clojure.test :refer :all]+ [metabase.models.serialization :as serdes]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(deftest import-table-fk-existing-test+ (testing "returns the existing table id when found"+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}+ :model/Table {table-id :id} {:db_id db-id :name "users" :schema "public"}]+ (is (= table-id (serdes/*import-table-fk* ["test-db" "public" "users"]))))))++(deftest import-table-fk-synthesizes-inactive-table-test+ (testing "creates an inactive table when the database exists but the table doesn't"+ (mt/with-model-cleanup [:model/Table]+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}]+ (let [table-id (serdes/*import-table-fk* ["test-db" "public" "missing"])]+ (is (=? {:id table-id+ :db_id db-id+ :schema "public"+ :name "missing"+ :active false}+ (t2/select-one :model/Table :id table-id))))))))++(deftest import-table-fk-no-schema-test+ (testing "synthesizes a table when schema is nil"+ (mt/with-model-cleanup [:model/Table]+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}]+ (let [table-id (serdes/*import-table-fk* ["test-db" nil "schemaless"])]+ (is (=? {:id table-id+ :db_id db-id+ :schema nil+ :name "schemaless"+ :active false}+ (t2/select-one :model/Table :id table-id))))))))++(deftest import-table-fk-throws-when-database-missing-test+ (testing "throws when the database itself does not exist"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"database not found"+ (serdes/*import-table-fk* ["does-not-exist" "public" "users"])))))++(deftest import-field-fk-existing-test+ (testing "returns the existing field id when found"+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}+ :model/Table {table-id :id} {:db_id db-id :name "users" :schema "public"}+ :model/Field {field-id :id} {:table_id table-id :name "email"}]+ (is (= field-id+ (serdes/*import-field-fk* ["test-db" "public" "users" "email"]))))))++(deftest import-field-fk-synthesizes-inactive-field-test+ (testing "creates an inactive field when the table exists but the field doesn't"+ (mt/with-model-cleanup [:model/Field]+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}+ :model/Table {table-id :id} {:db_id db-id :name "users" :schema "public"}]+ (let [field-id (serdes/*import-field-fk* ["test-db" "public" "users" "missing"])]+ (is (=? {:id field-id+ :table_id table-id+ :parent_id nil+ :name "missing"+ :active false}+ (t2/select-one :model/Field :id field-id))))))))++(deftest import-field-fk-synthesizes-nested-chain-test+ (testing "creates inactive parent fields along the chain when missing"+ (mt/with-model-cleanup [:model/Field]+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}+ :model/Table {table-id :id} {:db_id db-id :name "events" :schema "public"}]+ (let [field-id (serdes/*import-field-fk* ["test-db" "public" "events" "outer" "middle" "inner"])+ inner (t2/select-one :model/Field :id field-id)+ middle (t2/select-one :model/Field :id (:parent_id inner))+ outer (t2/select-one :model/Field :id (:parent_id middle))]+ (is (=? {:name "inner" :active false} inner))+ (is (=? {:name "middle" :active false :parent_id (:id outer)} middle))+ (is (=? {:name "outer" :active false :parent_id nil :table_id table-id} outer)))))))++(deftest import-field-fk-reuses-existing-parent-test+ (testing "uses an existing parent field rather than creating a duplicate"+ (mt/with-model-cleanup [:model/Field]+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}+ :model/Table {table-id :id} {:db_id db-id :name "events" :schema "public"}+ :model/Field {parent-id :id} {:table_id table-id :name "outer"}]+ (let [field-id (serdes/*import-field-fk* ["test-db" "public" "events" "outer" "inner"])]+ (is (=? {:id field-id+ :name "inner"+ :parent_id parent-id+ :active false}+ (t2/select-one :model/Field :id field-id)))+ (is (= 1 (t2/count :model/Field :table_id table-id :name "outer"))))))))diff --git a/test/metabase/transforms/models_test.clj b/test/metabase/transforms/models_test.cljindex 816aafd5ab3..f9a92957b18 100644--- a/test/metabase/transforms/models_test.clj+++ b/test/metabase/transforms/models_test.clj@@ -197,35 +197,39 @@:schema nil:name "test_table_null_schema":db_id db-id}}]-- (testing "Hydrates table with non-NULL schema"- (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]- (is (some? (:table hydrated))- "Transform should have hydrated table")- (is (= table-with-schema-id (-> hydrated :table :id))- "Should hydrate correct table with schema")))-- (testing "Hydrates table with NULL schema"- (let [hydrated (t2/hydrate transform2 :table-with-db-and-fields)]- (is (some? (:table hydrated))- "Transform with NULL schema should have hydrated table")- (is (= table-null-schema-id (-> hydrated :table :id))- "Should hydrate correct table with NULL schema")))-- (testing "Hydrates multiple transforms with mixed schemas in single batch"- (let [hydrated (t2/hydrate [transform1 transform2] :table-with-db-and-fields)]- (is (= 2 (count hydrated))- "Should hydrate both transforms")- (is (every? (comp some? :table) hydrated)- "Both transforms should have hydrated tables")- (is (= #{table-with-schema-id table-null-schema-id}- (set (map (comp :id :table) hydrated)))- "Should hydrate both tables correctly")))-- (testing "Does not hydrate unrelated tables"- (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]- (is (not= other-table-id (-> hydrated :table :id))- "Should not hydrate unrelated table"))))))+ (t2/update! :model/Transform (:id transform1) {:target_table_id table-with-schema-id})+ (t2/update! :model/Transform (:id transform2) {:target_table_id table-null-schema-id})+ (let [transform1 (t2/select-one :model/Transform :id (:id transform1))+ transform2 (t2/select-one :model/Transform :id (:id transform2))]++ (testing "Hydrates table with non-NULL schema"+ (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]+ (is (some? (:table hydrated))+ "Transform should have hydrated table")+ (is (= table-with-schema-id (-> hydrated :table :id))+ "Should hydrate correct table with schema")))++ (testing "Hydrates table with NULL schema"+ (let [hydrated (t2/hydrate transform2 :table-with-db-and-fields)]+ (is (some? (:table hydrated))+ "Transform with NULL schema should have hydrated table")+ (is (= table-null-schema-id (-> hydrated :table :id))+ "Should hydrate correct table with NULL schema")))++ (testing "Hydrates multiple transforms with mixed schemas in single batch"+ (let [hydrated (t2/hydrate [transform1 transform2] :table-with-db-and-fields)]+ (is (= 2 (count hydrated))+ "Should hydrate both transforms")+ (is (every? (comp some? :table) hydrated)+ "Both transforms should have hydrated tables")+ (is (= #{table-with-schema-id table-null-schema-id}+ (set (map (comp :id :table) hydrated)))+ "Should hydrate both tables correctly")))++ (testing "Does not hydrate unrelated tables"+ (let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]+ (is (not= other-table-id (-> hydrated :table :id))+ "Should not hydrate unrelated table")))))))(deftest creator-hydration-test(testing "Transform creator hydration returns user details"diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex 2bd6f5a9e19..464d2802fe5 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -210,10 +210,10 @@result (transforms-base.u/normalize-source-tables source-tables)](is (= 999 (:table_id (first result))))))- (testing "creates transform target table for non-existent table ref"+ (testing "leaves :table_id nil for non-existent table ref"(let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]result (transforms-base.u/normalize-source-tables source-tables)]- (is (int? (:table_id (first result))))))+ (is (nil? (:table_id (first result))))))(testing "handles entries needing different kinds of enrichment"(let [source-tables [{:alias "t1" :table_id (:id t1)}@@ -368,26 +368,26 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/with-premium-features #{:transforms-basic}(let [target {:type "table" :schema nil :name "test_output_table"}]- ;; The Transform after-insert hook creates a provisional table for the target,- ;; so we don't need to create one explicitly.- (mt/with-temp [:model/Transform {transform-id :id :as transform}+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id)+ :schema nil+ :name "test_output_table"}+ :model/Transform {transform-id :id :as transform}{:target target:source {:type "query":query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}]- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_output_table" :schema nil)]- ;; Mock execute-base! to return success without actually running a query,- ;; run-cancelable-transform! to bypass schema creation / cancellation infra,- ;; and sync-target! to skip driver calls but still return the provisional table- ;; so complete-execution! can set transform_id on it.- (mt/with-dynamic-fn-redefs- [transforms-base.i/execute-base! (constantly {:status :succeeded})- transforms-base.u/sync-target! (fn [_target _database]- (t2/select-one :model/Table table-id))- transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]- (run-fn (a/promise-chan) nil))]- (transforms.execute/execute! transform {:run-method :manual})- (is (= transform-id- (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))))+ ;; Mock execute-base! to return success without actually running a query,+ ;; run-cancelable-transform! to bypass schema creation / cancellation infra,+ ;; and sync-target! to skip driver calls but still return the target table+ ;; so complete-execution! can set transform_id on it.+ (mt/with-dynamic-fn-redefs+ [transforms-base.i/execute-base! (constantly {:status :succeeded})+ transforms-base.u/sync-target! (fn [_target _database]+ (t2/select-one :model/Table table-id))+ transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]+ (run-fn (a/promise-chan) nil))]+ (transforms.execute/execute! transform {:run-method :manual})+ (is (= transform-id+ (t2/select-one-fn :transform_id :model/Table :id table-id))))))))))(deftest transform-hydration-test(testing "hydrating :transform on a table"diff --git a/test/metabase/transforms_base/ordering_test.clj b/test/metabase/transforms_base/ordering_test.cljindex 0aa51fcdc16..e49c5c1dbe3 100644--- a/test/metabase/transforms_base/ordering_test.clj+++ b/test/metabase/transforms_base/ordering_test.clj@@ -42,11 +42,12 @@:name "orders_2"}:model/Field _ {:table_id table:name "foo"}- :model/Transform {parent :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table (mt/id :orders)}}- "orders_2")+ :model/Transform {parent :id} (-> (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table (mt/id :orders)}}+ "orders_2")+ (assoc :target_table_id table)):model/Transform {child :id} (make-transform{:database (mt/id):type "query"@@ -299,11 +300,12 @@:name "table_1"}:model/Field _ {:table_id table1:name "foo"}- :model/Transform {t1 :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table table1}}- "table_1")]+ :model/Transform {t1 :id} (-> (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table table1}}+ "table_1")+ (assoc :target_table_id table1))](is (= {:cycle-str "transform_table_1":cycle [t1]}(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))@@ -314,16 +316,18 @@:model/Table {table2 :id} {:schema (default-schema-or-public), :name "table_2"}:model/Field _ {:table_id table2:name "foo"}- :model/Transform {t1 :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table table1}}- "table_2")- :model/Transform {t2 :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table table2}}- "table_1")]+ :model/Transform {t1 :id} (-> (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table table1}}+ "table_2")+ (assoc :target_table_id table2))+ :model/Transform {t2 :id} (-> (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table table2}}+ "table_1")+ (assoc :target_table_id table1))](is (= {:cycle-str "transform_table_2 -> transform_table_1",:cycle [t1 t2]}(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))@@ -340,21 +344,24 @@:name "table_3"}:model/Field _ {:table_id table3:name "foo"}- :model/Transform {t1 :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table table1}}- "table_2")- :model/Transform {t2 :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table table2}}- "table_3")- :model/Transform {t3 :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table table3}}- "table_1")]+ :model/Transform {t1 :id} (-> (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table table1}}+ "table_2")+ (assoc :target_table_id table2))+ :model/Transform {t2 :id} (-> (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table table2}}+ "table_3")+ (assoc :target_table_id table3))+ :model/Transform {t3 :id} (-> (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table table3}}+ "table_1")+ (assoc :target_table_id table1))](is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",:cycle [t1 t3 t2]}(ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex a5f1b882223..786ac947b27 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -626,7 +626,8 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/with-temp [:model/Table {table :id} {:schema "public", :name "orders_2"}:model/Field _ {:table_id table, :name "foo"}- :model/Transform parent (->transform "transform1" (mt/mbql-query orders))+ :model/Transform parent (-> (->transform "transform1" (mt/mbql-query orders))+ (assoc :target_table_id table)):model/Transform child (-> (->transform "transform2" (mt/mbql-query nil {:source-table table}))(assoc-in [:target :name] "orders_3"))](let [deps-resp (mt/user-http-request :lucky :get 200 (format "transform/%s/dependencies" (:id child)))]