The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Unifying source tables transform format (#70298)
Ngoc Khuatc86caa092e2a
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex 21511052820..41260a6060a 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -47,8 +47,8 @@(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."[{{tables :source-tables} :source :as _py-transform}- :- [:map [:source-tables {:optional true} [:map-of :string [:or :int [:map [:table_id :int]]]]]]]- {:table (into #{} (keep (fn [v] (if (map? v) (:table_id v) v))) (vals tables))})+ :- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]+ {:table (into #{} (keep :table_id) tables)})(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps"Given a Transform (in Toucan form), return its upstream dependencies."@@ -58,8 +58,7 @@[:query[:map [:query ::lib.schema/query]]][:python- ;; If the upstream table doesn't exist yet, table_id will be nil- [:map [:source-tables {:optional true} [:map-of :string [:or :int [:map [:table_id [:maybe :int]]]]]]]]]]]]+ [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]]]]](let [source-type (transforms-base.u/transform-type transform)](case source-type:query (upstream-deps:query query)diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.cljindex c1518645217..5586b5e8696 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj@@ -99,7 +99,7 @@(when-let [source-database (get-in item [:source :source-database])](when-let [source-tables (not-empty (get-in item [:source :source-tables]))]{:database-id source-database- :table-ids (vals source-tables)}))))+ :table-ids (map :table_id source-tables)}))))(defn- python-transform-tables-for-context"Get tables for Python transform formatted for metabot context."diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljindex f8d556caae4..9ec54e49cd0 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -60,7 +60,7 @@[transform](into #{}(map source-table-value->dependency)- (vals (get-in transform [:source :source-tables]))))+ (get-in transform [:source :source-tables])));;; ------------------------------------------------- Message Log (in-memory only) -------------------------------------------------diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.cljindex 6df753785a3..c80663c759d 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj@@ -27,6 +27,7 @@[metabase.lib.core :as lib][metabase.queries.schema :as queries.schema][metabase.request.core :as request]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.core :as transforms][metabase.transforms.feature-gating :as transforms.gating][metabase.transforms.util :as transforms.u]@@ -79,7 +80,12 @@[:python[:map {:closed true}[:source-database {:optional true} :int]- [:source-tables [:map-of [:string {:min 1}] [:or :int :map]]]+ ;; TODO (Ngoc 2026-03-04) -- remove decode/normalize when FE sends array format for source-tables+ [:source-tables [:sequential {:decode/normalize (fn [st]+ (if (map? st)+ (transforms-base.u/source-tables-map->vec st)+ st))}+ [:map [:alias [:string {:min 1}]] [:table_id :int]]]][:type [:= "python"]][:body :string]]]])@@ -828,7 +834,10 @@(deferred-tru "Another transform in this workspace already targets that table"))transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body:ref-id ref-id)]- (attach-isolated-target (select-malli-keys WorkspaceTransform workspace-transform-alias transform)))))+ (-> (select-malli-keys WorkspaceTransform workspace-transform-alias transform)+ attach-isolated-target+ ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables+ transforms/source-tables-vec->map-for-fe))))(api.macros/defendpoint :post "/:ws-id/transform":- WorkspaceTransform@@ -876,7 +885,9 @@(-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)(t2/select-one :workspace_id ws-id :ref_id tx-id)api/check-404- attach-isolated-target))+ attach-isolated-target+ ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables+ transforms/source-tables-vec->map-for-fe))(api.macros/defendpoint :get "/:ws-id/transform/:tx-id" :- WorkspaceTransform"Get a specific transform in a workspace."diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.cljindex c97c323abc1..5f4f6a732dc 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj@@ -175,25 +175,20 @@(inputs-from-native-query query db-id)(inputs-from-mbql-query query)))-(defn- source-table-ref->table-ref- "Paper over a small unfortunate difference in shape"- [table-ref]- (-> (select-keys table-ref [:schema :table :table_id])- (assoc :db_id (:database_id table-ref))))-(defn- inputs-from-python-transform- "Extract table refs from a python transform's source-tables.- Uses metadata from map refs directly; only looks up the database for bare integer IDs."+ "Extract table refs from a python transform's source-tables vec.+ Entries with :database_id/:schema/:table just need a key rename;+ entries with only :table_id fall back to a batch lookup."[source-tables]- (let [{maps true ints false} (group-by (comp map? val) source-tables)- ;; Maps already have schema/table metadata- map-refs (map (comp source-table-ref->table-ref val) maps)- ;; Integers need a batch lookup- int-ids (map val ints)- int-refs (when (seq int-ids)- (let [lookup (batch-table-refs-from-ids int-ids)]- (keep lookup int-ids)))]- (into (vec map-refs) int-refs)))+ (let [{full true, partial false} (group-by (fn [e] (boolean (:table e))) source-tables)+ renamed (mapv (fn [{:keys [database_id schema table]}]+ {:db_id database_id :schema schema :table table})+ full)+ looked (when (seq partial)+ (let [ids (into #{} (keep :table_id) partial)+ lookup (batch-table-refs-from-ids ids)]+ (u/keepv lookup (map :table_id partial))))]+ (into renamed looked)))(mu/defn analyze-entity :- ::analysis"Analyze a workspace entity to find its dependencies.diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.cljindex ac636ef9e8f..13f27c7cb66 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj@@ -11,25 +11,27 @@[metabase.transforms-base.core :as transforms-base][metabase.transforms-base.util :as transforms-base.u][metabase.util.log :as log]+ [metabase.util.malli :as mu][toucan2.core :as t2]))(set! *warn-on-reflection* true)-(defn- remap-python-source- "Remap source-tables in a Python transform source to point to isolated tables.- Handles both legacy integer format and new map format (from PR #66934)."- [table-mapping source]- (letfn [(remap [{:keys [database_id schema table table_id] :as table-ref}]- (if-let [{:keys [id db-id schema table]} (or (table-mapping table-ref)- (some-> table_id table-mapping)- (when table- (table-mapping [database_id schema table])))]- ;; Prefer table ID when available; map format won't work until PR #66934 is merged:- ;; https://github.com/metabase/metabase/pull/66934- (or id {:database_id db-id, :schema schema, :table table})- ;; Leave it un-mapped if we don't have an override.- table-ref))]- (update source :source-tables update-vals remap)))+(mu/defn- remap-python-source+ "Remap source-tables in a Python transform source to point to isolated tables."+ [table-mapping source :- [:map [:source-tables [:sequential ::transforms-base.u/source-table-entry]]]]+ (letfn [(remap [{:keys [database_id schema table table_id] :as entry}]+ (if-let [{:keys [id db-id] target-schema :schema target-table :table}+ (or (table-mapping entry)+ (some-> table_id table-mapping)+ (when table+ (table-mapping [database_id schema table])))]+ (cond-> entry+ id (assoc :table_id id)+ db-id (assoc :database_id db-id)+ target-schema (assoc :schema target-schema)+ target-table (assoc :table target-table))+ entry))]+ (update source :source-tables (fn [entries] (mapv remap entries)))))(defn- remap-sql-source [table-mapping source](let [remapping (reducediff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.cljindex b0dabaf479f..5238f8f0dbd 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj@@ -199,8 +199,8 @@orders-id (mt/id :orders)](mt/with-temp [:model/Transform transform {:name "Test Transform":source {:type :python- :source-tables {"PRODUCTS" products-id- "ORDERS" orders-id}+ :source-tables [{:alias "PRODUCTS" :table_id products-id}+ {:alias "ORDERS" :table_id orders-id}];; A problematic field, hopefully removed again.:source-database (mt/id):body "..."}diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex bcd554ddc15..1cb90946923 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -323,10 +323,11 @@(fn [_](let [source {:type :python,:source-database (mt/id),- :source-tables {"orders" {:database_id (mt/id),- :schema "public",- :table "orders",- :table_id (mt/id :orders)}},+ :source-tables [{:alias "orders"+ :database_id (mt/id)+ :schema "public"+ :table "orders"+ :table_id (mt/id :orders)}],:body"import pandas as pd\n\ndef transform(orders):\n return orders"}](mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]@@ -366,10 +367,11 @@(fn [_](let [source {:type :python,:source-database (mt/id),- :source-tables {"orders" {:database_id (mt/id),- :schema "public",- :table "orders",- :table_id (mt/id :orders)}},+ :source-tables [{:alias "orders"+ :database_id (mt/id)+ :schema "public"+ :table "orders"+ :table_id (mt/id :orders)}],:body"import pandas as pd\n\ndef transform(orders):\n return orders"}target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.cljindex 80c7539e939..1bf446caaeb 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj@@ -143,8 +143,8 @@(let [input {:user_is_viewing [{:type "transform":source {:type "python":source-database 2- :source-tables {:orders 24- :products 31}}}]}+ :source-tables [{:alias "orders" :table_id 24}+ {:alias "products" :table_id 31}]}}]}result (#'context/enhance-context-with-schema input)used-tables (get-in result [:user_is_viewing 0 :used_tables])](is (= mock-tables used-tables)))))))@@ -157,7 +157,7 @@(let [input {:user_is_viewing [{:type "transform":source {:type "python":source-database 2- :source-tables {}}}]}+ :source-tables []}}]}result (#'context/enhance-context-with-schema input)](is (nil? (get-in result [:user_is_viewing 0 :used_tables])))(is (false? @called?)))))))@@ -169,7 +169,7 @@(fn [_] (reset! called? true) nil)](let [input {:user_is_viewing [{:type "transform":source {:type "python"- :source-tables {:orders 24}}}]}+ :source-tables [{:alias "orders" :table_id 24}]}}]}result (#'context/enhance-context-with-schema input)](is (nil? (get-in result [:user_is_viewing 0 :used_tables])))(is (false? @called?)))))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.cljindex 55b293ea6ed..e3553f054d7 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj@@ -19,6 +19,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.test :as mt]+ [metabase.transforms.crud :as transforms.crud][metabase.util :as u][metabase.util.malli.registry :as mr][toucan2.core :as t2]))@@ -1291,7 +1292,7 @@:source {:type "python":source-database (mt/id):body "print('hello world')"- :source-tables {}}+ :source-tables []}:target {:type "table":name "t2_table"}}](testing "With insufficient permissions"@@ -1302,7 +1303,9 @@(testing "With superuser permissions"(is (=? {:structured_output [(mt/obj->json->obj (select-keys t1 [:id :entity_id :name :description :source]));; note: t2 not included because it's a (non-native) MBQL query- (mt/obj->json->obj (select-keys t3 [:id :entity_id :name :description :source]))]+ ;; API converts source-tables from vec back to map for FE+ (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe+ (select-keys t3 [:id :entity_id :name :description :source])))]:conversation_id conversation-id}(-> (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transforms"{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}@@ -1328,7 +1331,7 @@:source {:type "python":body "print('hello world')":source-database (mt/id)- :source-tables {}}+ :source-tables []}:target {:type "table":name "t2_table":database (mt/id)}}]@@ -1347,7 +1350,9 @@(testing "With superuser permissions"(doseq [transform [t1 t2]](testing (:name transform)- (is (=? {:structured_output (mt/obj->json->obj (select-keys transform [:id :entity_id :name :description :source :target]))+ ;; API converts source-tables from vec back to map for FE+ (is (=? {:structured_output (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe+ (select-keys transform [:id :entity_id :name :description :source :target]))):conversation_id conversation-id}(mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transform-details"{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.cljindex 2dd412475dc..0c42a2c1831 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj@@ -251,13 +251,13 @@:source {:type "python":body "print('hello')":source-database (mt/id)- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}}+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}:target {:type "table":schema "public":name "python_transform_table"}}](let [modified-source {:type "python":body "print('modified')"- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}}+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}result (metabot.dependencies/check-transform-dependencies{:id python-transform-id:source modified-source})]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 0a69cfa3eac..445e60ff65a 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -535,6 +535,36 @@ serdes/meta:entity-id(str/replace (u/lower-case-en name) #"\s+" "_")))+(defn generate-python-transform-yaml+ "Generate YAML content for a python transform with old map-format source-tables.+ Used to test backward compatibility when importing legacy format data.+ `source-tables-yaml` should be a YAML string representing the source-tables map,+ e.g. \"{orders: 123, products: 456}\"."+ [entity-id name db-name source-tables-yaml & {:keys [collection-id]}]+ (format "name: %s+description: null+entity_id: %s+collection_id: %s+created_at: '2024-08-28T09:46:18.671622Z'+creator_id: [email protected]+source_database_id: %s+source:+ type: python+ body: 'import pandas as pd'+ source-tables: %s+target:+ type: table+ database: %s+ name: test_output+ schema: PUBLIC+serdes/meta:+- id: %s+ label: %s+ model: Transform+"+ name entity-id (or collection-id "null") db-name source-tables-yaml db-name+ entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))+(defn generate-transform-yaml"Generate YAML content for a transform with the given `entity-id` and `name`.Optionally accepts `:collection-id` for transforms inside a collection."diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex be52c6d80dc..0141acbd197 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -326,6 +326,39 @@ is_sample: false(is (t2/exists? :model/Transform :entity_id transform-entity-id)"Transform should be imported from transforms-namespace collection")))))))+(deftest import-python-transform-with-legacy-map-source-tables-test+ (testing "Import of a python transform with old map-format source-tables converts to vec format"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-model-cleanup [:model/Transform :model/Collection]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ coll-entity-id "xforms-coll-legacy-01"+ transform-entity-id "python-legacy-fmt-01x"+ ;; Use old map format: {alias: table_id} — this is how pre-unification exports look+ source-tables-yaml (format "{orders: %d}" (mt/id :orders))+ test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")+ (generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")+ (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_python_transform.yaml")+ (test-helpers/generate-python-transform-yaml transform-entity-id "Python Transform"+ "test-data (h2)" source-tables-yaml+ :collection-id coll-entity-id)}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (when-let [transform (t2/select-one :model/Transform :entity_id transform-entity-id)]+ (let [source-tables (get-in transform [:source :source-tables])]+ (testing "source-tables is stored as vec format after import"+ (is (sequential? source-tables)))+ (testing "entry has correct alias and table_id"+ (let [entry (first source-tables)]+ (is (= "orders" (:alias entry)))+ (is (= (mt/id :orders) (:table_id entry)))))+ (testing "entry is enriched with table metadata"+ (let [entry (first source-tables)]+ (is (= (mt/id) (:database_id entry)))+ (is (some? (:table entry)))))))))))))+(deftest archived-transforms-namespace-collection-excluded-from-export-test(testing "When a transforms-namespace collection is archived, it and its children are excluded from export"(mt/with-premium-features #{:transforms}diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex fb2a28e54f0..28e0ce4aec4 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -30,7 +30,7 @@{:name "Python Transform":source {:type "python":body "print('hello world')"- :source-tables {}+ :source-tables []:source-database (mt/id)}:target {:type "table":schema (get-test-schema)diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex a02eb8d6dcb..5957224e89d 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -153,7 +153,7 @@:source-incremental-strategy {:type "checkpoint":checkpoint-filter-unique-key lib-column-key}}:python {:type "python"- :source-tables {"transforms_products" (mt/id :transforms_products)}+ :source-tables [{:alias "transforms_products" :table_id (mt/id :transforms_products)}]:limit 10:body incremental-python-body:source-incremental-strategy {:type "checkpoint"diff --git a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/models_test.cljdeleted file mode 100644index f7ffcac24e2..00000000000--- a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj+++ /dev/null@@ -1,63 +0,0 @@-(ns metabase-enterprise.transforms.models-test- (:require- [clojure.test :refer [deftest testing is]]- [metabase.test :as mt]- [toucan2.core :as t2]))--(deftest python-transform-source-table-resolution-on-save-test- (testing "Python transform source-tables with name refs get table_id resolved on save"- (mt/with-temp [:model/Database {db-id :id} {}- :model/Table {table-id :id} {:db_id db-id- :name "existing_table"- :schema "public"}]- (testing "table_id is populated when table exists"- (mt/with-temp [:model/Transform transform {:name "Transform with ref"- :source {:type "python"- :body "def transform(): pass"- :source-database db-id- :source-tables {"input" {:database_id db-id- :schema "public"- :table "existing_table"}}}- :target {:type "table"- :schema "public"- :name "output"- :database db-id}}]- (let [saved (t2/select-one :model/Transform :id (:id transform))- source-table (get-in saved [:source :source-tables "input"])]- (is (= table-id (:table_id source-table))- "table_id should be resolved from database lookup"))))-- (testing "table_id remains nil when table doesn't exist"- (mt/with-temp [:model/Transform transform {:name "Transform with missing ref"- :source {:type "python"- :body "def transform(): pass"- :source-database db-id- :source-tables {"input" {:database_id db-id- :schema "public"- :table "nonexistent_table"}}}- :target {:type "table"- :schema "public"- :name "output"- :database db-id}}]- (let [saved (t2/select-one :model/Transform :id (:id transform))- source-table (get-in saved [:source :source-tables "input"])]- (is (nil? (:table_id source-table))- "table_id should be nil for non-existent table"))))-- (testing "existing table_id is preserved"- (mt/with-temp [:model/Transform transform {:name "Transform with explicit table_id"- :source {:type "python"- :body "def transform(): pass"- :source-database db-id- :source-tables {"input" {:database_id db-id- :schema "public"- :table "existing_table"- :table_id 999}}}- :target {:type "table"- :schema "public"- :name "output"- :database db-id}}]- (let [saved (t2/select-one :model/Transform :id (:id transform))- source-table (get-in saved [:source :source-tables "input"])]- (is (= 999 (:table_id source-table))- "explicit table_id should be preserved")))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex f6a81f187a7..75702b2064a 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -25,13 +25,14 @@(testing "Python transform with name-based source table ref resolves to producing transform"(mt/with-temp [;; Transform A produces table "intermediate_output":model/Transform {t-a :id} (make-python-transform- {"input" (mt/id :orders)}+ [{:alias "input" :table_id (mt/id :orders)}]"intermediate_output");; Transform B references intermediate_output by name (table doesn't exist yet):model/Transform {t-b :id} (make-python-transform- {"source" {:database_id (mt/id)- :schema "public"- :table "intermediate_output"}}+ [{:alias "source"+ :database_id (mt/id)+ :schema "public"+ :table "intermediate_output"}]"final_output")](testing "table-dependencies returns table-ref for unresolved name reference"(let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))]@@ -47,15 +48,14 @@(deftest python-transform-mixed-source-tables-test(testing "Python transform with mixed int and name-based refs"(mt/with-temp [:model/Transform {t-a :id} (make-python-transform- {"input" (mt/id :orders)}+ [{:alias "input" :table_id (mt/id :orders)}]"output_a"):model/Transform {t-b :id} (make-python-transform- {;; Direct table reference (existing table)- "existing" (mt/id :products)- ;; Name-based reference (table doesn't exist yet)- "from_transform" {:database_id (mt/id)- :schema "public"- :table "output_a"}}+ [{:alias "existing" :table_id (mt/id :products)}+ {:alias "from_transform"+ :database_id (mt/id)+ :schema "public"+ :table "output_a"}]"output_b")](testing "table-dependencies includes both types"(let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))]diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.cljindex 606a8789afe..7fbba9b52f6 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj@@ -457,9 +457,9 @@" # Return processed dataframe\n"" return df")- source-tables (cond-> {source-table-name source-table-id}+ source-tables (cond-> [{:alias source-table-name :table_id source-table-id}]exotic-table-id- (assoc (str source-table-name "_exotic") exotic-table-id))+ (conj {:alias (str source-table-name "_exotic") :table_id exotic-table-id}))result (execute-e2e-transform! table-name transform-code source-tables){:keys [columns] result-rows :rows} resultdiff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.cljindex 17eb49b1e3c..b164884b271 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj@@ -44,7 +44,7 @@transform-payload {:name "Library Test Transform":source {:type "python":source-database (mt/id)- :source-tables {}+ :source-tables []:body transform-body}:target (assoc target :database (mt/id))}{transform-id :id} (mt/user-http-request :crowberto :post 200 "transform"@@ -71,7 +71,7 @@(let [original {:name "Gadget Products":source {:type "python":source-database (mt/id)- :source-tables {"transforms_customers" (mt/id :transforms_customers)}+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]:body (str "import pandas as pd\n""\n""def transform():\n"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.cljindex 4fe6138a118..fd036fe1d1a 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj@@ -25,7 +25,7 @@:name "swap_tbl"}](let [initial-transform {:name "Python Transform Initial":source {:type "python"- :source-tables {}+ :source-tables []:source-database (mt/id):body (str "import pandas as pd\n""\n"@@ -41,7 +41,7 @@(t2/update! :model/Transform (:id transform){:source {:type "python"- :source-tables {}+ :source-tables []:source-database (mt/id):body (str "import pandas as pd\n""\n"@@ -77,7 +77,7 @@:name "cleanup_"}](let [transform-def {:name "Python Transform Cleanup":source {:type "python"- :source-tables {}+ :source-tables []:source-database (mt/id):body (str "import pandas as pd\n""\n"@@ -116,7 +116,7 @@" return pd.DataFrame({'result': ['should_not_reach_here']})")transform-def {:name "Python Transform Timeout Test":source {:type "python"- :source-tables {}+ :source-tables []:source-database (mt/id):body long-running-code}:target (assoc target :database (mt/id))}]@@ -146,9 +146,10 @@:source {:type "python":body "def transform(input): return input":source-database (mt/id)- :source-tables {"input" {:database_id (mt/id)- :schema schema- :table "nonexistent_table"}}}+ :source-tables [{:alias "input"+ :database_id (mt/id)+ :schema schema+ :table "nonexistent_table"}]}:target (assoc target :database (mt/id))}](testing "Execution throws with informative error message"(is (thrown-with-msg?@@ -168,9 +169,10 @@:source {:type "python":body "def transform(input): return input":source-database (mt/id)- :source-tables {"input" {:database_id (mt/id)- :schema nil- :table "missing_table"}}}+ :source-tables [{:alias "input"+ :database_id (mt/id)+ :schema nil+ :table "missing_table"}]}:target (assoc target :database (mt/id))}](is (thrown-with-msg?clojure.lang.ExceptionInfodiff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.cljindex af3c34383e4..473dd057f83 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj@@ -29,7 +29,7 @@:model/Transform {transform-id :id} {:name "Gadget Products":source {:type "python":source-database (mt/id)- :source-tables {"transforms_customers" (mt/id :transforms_customers)}+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]:body (str "import pandas as pd\n""\n""def transform():\n"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex 441ef4ab21a..fb46a322bac 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -37,8 +37,8 @@(deftest python-transform-basic-dependencies-test(testing "Python transforms with source-tables dependencies are extracted correctly"- (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)- "products" (mt/id :products)})]+ (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}+ {:alias "products" :table_id (mt/id :products)}])](is (= #{{:table (mt/id :orders)}{:table (mt/id :products)}}(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))@@ -51,8 +51,8 @@:name "output_1"}:model/Field _ {:table_id table1:name "foo"}- :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")- :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]+ :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")+ :model/Transform {t2 :id} (make-python-transform [{:alias "output_1" :table_id table1}] "output_2")](is (= {t1 #{}t2 #{t1}}(ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]])))))))))@@ -69,10 +69,10 @@:name "output_2"}:model/Field _ {:table_id table2:name "bar"}- :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")- :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")- :model/Transform {t3 :id} (make-python-transform {"output_1" table1- "output_2" table2} "final_output")]+ :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")+ :model/Transform {t2 :id} (make-python-transform [{:alias "products" :table_id (mt/id :products)}] "output_2")+ :model/Transform {t3 :id} (make-python-transform [{:alias "output_1" :table_id table1}+ {:alias "output_2" :table_id table2}] "final_output")](is (= {t1 #{}t2 #{}t3 #{t1 t2}}@@ -97,7 +97,7 @@:query {:source-table (mt/id :orders)}}"sql_output");; Python transform that depends on the SQL transform's output- :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")+ :model/Transform {t2 :id} (make-python-transform [{:alias "sql_output" :table_id table1}] "python_output");; Another SQL transform that depends on the Python transform's output:model/Transform {t3 :id} (make-transform{:database (mt/id)diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljindex a5cba2543be..b55d23c7673 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -32,7 +32,7 @@transform-payload {:name "My beautiful python runner":source {:type "python":body "print('hello world')"- :source-tables {}+ :source-tables []:source-database (mt/id)}:target {:type "table":schema schema@@ -49,7 +49,7 @@{:name "My beautiful python runner":source {:type "python":body "print('hello world')"- :source-tables {}+ :source-tables []:source-database (mt/id)}:target {:type "table":schema (get-test-schema)@@ -64,7 +64,7 @@{:name "My beautiful python runner":source {:type "python":body "print('hello world')"- :source-tables {}+ :source-tables []:source-database (mt/id)}:target {:type "table":schema (get-test-schema)@@ -80,7 +80,7 @@{:name "My beautiful python runner":source {:type "python":body "print('hello chris')"- :source-tables {}+ :source-tables []:source-database (mt/id)}:target {:type "table":schema (get-test-schema)@@ -96,7 +96,7 @@:as transform} {:name "Original Python Transform":source {:type "python":body "print('original')"- :source-tables {}+ :source-tables []:source-database (mt/id)}:target {:type "table":schema "scheam"@@ -120,7 +120,7 @@transform-payload {:name "Test Run Python Transform":source {:type "python":body "def transform():\n pass"- :source-tables {}+ :source-tables []:source-database (mt/id)}:target {:type "table":schema schema@@ -149,7 +149,7 @@(let [original {:name "Gadget Products":source {:type "python":source-database (mt/id)- :source-tables {"transforms_customers" (mt/id :transforms_customers)}+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]:body (str "import pandas as pd\n""\n""def transform():\n"@@ -223,7 +223,7 @@:source {:type "python":body (program->source program):source-database (mt/id)- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}}+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}:target (assoc target :database (mt/id))})))(block-on-run [{:keys [expect-status]} target transform-id]@@ -329,7 +329,7 @@:model/Transform transform {:name "Python Transform Cross DB":source {:type "python":source-database (mt/id)- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]:body "def transform():\n pass"}:target {:type "table":schema "PUBLIC"@@ -353,7 +353,7 @@:source {:type "python":body (str/join "\n" program):source-database (mt/id)- :source-tables {"transforms_customers" (mt/id :transforms_customers)}}+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]}:target (assoc target :database (mt/id))}));; using clojure-ey coordination with promises (I know j.u.c could be better here)@@ -526,7 +526,7 @@(let [initial-transform {:name "Schema Change Integration Test":source {:type "python":source-database (mt/id)- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]:body (str "import pandas as pd\n""\n""def transform():\n"@@ -545,7 +545,7 @@(let [updated-transform (assoc initial-transform:source {:type "python":source-database (mt/id)- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]:body (str "import pandas as pd\n""\n""def transform():\n"@@ -573,9 +573,10 @@:schema (get-test-schema):name "table_ref_test"}](let [;; Use a name-based ref instead of table ID- source-tables {"input" {:database_id (mt/id)- :schema (get-test-schema)- :table "transforms_products"}}+ source-tables [{:alias "input"+ :database_id (mt/id)+ :schema (get-test-schema)+ :table "transforms_products"}]transform-payload {:name "Transform with table ref":source {:type "python":body "def transform(input):\n return input"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.cljindex b7995fbdb5f..6389f26ef74 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj@@ -95,8 +95,8 @@(deftest ^:parallel analyze-entity-python-transform-test(testing "analyze-entity extracts dependencies from a python transform"(let [transform {:source {:type "python"- :source-tables {"orders" (mt/id :orders)- "products" (mt/id :products)}+ :source-tables [{:alias "orders" :table_id (mt/id :orders)}+ {:alias "products" :table_id (mt/id :products)}]:body "def transform(orders, products): return orders"}:target {:database (mt/id):schema "public"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.cljindex 7d105fdcf60..9fc705712e0 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj@@ -98,7 +98,7 @@db-id (:database_id workspace)body {:name "Python Dry-Run Transform":source {:type "python"- :source-tables {}+ :source-tables []:body (str "import pandas as pd\n""\n""def transform():\n"@@ -129,86 +129,53 @@(deftest remap-python-source-test(let [remap-python-source #'ws.execute/remap-python-source]- (testing "remaps to table ID when mapping has :id"+ (testing "remaps entry via triple lookup when mapping has :id"(let [table-mapping {[1 "public" "orders"] {:db-id 1:schema "ws_isolated_123":table "orders_isolated":id 456}}source {:type "python":body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "public"- :table "orders"- :table_id 123}}}]- (is (= {:type "python"- :body "import pandas as pd"- :source-tables {"orders" 456}}- (remap-python-source table-mapping source)))))-- (testing "remaps to map format when mapping has nil :id"- (let [table-mapping {[1 "public" "orders"] {:db-id 1- :schema "ws_isolated_123"- :table "orders_isolated"- :id nil}}- source {:type "python"- :body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "public"- :table "orders"- :table_id 123}}}]- (is (= {:type "python"- :body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "ws_isolated_123"- :table "orders_isolated"}}}- (remap-python-source table-mapping source)))))-- (testing "leaves source-tables unchanged when no mapping exists"- (let [table-mapping {}- source {:type "python"- :body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "public"- :table "orders"- :table_id 123}}}]- (is (= source (remap-python-source table-mapping source)))))+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}]+ (:source-tables (remap-python-source table-mapping source))))))- (testing "remaps legacy integer to table ID when mapping has :id"+ (testing "remaps entry via table_id lookup"(let [table-mapping {123 {:db-id 1:schema "ws_isolated_123":table "orders_isolated":id 456}}source {:type "python":body "import pandas as pd"- :source-tables {"orders" 123}}]- (is (= {:type "python"- :body "import pandas as pd"- :source-tables {"orders" 456}}- (remap-python-source table-mapping source)))))+ :source-tables [{:alias "orders" :table_id 123}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}]+ (:source-tables (remap-python-source table-mapping source))))))- (testing "remaps legacy integer to map format when mapping has nil :id"- (let [table-mapping {123 {:db-id 1- :schema "ws_isolated_123"- :table "orders_isolated"- :id nil}}+ (testing "remaps without table_id when mapping has nil :id"+ (let [table-mapping {[1 "public" "orders"] {:db-id 1+ :schema "ws_isolated_123"+ :table "orders_isolated"+ :id nil}}source {:type "python":body "import pandas as pd"- :source-tables {"orders" 123}}]- (is (= {:type "python"- :body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "ws_isolated_123"- :table "orders_isolated"}}}- (remap-python-source table-mapping source)))))-- (testing "leaves legacy integer table-id unchanged when no mapping exists"+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 123}]+ (:source-tables (remap-python-source table-mapping source))))))++ (testing "leaves source-tables unchanged when no mapping exists"(let [table-mapping {}source {:type "python":body "import pandas as pd"- :source-tables {"orders" 123}}]+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}]}](is (= source (remap-python-source table-mapping source)))))- (testing "handles mixed formats with mappings for both"+ (testing "handles multiple entries with mappings"(let [table-mapping {[1 "public" "orders"] {:db-id 1:schema "ws_isolated_123":table "orders_isolated"@@ -219,18 +186,16 @@:id 999}}source {:type "python":body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "public"- :table "orders"- :table_id 123}- "customers" 789}}]- (is (= {:type "python"- :body "import pandas as pd"- :source-tables {"orders" 456- "customers" 999}}- (remap-python-source table-mapping source)))))-- (testing "prefers table_id lookup over triple lookup for map format"+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}+ {:alias "customers" :table_id 789}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}+ {:alias "customers" :database_id 1+ :schema "ws_isolated_123" :table "customers_isolated" :table_id 999}]+ (:source-tables (remap-python-source table-mapping source))))))++ (testing "prefers table_id lookup over triple lookup"(let [table-mapping {123 {:db-id 1:schema "ws_isolated_123":table "orders_by_id"@@ -241,14 +206,11 @@:id 789}}source {:type "python":body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "public"- :table "orders"- :table_id 123}}}]- (is (= {:type "python"- :body "import pandas as pd"- :source-tables {"orders" 456}}- (remap-python-source table-mapping source)))))+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "ws_isolated_123" :table "orders_by_id" :table_id 456}]+ (:source-tables (remap-python-source table-mapping source))))))(testing "falls back to triple lookup when table_id not in mapping"(let [table-mapping {[1 "public" "orders"] {:db-id 1@@ -257,14 +219,34 @@:id 456}}source {:type "python":body "import pandas as pd"- :source-tables {"orders" {:database_id 1- :schema "public"- :table "orders"- :table_id 999}}}]- (is (= {:type "python"- :body "import pandas as pd"- :source-tables {"orders" 456}}- (remap-python-source table-mapping source)))))))+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 999}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}]+ (:source-tables (remap-python-source table-mapping source))))))++ (testing "keeps original values when mapping has nil :id"+ (let [table-mapping {123 {:db-id 1+ :schema "ws_isolated_123"+ :table "orders_isolated"+ :id nil}}+ source {:type "python"+ :body "import pandas as pd"+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 123}]+ (:source-tables (remap-python-source table-mapping source))))))++ (testing "entry with no matching mapping is left unchanged"+ (let [table-mapping {999 {:db-id 1 :schema "ws" :table "t" :id 456}}+ source {:type "python"+ :body "import pandas as pd"+ :source-tables [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}]}]+ (is (= [{:alias "orders" :database_id 1+ :schema "public" :table "orders" :table_id 123}]+ (:source-tables (remap-python-source table-mapping source))))))))(deftest remap-sql-source-test(let [remap-sql-source #'ws.execute/remap-sql-sourcediff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 12cce2e37c5..a7b9d07c88f 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -3574,6 +3574,14 @@ databaseChangeLog:path: instance_analytics_views/query_log/v2/h2-query_log.sqlrelativeToChangelogFile: true+ - changeSet:+ id: v59.2026-03-03T12:00:00+ author: qnkhuat+ comment: Unify source-tables format from map to ordered array+ changes:+ - customChange:+ class: "metabase.app_db.custom_migrations.UnifySourceTablesFormat"+- changeSet:id: v59.2026-03-04T00:00:00author: rileythompsondiff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex db8b2ca3911..5d6f1d70da5 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -1855,6 +1855,72 @@(define-migration MoveExistingAtSymbolUserAttributes(reserve-at-symbol-user-attributes/migrate!))+(define-reversible-migration UnifySourceTablesFormat+ (let [tables [{:table :transform :pks [:id] :where [:= :source_type "python"]}+ {:table :workspace_transform :pks [:workspace_id :ref_id]}]+ python? (fn [source]+ (= "python" (get (json-out source false) "type")))+ all-rows (into []+ (mapcat (fn [{:keys [table pks] w :where}]+ (->> (t2/query (cond-> {:select (into [:source] pks)+ :from [table]}+ w (assoc :where w)))+ (filter #(or w (python? (:source %))))+ (map #(assoc % ::table table ::pks pks)))))+ tables)+ all-ids (into #{}+ (mapcat (fn [{:keys [source]}]+ (let [st (get (json-out source false) "source-tables")]+ (when (map? st)+ (filter int? (vals st))))))+ all-rows)+ metadata (when (seq all-ids)+ (into {}+ (map (fn [{:keys [id db_id schema name]}]+ [id {"database_id" db_id "schema" schema "table" name}]))+ (t2/query {:select [:id :db_id :schema :name]+ :from [:metabase_table]+ :where [:in :id all-ids]})))]+ (doseq [{:keys [source] :as row} all-rows]+ (let [parsed (json-out source false)+ st (get parsed "source-tables")]+ (when (map? st)+ (let [entries (mapv (fn [[alias v]]+ (if (int? v)+ ;; Integer value: backfill all metadata (database_id, schema, table) from DB.+ (merge {"alias" alias "table_id" v}+ (get metadata v))+ ;; Ref-map value: already has database_id/schema/table; table_id is+ ;; resolved lazily by normalize-source-tables when the transform is saved via the API.+ (assoc v "alias" alias)))+ st)+ pks (::pks row)]+ (t2/query {:update (::table row)+ :set {:source (json-in (assoc parsed "source-tables" entries))}+ :where (into [:and] (map #(vector := % (get row %)) pks))}))))))+ (let [convert-back+ (fn [source-json]+ (let [parsed (json-out source-json false)]+ (when-let [st (get parsed "source-tables")]+ (when (sequential? st)+ (let [m (into {} (map (fn [entry]+ [(get entry "alias")+ (or (get entry "table_id") entry)]))+ st)]+ (json-in (assoc parsed "source-tables" m)))))))+ tables [{:table :transform :pks [:id] :where [:= :source_type "python"]}+ {:table :workspace_transform :pks [:workspace_id :ref_id]}]+ python? (fn [source]+ (= "python" (get (json-out source false) "type")))]+ (doseq [{:keys [table pks] w :where} tables]+ (doseq [row (cond->> (t2/query (cond-> {:select (into [:source] pks)+ :from [table]}+ w (assoc :where w)))+ (not w) (filter #(python? (:source %))))]+ (when-let [new-source (convert-back (:source row))]+ (t2/query {:update table+ :set {:source new-source}+ :where (into [:and] (map #(vector := % (get row %)) pks))}))))))(define-migration FixClickHouseUploadDBSchemaNames"This data migration is meant to fix the issues seen in #69667, #68298 and #65945.We made the driver feature `schemas` conditional on the `enable-multiple-db` DB connection setting.diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.cljindex 53ec780df1d..93bdc5cf98b 100644--- a/src/metabase/models/transforms/transform.clj+++ b/src/metabase/models/transforms/transform.clj@@ -68,29 +68,22 @@(perms/has-db-transforms-permission? api/*current-user-id* source-db-id)(remote-sync/transforms-editable?))))-(defn- keywordize-source-table-refs- "Keywordize keys in source-tables map values (refs are maps, ints pass through)."- [source-tables]- (update-vals source-tables #(if (map? %) (update-keys % keyword) %)))-(defn transform-source-out"Deserialize a transform source map from JSON storage format.- Normalizes queries, keywordizes types and source-table refs."+ Normalizes queries and keywordizes type fields."[m](-> mmi/json-out-without-keywordization(update-keys keyword)- (m/update-existing :source-tables keywordize-source-table-refs)(m/update-existing :query lib-be/normalize-query)- (m/update-existing :type keyword)- (m/update-existing :source-incremental-strategy #(update-keys % keyword))))+ (m/update-existing :source-incremental-strategy #(update-keys % keyword))+ (m/update-existing :source-tables (fn [st] (mapv #(update-keys % keyword) st)))+ (m/update-existing :type keyword)))(defn transform-source-in- "Serialize a transform source map for JSON storage.- Normalizes source-tables and prepares queries for serialization."+ "Serialize a transform source map for JSON storage."[m](-> m- (m/update-existing :source-tables transforms-base.u/normalize-source-tables)(m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))mi/json-in))@@ -337,7 +330,14 @@:collection_id (serdes/fk :model/Collection):source_database_id (serdes/fk :model/Database :name):source {:export #(update % :query serdes/export-mbql)- :import #(update % :query serdes/import-mbql)}+ :import (fn [source]+ (-> source+ (m/update-existing :source-tables+ (fn [st]+ (if (map? st)+ (transforms-base.u/source-tables-map->vec st)+ st)))+ (m/update-existing :query serdes/import-mbql)))}:target {:export serdes/export-mbql :import serdes/import-mbql}:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 813280d8245..88b43649a68 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -31,7 +31,7 @@add-source-readableis-temp-transform-table?][metabase.transforms.crud- python-source-table-ref->table-id+ source-tables-vec->map-for-fecheck-database-featurecheck-feature-enabled!extract-all-columns-from-querydiff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex 59f1438012d..38d21a3e6b5 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -26,16 +26,14 @@(set! *warn-on-reflection* true)-(defn python-source-table-ref->table-id- "Change source of python transform from name->table-ref to name->table-id.-- We now supported table-ref as source but since FE is still expecting table-id we need to temporarily do this.- Should update FE to fully use table-ref"+;; TODO(FE-source-tables): Remove this function and all call sites when FE adopts the array format for source-tables.+(defn source-tables-vec->map-for-fe+ "Convert source-tables from internal vec format to legacy map format for FE compatibility.+ Remove this when FE adopts the array format."[transform](if (transforms-base.u/python-transform? transform)(update-in transform [:source :source-tables]- (fn [source-tables]- (update-vals source-tables #(if (int? %) % (:table_id %)))))+ transforms-base.u/source-tables-vec->alias-id-map)transform))(defn check-database-feature@@ -157,8 +155,8 @@(transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)(transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)(map #(update % :last_run transforms-base.u/localize-run-timestamps))- (map python-source-table-ref->table-id)))- transforms.u/add-source-readable))))+ (map transforms.u/add-source-readable)+ (map source-tables-vec->map-for-fe))))))) ;; TODO(FE-source-tables): remove(defn get-transform"Get a specific transform."@@ -169,8 +167,8 @@(t2/hydrate :last_run :transform_tag_ids :creator :owner)(u/update-some :last_run transforms-base.u/localize-run-timestamps)(assoc :table target-table)- python-source-table-ref->table-id- transforms.u/add-source-readable)))+ transforms.u/add-source-readable+ source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove(defn create-transform!"Create new transform in the appdb.@@ -231,8 +229,8 @@(t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids :creator :owner))](events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})(-> transform- python-source-table-ref->table-id- transforms.u/add-source-readable)))+ transforms.u/add-source-readable+ source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove(defn delete-transform!"Delete a transform and publish the delete event."diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.cljindex 0bda7ccebd0..a235243e4b1 100644--- a/src/metabase/transforms/schema.clj+++ b/src/metabase/transforms/schema.clj@@ -3,22 +3,10 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.queries.schema :as queries.schema]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))-(mr/def ::source-table-ref- "A reference to a source table by name, for cases where table_id may not exist yet.- Also saves querying metadata in situations where we'll need the name."- [:map- [:database_id :int]- [:schema {:optional true} [:maybe :string]]- [:table :string]- [:table_id {:optional true} [:maybe :int]]])--(mr/def ::source-table-value- "Either a table ID (int) or a reference map."- [:or :int ::source-table-ref])-(mr/def ::checkpoint-strategy[:map[:type [:= "checkpoint"]]@@ -43,7 +31,12 @@[:map[:source-database {:optional true} :int];; NB: if source is checkpoint, only one table allowed- [:source-tables [:map-of :string ::source-table-value]]+ ;; decode/normalize: convert FE map format to vec and enrich with DB metadata+ [:source-tables [:sequential {:decode/normalize (fn [st]+ (if (map? st)+ (transforms-base.u/source-tables-map->vec st)+ st))}+ ::transforms-base.u/source-table-entry]][:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :python]][:body :string][:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]])@@ -77,8 +70,6 @@(mr/def ::id pos-int?)-(mr/def ::run-id pos-int?)-(mr/def ::transform[:map[:id ::id]diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex bdc495e2801..7692aa73a69 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -57,14 +57,7 @@(let [source-tables (:source-tables source)](if (empty? source-tables)true- (let [table-ids (into []- (comp (map val)- (map #(cond- (int? %) %- (map? %) (:table_id %)- :else nil))- (filter some?))- source-tables)]+ (let [table-ids (into [] (keep :table_id) source-tables)](and (seq table-ids)(every? (fn [table-id](when-let [table (t2/select-one :model/Table table-id)]diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 3eef908348b..c773f3fe390 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -553,6 +553,13 @@(execute-secondary-index-ddl-if-required!transform run-id database target with-stage-timing-fn)))+;;; ------------------------------------------------- Source Table Schemas -------------------------------------------------++(mu/defn source-tables-vec->alias-id-map :- [:map-of :string [:maybe :int]]+ "Convert source-table entries to `{alias -> table_id}` map."+ [entries :- [:sequential ::source-table-entry]]+ (into {} (map (juxt :alias :table_id)) entries))+;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------(def ^:private ^:const batch-lookup-chunk-size@@ -589,60 +596,87 @@[v](and (map? v) (nil? (:table_id v))))-(defn normalize-source-tables- "Normalize source-tables to consistent map format {:database_id :schema :table :table_id}.-- The old format stored just integer table IDs. New transforms store maps on write.- Old data is converted on read via transform-source-out for backwards compatibility.-+(mr/def ::source-table-entry+ "A source table entry in the array format. Combines alias with table reference."+ [:map+ [:alias :string]+ [:database_id {:optional true} :int]+ [:schema {:optional true} [:maybe :string]]+ [:table {:optional true} :string]+ [:table_id {:optional true} [:maybe :int]]])++(mu/defn normalize-source-tables :- [:sequential ::source-table-entry]+ "Normalize source-table entries by enriching them with full metadata.+ For entries with only :table_id, looks up :database_id/:schema/:table.+ For entries with only :database_id/:schema/:table, looks up :table_id.Throws if an integer table ID references a non-existent table.Map refs with non-existent tables get nil table_id (resolved later at execute time)."- [source-tables]- (let [int-table-ids (into #{} (filter int?) (vals source-tables))- int-id->metadata (when (seq int-table-ids)- (t2/select-pk->fn (fn [{:keys [db_id schema name]}]- {:database_id db_id :schema schema :table name})- [:model/Table :id :db_id :schema :name]- :id [:in int-table-ids]))- missing-ids (when (seq int-table-ids)- (remove int-id->metadata int-table-ids))- refs-needing-id (filter missing-table-id? (vals source-tables))+ [source-tables :- [:sequential ::source-table-entry]]+ (let [;; Entries that have table_id but lack table metadata need lookup+ needs-metadata (filter (fn [e] (and (:table_id e) (not (:table e)))) source-tables)+ int-id->metadata (when (seq needs-metadata)+ (let [ids (into #{} (map :table_id) needs-metadata)]+ (t2/select-pk->fn (fn [{:keys [db_id schema name]}]+ {:database_id db_id :schema schema :table name})+ [:model/Table :id :db_id :schema :name]+ :id [:in ids])))+ missing-ids (when (seq needs-metadata)+ (let [ids (into #{} (map :table_id) needs-metadata)]+ (remove (or int-id->metadata {}) ids)))+ refs-needing-id (filter missing-table-id? source-tables)ref-lookup (or (batch-lookup-table-ids refs-needing-id) {})](when (seq missing-ids)(throw (ex-info (str "Tables not found for ids: " (str/join ", " (sort missing-ids))){:table_ids (vec missing-ids)})))- (update-vals source-tables- (fn [v]- (cond- (int? v) (assoc (int-id->metadata v) :table_id v)- (:table_id v) v- :else (assoc v :table_id (ref-lookup (source-table-ref->key v))))))))--(defn resolve-source-tables- "Resolve source-tables to {alias -> table_id}. Throws if any table not found.- For execute time - all entries must resolve to valid table IDs.- Handles both integer IDs (old format) and map refs (new format)."- [source-tables]- (let [needs-lookup (filter missing-table-id? (vals source-tables))+ (mapv (fn [entry]+ (cond+ ;; Has table_id but no table metadata — enrich from DB+ (and (:table_id entry) (not (:table entry)))+ (merge (int-id->metadata (:table_id entry)) entry)++ ;; Has table metadata but no table_id — look it up+ (missing-table-id? entry)+ (assoc entry :table_id (ref-lookup (source-table-ref->key entry)))++ ;; Already fully populated+ :else entry))+ source-tables)))++(mu/defn resolve-source-tables :- [:map-of :string :int]+ "Resolve source-table entries to `{alias -> table_id}`. Throws if any table not found.+ For execute time — all entries must resolve to valid table IDs."+ [source-tables :- [:sequential ::source-table-entry]]+ (let [needs-lookup (filter missing-table-id? source-tables)lookup (or (batch-lookup-table-ids needs-lookup) {})- resolved (u/for-map [[alias v] source-tables]- [alias (if (int? v)- v- (or (:table_id v) (lookup (source-table-ref->key v))))])- unresolved (for [[alias table-id] resolved- :when (nil? table-id)- :let [v (get source-tables alias)]]- {:alias alias- :table (if-let [schema (:schema v)]- (str schema "." (:table v))- (:table v))- :ref v})]+ resolved (u/for-map [entry source-tables]+ [(:alias entry) (or (:table_id entry) (lookup (source-table-ref->key entry)))])+ unresolved (for [entry source-tables+ :let [table-id (or (:table_id entry) (lookup (source-table-ref->key entry)))]+ :when (nil? table-id)]+ {:alias (:alias entry)+ :table (if-let [schema (:schema entry)]+ (str schema "." (:table entry))+ (:table entry))+ :ref entry})](when (seq unresolved)(throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved))){:unresolved unresolved:transform-message "Input table not found"})))resolved))+(mu/defn source-tables-map->vec :- [:sequential ::source-table-entry]+ "Convert map format `{alias -> value}` to vec format `[{:alias alias ...}]`.+ Handles both int values (`{alias: table_id}`) and ref map values (`{alias: {:database_id ...}}`.+ Accepts both keyword and string keys for alias.+ Enriches entries with full metadata via [[normalize-source-tables]]."+ [m :- [:map-of [:or :string :keyword] [:or :int :map]]]+ (normalize-source-tables+ (mapv (fn [[alias v]]+ (if (int? v)+ {:alias (name alias) :table_id v}+ (assoc v :alias (name alias))))+ m)))+;;; ------------------------------------------------- Timestamp Helpers -------------------------------------------------(defn ->instantdiff --git a/src/metabase/transforms_inspector/context.clj b/src/metabase/transforms_inspector/context.cljindex d00ceee4e76..dd1d9f7e7c0 100644--- a/src/metabase/transforms_inspector/context.clj+++ b/src/metabase/transforms_inspector/context.clj@@ -64,7 +64,7 @@(try(let [source-tables (get-in transform [:source :source-tables])normalized (transforms-base.u/normalize-source-tables source-tables)- table-ids (keep (fn [[_ v]] (:table_id v)) normalized)]+ table-ids (keep :table_id normalized)](table-ids->source-info table-ids))(catch Exception e(log/warn e "Failed to extract sources from Python transform")diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 29a1523e572..4810afc61a6 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -174,8 +174,8 @@403(deferred-tru "A table with that name already exists."))(-> (transforms.core/create-transform! body)- transforms.core/python-source-table-ref->table-id- transforms.u/add-source-readable))+ transforms.u/add-source-readable+ transforms.core/source-tables-vec->map-for-fe)) ;; TODO(FE-source-tables): remove(api.macros/defendpoint :get "/:id" :- TransformResponse"Get a specific transform."@@ -193,8 +193,8 @@dep-ids (get global-ordering id)dependencies (map id->transform dep-ids)](->> (t2/hydrate dependencies :creator :owner)- (mapv transforms.core/python-source-table-ref->table-id)- transforms.u/add-source-readable)))+ transforms.u/add-source-readable+ (mapv transforms.core/source-tables-vec->map-for-fe)))) ;; TODO(FE-source-tables): remove(def ^:private MergeHistoryEntry[:mapdiff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex 1d35b53ee2b..38bfabdeba6 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -2776,6 +2776,86 @@(is (nil? (id->target none-id)))(is (nil? (id->target deleted-id))))))))+(deftest unify-source-tables-format-test+ (testing "v59.2026-03-03T12:00:00: convert source-tables from map to vec format"+ (impl/test-migrations ["v59.2026-03-03T12:00:00"] [migrate!]+ (let [user-id (:id (new-instance-with-default :core_user))+ db-id (:id (new-instance-with-default :metabase_database))+ ;; source-tables as map with int values (FE format)+ int-source (json/encode {:type "python" :body "x=1" :source-database db-id+ :source-tables {"orders" 42 "products" 99}})+ ;; source-tables as map with ref values (normalized format)+ ref-source (json/encode {:type "python" :body "x=1" :source-database db-id+ :source-tables {"input" {"database_id" db-id "schema" "public"+ "table" "my_table" "table_id" 7}}})+ ;; query transform (no source-tables) — should be untouched+ query-source (json/encode {:type "query" :query {:database db-id}})+ target (json/encode {:type "table" :schema "public" :name "out"})+ insert-transform!+ (fn [source]+ (t2/insert-returning-pk!+ :transform {:name (mt/random-name)+ :source source+ :target target+ :source_type "python"+ :source_database_id db-id+ :created_at :%now+ :updated_at :%now}))+ int-id (insert-transform! int-source)+ ref-id (insert-transform! ref-source)+ query-id (insert-transform! query-source)+ ;; Also test workspace_transform+ ws-id (:id (t2/insert-returning-instance!+ :workspace {:name "test-ws"+ :creator_id user-id+ :created_at :%now+ :updated_at :%now}))+ _ (t2/insert! :workspace_transform+ {:ref_id (str (random-uuid))+ :workspace_id ws-id+ :name "ws-transform"+ :source int-source+ :target target+ :created_at :%now+ :updated_at :%now})+ get-source-tables (fn [table-name pk-map]+ (let [where (into [:and] (map (fn [[k v]] [:= k v]) pk-map))+ row (first (t2/query {:select [:source] :from [table-name] :where where}))]+ (get (json/decode (:source row)) "source-tables")))]+ (testing "Before migration, source-tables are maps"+ (is (map? (get-source-tables :transform {:id int-id})))+ (is (map? (get-source-tables :transform {:id ref-id}))))++ (migrate!)++ (testing "After migration, int-value maps become vec of entries"+ (let [st (get-source-tables :transform {:id int-id})]+ (is (sequential? st))+ (is (= 2 (count st)))+ (is (= #{"orders" "products"} (set (map #(get % "alias") st))))+ (is (= #{42 99} (set (map #(get % "table_id") st))))))++ (testing "After migration, ref-value maps become vec of entries with alias"+ (let [st (get-source-tables :transform {:id ref-id})]+ (is (sequential? st))+ (is (= "input" (get (first st) "alias")))+ (is (= 7 (get (first st) "table_id")))))++ (testing "Query transforms are untouched"+ (is (nil? (get-source-tables :transform {:id query-id}))))++ (testing "workspace_transform is also migrated"+ (let [ws-rows (t2/query {:select [:source] :from [:workspace_transform] :where [:= :workspace_id ws-id]})+ st (get (json/decode (:source (first ws-rows))) "source-tables")]+ (is (sequential? st))+ (is (= 2 (count st)))))++ (testing "Rollback converts vec back to map"+ (migrate! :down 58)+ (let [st (get-source-tables :transform {:id int-id})]+ (is (map? st))+ (is (= 42 (get st "orders")))+ (is (= 99 (get st "products")))))))))(deftest fix-clickhouse-upload-db-schema-names-test(testing "FixClickHouseUploadDBSchemaNames, v59.2026-03-04T00:00:00: fix clickhouse upload db schema names"(encryption-test/with-secret-key "fake-secret-key"diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex b307976b702..a14716343a7 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -185,72 +185,69 @@(is (= {} result)))))))(deftest normalize-source-tables-test- (testing "normalize-source-tables converts all entries to map format"+ (testing "normalize-source-tables enriches vec entries"(mt/with-temp [:model/Database db {}:model/Table t1 {:db_id (:id db) :name "existing_table" :schema nil}]- (testing "converts integer table ID to map format"- (let [result (transforms-base.u/normalize-source-tables {"t" (:id t1)})]- (is (map? (get result "t")))- (is (= (:id db) (get-in result ["t" :database_id])))- (is (= "existing_table" (get-in result ["t" :table])))- (is (= (:id t1) (get-in result ["t" :table_id])))))-- (testing "throws for non-existent integer table ID"+ (testing "enriches entry with table_id but no table metadata"+ (let [result (transforms-base.u/normalize-source-tables [{:alias "t" :table_id (:id t1)}])+ entry (first result)]+ (is (= (:id db) (:database_id entry)))+ (is (= "existing_table" (:table entry)))+ (is (= (:id t1) (:table_id entry)))))++ (testing "throws for non-existent table_id"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found for ids: 999999"- (transforms-base.u/normalize-source-tables {"t" 999999}))))+ (transforms-base.u/normalize-source-tables [{:alias "t" :table_id 999999}]))))- (testing "populates table_id for existing table"- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table"}}+ (testing "populates table_id for existing table ref"+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "existing_table"}]result (transforms-base.u/normalize-source-tables source-tables)]- (is (= (:id t1) (get-in result ["t" :table_id])))))+ (is (= (:id t1) (:table_id (first result))))))- (testing "preserves existing table_id"- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table" :table_id 999}}+ (testing "preserves existing table_id when table metadata present"+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "existing_table" :table_id 999}]result (transforms-base.u/normalize-source-tables source-tables)]- (is (= 999 (get-in result ["t" :table_id])))))+ (is (= 999 (:table_id (first result))))))(testing "leaves table_id nil for non-existent table ref"- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]result (transforms-base.u/normalize-source-tables source-tables)]- (is (nil? (get-in result ["t" :table_id])))))+ (is (nil? (:table_id (first result))))))- (testing "handles mixed int and map entries"- (let [source-tables {"t1" (:id t1)- "t2" {:database_id (:id db) :schema nil :table "existing_table"}}+ (testing "handles entries needing different kinds of enrichment"+ (let [source-tables [{:alias "t1" :table_id (:id t1)}+ {:alias "t2" :database_id (:id db) :schema nil :table "existing_table"}]result (transforms-base.u/normalize-source-tables source-tables)]- (is (map? (get result "t1")))- (is (= (:id t1) (get-in result ["t1" :table_id])))- (is (= (:id t1) (get-in result ["t2" :table_id]))))))))+ (is (= "existing_table" (:table (first result))))+ (is (= (:id t1) (:table_id (first result))))+ (is (= (:id t1) (:table_id (second result)))))))))(deftest resolve-source-tables-test- (testing "resolve-source-tables returns {alias -> table_id} map"+ (testing "resolve-source-tables returns {alias -> table_id} map from vec input"(mt/with-temp [:model/Database db {}:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}:model/Table t2 {:db_id (:id db) :name "table_two" :schema nil}]- (testing "passes through integer entries (old format)"- (is (= {"t" 123} (transforms-base.u/resolve-source-tables {"t" 123}))))-- (testing "resolves map with table_id"- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}}]+ (testing "resolves entry with table_id"+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}]](is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))- (testing "looks up table_id for map without it"- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one"}}]+ (testing "looks up table_id for entry without it"+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}]](is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))(testing "throws for non-existent table"- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}]+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]](is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: nonexistent"(transforms-base.u/resolve-source-tables source-tables)))))(testing "throws with schema in error message"- (let [source-tables {"t" {:database_id (:id db) :schema "my_schema" :table "nonexistent"}}]+ (let [source-tables [{:alias "t" :database_id (:id db) :schema "my_schema" :table "nonexistent"}]](is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: my_schema\.nonexistent"(transforms-base.u/resolve-source-tables source-tables)))))- (testing "handles mixed entries (old and new format)"- (let [source-tables {"t1" (:id t1)- "t2" {:database_id (:id db) :schema nil :table "table_two"}}]+ (testing "handles multiple entries"+ (let [source-tables [{:alias "t1" :table_id (:id t1)}+ {:alias "t2" :database_id (:id db) :schema nil :table "table_two"}]](is (= {"t1" (:id t1) "t2" (:id t2)}(transforms-base.u/resolve-source-tables source-tables))))))))@@ -269,12 +266,12 @@(testing "returns true for python transform when user can read all source tables"(let [transform {:source {:type :python- :source-tables {"t1" table-id}}}]+ :source-tables [{:alias "t1" :table_id table-id}]}}](is (true? (transforms.u/source-tables-readable? transform)))))- (testing "handles source tables with table_id in map format"+ (testing "handles source tables with table_id"(let [transform {:source {:type :python- :source-tables {"t1" {:table_id table-id}}}}]+ :source-tables [{:alias "t1" :table_id table-id}]}}](is (true? (transforms.u/source-tables-readable? transform))))))))))))(deftest source-tables-readable-permissions-test@@ -298,7 +295,7 @@(testing "Python transforms - blocked database access"(let [transform {:source {:type :python- :source-tables {"t1" table1-id}}}]+ :source-tables [{:alias "t1" :table_id table1-id}]}}](mt/with-user-in-groups [group {:name "Blocked Group"}user [group]](mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked@@ -309,8 +306,8 @@(testing "Python transforms - granular access but missing some tables"(let [transform {:source {:type :python- :source-tables {"t1" table1-id- "t2" table2-id}}}]+ :source-tables [{:alias "t1" :table_id table1-id}+ {:alias "t2" :table_id table2-id}]}}](mt/with-user-in-groups [group {:name "Limited Granular Group"}user [group]];; Block all-users from viewing data at database level, then grant specific access via test groupdiff --git a/test/metabase/transforms_inspector/context_test.clj b/test/metabase/transforms_inspector/context_test.cljindex 30e05cda4b0..970763188bb 100644--- a/test/metabase/transforms_inspector/context_test.clj+++ b/test/metabase/transforms_inspector/context_test.clj@@ -176,9 +176,9 @@(is (contains? table-ids (mt/id :products))))))(deftest extract-sources-python-test- (testing "Python: extracts source tables from source-tables map"+ (testing "Python: extracts source tables from source-tables vec"(let [transform {:source {:type :python- :source-tables {"orders" (mt/id :orders)}+ :source-tables [{:alias "orders" :table_id (mt/id :orders)}]:source-database (mt/id)}:name "test"}sources (context/extract-sources transform)]@@ -189,8 +189,8 @@(deftest extract-sources-python-multiple-tables-test(testing "Python: extracts multiple source tables"(let [transform {:source {:type :python- :source-tables {"orders" (mt/id :orders)- "products" (mt/id :products)}+ :source-tables [{:alias "orders" :table_id (mt/id :orders)}+ {:alias "products" :table_id (mt/id :products)}]:source-database (mt/id)}:name "test"}sources (context/extract-sources transform)diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 08a232bb38d..0c1a1949f09 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -1750,7 +1750,7 @@:query (lib/native-query (mt/metadata-provider) "SELECT 1")}python-source {:type "python":body "print('hello world')"- :source-tables {}+ :source-tables []:source-database (mt/id)}](mt/with-temp [:model/Transform {query-id :id} {:name query-name:source query-source