Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Unifying source tables transform format (#70298)

Ngoc Khuatc86caa092e2a
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
index 21511052820..41260a6060a 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -47,8 +47,8 @@
(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
[{{tables :source-tables} :source :as _py-transform}
- :- [:map [:source-tables {:optional true} [:map-of :string [:or :int [:map [:table_id :int]]]]]]]
- {:table (into #{} (keep (fn [v] (if (map? v) (:table_id v) v))) (vals tables))})
+ :- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]
+ {:table (into #{} (keep :table_id) tables)})
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
"Given a Transform (in Toucan form), return its upstream dependencies."
@@ -58,8 +58,7 @@
[:query
[:map [:query ::lib.schema/query]]]
[:python
- ;; If the upstream table doesn't exist yet, table_id will be nil
- [:map [:source-tables {:optional true} [:map-of :string [:or :int [:map [:table_id [:maybe :int]]]]]]]]]]]]
+ [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]]]]]
(let [source-type (transforms-base.u/transform-type transform)]
(case source-type
:query (upstream-deps:query query)
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
index c1518645217..5586b5e8696 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
@@ -99,7 +99,7 @@
(when-let [source-database (get-in item [:source :source-database])]
(when-let [source-tables (not-empty (get-in item [:source :source-tables]))]
{:database-id source-database
- :table-ids (vals source-tables)}))))
+ :table-ids (map :table_id source-tables)}))))
(defn- python-transform-tables-for-context
"Get tables for Python transform formatted for metabot context."
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index f8d556caae4..9ec54e49cd0 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -60,7 +60,7 @@
[transform]
(into #{}
(map source-table-value->dependency)
- (vals (get-in transform [:source :source-tables]))))
+ (get-in transform [:source :source-tables])))
;;; ------------------------------------------------- Message Log (in-memory only) -------------------------------------------------
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
index 6df753785a3..c80663c759d 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
@@ -27,6 +27,7 @@
[metabase.lib.core :as lib]
[metabase.queries.schema :as queries.schema]
[metabase.request.core :as request]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.core :as transforms]
[metabase.transforms.feature-gating :as transforms.gating]
[metabase.transforms.util :as transforms.u]
@@ -79,7 +80,12 @@
[:python
[:map {:closed true}
[:source-database {:optional true} :int]
- [:source-tables [:map-of [:string {:min 1}] [:or :int :map]]]
+ ;; TODO (Ngoc 2026-03-04) -- remove decode/normalize when FE sends array format for source-tables
+ [:source-tables [:sequential {:decode/normalize (fn [st]
+ (if (map? st)
+ (transforms-base.u/source-tables-map->vec st)
+ st))}
+ [:map [:alias [:string {:min 1}]] [:table_id :int]]]]
[:type [:= "python"]]
[:body :string]]]])
@@ -828,7 +834,10 @@
(deferred-tru "Another transform in this workspace already targets that table"))
transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body
:ref-id ref-id)]
- (attach-isolated-target (select-malli-keys WorkspaceTransform workspace-transform-alias transform)))))
+ (-> (select-malli-keys WorkspaceTransform workspace-transform-alias transform)
+ attach-isolated-target
+ ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables
+ transforms/source-tables-vec->map-for-fe))))
(api.macros/defendpoint :post "/:ws-id/transform"
:- WorkspaceTransform
@@ -876,7 +885,9 @@
(-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)
(t2/select-one :workspace_id ws-id :ref_id tx-id)
api/check-404
- attach-isolated-target))
+ attach-isolated-target
+ ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables
+ transforms/source-tables-vec->map-for-fe))
(api.macros/defendpoint :get "/:ws-id/transform/:tx-id" :- WorkspaceTransform
"Get a specific transform in a workspace."
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
index c97c323abc1..5f4f6a732dc 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
@@ -175,25 +175,20 @@
(inputs-from-native-query query db-id)
(inputs-from-mbql-query query)))
-(defn- source-table-ref->table-ref
- "Paper over a small unfortunate difference in shape"
- [table-ref]
- (-> (select-keys table-ref [:schema :table :table_id])
- (assoc :db_id (:database_id table-ref))))
-
(defn- inputs-from-python-transform
- "Extract table refs from a python transform's source-tables.
- Uses metadata from map refs directly; only looks up the database for bare integer IDs."
+ "Extract table refs from a python transform's source-tables vec.
+ Entries with :database_id/:schema/:table just need a key rename;
+ entries with only :table_id fall back to a batch lookup."
[source-tables]
- (let [{maps true ints false} (group-by (comp map? val) source-tables)
- ;; Maps already have schema/table metadata
- map-refs (map (comp source-table-ref->table-ref val) maps)
- ;; Integers need a batch lookup
- int-ids (map val ints)
- int-refs (when (seq int-ids)
- (let [lookup (batch-table-refs-from-ids int-ids)]
- (keep lookup int-ids)))]
- (into (vec map-refs) int-refs)))
+ (let [{full true, partial false} (group-by (fn [e] (boolean (:table e))) source-tables)
+ renamed (mapv (fn [{:keys [database_id schema table]}]
+ {:db_id database_id :schema schema :table table})
+ full)
+ looked (when (seq partial)
+ (let [ids (into #{} (keep :table_id) partial)
+ lookup (batch-table-refs-from-ids ids)]
+ (u/keepv lookup (map :table_id partial))))]
+ (into renamed looked)))
(mu/defn analyze-entity :- ::analysis
"Analyze a workspace entity to find its dependencies.
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
index ac636ef9e8f..13f27c7cb66 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
@@ -11,25 +11,27 @@
[metabase.transforms-base.core :as transforms-base]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
-(defn- remap-python-source
- "Remap source-tables in a Python transform source to point to isolated tables.
- Handles both legacy integer format and new map format (from PR #66934)."
- [table-mapping source]
- (letfn [(remap [{:keys [database_id schema table table_id] :as table-ref}]
- (if-let [{:keys [id db-id schema table]} (or (table-mapping table-ref)
- (some-> table_id table-mapping)
- (when table
- (table-mapping [database_id schema table])))]
- ;; Prefer table ID when available; map format won't work until PR #66934 is merged:
- ;; https://github.com/metabase/metabase/pull/66934
- (or id {:database_id db-id, :schema schema, :table table})
- ;; Leave it un-mapped if we don't have an override.
- table-ref))]
- (update source :source-tables update-vals remap)))
+(mu/defn- remap-python-source
+ "Remap source-tables in a Python transform source to point to isolated tables."
+ [table-mapping source :- [:map [:source-tables [:sequential ::transforms-base.u/source-table-entry]]]]
+ (letfn [(remap [{:keys [database_id schema table table_id] :as entry}]
+ (if-let [{:keys [id db-id] target-schema :schema target-table :table}
+ (or (table-mapping entry)
+ (some-> table_id table-mapping)
+ (when table
+ (table-mapping [database_id schema table])))]
+ (cond-> entry
+ id (assoc :table_id id)
+ db-id (assoc :database_id db-id)
+ target-schema (assoc :schema target-schema)
+ target-table (assoc :table target-table))
+ entry))]
+ (update source :source-tables (fn [entries] (mapv remap entries)))))
(defn- remap-sql-source [table-mapping source]
(let [remapping (reduce
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
index b0dabaf479f..5238f8f0dbd 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
@@ -199,8 +199,8 @@
orders-id (mt/id :orders)]
(mt/with-temp [:model/Transform transform {:name "Test Transform"
:source {:type :python
- :source-tables {"PRODUCTS" products-id
- "ORDERS" orders-id}
+ :source-tables [{:alias "PRODUCTS" :table_id products-id}
+ {:alias "ORDERS" :table_id orders-id}]
;; A problematic field, hopefully removed again.
:source-database (mt/id)
:body "..."}
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index bcd554ddc15..1cb90946923 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -323,10 +323,11 @@
(fn [_]
(let [source {:type :python,
:source-database (mt/id),
- :source-tables {"orders" {:database_id (mt/id),
- :schema "public",
- :table "orders",
- :table_id (mt/id :orders)}},
+ :source-tables [{:alias "orders"
+ :database_id (mt/id)
+ :schema "public"
+ :table "orders"
+ :table_id (mt/id :orders)}],
:body
"import pandas as pd\n\ndef transform(orders):\n return orders"}]
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
@@ -366,10 +367,11 @@
(fn [_]
(let [source {:type :python,
:source-database (mt/id),
- :source-tables {"orders" {:database_id (mt/id),
- :schema "public",
- :table "orders",
- :table_id (mt/id :orders)}},
+ :source-tables [{:alias "orders"
+ :database_id (mt/id)
+ :schema "public"
+ :table "orders"
+ :table_id (mt/id :orders)}],
:body
"import pandas as pd\n\ndef transform(orders):\n return orders"}
target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj
index 80c7539e939..1bf446caaeb 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj
@@ -143,8 +143,8 @@
(let [input {:user_is_viewing [{:type "transform"
:source {:type "python"
:source-database 2
- :source-tables {:orders 24
- :products 31}}}]}
+ :source-tables [{:alias "orders" :table_id 24}
+ {:alias "products" :table_id 31}]}}]}
result (#'context/enhance-context-with-schema input)
used-tables (get-in result [:user_is_viewing 0 :used_tables])]
(is (= mock-tables used-tables)))))))
@@ -157,7 +157,7 @@
(let [input {:user_is_viewing [{:type "transform"
:source {:type "python"
:source-database 2
- :source-tables {}}}]}
+ :source-tables []}}]}
result (#'context/enhance-context-with-schema input)]
(is (nil? (get-in result [:user_is_viewing 0 :used_tables])))
(is (false? @called?)))))))
@@ -169,7 +169,7 @@
(fn [_] (reset! called? true) nil)]
(let [input {:user_is_viewing [{:type "transform"
:source {:type "python"
- :source-tables {:orders 24}}}]}
+ :source-tables [{:alias "orders" :table_id 24}]}}]}
result (#'context/enhance-context-with-schema input)]
(is (nil? (get-in result [:user_is_viewing 0 :used_tables])))
(is (false? @called?)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
index 55b293ea6ed..e3553f054d7 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
@@ -19,6 +19,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
+ [metabase.transforms.crud :as transforms.crud]
[metabase.util :as u]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
@@ -1291,7 +1292,7 @@
:source {:type "python"
:source-database (mt/id)
:body "print('hello world')"
- :source-tables {}}
+ :source-tables []}
:target {:type "table"
:name "t2_table"}}]
(testing "With insufficient permissions"
@@ -1302,7 +1303,9 @@
(testing "With superuser permissions"
(is (=? {:structured_output [(mt/obj->json->obj (select-keys t1 [:id :entity_id :name :description :source]))
;; note: t2 not included because it's a (non-native) MBQL query
- (mt/obj->json->obj (select-keys t3 [:id :entity_id :name :description :source]))]
+ ;; API converts source-tables from vec back to map for FE
+ (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe
+ (select-keys t3 [:id :entity_id :name :description :source])))]
:conversation_id conversation-id}
(-> (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transforms"
{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}
@@ -1328,7 +1331,7 @@
:source {:type "python"
:body "print('hello world')"
:source-database (mt/id)
- :source-tables {}}
+ :source-tables []}
:target {:type "table"
:name "t2_table"
:database (mt/id)}}]
@@ -1347,7 +1350,9 @@
(testing "With superuser permissions"
(doseq [transform [t1 t2]]
(testing (:name transform)
- (is (=? {:structured_output (mt/obj->json->obj (select-keys transform [:id :entity_id :name :description :source :target]))
+ ;; API converts source-tables from vec back to map for FE
+ (is (=? {:structured_output (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe
+ (select-keys transform [:id :entity_id :name :description :source :target])))
:conversation_id conversation-id}
(mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transform-details"
{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj
index 2dd412475dc..0c42a2c1831 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj
@@ -251,13 +251,13 @@
:source {:type "python"
:body "print('hello')"
:source-database (mt/id)
- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}}
+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}
:target {:type "table"
:schema "public"
:name "python_transform_table"}}]
(let [modified-source {:type "python"
:body "print('modified')"
- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}}
+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}
result (metabot.dependencies/check-transform-dependencies
{:id python-transform-id
:source modified-source})]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 0a69cfa3eac..445e60ff65a 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -535,6 +535,36 @@ serdes/meta:
entity-id
(str/replace (u/lower-case-en name) #"\s+" "_")))
+(defn generate-python-transform-yaml
+ "Generate YAML content for a python transform with old map-format source-tables.
+ Used to test backward compatibility when importing legacy format data.
+ `source-tables-yaml` should be a YAML string representing the source-tables map,
+ e.g. \"{orders: 123, products: 456}\"."
+ [entity-id name db-name source-tables-yaml & {:keys [collection-id]}]
+ (format "name: %s
+description: null
+entity_id: %s
+collection_id: %s
+created_at: '2024-08-28T09:46:18.671622Z'
+creator_id: [email protected]
+source_database_id: %s
+source:
+ type: python
+ body: 'import pandas as pd'
+ source-tables: %s
+target:
+ type: table
+ database: %s
+ name: test_output
+ schema: PUBLIC
+serdes/meta:
+- id: %s
+ label: %s
+ model: Transform
+"
+ name entity-id (or collection-id "null") db-name source-tables-yaml db-name
+ entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
+
(defn generate-transform-yaml
"Generate YAML content for a transform with the given `entity-id` and `name`.
Optionally accepts `:collection-id` for transforms inside a collection."
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index be52c6d80dc..0141acbd197 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -326,6 +326,39 @@ is_sample: false
(is (t2/exists? :model/Transform :entity_id transform-entity-id)
"Transform should be imported from transforms-namespace collection")))))))
+(deftest import-python-transform-with-legacy-map-source-tables-test
+ (testing "Import of a python transform with old map-format source-tables converts to vec format"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temporary-setting-values [remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/Transform :model/Collection]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ coll-entity-id "xforms-coll-legacy-01"
+ transform-entity-id "python-legacy-fmt-01x"
+ ;; Use old map format: {alias: table_id} — this is how pre-unification exports look
+ source-tables-yaml (format "{orders: %d}" (mt/id :orders))
+ test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")
+ (generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")
+ (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_python_transform.yaml")
+ (test-helpers/generate-python-transform-yaml transform-entity-id "Python Transform"
+ "test-data (h2)" source-tables-yaml
+ :collection-id coll-entity-id)}}
+ mock-source (test-helpers/create-mock-source :initial-files test-files)
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result))
+ (str "Import should succeed. Result: " result))
+ (when-let [transform (t2/select-one :model/Transform :entity_id transform-entity-id)]
+ (let [source-tables (get-in transform [:source :source-tables])]
+ (testing "source-tables is stored as vec format after import"
+ (is (sequential? source-tables)))
+ (testing "entry has correct alias and table_id"
+ (let [entry (first source-tables)]
+ (is (= "orders" (:alias entry)))
+ (is (= (mt/id :orders) (:table_id entry)))))
+ (testing "entry is enriched with table metadata"
+ (let [entry (first source-tables)]
+ (is (= (mt/id) (:database_id entry)))
+ (is (some? (:table entry)))))))))))))
+
(deftest archived-transforms-namespace-collection-excluded-from-export-test
(testing "When a transforms-namespace collection is archived, it and its children are excluded from export"
(mt/with-premium-features #{:transforms}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index fb2a28e54f0..28e0ce4aec4 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -30,7 +30,7 @@
{:name "Python Transform"
:source {:type "python"
:body "print('hello world')"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}
:target {:type "table"
:schema (get-test-schema)
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index a02eb8d6dcb..5957224e89d 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -153,7 +153,7 @@
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-unique-key lib-column-key}}
:python {:type "python"
- :source-tables {"transforms_products" (mt/id :transforms_products)}
+ :source-tables [{:alias "transforms_products" :table_id (mt/id :transforms_products)}]
:limit 10
:body incremental-python-body
:source-incremental-strategy {:type "checkpoint"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
deleted file mode 100644
index f7ffcac24e2..00000000000
--- a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
+++ /dev/null
@@ -1,63 +0,0 @@
-(ns metabase-enterprise.transforms.models-test
- (:require
- [clojure.test :refer [deftest testing is]]
- [metabase.test :as mt]
- [toucan2.core :as t2]))
-
-(deftest python-transform-source-table-resolution-on-save-test
- (testing "Python transform source-tables with name refs get table_id resolved on save"
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Table {table-id :id} {:db_id db-id
- :name "existing_table"
- :schema "public"}]
- (testing "table_id is populated when table exists"
- (mt/with-temp [:model/Transform transform {:name "Transform with ref"
- :source {:type "python"
- :body "def transform(): pass"
- :source-database db-id
- :source-tables {"input" {:database_id db-id
- :schema "public"
- :table "existing_table"}}}
- :target {:type "table"
- :schema "public"
- :name "output"
- :database db-id}}]
- (let [saved (t2/select-one :model/Transform :id (:id transform))
- source-table (get-in saved [:source :source-tables "input"])]
- (is (= table-id (:table_id source-table))
- "table_id should be resolved from database lookup"))))
-
- (testing "table_id remains nil when table doesn't exist"
- (mt/with-temp [:model/Transform transform {:name "Transform with missing ref"
- :source {:type "python"
- :body "def transform(): pass"
- :source-database db-id
- :source-tables {"input" {:database_id db-id
- :schema "public"
- :table "nonexistent_table"}}}
- :target {:type "table"
- :schema "public"
- :name "output"
- :database db-id}}]
- (let [saved (t2/select-one :model/Transform :id (:id transform))
- source-table (get-in saved [:source :source-tables "input"])]
- (is (nil? (:table_id source-table))
- "table_id should be nil for non-existent table"))))
-
- (testing "existing table_id is preserved"
- (mt/with-temp [:model/Transform transform {:name "Transform with explicit table_id"
- :source {:type "python"
- :body "def transform(): pass"
- :source-database db-id
- :source-tables {"input" {:database_id db-id
- :schema "public"
- :table "existing_table"
- :table_id 999}}}
- :target {:type "table"
- :schema "public"
- :name "output"
- :database db-id}}]
- (let [saved (t2/select-one :model/Transform :id (:id transform))
- source-table (get-in saved [:source :source-tables "input"])]
- (is (= 999 (:table_id source-table))
- "explicit table_id should be preserved")))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
index f6a81f187a7..75702b2064a 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
@@ -25,13 +25,14 @@
(testing "Python transform with name-based source table ref resolves to producing transform"
(mt/with-temp [;; Transform A produces table "intermediate_output"
:model/Transform {t-a :id} (make-python-transform
- {"input" (mt/id :orders)}
+ [{:alias "input" :table_id (mt/id :orders)}]
"intermediate_output")
;; Transform B references intermediate_output by name (table doesn't exist yet)
:model/Transform {t-b :id} (make-python-transform
- {"source" {:database_id (mt/id)
- :schema "public"
- :table "intermediate_output"}}
+ [{:alias "source"
+ :database_id (mt/id)
+ :schema "public"
+ :table "intermediate_output"}]
"final_output")]
(testing "table-dependencies returns table-ref for unresolved name reference"
(let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))]
@@ -47,15 +48,14 @@
(deftest python-transform-mixed-source-tables-test
(testing "Python transform with mixed int and name-based refs"
(mt/with-temp [:model/Transform {t-a :id} (make-python-transform
- {"input" (mt/id :orders)}
+ [{:alias "input" :table_id (mt/id :orders)}]
"output_a")
:model/Transform {t-b :id} (make-python-transform
- {;; Direct table reference (existing table)
- "existing" (mt/id :products)
- ;; Name-based reference (table doesn't exist yet)
- "from_transform" {:database_id (mt/id)
- :schema "public"
- :table "output_a"}}
+ [{:alias "existing" :table_id (mt/id :products)}
+ {:alias "from_transform"
+ :database_id (mt/id)
+ :schema "public"
+ :table "output_a"}]
"output_b")]
(testing "table-dependencies includes both types"
(let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
index 606a8789afe..7fbba9b52f6 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -457,9 +457,9 @@
" # Return processed dataframe\n"
" return df")
- source-tables (cond-> {source-table-name source-table-id}
+ source-tables (cond-> [{:alias source-table-name :table_id source-table-id}]
exotic-table-id
- (assoc (str source-table-name "_exotic") exotic-table-id))
+ (conj {:alias (str source-table-name "_exotic") :table_id exotic-table-id}))
result (execute-e2e-transform! table-name transform-code source-tables)
{:keys [columns] result-rows :rows} result
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
index 17eb49b1e3c..b164884b271 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
@@ -44,7 +44,7 @@
transform-payload {:name "Library Test Transform"
:source {:type "python"
:source-database (mt/id)
- :source-tables {}
+ :source-tables []
:body transform-body}
:target (assoc target :database (mt/id))}
{transform-id :id} (mt/user-http-request :crowberto :post 200 "transform"
@@ -71,7 +71,7 @@
(let [original {:name "Gadget Products"
:source {:type "python"
:source-database (mt/id)
- :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
index 4fe6138a118..fd036fe1d1a 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
@@ -25,7 +25,7 @@
:name "swap_tbl"}]
(let [initial-transform {:name "Python Transform Initial"
:source {:type "python"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)
:body (str "import pandas as pd\n"
"\n"
@@ -41,7 +41,7 @@
(t2/update! :model/Transform (:id transform)
{:source {:type "python"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)
:body (str "import pandas as pd\n"
"\n"
@@ -77,7 +77,7 @@
:name "cleanup_"}]
(let [transform-def {:name "Python Transform Cleanup"
:source {:type "python"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)
:body (str "import pandas as pd\n"
"\n"
@@ -116,7 +116,7 @@
" return pd.DataFrame({'result': ['should_not_reach_here']})")
transform-def {:name "Python Transform Timeout Test"
:source {:type "python"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)
:body long-running-code}
:target (assoc target :database (mt/id))}]
@@ -146,9 +146,10 @@
:source {:type "python"
:body "def transform(input): return input"
:source-database (mt/id)
- :source-tables {"input" {:database_id (mt/id)
- :schema schema
- :table "nonexistent_table"}}}
+ :source-tables [{:alias "input"
+ :database_id (mt/id)
+ :schema schema
+ :table "nonexistent_table"}]}
:target (assoc target :database (mt/id))}]
(testing "Execution throws with informative error message"
(is (thrown-with-msg?
@@ -168,9 +169,10 @@
:source {:type "python"
:body "def transform(input): return input"
:source-database (mt/id)
- :source-tables {"input" {:database_id (mt/id)
- :schema nil
- :table "missing_table"}}}
+ :source-tables [{:alias "input"
+ :database_id (mt/id)
+ :schema nil
+ :table "missing_table"}]}
:target (assoc target :database (mt/id))}]
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
index af3c34383e4..473dd057f83 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
@@ -29,7 +29,7 @@
:model/Transform {transform-id :id} {:name "Gadget Products"
:source {:type "python"
:source-database (mt/id)
- :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index 441ef4ab21a..fb46a322bac 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -37,8 +37,8 @@
(deftest python-transform-basic-dependencies-test
(testing "Python transforms with source-tables dependencies are extracted correctly"
- (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
- "products" (mt/id :products)})]
+ (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}
+ {:alias "products" :table_id (mt/id :products)}])]
(is (= #{{:table (mt/id :orders)}
{:table (mt/id :products)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
@@ -51,8 +51,8 @@
:name "output_1"}
:model/Field _ {:table_id table1
:name "foo"}
- :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
- :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+ :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")
+ :model/Transform {t2 :id} (make-python-transform [{:alias "output_1" :table_id table1}] "output_2")]
(is (= {t1 #{}
t2 #{t1}}
(ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]])))))))))
@@ -69,10 +69,10 @@
:name "output_2"}
:model/Field _ {:table_id table2
:name "bar"}
- :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
- :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
- :model/Transform {t3 :id} (make-python-transform {"output_1" table1
- "output_2" table2} "final_output")]
+ :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")
+ :model/Transform {t2 :id} (make-python-transform [{:alias "products" :table_id (mt/id :products)}] "output_2")
+ :model/Transform {t3 :id} (make-python-transform [{:alias "output_1" :table_id table1}
+ {:alias "output_2" :table_id table2}] "final_output")]
(is (= {t1 #{}
t2 #{}
t3 #{t1 t2}}
@@ -97,7 +97,7 @@
:query {:source-table (mt/id :orders)}}
"sql_output")
;; Python transform that depends on the SQL transform's output
- :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+ :model/Transform {t2 :id} (make-python-transform [{:alias "sql_output" :table_id table1}] "python_output")
;; Another SQL transform that depends on the Python transform's output
:model/Transform {t3 :id} (make-transform
{:database (mt/id)
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
index a5cba2543be..b55d23c7673 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -32,7 +32,7 @@
transform-payload {:name "My beautiful python runner"
:source {:type "python"
:body "print('hello world')"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}
:target {:type "table"
:schema schema
@@ -49,7 +49,7 @@
{:name "My beautiful python runner"
:source {:type "python"
:body "print('hello world')"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}
:target {:type "table"
:schema (get-test-schema)
@@ -64,7 +64,7 @@
{:name "My beautiful python runner"
:source {:type "python"
:body "print('hello world')"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}
:target {:type "table"
:schema (get-test-schema)
@@ -80,7 +80,7 @@
{:name "My beautiful python runner"
:source {:type "python"
:body "print('hello chris')"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}
:target {:type "table"
:schema (get-test-schema)
@@ -96,7 +96,7 @@
:as transform} {:name "Original Python Transform"
:source {:type "python"
:body "print('original')"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}
:target {:type "table"
:schema "scheam"
@@ -120,7 +120,7 @@
transform-payload {:name "Test Run Python Transform"
:source {:type "python"
:body "def transform():\n pass"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}
:target {:type "table"
:schema schema
@@ -149,7 +149,7 @@
(let [original {:name "Gadget Products"
:source {:type "python"
:source-database (mt/id)
- :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -223,7 +223,7 @@
:source {:type "python"
:body (program->source program)
:source-database (mt/id)
- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}}
+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}
:target (assoc target :database (mt/id))})))
(block-on-run [{:keys [expect-status]} target transform-id]
@@ -329,7 +329,7 @@
:model/Transform transform {:name "Python Transform Cross DB"
:source {:type "python"
:source-database (mt/id)
- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}
+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]
:body "def transform():\n pass"}
:target {:type "table"
:schema "PUBLIC"
@@ -353,7 +353,7 @@
:source {:type "python"
:body (str/join "\n" program)
:source-database (mt/id)
- :source-tables {"transforms_customers" (mt/id :transforms_customers)}}
+ :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]}
:target (assoc target :database (mt/id))}))
;; using clojure-ey coordination with promises (I know j.u.c could be better here)
@@ -526,7 +526,7 @@
(let [initial-transform {:name "Schema Change Integration Test"
:source {:type "python"
:source-database (mt/id)
- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}
+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -545,7 +545,7 @@
(let [updated-transform (assoc initial-transform
:source {:type "python"
:source-database (mt/id)
- :source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}
+ :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -573,9 +573,10 @@
:schema (get-test-schema)
:name "table_ref_test"}]
(let [;; Use a name-based ref instead of table ID
- source-tables {"input" {:database_id (mt/id)
- :schema (get-test-schema)
- :table "transforms_products"}}
+ source-tables [{:alias "input"
+ :database_id (mt/id)
+ :schema (get-test-schema)
+ :table "transforms_products"}]
transform-payload {:name "Transform with table ref"
:source {:type "python"
:body "def transform(input):\n return input"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
index b7995fbdb5f..6389f26ef74 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
@@ -95,8 +95,8 @@
(deftest ^:parallel analyze-entity-python-transform-test
(testing "analyze-entity extracts dependencies from a python transform"
(let [transform {:source {:type "python"
- :source-tables {"orders" (mt/id :orders)
- "products" (mt/id :products)}
+ :source-tables [{:alias "orders" :table_id (mt/id :orders)}
+ {:alias "products" :table_id (mt/id :products)}]
:body "def transform(orders, products): return orders"}
:target {:database (mt/id)
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
index 7d105fdcf60..9fc705712e0 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
@@ -98,7 +98,7 @@
db-id (:database_id workspace)
body {:name "Python Dry-Run Transform"
:source {:type "python"
- :source-tables {}
+ :source-tables []
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -129,86 +129,53 @@
(deftest remap-python-source-test
(let [remap-python-source #'ws.execute/remap-python-source]
- (testing "remaps to table ID when mapping has :id"
+ (testing "remaps entry via triple lookup when mapping has :id"
(let [table-mapping {[1 "public" "orders"] {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id 456}}
source {:type "python"
:body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "public"
- :table "orders"
- :table_id 123}}}]
- (is (= {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" 456}}
- (remap-python-source table-mapping source)))))
-
- (testing "remaps to map format when mapping has nil :id"
- (let [table-mapping {[1 "public" "orders"] {:db-id 1
- :schema "ws_isolated_123"
- :table "orders_isolated"
- :id nil}}
- source {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "public"
- :table "orders"
- :table_id 123}}}]
- (is (= {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "ws_isolated_123"
- :table "orders_isolated"}}}
- (remap-python-source table-mapping source)))))
-
- (testing "leaves source-tables unchanged when no mapping exists"
- (let [table-mapping {}
- source {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "public"
- :table "orders"
- :table_id 123}}}]
- (is (= source (remap-python-source table-mapping source)))))
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}]
+ (:source-tables (remap-python-source table-mapping source))))))
- (testing "remaps legacy integer to table ID when mapping has :id"
+ (testing "remaps entry via table_id lookup"
(let [table-mapping {123 {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id 456}}
source {:type "python"
:body "import pandas as pd"
- :source-tables {"orders" 123}}]
- (is (= {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" 456}}
- (remap-python-source table-mapping source)))))
+ :source-tables [{:alias "orders" :table_id 123}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}]
+ (:source-tables (remap-python-source table-mapping source))))))
- (testing "remaps legacy integer to map format when mapping has nil :id"
- (let [table-mapping {123 {:db-id 1
- :schema "ws_isolated_123"
- :table "orders_isolated"
- :id nil}}
+ (testing "remaps without table_id when mapping has nil :id"
+ (let [table-mapping {[1 "public" "orders"] {:db-id 1
+ :schema "ws_isolated_123"
+ :table "orders_isolated"
+ :id nil}}
source {:type "python"
:body "import pandas as pd"
- :source-tables {"orders" 123}}]
- (is (= {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "ws_isolated_123"
- :table "orders_isolated"}}}
- (remap-python-source table-mapping source)))))
-
- (testing "leaves legacy integer table-id unchanged when no mapping exists"
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 123}]
+ (:source-tables (remap-python-source table-mapping source))))))
+
+ (testing "leaves source-tables unchanged when no mapping exists"
(let [table-mapping {}
source {:type "python"
:body "import pandas as pd"
- :source-tables {"orders" 123}}]
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}]}]
(is (= source (remap-python-source table-mapping source)))))
- (testing "handles mixed formats with mappings for both"
+ (testing "handles multiple entries with mappings"
(let [table-mapping {[1 "public" "orders"] {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
@@ -219,18 +186,16 @@
:id 999}}
source {:type "python"
:body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "public"
- :table "orders"
- :table_id 123}
- "customers" 789}}]
- (is (= {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" 456
- "customers" 999}}
- (remap-python-source table-mapping source)))))
-
- (testing "prefers table_id lookup over triple lookup for map format"
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}
+ {:alias "customers" :table_id 789}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}
+ {:alias "customers" :database_id 1
+ :schema "ws_isolated_123" :table "customers_isolated" :table_id 999}]
+ (:source-tables (remap-python-source table-mapping source))))))
+
+ (testing "prefers table_id lookup over triple lookup"
(let [table-mapping {123 {:db-id 1
:schema "ws_isolated_123"
:table "orders_by_id"
@@ -241,14 +206,11 @@
:id 789}}
source {:type "python"
:body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "public"
- :table "orders"
- :table_id 123}}}]
- (is (= {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" 456}}
- (remap-python-source table-mapping source)))))
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "ws_isolated_123" :table "orders_by_id" :table_id 456}]
+ (:source-tables (remap-python-source table-mapping source))))))
(testing "falls back to triple lookup when table_id not in mapping"
(let [table-mapping {[1 "public" "orders"] {:db-id 1
@@ -257,14 +219,34 @@
:id 456}}
source {:type "python"
:body "import pandas as pd"
- :source-tables {"orders" {:database_id 1
- :schema "public"
- :table "orders"
- :table_id 999}}}]
- (is (= {:type "python"
- :body "import pandas as pd"
- :source-tables {"orders" 456}}
- (remap-python-source table-mapping source)))))))
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 999}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 456}]
+ (:source-tables (remap-python-source table-mapping source))))))
+
+ (testing "keeps original values when mapping has nil :id"
+ (let [table-mapping {123 {:db-id 1
+ :schema "ws_isolated_123"
+ :table "orders_isolated"
+ :id nil}}
+ source {:type "python"
+ :body "import pandas as pd"
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "ws_isolated_123" :table "orders_isolated" :table_id 123}]
+ (:source-tables (remap-python-source table-mapping source))))))
+
+ (testing "entry with no matching mapping is left unchanged"
+ (let [table-mapping {999 {:db-id 1 :schema "ws" :table "t" :id 456}}
+ source {:type "python"
+ :body "import pandas as pd"
+ :source-tables [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}]}]
+ (is (= [{:alias "orders" :database_id 1
+ :schema "public" :table "orders" :table_id 123}]
+ (:source-tables (remap-python-source table-mapping source))))))))
(deftest remap-sql-source-test
(let [remap-sql-source #'ws.execute/remap-sql-source
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index 12cce2e37c5..a7b9d07c88f 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -3574,6 +3574,14 @@ databaseChangeLog:
path: instance_analytics_views/query_log/v2/h2-query_log.sql
relativeToChangelogFile: true
+ - changeSet:
+ id: v59.2026-03-03T12:00:00
+ author: qnkhuat
+ comment: Unify source-tables format from map to ordered array
+ changes:
+ - customChange:
+ class: "metabase.app_db.custom_migrations.UnifySourceTablesFormat"
+
- changeSet:
id: v59.2026-03-04T00:00:00
author: rileythompson
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index db8b2ca3911..5d6f1d70da5 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -1855,6 +1855,72 @@
(define-migration MoveExistingAtSymbolUserAttributes
(reserve-at-symbol-user-attributes/migrate!))
+(define-reversible-migration UnifySourceTablesFormat
+ (let [tables [{:table :transform :pks [:id] :where [:= :source_type "python"]}
+ {:table :workspace_transform :pks [:workspace_id :ref_id]}]
+ python? (fn [source]
+ (= "python" (get (json-out source false) "type")))
+ all-rows (into []
+ (mapcat (fn [{:keys [table pks] w :where}]
+ (->> (t2/query (cond-> {:select (into [:source] pks)
+ :from [table]}
+ w (assoc :where w)))
+ (filter #(or w (python? (:source %))))
+ (map #(assoc % ::table table ::pks pks)))))
+ tables)
+ all-ids (into #{}
+ (mapcat (fn [{:keys [source]}]
+ (let [st (get (json-out source false) "source-tables")]
+ (when (map? st)
+ (filter int? (vals st))))))
+ all-rows)
+ metadata (when (seq all-ids)
+ (into {}
+ (map (fn [{:keys [id db_id schema name]}]
+ [id {"database_id" db_id "schema" schema "table" name}]))
+ (t2/query {:select [:id :db_id :schema :name]
+ :from [:metabase_table]
+ :where [:in :id all-ids]})))]
+ (doseq [{:keys [source] :as row} all-rows]
+ (let [parsed (json-out source false)
+ st (get parsed "source-tables")]
+ (when (map? st)
+ (let [entries (mapv (fn [[alias v]]
+ (if (int? v)
+ ;; Integer value: backfill all metadata (database_id, schema, table) from DB.
+ (merge {"alias" alias "table_id" v}
+ (get metadata v))
+ ;; Ref-map value: already has database_id/schema/table; table_id is
+ ;; resolved lazily by normalize-source-tables when the transform is saved via the API.
+ (assoc v "alias" alias)))
+ st)
+ pks (::pks row)]
+ (t2/query {:update (::table row)
+ :set {:source (json-in (assoc parsed "source-tables" entries))}
+ :where (into [:and] (map #(vector := % (get row %)) pks))}))))))
+ (let [convert-back
+ (fn [source-json]
+ (let [parsed (json-out source-json false)]
+ (when-let [st (get parsed "source-tables")]
+ (when (sequential? st)
+ (let [m (into {} (map (fn [entry]
+ [(get entry "alias")
+ (or (get entry "table_id") entry)]))
+ st)]
+ (json-in (assoc parsed "source-tables" m)))))))
+ tables [{:table :transform :pks [:id] :where [:= :source_type "python"]}
+ {:table :workspace_transform :pks [:workspace_id :ref_id]}]
+ python? (fn [source]
+ (= "python" (get (json-out source false) "type")))]
+ (doseq [{:keys [table pks] w :where} tables]
+ (doseq [row (cond->> (t2/query (cond-> {:select (into [:source] pks)
+ :from [table]}
+ w (assoc :where w)))
+ (not w) (filter #(python? (:source %))))]
+ (when-let [new-source (convert-back (:source row))]
+ (t2/query {:update table
+ :set {:source new-source}
+ :where (into [:and] (map #(vector := % (get row %)) pks))}))))))
(define-migration FixClickHouseUploadDBSchemaNames
"This data migration is meant to fix the issues seen in #69667, #68298 and #65945.
We made the driver feature `schemas` conditional on the `enable-multiple-db` DB connection setting.
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index 53ec780df1d..93bdc5cf98b 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -68,29 +68,22 @@
(perms/has-db-transforms-permission? api/*current-user-id* source-db-id)
(remote-sync/transforms-editable?))))
-(defn- keywordize-source-table-refs
- "Keywordize keys in source-tables map values (refs are maps, ints pass through)."
- [source-tables]
- (update-vals source-tables #(if (map? %) (update-keys % keyword) %)))
-
(defn transform-source-out
"Deserialize a transform source map from JSON storage format.
- Normalizes queries, keywordizes types and source-table refs."
+ Normalizes queries and keywordizes type fields."
[m]
(-> m
mi/json-out-without-keywordization
(update-keys keyword)
- (m/update-existing :source-tables keywordize-source-table-refs)
(m/update-existing :query lib-be/normalize-query)
- (m/update-existing :type keyword)
- (m/update-existing :source-incremental-strategy #(update-keys % keyword))))
+ (m/update-existing :source-incremental-strategy #(update-keys % keyword))
+ (m/update-existing :source-tables (fn [st] (mapv #(update-keys % keyword) st)))
+ (m/update-existing :type keyword)))
(defn transform-source-in
- "Serialize a transform source map for JSON storage.
- Normalizes source-tables and prepares queries for serialization."
+ "Serialize a transform source map for JSON storage."
[m]
(-> m
- (m/update-existing :source-tables transforms-base.u/normalize-source-tables)
(m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))
mi/json-in))
@@ -337,7 +330,14 @@
:collection_id (serdes/fk :model/Collection)
:source_database_id (serdes/fk :model/Database :name)
:source {:export #(update % :query serdes/export-mbql)
- :import #(update % :query serdes/import-mbql)}
+ :import (fn [source]
+ (-> source
+ (m/update-existing :source-tables
+ (fn [st]
+ (if (map? st)
+ (transforms-base.u/source-tables-map->vec st)
+ st)))
+ (m/update-existing :query serdes/import-mbql)))}
:target {:export serdes/export-mbql :import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index 813280d8245..88b43649a68 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -31,7 +31,7 @@
add-source-readable
is-temp-transform-table?]
[metabase.transforms.crud
- python-source-table-ref->table-id
+ source-tables-vec->map-for-fe
check-database-feature
check-feature-enabled!
extract-all-columns-from-query
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index 59f1438012d..38d21a3e6b5 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -26,16 +26,14 @@
(set! *warn-on-reflection* true)
-(defn python-source-table-ref->table-id
- "Change source of python transform from name->table-ref to name->table-id.
-
- We now supported table-ref as source but since FE is still expecting table-id we need to temporarily do this.
- Should update FE to fully use table-ref"
+;; TODO(FE-source-tables): Remove this function and all call sites when FE adopts the array format for source-tables.
+(defn source-tables-vec->map-for-fe
+ "Convert source-tables from internal vec format to legacy map format for FE compatibility.
+ Remove this when FE adopts the array format."
[transform]
(if (transforms-base.u/python-transform? transform)
(update-in transform [:source :source-tables]
- (fn [source-tables]
- (update-vals source-tables #(if (int? %) % (:table_id %)))))
+ transforms-base.u/source-tables-vec->alias-id-map)
transform))
(defn check-database-feature
@@ -157,8 +155,8 @@
(transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)
(transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)
(map #(update % :last_run transforms-base.u/localize-run-timestamps))
- (map python-source-table-ref->table-id)))
- transforms.u/add-source-readable))))
+ (map transforms.u/add-source-readable)
+ (map source-tables-vec->map-for-fe))))))) ;; TODO(FE-source-tables): remove
(defn get-transform
"Get a specific transform."
@@ -169,8 +167,8 @@
(t2/hydrate :last_run :transform_tag_ids :creator :owner)
(u/update-some :last_run transforms-base.u/localize-run-timestamps)
(assoc :table target-table)
- python-source-table-ref->table-id
- transforms.u/add-source-readable)))
+ transforms.u/add-source-readable
+ source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove
(defn create-transform!
"Create new transform in the appdb.
@@ -231,8 +229,8 @@
(t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids :creator :owner))]
(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
(-> transform
- python-source-table-ref->table-id
- transforms.u/add-source-readable)))
+ transforms.u/add-source-readable
+ source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove
(defn delete-transform!
"Delete a transform and publish the delete event."
diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.clj
index 0bda7ccebd0..a235243e4b1 100644
--- a/src/metabase/transforms/schema.clj
+++ b/src/metabase/transforms/schema.clj
@@ -3,22 +3,10 @@
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.queries.schema :as queries.schema]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
-(mr/def ::source-table-ref
- "A reference to a source table by name, for cases where table_id may not exist yet.
- Also saves querying metadata in situations where we'll need the name."
- [:map
- [:database_id :int]
- [:schema {:optional true} [:maybe :string]]
- [:table :string]
- [:table_id {:optional true} [:maybe :int]]])
-
-(mr/def ::source-table-value
- "Either a table ID (int) or a reference map."
- [:or :int ::source-table-ref])
-
(mr/def ::checkpoint-strategy
[:map
[:type [:= "checkpoint"]]
@@ -43,7 +31,12 @@
[:map
[:source-database {:optional true} :int]
;; NB: if source is checkpoint, only one table allowed
- [:source-tables [:map-of :string ::source-table-value]]
+ ;; decode/normalize: convert FE map format to vec and enrich with DB metadata
+ [:source-tables [:sequential {:decode/normalize (fn [st]
+ (if (map? st)
+ (transforms-base.u/source-tables-map->vec st)
+ st))}
+ ::transforms-base.u/source-table-entry]]
[:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :python]]
[:body :string]
[:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]])
@@ -77,8 +70,6 @@
(mr/def ::id pos-int?)
-(mr/def ::run-id pos-int?)
-
(mr/def ::transform
[:map
[:id ::id]
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index bdc495e2801..7692aa73a69 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -57,14 +57,7 @@
(let [source-tables (:source-tables source)]
(if (empty? source-tables)
true
- (let [table-ids (into []
- (comp (map val)
- (map #(cond
- (int? %) %
- (map? %) (:table_id %)
- :else nil))
- (filter some?))
- source-tables)]
+ (let [table-ids (into [] (keep :table_id) source-tables)]
(and (seq table-ids)
(every? (fn [table-id]
(when-let [table (t2/select-one :model/Table table-id)]
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 3eef908348b..c773f3fe390 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -553,6 +553,13 @@
(execute-secondary-index-ddl-if-required!
transform run-id database target with-stage-timing-fn)))
+;;; ------------------------------------------------- Source Table Schemas -------------------------------------------------
+
+(mu/defn source-tables-vec->alias-id-map :- [:map-of :string [:maybe :int]]
+ "Convert source-table entries to `{alias -> table_id}` map."
+ [entries :- [:sequential ::source-table-entry]]
+ (into {} (map (juxt :alias :table_id)) entries))
+
;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------
(def ^:private ^:const batch-lookup-chunk-size
@@ -589,60 +596,87 @@
[v]
(and (map? v) (nil? (:table_id v))))
-(defn normalize-source-tables
- "Normalize source-tables to consistent map format {:database_id :schema :table :table_id}.
-
- The old format stored just integer table IDs. New transforms store maps on write.
- Old data is converted on read via transform-source-out for backwards compatibility.
-
+(mr/def ::source-table-entry
+ "A source table entry in the array format. Combines alias with table reference."
+ [:map
+ [:alias :string]
+ [:database_id {:optional true} :int]
+ [:schema {:optional true} [:maybe :string]]
+ [:table {:optional true} :string]
+ [:table_id {:optional true} [:maybe :int]]])
+
+(mu/defn normalize-source-tables :- [:sequential ::source-table-entry]
+ "Normalize source-table entries by enriching them with full metadata.
+ For entries with only :table_id, looks up :database_id/:schema/:table.
+ For entries with only :database_id/:schema/:table, looks up :table_id.
Throws if an integer table ID references a non-existent table.
Map refs with non-existent tables get nil table_id (resolved later at execute time)."
- [source-tables]
- (let [int-table-ids (into #{} (filter int?) (vals source-tables))
- int-id->metadata (when (seq int-table-ids)
- (t2/select-pk->fn (fn [{:keys [db_id schema name]}]
- {:database_id db_id :schema schema :table name})
- [:model/Table :id :db_id :schema :name]
- :id [:in int-table-ids]))
- missing-ids (when (seq int-table-ids)
- (remove int-id->metadata int-table-ids))
- refs-needing-id (filter missing-table-id? (vals source-tables))
+ [source-tables :- [:sequential ::source-table-entry]]
+ (let [;; Entries that have table_id but lack table metadata need lookup
+ needs-metadata (filter (fn [e] (and (:table_id e) (not (:table e)))) source-tables)
+ int-id->metadata (when (seq needs-metadata)
+ (let [ids (into #{} (map :table_id) needs-metadata)]
+ (t2/select-pk->fn (fn [{:keys [db_id schema name]}]
+ {:database_id db_id :schema schema :table name})
+ [:model/Table :id :db_id :schema :name]
+ :id [:in ids])))
+ missing-ids (when (seq needs-metadata)
+ (let [ids (into #{} (map :table_id) needs-metadata)]
+ (remove (or int-id->metadata {}) ids)))
+ refs-needing-id (filter missing-table-id? source-tables)
ref-lookup (or (batch-lookup-table-ids refs-needing-id) {})]
(when (seq missing-ids)
(throw (ex-info (str "Tables not found for ids: " (str/join ", " (sort missing-ids)))
{:table_ids (vec missing-ids)})))
- (update-vals source-tables
- (fn [v]
- (cond
- (int? v) (assoc (int-id->metadata v) :table_id v)
- (:table_id v) v
- :else (assoc v :table_id (ref-lookup (source-table-ref->key v))))))))
-
-(defn resolve-source-tables
- "Resolve source-tables to {alias -> table_id}. Throws if any table not found.
- For execute time - all entries must resolve to valid table IDs.
- Handles both integer IDs (old format) and map refs (new format)."
- [source-tables]
- (let [needs-lookup (filter missing-table-id? (vals source-tables))
+ (mapv (fn [entry]
+ (cond
+ ;; Has table_id but no table metadata — enrich from DB
+ (and (:table_id entry) (not (:table entry)))
+ (merge (int-id->metadata (:table_id entry)) entry)
+
+ ;; Has table metadata but no table_id — look it up
+ (missing-table-id? entry)
+ (assoc entry :table_id (ref-lookup (source-table-ref->key entry)))
+
+ ;; Already fully populated
+ :else entry))
+ source-tables)))
+
+(mu/defn resolve-source-tables :- [:map-of :string :int]
+ "Resolve source-table entries to `{alias -> table_id}`. Throws if any table not found.
+ For execute time — all entries must resolve to valid table IDs."
+ [source-tables :- [:sequential ::source-table-entry]]
+ (let [needs-lookup (filter missing-table-id? source-tables)
lookup (or (batch-lookup-table-ids needs-lookup) {})
- resolved (u/for-map [[alias v] source-tables]
- [alias (if (int? v)
- v
- (or (:table_id v) (lookup (source-table-ref->key v))))])
- unresolved (for [[alias table-id] resolved
- :when (nil? table-id)
- :let [v (get source-tables alias)]]
- {:alias alias
- :table (if-let [schema (:schema v)]
- (str schema "." (:table v))
- (:table v))
- :ref v})]
+ resolved (u/for-map [entry source-tables]
+ [(:alias entry) (or (:table_id entry) (lookup (source-table-ref->key entry)))])
+ unresolved (for [entry source-tables
+ :let [table-id (or (:table_id entry) (lookup (source-table-ref->key entry)))]
+ :when (nil? table-id)]
+ {:alias (:alias entry)
+ :table (if-let [schema (:schema entry)]
+ (str schema "." (:table entry))
+ (:table entry))
+ :ref entry})]
(when (seq unresolved)
(throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved)))
{:unresolved unresolved
:transform-message "Input table not found"})))
resolved))
+(mu/defn source-tables-map->vec :- [:sequential ::source-table-entry]
+ "Convert map format `{alias -> value}` to vec format `[{:alias alias ...}]`.
+ Handles both int values (`{alias: table_id}`) and ref map values (`{alias: {:database_id ...}}`.
+ Accepts both keyword and string keys for alias.
+ Enriches entries with full metadata via [[normalize-source-tables]]."
+ [m :- [:map-of [:or :string :keyword] [:or :int :map]]]
+ (normalize-source-tables
+ (mapv (fn [[alias v]]
+ (if (int? v)
+ {:alias (name alias) :table_id v}
+ (assoc v :alias (name alias))))
+ m)))
+
;;; ------------------------------------------------- Timestamp Helpers -------------------------------------------------
(defn ->instant
diff --git a/src/metabase/transforms_inspector/context.clj b/src/metabase/transforms_inspector/context.clj
index d00ceee4e76..dd1d9f7e7c0 100644
--- a/src/metabase/transforms_inspector/context.clj
+++ b/src/metabase/transforms_inspector/context.clj
@@ -64,7 +64,7 @@
(try
(let [source-tables (get-in transform [:source :source-tables])
normalized (transforms-base.u/normalize-source-tables source-tables)
- table-ids (keep (fn [[_ v]] (:table_id v)) normalized)]
+ table-ids (keep :table_id normalized)]
(table-ids->source-info table-ids))
(catch Exception e
(log/warn e "Failed to extract sources from Python transform")
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 29a1523e572..4810afc61a6 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -174,8 +174,8 @@
403
(deferred-tru "A table with that name already exists."))
(-> (transforms.core/create-transform! body)
- transforms.core/python-source-table-ref->table-id
- transforms.u/add-source-readable))
+ transforms.u/add-source-readable
+ transforms.core/source-tables-vec->map-for-fe)) ;; TODO(FE-source-tables): remove
(api.macros/defendpoint :get "/:id" :- TransformResponse
"Get a specific transform."
@@ -193,8 +193,8 @@
dep-ids (get global-ordering id)
dependencies (map id->transform dep-ids)]
(->> (t2/hydrate dependencies :creator :owner)
- (mapv transforms.core/python-source-table-ref->table-id)
- transforms.u/add-source-readable)))
+ transforms.u/add-source-readable
+ (mapv transforms.core/source-tables-vec->map-for-fe)))) ;; TODO(FE-source-tables): remove
(def ^:private MergeHistoryEntry
[:map
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index 1d35b53ee2b..38bfabdeba6 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -2776,6 +2776,86 @@
(is (nil? (id->target none-id)))
(is (nil? (id->target deleted-id))))))))
+(deftest unify-source-tables-format-test
+ (testing "v59.2026-03-03T12:00:00: convert source-tables from map to vec format"
+ (impl/test-migrations ["v59.2026-03-03T12:00:00"] [migrate!]
+ (let [user-id (:id (new-instance-with-default :core_user))
+ db-id (:id (new-instance-with-default :metabase_database))
+ ;; source-tables as map with int values (FE format)
+ int-source (json/encode {:type "python" :body "x=1" :source-database db-id
+ :source-tables {"orders" 42 "products" 99}})
+ ;; source-tables as map with ref values (normalized format)
+ ref-source (json/encode {:type "python" :body "x=1" :source-database db-id
+ :source-tables {"input" {"database_id" db-id "schema" "public"
+ "table" "my_table" "table_id" 7}}})
+ ;; query transform (no source-tables) — should be untouched
+ query-source (json/encode {:type "query" :query {:database db-id}})
+ target (json/encode {:type "table" :schema "public" :name "out"})
+ insert-transform!
+ (fn [source]
+ (t2/insert-returning-pk!
+ :transform {:name (mt/random-name)
+ :source source
+ :target target
+ :source_type "python"
+ :source_database_id db-id
+ :created_at :%now
+ :updated_at :%now}))
+ int-id (insert-transform! int-source)
+ ref-id (insert-transform! ref-source)
+ query-id (insert-transform! query-source)
+ ;; Also test workspace_transform
+ ws-id (:id (t2/insert-returning-instance!
+ :workspace {:name "test-ws"
+ :creator_id user-id
+ :created_at :%now
+ :updated_at :%now}))
+ _ (t2/insert! :workspace_transform
+ {:ref_id (str (random-uuid))
+ :workspace_id ws-id
+ :name "ws-transform"
+ :source int-source
+ :target target
+ :created_at :%now
+ :updated_at :%now})
+ get-source-tables (fn [table-name pk-map]
+ (let [where (into [:and] (map (fn [[k v]] [:= k v]) pk-map))
+ row (first (t2/query {:select [:source] :from [table-name] :where where}))]
+ (get (json/decode (:source row)) "source-tables")))]
+ (testing "Before migration, source-tables are maps"
+ (is (map? (get-source-tables :transform {:id int-id})))
+ (is (map? (get-source-tables :transform {:id ref-id}))))
+
+ (migrate!)
+
+ (testing "After migration, int-value maps become vec of entries"
+ (let [st (get-source-tables :transform {:id int-id})]
+ (is (sequential? st))
+ (is (= 2 (count st)))
+ (is (= #{"orders" "products"} (set (map #(get % "alias") st))))
+ (is (= #{42 99} (set (map #(get % "table_id") st))))))
+
+ (testing "After migration, ref-value maps become vec of entries with alias"
+ (let [st (get-source-tables :transform {:id ref-id})]
+ (is (sequential? st))
+ (is (= "input" (get (first st) "alias")))
+ (is (= 7 (get (first st) "table_id")))))
+
+ (testing "Query transforms are untouched"
+ (is (nil? (get-source-tables :transform {:id query-id}))))
+
+ (testing "workspace_transform is also migrated"
+ (let [ws-rows (t2/query {:select [:source] :from [:workspace_transform] :where [:= :workspace_id ws-id]})
+ st (get (json/decode (:source (first ws-rows))) "source-tables")]
+ (is (sequential? st))
+ (is (= 2 (count st)))))
+
+ (testing "Rollback converts vec back to map"
+ (migrate! :down 58)
+ (let [st (get-source-tables :transform {:id int-id})]
+ (is (map? st))
+ (is (= 42 (get st "orders")))
+ (is (= 99 (get st "products")))))))))
(deftest fix-clickhouse-upload-db-schema-names-test
(testing "FixClickHouseUploadDBSchemaNames, v59.2026-03-04T00:00:00: fix clickhouse upload db schema names"
(encryption-test/with-secret-key "fake-secret-key"
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index b307976b702..a14716343a7 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -185,72 +185,69 @@
(is (= {} result)))))))
(deftest normalize-source-tables-test
- (testing "normalize-source-tables converts all entries to map format"
+ (testing "normalize-source-tables enriches vec entries"
(mt/with-temp [:model/Database db {}
:model/Table t1 {:db_id (:id db) :name "existing_table" :schema nil}]
- (testing "converts integer table ID to map format"
- (let [result (transforms-base.u/normalize-source-tables {"t" (:id t1)})]
- (is (map? (get result "t")))
- (is (= (:id db) (get-in result ["t" :database_id])))
- (is (= "existing_table" (get-in result ["t" :table])))
- (is (= (:id t1) (get-in result ["t" :table_id])))))
-
- (testing "throws for non-existent integer table ID"
+ (testing "enriches entry with table_id but no table metadata"
+ (let [result (transforms-base.u/normalize-source-tables [{:alias "t" :table_id (:id t1)}])
+ entry (first result)]
+ (is (= (:id db) (:database_id entry)))
+ (is (= "existing_table" (:table entry)))
+ (is (= (:id t1) (:table_id entry)))))
+
+ (testing "throws for non-existent table_id"
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found for ids: 999999"
- (transforms-base.u/normalize-source-tables {"t" 999999}))))
+ (transforms-base.u/normalize-source-tables [{:alias "t" :table_id 999999}]))))
- (testing "populates table_id for existing table"
- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table"}}
+ (testing "populates table_id for existing table ref"
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "existing_table"}]
result (transforms-base.u/normalize-source-tables source-tables)]
- (is (= (:id t1) (get-in result ["t" :table_id])))))
+ (is (= (:id t1) (:table_id (first result))))))
- (testing "preserves existing table_id"
- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table" :table_id 999}}
+ (testing "preserves existing table_id when table metadata present"
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "existing_table" :table_id 999}]
result (transforms-base.u/normalize-source-tables source-tables)]
- (is (= 999 (get-in result ["t" :table_id])))))
+ (is (= 999 (:table_id (first result))))))
(testing "leaves table_id nil for non-existent table ref"
- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]
result (transforms-base.u/normalize-source-tables source-tables)]
- (is (nil? (get-in result ["t" :table_id])))))
+ (is (nil? (:table_id (first result))))))
- (testing "handles mixed int and map entries"
- (let [source-tables {"t1" (:id t1)
- "t2" {:database_id (:id db) :schema nil :table "existing_table"}}
+ (testing "handles entries needing different kinds of enrichment"
+ (let [source-tables [{:alias "t1" :table_id (:id t1)}
+ {:alias "t2" :database_id (:id db) :schema nil :table "existing_table"}]
result (transforms-base.u/normalize-source-tables source-tables)]
- (is (map? (get result "t1")))
- (is (= (:id t1) (get-in result ["t1" :table_id])))
- (is (= (:id t1) (get-in result ["t2" :table_id]))))))))
+ (is (= "existing_table" (:table (first result))))
+ (is (= (:id t1) (:table_id (first result))))
+ (is (= (:id t1) (:table_id (second result)))))))))
(deftest resolve-source-tables-test
- (testing "resolve-source-tables returns {alias -> table_id} map"
+ (testing "resolve-source-tables returns {alias -> table_id} map from vec input"
(mt/with-temp [:model/Database db {}
:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}
:model/Table t2 {:db_id (:id db) :name "table_two" :schema nil}]
- (testing "passes through integer entries (old format)"
- (is (= {"t" 123} (transforms-base.u/resolve-source-tables {"t" 123}))))
-
- (testing "resolves map with table_id"
- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}}]
+ (testing "resolves entry with table_id"
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}]]
(is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))
- (testing "looks up table_id for map without it"
- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one"}}]
+ (testing "looks up table_id for entry without it"
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}]]
(is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))
(testing "throws for non-existent table"
- (let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}]
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]]
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: nonexistent"
(transforms-base.u/resolve-source-tables source-tables)))))
(testing "throws with schema in error message"
- (let [source-tables {"t" {:database_id (:id db) :schema "my_schema" :table "nonexistent"}}]
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema "my_schema" :table "nonexistent"}]]
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: my_schema\.nonexistent"
(transforms-base.u/resolve-source-tables source-tables)))))
- (testing "handles mixed entries (old and new format)"
- (let [source-tables {"t1" (:id t1)
- "t2" {:database_id (:id db) :schema nil :table "table_two"}}]
+ (testing "handles multiple entries"
+ (let [source-tables [{:alias "t1" :table_id (:id t1)}
+ {:alias "t2" :database_id (:id db) :schema nil :table "table_two"}]]
(is (= {"t1" (:id t1) "t2" (:id t2)}
(transforms-base.u/resolve-source-tables source-tables))))))))
@@ -269,12 +266,12 @@
(testing "returns true for python transform when user can read all source tables"
(let [transform {:source {:type :python
- :source-tables {"t1" table-id}}}]
+ :source-tables [{:alias "t1" :table_id table-id}]}}]
(is (true? (transforms.u/source-tables-readable? transform)))))
- (testing "handles source tables with table_id in map format"
+ (testing "handles source tables with table_id"
(let [transform {:source {:type :python
- :source-tables {"t1" {:table_id table-id}}}}]
+ :source-tables [{:alias "t1" :table_id table-id}]}}]
(is (true? (transforms.u/source-tables-readable? transform))))))))))))
(deftest source-tables-readable-permissions-test
@@ -298,7 +295,7 @@
(testing "Python transforms - blocked database access"
(let [transform {:source {:type :python
- :source-tables {"t1" table1-id}}}]
+ :source-tables [{:alias "t1" :table_id table1-id}]}}]
(mt/with-user-in-groups [group {:name "Blocked Group"}
user [group]]
(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked
@@ -309,8 +306,8 @@
(testing "Python transforms - granular access but missing some tables"
(let [transform {:source {:type :python
- :source-tables {"t1" table1-id
- "t2" table2-id}}}]
+ :source-tables [{:alias "t1" :table_id table1-id}
+ {:alias "t2" :table_id table2-id}]}}]
(mt/with-user-in-groups [group {:name "Limited Granular Group"}
user [group]]
;; Block all-users from viewing data at database level, then grant specific access via test group
diff --git a/test/metabase/transforms_inspector/context_test.clj b/test/metabase/transforms_inspector/context_test.clj
index 30e05cda4b0..970763188bb 100644
--- a/test/metabase/transforms_inspector/context_test.clj
+++ b/test/metabase/transforms_inspector/context_test.clj
@@ -176,9 +176,9 @@
(is (contains? table-ids (mt/id :products))))))
(deftest extract-sources-python-test
- (testing "Python: extracts source tables from source-tables map"
+ (testing "Python: extracts source tables from source-tables vec"
(let [transform {:source {:type :python
- :source-tables {"orders" (mt/id :orders)}
+ :source-tables [{:alias "orders" :table_id (mt/id :orders)}]
:source-database (mt/id)}
:name "test"}
sources (context/extract-sources transform)]
@@ -189,8 +189,8 @@
(deftest extract-sources-python-multiple-tables-test
(testing "Python: extracts multiple source tables"
(let [transform {:source {:type :python
- :source-tables {"orders" (mt/id :orders)
- "products" (mt/id :products)}
+ :source-tables [{:alias "orders" :table_id (mt/id :orders)}
+ {:alias "products" :table_id (mt/id :products)}]
:source-database (mt/id)}
:name "test"}
sources (context/extract-sources transform)
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 08a232bb38d..0c1a1949f09 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -1750,7 +1750,7 @@
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
python-source {:type "python"
:body "print('hello world')"
- :source-tables {}
+ :source-tables []
:source-database (mt/id)}]
(mt/with-temp [:model/Transform {query-id :id} {:name query-name
:source query-source