The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
use array/list for sources table in transforms (#70100)
Jakub Chodorowicz80d08cce82eb
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.tsindex 7016d2c8c66..d3f3385c2ff 100644--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts@@ -1537,7 +1537,7 @@ LIMITdef transform(foo):return pd.DataFrame([{"foo": 42 }])`,- sourceTables: { foo: id },+ sourceTables: pythonSourceTables("foo", id),visitTransform: true,});},@@ -1575,7 +1575,7 @@ LIMITdef transform(foo):return pd.DataFrame([{"foo": 42 }])`,- sourceTables: { foo: id },+ sourceTables: pythonSourceTables("foo", id),visitTransform: true,});},@@ -1615,7 +1615,7 @@ LIMITdef transform(foo):return pd.DataFrame([{"foo": 42 }])`,- sourceTables: { foo: id },+ sourceTables: pythonSourceTables("foo", id),visitTransform: true,});},@@ -1651,7 +1651,7 @@ LIMITdef transform(foo):return pd.DataFrame([{"foo": 42 }])`,- sourceTables: { foo: id },+ sourceTables: pythonSourceTables("foo", id),visitTransform: true,});},@@ -3530,7 +3530,7 @@ describe(print("Hello, world!")return pd.DataFrame([{"foo": common.useful_calculation(40, 2) }])`,- sourceTables: { foo: id },+ sourceTables: pythonSourceTables("foo", id),visitTransform: true,});},@@ -3572,7 +3572,7 @@ describe(def transform(foo):return pd.DataFrame([{"foo": 42}])`,- sourceTables: { foo: id },+ sourceTables: pythonSourceTables("foo", id),visitTransform: true,});},@@ -3848,6 +3848,20 @@ function createPythonTransform(opts: {});}+function pythonSourceTables(+ alias: string,+ tableId: number,+): PythonTransformTableAliases {+ return [+ {+ alias,+ table_id: tableId,+ database_id: WRITABLE_DB_ID,+ schema: TARGET_SCHEMA,+ },+ ];+}+function visitTableQuestion({targetTable = TARGET_TABLE,targetSchema = TARGET_SCHEMA,diff --git a/e2e/test/scenarios/data-studio/workspaces.cy.spec.ts b/e2e/test/scenarios/data-studio/workspaces.cy.spec.tsindex 7162d958203..3e21bd7f07c 100644--- a/e2e/test/scenarios/data-studio/workspaces.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/workspaces.cy.spec.ts@@ -1765,7 +1765,20 @@ describe("scenarios > data studio > workspaces", () => {}).then((id2) => {createPythonTransform({body: TEST_PYTHON_TRANSFORM,- sourceTables: { foo: id1, bar: id2 },+ sourceTables: [+ {+ alias: "foo",+ table_id: id1,+ database_id: WRITABLE_DB_ID,+ schema: "Schema A",+ },+ {+ alias: "bar",+ table_id: id2,+ database_id: WRITABLE_DB_ID,+ schema: "Schema B",+ },+ ],visitTransform: true,});});@@ -1908,7 +1921,20 @@ describe("scenarios > data studio > workspaces", () => {}).then((id2) => {createPythonTransform({body: TEST_PYTHON_TRANSFORM_MULTI_TABLE,- sourceTables: { animals_a: id1, animals_b: id2 },+ sourceTables: [+ {+ alias: "animals_a",+ table_id: id1,+ database_id: WRITABLE_DB_ID,+ schema: "Schema A",+ },+ {+ alias: "animals_b",+ table_id: id2,+ database_id: WRITABLE_DB_ID,+ schema: "Schema B",+ },+ ],visitTransform: true,});});@@ -2517,7 +2543,14 @@ function createTransforms({ visit }: { visit?: boolean } = { visit: false }) {H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then((id) => {createPythonTransform({body: TEST_PYTHON_TRANSFORM,- sourceTables: { foo: id },+ sourceTables: [+ {+ alias: "foo",+ table_id: id,+ database_id: WRITABLE_DB_ID,+ schema: TARGET_SCHEMA,+ },+ ],});});diff --git a/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.ts b/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.tsindex 1b2caab2772..d87f5e3ea42 100644--- a/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.ts+++ b/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.ts@@ -1,6 +1,7 @@import dedent from "ts-dedent";import { WRITABLE_DB_ID } from "e2e/support/cypress_data";+import type { PythonTransformTableAliases } from "metabase-types/api";const { H } = cy;@@ -168,7 +169,7 @@ describe(createMockPythonTransformJSON(null,WRITABLE_DB_ID,- { metabase_table_df: 152 },+ pythonSourceTables("metabase_table_df", 152),"import pandas as pd\\n\\ndef transform(metabase_table_df):\\n return pd.DataFrame({'value': [1]})",),),@@ -194,7 +195,7 @@ describe(createMockPythonTransformJSON(null,WRITABLE_DB_ID,- { metabase_table_df: 152 },+ pythonSourceTables("metabase_table_df", 152),"import pandas as pd\\n\\ndef transform(metabase_table_df):\\n return pd.DataFrame({'value': [2]})",),),@@ -228,7 +229,7 @@ describe(createMockPythonTransformJSON(null,WRITABLE_DB_ID,- { metabase_table_df: 152 },+ pythonSourceTables("metabase_table_df", 152),"import pandas as pd\\n\\ndef transform(metabase_table_df):\\n return pd.DataFrame({'value': [4]})",),),@@ -385,7 +386,7 @@ describe(def transform(foo):return pd.DataFrame({'value': [1]})`,- sourceTables: { foo: tableId },+ sourceTables: pythonSourceTables("foo", Number(tableId)),}).then(({ body: transform }) => {visitTransformListPage();getMetabotButton().click();@@ -397,7 +398,7 @@ describe(createMockPythonTransformJSON(Number(transform.id),WRITABLE_DB_ID,- { foo: tableId },+ pythonSourceTables("foo", Number(tableId)),"import pandas as pd\\n\\ndef transform(foo):\\n return pd.DataFrame({'value': [2]})",),),@@ -439,7 +440,7 @@ describe(createMockPythonTransformJSON(Number(transform.id),WRITABLE_DB_ID,- { metabase_table_df: 152 },+ pythonSourceTables("metabase_table_df", 152),"import pandas as pd\\n\\ndef transform(foo):\\n return pd.DataFrame({'value': [4]})",),),@@ -474,11 +475,27 @@ const createMockNativeTransformJSON = (const createMockPythonTransformJSON = (id: number | null,databaseId: number,- sourceTables: { [tableName: string]: number },+ sourceTables: PythonTransformTableAliases,body: string,) =>`{"id":${id},"name":"A number","entity_id":null,"description":"","source":{"type":"python","source-database":${databaseId},"source-tables":${JSON.stringify(sourceTables)},"body":"${body}"},"target":{"type":"table","name":""},"created_at":null,"updated_at":null}`;+function pythonSourceTables(+ alias: string,+ tableId: number,+ schema = "Schema A",+ databaseId = WRITABLE_DB_ID,+): PythonTransformTableAliases {+ return [+ {+ alias,+ table_id: tableId,+ database_id: databaseId,+ schema,+ },+ ];+}+const createMockTransformSuggestionResponse = (text: string,transformJSON: string,diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/api.cljindex 6916ea08e35..a0c8bd9f069 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj@@ -9,6 +9,7 @@[metabase.api.routes.common :refer [+auth]][metabase.api.util.handlers :as handlers][metabase.permissions.core :as perms]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util.i18n :as i18n][metabase.util.malli.schema :as ms][toucan2.core :as t2]))@@ -63,10 +64,10 @@per_input_row_limit 100}}:- [:map[:code :string]- [:source_tables [:map-of {:min 1} :string :int]]+ [:source_tables [:sequential {:min 1} ::transforms-base.u/source-table-entry]][:output_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]][:per_input_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]]]]- (let [db-ids (t2/select-fn-set :db_id [:model/Table :db_id] :id [:in (vals source_tables)])]+ (let [db-ids (t2/select-fn-set :db_id [:model/Table :db_id] :id [:in (map :table_id source_tables)])](api/check-400 (= (count db-ids) 1) (i18n/deferred-tru "All source tables must belong to the same database."))(api/check-403 (perms/has-db-transforms-permission? api/*current-user-id* (first db-ids))));; NOTE: we do not test database support, as there is no write target.diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljindex 1f501795297..1620e9be98c 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -240,7 +240,7 @@{:server-url server-url:code (:body source):run-id run-id- :table-name->id resolved-source-tables+ :source-tables resolved-source-tables:shared-storage @shared-storage-ref})output-manifest (python-runner/read-output-manifest @shared-storage-ref)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex aea8c43f44b..d8249805e4a 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -200,12 +200,16 @@(defn execute-python-code-http-call!"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returnsthe response from the server."- [{:keys [server-url code request-id run-id table-name->id shared-storage timeout-secs]}]+ [{:keys [server-url code request-id run-id source-tables shared-storage timeout-secs]}](let [{:keys [objects]} shared-storage{:keys [output output-manifest events]} objectsurl-for-path (fn [path] (:url (get objects path)))- table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))- table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))+ table-name->url (into {} (map (fn [{:keys [alias table_id]}]+ [alias (url-for-path [:table table_id :data])]))+ source-tables)+ table-name->manifest-url (into {} (map (fn [{:keys [alias table_id]}]+ [alias (url-for-path [:table table_id :manifest])]))+ source-tables)payload {:code code:library (t2/select-fn->fn :path :source :model/PythonLibrary):timeout (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))@@ -292,21 +296,19 @@(when (and (:source-incremental-strategy source)(> (count (:source-tables source)) 1))(throw (ex-info "Incremental transforms for python only supports one source table" {})))- (doseq [[table-name v] (:source-tables source)- ;; TODO (Chris 2026-03-03) We should also handle the case where table-id has not been backfilled yet...- :let [table-id (if (int? v) v (:table_id v))- {:keys [s3-client bucket-name objects]} shared-storage- {data-path :path} (get objects [:table table-id :data])- {manifest-path :path} (get objects [:table table-id :manifest])]]+ (doseq [{:keys [alias table_id]} (:source-tables source)+ :let [{:keys [s3-client bucket-name objects]} shared-storage+ {data-path :path} (get objects [:table table_id :data])+ {manifest-path :path} (get objects [:table table_id :manifest])]](let [tmp-data-file (File/createTempFile data-path "")tmp-meta-file (File/createTempFile manifest-path "")](try- (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)+ (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table_id)driver (t2/select-one-fn :engine :model/Database db-id)- fields-meta (fields-metadata driver table-id)- manifest (generate-manifest table-id fields-meta)]+ fields-meta (fields-metadata driver table_id)+ manifest (generate-manifest table_id fields-meta)](transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]]- (let [query (build-table-query table-id (:source-incremental-strategy source) source-range-params limit)]+ (let [query (build-table-query table_id (:source-incremental-strategy source) source-range-params limit)](write-query-data-to-file!{:query query:fields-meta fields-meta@@ -324,10 +326,10 @@(catch InterruptedException ie (throw ie))(catch Throwable t(throw (ex-info "An error occurred while copying table data to S3"- {:table-id table-id+ {:table-id table_id:transform-message (or (:transform-message (ex-data t));; Cast table-id to string manually, to avoid thousands separators.- (i18n/tru "Failed to copy table contents to shared storage: {0} ({1})" table-name (str table-id)))}+ (i18n/tru "Failed to copy table contents to shared storage: {0} ({1})" alias (str table_id)))}t)))(finally(safe-delete tmp-data-file)@@ -339,7 +341,7 @@Args::code - Python code to execute- :source-tables - Map of table-name -> table-id (already resolved)+ :source-tables - Sequential of source-table entries [{:alias ... :table_id ...} ...]:row-limit - Max rows to return (also limits input rows):timeout-secs - Optional timeout override@@ -361,7 +363,7 @@{:server-url server-url:code code:request-id (u/generate-nano-id)- :table-name->id source-tables+ :source-tables source-tables:timeout-secs timeout-secs:shared-storage @shared-storage-ref})events (read-events @shared-storage-ref)]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.cljindex 2992c7cfffa..09b8dadf76e 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj@@ -123,7 +123,7 @@(.build))](.toString (.url (.presignPutObject presigner request)))))-(defn- s3-shared-storage [table-name->id]+(defn- s3-shared-storage [source-tables](let [work-dir-name (working-dir-for-run)container-presigner (create-s3-presigner-for-container)bucket-name (transforms-python.settings/python-storage-s-3-bucket)@@ -142,9 +142,9 @@{:output (ref :put "output.csv"):output-manifest (ref :put "output-manifest.json"):events (ref :put "events.jsonl")}- (for [[table-name id] table-name->id]- {[:table id :manifest] (ref :get (str "table-" (name table-name) "-" id ".manifest.json"))- [:table id :data] (ref :get (str "table-" (name table-name) "-" id ".jsonl"))}))}))+ (for [{:keys [alias table_id]} source-tables]+ {[:table table_id :manifest] (ref :get (str "table-" alias "-" table_id ".manifest.json"))+ [:table table_id :data] (ref :get (str "table-" alias "-" table_id ".jsonl"))}))}))(declare delete-many)@@ -172,8 +172,8 @@When the value is closed, all the relevant keys will be deleted, but this is best-effort only.We rely on the bucket retention policy to ensure any stragglers are eventually deleted."- ^Closeable [table-name->id]- (let [shared-storage (s3-shared-storage table-name->id)]+ ^Closeable [source-tables]+ (let [shared-storage (s3-shared-storage source-tables)](reify IDeref(deref [_] shared-storage)Closeablediff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.cljindex 17ba26b8560..ed029872d4c 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj@@ -53,8 +53,7 @@[:python[:map {:closed true}[:source-database {:optional true} :int]- [:source-tables [:sequential {:decode/normalize transforms-base.u/normalize-source-tables-structure}- [:map [:alias [:string {:min 1}]] [:table_id :int]]]]+ [:source-tables [:sequential ::transforms-base.u/source-table-entry]][:type [:= "python"]][:body :string]]]])@@ -193,9 +192,7 @@(-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)(t2/select-one :workspace_id ws-id :ref_id tx-id)api/check-404- attach-isolated-target- ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables- transforms/source-tables-vec->map-for-fe))+ attach-isolated-target))(defn create-workspace-transform!"Shared logic for creating a new workspace transform.@@ -236,9 +233,7 @@transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body:ref-id ref-id)](-> (select-malli-keys WorkspaceTransform workspace-transform-alias transform)- attach-isolated-target- ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables- transforms/source-tables-vec->map-for-fe))))+ attach-isolated-target))));;; ---------------------------------------- Pending inputs ----------------------------------------diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.cljindex 6440ef0923e..3226b7a670a 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj@@ -94,14 +94,14 @@:keys [cols rows]} (python-runner/execute-and-read-output!{:code (:body remapped-source):source-tables resolved-source-tables- :row-limit preview-row-limit})- flatrows (apply juxt (map (fn [c] (let [cname (:name c)] #(get % cname))) cols))]+ :row-limit preview-row-limit})](if (= :succeeded (:status result))- {:status :succeeded- ;; return logs for debugging- :logs (str/join "\n" (:logs result))- :data {:cols cols- :rows (mapv flatrows rows)}}+ (let [flatrows (apply juxt (map (fn [c] (let [cname (:name c)] #(get % cname))) cols))]+ {:status :succeeded+ ;; return logs for debugging+ :logs (str/join "\n" (:logs result))+ :data {:cols cols+ :rows (mapv flatrows rows)}}){:status :failed:message (:message result)})))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.cljindex 2b3dee503f4..a2b8f3fa31b 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj@@ -6,6 +6,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.test :as mt]+ [metabase.transforms.test-util :as transforms.tu][toucan2.core :as t2]))(deftest ^:parallel upstream-deps-card-test@@ -199,9 +200,8 @@orders-id (mt/id :orders)](mt/with-temp [:model/Transform transform {:name "Test Transform":source {:type :python- :source-tables [{:alias "PRODUCTS" :table_id products-id}- {:alias "ORDERS" :table_id orders-id}]- ;; A problematic field, hopefully removed again.+ :source-tables [(transforms.tu/source-table-entry "PRODUCTS" products-id)+ (transforms.tu/source-table-entry "ORDERS" orders-id)]:source-database (mt/id):body "..."}:target {:schema "PUBLIC"diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.cljindex a50ab52d0dd..f32e2a0c78d 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj@@ -17,7 +17,6 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.test :as mt]- [metabase.transforms.crud :as transforms.crud][metabase.util :as u][metabase.util.malli.registry :as mr][toucan2.core :as t2]))@@ -1282,9 +1281,7 @@(testing "With superuser permissions"(is (=? {:structured_output [(mt/obj->json->obj (select-keys t1 [:id :entity_id :name :description :source]));; note: t2 not included because it's a (non-native) MBQL query- ;; API converts source-tables from vec back to map for FE- (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe- (select-keys t3 [:id :entity_id :name :description :source])))]+ (mt/obj->json->obj (select-keys t3 [:id :entity_id :name :description :source]))]:conversation_id conversation-id}(-> (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transforms"{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}@@ -1329,9 +1326,7 @@(testing "With superuser permissions"(doseq [transform [t1 t2]](testing (:name transform)- ;; API converts source-tables from vec back to map for FE- (is (=? {:structured_output (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe- (select-keys transform [:id :entity_id :name :description :source :target])))+ (is (=? {:structured_output (mt/obj->json->obj (select-keys transform [:id :entity_id :name :description :source :target])):conversation_id conversation-id}(mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transform-details"{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.cljindex 3372a8cf837..40993e5cfe6 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj@@ -7,6 +7,7 @@[metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt]+ [metabase.transforms.test-util :as transforms.tu][metabase.util :as u][toucan2.core :as t2]))@@ -251,13 +252,13 @@:source {:type "python":body "print('hello')":source-database (mt/id)- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}:target {:type "table":schema "public":name "python_transform_table"}}](let [modified-source {:type "python":body "print('modified')"- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}result (metabot.dependencies/check-transform-dependencies{:id python-transform-id:source modified-source})]diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex ab502aea2a7..c27ce039172 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -130,7 +130,7 @@:source-incremental-strategy {:type "checkpoint":checkpoint-filter-field-id (checkpoint-filter-field)}}:python {:type "python"- :source-tables [{:alias "transforms_products" :table_id (mt/id :transforms_products)}]+ :source-tables [(transforms.tu/source-table-entry "transforms_products" (mt/id :transforms_products))]:body incremental-python-body:source-incremental-strategy {:type "checkpoint":checkpoint-filter-field-id (checkpoint-filter-field)}})diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex 75702b2064a..5bb0942f80d 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -4,6 +4,7 @@[metabase.test :as mt][metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.ordering :as ordering]+ [metabase.transforms.test-util :as transforms.tu][toucan2.core :as t2]))(defn- make-python-transform@@ -25,7 +26,7 @@(testing "Python transform with name-based source table ref resolves to producing transform"(mt/with-temp [;; Transform A produces table "intermediate_output":model/Transform {t-a :id} (make-python-transform- [{:alias "input" :table_id (mt/id :orders)}]+ [(transforms.tu/source-table-entry "input" (mt/id :orders))]"intermediate_output");; Transform B references intermediate_output by name (table doesn't exist yet):model/Transform {t-b :id} (make-python-transform@@ -48,10 +49,10 @@(deftest python-transform-mixed-source-tables-test(testing "Python transform with mixed int and name-based refs"(mt/with-temp [:model/Transform {t-a :id} (make-python-transform- [{:alias "input" :table_id (mt/id :orders)}]+ [(transforms.tu/source-table-entry "input" (mt/id :orders))]"output_a"):model/Transform {t-b :id} (make-python-transform- [{:alias "existing" :table_id (mt/id :products)}+ [(transforms.tu/source-table-entry "existing" (mt/id :products)){:alias "from_transform":database_id (mt/id):schema "public"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.cljindex d67d887922f..8b96157d4a6 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj@@ -7,6 +7,7 @@[metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt][metabase.transforms.test-dataset :as transforms-dataset]+ [metabase.transforms.test-util :as transforms.tu][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -97,7 +98,9 @@" print(\"out2\")"" print(\"err2\", file=sys.stderr)"" return pd.DataFrame({'x': [42, 43]})"]- body {:source_tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}, :code (str/join "\n" program)}+ table-id (t2/select-one-pk :model/Table :db_id (mt/id))+ body {:source_tables [(transforms.tu/source-table-entry "test" table-id)]+ :code (str/join "\n" program)}{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/transforms-python/test-run" body)](is (nil? error))(is (str/includes? logs "out1\nerr1\nout2\nerr2"))@@ -106,9 +109,10 @@(defn- test-run [& {:keys [program user features source-tables extra-opts]:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]user :crowberto- source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}- features #{:transforms-basic :transforms-python}}}]- (let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]+ features #{:transforms :transforms-python}}}]+ (let [source-tables (or source-tables+ [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id) :active true))])+ body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)](mt/with-premium-features features(mt/user-http-request-full-response user :post "ee/transforms-python/test-run" body))))@@ -116,7 +120,7 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)(mt/dataset transforms-dataset/transforms-test(let [program ["def transform(customers):" " return customers"]- source-tables {"customers" (mt/id :transforms_customers)}+ source-tables [(transforms.tu/source-table-entry "customers" (mt/id :transforms_customers))]{:keys [status body]} (test-run :program program :source-tables source-tables)](is (= 200 status))(is (nil? (:error body)))@@ -126,7 +130,7 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)(mt/dataset transforms-dataset/transforms-test(let [program ["import pandas as pd" "def transform(customers):" " return pd.DataFrame()"]- source-tables {"customers" (mt/id :transforms_customers)}+ source-tables [(transforms.tu/source-table-entry "customers" (mt/id :transforms_customers))]{:keys [status body]} (test-run :program program :source-tables source-tables)](is (= 200 status))(is (nil? (:error body)))@@ -161,7 +165,7 @@(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))(testing "truncates sources"(let [program ["def transform(customers):" " return customers"]- source-tables {"customers" (mt/id :transforms_customers)}+ source-tables [(transforms.tu/source-table-entry "customers" (mt/id :transforms_customers))]response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})](is (=? {:body {:output {:rows #(= 2 (count %))}}} response))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.cljindex 1407c6e2ca3..610af8e1763 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj@@ -457,9 +457,9 @@" # Return processed dataframe\n"" return df")- source-tables (cond-> [{:alias source-table-name :table_id source-table-id}]+ source-tables (cond-> [(transforms.tu/source-table-entry source-table-name source-table-id)]exotic-table-id- (conj {:alias (str source-table-name "_exotic") :table_id exotic-table-id}))+ (conj (transforms.tu/source-table-entry (str source-table-name "_exotic") exotic-table-id)))result (execute-e2e-transform! table-name transform-code source-tables){:keys [columns] result-rows :rows} resultdiff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.cljindex 8dda61aa7e3..0bd87924394 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj@@ -71,7 +71,7 @@(let [original {:name "Gadget Products":source {:type "python":source-database (mt/id)- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]:body (str "import pandas as pd\n""\n""def transform():\n"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.cljindex 3cf81e7f0fe..e239e0991b4 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj@@ -29,7 +29,7 @@:model/Transform {transform-id :id} {:name "Gadget Products":source {:type "python":source-database (mt/id)- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]:body (str "import pandas as pd\n""\n""def transform():\n"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex fb46a322bac..dd63ecdce62 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -37,8 +37,8 @@(deftest python-transform-basic-dependencies-test(testing "Python transforms with source-tables dependencies are extracted correctly"- (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}- {:alias "products" :table_id (mt/id :products)}])]+ (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))+ (transforms.tu/source-table-entry "products" (mt/id :products))])](is (= #{{:table (mt/id :orders)}{:table (mt/id :products)}}(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))@@ -51,8 +51,8 @@:name "output_1"}:model/Field _ {:table_id table1:name "foo"}- :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")- :model/Transform {t2 :id} (make-python-transform [{:alias "output_1" :table_id table1}] "output_2")]+ :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")+ :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)] "output_2")](is (= {t1 #{}t2 #{t1}}(ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]])))))))))@@ -69,10 +69,10 @@:name "output_2"}:model/Field _ {:table_id table2:name "bar"}- :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")- :model/Transform {t2 :id} (make-python-transform [{:alias "products" :table_id (mt/id :products)}] "output_2")- :model/Transform {t3 :id} (make-python-transform [{:alias "output_1" :table_id table1}- {:alias "output_2" :table_id table2}] "final_output")]+ :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")+ :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2")+ :model/Transform {t3 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)+ (transforms.tu/source-table-entry "output_2" table2)] "final_output")](is (= {t1 #{}t2 #{}t3 #{t1 t2}}@@ -97,7 +97,7 @@:query {:source-table (mt/id :orders)}}"sql_output");; Python transform that depends on the SQL transform's output- :model/Transform {t2 :id} (make-python-transform [{:alias "sql_output" :table_id table1}] "python_output")+ :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "sql_output" table1)] "python_output");; Another SQL transform that depends on the Python transform's output:model/Transform {t3 :id} (make-transform{:database (mt/id)diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.cljindex f5daedf29b3..1dfc3b5027f 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj@@ -16,6 +16,7 @@[metabase.test.data.sql :as sql.tx][metabase.test.util :as tu][metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.test-util :as transforms.tu][metabase.util :as u][metabase.util.json :as json][toucan2.core :as t2]))@@ -55,29 +56,36 @@(defn- jsonl-output [expected] #(= expected (parse-jsonl %)))+(defn- tables-map->source-tables+ "Convert old test map format {\"alias\" table_id} to new source-table-entry format."+ [tables]+ (mapv (fn [[alias table-id]]+ (transforms.tu/source-table-entry (name alias) table-id))+ tables))+(defn execute! [{:keys [code tables]}]- (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]- (let [server-url (transforms-python.settings/python-runner-url)- cancel-chan (a/promise-chan)- table-name->id (or tables {})- test-id (next-job-run-id)- _ (python-runner/copy-tables-to-s3! {:run-id test-id- :shared-storage @shared-storage-ref- :source {:source-tables table-name->id}- :cancel-chan cancel-chan})- response (python-runner/execute-python-code-http-call! {:server-url server-url- :code code- :run-id test-id- :table-name->id table-name->id- :shared-storage @shared-storage-ref})- events (python-runner/read-events @shared-storage-ref)- output-manifest (python-runner/read-output-manifest @shared-storage-ref)]+ (let [source-tables (if (seq tables) (tables-map->source-tables tables) [])]+ (with-open [shared-storage-ref (s3/open-shared-storage! source-tables)]+ (let [server-url (transforms-python.settings/python-runner-url)+ cancel-chan (a/promise-chan)+ test-id (next-job-run-id)+ _ (python-runner/copy-tables-to-s3! {:run-id test-id+ :shared-storage @shared-storage-ref+ :source {:source-tables source-tables}+ :cancel-chan cancel-chan})+ response (python-runner/execute-python-code-http-call! {:server-url server-url+ :code code+ :run-id test-id+ :source-tables source-tables+ :shared-storage @shared-storage-ref})+ events (python-runner/read-events @shared-storage-ref)+ output-manifest (python-runner/read-output-manifest @shared-storage-ref)];; not sure about munging this all together but its what tests expect for now- (merge (:body response)- {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))- :output-manifest output-manifest- :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))- :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))+ (merge (:body response)+ {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))+ :output-manifest output-manifest+ :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))+ :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))})))))(defn ok-stdout [num-rows num-cols](format (str "Successfully saved %d rows to S3\n"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.cljindex c159d9100dc..4bf49a76597 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj@@ -64,9 +64,9 @@(deftest open-s3-shared-storage-test(testing "open-s3-shared-storage! returns closeable derefable"- (mt/with-premium-features #{:transforms-python :transforms-basic}- (let [table-name->id {"users" 1}- storage-ref (s3/open-shared-storage! table-name->id)+ (mt/with-premium-features #{:transforms-python :transforms}+ (let [source-tables [{:alias "users" :table_id 1 :database_id 1 :schema nil}]+ storage-ref (s3/open-shared-storage! source-tables){:keys [s3-client bucket-name objects]} @storage-ref](testing "can be dereferenced"(doseq [[k {:keys [method path url]}] objectsdiff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljindex 8d2be07fbba..fa62b6f6c38 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -149,7 +149,7 @@(let [original {:name "Gadget Products":source {:type "python":source-database (mt/id)- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]:body (str "import pandas as pd\n""\n""def transform():\n"@@ -223,7 +223,7 @@:source {:type "python":body (program->source program):source-database (mt/id)- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}:target (assoc target :database (mt/id))})))(block-on-run [{:keys [expect-status]} target transform-id]@@ -329,7 +329,7 @@:model/Transform transform {:name "Python Transform Cross DB":source {:type "python":source-database (mt/id)- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]:body "def transform():\n pass"}:target {:type "table":schema "PUBLIC"@@ -353,7 +353,7 @@:source {:type "python":body (str/join "\n" program):source-database (mt/id)- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]}+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]}:target (assoc target :database (mt/id))}));; using clojure-ey coordination with promises (I know j.u.c could be better here)@@ -526,7 +526,7 @@(let [initial-transform {:name "Schema Change Integration Test":source {:type "python":source-database (mt/id)- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]:body (str "import pandas as pd\n""\n""def transform():\n"@@ -545,7 +545,7 @@(let [updated-transform (assoc initial-transform:source {:type "python":source-database (mt/id)- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]:body (str "import pandas as pd\n""\n""def transform():\n"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.cljindex 6389f26ef74..d8e2030fd23 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj@@ -7,6 +7,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.test :as mt]+ [metabase.transforms.test-util :as transforms.tu][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -95,8 +96,8 @@(deftest ^:parallel analyze-entity-python-transform-test(testing "analyze-entity extracts dependencies from a python transform"(let [transform {:source {:type "python"- :source-tables [{:alias "orders" :table_id (mt/id :orders)}- {:alias "products" :table_id (mt/id :products)}]+ :source-tables [(transforms.tu/source-table-entry "orders" (mt/id :orders))+ (transforms.tu/source-table-entry "products" (mt/id :products))]:body "def transform(orders, products): return orders"}:target {:database (mt/id):schema "public"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.cljindex b22f8e48dea..7a1f59a73f3 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj@@ -182,7 +182,7 @@:id 456}}source {:type "python":body "import pandas as pd"- :source-tables [{:alias "orders" :table_id 123}]}]+ :source-tables [{:alias "orders" :table_id 123 :database_id 1 :schema "public"}]}](is (= [{:alias "orders" :database_id 1:schema "ws_isolated_123" :table "orders_isolated" :table_id 456}](:source-tables (remap-python-source table-mapping source))))))@@ -221,7 +221,7 @@:body "import pandas as pd":source-tables [{:alias "orders" :database_id 1:schema "public" :table "orders" :table_id 123}- {:alias "customers" :table_id 789}]}]+ {:alias "customers" :table_id 789 :database_id 1 :schema "public"}]}](is (= [{:alias "orders" :database_id 1:schema "ws_isolated_123" :table "orders_isolated" :table_id 456}{:alias "customers" :database_id 1diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsxindex 809f6aa58e3..b72c7e53e5c 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsx@@ -15,17 +15,7 @@ export const TransformSourceTablesDiff = ({newSourceTables,oldSourceTables,}: Props) => {- const oldSourceTablesIds = Object.values(oldSourceTables);- const newSourceTablesIds = Object.values(newSourceTables);-- const oldSourceNames = Object.fromEntries(- Object.entries(oldSourceTables).map(([k, v]) => [v, k]),- );- const newSourceNames = Object.fromEntries(- Object.entries(newSourceTables).map(([k, v]) => [v, k]),- );-- const removedTablesIds = oldSourceTablesIds.slice(newSourceTablesIds.length);+ const removedEntries = oldSourceTables.slice(newSourceTables.length);return (<Box@@ -37,41 +27,42 @@ export const TransformSourceTablesDiff = ({columnGap: 32,}}>- {newSourceTablesIds.map((id, index) => {- const oldTableId = oldSourceTablesIds[index];- const oldSourceName = oldSourceNames[oldTableId];- const newSourceName = newSourceNames[id];- const sourceNameChanged = oldSourceName !== newSourceName;+ {newSourceTables.map((entry, index) => {+ const oldEntry = oldSourceTables[index];+ const oldAlias = oldEntry?.alias;+ const newAlias = entry.alias;+ const sourceNameChanged = oldAlias !== newAlias;return (- <Fragment key={id}>+ <Fragment key={entry.alias}><Group gap="xs">- {sourceNameChanged && oldSourceName && (+ {sourceNameChanged && oldAlias && (<Text c="danger" component="s" td="line-through">- {oldSourceName}+ {oldAlias}</Text>)}<Text c={sourceNameChanged ? "success" : undefined}>- {newSourceName}+ {newAlias}</Text></Group>- <TableItem oldTableId={oldTableId} tableId={id} />+ <TableItem+ oldTableId={oldEntry?.table_id}+ tableId={entry.table_id}+ /></Fragment>);})}- {removedTablesIds.map((id) => {- const oldSourceName = oldSourceNames[id];-+ {removedEntries.map((entry) => {return (- <Fragment key={id}>+ <Fragment key={entry.alias}><Text c="danger" component="s" td="line-through">- {oldSourceName}+ {entry.alias}</Text>- <TableItem oldTableId={id} tableId={undefined} />+ <TableItem oldTableId={entry.table_id} tableId={undefined} /></Fragment>);})}diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.tsindex ae8b6e10a2e..020affdbe2a 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts@@ -3,7 +3,6 @@ import { useMemo } from "react";import { useGetTransformQuery } from "metabase/api";import { useSelector } from "metabase/lib/redux";import { getMetadata } from "metabase/selectors/metadata";-import { collapseSourceTableReferences } from "metabase/transforms/utils";import { useGetWorkspaceTransformQuery } from "metabase-enterprise/api";import type {PythonTransformSource,@@ -79,14 +78,12 @@ export function useTransformSources(return {oldSource,oldSourceTables: isPythonTransformSource(globalTransform?.source)- ? collapseSourceTableReferences(globalTransform.source["source-tables"])+ ? globalTransform.source["source-tables"]: undefined,oldTarget: globalTransform?.target,newSource,newSourceTables: isPythonTransformSource(workspaceTransform?.source)- ? collapseSourceTableReferences(- workspaceTransform.source["source-tables"],- )+ ? workspaceTransform.source["source-tables"]: undefined,newTarget: workspaceTransform?.target,hasError,diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.tsindex a0258aa2dba..f727e58fdc7 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.ts@@ -55,14 +55,12 @@ export function areSourceTablesEqual(return a === b;}- const aEntries = Object.entries(a);- const bEntries = Object.entries(b);-- if (aEntries.length !== bEntries.length) {+ if (a.length !== b.length) {return false;}- return aEntries.every(([key, value], index) => {- return bEntries[index][0] === key && bEntries[index][1] === value;- });+ return a.every(+ (entry, index) =>+ b[index].alias === entry.alias && b[index].table_id === entry.table_id,+ );}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsxindex 71f035beee5..97c8639df8f 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx@@ -1,12 +1,10 @@-import { useEffect, useMemo, useState } from "react";+import { useEffect, useState } from "react";import { t } from "ttag";import { skipToken, useListTablesQuery } from "metabase/api";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";-import { extractTableId } from "metabase/transforms/utils";import { Box, Button, Icon, Stack, Text } from "metabase/ui";import type {- ConcreteTableId,DatabaseId,PythonTransformTableAliases,Table,@@ -66,7 +64,10 @@ export function PythonDataPicker({const handleChange = (selections: TableSelection[]) => {if (database) {- const tableAliases = selectionsToTableAliases(selections);+ const tableAliases = selectionsToTableAliases(+ selections,+ tablesData ?? [],+ );onChange(database, tableAliases, tablesData ?? []);}};@@ -176,13 +177,6 @@ function SelectionInput({}) {const table = availableTables.find((table) => table.id === selection.tableId);- const selectedTableIds = useMemo((): ConcreteTableId[] => {- return Object.values(tables).flatMap((v) => {- const id = extractTableId(v);- return id != null ? [id] : [];- });- }, [tables]);-function handleAliasChange(newAlias: string) {const newSelection = {...selection,@@ -227,7 +221,7 @@ function SelectionInput({<TableSelectordatabase={database}table={table}- selectedTableIds={selectedTableIds}+ selectedTableIds={tables.map((t) => t.table_id)}onChange={handleTableChange}onRemove={onRemove}availableTables={availableTables}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.tsindex 793ac59e6d2..360319ec3b9 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts@@ -1,15 +1,18 @@import { slugify as toSlug } from "metabase/lib/formatting";-import { extractTableId } from "metabase/transforms/utils";-import type { PythonTransformTableAliases } from "metabase-types/api";+import type {+ PythonTransformTableAliases,+ PythonTransformTableEntry,+ Table,+} from "metabase-types/api";import type { TableSelection } from "./types";export function getInitialTableSelections(tables: PythonTransformTableAliases | undefined,) {- if (tables && Object.keys(tables).length > 0) {- return Object.entries(tables).map(([alias, value]) => ({- tableId: extractTableId(value),+ if (tables && tables.length > 0) {+ return tables.map(({ alias, table_id }) => ({+ tableId: table_id,alias,}));}@@ -24,16 +27,23 @@ export function getInitialTableSelections(export function selectionsToTableAliases(selections: TableSelection[],+ tableInfo: Table[],): PythonTransformTableAliases {- const tableAliases: PythonTransformTableAliases = {};-- for (const selection of selections) {- const { alias, tableId } = selection;- if (tableId !== undefined && alias !== "") {- tableAliases[alias] = tableId;- }- }- return tableAliases;+ return selections+ .filter((s) => s.tableId !== undefined && s.alias !== "")+ .map((s) => {+ const table = tableInfo.find((tbl) => tbl.id === s.tableId);+ if (!table || !s.tableId) {+ return null;+ }+ return {+ alias: s.alias,+ table_id: s.tableId,+ schema: table.schema,+ database_id: table.db_id,+ } satisfies PythonTransformTableEntry;+ })+ .filter((t) => t !== null);}export function slugify(diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsxindex c85c47e9ace..8729caae3e6 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx@@ -57,7 +57,7 @@ export function PythonTransformEditor({const newSource = {...source,"source-database": databaseId,- "source-tables": {},+ "source-tables": [],};onChangeSource(newSource);};diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsxindex 37783fac8c9..fa58fee6c57 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsx@@ -41,7 +41,7 @@ const mockPythonSource: PythonTransformSourceDraft = {type: "python",body: "# Python script\nprint('Hello, world!')","source-database": DATABASE_ID,- "source-tables": {},+ "source-tables": [],};const mockPythonTransform = createMockTransform({@@ -52,7 +52,7 @@ const mockPythonTransform = createMockTransform({type: "python",body: "def transform(): pass","source-database": DATABASE_ID,- "source-tables": {},+ "source-tables": [],},});diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsxindex e2474b04280..85f888883fe 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsx@@ -40,7 +40,7 @@ const mockPythonTransform = createMockTransform({type: "python",body: "def transform(): pass","source-database": DATABASE_ID,- "source-tables": {},+ "source-tables": [],},});diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.tsindex 96446a33de1..4442187e65f 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts@@ -9,7 +9,7 @@ export function updateTransformSignature(tables: PythonTransformTableAliases,tableInfo: Table[],): string {- const tableAliases = Object.keys(tables);+ const tableAliases = tables.map((t) => t.alias);const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;@@ -46,8 +46,8 @@ export function updateTransformSignature(const newArgsSection = tableAliases.map((alias) => {const padding = " ".repeat(maxAliasLength - alias.length);- const tableId = tables[alias];- const tableName = getTableName(tableId);+ const tableId = tables.find((t) => t.alias === alias)?.table_id;+ const tableName = tableId != null ? getTableName(tableId) : undefined;return ` ${alias}:${padding} DataFrame containing the data from the "${tableName}" table`;}).join("\n");@@ -118,8 +118,8 @@ ${newSignature}${tableAliases.map((alias) => {const padding = " ".repeat(maxAliasLength - alias.length);- const tableId = tables[alias];- const tableName = getTableName(tableId);+ const tableId = tables.find((t) => t.alias === alias)?.table_id;+ const tableName = tableId != null ? getTableName(tableId) : undefined;if (alias === tableName) {return ` ${alias}:${padding} DataFrame containing the data from the corresponding table`;}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.tsindex 59cacbdd91a..137e42561e5 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.ts@@ -20,7 +20,7 @@ export function getPythonSourceValidationResult(};}- if (Object.keys(source["source-tables"]).length === 0) {+ if (source["source-tables"].length === 0) {return {isValid: false,errorMessage: t`Select at least one table to alias`,diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.tsindex 69282f86dad..6fc4ae0730e 100644--- a/frontend/src/metabase-types/api/mocks/transform.ts+++ b/frontend/src/metabase-types/api/mocks/transform.ts@@ -40,7 +40,7 @@ export function createMockTransformSource(): TransformSource {export function createMockPythonTransformSource({sourceDatabase = 1,- sourceTables = {},+ sourceTables = [],body = "# Python script\nprint('Hello, world!')",}: {sourceDatabase?: DatabaseId;diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex 80df541c1f0..3c5ab4c2300 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -61,7 +61,14 @@ export type Transform = {export type SuggestedTransform = Partial<Pick<Transform, "id">> &Pick<Transform, "name" | "description" | "source" | "target">;-export type PythonTransformTableAliases = Record<string, ConcreteTableId>;+export type PythonTransformTableEntry = {+ alias: string;+ table_id: ConcreteTableId;+ schema?: SchemaName | null;+ database_id: DatabaseId;+};++export type PythonTransformTableAliases = PythonTransformTableEntry[];export type TransformSourceCheckpointStrategy = {type: "checkpoint";diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsxindex fcc150acd41..04879b040d7 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx@@ -300,7 +300,7 @@ function SourceStrategyFields({function getIsPythonTransformWithMultipleTables(source: TransformSource) {const isPythonTransform = source.type === "python";const isMultiTablePythonTransform =- isPythonTransform && Object.keys(source["source-tables"]).length > 1;+ isPythonTransform && source["source-tables"].length > 1;return isMultiTablePythonTransform;}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsxindex 066f14cc607..828971cf4b5 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx@@ -34,8 +34,7 @@ export function PythonKeysetColumnSelect({// Get the first (and should be only) table ID// Incremental transforms are only supported for single-table Python transformsconst tableId = useMemo(() => {- const tableIds = Object.values(sourceTables);- return tableIds.length === 1 ? tableIds[0] : null;+ return sourceTables.length === 1 ? sourceTables[0].table_id : null;}, [sourceTables]);// Fetch metadata for the tablediff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.tsindex 4a784950e54..3ef5d024a1f 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts@@ -39,7 +39,9 @@ export const useHasCheckpointOptions = (source: TransformSource) => {useNativeHasCheckpointFieldOptions(libQuery);const pythonTableId = isPythonTransform- ? Object.values(source["source-tables"]).find(isConcreteTableId)+ ? Object.values(source["source-tables"]).find(({ table_id }) =>+ isConcreteTableId(table_id),+ )?.table_id: null;const { data: pythonTable } = useGetTableQueryMetadataQuery(pythonTableId ? { id: pythonTableId } : skipToken,diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/utils.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/utils.tsindex 4b33783cb5b..f6813e202d0 100644--- a/frontend/src/metabase/transforms/pages/NewTransformPage/utils.ts+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/utils.ts@@ -28,7 +28,7 @@ export function getInitialPythonSource(): PythonTransformSourceDraft {return {type: "python","source-database": undefined,- "source-tables": {},+ "source-tables": [],body: `# Write your Python transformation script hereimport commonimport pandas as pddiff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsxindex c602054ebf0..0dc7bb935e3 100644--- a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsx+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsx@@ -26,7 +26,7 @@ const mockPythonSource: DraftTransformSource = {type: "python",body: "# Python script","source-database": 1,- "source-tables": {},+ "source-tables": [],};type SetupOpts = {diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.tsindex 7ac1df46c8b..a89a88d86fb 100644--- a/frontend/src/metabase/transforms/utils.ts+++ b/frontend/src/metabase/transforms/utils.ts@@ -8,11 +8,9 @@ import * as Lib from "metabase-lib";import type Metadata from "metabase-lib/v1/metadata/Metadata";import type {CollectionNamespace,- ConcreteTableId,Database,DatabaseId,DraftTransformSource,- PythonTransformTableAliases,TemplateTag,Transform,TransformRun,@@ -135,78 +133,11 @@ export function isSameSource(return Lib.areLegacyQueriesEqual(source1.query, source2.query);}if (source1.type === "python" && source2.type === "python") {- return _.isEqual(- {- ...source1,- "source-tables": collapseSourceTableReferences(- source1["source-tables"],- ),- },- {- ...source2,- "source-tables": collapseSourceTableReferences(- source2["source-tables"],- ),- },- );+ return _.isEqual(source1, source2);}return false;}-/**- * Extract a ConcreteTableId from a source-tables value, which may be either- * a bare integer or a map with table_id (from backend normalization).- */-export function extractTableId(- value: ConcreteTableId | Record<string, unknown>,-): ConcreteTableId | undefined {- if (typeof value === "number") {- return value;- }- if (typeof value === "object" && value != null) {- const id = (value as Record<string, unknown>)["table_id"];- if (typeof id === "number") {- return id as ConcreteTableId;- }- }- return undefined;-}--const collapseCache = new WeakMap<- Record<string, unknown>,- PythonTransformTableAliases->();--/**- * Collapse source-table references to plain { alias: tableId } form.- * The backend may return map refs (with table_id, schema, etc.) instead of- * bare integer IDs. Returns the original map unchanged if all values are- * already plain integers. Results are memoized by object reference.- */-export function collapseSourceTableReferences(- tables: Record<string, unknown>,-): PythonTransformTableAliases {- const cached = collapseCache.get(tables);- if (cached) {- return cached;- }- const values = Object.values(tables);- if (values.every((v) => typeof v === "number")) {- return tables as PythonTransformTableAliases;- }- const result: PythonTransformTableAliases = {};- for (const [key, value] of Object.entries(tables)) {- const id = extractTableId(- value as ConcreteTableId | Record<string, unknown>,- );- if (id != null) {- result[key] = id;- }- }- collapseCache.set(tables, result);- return result;-}-export function isSourceEmpty(source: DraftTransformSource,databaseId: DatabaseId,diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 801ad9b2c78..6f3de058de7 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -31,7 +31,6 @@add-source-readableis-temp-transform-table?][metabase.transforms.crud- source-tables-vec->map-for-fecheck-database-featurecheck-feature-enabled!validate-incremental-column-type!diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex e612fb2d9a3..c4ffd1468f7 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -19,16 +19,6 @@(set! *warn-on-reflection* true)-;; TODO(FE-source-tables): Remove this function and all call sites when FE adopts the array format for source-tables.-(defn source-tables-vec->map-for-fe- "Convert source-tables from internal vec format to legacy map format for FE compatibility.- Remove this when FE adopts the array format."- [transform]- (if (transforms-base.u/python-transform? transform)- (update-in transform [:source :source-tables]- transforms-base.u/source-tables-vec->alias-id-map)- transform))-(defn check-database-feature"Check that the target database supports the required features for this transform."[transform]@@ -87,8 +77,7 @@(transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)(transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)(map #(update % :last_run transforms-base.u/localize-run-timestamps))- (map transforms.u/add-source-readable)- (map source-tables-vec->map-for-fe))))))) ;; TODO(FE-source-tables): remove+ (map transforms.u/add-source-readable)))))))(defn get-transform"Get a specific transform."@@ -99,8 +88,7 @@(t2/hydrate :last_run :transform_tag_ids :creator :owner)(u/update-some :last_run transforms-base.u/localize-run-timestamps)(assoc :table target-table)- transforms.u/add-source-readable- source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove+ transforms.u/add-source-readable)))(defn create-transform!"Create new transform in the appdb.@@ -161,8 +149,7 @@(t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids :creator :owner))](events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})(-> transform- transforms.u/add-source-readable- source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove+ transforms.u/add-source-readable)))(defn delete-transform!"Delete a transform and publish the delete event."diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.cljindex 46679487d71..f38427e8e42 100644--- a/src/metabase/transforms/schema.clj+++ b/src/metabase/transforms/schema.clj@@ -27,12 +27,7 @@[:map[:source-database {:optional true} :int];; NB: if source is checkpoint, only one table allowed- ;; decode/normalize: convert FE map format to vec and enrich with DB metadata- [:source-tables [:sequential {:decode/normalize (fn [st]- (if (map? st)- (transforms-base.u/source-tables-map->vec st)- st))}- ::transforms-base.u/source-table-entry]]+ [:source-tables [:sequential ::transforms-base.u/source-table-entry]][:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :python]][:body :string][:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]])diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex a62fa0cff3c..56a3da85774 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -478,11 +478,6 @@;;; ------------------------------------------------- Source Table Schemas --------------------------------------------------(mu/defn source-tables-vec->alias-id-map :- [:map-of :string [:maybe :int]]- "Convert source-table entries to `{alias -> table_id}` map."- [entries :- [:sequential ::source-table-entry]]- (into {} (map (juxt :alias :table_id)) entries))-;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------(def ^:private ^:const batch-lookup-chunk-size@@ -523,8 +518,8 @@"A source table entry in the array format. Combines alias with table reference."[:map[:alias :string]- [:database_id {:optional true} :int]- [:schema {:optional true} [:maybe :string]]+ [:database_id :int]+ [:schema [:maybe :string]][:table {:optional true} :string][:table_id {:optional true} [:maybe :int]]])@@ -534,7 +529,7 @@For entries with only :database_id/:schema/:table, looks up :table_id.Throws if an integer table ID references a non-existent table.Map refs with non-existent tables get nil table_id (resolved later at execute time)."- [source-tables :- [:sequential ::source-table-entry]]+ [source-tables :- [:sequential [:map [:alias :string]]]](let [;; Entries that have table_id but lack table metadata need lookupneeds-metadata (filter (fn [e] (and (:table_id e) (not (:table e)))) source-tables)int-id->metadata (when (seq needs-metadata)@@ -565,24 +560,23 @@:else entry))source-tables)))-(mu/defn resolve-source-tables :- [:map-of :string :int]- "Resolve source-table entries to `{alias -> table_id}`. Throws if any table not found.+(mu/defn resolve-source-tables :- [:sequential ::source-table-entry]+ "Resolve source-table entries to entries with :table_id filled in. Throws if any table not found.For execute time — all entries must resolve to valid table IDs."[source-tables :- [:sequential ::source-table-entry]](let [needs-lookup (filter missing-table-id? source-tables)lookup (or (batch-lookup-table-ids needs-lookup) {})- resolved (u/for-map [entry source-tables]- [(:alias entry) (or (:table_id entry) (lookup (source-table-ref->key entry)))])- unresolved (for [entry source-tables- :let [table-id (or (:table_id entry) (lookup (source-table-ref->key entry)))]- :when (nil? table-id)]- {:alias (:alias entry)- :table (if-let [schema (:schema entry)]- (str schema "." (:table entry))- (:table entry))- :ref entry})]+ resolved (mapv (fn [entry]+ (let [table-id (or (:table_id entry) (lookup (source-table-ref->key entry)))]+ (assoc entry :table_id table-id)))+ source-tables)+ unresolved (filter #(nil? (:table_id %)) resolved)](when (seq unresolved)- (throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved)))+ (throw (ex-info (str "Tables not found: " (str/join ", " (map (fn [{:keys [schema table]}]+ (if schema+ (str schema "." table)+ table))+ unresolved))){:unresolved unresolved:transform-message "Input table not found"})))resolved))diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 159d7b636b4..30d711e09c7 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -179,8 +179,7 @@403(deferred-tru "A table with that name already exists."))(-> (transforms.core/create-transform! body)- transforms.u/add-source-readable- transforms.core/source-tables-vec->map-for-fe)) ;; TODO(FE-source-tables): remove+ transforms.u/add-source-readable))(api.macros/defendpoint :get "/:id" :- TransformResponse"Get a specific transform."@@ -198,8 +197,7 @@dep-ids (get global-ordering id)dependencies (map id->transform dep-ids)](->> (t2/hydrate dependencies :creator :owner)- transforms.u/add-source-readable- (mapv transforms.core/source-tables-vec->map-for-fe)))) ;; TODO(FE-source-tables): remove+ transforms.u/add-source-readable)))(def ^:private MergeHistoryEntry[:mapdiff --git a/test/metabase/transforms/test_util.clj b/test/metabase/transforms/test_util.cljindex 75f773c0d0d..07402356171 100644--- a/test/metabase/transforms/test_util.clj+++ b/test/metabase/transforms/test_util.clj@@ -15,6 +15,12 @@(set! *warn-on-reflection* true)+(defn source-table-entry+ "Build a full source-table-entry from alias + table_id by looking up database_id and schema."+ [alias table-id]+ (let [{:keys [db_id schema]} (t2/select-one [:model/Table :db_id :schema] :id table-id)]+ {:alias alias :table_id table-id :database_id db_id :schema schema}))+(defn drop-target!"Drop transform target `target` and clean up its metadata.`target` can be a string or a map. If `target` is a string, type :table is assumed.diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex 45ae3057eb2..13fbf0029e1 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -224,17 +224,19 @@(is (= (:id t1) (:table_id (second result)))))))))(deftest resolve-source-tables-test- (testing "resolve-source-tables returns {alias -> table_id} map from vec input"+ (testing "resolve-source-tables returns entries with :table_id filled in"(mt/with-temp [:model/Database db {}:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}:model/Table t2 {:db_id (:id db) :name "table_two" :schema nil}](testing "resolves entry with table_id"- (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}]]- (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}]+ result (transforms-base.u/resolve-source-tables source-tables)]+ (is (= (:id t1) (:table_id (first result))))))(testing "looks up table_id for entry without it"- (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}]]- (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}]+ result (transforms-base.u/resolve-source-tables source-tables)]+ (is (= (:id t1) (:table_id (first result))))))(testing "throws for non-existent table"(let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]]@@ -247,10 +249,11 @@(transforms-base.u/resolve-source-tables source-tables)))))(testing "handles multiple entries"- (let [source-tables [{:alias "t1" :table_id (:id t1)}- {:alias "t2" :database_id (:id db) :schema nil :table "table_two"}]]- (is (= {"t1" (:id t1) "t2" (:id t2)}- (transforms-base.u/resolve-source-tables source-tables))))))))+ (let [source-tables [{:alias "t1" :table_id (:id t1) :database_id (:id db) :schema nil}+ {:alias "t2" :database_id (:id db) :schema nil :table "table_two"}]+ result (transforms-base.u/resolve-source-tables source-tables)]+ (is (= (:id t1) (:table_id (first result))))+ (is (= (:id t2) (:table_id (second result)))))))))(deftest source-tables-readable?-test(testing "source-tables-readable? function"