Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

use array/list for sources table in transforms (#70100)

Jakub Chodorowicz80d08cce82eb
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
index 7016d2c8c66..d3f3385c2ff 100644
--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
@@ -1537,7 +1537,7 @@ LIMIT
def transform(foo):
return pd.DataFrame([{"foo": 42 }])
`,
- sourceTables: { foo: id },
+ sourceTables: pythonSourceTables("foo", id),
visitTransform: true,
});
},
@@ -1575,7 +1575,7 @@ LIMIT
def transform(foo):
return pd.DataFrame([{"foo": 42 }])
`,
- sourceTables: { foo: id },
+ sourceTables: pythonSourceTables("foo", id),
visitTransform: true,
});
},
@@ -1615,7 +1615,7 @@ LIMIT
def transform(foo):
return pd.DataFrame([{"foo": 42 }])
`,
- sourceTables: { foo: id },
+ sourceTables: pythonSourceTables("foo", id),
visitTransform: true,
});
},
@@ -1651,7 +1651,7 @@ LIMIT
def transform(foo):
return pd.DataFrame([{"foo": 42 }])
`,
- sourceTables: { foo: id },
+ sourceTables: pythonSourceTables("foo", id),
visitTransform: true,
});
},
@@ -3530,7 +3530,7 @@ describe(
print("Hello, world!")
return pd.DataFrame([{"foo": common.useful_calculation(40, 2) }])
`,
- sourceTables: { foo: id },
+ sourceTables: pythonSourceTables("foo", id),
visitTransform: true,
});
},
@@ -3572,7 +3572,7 @@ describe(
def transform(foo):
return pd.DataFrame([{"foo": 42}])
`,
- sourceTables: { foo: id },
+ sourceTables: pythonSourceTables("foo", id),
visitTransform: true,
});
},
@@ -3848,6 +3848,20 @@ function createPythonTransform(opts: {
});
}
+function pythonSourceTables(
+ alias: string,
+ tableId: number,
+): PythonTransformTableAliases {
+ return [
+ {
+ alias,
+ table_id: tableId,
+ database_id: WRITABLE_DB_ID,
+ schema: TARGET_SCHEMA,
+ },
+ ];
+}
+
function visitTableQuestion({
targetTable = TARGET_TABLE,
targetSchema = TARGET_SCHEMA,
diff --git a/e2e/test/scenarios/data-studio/workspaces.cy.spec.ts b/e2e/test/scenarios/data-studio/workspaces.cy.spec.ts
index 7162d958203..3e21bd7f07c 100644
--- a/e2e/test/scenarios/data-studio/workspaces.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/workspaces.cy.spec.ts
@@ -1765,7 +1765,20 @@ describe("scenarios > data studio > workspaces", () => {
}).then((id2) => {
createPythonTransform({
body: TEST_PYTHON_TRANSFORM,
- sourceTables: { foo: id1, bar: id2 },
+ sourceTables: [
+ {
+ alias: "foo",
+ table_id: id1,
+ database_id: WRITABLE_DB_ID,
+ schema: "Schema A",
+ },
+ {
+ alias: "bar",
+ table_id: id2,
+ database_id: WRITABLE_DB_ID,
+ schema: "Schema B",
+ },
+ ],
visitTransform: true,
});
});
@@ -1908,7 +1921,20 @@ describe("scenarios > data studio > workspaces", () => {
}).then((id2) => {
createPythonTransform({
body: TEST_PYTHON_TRANSFORM_MULTI_TABLE,
- sourceTables: { animals_a: id1, animals_b: id2 },
+ sourceTables: [
+ {
+ alias: "animals_a",
+ table_id: id1,
+ database_id: WRITABLE_DB_ID,
+ schema: "Schema A",
+ },
+ {
+ alias: "animals_b",
+ table_id: id2,
+ database_id: WRITABLE_DB_ID,
+ schema: "Schema B",
+ },
+ ],
visitTransform: true,
});
});
@@ -2517,7 +2543,14 @@ function createTransforms({ visit }: { visit?: boolean } = { visit: false }) {
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then((id) => {
createPythonTransform({
body: TEST_PYTHON_TRANSFORM,
- sourceTables: { foo: id },
+ sourceTables: [
+ {
+ alias: "foo",
+ table_id: id,
+ database_id: WRITABLE_DB_ID,
+ schema: TARGET_SCHEMA,
+ },
+ ],
});
});
diff --git a/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.ts b/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.ts
index 1b2caab2772..d87f5e3ea42 100644
--- a/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.ts
+++ b/e2e/test/scenarios/metabot/transforms-codegen.cy.spec.ts
@@ -1,6 +1,7 @@
import dedent from "ts-dedent";
import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
+import type { PythonTransformTableAliases } from "metabase-types/api";
const { H } = cy;
@@ -168,7 +169,7 @@ describe(
createMockPythonTransformJSON(
null,
WRITABLE_DB_ID,
- { metabase_table_df: 152 },
+ pythonSourceTables("metabase_table_df", 152),
"import pandas as pd\\n\\ndef transform(metabase_table_df):\\n return pd.DataFrame({'value': [1]})",
),
),
@@ -194,7 +195,7 @@ describe(
createMockPythonTransformJSON(
null,
WRITABLE_DB_ID,
- { metabase_table_df: 152 },
+ pythonSourceTables("metabase_table_df", 152),
"import pandas as pd\\n\\ndef transform(metabase_table_df):\\n return pd.DataFrame({'value': [2]})",
),
),
@@ -228,7 +229,7 @@ describe(
createMockPythonTransformJSON(
null,
WRITABLE_DB_ID,
- { metabase_table_df: 152 },
+ pythonSourceTables("metabase_table_df", 152),
"import pandas as pd\\n\\ndef transform(metabase_table_df):\\n return pd.DataFrame({'value': [4]})",
),
),
@@ -385,7 +386,7 @@ describe(
def transform(foo):
return pd.DataFrame({'value': [1]})
`,
- sourceTables: { foo: tableId },
+ sourceTables: pythonSourceTables("foo", Number(tableId)),
}).then(({ body: transform }) => {
visitTransformListPage();
getMetabotButton().click();
@@ -397,7 +398,7 @@ describe(
createMockPythonTransformJSON(
Number(transform.id),
WRITABLE_DB_ID,
- { foo: tableId },
+ pythonSourceTables("foo", Number(tableId)),
"import pandas as pd\\n\\ndef transform(foo):\\n return pd.DataFrame({'value': [2]})",
),
),
@@ -439,7 +440,7 @@ describe(
createMockPythonTransformJSON(
Number(transform.id),
WRITABLE_DB_ID,
- { metabase_table_df: 152 },
+ pythonSourceTables("metabase_table_df", 152),
"import pandas as pd\\n\\ndef transform(foo):\\n return pd.DataFrame({'value': [4]})",
),
),
@@ -474,11 +475,27 @@ const createMockNativeTransformJSON = (
const createMockPythonTransformJSON = (
id: number | null,
databaseId: number,
- sourceTables: { [tableName: string]: number },
+ sourceTables: PythonTransformTableAliases,
body: string,
) =>
`{"id":${id},"name":"A number","entity_id":null,"description":"","source":{"type":"python","source-database":${databaseId},"source-tables":${JSON.stringify(sourceTables)},"body":"${body}"},"target":{"type":"table","name":""},"created_at":null,"updated_at":null}`;
+function pythonSourceTables(
+ alias: string,
+ tableId: number,
+ schema = "Schema A",
+ databaseId = WRITABLE_DB_ID,
+): PythonTransformTableAliases {
+ return [
+ {
+ alias,
+ table_id: tableId,
+ database_id: databaseId,
+ schema,
+ },
+ ];
+}
+
const createMockTransformSuggestionResponse = (
text: string,
transformJSON: string,
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
index 6916ea08e35..a0c8bd9f069 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/api.clj
@@ -9,6 +9,7 @@
[metabase.api.routes.common :refer [+auth]]
[metabase.api.util.handlers :as handlers]
[metabase.permissions.core :as perms]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util.i18n :as i18n]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -63,10 +64,10 @@
per_input_row_limit 100}}
:- [:map
[:code :string]
- [:source_tables [:map-of {:min 1} :string :int]]
+ [:source_tables [:sequential {:min 1} ::transforms-base.u/source-table-entry]]
[:output_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]]
[:per_input_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]]]]
- (let [db-ids (t2/select-fn-set :db_id [:model/Table :db_id] :id [:in (vals source_tables)])]
+ (let [db-ids (t2/select-fn-set :db_id [:model/Table :db_id] :id [:in (map :table_id source_tables)])]
(api/check-400 (= (count db-ids) 1) (i18n/deferred-tru "All source tables must belong to the same database."))
(api/check-403 (perms/has-db-transforms-permission? api/*current-user-id* (first db-ids))))
;; NOTE: we do not test database support, as there is no write target.
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index 1f501795297..1620e9be98c 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -240,7 +240,7 @@
{:server-url server-url
:code (:body source)
:run-id run-id
- :table-name->id resolved-source-tables
+ :source-tables resolved-source-tables
:shared-storage @shared-storage-ref})
output-manifest (python-runner/read-output-manifest @shared-storage-ref)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index aea8c43f44b..d8249805e4a 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -200,12 +200,16 @@
(defn execute-python-code-http-call!
"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
the response from the server."
- [{:keys [server-url code request-id run-id table-name->id shared-storage timeout-secs]}]
+ [{:keys [server-url code request-id run-id source-tables shared-storage timeout-secs]}]
(let [{:keys [objects]} shared-storage
{:keys [output output-manifest events]} objects
url-for-path (fn [path] (:url (get objects path)))
- table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))
- table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
+ table-name->url (into {} (map (fn [{:keys [alias table_id]}]
+ [alias (url-for-path [:table table_id :data])]))
+ source-tables)
+ table-name->manifest-url (into {} (map (fn [{:keys [alias table_id]}]
+ [alias (url-for-path [:table table_id :manifest])]))
+ source-tables)
payload {:code code
:library (t2/select-fn->fn :path :source :model/PythonLibrary)
:timeout (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))
@@ -292,21 +296,19 @@
(when (and (:source-incremental-strategy source)
(> (count (:source-tables source)) 1))
(throw (ex-info "Incremental transforms for python only supports one source table" {})))
- (doseq [[table-name v] (:source-tables source)
- ;; TODO (Chris 2026-03-03) We should also handle the case where table-id has not been backfilled yet...
- :let [table-id (if (int? v) v (:table_id v))
- {:keys [s3-client bucket-name objects]} shared-storage
- {data-path :path} (get objects [:table table-id :data])
- {manifest-path :path} (get objects [:table table-id :manifest])]]
+ (doseq [{:keys [alias table_id]} (:source-tables source)
+ :let [{:keys [s3-client bucket-name objects]} shared-storage
+ {data-path :path} (get objects [:table table_id :data])
+ {manifest-path :path} (get objects [:table table_id :manifest])]]
(let [tmp-data-file (File/createTempFile data-path "")
tmp-meta-file (File/createTempFile manifest-path "")]
(try
- (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
+ (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table_id)
driver (t2/select-one-fn :engine :model/Database db-id)
- fields-meta (fields-metadata driver table-id)
- manifest (generate-manifest table-id fields-meta)]
+ fields-meta (fields-metadata driver table_id)
+ manifest (generate-manifest table_id fields-meta)]
(transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]]
- (let [query (build-table-query table-id (:source-incremental-strategy source) source-range-params limit)]
+ (let [query (build-table-query table_id (:source-incremental-strategy source) source-range-params limit)]
(write-query-data-to-file!
{:query query
:fields-meta fields-meta
@@ -324,10 +326,10 @@
(catch InterruptedException ie (throw ie))
(catch Throwable t
(throw (ex-info "An error occurred while copying table data to S3"
- {:table-id table-id
+ {:table-id table_id
:transform-message (or (:transform-message (ex-data t))
;; Cast table-id to string manually, to avoid thousands separators.
- (i18n/tru "Failed to copy table contents to shared storage: {0} ({1})" table-name (str table-id)))}
+ (i18n/tru "Failed to copy table contents to shared storage: {0} ({1})" alias (str table_id)))}
t)))
(finally
(safe-delete tmp-data-file)
@@ -339,7 +341,7 @@
Args:
:code - Python code to execute
- :source-tables - Map of table-name -> table-id (already resolved)
+ :source-tables - Sequential of source-table entries [{:alias ... :table_id ...} ...]
:row-limit - Max rows to return (also limits input rows)
:timeout-secs - Optional timeout override
@@ -361,7 +363,7 @@
{:server-url server-url
:code code
:request-id (u/generate-nano-id)
- :table-name->id source-tables
+ :source-tables source-tables
:timeout-secs timeout-secs
:shared-storage @shared-storage-ref})
events (read-events @shared-storage-ref)]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
index 2992c7cfffa..09b8dadf76e 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
@@ -123,7 +123,7 @@
(.build))]
(.toString (.url (.presignPutObject presigner request)))))
-(defn- s3-shared-storage [table-name->id]
+(defn- s3-shared-storage [source-tables]
(let [work-dir-name (working-dir-for-run)
container-presigner (create-s3-presigner-for-container)
bucket-name (transforms-python.settings/python-storage-s-3-bucket)
@@ -142,9 +142,9 @@
{:output (ref :put "output.csv")
:output-manifest (ref :put "output-manifest.json")
:events (ref :put "events.jsonl")}
- (for [[table-name id] table-name->id]
- {[:table id :manifest] (ref :get (str "table-" (name table-name) "-" id ".manifest.json"))
- [:table id :data] (ref :get (str "table-" (name table-name) "-" id ".jsonl"))}))}))
+ (for [{:keys [alias table_id]} source-tables]
+ {[:table table_id :manifest] (ref :get (str "table-" alias "-" table_id ".manifest.json"))
+ [:table table_id :data] (ref :get (str "table-" alias "-" table_id ".jsonl"))}))}))
(declare delete-many)
@@ -172,8 +172,8 @@
When the value is closed, all the relevant keys will be deleted, but this is best-effort only.
We rely on the bucket retention policy to ensure any stragglers are eventually deleted."
- ^Closeable [table-name->id]
- (let [shared-storage (s3-shared-storage table-name->id)]
+ ^Closeable [source-tables]
+ (let [shared-storage (s3-shared-storage source-tables)]
(reify IDeref
(deref [_] shared-storage)
Closeable
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj
index 17ba26b8560..ed029872d4c 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj
@@ -53,8 +53,7 @@
[:python
[:map {:closed true}
[:source-database {:optional true} :int]
- [:source-tables [:sequential {:decode/normalize transforms-base.u/normalize-source-tables-structure}
- [:map [:alias [:string {:min 1}]] [:table_id :int]]]]
+ [:source-tables [:sequential ::transforms-base.u/source-table-entry]]
[:type [:= "python"]]
[:body :string]]]])
@@ -193,9 +192,7 @@
(-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)
(t2/select-one :workspace_id ws-id :ref_id tx-id)
api/check-404
- attach-isolated-target
- ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables
- transforms/source-tables-vec->map-for-fe))
+ attach-isolated-target))
(defn create-workspace-transform!
"Shared logic for creating a new workspace transform.
@@ -236,9 +233,7 @@
transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body
:ref-id ref-id)]
(-> (select-malli-keys WorkspaceTransform workspace-transform-alias transform)
- attach-isolated-target
- ;; TODO (Ngoc 2026-03-04) -- remove when FE sends/expects array format for source-tables
- transforms/source-tables-vec->map-for-fe))))
+ attach-isolated-target))))
;;; ---------------------------------------- Pending inputs ----------------------------------------
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
index 6440ef0923e..3226b7a670a 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
@@ -94,14 +94,14 @@
:keys [cols rows]} (python-runner/execute-and-read-output!
{:code (:body remapped-source)
:source-tables resolved-source-tables
- :row-limit preview-row-limit})
- flatrows (apply juxt (map (fn [c] (let [cname (:name c)] #(get % cname))) cols))]
+ :row-limit preview-row-limit})]
(if (= :succeeded (:status result))
- {:status :succeeded
- ;; return logs for debugging
- :logs (str/join "\n" (:logs result))
- :data {:cols cols
- :rows (mapv flatrows rows)}}
+ (let [flatrows (apply juxt (map (fn [c] (let [cname (:name c)] #(get % cname))) cols))]
+ {:status :succeeded
+ ;; return logs for debugging
+ :logs (str/join "\n" (:logs result))
+ :data {:cols cols
+ :rows (mapv flatrows rows)}})
{:status :failed
:message (:message result)})))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
index 2b3dee503f4..a2b8f3fa31b 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
@@ -6,6 +6,7 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.test :as mt]
+ [metabase.transforms.test-util :as transforms.tu]
[toucan2.core :as t2]))
(deftest ^:parallel upstream-deps-card-test
@@ -199,9 +200,8 @@
orders-id (mt/id :orders)]
(mt/with-temp [:model/Transform transform {:name "Test Transform"
:source {:type :python
- :source-tables [{:alias "PRODUCTS" :table_id products-id}
- {:alias "ORDERS" :table_id orders-id}]
- ;; A problematic field, hopefully removed again.
+ :source-tables [(transforms.tu/source-table-entry "PRODUCTS" products-id)
+ (transforms.tu/source-table-entry "ORDERS" orders-id)]
:source-database (mt/id)
:body "..."}
:target {:schema "PUBLIC"
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
index a50ab52d0dd..f32e2a0c78d 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
@@ -17,7 +17,6 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
- [metabase.transforms.crud :as transforms.crud]
[metabase.util :as u]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
@@ -1282,9 +1281,7 @@
(testing "With superuser permissions"
(is (=? {:structured_output [(mt/obj->json->obj (select-keys t1 [:id :entity_id :name :description :source]))
;; note: t2 not included because it's a (non-native) MBQL query
- ;; API converts source-tables from vec back to map for FE
- (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe
- (select-keys t3 [:id :entity_id :name :description :source])))]
+ (mt/obj->json->obj (select-keys t3 [:id :entity_id :name :description :source]))]
:conversation_id conversation-id}
(-> (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transforms"
{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}
@@ -1329,9 +1326,7 @@
(testing "With superuser permissions"
(doseq [transform [t1 t2]]
(testing (:name transform)
- ;; API converts source-tables from vec back to map for FE
- (is (=? {:structured_output (mt/obj->json->obj (transforms.crud/source-tables-vec->map-for-fe
- (select-keys transform [:id :entity_id :name :description :source :target])))
+ (is (=? {:structured_output (mt/obj->json->obj (select-keys transform [:id :entity_id :name :description :source :target]))
:conversation_id conversation-id}
(mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-transform-details"
{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj
index 3372a8cf837..40993e5cfe6 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/dependencies_test.clj
@@ -7,6 +7,7 @@
[metabase.permissions.models.permissions :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
+ [metabase.transforms.test-util :as transforms.tu]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -251,13 +252,13 @@
:source {:type "python"
:body "print('hello')"
:source-database (mt/id)
- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}
+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}
:target {:type "table"
:schema "public"
:name "python_transform_table"}}]
(let [modified-source {:type "python"
:body "print('modified')"
- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}
+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}
result (metabot.dependencies/check-transform-dependencies
{:id python-transform-id
:source modified-source})]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index ab502aea2a7..c27ce039172 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -130,7 +130,7 @@
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id (checkpoint-filter-field)}}
:python {:type "python"
- :source-tables [{:alias "transforms_products" :table_id (mt/id :transforms_products)}]
+ :source-tables [(transforms.tu/source-table-entry "transforms_products" (mt/id :transforms_products))]
:body incremental-python-body
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-field-id (checkpoint-filter-field)}})
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
index 75702b2064a..5bb0942f80d 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
@@ -4,6 +4,7 @@
[metabase.test :as mt]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.ordering :as ordering]
+ [metabase.transforms.test-util :as transforms.tu]
[toucan2.core :as t2]))
(defn- make-python-transform
@@ -25,7 +26,7 @@
(testing "Python transform with name-based source table ref resolves to producing transform"
(mt/with-temp [;; Transform A produces table "intermediate_output"
:model/Transform {t-a :id} (make-python-transform
- [{:alias "input" :table_id (mt/id :orders)}]
+ [(transforms.tu/source-table-entry "input" (mt/id :orders))]
"intermediate_output")
;; Transform B references intermediate_output by name (table doesn't exist yet)
:model/Transform {t-b :id} (make-python-transform
@@ -48,10 +49,10 @@
(deftest python-transform-mixed-source-tables-test
(testing "Python transform with mixed int and name-based refs"
(mt/with-temp [:model/Transform {t-a :id} (make-python-transform
- [{:alias "input" :table_id (mt/id :orders)}]
+ [(transforms.tu/source-table-entry "input" (mt/id :orders))]
"output_a")
:model/Transform {t-b :id} (make-python-transform
- [{:alias "existing" :table_id (mt/id :products)}
+ [(transforms.tu/source-table-entry "existing" (mt/id :products))
{:alias "from_transform"
:database_id (mt/id)
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
index d67d887922f..8b96157d4a6 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
@@ -7,6 +7,7 @@
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
[metabase.transforms.test-dataset :as transforms-dataset]
+ [metabase.transforms.test-util :as transforms.tu]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -97,7 +98,9 @@
" print(\"out2\")"
" print(\"err2\", file=sys.stderr)"
" return pd.DataFrame({'x': [42, 43]})"]
- body {:source_tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}, :code (str/join "\n" program)}
+ table-id (t2/select-one-pk :model/Table :db_id (mt/id))
+ body {:source_tables [(transforms.tu/source-table-entry "test" table-id)]
+ :code (str/join "\n" program)}
{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/transforms-python/test-run" body)]
(is (nil? error))
(is (str/includes? logs "out1\nerr1\nout2\nerr2"))
@@ -106,9 +109,10 @@
(defn- test-run [& {:keys [program user features source-tables extra-opts]
:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]
user :crowberto
- source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}
- features #{:transforms-basic :transforms-python}}}]
- (let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
+ features #{:transforms :transforms-python}}}]
+ (let [source-tables (or source-tables
+ [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id) :active true))])
+ body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
(mt/with-premium-features features
(mt/user-http-request-full-response user :post "ee/transforms-python/test-run" body))))
@@ -116,7 +120,7 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/dataset transforms-dataset/transforms-test
(let [program ["def transform(customers):" " return customers"]
- source-tables {"customers" (mt/id :transforms_customers)}
+ source-tables [(transforms.tu/source-table-entry "customers" (mt/id :transforms_customers))]
{:keys [status body]} (test-run :program program :source-tables source-tables)]
(is (= 200 status))
(is (nil? (:error body)))
@@ -126,7 +130,7 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/dataset transforms-dataset/transforms-test
(let [program ["import pandas as pd" "def transform(customers):" " return pd.DataFrame()"]
- source-tables {"customers" (mt/id :transforms_customers)}
+ source-tables [(transforms.tu/source-table-entry "customers" (mt/id :transforms_customers))]
{:keys [status body]} (test-run :program program :source-tables source-tables)]
(is (= 200 status))
(is (nil? (:error body)))
@@ -161,7 +165,7 @@
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))
(testing "truncates sources"
(let [program ["def transform(customers):" " return customers"]
- source-tables {"customers" (mt/id :transforms_customers)}
+ source-tables [(transforms.tu/source-table-entry "customers" (mt/id :transforms_customers))]
response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})]
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
index 1407c6e2ca3..610af8e1763 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -457,9 +457,9 @@
" # Return processed dataframe\n"
" return df")
- source-tables (cond-> [{:alias source-table-name :table_id source-table-id}]
+ source-tables (cond-> [(transforms.tu/source-table-entry source-table-name source-table-id)]
exotic-table-id
- (conj {:alias (str source-table-name "_exotic") :table_id exotic-table-id}))
+ (conj (transforms.tu/source-table-entry (str source-table-name "_exotic") exotic-table-id)))
result (execute-e2e-transform! table-name transform-code source-tables)
{:keys [columns] result-rows :rows} result
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
index 8dda61aa7e3..0bd87924394 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
@@ -71,7 +71,7 @@
(let [original {:name "Gadget Products"
:source {:type "python"
:source-database (mt/id)
- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]
+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
index 3cf81e7f0fe..e239e0991b4 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
@@ -29,7 +29,7 @@
:model/Transform {transform-id :id} {:name "Gadget Products"
:source {:type "python"
:source-database (mt/id)
- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]
+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index fb46a322bac..dd63ecdce62 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -37,8 +37,8 @@
(deftest python-transform-basic-dependencies-test
(testing "Python transforms with source-tables dependencies are extracted correctly"
- (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}
- {:alias "products" :table_id (mt/id :products)}])]
+ (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))
+ (transforms.tu/source-table-entry "products" (mt/id :products))])]
(is (= #{{:table (mt/id :orders)}
{:table (mt/id :products)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
@@ -51,8 +51,8 @@
:name "output_1"}
:model/Field _ {:table_id table1
:name "foo"}
- :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")
- :model/Transform {t2 :id} (make-python-transform [{:alias "output_1" :table_id table1}] "output_2")]
+ :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
+ :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)] "output_2")]
(is (= {t1 #{}
t2 #{t1}}
(ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]])))))))))
@@ -69,10 +69,10 @@
:name "output_2"}
:model/Field _ {:table_id table2
:name "bar"}
- :model/Transform {t1 :id} (make-python-transform [{:alias "orders" :table_id (mt/id :orders)}] "output_1")
- :model/Transform {t2 :id} (make-python-transform [{:alias "products" :table_id (mt/id :products)}] "output_2")
- :model/Transform {t3 :id} (make-python-transform [{:alias "output_1" :table_id table1}
- {:alias "output_2" :table_id table2}] "final_output")]
+ :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
+ :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2")
+ :model/Transform {t3 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)
+ (transforms.tu/source-table-entry "output_2" table2)] "final_output")]
(is (= {t1 #{}
t2 #{}
t3 #{t1 t2}}
@@ -97,7 +97,7 @@
:query {:source-table (mt/id :orders)}}
"sql_output")
;; Python transform that depends on the SQL transform's output
- :model/Transform {t2 :id} (make-python-transform [{:alias "sql_output" :table_id table1}] "python_output")
+ :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "sql_output" table1)] "python_output")
;; Another SQL transform that depends on the Python transform's output
:model/Transform {t3 :id} (make-transform
{:database (mt/id)
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
index f5daedf29b3..1dfc3b5027f 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
@@ -16,6 +16,7 @@
[metabase.test.data.sql :as sql.tx]
[metabase.test.util :as tu]
[metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.test-util :as transforms.tu]
[metabase.util :as u]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -55,29 +56,36 @@
(defn- jsonl-output [expected] #(= expected (parse-jsonl %)))
+(defn- tables-map->source-tables
+ "Convert old test map format {\"alias\" table_id} to new source-table-entry format."
+ [tables]
+ (mapv (fn [[alias table-id]]
+ (transforms.tu/source-table-entry (name alias) table-id))
+ tables))
+
(defn execute! [{:keys [code tables]}]
- (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
- (let [server-url (transforms-python.settings/python-runner-url)
- cancel-chan (a/promise-chan)
- table-name->id (or tables {})
- test-id (next-job-run-id)
- _ (python-runner/copy-tables-to-s3! {:run-id test-id
- :shared-storage @shared-storage-ref
- :source {:source-tables table-name->id}
- :cancel-chan cancel-chan})
- response (python-runner/execute-python-code-http-call! {:server-url server-url
- :code code
- :run-id test-id
- :table-name->id table-name->id
- :shared-storage @shared-storage-ref})
- events (python-runner/read-events @shared-storage-ref)
- output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
+ (let [source-tables (if (seq tables) (tables-map->source-tables tables) [])]
+ (with-open [shared-storage-ref (s3/open-shared-storage! source-tables)]
+ (let [server-url (transforms-python.settings/python-runner-url)
+ cancel-chan (a/promise-chan)
+ test-id (next-job-run-id)
+ _ (python-runner/copy-tables-to-s3! {:run-id test-id
+ :shared-storage @shared-storage-ref
+ :source {:source-tables source-tables}
+ :cancel-chan cancel-chan})
+ response (python-runner/execute-python-code-http-call! {:server-url server-url
+ :code code
+ :run-id test-id
+ :source-tables source-tables
+ :shared-storage @shared-storage-ref})
+ events (python-runner/read-events @shared-storage-ref)
+ output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
;; not sure about munging this all together but its what tests expect for now
- (merge (:body response)
- {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
- :output-manifest output-manifest
- :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
- :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+ (merge (:body response)
+ {:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
+ :output-manifest output-manifest
+ :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+ :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))})))))
(defn ok-stdout [num-rows num-cols]
(format (str "Successfully saved %d rows to S3\n"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
index c159d9100dc..4bf49a76597 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
@@ -64,9 +64,9 @@
(deftest open-s3-shared-storage-test
(testing "open-s3-shared-storage! returns closeable derefable"
- (mt/with-premium-features #{:transforms-python :transforms-basic}
- (let [table-name->id {"users" 1}
- storage-ref (s3/open-shared-storage! table-name->id)
+ (mt/with-premium-features #{:transforms-python :transforms}
+ (let [source-tables [{:alias "users" :table_id 1 :database_id 1 :schema nil}]
+ storage-ref (s3/open-shared-storage! source-tables)
{:keys [s3-client bucket-name objects]} @storage-ref]
(testing "can be dereferenced"
(doseq [[k {:keys [method path url]}] objects
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
index 8d2be07fbba..fa62b6f6c38 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -149,7 +149,7 @@
(let [original {:name "Gadget Products"
:source {:type "python"
:source-database (mt/id)
- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]
+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -223,7 +223,7 @@
:source {:type "python"
:body (program->source program)
:source-database (mt/id)
- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]}
+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}
:target (assoc target :database (mt/id))})))
(block-on-run [{:keys [expect-status]} target transform-id]
@@ -329,7 +329,7 @@
:model/Transform transform {:name "Python Transform Cross DB"
:source {:type "python"
:source-database (mt/id)
- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]
+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]
:body "def transform():\n pass"}
:target {:type "table"
:schema "PUBLIC"
@@ -353,7 +353,7 @@
:source {:type "python"
:body (str/join "\n" program)
:source-database (mt/id)
- :source-tables [{:alias "transforms_customers" :table_id (mt/id :transforms_customers)}]}
+ :source-tables [(transforms.tu/source-table-entry "transforms_customers" (mt/id :transforms_customers))]}
:target (assoc target :database (mt/id))}))
;; using clojure-ey coordination with promises (I know j.u.c could be better here)
@@ -526,7 +526,7 @@
(let [initial-transform {:name "Schema Change Integration Test"
:source {:type "python"
:source-database (mt/id)
- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]
+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -545,7 +545,7 @@
(let [updated-transform (assoc initial-transform
:source {:type "python"
:source-database (mt/id)
- :source-tables [{:alias "test" :table_id (t2/select-one-pk :model/Table :db_id (mt/id))}]
+ :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
index 6389f26ef74..d8e2030fd23 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/dependencies_test.clj
@@ -7,6 +7,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
+ [metabase.transforms.test-util :as transforms.tu]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -95,8 +96,8 @@
(deftest ^:parallel analyze-entity-python-transform-test
(testing "analyze-entity extracts dependencies from a python transform"
(let [transform {:source {:type "python"
- :source-tables [{:alias "orders" :table_id (mt/id :orders)}
- {:alias "products" :table_id (mt/id :products)}]
+ :source-tables [(transforms.tu/source-table-entry "orders" (mt/id :orders))
+ (transforms.tu/source-table-entry "products" (mt/id :products))]
:body "def transform(orders, products): return orders"}
:target {:database (mt/id)
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
index b22f8e48dea..7a1f59a73f3 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
@@ -182,7 +182,7 @@
:id 456}}
source {:type "python"
:body "import pandas as pd"
- :source-tables [{:alias "orders" :table_id 123}]}]
+ :source-tables [{:alias "orders" :table_id 123 :database_id 1 :schema "public"}]}]
(is (= [{:alias "orders" :database_id 1
:schema "ws_isolated_123" :table "orders_isolated" :table_id 456}]
(:source-tables (remap-python-source table-mapping source))))))
@@ -221,7 +221,7 @@
:body "import pandas as pd"
:source-tables [{:alias "orders" :database_id 1
:schema "public" :table "orders" :table_id 123}
- {:alias "customers" :table_id 789}]}]
+ {:alias "customers" :table_id 789 :database_id 1 :schema "public"}]}]
(is (= [{:alias "orders" :database_id 1
:schema "ws_isolated_123" :table "orders_isolated" :table_id 456}
{:alias "customers" :database_id 1
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsx
index 809f6aa58e3..b72c7e53e5c 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/components/TransformSourceTablesDiff.tsx
@@ -15,17 +15,7 @@ export const TransformSourceTablesDiff = ({
newSourceTables,
oldSourceTables,
}: Props) => {
- const oldSourceTablesIds = Object.values(oldSourceTables);
- const newSourceTablesIds = Object.values(newSourceTables);
-
- const oldSourceNames = Object.fromEntries(
- Object.entries(oldSourceTables).map(([k, v]) => [v, k]),
- );
- const newSourceNames = Object.fromEntries(
- Object.entries(newSourceTables).map(([k, v]) => [v, k]),
- );
-
- const removedTablesIds = oldSourceTablesIds.slice(newSourceTablesIds.length);
+ const removedEntries = oldSourceTables.slice(newSourceTables.length);
return (
<Box
@@ -37,41 +27,42 @@ export const TransformSourceTablesDiff = ({
columnGap: 32,
}}
>
- {newSourceTablesIds.map((id, index) => {
- const oldTableId = oldSourceTablesIds[index];
- const oldSourceName = oldSourceNames[oldTableId];
- const newSourceName = newSourceNames[id];
- const sourceNameChanged = oldSourceName !== newSourceName;
+ {newSourceTables.map((entry, index) => {
+ const oldEntry = oldSourceTables[index];
+ const oldAlias = oldEntry?.alias;
+ const newAlias = entry.alias;
+ const sourceNameChanged = oldAlias !== newAlias;
return (
- <Fragment key={id}>
+ <Fragment key={entry.alias}>
<Group gap="xs">
- {sourceNameChanged && oldSourceName && (
+ {sourceNameChanged && oldAlias && (
<Text c="danger" component="s" td="line-through">
- {oldSourceName}
+ {oldAlias}
</Text>
)}
<Text c={sourceNameChanged ? "success" : undefined}>
- {newSourceName}
+ {newAlias}
</Text>
</Group>
- <TableItem oldTableId={oldTableId} tableId={id} />
+ <TableItem
+ oldTableId={oldEntry?.table_id}
+ tableId={entry.table_id}
+ />
</Fragment>
);
})}
- {removedTablesIds.map((id) => {
- const oldSourceName = oldSourceNames[id];
-
+ {removedEntries.map((entry) => {
return (
- <Fragment key={id}>
+ <Fragment key={entry.alias}>
<Text c="danger" component="s" td="line-through">
- {oldSourceName}
+ {entry.alias}
</Text>
- <TableItem oldTableId={id} tableId={undefined} />
+ <TableItem oldTableId={entry.table_id} tableId={undefined} />
</Fragment>
);
})}
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts
index ae8b6e10a2e..020affdbe2a 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts
@@ -3,7 +3,6 @@ import { useMemo } from "react";
import { useGetTransformQuery } from "metabase/api";
import { useSelector } from "metabase/lib/redux";
import { getMetadata } from "metabase/selectors/metadata";
-import { collapseSourceTableReferences } from "metabase/transforms/utils";
import { useGetWorkspaceTransformQuery } from "metabase-enterprise/api";
import type {
PythonTransformSource,
@@ -79,14 +78,12 @@ export function useTransformSources(
return {
oldSource,
oldSourceTables: isPythonTransformSource(globalTransform?.source)
- ? collapseSourceTableReferences(globalTransform.source["source-tables"])
+ ? globalTransform.source["source-tables"]
: undefined,
oldTarget: globalTransform?.target,
newSource,
newSourceTables: isPythonTransformSource(workspaceTransform?.source)
- ? collapseSourceTableReferences(
- workspaceTransform.source["source-tables"],
- )
+ ? workspaceTransform.source["source-tables"]
: undefined,
newTarget: workspaceTransform?.target,
hasError,
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.ts
index a0258aa2dba..f727e58fdc7 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/utils.ts
@@ -55,14 +55,12 @@ export function areSourceTablesEqual(
return a === b;
}
- const aEntries = Object.entries(a);
- const bEntries = Object.entries(b);
-
- if (aEntries.length !== bEntries.length) {
+ if (a.length !== b.length) {
return false;
}
- return aEntries.every(([key, value], index) => {
- return bEntries[index][0] === key && bEntries[index][1] === value;
- });
+ return a.every(
+ (entry, index) =>
+ b[index].alias === entry.alias && b[index].table_id === entry.table_id,
+ );
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
index 71f035beee5..97c8639df8f 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
@@ -1,12 +1,10 @@
-import { useEffect, useMemo, useState } from "react";
+import { useEffect, useState } from "react";
import { t } from "ttag";
import { skipToken, useListTablesQuery } from "metabase/api";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
-import { extractTableId } from "metabase/transforms/utils";
import { Box, Button, Icon, Stack, Text } from "metabase/ui";
import type {
- ConcreteTableId,
DatabaseId,
PythonTransformTableAliases,
Table,
@@ -66,7 +64,10 @@ export function PythonDataPicker({
const handleChange = (selections: TableSelection[]) => {
if (database) {
- const tableAliases = selectionsToTableAliases(selections);
+ const tableAliases = selectionsToTableAliases(
+ selections,
+ tablesData ?? [],
+ );
onChange(database, tableAliases, tablesData ?? []);
}
};
@@ -176,13 +177,6 @@ function SelectionInput({
}) {
const table = availableTables.find((table) => table.id === selection.tableId);
- const selectedTableIds = useMemo((): ConcreteTableId[] => {
- return Object.values(tables).flatMap((v) => {
- const id = extractTableId(v);
- return id != null ? [id] : [];
- });
- }, [tables]);
-
function handleAliasChange(newAlias: string) {
const newSelection = {
...selection,
@@ -227,7 +221,7 @@ function SelectionInput({
<TableSelector
database={database}
table={table}
- selectedTableIds={selectedTableIds}
+ selectedTableIds={tables.map((t) => t.table_id)}
onChange={handleTableChange}
onRemove={onRemove}
availableTables={availableTables}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
index 793ac59e6d2..360319ec3b9 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
@@ -1,15 +1,18 @@
import { slugify as toSlug } from "metabase/lib/formatting";
-import { extractTableId } from "metabase/transforms/utils";
-import type { PythonTransformTableAliases } from "metabase-types/api";
+import type {
+ PythonTransformTableAliases,
+ PythonTransformTableEntry,
+ Table,
+} from "metabase-types/api";
import type { TableSelection } from "./types";
export function getInitialTableSelections(
tables: PythonTransformTableAliases | undefined,
) {
- if (tables && Object.keys(tables).length > 0) {
- return Object.entries(tables).map(([alias, value]) => ({
- tableId: extractTableId(value),
+ if (tables && tables.length > 0) {
+ return tables.map(({ alias, table_id }) => ({
+ tableId: table_id,
alias,
}));
}
@@ -24,16 +27,23 @@ export function getInitialTableSelections(
export function selectionsToTableAliases(
selections: TableSelection[],
+ tableInfo: Table[],
): PythonTransformTableAliases {
- const tableAliases: PythonTransformTableAliases = {};
-
- for (const selection of selections) {
- const { alias, tableId } = selection;
- if (tableId !== undefined && alias !== "") {
- tableAliases[alias] = tableId;
- }
- }
- return tableAliases;
+ return selections
+ .filter((s) => s.tableId !== undefined && s.alias !== "")
+ .map((s) => {
+ const table = tableInfo.find((tbl) => tbl.id === s.tableId);
+ if (!table || !s.tableId) {
+ return null;
+ }
+ return {
+ alias: s.alias,
+ table_id: s.tableId,
+ schema: table.schema,
+ database_id: table.db_id,
+ } satisfies PythonTransformTableEntry;
+ })
+ .filter((t) => t !== null);
}
export function slugify(
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx
index c85c47e9ace..8729caae3e6 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx
@@ -57,7 +57,7 @@ export function PythonTransformEditor({
const newSource = {
...source,
"source-database": databaseId,
- "source-tables": {},
+ "source-tables": [],
};
onChangeSource(newSource);
};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsx
index 37783fac8c9..fa58fee6c57 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.unit.spec.tsx
@@ -41,7 +41,7 @@ const mockPythonSource: PythonTransformSourceDraft = {
type: "python",
body: "# Python script\nprint('Hello, world!')",
"source-database": DATABASE_ID,
- "source-tables": {},
+ "source-tables": [],
};
const mockPythonTransform = createMockTransform({
@@ -52,7 +52,7 @@ const mockPythonTransform = createMockTransform({
type: "python",
body: "def transform(): pass",
"source-database": DATABASE_ID,
- "source-tables": {},
+ "source-tables": [],
},
});
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsx
index e2474b04280..85f888883fe 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformTopBar/PythonTransformTopBar.unit.spec.tsx
@@ -40,7 +40,7 @@ const mockPythonTransform = createMockTransform({
type: "python",
body: "def transform(): pass",
"source-database": DATABASE_ID,
- "source-tables": {},
+ "source-tables": [],
},
});
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts
index 96446a33de1..4442187e65f 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts
@@ -9,7 +9,7 @@ export function updateTransformSignature(
tables: PythonTransformTableAliases,
tableInfo: Table[],
): string {
- const tableAliases = Object.keys(tables);
+ const tableAliases = tables.map((t) => t.alias);
const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
@@ -46,8 +46,8 @@ export function updateTransformSignature(
const newArgsSection = tableAliases
.map((alias) => {
const padding = " ".repeat(maxAliasLength - alias.length);
- const tableId = tables[alias];
- const tableName = getTableName(tableId);
+ const tableId = tables.find((t) => t.alias === alias)?.table_id;
+ const tableName = tableId != null ? getTableName(tableId) : undefined;
return ` ${alias}:${padding} DataFrame containing the data from the "${tableName}" table`;
})
.join("\n");
@@ -118,8 +118,8 @@ ${newSignature}
${tableAliases
.map((alias) => {
const padding = " ".repeat(maxAliasLength - alias.length);
- const tableId = tables[alias];
- const tableName = getTableName(tableId);
+ const tableId = tables.find((t) => t.alias === alias)?.table_id;
+ const tableName = tableId != null ? getTableName(tableId) : undefined;
if (alias === tableName) {
return ` ${alias}:${padding} DataFrame containing the data from the corresponding table`;
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.ts
index 59cacbdd91a..137e42561e5 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/utils.ts
@@ -20,7 +20,7 @@ export function getPythonSourceValidationResult(
};
}
- if (Object.keys(source["source-tables"]).length === 0) {
+ if (source["source-tables"].length === 0) {
return {
isValid: false,
errorMessage: t`Select at least one table to alias`,
diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.ts
index 69282f86dad..6fc4ae0730e 100644
--- a/frontend/src/metabase-types/api/mocks/transform.ts
+++ b/frontend/src/metabase-types/api/mocks/transform.ts
@@ -40,7 +40,7 @@ export function createMockTransformSource(): TransformSource {
export function createMockPythonTransformSource({
sourceDatabase = 1,
- sourceTables = {},
+ sourceTables = [],
body = "# Python script\nprint('Hello, world!')",
}: {
sourceDatabase?: DatabaseId;
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index 80df541c1f0..3c5ab4c2300 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -61,7 +61,14 @@ export type Transform = {
export type SuggestedTransform = Partial<Pick<Transform, "id">> &
Pick<Transform, "name" | "description" | "source" | "target">;
-export type PythonTransformTableAliases = Record<string, ConcreteTableId>;
+export type PythonTransformTableEntry = {
+ alias: string;
+ table_id: ConcreteTableId;
+ schema?: SchemaName | null;
+ database_id: DatabaseId;
+};
+
+export type PythonTransformTableAliases = PythonTransformTableEntry[];
export type TransformSourceCheckpointStrategy = {
type: "checkpoint";
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
index fcc150acd41..04879b040d7 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
@@ -300,7 +300,7 @@ function SourceStrategyFields({
function getIsPythonTransformWithMultipleTables(source: TransformSource) {
const isPythonTransform = source.type === "python";
const isMultiTablePythonTransform =
- isPythonTransform && Object.keys(source["source-tables"]).length > 1;
+ isPythonTransform && source["source-tables"].length > 1;
return isMultiTablePythonTransform;
}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
index 066f14cc607..828971cf4b5 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
@@ -34,8 +34,7 @@ export function PythonKeysetColumnSelect({
// Get the first (and should be only) table ID
// Incremental transforms are only supported for single-table Python transforms
const tableId = useMemo(() => {
- const tableIds = Object.values(sourceTables);
- return tableIds.length === 1 ? tableIds[0] : null;
+ return sourceTables.length === 1 ? sourceTables[0].table_id : null;
}, [sourceTables]);
// Fetch metadata for the table
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts
index 4a784950e54..3ef5d024a1f 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts
@@ -39,7 +39,9 @@ export const useHasCheckpointOptions = (source: TransformSource) => {
useNativeHasCheckpointFieldOptions(libQuery);
const pythonTableId = isPythonTransform
- ? Object.values(source["source-tables"]).find(isConcreteTableId)
+ ? Object.values(source["source-tables"]).find(({ table_id }) =>
+ isConcreteTableId(table_id),
+ )?.table_id
: null;
const { data: pythonTable } = useGetTableQueryMetadataQuery(
pythonTableId ? { id: pythonTableId } : skipToken,
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/utils.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/utils.ts
index 4b33783cb5b..f6813e202d0 100644
--- a/frontend/src/metabase/transforms/pages/NewTransformPage/utils.ts
+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/utils.ts
@@ -28,7 +28,7 @@ export function getInitialPythonSource(): PythonTransformSourceDraft {
return {
type: "python",
"source-database": undefined,
- "source-tables": {},
+ "source-tables": [],
body: `# Write your Python transformation script here
import common
import pandas as pd
diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsx
index c602054ebf0..0dc7bb935e3 100644
--- a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformPaneHeaderActions.unit.spec.tsx
@@ -26,7 +26,7 @@ const mockPythonSource: DraftTransformSource = {
type: "python",
body: "# Python script",
"source-database": 1,
- "source-tables": {},
+ "source-tables": [],
};
type SetupOpts = {
diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.ts
index 7ac1df46c8b..a89a88d86fb 100644
--- a/frontend/src/metabase/transforms/utils.ts
+++ b/frontend/src/metabase/transforms/utils.ts
@@ -8,11 +8,9 @@ import * as Lib from "metabase-lib";
import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type {
CollectionNamespace,
- ConcreteTableId,
Database,
DatabaseId,
DraftTransformSource,
- PythonTransformTableAliases,
TemplateTag,
Transform,
TransformRun,
@@ -135,78 +133,11 @@ export function isSameSource(
return Lib.areLegacyQueriesEqual(source1.query, source2.query);
}
if (source1.type === "python" && source2.type === "python") {
- return _.isEqual(
- {
- ...source1,
- "source-tables": collapseSourceTableReferences(
- source1["source-tables"],
- ),
- },
- {
- ...source2,
- "source-tables": collapseSourceTableReferences(
- source2["source-tables"],
- ),
- },
- );
+ return _.isEqual(source1, source2);
}
return false;
}
-/**
- * Extract a ConcreteTableId from a source-tables value, which may be either
- * a bare integer or a map with table_id (from backend normalization).
- */
-export function extractTableId(
- value: ConcreteTableId | Record<string, unknown>,
-): ConcreteTableId | undefined {
- if (typeof value === "number") {
- return value;
- }
- if (typeof value === "object" && value != null) {
- const id = (value as Record<string, unknown>)["table_id"];
- if (typeof id === "number") {
- return id as ConcreteTableId;
- }
- }
- return undefined;
-}
-
-const collapseCache = new WeakMap<
- Record<string, unknown>,
- PythonTransformTableAliases
->();
-
-/**
- * Collapse source-table references to plain { alias: tableId } form.
- * The backend may return map refs (with table_id, schema, etc.) instead of
- * bare integer IDs. Returns the original map unchanged if all values are
- * already plain integers. Results are memoized by object reference.
- */
-export function collapseSourceTableReferences(
- tables: Record<string, unknown>,
-): PythonTransformTableAliases {
- const cached = collapseCache.get(tables);
- if (cached) {
- return cached;
- }
- const values = Object.values(tables);
- if (values.every((v) => typeof v === "number")) {
- return tables as PythonTransformTableAliases;
- }
- const result: PythonTransformTableAliases = {};
- for (const [key, value] of Object.entries(tables)) {
- const id = extractTableId(
- value as ConcreteTableId | Record<string, unknown>,
- );
- if (id != null) {
- result[key] = id;
- }
- }
- collapseCache.set(tables, result);
- return result;
-}
-
export function isSourceEmpty(
source: DraftTransformSource,
databaseId: DatabaseId,
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index 801ad9b2c78..6f3de058de7 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -31,7 +31,6 @@
add-source-readable
is-temp-transform-table?]
[metabase.transforms.crud
- source-tables-vec->map-for-fe
check-database-feature
check-feature-enabled!
validate-incremental-column-type!
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index e612fb2d9a3..c4ffd1468f7 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -19,16 +19,6 @@
(set! *warn-on-reflection* true)
-;; TODO(FE-source-tables): Remove this function and all call sites when FE adopts the array format for source-tables.
-(defn source-tables-vec->map-for-fe
- "Convert source-tables from internal vec format to legacy map format for FE compatibility.
- Remove this when FE adopts the array format."
- [transform]
- (if (transforms-base.u/python-transform? transform)
- (update-in transform [:source :source-tables]
- transforms-base.u/source-tables-vec->alias-id-map)
- transform))
-
(defn check-database-feature
"Check that the target database supports the required features for this transform."
[transform]
@@ -87,8 +77,7 @@
(transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)
(transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)
(map #(update % :last_run transforms-base.u/localize-run-timestamps))
- (map transforms.u/add-source-readable)
- (map source-tables-vec->map-for-fe))))))) ;; TODO(FE-source-tables): remove
+ (map transforms.u/add-source-readable)))))))
(defn get-transform
"Get a specific transform."
@@ -99,8 +88,7 @@
(t2/hydrate :last_run :transform_tag_ids :creator :owner)
(u/update-some :last_run transforms-base.u/localize-run-timestamps)
(assoc :table target-table)
- transforms.u/add-source-readable
- source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove
+ transforms.u/add-source-readable)))
(defn create-transform!
"Create new transform in the appdb.
@@ -161,8 +149,7 @@
(t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids :creator :owner))]
(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
(-> transform
- transforms.u/add-source-readable
- source-tables-vec->map-for-fe))) ;; TODO(FE-source-tables): remove
+ transforms.u/add-source-readable)))
(defn delete-transform!
"Delete a transform and publish the delete event."
diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.clj
index 46679487d71..f38427e8e42 100644
--- a/src/metabase/transforms/schema.clj
+++ b/src/metabase/transforms/schema.clj
@@ -27,12 +27,7 @@
[:map
[:source-database {:optional true} :int]
;; NB: if source is checkpoint, only one table allowed
- ;; decode/normalize: convert FE map format to vec and enrich with DB metadata
- [:source-tables [:sequential {:decode/normalize (fn [st]
- (if (map? st)
- (transforms-base.u/source-tables-map->vec st)
- st))}
- ::transforms-base.u/source-table-entry]]
+ [:source-tables [:sequential ::transforms-base.u/source-table-entry]]
[:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :python]]
[:body :string]
[:source-incremental-strategy {:optional true} ::source-incremental-strategy]]]])
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index a62fa0cff3c..56a3da85774 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -478,11 +478,6 @@
;;; ------------------------------------------------- Source Table Schemas -------------------------------------------------
-(mu/defn source-tables-vec->alias-id-map :- [:map-of :string [:maybe :int]]
- "Convert source-table entries to `{alias -> table_id}` map."
- [entries :- [:sequential ::source-table-entry]]
- (into {} (map (juxt :alias :table_id)) entries))
-
;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------
(def ^:private ^:const batch-lookup-chunk-size
@@ -523,8 +518,8 @@
"A source table entry in the array format. Combines alias with table reference."
[:map
[:alias :string]
- [:database_id {:optional true} :int]
- [:schema {:optional true} [:maybe :string]]
+ [:database_id :int]
+ [:schema [:maybe :string]]
[:table {:optional true} :string]
[:table_id {:optional true} [:maybe :int]]])
@@ -534,7 +529,7 @@
For entries with only :database_id/:schema/:table, looks up :table_id.
Throws if an integer table ID references a non-existent table.
Map refs with non-existent tables get nil table_id (resolved later at execute time)."
- [source-tables :- [:sequential ::source-table-entry]]
+ [source-tables :- [:sequential [:map [:alias :string]]]]
(let [;; Entries that have table_id but lack table metadata need lookup
needs-metadata (filter (fn [e] (and (:table_id e) (not (:table e)))) source-tables)
int-id->metadata (when (seq needs-metadata)
@@ -565,24 +560,23 @@
:else entry))
source-tables)))
-(mu/defn resolve-source-tables :- [:map-of :string :int]
- "Resolve source-table entries to `{alias -> table_id}`. Throws if any table not found.
+(mu/defn resolve-source-tables :- [:sequential ::source-table-entry]
+ "Resolve source-table entries to entries with :table_id filled in. Throws if any table not found.
For execute time — all entries must resolve to valid table IDs."
[source-tables :- [:sequential ::source-table-entry]]
(let [needs-lookup (filter missing-table-id? source-tables)
lookup (or (batch-lookup-table-ids needs-lookup) {})
- resolved (u/for-map [entry source-tables]
- [(:alias entry) (or (:table_id entry) (lookup (source-table-ref->key entry)))])
- unresolved (for [entry source-tables
- :let [table-id (or (:table_id entry) (lookup (source-table-ref->key entry)))]
- :when (nil? table-id)]
- {:alias (:alias entry)
- :table (if-let [schema (:schema entry)]
- (str schema "." (:table entry))
- (:table entry))
- :ref entry})]
+ resolved (mapv (fn [entry]
+ (let [table-id (or (:table_id entry) (lookup (source-table-ref->key entry)))]
+ (assoc entry :table_id table-id)))
+ source-tables)
+ unresolved (filter #(nil? (:table_id %)) resolved)]
(when (seq unresolved)
- (throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved)))
+ (throw (ex-info (str "Tables not found: " (str/join ", " (map (fn [{:keys [schema table]}]
+ (if schema
+ (str schema "." table)
+ table))
+ unresolved)))
{:unresolved unresolved
:transform-message "Input table not found"})))
resolved))
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 159d7b636b4..30d711e09c7 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -179,8 +179,7 @@
403
(deferred-tru "A table with that name already exists."))
(-> (transforms.core/create-transform! body)
- transforms.u/add-source-readable
- transforms.core/source-tables-vec->map-for-fe)) ;; TODO(FE-source-tables): remove
+ transforms.u/add-source-readable))
(api.macros/defendpoint :get "/:id" :- TransformResponse
"Get a specific transform."
@@ -198,8 +197,7 @@
dep-ids (get global-ordering id)
dependencies (map id->transform dep-ids)]
(->> (t2/hydrate dependencies :creator :owner)
- transforms.u/add-source-readable
- (mapv transforms.core/source-tables-vec->map-for-fe)))) ;; TODO(FE-source-tables): remove
+ transforms.u/add-source-readable)))
(def ^:private MergeHistoryEntry
[:map
diff --git a/test/metabase/transforms/test_util.clj b/test/metabase/transforms/test_util.clj
index 75f773c0d0d..07402356171 100644
--- a/test/metabase/transforms/test_util.clj
+++ b/test/metabase/transforms/test_util.clj
@@ -15,6 +15,12 @@
(set! *warn-on-reflection* true)
+(defn source-table-entry
+ "Build a full source-table-entry from alias + table_id by looking up database_id and schema."
+ [alias table-id]
+ (let [{:keys [db_id schema]} (t2/select-one [:model/Table :db_id :schema] :id table-id)]
+ {:alias alias :table_id table-id :database_id db_id :schema schema}))
+
(defn drop-target!
"Drop transform target `target` and clean up its metadata.
`target` can be a string or a map. If `target` is a string, type :table is assumed.
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index 45ae3057eb2..13fbf0029e1 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -224,17 +224,19 @@
(is (= (:id t1) (:table_id (second result)))))))))
(deftest resolve-source-tables-test
- (testing "resolve-source-tables returns {alias -> table_id} map from vec input"
+ (testing "resolve-source-tables returns entries with :table_id filled in"
(mt/with-temp [:model/Database db {}
:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}
:model/Table t2 {:db_id (:id db) :name "table_two" :schema nil}]
(testing "resolves entry with table_id"
- (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}]]
- (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}]
+ result (transforms-base.u/resolve-source-tables source-tables)]
+ (is (= (:id t1) (:table_id (first result))))))
(testing "looks up table_id for entry without it"
- (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}]]
- (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))
+ (let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "table_one"}]
+ result (transforms-base.u/resolve-source-tables source-tables)]
+ (is (= (:id t1) (:table_id (first result))))))
(testing "throws for non-existent table"
(let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]]
@@ -247,10 +249,11 @@
(transforms-base.u/resolve-source-tables source-tables)))))
(testing "handles multiple entries"
- (let [source-tables [{:alias "t1" :table_id (:id t1)}
- {:alias "t2" :database_id (:id db) :schema nil :table "table_two"}]]
- (is (= {"t1" (:id t1) "t2" (:id t2)}
- (transforms-base.u/resolve-source-tables source-tables))))))))
+ (let [source-tables [{:alias "t1" :table_id (:id t1) :database_id (:id db) :schema nil}
+ {:alias "t2" :database_id (:id db) :schema nil :table "table_two"}]
+ result (transforms-base.u/resolve-source-tables source-tables)]
+ (is (= (:id t1) (:table_id (first result))))
+ (is (= (:id t2) (:table_id (second result)))))))))
(deftest source-tables-readable?-test
(testing "source-tables-readable? function"