The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Python Transforms (#63700)
Nicola Mometto7b6fbd6374a1
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 81ce3c877b4..cbbf8a98290 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -173,6 +173,7 @@metabase.driver.sql-jdbc.execute/execute-prepared-statement!metabase.notification.models/create-notification!metabase.pulse.send/send-pulse!+ metabase-enterprise.transforms-python.python-runner-test/execute!;; Safe in tests, where we typically use a private, temporary index per thread.metabase.search.core/reindex!metabase.util/run-count!@@ -418,6 +419,7 @@metabase-enterprise.transforms.api-test/with-transform-tags clojure.core/letmetabase-enterprise.transforms.api-test/with-worker-runs clojure.core/letmetabase-enterprise.transforms.test-util/with-transform-cleanup! clojure.core/let+ metabase-enterprise.transforms.execute/with-transform-lifecycle clojure.core/letmetabase.actions.test-util/with-actions clojure.core/letmetabase.api.common/let-404 clojure.core/letmetabase.app-db.custom-migrations.util/with-temp-scheduler! clojure.core/fn@@ -519,6 +521,7 @@metabase-enterprise.cache.config-test/with-temp-persist-models hooks.common/with-seven-bindingsmetabase-enterprise.semantic-search.db.connection/with-migrate-tx hooks.common/let-one-with-optional-valuemetabase-enterprise.serialization.test-util/with-temp-dpc hooks.toucan2.tools.with-temp/with-temp+ metabase-enterprise.transforms-python.drivers-test/with-test-table hooks.common/with-two-bindingsmetabase.api.common.internal-test/with-jetty-server hooks.common/with-one-bindingmetabase.api.macros/defendpoint hooks.metabase.api.macros/defendpointmetabase.app-db.schema-migrations-test.impl/test-migrations hooks.metabase.app-db.schema-migrations-test.impl/test-migrationsdiff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 5d92ee979bf..c98b29ea88d 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -894,6 +894,19 @@enterprise/advanced-permissionsenterprise/sandbox}}+ python-runner+ {:team "Workflows"+ :api #{metabase.python-runner.api+ metabase.python-runner.core+ metabase.python-runner.init}+ :uses #{api+ driver+ models+ query-processor+ request+ util+ enterprise/transforms}}+queries{:team "Querying":api #{metabase.queries.api@@ -1984,6 +1997,7 @@:api #{metabase-enterprise.transforms.apimetabase-enterprise.transforms.coremetabase-enterprise.transforms.init+ metabase-enterprise.transforms.instrumentationmetabase-enterprise.transforms.util}:uses #{analyticsapi@@ -2009,6 +2023,7 @@enterprise/transforms-python{:team "Querying":api #{metabase-enterprise.transforms-python.api+ metabase-enterprise.transforms-python.executemetabase-enterprise.transforms-python.initmetabase-enterprise.transforms-python.settings}:uses #{apidiff --git a/.github/scripts/setup-python-runner.sh b/.github/scripts/setup-python-runner.shnew file mode 100755index 00000000000..518b3cde0fd--- /dev/null+++ b/.github/scripts/setup-python-runner.sh@@ -0,0 +1,29 @@+#!/bin/bash+set -e++# Parse command line arguments+SKIP_START=false++while [[ $# -gt 0 ]]; do+ case $1 in+ --skip-start)+ SKIP_START=true+ shift+ ;;+ *)+ echo "Unknown option: $1"+ echo "Usage: $0 [--skip-start]"+ exit 1+ ;;+ esac+done++echo "Cloning python-runner repository..."+git clone --depth 1 https://${METABASE_AUTOMATION_USER_TOKEN}@github.com/metabase/python-runner-container.git python-runner++if [ "$SKIP_START" = false ]; then+ echo "Starting python-runner..."+ cd python-runner+ make run-ci+ cd ..+fidiff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.ymlindex a2587bbbde1..55853b0f845 100644--- a/.github/workflows/app-db.yml+++ b/.github/workflows/app-db.yml@@ -35,21 +35,21 @@ jobs:build-static-viz: falsetest-args: >-:only '"enterprise/backend/test"'- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'- name: EE App DB Tests (Part 1)build-static-viz: truetest-args: >-:only '"test"':partition/total 2:partition/index 0- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'- name: EE App DB Tests (Part 2)build-static-viz: truetest-args: >-:only '"test"':partition/total 2:partition/index 1- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'services:mysql:@@ -147,21 +147,21 @@ jobs:build-static-viz: falsetest-args: >-:only '"enterprise/backend/test"'- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'- name: EE App DB Tests (Part 1)build-static-viz: truetest-args: >-:only '"test"':partition/total 2:partition/index 0- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'- name: EE App DB Tests (Part 2)build-static-viz: truetest-args: >-:only '"test"':partition/total 2:partition/index 1- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'services:mysql:@@ -259,11 +259,13 @@ jobs:build-static-viz: falsetest-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ exclude-tag: ':mb/transforms-python-test'- name: Enterprise Testsbuild-static-viz: falsetest-args: >-:only '"enterprise/backend/test"'- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'- name: EE App DB Tests (Part 1)build-static-viz: truetest-args: >-@@ -271,14 +273,14 @@ jobs::exclude-tags [:mb/driver-tests]:partition/total 2:partition/index 0- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'- name: EE App DB Tests (Part 2)build-static-viz: truetest-args: >-:only '"test"':partition/total 2:partition/index 1- exclude-tag: ':mb/driver-tests'+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'name: "${{ matrix.version.name }} ${{ matrix.job.name }}"env:# Only run old migrations tests on pushes to master or release branches. All other branches should skip testsdiff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex a1f8e7d9d97..c35e0121fb9 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -90,6 +90,7 @@ jobs:junit-name: 'be-tests-athena-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -106,6 +107,18 @@ jobs:if: ${{ !inputs.skip }}runs-on: ubuntu-22.04timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ job:+ - name: Driver Tests+ test-args: >-+ :only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ - name: Transforms Python Tests+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ needs-python-runner: trueenv:CI: 'true'DRIVERS: bigquery-cloud-sdk@@ -116,15 +129,19 @@ jobs:MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}- name: BigQuery+ name: BigQuery ${{ matrix.job.name }}steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test BigQuery Cloud SDK driveruses: ./.github/actions/test-driverwith:junit-name: 'be-tests-bigquery-cloud-sdk-ee'- test-args: >-- :only-tags [:mb/driver-tests]+ test-args: ${{ matrix.job.test-args }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -141,6 +158,18 @@ jobs:if: ${{ !inputs.skip }}runs-on: ubuntu-22.04timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ job:+ - name: Driver Tests+ test-args: >-+ :only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ - name: Transforms Python Tests+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ needs-python-runner: trueenv:CI: 'true'DRIVERS: clickhouse@@ -148,9 +177,14 @@ jobs:MB_CLICKHOUSE_TEST_PORT: 8123MB_CLICKHOUSE_TEST_OLD_PORT: 8124MB_CLICKHOUSE_TEST_NGINX_PORT: 8127- name: Clickhouse+ name: Clickhouse ${{ matrix.job.name }}steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Add ClickHouse TLS instance to /etc/hostsrun: |sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts@@ -170,8 +204,7 @@ jobs:uses: ./.github/actions/test-driverwith:junit-name: 'be-tests-clickhouse-ee'- test-args: >-- :only-tags [:mb/driver-tests]+ test-args: ${{ matrix.job.test-args }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -208,6 +241,7 @@ jobs:junit-name: 'be-tests-druid-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -244,6 +278,7 @@ jobs:junit-name: 'be-tests-druid-jdbc-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -280,6 +315,7 @@ jobs:junit-name: 'be-tests-databricks-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -317,6 +353,7 @@ jobs:junit-name: 'be-tests-databricks-multi-catalog-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -338,12 +375,17 @@ jobs:name: H2steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test H2 driveruses: ./.github/actions/test-driverwith:junit-name: 'be-tests-h2'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -388,6 +430,11 @@ jobs:- name: Driver Teststest-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ - name: Transforms Python Tests+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ needs-python-runner: trueservices:mysql:@@ -442,6 +489,11 @@ jobs:name: "${{ matrix.version.name }} ${{ matrix.job.name }}"steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ #if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test ${{ matrix.version.name }}uses: ./.github/actions/test-driverwith:@@ -476,6 +528,15 @@ jobs:- name: MongoDB Latestjunit-name: be-tests-mongo-latest-eeimage: mongo:latest+ job:+ - name: Driver Tests+ test-args: >-+ :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'+ :only-tags [:mb/driver-tests]+ - name: Transforms Python Tests+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ needs-python-runner: trueenv:CI: 'true'DRIVERS: mongo@@ -489,16 +550,19 @@ jobs:env:MONGO_INITDB_ROOT_USERNAME: metabaseMONGO_INITDB_ROOT_PASSWORD: metasample123- name: ${{ matrix.version.name }}+ name: ${{ matrix.version.name }} ${{ matrix.job.name }}steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test ${{ matrix.version.name }}uses: ./.github/actions/test-driverwith:junit-name: ${{ matrix.version.junit-name }}- test-args: >-- :exclude-tags [:mongo-sharded-cluster-tests]- :only-tags [:mb/driver-tests]+ test-args: ${{ matrix.job.test-args }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -562,7 +626,7 @@ jobs:with:junit-name: ${{ matrix.version.junit-name }}test-args: >-- :exclude-tags [:mongo-sharded-cluster-tests]+ :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]':only-tags [:mb/driver-tests]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results@@ -670,6 +734,7 @@ jobs:junit-name: ${{ matrix.version.junit-name }}test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -704,6 +769,11 @@ jobs:- name: Driver Teststest-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ - name: Transforms Python Tests+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ needs-python-runner: truename: "${{ matrix.version.name }} ${{ matrix.job.name }}"env:# Only run old migrations tests on pushes to master or release branches. All other branches should skip tests@@ -745,6 +815,11 @@ jobs:POSTGRES_HOST_AUTH_METHOD: truststeps:- uses: actions/checkout@v4+ - name: Setup python-runner+ #if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})uses: ./.github/actions/test-driverwith:@@ -821,6 +896,7 @@ jobs:junit-name: 'be-tests-presto-jdbc-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -850,8 +926,14 @@ jobs:skip: falsetest-args: >-:only-tags [:mb/driver-tests]- :exclude-tags [:mb/upload-tests]+ :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]':fail-fast? true+ - name: Redshift Transforms Python Tests+ skip: false+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ :fail-fast? true+ needs-python-runner: trueenv:CI: 'true'@@ -864,6 +946,11 @@ jobs:name: ${{ matrix.job.name }}steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test ${{ matrix.job.name }}if: ${{ !matrix.job.skip }}uses: ./.github/actions/test-driver@@ -887,6 +974,18 @@ jobs:if: ${{ !inputs.skip }}runs-on: ubuntu-22.04timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ job:+ - name: Driver Tests+ test-args: >-+ :only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ - name: Transforms Python Tests+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ needs-python-runner: trueenv:CI: 'true'DRIVERS: snowflake@@ -905,15 +1004,19 @@ jobs:MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USERMB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLEMB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB- name: Snowflake+ name: Snowflake ${{ matrix.job.name }}steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test Snowflake driveruses: ./.github/actions/test-driverwith:junit-name: 'be-tests-snowflake-ee'- test-args: >-- :only-tags [:mb/driver-tests]+ test-args: ${{ matrix.job.test-args }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -948,6 +1051,7 @@ jobs:junit-name: 'be-tests-sparksql-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -977,6 +1081,7 @@ jobs:junit-name: 'be-tests-sqlite-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -1004,6 +1109,15 @@ jobs:- name: SQL Server 2022junit-name: be-tests-sqlserver-2022-eeimage: mcr.microsoft.com/mssql/server:2022-latest+ job:+ - name: Driver Tests+ test-args: >-+ :only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ - name: Transforms Python Tests+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ needs-python-runner: trueenv:CI: 'true'DRIVERS: sqlserver@@ -1019,15 +1133,19 @@ jobs:ACCEPT_EULA: YSA_PASSWORD: 'P@ssw0rd'MSSQL_MEMORY_LIMIT_MB: 1024- name: ${{ matrix.version.name }}+ name: ${{ matrix.version.name }} ${{ matrix.job.name }}steps:- uses: actions/checkout@v4+ - name: Setup python-runner+ if: ${{ matrix.job.needs-python-runner == true }}+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Test ${{ matrix.version.name }}uses: ./.github/actions/test-driverwith:junit-name: ${{ matrix.version.junit-name }}- test-args: >-- :only-tags [:mb/driver-tests]+ test-args: ${{ matrix.job.test-args }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()@@ -1136,6 +1254,7 @@ jobs:junit-name: 'be-tests-vertica-ee'test-args: >-:only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always()diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex 8cda8b9f24f..196788c485d 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -60,6 +60,14 @@ jobs:CYPRESS_IS_EMBEDDING_SDK: "${{ inputs.name == 'embedding-sdk' }}"steps:- uses: actions/checkout@v4+ with:+ submodules: true+ token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}++ - name: Setup python-runner+ env:+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: ./.github/scripts/setup-python-runner.sh- name: Prepare Docker containersuses: ./.github/actions/e2e-prepare-containersdiff --git a/deps.edn b/deps.ednindex 713c03773e5..b82eabaf744 100644--- a/deps.edn+++ b/deps.edn@@ -198,6 +198,8 @@org.eclipse.jetty.ee9/jetty-ee9-servletorg.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}slingshot/slingshot {:mvn/version "0.12.2"} ; enhanced throw/catch, used by other deps+ software.amazon.awssdk/s3 {:mvn/version "2.33.0"} ; AWS SDK v2 for S3+ software.amazon.awssdk/sts {:mvn/version "2.33.0"} ; Required for S3 client to use service accountstencil/stencil {:mvn/version "0.5.0"} ; Mustache templates for Clojureuser-agent/user-agent {:mvn/version "0.1.1"} ; User-Agent string parser, for Login History page & elsewhereweavejester/dependency {:mvn/version "1.0.0"} ; Dependency graphs and topological sortingdiff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex a91cecc05b3..36bd741b38a 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -20,7 +20,22 @@ title: Driver interface changelog- Added `metabase.driver/rename-tables!*` multimethod for atomic table renaming operations. Takes a map of {from-table to-table}pairs that has been topologically sorted.-- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.+- Added `metabase.driver/rename-table!` multimethod for table renaming operations. Takes a single from-table to-table pair.+ Fallbacks to singleton call to `rename-tables!*` if available.++- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!`+ which should be implemented by drivers that support `:schemas` and `:transforms/table`.++- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase+ type (from the type hierarchy) as a HoneySQL spec. This method handles general Metabase base types.++- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources+ into existing tables. This multimethod dispatches on both the driver and the data source type+ (`:rows` or `:jsonl-file`). It allows drivers to optimize based on the data source type and returns the number+ of rows inserted. Default implementations are provided for both `:rows` and `:jsonl-file` source types.++- Added `metabase.driver/insert-col->val` multimethod to parse values for insertion based on driver, data source type+ and column definition. Drivers should implement this when their insertion mechanism needs values converted to proper types.- `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,instead of just the top-level message as a string.diff --git a/e2e/support/helpers/api/index.ts b/e2e/support/helpers/api/index.tsindex ee10d63d045..0624a12c28d 100644--- a/e2e/support/helpers/api/index.ts+++ b/e2e/support/helpers/api/index.ts@@ -38,4 +38,4 @@ export { getCurrentUser } from "./getCurrentUser";export { remapDisplayValueToFK } from "./remapDisplayValueToFK";export { updateComment } from "./updateComment";export { updateDashboardCards } from "./updateDashboardCards";-export { updateSetting } from "./updateSetting";+export { updateSetting, updateEnterpriseSetting } from "./updateSetting";diff --git a/e2e/support/helpers/api/updateSetting.ts b/e2e/support/helpers/api/updateSetting.tsindex e20773441bb..d7b8bb23851 100644--- a/e2e/support/helpers/api/updateSetting.ts+++ b/e2e/support/helpers/api/updateSetting.ts@@ -1,4 +1,4 @@-import type { Settings } from "metabase-types/api";+import type { EnterpriseSettings, Settings } from "metabase-types/api";export const updateSetting = <TKey extends keyof Settings,@@ -9,3 +9,13 @@ export const updateSetting = <): Cypress.Chainable<Cypress.Response<never>> => {return cy.request<never>("PUT", `/api/setting/${setting}`, { value });};++export const updateEnterpriseSetting = <+ TKey extends keyof EnterpriseSettings,+ TValue extends EnterpriseSettings[TKey],+>(+ setting: TKey,+ value: TValue,+): Cypress.Chainable<Cypress.Response<never>> => {+ return cy.request<never>("PUT", `/api/setting/${setting}`, { value });+};diff --git a/e2e/support/helpers/e2e-codemirror-helpers.ts b/e2e/support/helpers/e2e-codemirror-helpers.tsindex 088a6928a83..6da51822fcc 100644--- a/e2e/support/helpers/e2e-codemirror-helpers.ts+++ b/e2e/support/helpers/e2e-codemirror-helpers.ts@@ -1,6 +1,208 @@+type TypeOptions = {+ delay?: number;+ focus?: boolean;+ allowFastSet?: boolean;+};++export function codeMirrorHelpers<T extends object>(testId: string, extra: T) {+ const helpers = {+ get() {+ cy.findAllByTestId("loading-indicator").should("not.exist");+ return cy.get(`[data-testid=${testId}] .cm-content`);+ },+ focus() {+ helpers.get().should("be.visible").click("right");+ helpers.get().get(".cm-editor").should("have.class", "cm-focused");+ return helpers;+ },+ blur() {+ helpers.get().get(".cm-content").blur();+ return helpers;+ },+ selectAll() {+ const isMac = Cypress.platform === "darwin";+ const metaKey = isMac ? "Meta" : "Control";+ helpers.focus();+ cy.realPress([metaKey, "A"]);+ return helpers;+ },+ clear() {+ helpers.selectAll();+ cy.realPress(["Backspace"]);+ return helpers;+ },+ type(+ text: string,+ { focus = true, delay = 10, allowFastSet = false }: TypeOptions = {},+ ) {+ if (focus) {+ helpers.focus();+ }++ if (allowFastSet) {+ // Enter the formula in one go+ // HACK: we do invoke("text") instead of type() because type() does not work on+ // CodeMirror elements in Cypress. realType() would work but some of the formulas+ // contain special characters that are not supported by realType().+ helpers.get().invoke("text", text);++ // invoke("text") does not trigger the validator, so we need to trigger it manually+ // by typing something+ helpers.type(" {backspace}");++ return helpers;+ }++ const isMac = Cypress.platform === "darwin";+ const metaKey = isMac ? "Meta" : "Control";++ const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);++ function insert(text: string) {+ cy.realType(text, { pressDelay: delay });+ }++ // HACK: realType does not accept a lot of common escape sequences,+ // so we implement them manually here for the native editor.+ parts.forEach((part) => {+ switch (part.toLowerCase()) {+ case "":+ return;++ case "{clear}":+ return helpers.clear();++ case "{selectall}":+ return helpers.selectAll();++ case "{leftarrow}":+ return cy.realPress(["ArrowLeft"]);++ case "{rightarrow}":+ return cy.realPress(["ArrowRight"]);++ case "{downarrow}":+ return cy.realPress(["ArrowDown"]);++ case "{uparrow}":+ return cy.realPress(["ArrowUp"]);++ case "{enter}":+ return cy.realPress(["Enter"]);++ case "{home}":+ case "{movetostart}":+ return cy.realPress(["Home"]);++ case "{end}":+ case "{movetoend}":+ return cy.realPress(["End"]);++ case "{backspace}":+ return cy.realPress(["Backspace"]);++ case "{tab}":+ return cy.realPress(["Tab"]);++ case "{nextcompletion}":+ cy.wait(50);+ return cy.realPress([metaKey, "j"]);++ case "{prevcompletion}":+ cy.wait(50);+ return cy.realPress([metaKey, "k"]);++ case "{{}":+ return cy.realType("{");+ }++ if (part.startsWith("{") && part.endsWith("}")) {+ // unknown escape sequence, let's try typing it+ insert("{");+ insert(part.slice(1));+ return;+ }++ // realType does not support → arrow, so we replace it with ->, which the editor+ // expands into →+ const unexpanded = part.replaceAll(/→/g, "->");+ const alphabet =+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789^[]()-,.;_!@#$%&*+=/<>\" ':;\\\n{}";+ if (unexpanded.split("").some((char) => !alphabet.includes(char))) {+ throw new Error(+ `unknown character in codeMirrorHelpers.type in ${part}`,+ );+ }++ insert(unexpanded);+ });++ return helpers;+ },+ textbox() {+ return helpers.get().get("[role='textbox']");+ },+ value() {+ // Get the multiline text content of the editor+ return helpers+ .textbox()+ .get(".cm-line")+ .then((lines) => {+ const text: string[] = [];+ lines.each((_, line) => {+ const placeholder = line.querySelector(".cm-placeholder");+ if (placeholder) {+ return;+ }+ text.push(line.textContent ?? "");+ });+ return text.join("\n");+ });+ },+ completions() {+ return cy.get(".cm-tooltip-autocomplete").should("be.visible");+ },+ completion(label: string) {+ return cy.get(".cm-completionLabel").contains(label).parent();+ },+ acceptCompletion(key: "enter" | "tab" = "enter") {+ helpers.completions().should("be.visible");++ // Avoid flakiness with CodeMirror not accepting the suggestion immediately+ cy.wait(300);+ helpers.type(`{${key}}`, { focus: false });+ },+ selectCompletion(name: string) {+ helpers.completions().should("be.visible");++ // Avoid flakiness with CodeMirror not accepting the suggestion immediately+ cy.wait(300);+ helpers.completion(name).click();+ },+ paste(content: string) {+ helpers.textbox().then((el) => {+ const clipboardData = new DataTransfer();+ clipboardData.setData("text/plain", content);++ const pasteEvent = new ClipboardEvent("paste", {+ bubbles: true,+ cancelable: true,+ clipboardData,+ });++ el[0].dispatchEvent(pasteEvent);+ });+ },+ ...extra,+ };++ return helpers;+}+export function codeMirrorEditor() {return cy.get(".cm-content");}+export function codeMirrorValue() {// Get the multiline text content of the editorreturn codeMirrorEditor()diff --git a/e2e/support/helpers/e2e-custom-column-helpers.ts b/e2e/support/helpers/e2e-custom-column-helpers.tsindex 66431f6ce19..92f23114ccf 100644--- a/e2e/support/helpers/e2e-custom-column-helpers.ts+++ b/e2e/support/helpers/e2e-custom-column-helpers.ts@@ -1,5 +1,7 @@import { popover } from "e2e/support/helpers/e2e-ui-elements-helpers";+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";+export function expressionEditorWidget() {return cy.findByTestId("expression-editor");}@@ -8,9 +10,6 @@ export function expressionEditorTextfield() {return CustomExpressionEditor.get();}-const isMac = Cypress.platform === "darwin";-const metaKey = isMac ? "Meta" : "Control";-export function enterCustomColumnDetails({formula,name,@@ -71,224 +70,43 @@ export function enterCustomColumnDetails({}}-export const CustomExpressionEditor = {- get() {- return cy.findByTestId("custom-expression-query-editor");- },- /**- * @param {string} text the formula to type- * @param {Object} options- * @param {boolean} [options.focus]- * true by default. Set to false to stop the helper from explicitly focussing the formula input.- * @param {boolean} [options.allowFastSet]- * true by default. However, if you need to examine the popover in the test, it should be set to false so the popover is not dismissed- * Because CodeMirror uses a contenteditable div, and it is not possible to use cy.type() on it, we emulate .type with realPress.- * This does not always work, since realPress() does not support all characters. Setting this to true will enable an escape hatch- * that uses cy.invoke('text') under the hood, to allow for formulas that contain unsupported characters.- * This has some other sideeffects however, so use it sparingly.- */- type(- text: string,- {- allowFastSet = false,- focus = true,- delay = 0,- }: {- focus?: boolean;- allowFastSet?: boolean;- delay?: number;- } = {},- ) {- if (focus) {- CustomExpressionEditor.focus();- }-- if (allowFastSet) {- // Enter the formula in one go- // HACK: we do invoke("text") instead of type() because type() does not work on- // CodeMirror elements in Cypress. realType() would work but some of the formulas- // contain special characters that are not supported by realType().- CustomExpressionEditor.get().findByRole("textbox").invoke("text", text);-- // invoke("text") does not trigger the validator, so we need to trigger it manually- // by typing something- CustomExpressionEditor.type(" {backspace}");-+export const CustomExpressionEditor = codeMirrorHelpers(+ "custom-expression-query-editor",+ {+ formatButton() {+ return cy.findByLabelText("Auto-format");+ },+ format() {+ CustomExpressionEditor.formatButton().should("be.visible");+ CustomExpressionEditor.formatButton().click();return CustomExpressionEditor;- }-- const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);-- parts.forEach((part) => {- cy.wait(delay);- switch (part.toLowerCase()) {- case "":- return;-- case "{clear}":- return CustomExpressionEditor.clear();-- case "{selectall}":- return CustomExpressionEditor.selectAll();-- case "{leftarrow}":- return cy.realPress(["ArrowLeft"]);-- case "{rightarrow}":- return cy.realPress(["ArrowRight"]);-- case "{downarrow}":- return cy.realPress(["ArrowDown"]);-- case "{uparrow}":- return cy.realPress(["ArrowUp"]);-- case "{enter}":- return cy.realPress(["Enter"]);-- case "{home}":- case "{movetostart}":- return cy.realPress(["Home"]);-- case "{end}":- case "{movetoend}":- return cy.realPress(["End"]);-- case "{backspace}":- return cy.realPress(["Backspace"]);-- case "{tab}":- return cy.realPress(["Tab"]);-- case "{nextcompletion}":- cy.wait(50);- return cy.realPress([metaKey, "j"]);-- case "{prevcompletion}":- cy.wait(50);- return cy.realPress([metaKey, "k"]);-- case "{{}":- return cy.realType("{");- }-- if (part.startsWith("{") && part.endsWith("}")) {- // unknown escape sequence, let's try typing it- throw new Error(- `unknown escape sequence in CustomExpressionEditor.type: ${part}`,- );- }-- // realType does not support → arrow, so we replace it with ->, which the editor- // expands into →- const unexpanded = part.replaceAll(/→/g, "->");-- const alphabet =- "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789^[]()-,.;_!@#$%&*+=/<>\" ':;\\\n";- if (unexpanded.split("").some((char) => !alphabet.includes(char))) {- throw new Error(- `unknown character in CustomExpressionEditor.type in ${part}`,- );- }-- cy.realType(unexpanded, { delay });- });- return CustomExpressionEditor;- },- focus() {- CustomExpressionEditor.get().click("right");- return CustomExpressionEditor;- },- blur() {- // click outside the expression editor- cy.findByTestId("expression-editor").click("bottomRight", { force: true });- return CustomExpressionEditor;- },- formatButton() {- return cy.findByLabelText("Auto-format");- },- format() {- CustomExpressionEditor.formatButton().should("be.visible");- CustomExpressionEditor.formatButton().click();- return CustomExpressionEditor;- },- selectAll() {- CustomExpressionEditor.focus();- cy.realPress([metaKey, "A"]);- return CustomExpressionEditor;- },- clear() {- CustomExpressionEditor.selectAll();- cy.realPress(["Backspace"]);- return CustomExpressionEditor;- },- textbox() {- return CustomExpressionEditor.get().get("[role='textbox']");- },- value() {- // Get the multiline text content of the editor- return CustomExpressionEditor.textbox()- .get(".cm-line")- .then((lines) => {- const text: string[] = [];- lines.each((_, line) => {- text.push(line.textContent ?? "");- });- const value = text.join("\n");- const placeholder = "Type your expression, press '[' for columns…";- if (value === placeholder) {- return "";- }- return value;+ },+ helpTextHeader() {+ return cy.findByTestId("expression-helper-popover-structure");+ },+ helpText() {+ return cy.findByTestId("expression-helper");+ },+ nameInput() {+ return cy.findByTestId("expression-name");+ },+ functionBrowser() {+ return cy.findByTestId("expression-editor-function-browser");+ },+ completions() {+ return cy.findByTestId("custom-expression-editor-suggestions");+ },+ completion(name: string) {+ return CustomExpressionEditor.completions()+ .findAllByRole("option")+ .contains(name)+ .first();+ },+ blur() {+ cy.findByTestId("expression-editor").click("bottomRight", {+ force: true,});+ return CustomExpressionEditor;+ },},- completions() {- return cy.findByTestId("custom-expression-editor-suggestions");- },- completion(name: string) {- return CustomExpressionEditor.completions()- .findAllByRole("option")- .contains(name)- .first();- },- acceptCompletion(key: "enter" | "tab" = "enter") {- CustomExpressionEditor.completions().should("be.visible");-- // Avoid flakiness with CodeMirror not accepting the suggestion immediately- cy.wait(300);- CustomExpressionEditor.type(`{${key}}`, { focus: false });- },- selectCompletion(name: string) {- CustomExpressionEditor.completions().should("be.visible");-- // Avoid flakiness with CodeMirror not accepting the suggestion immediately- cy.wait(300);- CustomExpressionEditor.completion(name).click();- },- helpTextHeader() {- return cy.findByTestId("expression-helper-popover-structure");- },- helpText() {- return cy.findByTestId("expression-helper");- },- paste(content: string) {- CustomExpressionEditor.textbox().then((el) => {- const clipboardData = new DataTransfer();- clipboardData.setData("text/plain", content);-- const pasteEvent = new ClipboardEvent("paste", {- bubbles: true,- cancelable: true,- clipboardData,- });-- el[0].dispatchEvent(pasteEvent);- });- },- nameInput() {- return cy.findByTestId("expression-name");- },- functionBrowser() {- return cy.findByTestId("expression-editor-function-browser");- },-};+);diff --git a/e2e/support/helpers/e2e-native-editor-helpers.ts b/e2e/support/helpers/e2e-native-editor-helpers.tsindex d55a4cc6f3a..33b4458f994 100644--- a/e2e/support/helpers/e2e-native-editor-helpers.ts+++ b/e2e/support/helpers/e2e-native-editor-helpers.ts@@ -1,42 +1,6 @@+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";import { popover } from "./e2e-ui-elements-helpers";-function nativeEditor() {- cy.findAllByTestId("loading-indicator").should("not.exist");- return cy.get("[data-testid=native-query-editor] .cm-content");-}--function focusNativeEditor() {- nativeEditor().should("be.visible").click();-- nativeEditor().get(".cm-editor").should("have.class", "cm-focused");-- return nativeEditor();-}--function blurNativeEditor() {- nativeEditor().get(".cm-editor").blur();-}--function nativeEditorCompletions() {- return cy.get(".cm-tooltip-autocomplete").should("be.visible");-}--function nativeEditorCompletion(label: string) {- return cy.get(".cm-completionLabel").contains(label).parent();-}--function nativeEditorSelectAll() {- const isMac = Cypress.platform === "darwin";- const metaKey = isMac ? "Meta" : "Control";- focusNativeEditor().realPress([metaKey, "A"]);- cy.get(".cm-selectionBackground").should("exist");-}--function clearNativeEditor() {- nativeEditorSelectAll();- cy.realPress(["Backspace"]);-}-export function nativeEditorDataSource() {return cy.findAllByTestId("gui-builder-data").first();}@@ -46,139 +10,7 @@ export function selectNativeEditorDataSource(name: string) {popover().findByText(name).click();}-type TypeOptions = {- delay?: number;- focus?: boolean;-};--function nativeEditorType(- text: string,- { delay = 10, focus = true }: TypeOptions = {},-) {- if (focus) {- focusNativeEditor();- }-- const isMac = Cypress.platform === "darwin";- const metaKey = isMac ? "Meta" : "Control";-- const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);-- function type(text: string) {- cy.realType(text, {- pressDelay: delay,- });- }-- // HACK: realType does not accept a lot of common escape sequences,- // so we implement them manually here for the native editor.- parts.forEach((part) => {- switch (part.toLowerCase()) {- case "":- return;-- case "{clear}":- return clearNativeEditor();-- case "{selectall}":- return nativeEditorSelectAll();-- case "{leftarrow}":- return cy.realPress(["ArrowLeft"]);-- case "{rightarrow}":- return cy.realPress(["ArrowRight"]);-- case "{downarrow}":- return cy.realPress(["ArrowDown"]);-- case "{uparrow}":- return cy.realPress(["ArrowUp"]);-- case "{enter}":- return cy.realPress(["Enter"]);-- case "{home}":- case "{movetostart}":- return cy.realPress(["Home"]);-- case "{end}":- case "{movetoend}":- return cy.realPress(["End"]);-- case "{backspace}":- return cy.realPress(["Backspace"]);-- case "{tab}":- return cy.realPress(["Tab"]);-- case "{nextcompletion}":- cy.wait(50);- return cy.realPress([metaKey, "j"]);-- case "{prevcompletion}":- cy.wait(50);- return cy.realPress([metaKey, "k"]);-- case "{{}":- return cy.realType("{");- }-- if (part.startsWith("{") && part.endsWith("}")) {- // unknown escape sequence, let's try typing it- type("{");- type(part.slice(1));- return;- }-- type(part);- });-- return nativeEditor();-}--export const NativeEditor = {- get: nativeEditor,- type(text: string, options: TypeOptions = {}) {- nativeEditorType(text, options);- return NativeEditor;- },- focus() {- focusNativeEditor();- return NativeEditor;- },- blur() {- blurNativeEditor();- return NativeEditor;- },- selectAll() {- nativeEditorSelectAll();- return NativeEditor;- },- clear() {- clearNativeEditor();- return NativeEditor;- },- textbox() {- return NativeEditor.get().get("[role='textbox']");- },- value() {- // Get the multiline text content of the editor- return NativeEditor.textbox()- .get(".cm-line")- .then((lines) => {- const text: string[] = [];- lines.each((_, line) => {- text.push(line.textContent ?? "");- });- const value = text.join("\n");- const placeholder = "SELECT * FROM TABLE_NAME";- if (value === placeholder) {- return "";- }- return value;- });- },- completions: nativeEditorCompletions,- completion: nativeEditorCompletion,-};+export const NativeEditor = codeMirrorHelpers("native-query-editor", {+ dataSource: nativeEditorDataSource,+ selectDataSource: selectNativeEditorDataSource,+});diff --git a/e2e/support/helpers/e2e-python-helpers.ts b/e2e/support/helpers/e2e-python-helpers.tsnew file mode 100644index 00000000000..6b073adb074--- /dev/null+++ b/e2e/support/helpers/e2e-python-helpers.ts@@ -0,0 +1,3 @@+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";++export const PythonEditor = codeMirrorHelpers("python-editor", {});diff --git a/e2e/support/helpers/index.ts b/e2e/support/helpers/index.tsindex dad0c1cd59b..36e00b8db54 100644--- a/e2e/support/helpers/index.ts+++ b/e2e/support/helpers/index.ts@@ -36,6 +36,7 @@ export * from "./e2e-native-editor-helpers";export * from "./e2e-notebook-helpers";export * from "./e2e-notification-helpers";export * from "./e2e-permissions-helpers";+export * from "./e2e-python-helpers";export * from "./e2e-qa-databases-helpers";export * from "./e2e-relative-date-picker-helpers";export * from "./e2e-request-helpers";diff --git a/e2e/test/scenarios/admin/transforms.cy.spec.ts b/e2e/test/scenarios/admin/transforms.cy.spec.tsindex f98372f402e..d680d7451a7 100644--- a/e2e/test/scenarios/admin/transforms.cy.spec.ts+++ b/e2e/test/scenarios/admin/transforms.cy.spec.ts@@ -1,10 +1,13 @@const { H } = cy;+import dedent from "ts-dedent";+import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";import type {CardType,ListTransformRunsResponse,+ PythonTransformTableAliases,TransformTagId,} from "metabase-types/api";@@ -187,6 +190,123 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {testSnippets();});+ it(+ "should be possible to create and run a Python transform",+ { tags: ["@transforms-python"] },+ () => {+ setPythonRunnerSettings();+ cy.log("create a new transform");+ visitTransformListPage();+ getTransformListPage().button("Create a transform").click();+ H.popover().findByText("Python script").click();++ H.expectUnstructuredSnowplowEvent({+ event: "transform_create",+ event_detail: "python",+ triggered_from: "transform-page-create-menu",+ });++ cy.findByTestId("python-data-picker")+ .findByText("Select a database")+ .click();++ cy.log("Unsupported databases should be disabled");+ H.popover()+ .findByRole("option", { name: "Sample Database" })+ .should("have.attr", "aria-disabled", "true");++ cy.log("Select database");+ H.popover().findByText(DB_NAME).click();++ getPythonDataPicker().button("Select a table…").click();+ H.entityPickerModal().findByText("Animals").click();++ getPythonDataPicker().within(() => {+ cy.findByText("Writable Postgres12 / Schema A").should("be.visible");+ cy.findByText("Animals").should("be.visible");+ cy.findByPlaceholderText("Enter alias").should(+ "have.value",+ "animals",+ );+ cy.findByPlaceholderText("Enter alias")+ .clear()+ .type("foo bar")+ .blur();+ cy.findByPlaceholderText("Enter alias").should(+ "have.value",+ "foo_bar",+ );+ });++ H.PythonEditor.value()+ .should("contain", "def transform(foo_bar):")+ .should(+ "contain",+ 'foo_bar: DataFrame containing the data from the "Writable Postgres12.Schema A.Animals" table',+ );++ getPythonDataPicker().within(() => {+ cy.findByText("Add a table").click();+ cy.button("Select a table…").click();+ });++ H.entityPickerModal().within(() => {+ cy.log("Selecting the same table should not be possible");+ cy.findByText("Animals")+ .parent()+ .parent()+ .parent()+ .should("have.attr", "data-disabled", "true");++ cy.findByText("Schema B").click();+ cy.findByText("Animals").click();+ });++ getPythonDataPicker().within(() => {+ cy.icon("refresh").click();+ cy.findAllByPlaceholderText("Enter alias")+ .first()+ .should("have.value", "animals_1")+ .clear()+ .type("foo bar")+ .blur()+ .should("have.value", "foo_bar");+ cy.findAllByPlaceholderText("Enter alias")+ .eq(1)+ .should("have.value", "animals")+ .clear()+ .type("foo bar")+ .blur()+ .should("have.value", "foo_bar_1");+ });++ getQueryEditor().button("Save").click();++ H.modal().within(() => {+ cy.findByLabelText("Name").clear().type("Python transform");+ cy.findByLabelText("Table name").clear().type("python_transform");+ cy.button("Save").click();+ });++ H.expectUnstructuredSnowplowEvent({+ event: "transform_created",+ });++ cy.log("run the transform and make sure its table can be queried");+ runTransformAndWaitForSuccess();+ H.expectUnstructuredSnowplowEvent({+ event: "transform_trigger_manual_run",+ triggered_from: "transform-page",+ });++ getRunSection().should("contain", "Executing Python transform");++ getTableLink().click();+ H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");+ H.assertQueryBuilderRowCount(1);+ },+ );+it("should be able to create and run a transform from a question or a model", () => {function testCardSource({type,@@ -1169,6 +1289,38 @@ LIMITH.queryBuilderHeader().findByText(DB_NAME).should("be.visible");H.assertQueryBuilderRowCount(1);});++ it("should be able to update a Python query", () => {+ setPythonRunnerSettings();+ cy.log("create a new transform");+ H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(+ (id) => {+ createPythonTransform({+ body: dedent`+ import pandas as pd++ def transform(foo):+ return pd.DataFrame([{"foo": 42 }])+ `,+ sourceTables: { foo: id },+ visitTransform: true,+ });+ },+ );++ cy.log("update the query");+ getTransformPage().findByRole("link", { name: "Edit script" }).click();+ H.PythonEditor.type("{backspace}{backspace}{backspace} + 10 }])");++ getQueryEditor().button("Save changes").click();+ cy.wait("@updateTransform");++ cy.log("run the transform and make sure the query has changed");+ runTransformAndWaitForSuccess();+ getTableLink().click();+ H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");+ H.assertQueryBuilderRowCount(1);+ });});describe("runs", () => {@@ -1395,6 +1547,136 @@ LIMITH.main().findByText("Dependencies").should("not.exist");});});++ describe("python > common library", () => {+ it(+ "should be possible to edit and save the common library",+ { tags: ["@transforms-python"] },+ () => {+ visitCommonLibrary();++ cy.log("updating the library should be possible");+ H.PythonEditor.clear().type(+ dedent`+ def useful_calculation(a, b):+ return a + b+ `,+ );+ getLibraryEditorHeader().findByText("Save").click();++ cy.log("the contents should be saved properly");+ visitCommonLibrary();+ H.PythonEditor.value().should(+ "eq",+ dedent`+ def useful_calculation(a, b):+ return a + b+ `,+ );++ cy.log("reverting the changes should be possible");+ H.PythonEditor.clear().type("# oops");+ getLibraryEditorHeader().findByText("Revert").click();+ H.PythonEditor.value().should(+ "eq",+ dedent`+ def useful_calculation(a, b):+ return a + b+ `,+ );+ },+ );++ it(+ "should be possible to use the common library",+ { tags: ["@transforms-python"] },+ () => {+ setPythonRunnerSettings();+ createPythonLibrary(+ "common.py",+ dedent`+ def useful_calculation(a, b):+ return a + b+ `,+ );++ visitTransformListPage();+ getTransformListPage().button("Create a transform").click();+ H.popover().findByText("Python script").click();++ H.PythonEditor.clear().type(+ dedent`+ import pandas as pd++ def transform():+ return pd.DataFrame([{"foo": common.useful_calculation(1, 2)}])+ `,+ );++ getQueryEditor().findByText("Import common library").click();+ H.PythonEditor.value().should("contain", "import common");++ cy.findByTestId("python-data-picker")+ .findByText("Select a database")+ .click();++ H.popover().findByText(DB_NAME).click();++ cy.findByTestId("python-data-picker")+ .findByText("Select a table…")+ .click();++ H.entityPickerModal().within(() => {+ cy.findByText("Schema a").click();+ cy.findByText("Animals").click();+ });++ getQueryEditor().button("Save").click();++ H.modal().within(() => {+ cy.findByLabelText("Name").clear().type("Python transform");+ cy.findByLabelText("Table name").clear().type("python_transform");+ cy.button("Save").click();+ });++ runTransformAndWaitForSuccess();+ getTableLink().click();+ H.queryBuilderHeader()+ .findByText("Python Transform")+ .should("be.visible");+ H.assertQueryBuilderRowCount(1);+ cy.findByTestId("scalar-value").should("have.text", "3");+ H.expectUnstructuredSnowplowEvent({+ event: "transform_created",+ });++ cy.log("update the common library and run the transform again");+ cy.go("back");+ createPythonLibrary(+ "common.py",+ dedent`+ def useful_calculation(a, b):+ return a + b + 40+ `,+ );+ runTransformAndWaitForSuccess();+ getTableLink().click();+ H.queryBuilderHeader()+ .findByText("Python Transform")+ .should("be.visible");+ H.assertQueryBuilderRowCount(1);+ cy.findByTestId("scalar-value").should("have.text", "43");+ },+ );++ function visitCommonLibrary(path = "common.py") {+ cy.visit(`/admin/transforms/library/${path}`);+ }++ function getLibraryEditorHeader() {+ return cy.findByTestId("library-editor-header");+ }+ });});describe("scenarios > admin > transforms > databases without :schemas", () => {@@ -2476,6 +2758,41 @@ function createSqlTransform({{ wrapId: true, visitTransform },);}+function createPythonTransform({+ body,+ sourceTables,+ targetTable = TARGET_TABLE,+ targetSchema = TARGET_SCHEMA,+ tagIds,+ visitTransform,+}: {+ body: string;+ sourceTables: PythonTransformTableAliases;+ targetTable?: string;+ targetSchema?: string;+ tagIds?: TransformTagId[];+ visitTransform?: boolean;+}) {+ H.createTransform(+ {+ name: "Python transform",+ source: {+ type: "python",+ "source-database": WRITABLE_DB_ID,+ "source-tables": sourceTables,+ body,+ },+ target: {+ type: "table",+ database: WRITABLE_DB_ID,+ name: targetTable,+ schema: targetSchema,+ },+ tag_ids: tagIds,+ },+ { wrapId: true, visitTransform },+ );+}function visitTableQuestion({targetTable = TARGET_TABLE,@@ -2531,3 +2848,35 @@ function assertOptionNotSelected(name: string) {function editorSidebar() {return cy.findByTestId("editor-sidebar");}++function getPythonDataPicker() {+ return cy.findByTestId("python-data-picker");+}++function createPythonLibrary(path: string, source: string) {+ cy.request("PUT", `/api/ee/transforms-python/library/${path}`, {+ source,+ });+}++function setPythonRunnerSettings() {+ H.updateEnterpriseSetting("python-runner-url", "http://localhost:5001");+ H.updateEnterpriseSetting("python-runner-api-token", "dev-token-12345");+ H.updateEnterpriseSetting(+ "python-storage-s-3-endpoint",+ "http://localhost:4566",+ );+ H.updateEnterpriseSetting("python-storage-s-3-region", "us-east-1");+ H.updateEnterpriseSetting(+ "python-storage-s-3-bucket",+ "metabase-python-runner",+ );+ H.updateEnterpriseSetting("python-storage-s-3-prefix", "test-prefix");+ H.updateEnterpriseSetting("python-storage-s-3-access-key", "test");+ H.updateEnterpriseSetting("python-storage-s-3-secret-key", "test");+ H.updateEnterpriseSetting(+ "python-storage-s-3-container-endpoint",+ "http://localstack:4566",+ );+ H.updateEnterpriseSetting("python-storage-s-3-path-style-access", true);+}diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.cljindex e9522faf3b1..28e174d6cf3 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj@@ -1,7 +1,9 @@(ns metabase-enterprise.transforms.api(:require+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase-enterprise.transforms.api.transform-job][metabase-enterprise.transforms.api.transform-tag]+ [metabase-enterprise.transforms.canceling :as transforms.canceling][metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.models.transform :as transform.model][metabase-enterprise.transforms.models.transform-run :as transform-run]@@ -29,12 +31,21 @@(set! *warn-on-reflection* true)(mr/def ::transform-source- [:map- [:type [:= "query"]]- [:query [:map [:database :int]]]])+ [:multi {:dispatch (comp keyword :type)}+ [:query+ [:map+ [:type [:= "query"]]+ [:query [:map [:database :int]]]]]+ [:python+ [:map {:closed true}+ [:source-database {:optional true} :int]+ [:source-tables [:map-of :string :int]]+ [:type [:= "python"]]+ [:body :string]]]])(mr/def ::transform-target[:map+ [:database {:optional true} :int][:type [:enum "table"]][:schema {:optional true} [:or ms/NonBlankString :nil]][:name :string]])@@ -55,14 +66,10 @@:schema "transforms":name "gadget_products"}}])-(defn- source-database-id- [transform]- (-> transform :source :query :database))-(defn- check-database-feature[transform]- (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))- (deferred-tru "The source database cannot be found."))+ (let [database (api/check-400 (t2/select-one :model/Database (transforms.util/target-database-id transform))+ (deferred-tru "The target database cannot be found."))feature (transforms.util/required-database-feature transform)](api/check-400 (not (:is_sample database))(deferred-tru "Cannot run transforms on the sample database."))@@ -73,6 +80,11 @@(api/check-400 (not (transforms.util/db-routing-enabled? database))(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))+(defn- check-feature-enabled!+ [transform]+ (api/check (transforms.util/check-feature-enabled transform)+ [402 (deferred-tru "Premium features required for this transform type are not enabled.")]))+(api.macros/defendpoint :get "/""Get a list of transforms."[_route-params@@ -95,6 +107,7 @@[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]](api/check-superuser)(check-database-feature body)+ (check-feature-enabled! body)(api/check (not (transforms.util/target-table-exists? body))403(deferred-tru "A table with that name already exists."))@@ -115,8 +128,7 @@(log/info "get transform" id)(api/check-superuser)(let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))- database-id (source-database-id transform)- target-table (transforms.util/target-table database-id target :active true)]+ target-table (transforms.util/target-table (transforms.util/target-database-id transform) target :active true)](-> transform(t2/hydrate :last_run :transform_tag_ids)(u/update-some :last_run transforms.util/localize-run-timestamps)@@ -173,10 +185,13 @@(let [old (t2/select-one :model/Transform id)new (merge old body)target-fields #(-> % :target (select-keys [:schema :name]))]+ ;; we must validate on a full transform object+ (check-feature-enabled! new)(check-database-feature new)- (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]- (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)- {:status-code 400})))+ (when (transforms.util/query-transform? old)+ (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]+ (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)+ {:status-code 400}))))(api/check (not (and (not= (target-fields old) (target-fields new))(transforms.util/target-table-exists? new)))403@@ -212,8 +227,11 @@[:id ms/PositiveInt]]](log/info "canceling transform " id)(api/check-superuser)- (let [run (api/check-404 (transform-run/running-run-for-transform-id id))]- (transform-run-cancelation/mark-cancel-started-run! (:id run)))+ (let [transform (api/check-404 (t2/select-one :model/Transform id))+ run (api/check-404 (transform-run/running-run-for-transform-id id))]+ (transform-run-cancelation/mark-cancel-started-run! (:id run))+ (when (transforms.util/python-transform? transform)+ (transforms.canceling/cancel-run! (:id run))))nil)(api.macros/defendpoint :post "/:id/run"@@ -223,10 +241,15 @@(log/info "run transform" id)(api/check-superuser)(let [transform (api/check-404 (t2/select-one :model/Transform id))+ _ (check-feature-enabled! transform)start-promise (promise)]- (u.jvm/in-virtual-thread*- (transforms.execute/run-mbql-transform! transform {:start-promise start-promise- :run-method :manual}))+ (if (transforms.util/python-transform? transform)+ (u.jvm/in-virtual-thread*+ (transforms-python.execute/execute-python-transform! transform {:start-promise start-promise+ :run-method :manual}))+ (u.jvm/in-virtual-thread*+ (transforms.execute/run-mbql-transform! transform {:start-promise start-promise+ :run-method :manual})))(when (instance? Throwable @start-promise)(throw @start-promise))(let [result @start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/canceling.clj b/enterprise/backend/src/metabase_enterprise/transforms/canceling.cljindex 0c62714a8d6..f4a2612ccbc 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/canceling.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/canceling.clj@@ -49,7 +49,9 @@(chan-signal-cancel! run-id)(transform-run/timeout-run! run-id)))-(defn- cancel-run! [run-id]+(defn cancel-run!+ "Cancel a run with id."+ [run-id](when (chan-signal-cancel! run-id)(transform-run/cancel-run! run-id)))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.cljindex bc1b82fdca0..bfc527b5349 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.transforms.execute(:require+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver][metabase.driver.util :as driver.u]@@ -54,7 +55,8 @@(transforms.util/run-cancelable-transform!run-id driver transform-details(fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))- (transforms.util/sync-target! target database run-id)))+ (transforms.instrumentation/with-stage-timing [run-id :table-sync]+ (transforms.util/sync-target! target database run-id))))(catch Throwable t(log/error t "Error executing transform")(when start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.cljnew file mode 100644index 00000000000..dbc32f09081--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj@@ -0,0 +1,193 @@+(ns metabase-enterprise.transforms.instrumentation+ (:require+ [metabase.analytics.prometheus :as prometheus]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++(set! *warn-on-reflection* true)++(mr/def ::stage-type+ [:enum :export :computation :import])++(def ^:private label-to-stage+ {:dwh-to-file :export+ :file-to-s3 :export+ :file-to-dwh :import+ :python-execution :computation+ :mbql-query :computation+ :table-sync :import})++(mr/def ::stage-label+ [:enum+ :dwh-to-file+ :file-to-s3+ :file-to-dwh++ :python-execution+ :mbql-query+ :table-sync])++(mu/defn record-stage-start!+ "Record the start of a transform stage."+ [job-run-id :- pos-int?+ stage-label :- ::stage-label]+ (let [stage-type (label-to-stage stage-label)]+ (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))+ (prometheus/inc! :metabase-transforms/stage-started+ {:job-run-id (str job-run-id)+ :stage-type (name stage-type)+ :stage-label (name stage-label)})))++(mu/defn record-stage-completion!+ "Record the successful completion of a transform stage."+ [job-run-id :- pos-int?+ stage-label :- ::stage-label+ duration-ms :- int?]+ (let [stage-type (label-to-stage stage-label)]+ (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)+ (let [labels {:job-run-id (str job-run-id)+ :stage-type (name stage-type)+ :stage-label (name stage-label)}]+ (prometheus/inc! :metabase-transforms/stage-completed labels)+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))++(mu/defn record-stage-failure!+ "Record the failure of a transform stage."+ [job-run-id :- pos-int?+ stage-label :- ::stage-label+ duration-ms :- int?]+ (let [stage-type (label-to-stage stage-label)]+ (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))+ (let [labels {:job-run-id (str job-run-id)+ :stage-type (name stage-type)+ :stage-label (name stage-label)}]+ (prometheus/inc! :metabase-transforms/stage-failed labels)+ (when duration-ms+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))))++(defn with-timing+ "Generic timing function that executes body while recording metrics.++ Takes a map of optional functions:+ - :start-fn - function to call when starting+ - :success-fn - function to call on success (receives duration-ms as last arg)+ - :error-fn - function to call on error (receives duration-ms as last arg)++ Usage:+ (with-timing {:start-fn #(record-stage-start! %1 %3)+ :success-fn #(record-stage-completion! %1 %2 %3)+ :error-fn #(record-stage-failure! %1 %2 %3)}+ [job-run-id stage-label]+ #(do-work)) "+ [callbacks args body-fn]+ (let [start-time (u/start-timer)]+ (when-let [start-fn (:start-fn callbacks)]+ (apply start-fn args))+ (try+ (let [result (body-fn)]+ (when-let [success-fn (:success-fn callbacks)]+ (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))+ result)+ (catch Throwable t+ (when-let [error-fn (:error-fn callbacks)]+ (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))+ (throw t)))))++(defmacro with-stage-timing+ "Execute body while timing a transform stage. Automatically records start, completion or failure.++ Usage:+ (with-stage-timing [job-run-id :dwh-to-file]+ ;; stage implementation+ result)"+ [args & body]+ `(with-timing {:start-fn record-stage-start!+ :success-fn record-stage-completion!+ :error-fn record-stage-failure!}+ ~args+ (^:once fn* [] ~@body)))++(mu/defn record-data-transfer!+ "Record metrics about data transfer (size and rows)."+ [job-run-id :- pos-int?+ stage-label :- ::stage-label+ bytes :- [:maybe int?]+ rows :- [:maybe int?]]+ (if (and (nil? bytes) (nil? rows))+ (log/warnf "Data transfer recorded: run-id=%d stage=%s but no bytes or rows provided"+ job-run-id+ (name stage-label))+ (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"+ job-run-id+ (name stage-label)+ (if bytes (str " bytes=" bytes) "")+ (if rows (str " rows=" rows) "")))+ (let [labels {:job-run-id (str job-run-id)+ :stage-label (name stage-label)}]+ (when bytes+ (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))+ (when rows+ (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))++(defn record-job-start!+ "Record the start of a transform job run."+ [job-id run-method]+ (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))+ (prometheus/inc! :metabase-transforms/job-runs-total+ {:job-id (str job-id)+ :run-method (name run-method)}))++(defn record-job-completion!+ "Record the successful completion of a transform job run."+ [job-id run-method duration-ms]+ (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)+ (let [labels {:job-id (str job-id)+ :run-method (name run-method)}]+ (prometheus/inc! :metabase-transforms/job-runs-completed labels)+ (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))++(defn record-job-failure!+ "Record the failure of a transform job run."+ [job-id run-method duration-ms]+ (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)+ (let [labels {:job-id (str job-id)+ :run-method (name run-method)}]+ (prometheus/inc! :metabase-transforms/job-runs-failed labels)+ (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))++(defmacro with-job-timing+ "Execute body while timing a transform job run. Automatically records start, completion or failure.++ Usage:+ (with-job-timing [job-id run-method]+ ;; job implementation+ result)"+ [[job-id run-method] & body]+ `(with-timing {:start-fn record-job-start!+ :success-fn record-job-completion!+ :error-fn record-job-failure!}+ [~job-id ~run-method]+ (^:once fn* [] ~@body)))++(mu/defn record-python-api-call!+ "Record metrics about Python API calls."+ [job-run-id :- pos-int?+ duration-ms :- int?+ status :- [:enum :success :error :timeout]]+ (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)+ (let [labels {:job-run-id (str job-run-id)}]+ (prometheus/inc! :metabase-transforms/python-api-calls-total+ (assoc labels :status (name status)))+ (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))++(defmacro with-python-api-timing+ "Execute body while timing a Python API call."+ [[job-run-id] & body]+ `(with-timing {:success-fn (fn [job-run-id# duration-ms#]+ (record-python-api-call! job-run-id# duration-ms# :success))+ :error-fn (fn [job-run-id# duration-ms#]+ (record-python-api-call! job-run-id# duration-ms# :error))}+ [~job-run-id]+ (^:once fn* [] ~@body)))diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.cljindex 68ab07b40af..7740701daad 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj@@ -5,11 +5,14 @@[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval][clojurewerkz.quartzite.triggers :as triggers][flatland.ordered.set :as ordered-set]+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase-enterprise.transforms.execute :as transforms.execute]+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation][metabase-enterprise.transforms.models.job-run :as transforms.job-run][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.ordering :as transforms.ordering][metabase-enterprise.transforms.settings :as transforms.settings]+ [metabase-enterprise.transforms.util :as transforms.util][metabase.task.core :as task][metabase.util.log :as log][toucan2.core :as t2]))@@ -64,18 +67,22 @@(map transforms-by-id complete)))))(defn- run-transform! [run-id run-method {transform-id :id :as transform}]- (when- (transform-run/running-run-for-transform-id transform-id)- (log/warn "Transform" (pr-str transform-id) "already running, waiting")- (loop []- (Thread/sleep 2000)+ (if-not (transforms.util/check-feature-enabled transform)+ (log/warnf "Skip running transform %d due to lacking premium features" transform-id)+ (do(when (transform-run/running-run-for-transform-id transform-id)- (recur))))-- (log/info "Executing job transform" (pr-str transform-id))- (transforms.execute/run-mbql-transform! transform {:run-method run-method})-- (transforms.job-run/add-run-activity! run-id))+ (log/warn "Transform" (pr-str transform-id) "already running, waiting")+ (loop []+ (Thread/sleep 2000)+ (when (transform-run/running-run-for-transform-id transform-id)+ (recur))))+ (log/info "Executing job transform" (pr-str transform-id))+ (if (transforms.util/python-transform? transform)+ (transforms.instrumentation/with-stage-timing [run-id :python-execution]+ (transforms-python.execute/execute-python-transform! transform {:run-method run-method}))+ (transforms.instrumentation/with-stage-timing [run-id :mbql-query]+ (transforms.execute/run-mbql-transform! transform {:run-method run-method})))+ (transforms.job-run/add-run-activity! run-id))))(defn run-transforms!"Run a series of transforms and their dependencies.@@ -113,12 +120,13 @@(let [transforms (job-transform-ids job-id)](log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))(let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]- (try- (run-transforms! run-id transforms opts)- (transforms.job-run/succeed-started-run! run-id)- (catch Throwable t- (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})- (throw t)))))))+ (transforms.instrumentation/with-job-timing [job-id run-method]+ (try+ (run-transforms! run-id transforms opts)+ (transforms.job-run/succeed-started-run! run-id)+ (catch Throwable t+ (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})+ (throw t))))))))(def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")diff --git a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj b/enterprise/backend/src/metabase_enterprise/transforms/ordering.cljindex c7bf77c2f2b..ae184e38de3 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj@@ -16,23 +16,29 @@(set! *warn-on-reflection* true)(defn- transform-deps [transform]- (let [query (-> (get-in transform [:source :query])- transforms.util/massage-sql-query- qp.preprocess/preprocess- ;; legacy usage -- don't do things like this going forward- #_{:clj-kondo/ignore [:discouraged-var]}- lib/->legacy-MBQL)]- (case (:type query)- :native (driver/native-query-deps (-> (qp.store/metadata-provider)- lib.metadata/database- :engine)- (get-in query [:native :query]))- :query (into #{}- (keep #(clojure.core.match/match %- [:source-table source] (when (int? source)- {:table source})- _ nil))- (tree-seq coll? seq query)))))+ (if (transforms.util/python-transform? transform)+ ;; Python transforms have dependencies via source-tables mapping+ (into #{}+ (map #(hash-map :table %))+ (vals (get-in transform [:source :source-tables])))+ ;; Query transforms have dependencies via SQL query analysis+ (let [query (-> (get-in transform [:source :query])+ transforms.util/massage-sql-query+ qp.preprocess/preprocess+ ;; legacy usage -- don't do things like this going forward+ #_{:clj-kondo/ignore [:discouraged-var]}+ lib/->legacy-MBQL)]+ (case (:type query)+ :native (driver/native-query-deps (-> (qp.store/metadata-provider)+ lib.metadata/database+ :engine)+ (get-in query [:native :query]))+ :query (into #{}+ (keep #(clojure.core.match/match %+ [:source-table source] (when (int? source)+ {:table source})+ _ nil))+ (tree-seq coll? seq query))))))(defn- dependency-map [transforms](into {}@@ -56,12 +62,15 @@The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to justthe transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'- dependency is ignored."+ dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."[transforms]- (let [transforms-by-db (->> transforms+ (let [;; Group all transforms by their database+ transforms-by-db (->> transforms(map (fn [transform]- {(get-in transform [:source :query :database]) [transform]}))+ (let [db-id (transforms.util/target-database-id transform)]+ {db-id [transform]})))(apply merge-with into))+transform-ids (into #{} (map :id) transforms){:keys [output-tables dependencies]} (->> transforms-by-db(map (fn [[db-id db-transforms]]diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex b22684bd615..ccd5a47bdef 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -7,12 +7,16 @@[metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.settings :as transforms.settings][metabase.driver :as driver]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.premium-features.core :as premium-features][metabase.query-processor.compile :as qp.compile][metabase.query-processor.pipeline :as qp.pipeline][metabase.sync.core :as sync][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][toucan2.core :as t2])(:import(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)@@ -27,6 +31,24 @@(keyword schema name)(keyword name)))+(defn query-transform?+ "Check if this is a query transform: native query / mbql query."+ [transform]+ (= :query (-> transform :source :type keyword)))++(defn python-transform?+ "Check if this is a Python transform."+ [transform]+ (= :python (-> transform :source :type keyword)))++(defn check-feature-enabled+ "Checking whether we have proper feature flags for using a given transform."+ [transform]+ (if (python-transform? transform)+ (and (premium-features/has-feature? :transforms)+ (premium-features/has-feature? :transforms-python))+ (premium-features/has-feature? :transforms)))+(defn try-start-unless-already-running"Start a transform run, throwing an informative error if already running."[id run-method]@@ -76,10 +98,18 @@(log/info "Syncing target" (pr-str target) "for transform")(activate-table-and-mark-computed! database target)))+;; TODO this and target-database-id can be transforms multimethods?+(defn target-database-id+ "Return the target database id of a transform"+ [transform]+ (if (python-transform? transform)+ (-> transform :target :database)+ (-> transform :source :query :database)))+(defn target-table-exists?"Test if the target table of a transform already exists."- [{:keys [source target] :as _transform}]- (let [db-id (-> source :query :database)+ [{:keys [target] :as transform}]+ (let [db-id (target-database-id transform){driver :engine :as database} (t2/select-one :model/Database db-id)](driver/table-exists? driver database target)))@@ -118,10 +148,10 @@(defn delete-target-table!"Delete the target table of a transform and sync it from the app db."- [{:keys [id target source], :as _transform}]+ [{:keys [id target], :as transform}](when target(let [target (update target :type keyword)- database-id (-> source :query :database)+ database-id (target-database-id transform){driver :engine :as database} (t2/select-one :model/Database database-id)](driver/drop-transform-target! driver database target)(log/info "Deactivating target " (pr-str target) "for transform" id)@@ -146,8 +176,10 @@(defn required-database-feature"Returns the database feature necessary to execute `transform`."[transform]- (case (-> transform :target :type)- "table" :transforms/table))+ (if (python-transform? transform)+ :transforms/python+ (case (-> transform :target :type)+ "table" :transforms/table)))(defn ->instant"Convert a temporal value `t` to an Instant in the system timezone."@@ -178,6 +210,65 @@(u/update-some :start_time utc-timestamp-string)(u/update-some :end_time utc-timestamp-string)))+(mr/def ::column-definition+ [:map+ [:name :string]+ [:type ::lib.schema.common/base-type]+ [:nullable? {:optional true} :boolean]])++(mr/def ::table-definition+ [:map+ [:name :keyword]+ [:columns [:sequential ::column-definition]]+ [:primary-key {:optional true} [:sequential :string]]])++(mu/defn create-table-from-schema!+ "Create a table from a table-schema"+ [driver :- :keyword+ database-id :- pos-int?+ table-schema :- ::table-definition]+ (let [{:keys [columns] table-name :name} table-schema+ column-definitions (into {} (map (fn [{:keys [name type database-type]}]+ (let [db-type (if database-type+ [[:raw database-type]]+ (try+ (driver/type->database-type driver type)+ (catch IllegalArgumentException _+ (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)+ (driver/type->database-type driver :type/Text))))]+ [name db-type])))+ columns)+ primary-key-opts (select-keys table-schema [:primary-key])]+ (log/infof "Creating table %s with %d columns" table-name (count columns))+ (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))++(defn drop-table!+ "Drop a table in the database."+ [driver database-id table-name]+ (log/infof "Dropping table %s" table-name)+ (driver/drop-table! driver database-id table-name))++(defn temp-table-name+ "Generate a temporary table name with the given suffix and current timestamp in seconds.+ If table name would exceed max table name length for the driver, fallback to using a shorter base-name."+ [driver base-table-name suffix]+ {:pre [(keyword? base-table-name)]}+ (let [suffix (str "_" suffix "_" (quot (System/currentTimeMillis) 1000))+ max-len (or (driver/table-name-length-limit driver) Integer/MAX_VALUE)+ max-prefix-len (- max-len (count suffix))+ table-name (name base-table-name)+ prefix (subs table-name 0 (min (count table-name) max-prefix-len))]+ (assert (pos? max-prefix-len))+ (keyword (namespace base-table-name)+ (str prefix suffix))))++(defn rename-tables!+ "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.+ This is a simpler, composable operation that only handles renaming."+ [driver database-id rename-map]+ (log/infof "Renaming tables: %s" (pr-str rename-map))+ (driver/rename-tables! driver database-id rename-map))+(defn db-routing-enabled?"Returns whether or not the given database is either a router or destination database"[db-or-id]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljnew file mode 100644index 00000000000..07a36320bf3--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -0,0 +1,328 @@+(ns metabase-enterprise.transforms-python.execute+ (:require+ [clojure.core.async :as a]+ [clojure.java.io :as io]+ [clojure.string :as str]+ [metabase-enterprise.transforms-python.python-runner :as python-runner]+ [metabase-enterprise.transforms-python.s3 :as s3]+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]+ [metabase-enterprise.transforms.util :as transforms.util]+ [metabase.driver :as driver]+ [metabase.driver.util :as driver.u]+ [metabase.util :as u]+ [metabase.util.jvm :as u.jvm]+ [metabase.util.log :as log]+ [toucan2.core :as t2])+ (:import+ (java.io Closeable File)+ (java.net SocketException)+ (java.time Duration)))++(set! *warn-on-reflection* true)++(def ^:const temp-table-suffix-new+ "Suffix used for temporary tables containing new data during swap operations."+ "new")++(def ^:const temp-table-suffix-old+ "Suffix used for temporary tables containing old data during swap operations."+ "old")++(defn- empty-message-log+ "Returns a new message log.++ Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.+ The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.++ Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.++ The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."+ []+ (atom {:pre-python [] ; log! outputs previous to the python execution, i.e table read progress+ :python nil ; events json structured logs from the /logs endpoint+ :post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target++(defn- log!+ "Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."+ [message-log s]+ (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))+ nil)++(defn- replace-python-logs!+ "Called when events output is received from a /logs call to copy those events into the message-log."+ [message-log events]+ (swap! message-log assoc :python events)+ nil)++(defn- message-log->transform-run-message+ "Presents the content of the log as a string suitable for storage in transform_run.message."+ [message-log]+ (let [{:keys [pre-python python post-python]} @message-log]+ (str/join "\n" (concat pre-python+ (map :message python)+ post-python))))++(defn- save-log-to-transform-run-message!+ "Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."+ [run-id message-log]+ (t2/update! :model/TransformRun+ :id run-id+ {:message (message-log->transform-run-message message-log)}))++(def ^:private ^Duration python-message-loop-sleep-duration+ (Duration/ofMillis 1000))++(defn- python-message-update-loop!+ "Block while relevant log data is replicated from the runner into the message log.+ When new logs are received, the log data will be flushed to the transform_run.message field as a string.++ The loop will exit on error or when interrupted."+ [run-id message-log]+ (try+ (loop []+ (if (.isInterrupted (Thread/currentThread))+ (log/debug "Message update loop interrupted")+ (do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]+ (when (pos? sleep-ms) (Thread/sleep sleep-ms)))+ (let [{:keys [status body]} (python-runner/get-logs run-id)]+ (cond+ (<= 200 status 299)+ (let [{:keys [execution_id events]} body]+ (if-not (= run-id execution_id)+ (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)+ (recur))+ (do+ (replace-python-logs! message-log events)+ (save-log-to-transform-run-message! run-id message-log)+ (recur))))+ (= 404 status)+ (do+ (log/debugf "No logs yet (or run finished), run-id: %s" run-id)+ (recur))+ :else+ (do+ (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)+ (log/debug "Exiting due to poll error")))))))+ (catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))+ (catch InterruptedException _)+ (catch Throwable e+ (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))++(defn- open-python-message-update-future! ^Closeable [run-id message-log]+ (let [cleanup (fn [fut]+ (future-cancel fut)+ (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))+ (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)+ (log/debugf "Log polling task done, run-id: %s" run-id)))+ fut (u.jvm/in-virtual-thread*+ (python-message-update-loop! run-id message-log))]+ (reify Closeable+ (close [_] (cleanup fut)))))++(defn- maybe-retry-with-backoff+ [driver thunk]+ ;; bigquery can have eventual consistency hiccups where the table is not yet available for insertion+ ;; immediately after creation, so we retry a few times with backoff+ (if (= driver :bigquery-cloud-sdk)+ (loop [attempt 0]+ (let [ret (try+ (thunk)+ (catch Exception e+ (if (and (< attempt 5)+ (or (str/includes? (str e) "not found")+ (str/includes? (str e) "Not found")))+ (let [delay-ms (* 150 (Math/pow 2 attempt))]+ (log/debugf "BigQuery operation failed (attempt %d), retrying in %dms" (inc attempt) delay-ms)+ (Thread/sleep (long delay-ms))+ ::recur)+ (throw e))))]+ (if (= ::recur ret)+ (recur (inc attempt))+ ret)))+ (thunk)))++(defn- create-table-and-insert-data!+ "Create a table from metadata and insert data from source."+ [driver db-id table-name metadata data-source]+ (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))+ :columns (mapv (fn [{:keys [name base_type #_database_type]}]+ {:name name+ :type (python-runner/restricted-insert-type base_type)+ ;; :database-type database_type+ :nullable? true})+ (:fields metadata))}+ data-source (assoc data-source :table-schema table-schema)]+ (transforms.util/create-table-from-schema! driver db-id table-schema)++ (->> #(driver/insert-from-source! driver db-id table-schema data-source)+ (maybe-retry-with-backoff driver))))++(defn- transfer-with-rename-tables-strategy!+ "Transfer data using the rename-tables*! multimethod with atomicity guarantees.+ Creates new table, then atomically renames target->old and new->target, then drops old."+ [driver db-id table-name metadata data-source]+ (let [source-table-name (transforms.util/temp-table-name driver table-name temp-table-suffix-new)+ temp-table-name (transforms.util/temp-table-name driver table-name temp-table-suffix-old)]+ (log/info "Using rename-tables strategy with atomicity guarantees")+ (try++ (create-table-and-insert-data! driver db-id source-table-name metadata data-source)+ (transforms.util/rename-tables! driver db-id {table-name temp-table-name+ source-table-name table-name})+ (transforms.util/drop-table! driver db-id temp-table-name)++ (catch Exception e+ (log/error e "Failed to transfer data using rename-tables strategy")+ (try+ (transforms.util/drop-table! driver db-id source-table-name)+ (catch Exception _))+ (throw e)))))++(defn- transfer-with-create-drop-rename-strategy!+ "Transfer data using create + drop + rename to minimize time without data.+ Creates new table, drops old table, then renames new->target."+ [driver db-id table-name metadata data-source]+ (let [source-table-name (transforms.util/temp-table-name driver table-name temp-table-suffix-new)]+ (log/info "Using create-drop-rename strategy to minimize downtime")+ (try++ (create-table-and-insert-data! driver db-id source-table-name metadata data-source)+ (transforms.util/drop-table! driver db-id table-name)+ (driver/rename-table! driver db-id source-table-name table-name)++ (catch Exception e+ (log/error e "Failed to transfer data using create-drop-rename strategy")+ (try+ (transforms.util/drop-table! driver db-id source-table-name)+ (catch Exception _))+ (throw e)))))++(defn- transfer-with-drop-create-fallback-strategy!+ "Transfer data using drop + create fallback strategy.+ Drops old table, then creates new table directly with target name."+ [driver db-id table-name metadata data-source]+ (log/info "Using drop-create fallback strategy")+ (try++ (transforms.util/drop-table! driver db-id table-name)+ (create-table-and-insert-data! driver db-id table-name metadata data-source)++ (catch Exception e+ (log/error e "Failed to transfer data using drop-create fallback strategy")+ (throw e))))++(defn- transfer-file-to-db [driver {db-id :id :as db} {:keys [target] :as transform} metadata temp-file]+ (let [table-name (transforms.util/qualified-table-name driver target)+ table-exists? (transforms.util/target-table-exists? transform)+ data-source {:type :jsonl-file+ :file temp-file}]+ (cond+ (not table-exists?)+ (do+ (log/info "New table")+ (create-table-and-insert-data! driver db-id table-name metadata data-source))++ (driver.u/supports? driver :atomic-renames db)+ (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)++ (driver.u/supports? driver :rename db)+ (transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)++ :else+ (transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))++(defn- start-cancellation-process!+ "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.+ Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."+ [server-url run-id cancel-chan]+ (a/go (when (a/<! cancel-chan)+ (python-runner/cancel-python-code-http-call! server-url run-id))))++(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]+ ;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms+ (with-open [log-future-ref (open-python-message-update-future! run-id message-log)+ shared-storage-ref (s3/open-shared-storage! (:source-tables source))]+ (let [driver (:engine db)+ server-url (transforms-python.settings/python-runner-url)+ _ (python-runner/copy-tables-to-s3! {:run-id run-id+ :shared-storage @shared-storage-ref+ :table-name->id (:source-tables source)+ :cancel-chan cancel-chan})+ _ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan+ {:keys [status body] :as response}+ (python-runner/execute-python-code-http-call!+ {:server-url server-url+ :code (:body source)+ :run-id run-id+ :table-name->id (:source-tables source)+ :shared-storage @shared-storage-ref})+ ;; TODO temporary to keep more code stable while refactoring+ ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)+ {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]+ (.close log-future-ref) ; early close to force any writes to flush+ (when (seq events)+ (replace-python-logs! message-log events))+ (if (not= 200 status)+ (throw (ex-info "Python runner call failed"+ {:status-code 400+ :api-status-code status+ :body body+ :events events+ :transform-run-message (message-log->transform-run-message message-log)}))+ (try+ (let [temp-file (File/createTempFile "transform-output-" ".jsonl")]+ (when-not (seq (:fields output-manifest))+ (throw (ex-info "No fields in metadata"+ {:metadata output-manifest+ :raw-body body+ :events events+ :transform-run-message (message-log->transform-run-message message-log)})))+ (try+ (with-open [writer (io/writer temp-file)]+ (.write writer ^String output))+ (let [file-size (.length temp-file)]+ (transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]+ (transfer-file-to-db driver db transform output-manifest temp-file))+ (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))+ (finally+ (.delete temp-file))))+ response+ (catch Exception e+ (log/error e "Failed to to create resulting table")+ (throw (ex-info "Failed to create the resulting table"+ {:transform-run-message (message-log->transform-run-message message-log)}+ e))))))))++(defn execute-python-transform!+ "Execute a Python transform by calling the python runner.++ Blocks until the transform returns."+ [transform {:keys [run-method start-promise]}]+ (assert (transforms.util/python-transform? transform) "Transform must be a python transform")+ (try+ (let [message-log (empty-message-log)+ {:keys [target] transform-id :id} transform+ {driver :engine :as db} (t2/select-one :model/Database (:database target))+ {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method)]+ (some-> start-promise (deliver [:started run-id]))+ (log! message-log "Executing Python transform")+ (log/info "Executing Python transform" transform-id "with target" (pr-str target))+ (let [start-ms (u/start-timer)+ transform-details {:db-id (:id db)+ :transform-type (keyword (:type target))+ :conn-spec (driver/connection-spec driver db)+ :output-schema (:schema target)+ :output-table (transforms.util/qualified-table-name driver target)}+ run-fn (fn [cancel-chan]+ (run-python-transform! transform db run-id cancel-chan message-log)+ (log! message-log (format "Python execution finished successfully in %s" (Duration/ofMillis (u/since-ms start-ms))))+ (save-log-to-transform-run-message! run-id message-log))+ result (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn)]+ (transforms.instrumentation/with-stage-timing [run-id :table-sync]+ (transforms.util/sync-target! target db run-id))+ {:run_id run-id+ :result result}))+ (catch Throwable t+ (log/error t "Error executing Python transform")+ (throw t))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljnew file mode 100644index 00000000000..60dc6d6e8dc--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -0,0 +1,283 @@+(ns metabase-enterprise.transforms-python.python-runner+ (:require+ [clj-http.client :as http]+ [clojure.java.io :as io]+ [clojure.string :as str]+ [medley.core :as m]+ [metabase-enterprise.transforms-python.s3 :as s3]+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]+ [metabase.config.core :as config]+ [metabase.driver :as driver]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.pipeline :as qp.pipeline]+ ;; TODO check that querying team are ok with us accessing this directly, otherwise make another plan+ ^{:clj-kondo/ignore [:discouraged-namespace]}+ [metabase.query-processor.store :as qp.store]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [toucan2.core :as t2])+ (:import+ (clojure.lang PersistentQueue)+ (java.io BufferedWriter File OutputStream OutputStreamWriter)+ (java.nio.charset StandardCharsets)))++(set! *warn-on-reflection* true)++(defn- authorization-headers+ "Returns HTTP headers with Authorization bearer token if configured.+ Throws configuration error in production if token is not set."+ []+ (let [api-token (transforms-python.settings/python-runner-api-token)]+ (if api-token+ {"Authorization" (str "Bearer " api-token)}+ (if config/is-prod?+ (throw (ex-info "Python runner API token is required in production but not configured"+ {:error-type :configuration-error}))+ {}))))++(defn- python-runner-request+ "Helper function for making HTTP requests to the python runner service."+ [server-url method endpoint & [request-options]]+ (let [url (str server-url "/v1" endpoint)+ base-options {:content-type :json+ :accept :json+ :throw-exceptions false+ :as :json+ :headers (authorization-headers)}]+ (http/request (merge base-options request-options {:method method, :url url}))))++(defn- write-jsonl-to-stream! [^OutputStream os col-names reducible-rows]+ (let [none? (volatile! true)+ writer (-> os+ (OutputStreamWriter. StandardCharsets/UTF_8)+ (BufferedWriter.))]++ (run! (fn [row]+ (when @none? (vreset! none? false))+ (let [row-map (zipmap col-names row)]+ (json/encode-to row-map writer {})+ (.newLine writer)))+ reducible-rows)++ ;; Workaround for LocalStack, which doesn't support zero byte files.+ (when @none?+ (.write writer " "))++ (doto writer+ (.flush)+ (.close))))++(defn- execute-mbql-query+ [driver db-id query respond cancel-chan]+ (driver/with-driver driver+ (let [native (qp.compile/compile {:type :query, :database db-id :query query})+ query {:database db-id+ :type :native+ :native native}]+ (qp.store/with-metadata-provider db-id+ (binding [qp.pipeline/*canceled-chan* cancel-chan]+ (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))++(defn root-type+ "Supported type for roundtrip/insertion"+ [base-type]+ (when base-type+ (some #(when (isa? base-type %) %)+ [:type/Number+ :type/Date+ :type/DateTime+ :type/Instant+ :type/DateTimeWithTZ+ :type/Text+ :type/Boolean])))++(defn restricted-insert-type+ "Type for insertion restricted to supported"+ [base_type]+ (if base_type+ (let [base-type-kw (keyword "type" base_type)]+ (if (root-type base-type-kw)+ base-type-kw+ :type/Text))+ :type/Text))++(defn- closest-ancestor [t pred]+ (loop [remaining (conj PersistentQueue/EMPTY [t])]+ (when-let [t (first remaining)]+ (if (pred t)+ t+ (recur (into (pop t) (parents t)))))))++(defn- effective-semantic-type-i-think+ "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"+ [{:keys [base_type effective_type semantic_type]}]+ (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))++(defn- generate-manifest+ "Generate a manifest to communicate schema and metadata information around the table."+ [table-id cols-meta]+ {:schema_version 1+ :data_format "jsonl"+ :data_version 1+ :table_metadata {:table_id table-id}+ :fields (mapv (fn [col-meta]+ {:name (:name col-meta)+ :base_type (some-> (:base_type col-meta) name)+ :database_type (some-> (:database_type col-meta) name)+ :root_type (some-> (root-type (:base_type col-meta)) name)+ :semantic_type (some-> (effective-semantic-type-i-think col-meta) name)+ :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)+ :field_id (:id col-meta)})+ cols-meta)})++(defn- maybe-fixup-value [col v]+ (cond+ (nil? (root-type (:base_type col)))+ ;; we're not a supported base type, so we just stringify it+ (when v (json/encode v))++ ;; the clickhouse driver returns bigdecimals for int64 values+ (and (isa? (:base_type col) :type/Integer)+ (or (instance? BigDecimal v)+ (float? v)))+ (bigint v)++ :else+ v))++(defn- write-table-data-to-file! [{:keys [db-id driver table-id fields-meta temp-file cancel-chan]}]+ (let [query {:source-table table-id}]+ (execute-mbql-query driver db-id query+ (fn [{cols-meta :cols} reducible-rows]+ (with-open [os (io/output-stream temp-file)]+ (let [filtered-col-meta (m/index-by :name fields-meta)+ col-names (map :name cols-meta)+ filtered-rows (eduction (map (fn [row]+ (->>+ (map vector col-names row)+ (filter (fn [[n _]]+ (contains? filtered-col-meta n)))+ (map (fn [[n v]]+ (maybe-fixup-value (filtered-col-meta n) v))))))+ reducible-rows)]+ (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))+ cancel-chan)))++(defn get-logs+ "Return the logs of the current running python process"+ [run-id]+ (let [server-url (transforms-python.settings/python-runner-url)]+ (python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))++(defn execute-python-code-http-call!+ "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns+ the response from the server."+ [{:keys [server-url code run-id table-name->id shared-storage]}]+ (let [{:keys [objects]} shared-storage+ {:keys [output output-manifest events]} objects++ url-for-path (fn [path] (:url (get objects path)))+ table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))+ table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))++ payload {:code code+ :library (t2/select-fn->fn :path :source :model/PythonLibrary)+ :timeout (* 30 60) ;; 30 mins harcoded for now+ :request_id run-id+ :output_url (:url output)+ :output_manifest_url (:url output-manifest)+ :events_url (:url events)+ :table_mapping table-name->url+ :manifest_mapping table-name->manifest-url}++ response (transforms.instrumentation/with-python-api-timing [run-id]+ (python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]+ ;; when a 500 is returned we observe a string in the body (despite the python returning json)+ ;; always try to parse the returned string as json before yielding (could tighten this up at some point)+ (update response :body (fn [string-if-error]+ (if (string? string-if-error)+ (try+ (json/decode+kw string-if-error)+ (catch Exception _+ {:error string-if-error}))+ string-if-error)))))++;; temporary, we should not need to realize data/events files into memory longer term+(defn read-output-objects+ "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."+ [{:keys [s3-client bucket-name objects]}]+ (let [{:keys [output output-manifest events]} objects+ output-content (s3/read-to-string s3-client bucket-name (:path output) nil)+ output-manifest-content (s3/read-to-string s3-client bucket-name (:path output-manifest) "{}")+ events-content (s3/read-to-string s3-client bucket-name (:path events))]+ {:output output-content+ :output-manifest (json/decode+kw output-manifest-content)+ :events (mapv json/decode+kw (str/split-lines events-content))}))++(defn cancel-python-code-http-call!+ "Calls the /cancel endpoint of the python runner. Returns immediately."+ [server-url run-id]+ (python-runner-request server-url :post "/cancel" {:body (json/encode {:request_id run-id})+ :async? true}+ #_success #(log/debug %)+ #_failure #(log/error %)))++(defn- safe-delete+ "Safely delete a file."+ [^File file]+ (try (.delete file) (catch Exception _)))++(defn- fields-metadata [_driver table-id]+ (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position]+ :table_id table-id+ :active true+ ;; we are only interested in top-level objects, so filter out nested fields (parent or path)+ :parent_id nil+ :nfc_path nil+ {:order-by [[:database_position :asc]]}))++;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.+(defn copy-tables-to-s3!+ "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).+ Blocks until all tables are fully written and committed to shared storage."+ [{:keys [run-id+ shared-storage+ table-name->id+ cancel-chan]}]+ ;; TODO there's scope for some parallelism here, in particular across different databases+ (doseq [table-id (vals table-name->id)+ :let [{:keys [s3-client bucket-name objects]} shared-storage+ {data-path :path} (get objects [:table table-id :data])+ {manifest-path :path} (get objects [:table table-id :manifest])]]+ (let [tmp-data-file (File/createTempFile data-path "")+ tmp-meta-file (File/createTempFile manifest-path "")]+ (try+ (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)+ driver (t2/select-one-fn :engine :model/Database db-id)+ fields-meta (fields-metadata driver table-id)+ manifest (generate-manifest table-id fields-meta)]++ (transforms.instrumentation/with-stage-timing [run-id :dwh-to-file]+ (write-table-data-to-file!+ {:db-id db-id+ :driver driver+ :table-id table-id+ :fields-meta fields-meta+ :temp-file tmp-data-file+ :cancel-chan cancel-chan}))++ (with-open [writer (io/writer tmp-meta-file)]+ (json/encode-to manifest writer {}))+ (let [data-size (.length tmp-data-file)+ meta-size (.length tmp-meta-file)]+ (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file data-size nil)++ (transforms.instrumentation/with-stage-timing [run-id :file-to-s3]+ (s3/upload-file s3-client bucket-name data-path tmp-data-file)+ (s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))++ (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))+ (finally+ (safe-delete tmp-data-file)+ (safe-delete tmp-meta-file))))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.cljnew file mode 100644index 00000000000..75448593736--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj@@ -0,0 +1,215 @@+(ns metabase-enterprise.transforms-python.s3+ (:require+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase.util.log :as log])+ (:import+ (clojure.lang IDeref)+ (java.io Closeable File)+ (java.net URI)+ (java.time Duration)+ (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)+ (software.amazon.awssdk.auth.credentials AwsCredentialsProvider DefaultCredentialsProvider)+ (software.amazon.awssdk.core.sync RequestBody)+ (software.amazon.awssdk.regions Region)+ (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)+ (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)+ (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)+ (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PutObjectPresignRequest)))++(set! *warn-on-reflection* true)++;; TODO We should make these durations configurable in future.++;; Longer duration for inputs than for outputs, to compensate for the duration of the code execution itself.+(def ^:private ^Duration presigned-get-duration (Duration/ofMinutes 30))+(def ^:private ^Duration presigned-put-duration (Duration/ofHours 5))++;; We should consider namespacing these paths, for example, using the run id to simplify cleanup.+;; Another idea is to have deterministic paths, so that it's easier to resume incomplete runs.+(defn- working-dir-for-run+ "The path within the target bucket within which to store all the files for this run."+ []+ (let [shared-prefix (some-> (transforms-python.settings/python-storage-s-3-prefix) (str "/"))]+ (str shared-prefix "run-" (System/nanoTime) "-" (rand-int 10000))))++(defmacro ^:private maybe-with-endpoint* [builder endpoint]+ `(let [builder# ~builder]+ (when-let [region# (transforms-python.settings/python-storage-s-3-region)]+ (.region builder# (Region/of region#)))+ (when ~endpoint (.endpointOverride builder# (URI/create ~endpoint)))+ builder#))++(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]+ (maybe-with-endpoint* builder endpoint))++(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]+ (maybe-with-endpoint* builder endpoint))++(defn- s3-configuration ^S3Configuration []+ (-> (S3Configuration/builder)+ (.pathStyleAccessEnabled (transforms-python.settings/python-storage-s-3-path-style-access))+ (.build)))++(defn- put-object-request ^PutObjectRequest [^String bucket-name ^String key]+ (-> (PutObjectRequest/builder) (.bucket bucket-name) (.key key) .build))++(defn- get-object-request ^GetObjectRequest [^String bucket-name ^String key]+ (-> (GetObjectRequest/builder) (.bucket bucket-name) (.key key) .build))++(defn- delete-object-request ^DeleteObjectRequest [^String bucket-name ^String key]+ (-> (DeleteObjectRequest/builder) (.bucket bucket-name) (.key key) .build))++(defn- maybe-with-credentials*+ [credentials-provider]+ (let [access-key (transforms-python.settings/python-storage-s-3-access-key)+ secret-key (transforms-python.settings/python-storage-s-3-secret-key)]+ (if (or access-key secret-key)+ (if-not (and access-key secret-key)+ (do (log/warnf "Ignoring %s because %s is not defined"+ (if access-key "access-key" "secret-key")+ (if (not access-key) "access-key" "secret-key"))+ (credentials-provider (DefaultCredentialsProvider/create)))+ (credentials-provider+ (StaticCredentialsProvider/create+ (AwsBasicCredentials/create access-key secret-key))))+ (credentials-provider (DefaultCredentialsProvider/create)))))++(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]+ (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))++(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]+ (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))++;; We just recreate the client every time, to keep things simple if config is changed.+(defn create-s3-client+ "Create S3 client for transferring table data and manifests."+ ^S3Client []+ (.build+ (doto (S3Client/builder)+ (maybe-with-endpoint-s3-client (transforms-python.settings/python-storage-s-3-endpoint))+ maybe-with-credentials-s3-client+ (.serviceConfiguration (s3-configuration)))))++(defn- create-s3-presigner-for-container+ "Create S3 presigner for container operations (presigned URLs). Uses distinct container-endpoint if relevant."+ ^S3Presigner []+ (let [container-endpoint (transforms-python.settings/python-storage-s-3-container-endpoint)+ endpoint (or container-endpoint (transforms-python.settings/python-storage-s-3-endpoint))]+ (.build+ (doto (S3Presigner/builder)+ (maybe-with-endpoint-s3-presigner endpoint)+ maybe-with-credentials-s3-presigner+ (.serviceConfiguration (s3-configuration))))))++(defn- generate-presigned-get-url+ "Generate GET URL using container presigner"+ [^S3Presigner presigner ^String bucket-name ^String key]+ (let [request (-> (GetObjectPresignRequest/builder)+ (.signatureDuration presigned-get-duration)+ (.getObjectRequest (get-object-request bucket-name key))+ (.build))]+ (.toString (.url (.presignGetObject presigner request)))))++(defn- generate-presigned-put-url+ "Generate PUT URL using container presigner"+ [^S3Presigner presigner ^String bucket-name ^String key]+ (let [request (-> (PutObjectPresignRequest/builder)+ (.signatureDuration presigned-put-duration)+ (.putObjectRequest (put-object-request bucket-name key))+ (.build))]+ (.toString (.url (.presignPutObject presigner request)))))++(defn- s3-shared-storage [table-name->id]+ (let [work-dir-name (working-dir-for-run)+ container-presigner (create-s3-presigner-for-container)+ bucket-name (transforms-python.settings/python-storage-s-3-bucket)+ ref (fn [method relative-path]+ (let [path (str work-dir-name "/" relative-path)]+ {:path path+ :method method+ :url (case method+ :put (generate-presigned-put-url container-presigner bucket-name path)+ :get (generate-presigned-get-url container-presigner bucket-name path))}))]+ ;; It feels dirty mixing the S3 client object with the other pure values, but it is convenient for the caller+ {:s3-client (create-s3-client)+ :bucket-name bucket-name+ :objects+ (into+ {:output (ref :put "output.csv")+ :output-manifest (ref :put "output-manifest.json")+ :events (ref :put "events.jsonl")}+ (for [[table-name id] table-name->id]+ {[:table id :manifest] (ref :get (str "table-" (name table-name) "-" id ".manifest.json"))+ [:table id :data] (ref :get (str "table-" (name table-name) "-" id ".jsonl"))}))}))++(declare delete-many)++;; TODO this feels like it's a hair's width from abstracting away s3 versus other transfer mechanisms - go all the way+(defn open-shared-storage!+ "Returns a map wrapped in a deref-able auto-closable, to be used in a with-open.++ The map contains:+ - :s3-client - an S3Client that can be used to upload/download files+ - :bucket-name - the S3 bucket name+ - :objects - a map of keys to {:path :method :url} maps, where:+ - :path is the S3 key+ - :method is either :get or :put, indicating whether the python-runner should fetch or upload the file.+ - :url is the presigned URL to be used by the python-runner to fetch or upload the file.++ The objects are broken into the following transform inputs:++ - [:table id :manifest] - for each input table, the manifest JSON file+ - [:table id :data] - for each input table, the data file, in format specified by the manifest.++ And the following transform outputs:+ - :output-manifest - the output manifest JSON file+ - :output - the output data file, in format specified by the output manifest.+ - :events - a JSONL file containing the events logged during python execution.++ When the value is closed, all the relevant keys will be deleted, but this is best-effort only.+ We rely on the bucket retention policy to ensure any stragglers are eventually deleted."+ ^Closeable [table-name->id]+ (let [shared-storage (s3-shared-storage table-name->id)]+ (reify IDeref+ (deref [_] shared-storage)+ Closeable+ (close [_] (delete-many (:s3-client shared-storage)+ (:bucket-name shared-storage)+ (map :path (vals (:objects shared-storage))))))))++(defn upload-file+ "Upload the given file to s3"+ [^S3Client s3-client ^String bucket-name ^String key ^File file]+ (let [^PutObjectRequest request (put-object-request bucket-name key)]+ (.putObject s3-client request (RequestBody/fromFile file))))++;; TODO optimize our ingestion to stream the data. reading a string will not work in any case for binary files.+(defn read-to-string+ "Get back the contents of the given key as a string."+ ([s3-client bucket-name key] (read-to-string s3-client bucket-name key ::throw))+ ([^S3Client s3-client ^String bucket-name ^String key not-found]+ (try+ (let [^GetObjectRequest request (get-object-request bucket-name key)+ response (.getObject s3-client request)]+ (slurp response))+ (catch NoSuchKeyException e+ (if (identical? ::throw not-found)+ (throw e)+ not-found)))))++(defn delete+ ;; TODO better error handling+ "Delete the given key from s3, ignoring any errors for now"+ [^S3Client s3-client ^String bucket-name ^String key]+ (try+ (.deleteObject s3-client (delete-object-request bucket-name key))+ (catch Exception e+ (log/debugf e "Error deleting s3 object %s" key)+ ;; Ignore deletion errors - object might not exist, or we might not have permissions+ ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted+ nil)))++(defn delete-many+ "Best effort delete the given s3 keys"+ [^S3Client s3-client bucket-name s3-keys]+ (run! (partial delete s3-client bucket-name) s3-keys))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.cljindex 2ae3fa46444..f451e84890e 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj@@ -18,7 +18,7 @@(setting/defsetting python-runner-api-token(deferred-tru "API token for authorizing with the python-runner service."):type :string- :visibility :internal+ :visibility :admin:default (when (not config/is-prod?) "dev-token-12345"):feature :transforms-python:doc false@@ -52,7 +52,7 @@(deferred-tru "S3 bucket name for storing Python execution artifacts."):type :string:visibility :admin- :default (if (not config/is-prod?) "metabase-python-runner" "metabase-cloud-python-runner-staging")+ :default (when (not config/is-prod?) "metabase-python-runner"):feature :transforms-python:doc false:export? false@@ -64,7 +64,7 @@:type :string:visibility :admin:feature :transforms-python- :default (if (not config/is-prod?) "test-prefix" "hosting-4b9351bb22b84791")+ :default (when (not config/is-prod?) "test-prefix"):doc false:export? false:encryption :nodiff --git a/enterprise/backend/test/metabase_enterprise/api/session_test.clj b/enterprise/backend/test/metabase_enterprise/api/session_test.cljindex c7df48de095..e33028e8e31 100644--- a/enterprise/backend/test/metabase_enterprise/api/session_test.clj+++ b/enterprise/backend/test/metabase_enterprise/api/session_test.clj@@ -47,6 +47,7 @@:sso-ldap:sso-saml:transforms+ :transforms-python:upload-management:whitelabel:collection-cleanup@@ -91,6 +92,7 @@:sso_saml true:table_data_editing false:transforms true+ :transforms-python true:upload_management true:whitelabel true:collection_cleanup truediff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex c626c008be4..225a25ea430 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -8,7 +8,7 @@[metabase-enterprise.transforms.models.transform-transform-tag][metabase-enterprise.transforms.query-test-util :as query-test-util][metabase-enterprise.transforms.test-dataset :as transforms-dataset]- [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp]]+ [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp get-test-schema]][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver][metabase.legacy-mbql.normalize :as mbql.normalize]@@ -52,12 +52,6 @@;; Convert to legacy MBQL which the transform API expects(lib.convert/->legacy-MBQL query)))-(defn- get-test-schema- "Get the schema from the products table in the test dataset.- This is needed for databases like BigQuery that require a schema/dataset."- []- (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)))-(deftest create-transform-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/with-premium-features #{:transforms}@@ -73,6 +67,80 @@:schema schema:name table-name}})))))))+(deftest create-transform-feature-flag-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "Creating a query transform without :transforms feature returns 403"+ (mt/with-premium-features #{}+ (mt/dataset transforms-dataset/transforms-test+ (let [query (make-query "Gadget")+ schema (get-test-schema)+ response (mt/user-http-request :crowberto :post 402 "ee/transform"+ {:name "Test Transform"+ :source {:type "query"+ :query query}+ :target {:type "table"+ :schema schema+ :name "test_transform"}})]+ (is (= "error-premium-feature-not-available" (:status response)))))))++ (testing "Creating a query transform with :transforms feature succeeds"+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [table-name "test_query_transform"]+ (let [query (make-query "Gadget")+ schema (get-test-schema)+ response (mt/user-http-request :crowberto :post 200 "ee/transform"+ {:name "Test Transform"+ :source {:type "query"+ :query query}+ :target {:type "table"+ :schema schema+ :name table-name}})]+ (is (some? (:id response))))))))))++(deftest update-transform-feature-flag-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "Updating a query transform requires :transforms feature"+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [table-name "test_update"]+ (let [query (make-query "Gadget")+ schema (get-test-schema)+ transform-payload {:name "Original Transform"+ :source {:type "query"+ :query query}+ :target {:type "table"+ :schema schema+ :name table-name}}+ created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]+ ;; Now test update without feature flag+ (mt/with-premium-features #{}+ (let [response (mt/user-http-request :crowberto :put+ (format "ee/transform/%d" (:id created))+ (assoc transform-payload :name "Updated Transform"))]+ (is (= "error-premium-feature-not-available" (:status response))))))))))))++(deftest run-transform-feature-flag-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "Running a query transform requires :transforms feature"+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [table-name "test_run"]+ (let [query (make-query "Gadget")+ schema (get-test-schema)+ transform-payload {:name "Test Run Transform"+ :source {:type "query"+ :query query}+ :target {:type "table"+ :schema schema+ :name table-name}}+ created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]+ ;; Now test run without feature flag+ (mt/with-premium-features #{}+ (let [response (mt/user-http-request :crowberto :post+ (format "ee/transform/%d/run" (:id created)))]+ (is (= "error-premium-feature-not-available" (:status response))))))))))))+(deftest list-transforms-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(testing "Can list without query parameters"diff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.cljindex 4c3630ca2d8..f7796ce3825 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj@@ -1,11 +1,11 @@(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test(:require[clojure.test :refer :all]- [medley.core :as m][metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.query-test-util :as query-test-util][metabase-enterprise.transforms.test-dataset :as transforms-dataset]- [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup! delete-schema!]]+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! delete-schema!]]+ [metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.lib.core :as lib]@@ -13,6 +13,7 @@[metabase.query-processor :as qp][metabase.sync.core :as sync][metabase.test :as mt]+ [metabase.test.data.sql :as sql.tx][toucan2.core :as t2][toucan2.util :as u]))@@ -34,19 +35,6 @@[mp table-name](->> table-name mt/id (lib.metadata/table mp)))-(defn- wait-for-table- "Wait for a table to appear in metadata, with timeout.- Copied from execute_test.clj - will consolidate later."- [table-name timeout-ms]- (let [mp (mt/metadata-provider)- limit (+ (System/currentTimeMillis) timeout-ms)]- (loop []- (Thread/sleep 200)- (when (> (System/currentTimeMillis) limit)- (throw (ex-info "table has not been created" {:table-name table-name, :timeout-ms timeout-ms})))- (or (m/find-first (comp #{table-name} :name) (lib.metadata/tables mp))- (recur)))))-(deftest execute-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/dataset transforms-dataset/transforms-test@@ -65,14 +53,14 @@:query t1-query}:target target1}](transforms.execute/run-mbql-transform! t1 {:run-method :manual})- (let [table1 (wait-for-table table1-name 10000)+ (let [table1 (transforms.tu/wait-for-table table1-name 10000)t2-query (make-query (->> table1 :name (table-name->qp-table (mt/metadata-provider))) "category" lib/= "Gizmo")](mt/with-temp [:model/Transform t2 {:name "transform2":source {:type :query:query t2-query}:target target2}](transforms.execute/run-mbql-transform! t2 {:run-method :cron})- (let [table2 (wait-for-table table2-name 10000)+ (let [table2 (transforms.tu/wait-for-table table2-name 10000)check-query (lib/aggregate (make-query (->> table2 :name (table-name->qp-table (mt/metadata-provider)))) (lib/count))query-result (qp/process-query check-query)];; The transforms-test dataset has exactly 4 Gizmo products (IDs 6, 8, 12, 14)@@ -123,7 +111,7 @@(doseq [transform [transform-no-limit transform-limit]](transforms.execute/run-mbql-transform! transform {:run-method :manual})(let [table-name (-> transform :target :name)- _ (wait-for-table table-name 10000)+ _ (transforms.tu/wait-for-table table-name 10000)table-result (lib.metadata/table mp (mt/id (keyword table-name)))query-result (->> (lib/query mp table-result)(qp/process-query)@@ -137,6 +125,24 @@["2024-01-23T00:00:00Z" 14.99]]query-result)))))))))))+(deftest create-table-from-schema!-test+ (mt/test-drivers (mt/normal-drivers-with-feature :uploads)+ (let [driver driver/*driver*+ db-id (mt/id)+ table-name (mt/random-name)+ schema-name (sql.tx/session-schema driver)+ table-schema {:name (if schema-name+ (keyword schema-name table-name)+ (keyword table-name))+ :columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "name" :type :type/Text :nullable? true}]}]+ (mt/as-admin+ (testing "create-table-from-schema! should create the table successfully"+ (transforms.util/create-table-from-schema! driver db-id table-schema)+ (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]+ (is (some? table-exists?) "Table should exist in the database schema")+ (driver/drop-table! driver db-id (:name table-schema))))))))+(deftest transform-schema-created-if-needed-test(mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})(mt/dataset transforms-dataset/transforms-test@@ -157,7 +163,8 @@:query query}:target target-table}](transforms.execute/run-mbql-transform! transform {:run-method :manual})- (let [table-result (wait-for-table (:name target-table) 10000)+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)+ table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))query-result (->> (lib/query mp table-result)(qp/process-query)(mt/formatted-rows [int str str 2.0 str])@@ -207,7 +214,7 @@:query query}:target target-table}](transforms.execute/run-mbql-transform! transform {:run-method :manual})- (let [_ (wait-for-table (:name target-table) 10000)+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))query-result (->> (lib/query mp table-result)(qp/process-query)diff --git a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.cljindex a12d860b401..aeee55451f7 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj@@ -1,7 +1,12 @@(ns metabase-enterprise.transforms.jobs-test+ #_{:clj-kondo/ignore [:discouraged-namespace]}(:require[clojure.test :refer :all]- [metabase-enterprise.transforms.jobs :as jobs]))+ [clojure.tools.logging :as log]+ [metabase-enterprise.transforms.execute :as transforms.execute]+ [metabase-enterprise.transforms.jobs :as jobs]+ [metabase-enterprise.transforms.models.transform-run :as transform-run]+ [metabase.test :as mt]))(deftest basic-deps-test(let [ordering {1 #{2 3}@@ -75,3 +80,64 @@(-> (#'jobs/next-transform sorted-ordering transforms-by-id #{2 3}):id)))(is (nil? (#'jobs/next-transform sorted-ordering transforms-by-id #{1 2 3})))))++(def ^:private query-source {:type "query"})+(def ^:private python-source {:type "python"})++(deftest run-transform-feature-flag-test+ (testing "Query transforms are skipped without :transforms feature"+ (mt/with-premium-features #{}+ (let [query-transform {:id 1+ :source query-source+ :name "Test Query Transform"}+ run-id 100+ logged-messages (atom [])]+ (with-redefs [log/log* (fn [_ level _ message]+ (swap! logged-messages conj {:level level :message message}))+ transform-run/running-run-for-transform-id (constantly nil)]+ (#'jobs/run-transform! run-id :scheduled query-transform)+ (is (= 1 (count @logged-messages))+ "Should log exactly one warning")+ (is (= :warn (:level (first @logged-messages)))+ "Should log at warn level")+ (is (re-matches #".*Skip running transform 1 due to lacking premium features.*"+ (:message (first @logged-messages)))+ "Warning message should indicate transform was skipped due to missing features")))))++ (testing "Python transforms are skipped without :transforms-python feature"+ (mt/with-premium-features #{:transforms}+ (let [python-transform {:id 2+ :source python-source+ :name "Test Python Transform"}+ run-id 101+ logged-messages (atom [])]+ (with-redefs [log/log* (fn [_ level _ message]+ (swap! logged-messages conj {:level level :message message}))+ transform-run/running-run-for-transform-id (constantly nil)]+ (#'jobs/run-transform! run-id :scheduled python-transform)+ (is (= 1 (count @logged-messages))+ "Should log exactly one warning")+ (is (= :warn (:level (first @logged-messages)))+ "Should log at warn level")+ (is (re-matches #".*Skip running transform 2 due to lacking premium features.*"+ (:message (first @logged-messages)))+ "Warning message should indicate transform was skipped due to missing features")))))++ (testing "Query transforms run with :transforms feature"+ (mt/with-premium-features #{:transforms}+ (let [query-transform {:id 3+ :source query-source+ :name "Test Query Transform"}+ run-id 102+ logged-messages (atom [])+ run-called? (atom false)]+ (with-redefs [log/log* (fn [_ level _ message]+ (swap! logged-messages conj {:level level :message message}))+ transform-run/running-run-for-transform-id (constantly nil)+ transforms.execute/run-mbql-transform! (fn [_ _]+ (reset! run-called? true))]+ (#'jobs/run-transform! run-id :scheduled query-transform)+ (is (empty? (filter (comp #{:warn} :level) @logged-messages))+ "Should not log warnings when feature is enabled")+ (is @run-called?+ "Should call run-mbql-transform! when feature is enabled"))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex fa74e8c8306..71d110792bc 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -12,7 +12,8 @@(defn- make-transform [query & [name schema]](let [name (or name (mt/random-name))- default-schema (some-> driver/*driver* driver.sql/default-schema)+ default-schema (when (get-method driver.sql/default-schema driver/*driver*)+ (driver.sql/default-schema driver/*driver*))schema (or schema default-schema "public")]{:source {:type :query:query query}diff --git a/enterprise/backend/test/metabase_enterprise/transforms/test_util.clj b/enterprise/backend/test/metabase_enterprise/transforms/test_util.cljindex 91b825732ca..fa04c43a593 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/test_util.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.transforms.test-util(:require[clojure.string :as str]+ [clojure.test :refer :all][metabase.api.common :as api][metabase.driver :as driver][metabase.test :as mt]@@ -53,6 +54,14 @@`(mt/with-model-cleanup [:model/Transform]~@body)))+(defn table-rows+ [table-name]+ (->>+ (mt/rows (mt/process-query {:database (mt/id)+ :query {:source-table (t2/select-one-pk :model/Table :name table-name)}+ :type :query}))+ (map (fn [x] (if (= :mongo driver/*driver*) (rest x) x)))))+(defn parse-timestamp"Parse a local datetime and convert it to a ZonedDateTime in the default timezone."^ZonedDateTime [timestamp-string]@@ -70,6 +79,62 @@^String [timestamp-string](-> timestamp-string parse-instant str))+(defn wait-for-table+ "Wait for a table to appear in metadata, with timeout."+ [^String table-name timeout-ms]+ (let [timer (u/start-timer)]+ (loop []+ (let [table (t2/select-one :model/Table :name table-name)+ fields (t2/select :model/Field :table_id (:id table))]+ (cond+ (and table (seq fields)) table+ (> (u/since-ms timer) timeout-ms)+ (throw (ex-info (format "Table %s did not appear after %dms" table-name timeout-ms)+ {:table-name table-name :timeout-ms timeout-ms}))+ :else (do (Thread/sleep 100)+ (recur)))))))++(defn test-run+ [transform-id]+ (let [resp (mt/user-http-request :crowberto :post 202 (format "ee/transform/%s/run" transform-id))+ timeout-s 10 ; 10 seconds is our timeout to finish execution and sync+ limit (+ (System/currentTimeMillis) (* timeout-s 1000))]+ (is (=? {:message "Transform run started"}+ resp))+ (loop [last-resp nil]+ (when (> (System/currentTimeMillis) limit)+ (throw (ex-info (str "Transform run timed out after " timeout-s " seconds") {:resp last-resp})))+ (let [resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" transform-id))+ status (some-> resp :last_run :status keyword)]+ (when-not (contains? #{:started :succeeded} status)+ (throw (ex-info (str "Transform run failed with status " status) {:resp resp :status status})))+ (when-not (some? (:table resp))+ (Thread/sleep 100)+ (recur resp))))))++(defn wait-for-transform-completion+ "Wait for a transform run to complete without triggering a new run.+ Polls the transform status until it succeeds or times out."+ [transform-id timeout-ms]+ (let [start-time (u/start-timer)]+ (loop []+ (when (> (u/since-ms start-time) timeout-ms)+ (throw (ex-info (format "Transform %d did not complete after %dms" transform-id timeout-ms)+ {:transform-id transform-id :timeout-ms timeout-ms})))+ (let [resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" transform-id))+ status (some-> resp :last_run :status keyword)]+ (case status+ :succeeded resp+ (:started :running) (do (Thread/sleep 100) (recur))+ (throw (ex-info (format "Transform run failed with status %s" status)+ {:resp resp :status status})))))))++(defn get-test-schema+ "Get the schema from the products table in the test dataset.+ This is needed for databases like BigQuery that require a schema/dataset."+ []+ (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)))+(defmulti delete-schema!"Deletes a schema."{:arglists '([driver db schema])}diff --git a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/util_test.cljnew file mode 100644index 00000000000..af55f2d8e8d--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj@@ -0,0 +1,64 @@+(ns ^:mb/driver-tests metabase-enterprise.transforms.util-test+ "Tests for transform utility functions."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.transforms.util :as transforms.util]+ [metabase.driver :as driver]+ [metabase.test :as mt]+ [metabase.test.data.sql :as sql.tx]))++(set! *warn-on-reflection* true)++(deftest temp-table-name-test+ (testing "temp-table-name generates valid table names respecting driver limits"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (let [driver driver/*driver*]++ (testing "Basic table name generation"+ (let [result (transforms.util/temp-table-name driver :base_table "test")]+ (is (keyword? result))+ (is (nil? (namespace result)))+ (is (str/starts-with? (name result) "base_table_test_"))))++ (testing "Table name with very large base name respects driver limits"+ (let [very-long-base-name (keyword (apply str (repeat 200 "a")))+ result (transforms.util/temp-table-name driver very-long-base-name "test")+ max-len (or (driver/table-name-length-limit driver) Integer/MAX_VALUE)+ actual-name (name result)]+ (is (keyword? result))+ (is (<= (count actual-name) max-len))+ (is (re-find #"_test_\d+$" actual-name))))++ (testing "Table name preserves namespace when present"+ (let [result (transforms.util/temp-table-name driver :schema/base_table "test")]+ (is (= "schema" (namespace result)))+ (is (str/starts-with? (name result) "base_table_test_"))))))))++(deftest temp-table-name-creates-table-test+ (testing "temp-table-name produces names that can actually create tables"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (let [driver driver/*driver*+ db-id (mt/id)++ long-base-name (keyword (apply str (repeat 1000 "test")))++ table-name (transforms.util/temp-table-name driver long-base-name "temp")+ schema-name (when (get-method sql.tx/session-schema driver)+ (sql.tx/session-schema driver))+ qualified-table-name (if schema-name+ (keyword schema-name (name table-name))+ table-name)+ column-definitions {"id" (driver/type->database-type driver :type/Integer)}]+ (mt/as-admin+ (try+ (testing "Can create table with generated temp name"+ (driver/create-table! driver db-id qualified-table-name column-definitions {})+ (when-not (= driver :mongo) ;; mongo doesn't actually create tables+ (is (driver/table-exists? driver (mt/db) {:schema schema-name :name (name table-name)}))))+ (finally+ (try+ (driver/drop-table! driver db-id qualified-table-name)+ (catch Exception _e+ ;; Ignore cleanup errors+ nil)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.cljnew file mode 100644index 00000000000..0a155546169--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj@@ -0,0 +1,528 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.drivers-test+ "Comprehensive tests for Python transforms across all supported drivers with all base and exotic types."+ (:require+ [clojure.core.async :as a]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.transforms-python.execute :as transforms.execute]+ [metabase-enterprise.transforms-python.python-runner :as python-runner]+ [metabase-enterprise.transforms-python.s3 :as s3]+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]+ [metabase-enterprise.transforms.util :as transforms.util]+ [metabase.driver :as driver]+ [metabase.driver.mysql :as mysql]+ [metabase.sync.core :as sync]+ [metabase.test :as mt]+ [metabase.test.data.interface :as tx]+ [metabase.test.data.sql :as sql.tx]+ [metabase.util :as u]+ [metabase.util.date-2 :as u.date]+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def test-id 42)++(defn- execute!+ "Execute a Python transform with the given code and tables"+ [{:keys [code tables]}]+ (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]+ (let [server-url (transforms-python.settings/python-runner-url)+ cancel-chan (a/promise-chan)+ table-name->id (or tables {})+ _ (python-runner/copy-tables-to-s3! {:run-id test-id+ :shared-storage @shared-storage-ref+ :table-name->id table-name->id+ :cancel-chan cancel-chan})+ response (python-runner/execute-python-code-http-call! {:server-url server-url+ :code code+ :run-id test-id+ :table-name->id table-name->id+ :shared-storage @shared-storage-ref})+ {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]+ (merge (:body response)+ {:output output+ :output-manifest output-manifest+ :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))+ :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))++(defn- execute-e2e-transform!+ "Execute an e2e Python transform test using execute-python-transform!"+ [table-name transform-code source-tables]+ (let [schema (when (#{:postgres :bigquery-cloud-sdk} driver/*driver*)+ (sql.tx/session-schema driver/*driver*))+ target {:type "table"+ :schema schema+ :name table-name+ :database (mt/id)}+ transform-def {:name (str "E2E Transform Test " table-name)+ :source {:type "python"+ :source-tables source-tables+ :body transform-code}+ :target target}]+ (with-transform-cleanup! [_target target]+ (mt/with-temp [:model/Transform transform transform-def]+ (transforms.execute/execute-python-transform! transform {:run-method :manual})+ (let [table (transforms.tu/wait-for-table table-name 10000)+ columns (t2/select :model/Field :table_id (:id table) {:order-by [:position]})+ column-names (filterv (fn [x] (not= x "_id")) ;; for mongo+ (map :name columns))+ rows (transforms.tu/table-rows table-name)]+ {:columns column-names+ :rows rows})))))++(def ^:private base-type-test-data+ "Base types that all drivers should support with test data."+ {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "name" :type :type/Text :nullable? true}+ {:name "price" :type :type/Float :nullable? true}+ {:name "active" :type :type/Boolean :nullable? true}+ {:name "created_date" :type :type/Date :nullable? true}+ {:name "created_at" :type :type/DateTime :nullable? true}]+ :data [[1 "Product A" 19.99 true "2024-01-01" "2024-01-01T12:00:00"]+ [2 "Product B" 15.50 false "2024-02-01" "2024-02-01T09:15:30"]+ [3 nil nil nil nil nil]]})++(def ^:private driver-exotic-types+ "Driver-specific exotic types with test data."+ {:postgres {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "bigint" :type :type/BigInteger :nullable? false}+ {:name "created_tz" :type :type/DateTimeWithTZ :nullable? true}+ {:name "uuid_field" :type :type/UUID :nullable? true}+ {:name "json_field" :type :type/JSON :nullable? true}+ {:name "ip_field" :type :type/IPAddress :nullable? true}+ {:name "int_array" :type :type/* :nullable? true :database-type "integer[]"}+ {:name "text_array" :type :type/* :nullable? true :database-type "text[]"}+ {:name "uuid_array" :type :type/* :nullable? true :database-type "uuid[]"}]+ :data [[1 -9223372036854775808 "2024-01-01T12:00:00Z" "550e8400-e29b-41d4-a716-446655440000" "{\"key\": \"value\"}" "192.168.1.1"+ "{1,2,3,4,5}" "{\"hello\",\"world\",\"test\"}" "{550e8400-e29b-41d4-a716-446655440000,123e4567-e89b-12d3-a456-426614174000}"]+ [2 0 nil nil nil nil nil nil nil]+ [3 nil "2024-02-01T09:15:30-05:00" nil nil nil "{}" "{}" "{}"]]}++ :mysql {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "json_field" :type :type/JSON :nullable? true}+ {:name "timestamp" :type :type/DateTimeWithLocalTZ :nullable? true :database-type "timestamp"}]+ :data [[1 "{\"key\": \"value\"}" "2024-01-01 12:00:00"]+ [2 nil+ nil]]}+ :mariadb {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "json_field" :type :type/JSON :nullable? true}]+ :data [[1 "{\"key\": \"mariadb_value\"}"]+ [2 nil]]}++ :bigquery-cloud-sdk {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "json_field" :type :type/JSON :nullable? true}+ {:name "dict_field" :type :type/Dictionary :nullable? true :database-type "STRUCT<key STRING, value INT64>"}]+ :data [[1 "{\"key\": \"value\"}" {"key" "test", "value" 42}]+ [2 nil nil]]}+ :sqlserver {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "uuid_field" :type :type/UUID :nullable? true}+ {:name "datetimeoffset_field" :type :type/DateTimeWithTZ :nullable? true :database-type "datetimeoffset"}]+ :data [[1 "550e8400-e29b-41d4-a716-446655440000" "2024-01-01 12:00:00 -05:00"]+ [2 nil nil]]}+ :mongo {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "uuid_field" :type :type/UUID :nullable? true}+ {:name "array_field" :type :type/Array :nullable? true :database-type "array"}+ {:name "dict_field" :type :type/Dictionary :nullable? true :database-type "object"}]+ :data [[1 #uuid "550e8400-e29b-41d4-a716-446655440000" [1, 2, 3] {"nested" "object"}]+ [2 nil nil nil]]}})++(defn- create-test-table-with-data!+ "Create a test table with the given schema and data for the current driver."+ [table-name schema data]+ (let [driver driver/*driver*+ db-id (mt/id)+ schema-name (case driver++ :clickhouse (-> (mt/db) :details :db)++ :mongo nil++ (sql.tx/session-schema driver))+ qualified-table-name (if schema-name+ (keyword schema-name table-name)+ (keyword table-name))+ table-schema {:name qualified-table-name+ :columns (:columns schema)}]+ (mt/as-admin+ (transforms.util/create-table-from-schema! driver db-id table-schema))++ (when (seq data)+ (driver/insert-from-source! driver db-id table-schema+ {:type :rows+ :data (map (fn [row]+ (map #(cond+ (and (= :sqlserver driver) (boolean? %))+ (if % 1 0)++ (and (= :mongo driver) (string? %) (try (u.date/parse %) (catch Exception _)))+ (u.date/parse %)++ :else+ %) row))+ data)}))++ (sync/sync-database! (mt/db) {:scan :schema})++ (t2/select-one-pk :model/Table :name (name qualified-table-name) :db_id db-id)))++(defn- cleanup-table!+ "Drop the test table by table ID."+ [table-id]+ (try+ (when-let [table (t2/select-one :model/Table :id table-id)]+ (let [table-name (:name table)+ qualified-name (case driver/*driver*+ :bigquery-cloud-sdk+ (let [schema (sql.tx/session-schema driver/*driver*)]+ (if schema+ (keyword (str schema "." table-name))+ (keyword table-name)))++ (let [schema (sql.tx/session-schema driver/*driver*)]+ (if schema+ (keyword schema table-name)+ (keyword table-name))))]+ (driver/drop-table! driver/*driver* (mt/id) qualified-name)))+ (catch Exception _e+ nil)))++(defn- validate-transform-output+ "Validate that the Python transform output preserves types and data correctly,+ assuming output is in JSONL format."+ [result expected-columns expected-row-count]+ (testing "Transform execution succeeded"+ (is (some? result))+ (is (contains? result :output))+ (is (contains? result :output-manifest)))++ (when result+ (let [lines (str/split-lines (:output result))+ rows (map json/decode lines)+ metadata (:output-manifest result)+ headers (map :name (:fields metadata))]++ (testing "Column headers are correct"+ (is (= (set expected-columns) (disj (set headers) "_id"))))++ (testing "Row count is correct"+ (is (= expected-row-count (count rows))))++ (testing "Metadata contains all expected columns"+ (is (= (set expected-columns)+ (disj (set (map :name (:fields metadata))) "_id"))))++ {:headers headers+ :rows rows+ :metadata metadata})))++#_:clj-kondo/ignore+(defmacro with-test-table+ [[table-id table-name] [schema data] & body]+ `(let [table-name# (if (= :redshift driver/*driver*)+ (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))+ (mt/random-name))+ table-id# (create-test-table-with-data! table-name# ~schema ~data)]+ (try+ (let [~table-id table-id#+ ~table-name table-name#]+ ~@body)+ (finally+ (cleanup-table! table-id#)))))++(defn- simple-identity-transform-code+ "Generate Python code for a simple identity transform."+ [table-name]+ (str "import pandas as pd\n"+ "\n"+ "def transform(" table-name "):\n"+ " df = " table-name ".copy()\n"+ " return df"))++(defn- execute-and-validate-transform!+ "Execute a Python transform and validate the output, returning validation results."+ [transform-code table-name table-id expected-columns expected-row-count]+ (let [result (execute! {:code transform-code+ :tables {table-name table-id}})]+ (validate-transform-output result expected-columns expected-row-count)))++(defn- test-exotic-types-for-driver!+ "Helper to test exotic types for the current driver."+ [driver-key]+ (when-let [exotic-config (get driver-exotic-types (if (mysql/mariadb? (mt/db)) :mariadb driver-key))]+ (with-test-table [table-id table-name] [exotic-config (:data exotic-config)]+ (let [transform-code (simple-identity-transform-code table-name)+ expected-columns (map :name (:columns exotic-config))+ expected-row-count (count (:data exotic-config))+ validation (execute-and-validate-transform!+ transform-code table-name table-id+ expected-columns expected-row-count)]++ (when validation+ (testing (str "Exotic types for " driver-key)+ (let [{:keys [metadata]} validation+ type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]+ [name (python-runner/restricted-insert-type base_type)])]++ (is (isa? :type/Integer (type-map "id")))++ (case driver-key+ :postgres (do+ (is (isa? (type-map "bigint") :type/BigInteger))+ (is (isa? (type-map "uuid_field") :type/Text #_:type/UUID))+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))+ (is (isa? (type-map "ip_field") :type/Text #_:type/IPAddress)))+ :mysql (if (mysql/mariadb? (mt/db))+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))+ (do+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))+ (is (isa? (type-map "timestamp") :type/DateTimeWithLocalTZ))))++ :bigquery-cloud-sdk (do+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))+ (is (isa? (type-map "dict_field") :type/Text #_:type/JSON)))++ :mongo (do+ (is (isa? (type-map "array_field") :type/Text #_:type/Array))+ (is (isa? (type-map "dict_field") :type/Text #_:type/Dictionary)))++ :sqlserver (do+ (is (isa? (type-map "uuid_field") :type/Text #_:type/UUID))+ (is (isa? (type-map "datetimeoffset_field") :type/DateTimeWithTZ)))))))+ validation))))++(deftest create-table-test+ (testing "Test we can create base table"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (mt/with-empty-db+ (with-test-table [table-id _table-name] [base-type-test-data (:data base-type-test-data)]+ (is table-id "Table should be created and have an ID"))))))++(deftest base-types-python-transform-test+ (testing "Test Python transforms with base types across all supported drivers"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (mt/with-empty-db+ (with-test-table [table-id table-name] [base-type-test-data (:data base-type-test-data)]+ (let [transform-code (simple-identity-transform-code table-name)+ expected-columns ["id" "name" "price" "active" "created_date" "created_at"]+ validation (execute-and-validate-transform!+ transform-code table-name table-id expected-columns 3)]++ (when validation+ (let [{:keys [metadata]} validation]+ (testing "Base type preservation"+ (let [type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]+ [name (python-runner/restricted-insert-type base_type)])]+ (is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))+ (is (isa? (type-map "name") :type/Text))+ (is (isa? (type-map "price") :type/Float))+ (is (isa? (type-map "active") :type/Boolean))+ (is (isa? (type-map "created_date") (if (= driver/*driver* :mongo) :type/Instant :type/Date)))+ (is (isa? (type-map "created_at") :type/DateTime))))))))))))++(deftest exotic-types-python-transform-test+ (testing "Test Python transforms with driver-specific exotic types"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (mt/with-empty-db+ (test-exotic-types-for-driver! driver/*driver*)))))++(deftest edge-cases-python-transform-test+ (testing "Test Python transforms with edge cases: null values, empty strings, extreme values"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (mt/with-empty-db+ (let [edge-case-schema {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "text_field" :type :type/Text :nullable? true}+ {:name "int_field" :type :type/Integer :nullable? true}+ {:name "float_field" :type :type/Float :nullable? true}+ {:name "bool_field" :type :type/Boolean :nullable? true}+ {:name "date_field" :type :type/Date :nullable? true}]+ :data [[1 "" 0 0.0 false "2024-01-01"]+ [2 "Very long text with special chars: !@#$%^&*(){}[]|\\:;\"'<>,.?/~`"+ 2147483647 1.79769313486 true "2222-12-31"]+ [3 nil nil nil nil nil]]}]+ (with-test-table [table-id table-name] [edge-case-schema (:data edge-case-schema)]+ (let [transform-code (str "import pandas as pd\n"+ "import numpy as np\n"+ "\n"+ "def transform(" table-name "):\n"+ " df = " table-name ".copy()\n"+ " \n"+ " # Handle text operations safely\n"+ " df['text_length'] = df['text_field'].fillna(\"\").astype(str).str.len()"+ " \n"+ " # Handle numeric operations with null safety\n"+ " df['int_doubled'] = df['int_field'] * 2\n"+ " df['float_squared'] = df['float_field'] ** 2\n"+ " \n"+ " # Boolean operations\n"+ " df['bool_inverted'] = ~df['bool_field'].fillna(False)\n"+ " \n"+ " return df")+ expected-columns ["id" "text_field" "int_field" "float_field" "bool_field" "date_field"+ "text_length" "int_doubled" "float_squared" "bool_inverted"]+ validation (execute-and-validate-transform!+ transform-code table-name table-id expected-columns 3)]++ (when validation+ (let [{:keys [rows metadata]} validation+ type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]+ [name (python-runner/restricted-insert-type base_type)])]++ (testing "Original columns preserved"+ (is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))+ (is (isa? (type-map "text_field") :type/Text))+ (is (isa? (type-map "int_field") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))+ (is (isa? (type-map "float_field") :type/Float))+ (is (isa? (type-map "bool_field") :type/Boolean))+ (is (isa? (type-map "date_field") (if (= driver/*driver* :mongo) :type/Instant :type/Date))))++ (testing "Computed columns have correct types"+ (is (isa? (type-map "text_length") :type/Integer))+ (is (isa? (type-map "int_doubled") :type/Integer))+ (is (isa? (type-map "float_squared") :type/Float))+ (is (isa? (type-map "bool_inverted") :type/Boolean)))++ (testing "Edge case data handling"+ (let [[row1 row2 row3] rows]+ (is (= 1 (get row1 "id")))+ (is (= 0 (get row1 "text_length"))) ; empty string length+ (is (= 0 (get row1 "int_doubled"))) ; 0 * 2++ (is (= 2 (get row2 "id")))+ (is (not= "" (get row2 "text_length"))) ; long string has length++ (is (= 3 (get row3 "id"))))))))))))))++(deftest idempotent-transform-test+ (testing "Test that running the same transform multiple times produces identical results"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (mt/with-empty-db+ (with-test-table [table-id table-name] [base-type-test-data (:data base-type-test-data)]+ (let [transform-code (str "import pandas as pd\n"+ "\n"+ "def transform(" table-name "):\n"+ " df = " table-name ".copy()\n"+ " df['computed_field'] = df['price'] * 1.1 # 10% markup\n"+ " df['name_upper'] = df['name'].str.upper()\n"+ " return df")+ ;; Run the transform twice+ result1 (execute!+ {:code transform-code+ :tables {table-name table-id}})++ result2 (execute!+ {:code transform-code+ :tables {table-name table-id}})]+ (testing "Both transforms succeeded"+ (is (some? result1))+ (is (some? result2)))++ (when (and result1 result2)+ (testing "Results are identical"+ (is (= (:output result1) (:output result2)))+ (is (= (count (:fields (:output-manifest result1)))+ (count (:fields (:output-manifest result2))))))++ (let [expected-columns ["id" "name" "price" "active" "created_date" "created_at"+ "computed_field" "name_upper"]+ validation (validate-transform-output result1 expected-columns 3)]++ (when validation+ (testing "Computed columns are added correctly"+ (let [{:keys [metadata]} validation+ type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]+ [name (python-runner/restricted-insert-type base_type)])]+ (is (= :type/Float (type-map "computed_field")))+ (is (= :type/Text (type-map "name_upper"))))))))))))))++(deftest comprehensive-e2e-python-transform-test+ (testing "End-to-end test using execute-python-transform! across all supported drivers with comprehensive type coverage"+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/python)+ ;; we sometimes get I/O error in CI due to it taking too long. it's too slow, too flakey to keep enabled+ :redshift)+ (mt/with-empty-db+ (mt/with-premium-features #{:transforms-python}+ (with-test-table [source-table-id source-table-name] [base-type-test-data (:data base-type-test-data)]+ (let [table-name (mt/random-name)+ exotic-config (get driver-exotic-types driver/*driver*)+ exotic-table-id (when exotic-config+ (create-test-table-with-data!+ (str source-table-name "_exotic")+ exotic-config+ (:data exotic-config)))]++ (try+ (let [transform-code (str "import pandas as pd\n"+ "\n"+ "def transform(" source-table-name+ (when exotic-config (str ", " source-table-name "_exotic"))+ "):\n"+ " # Start with base types table\n"+ " df = " source-table-name ".copy()\n"+ " \n"+ " # Add computed columns to test type handling\n"+ " df['price_with_tax'] = df['price'] * 1.08 # Float computation\n"+ " df['name_length'] = df['name'].str.len() # String operations\n"+ " df['is_expensive'] = df['price'] > 18.0 # Boolean computation\n"+ " \n"+ " # Date operations\n"+ " df['created_year'] = pd.to_datetime(df['created_date']).dt.year\n"+ " \n"+ (when exotic-config+ (str " # Merge with exotic types if available\n"+ " exotic_df = " source-table-name "_exotic.copy()\n"+ " df = df.merge(exotic_df, on='id', how='left', suffixes=('', '_exotic'))\n"+ " df.attrs[\"source_metadata\"] = {**df.attrs.get(\"source_metadata\", {}), **exotic_df.attrs.get(\"source_metadata\", {})}"+ " \n"))+ " # Return processed dataframe\n"+ " return df")++ source-tables (cond-> {source-table-name source-table-id}+ exotic-table-id+ (assoc (str source-table-name "_exotic") exotic-table-id))++ result (execute-e2e-transform! table-name transform-code source-tables)+ {:keys [columns] result-rows :rows} result]++ (testing "E2E transform execution succeeded"+ (is (seq result-rows) "Transform should produce results"))++ (testing "Expected data transformations"+ (let [first-row (first result-rows)+ second-row (second result-rows)+ third-row (nth result-rows 2)+ get-column (fn [row col-name] (get (zipmap columns row) col-name))]+ (testing "Computed columns are present and have reasonable values"+ (is (> (count columns) 7) "Should have additional computed columns")+ (is (= 3 (count result-rows)) "Should have 3 rows from source data"))++ (testing "Computed column values are mathematically correct"+ (testing "Row 1 price_with_tax calculation"+ (let [price-with-tax (get-column first-row "price_with_tax")]+ (is (and (number? price-with-tax) (> price-with-tax 21.5) (< price-with-tax 21.7))+ "First row price_with_tax should be ~21.59")))++ (testing "Row 2 price_with_tax calculation"+ (let [price-with-tax (get-column second-row "price_with_tax")]+ (is (and (number? price-with-tax) (> price-with-tax 16.7) (< price-with-tax 16.8))+ "Second row price_with_tax should be ~16.74")))++ (testing "Name length calculations"+ (is (== 9 (get-column first-row "name_length")) "First row name_length should be 9")+ (is (== 9 (get-column second-row "name_length")) "Second row name_length should be 9"))++ (testing "Boolean expense calculations"+ (is (true? (get-column first-row "is_expensive")) "First row is_expensive should be true")+ (is (false? (get-column second-row "is_expensive")) "Second row is_expensive should be false"))++ (testing "Date year extraction"+ (is (== 2024 (get-column first-row "created_year")) "First row created_year should be 2024")+ (is (== 2024 (get-column second-row "created_year")) "Second row created_year should be 2024"))++ (testing "Null value handling"+ (is (seq third-row) "Third row should contain values (nulls handled gracefully)"))))))+ (finally+ (when exotic-table-id+ (cleanup-table! exotic-table-id)))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.cljnew file mode 100644index 00000000000..55a55151371--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj@@ -0,0 +1,86 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.e2e-test+ "End-to-end tests for Python library functionality."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.transforms-python.models.python-library :as python-library]+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]+ [metabase.driver :as driver]+ [metabase.test :as mt]+ [metabase.test.util :as mt.util]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(deftest transforms-python-with-library-test+ (testing "Python transform execution with common library"+ (mt/test-drivers #{:postgres}+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (mt.util/with-discard-model-updates! [:model/PythonLibrary]+ ;; Create or update the python library+ (python-library/update-python-library-source! "common"+ (str "def multiply_by_two(x):\n"+ " return x * 2\n"+ "\n"+ "def add_greeting(name):\n"+ " return f'Hello, {name}!'\n"))+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ ;; Create and run a transform that uses the library+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"+ :schema schema+ :name "library_test_output"}]+ (let [transform-body (str "import pandas as pd\n"+ "from common import multiply_by_two, add_greeting\n"+ "\n"+ "def transform():\n"+ " data = {\n"+ " 'number': [1, 2, 3, 4, 5],\n"+ " 'doubled': [multiply_by_two(x) for x in [1, 2, 3, 4, 5]],\n"+ " 'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],\n"+ " 'greeting': [add_greeting(name) for name in ['Alice', 'Bob', 'Charlie', 'David', 'Eve']]\n"+ " }\n"+ " return pd.DataFrame(data)")+ transform-payload {:name "Library Test Transform"+ :source {:type "python"+ :source-database (mt/id)+ :source-tables {}+ :body transform-body}+ :target (assoc target :database (mt/id))}+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"+ transform-payload)]+ (transforms.tu/test-run transform-id)+ (transforms.tu/wait-for-table table-name 5000)+ (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))++ (let [rows (transforms.tu/table-rows table-name)]+ (is (= 5 (count rows)))+ (is (every? #(= (* 2 (first %)) (second %)) rows))+ (is (= ["Hello, Alice!" "Hello, Bob!" "Hello, Charlie!" "Hello, David!" "Hello, Eve!"]+ (map #(nth % 3) rows)))))))))))))++(deftest execute-transforms-python-test+ (testing "transform execution with :transforms/table target"+ (mt/test-drivers #{:postgres}+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"+ :schema schema+ :name "target_table"}]+ (let [original {:name "Gadget Products"+ :source {:type "python"+ :source-database (mt/id)+ :source-tables {"transforms_customers" (mt/id :transforms_customers)}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}+ :target (assoc target :database (mt/id))}+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"+ original)]+ (transforms.tu/test-run transform-id)+ (transforms.tu/wait-for-table table-name 5000)+ (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))+ (is (= [["Alice" 25] ["Bob" 30]]+ (transforms.tu/table-rows table-name)))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.cljnew file mode 100644index 00000000000..4da89bb4cb4--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj@@ -0,0 +1,97 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.execute-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]+ [metabase-enterprise.transforms.util :as transforms.util]+ [metabase.test :as mt]+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent CountDownLatch)))++(set! *warn-on-reflection* true)++(deftest atomic-python-transform-swap-test+ (testing "Python transform execution with atomic table swap"+ (mt/test-drivers #{:mysql :postgres}+ (mt/with-premium-features #{:transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"+ :schema schema+ :name "swap_tbl"}]+ (let [initial-transform {:name "Python Transform Initial"+ :source {:type "python"+ :source-tables {}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}+ :target (assoc target :database (mt/id))}]+ (mt/with-temp [:model/Transform transform initial-transform]+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})+ (transforms.tu/wait-for-table table-name 10000)++ (let [initial-rows (transforms.tu/table-rows table-name)]+ (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")++ (t2/update! :model/Transform (:id transform)+ {:source {:type "python"+ :source-tables {}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))++ (let [swap-latch (CountDownLatch. 1)+ original-rename-tables-atomic! transforms.util/rename-tables!]+ (with-redefs [transforms.util/rename-tables! (fn [driver db-id rename-pairs]+ (.await swap-latch)+ (original-rename-tables-atomic! driver db-id rename-pairs))]+ (let [transform-future (future+ (transforms-python.execute/execute-python-transform!+ (t2/select-one :model/Transform (:id transform))+ {:run-method :manual}))]+ (is (= [["Alice" 25] ["Bob" 30]]+ (transforms.tu/table-rows table-name))+ "Original data should still be accessible during transform")+ (.countDown swap-latch)+ @transform-future+ (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]+ (transforms.tu/table-rows table-name))+ "Table should now contain Charlie, Diana, and Eve")))))))))))))++(deftest python-transform-temp-table-cleanup-test+ (testing "Python transform cleans up temp tables on success"+ (mt/test-drivers #{:mysql :postgres}+ (mt/with-premium-features #{:transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"+ :schema schema+ :name "cleanup_"}]+ (let [transform-def {:name "Python Transform Cleanup"+ :source {:type "python"+ :source-tables {}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}+ :target (assoc target :database (mt/id))}]+ (mt/with-temp [:model/Transform transform transform-def]+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})+ (transforms.tu/wait-for-table table-name 10000)++ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})++ (let [db-id (mt/id)+ tables (t2/select :model/Table :db_id db-id :active true)+ new-table-pattern (re-pattern (str ".*" table-name "_" transforms-python.execute/temp-table-suffix-new "_.*"))+ old-table-pattern (re-pattern (str ".*" table-name "_" transforms-python.execute/temp-table-suffix-old "_.*"))]+ (is (not-any? #(or (re-matches new-table-pattern (:name %))+ (re-matches old-table-pattern (:name %))) tables)+ (str "No temp tables (_" transforms-python.execute/temp-table-suffix-new "_ or _" transforms-python.execute/temp-table-suffix-old "_) should remain after successful Python transform"))++ (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))+ "Table should contain the expected data after swap")))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.cljnew file mode 100644index 00000000000..6bca42e7ddd--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj@@ -0,0 +1,46 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.job-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.transforms.schedule :as transforms.schedule]+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]+ [metabase-enterprise.transforms.test-util :as transforms.tu]+ [metabase.driver :as driver]+ [metabase.task.core :as task]+ [metabase.test :as mt]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(deftest python-transform-scheduled-job-test+ (mt/test-helpers-set-global-values!+ (mt/with-temp-scheduler!+ (task/init! ::transforms.schedule/RunTransform)+ (mt/test-drivers #{:postgres}+ (mt/with-premium-features #{:transforms-python :transforms}+ (mt/dataset transforms-dataset/transforms-test+ (transforms.tu/with-transform-cleanup! [target {:type "table"+ :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ :name "target_table"}]+ (mt/with-temp+ [:model/TransformTag {tag-id :id} {:name "every second"}+ :model/Transform {transform-id :id} {:name "Gadget Products"+ :source {:type "python"+ :source-database (mt/id)+ :source-tables {"transforms_customers" (mt/id :transforms_customers)}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}+ :target (assoc target :database (mt/id))}+ :model/TransformTransformTag _ {:transform_id transform-id :tag_id tag-id :position 0}+ :model/TransformJob {job-id :id :as job} {:schedule "* * * * * ? *"}+ :model/TransformJobTransformTag _ {:job_id job-id :tag_id tag-id :position 0}]+ (transforms.schedule/initialize-job! job)+ (transforms.schedule/update-job! job-id "* * * * * ? *")+ (is (true? (u/poll {:thunk (fn [] (driver/table-exists? driver/*driver* (mt/db) target))+ :done? true?+ :timeout-ms 20000+ :interval-ms 1000})))+ (is (true? (u/poll {:thunk (fn [] (t2/exists? :model/Table :name (:name target)))+ :done? true?+ :timeout-ms 10000+ :interval-ms 1000})))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljnew file mode 100644index 00000000000..fabddc94569--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -0,0 +1,110 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.ordering-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.transforms.ordering :as ordering]+ [metabase.driver :as driver]+ [metabase.driver.sql :as driver.sql]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(defn- make-transform [query & [name schema]]+ (let [name (or name (mt/random-name))+ default-schema (when (get-method driver.sql/default-schema driver/*driver*)+ (driver.sql/default-schema driver/*driver*))+ schema (or schema default-schema "public")]+ {:source {:type :query+ :query query}+ :name (str "transform_" name)+ :target {:schema schema+ :name name+ :type :table}}))++(defn- make-python-transform [source-tables & [name schema target-db]]+ (let [name (or name (mt/random-name))+ default-schema (when (get-method driver.sql/default-schema driver/*driver*)+ (driver.sql/default-schema driver/*driver*))+ schema (or schema default-schema "public")+ target-db (or target-db (mt/id))]+ {:source {:type "python"+ :source-tables source-tables+ :body "# Python code here\nresult = df"}+ :name (str "python_transform_" name)+ :target {:database target-db+ :schema schema+ :name name+ :type "table"}}))++(defn- transform-deps-for-db [transform]+ (mt/with-metadata-provider (mt/id)+ (#'ordering/transform-deps transform)))++(deftest python-transform-basic-dependencies-test+ (testing "Python transforms with source-tables dependencies are extracted correctly"+ (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)+ "products" (mt/id :products)})]+ (is (= #{{:table (mt/id :orders)}+ {:table (mt/id :products)}}+ (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))++(deftest python-transform-ordering-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (testing "Python transforms are ordered correctly based on source-tables dependencies"+ (mt/with-temp [:model/Table {table1 :id} {:schema "public"+ :name "output_1"}+ :model/Field _ {:table_id table1+ :name "foo"}+ :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")+ :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]+ (is (= {t1 #{}+ t2 #{t1}}+ (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))++(deftest python-transform-multiple-dependencies-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (testing "Python transforms with multiple table dependencies"+ (mt/with-temp [:model/Table {table1 :id} {:schema "public"+ :name "output_1"}+ :model/Field _ {:table_id table1+ :name "foo"}+ :model/Table {table2 :id} {:schema "public"+ :name "output_2"}+ :model/Field _ {:table_id table2+ :name "bar"}+ :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")+ :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")+ :model/Transform {t3 :id} (make-python-transform {"output_1" table1+ "output_2" table2} "final_output")]+ (is (= {t1 #{}+ t2 #{}+ t3 #{t1 t2}}+ (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))++(deftest mixed-transform-ordering-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python :transforms/table)+ (testing "Python and query transforms are ordered together correctly"+ (mt/with-temp [:model/Table {table1 :id} {:schema "public"+ :name "sql_output"}+ :model/Field _ {:table_id table1+ :name "foo"}+ :model/Table {table2 :id} {:schema "public"+ :name "python_output"}+ :model/Field _ {:table_id table2+ :name "bar"}+ ;; SQL transform that depends on orders table+ :model/Transform {t1 :id} (make-transform+ {:database (mt/id),+ :type "query",+ :query {:source-table (mt/id :orders)}}+ "sql_output")+ ;; Python transform that depends on the SQL transform's output+ :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")+ ;; Another SQL transform that depends on the Python transform's output+ :model/Transform {t3 :id} (make-transform+ {:database (mt/id)+ :type "query"+ :query {:source-table table2}}+ "final_output")]+ (is (= {t1 #{}+ t2 #{t1}+ t3 #{t2}}+ (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.cljnew file mode 100644index 00000000000..52c8e04a810--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj@@ -0,0 +1,495 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.python-runner-test+ (:require+ [clojure.core.async :as a]+ [clojure.java.jdbc :as jdbc]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.transforms-python.python-runner :as python-runner]+ [metabase-enterprise.transforms-python.s3 :as s3]+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase-enterprise.transforms.util :as transforms.util]+ [metabase.driver :as driver]+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]+ [metabase.sync.core :as sync]+ [metabase.test :as mt]+ [metabase.test.data.interface :as tx]+ [metabase.test.data.sql :as sql.tx]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private last-job-run-id (atom 0))++(defn next-job-run-id [] (swap! last-job-run-id inc))++(defn- parse-jsonl [s] (map json/decode+kw (str/split-lines s)))++(defn template->regex+ "Convert a template string with $var$ placeholders to a regex pattern.+ Example: template->regex 'File ___PATH___/script.py, line ___LINE___'+ will match 'File /tmp/unique-path/script.py, line 123'"+ [template]+ (-> template+ (str/replace #"(?m)^\d{13}\s*" "")+ (str/replace "\\" "\\\\")+ (str/replace "." "\\.")+ (str/replace "*" "\\*")+ (str/replace "+" "\\+")+ (str/replace "?" "\\?")+ (str/replace "^" "\\^")+ (str/replace "$" "\\$")+ (str/replace "{" "\\{")+ (str/replace "}" "\\}")+ (str/replace "(" "\\(")+ (str/replace ")" "\\)")+ (str/replace "|" "\\|")+ (str/replace "[" "\\[")+ (str/replace "]" "\\]")+ (str/replace #"___PATH___" "(/[\\\\w-]+)+")+ (str/replace #"___LINE___" "\\\\d+")+ re-pattern))++(defn- jsonl-output [expected] #(= expected (parse-jsonl %)))++(defn- execute! [{:keys [code tables]}]+ (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]+ (let [server-url (transforms-python.settings/python-runner-url)+ cancel-chan (a/promise-chan)+ table-name->id (or tables {})+ test-id (next-job-run-id)+ _ (python-runner/copy-tables-to-s3! {:run-id test-id+ :shared-storage @shared-storage-ref+ :table-name->id table-name->id+ :cancel-chan cancel-chan})+ response (python-runner/execute-python-code-http-call! {:server-url server-url+ :code code+ :run-id test-id+ :table-name->id table-name->id+ :shared-storage @shared-storage-ref})+ {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]+ ;; not sure about munging this all together but its what tests expect for now+ (merge (:body response)+ {:output output+ :output-manifest output-manifest+ :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))+ :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))++(defn ok-stdout [num-rows num-cols]+ (format (str "Successfully saved %d rows to S3\n"+ "Successfully saved output manifest with %d fields to S3")+ num-rows num-cols))++(deftest ^:parallel transform-function-basic-test+ (testing "executes transform function and returns CSV output"+ (let [transform-code (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")+ result (execute! {:code transform-code})]+ (is (=? {:output (jsonl-output [{:name "Alice", :age 25}+ {:name "Bob", :age 30}])+ :stdout (ok-stdout 2 2)}+ result)))))++(deftest ^:parallel transform-function-missing-test+ (testing "handles missing transform function"+ (let [result (execute! {:code (str "import pandas as pd\n"+ "\n"+ "# No transform function defined")})]+ (is (=? {:exit_code 1+ :stderr "ERROR: User script must define a 'transform()' function"+ :stdout ""}+ result)))))++(deftest ^:parallel transform-function-wrong-return-type-test+ (testing "handles transform function returning non-DataFrame"+ (let [result (execute! {:code (str "def transform():\n"+ " return 'not a dataframe'")})]+ (is (=? {:exit_code 1+ :stderr "ERROR: Transform function must return a pandas DataFrame, got <class 'str'>"+ :stdout ""}+ result)))))++(deftest ^:parallel transform-function-error-test+ (testing "handles transform function with error"+ (let [result (execute! {:code (str "def transform():\n"+ " raise ValueError('Something went wrong')")})+ expected-template (str "ERROR: Transform function failed: Something went wrong\n"+ "Traceback (most recent call last):\n"+ " File \"___PATH___/transform_runner.py\", line ___LINE___, in main\n"+ " result = script.transform()\n"+ " ^^^^^^^^^^^^^^^^^^\n"+ " File \"___PATH___/script.py\", line 2, in transform\n"+ " raise ValueError('Something went wrong')\n"+ "ValueError: Something went wrong")+ stderr-pattern (template->regex expected-template)]+ (is (=? {:exit_code 1+ :stderr #(re-matches stderr-pattern %)+ :stdout ""+ :timeout false}+ result)))))++(deftest ^:parallel transform-function-complex-dataframe-test+ (testing "can create complex DataFrames with transform"+ (let [transform-code (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " data = {'x': [1, 2, 3], 'y': [10, 20, 30], 'z': ['a', 'b', 'c']}\n"+ " return pd.DataFrame(data)")+ result (execute! {:code transform-code})]+ (is (=? {:output (jsonl-output [{:x 1, :y 10, :z "a"}+ {:x 2, :y 20, :z "b"}+ {:x 3, :y 30, :z "c"}])+ :stdout (ok-stdout 3 3)}+ result)))))++(deftest ^:parallel transform-function-with-db-parameter-test+ (mt/test-drivers #{:postgres}+ (testing "transform function can accept db parameter for forward compatibility"+ (let [transform-code (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " data = {'name': ['Charlie', 'Dana'], 'score': [85, 92]}\n"+ " return pd.DataFrame(data)")+ result (execute! {:code transform-code})]+ (is (=? {:output (jsonl-output [{:name "Charlie", :score 85}+ {:name "Dana", :score 92}])+ :stdout (ok-stdout 2 2)}+ result))))))++(deftest transform-function-with-pass-thru+ (testing "transform function successfully connects to PostgreSQL database and reads data"+ (mt/test-drivers #{:postgres}+ (mt/with-empty-db+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])+ _ (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])+ _ (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])++ _ (sync/sync-database! (mt/db))++ transform-code (str "import pandas as pd\n"+ "\n"+ "def transform(students):\n"+ " return students")+ result (execute! {:code transform-code+ :tables {"students" (mt/id :students)}})]++ (is (=? {:output (jsonl-output [{:id 1 :name "Alice" :score 85}+ {:id 2 :name "Bob" :score 92}+ {:id 3 :name "Charlie" :score 88}+ {:id 4 :name "Dana" :score 90}])+ :output-manifest {:fields [{:base_type "Integer",+ :database_type "int4",+ :effective_type "Integer",+ :name "id",+ :root_type "Number",+ :semantic_type "PK"}+ {:base_type "Text",+ :database_type "varchar",+ :effective_type "Text",+ :name "name",+ :root_type "Text",+ :semantic_type "Name"}+ {:base_type "Integer",+ :database_type "int4",+ :effective_type "Integer",+ :name "score",+ :root_type "Number",+ :semantic_type "Score"}],+ :source_metadata {:fields [{:base_type "Integer",+ :database_type "int4",+ :effective_type "Integer",+ :name "id",+ :root_type "Number",+ :semantic_type "PK"}+ {:base_type "Text",+ :database_type "varchar",+ :effective_type "Text",+ :name "name",+ :root_type "Text",+ :semantic_type "Name"}+ {:base_type "Integer",+ :database_type "int4",+ :effective_type "Integer",+ :name "score",+ :root_type "Number",+ :semantic_type "Score"}],+ :table_metadata {:table_id (mt/malli=? int?)},+ :schema_version 1+ :data_format "jsonl"+ :data_version 1}+ :schema_version 1+ :data_format "jsonl"+ :data_version 1}+ :stdout (ok-stdout 4 3)}+ result)))))))++(deftest transform-function-with-empty-table-test+ (testing "transform function successfully connects to PostgreSQL database and reads data"+ (mt/test-drivers #{:postgres}+ (mt/with-empty-db+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])+ _ (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])++ _ (sync/sync-database! (mt/db))++ transform-code (str "import pandas as pd\n"+ "\n"+ "def transform(students):\n"+ " # Calculate average score\n"+ " avg_score = students['score'].mean()\n"+ " result = pd.DataFrame({\n"+ " 'student_count': [len(students)],\n"+ " 'average_score': [round(avg_score, 2)]\n"+ " })\n"+ " return result")+ result (execute! {:code transform-code+ :tables {"students" (mt/id :students)}})]++ (is (=? {:output "{\"student_count\":0,\"average_score\":null}\n"+ :output-manifest {:schema_version 1+ :data_format "jsonl"+ :data_version 1+ :fields [{:name "student_count", :base_type "Integer"}+ {:name "average_score", :base_type "Float"}]}+ :stdout (ok-stdout 1 2)}+ result)))))))++(deftest transform-function-with-working-database-test+ (testing "transform function successfully connects to PostgreSQL database and reads data"+ (mt/test-drivers #{:postgres}+ (mt/with-empty-db+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])+ _ (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])+ _ (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])++ _ (sync/sync-database! (mt/db))++ transform-code (str "import pandas as pd\n"+ "\n"+ "def transform(students):\n"+ " # Calculate average score\n"+ " avg_score = students['score'].mean()\n"+ " result = pd.DataFrame({\n"+ " 'student_count': [len(students)],\n"+ " 'average_score': [round(avg_score, 2)]\n"+ " })\n"+ " return result")+ result (execute! {:code transform-code+ :tables {"students" (mt/id :students)}})]++ (is (=? {:output "{\"student_count\":4,\"average_score\":88.75}\n"+ :output-manifest {:schema_version 1+ :data_format "jsonl"+ :data_version 1+ :fields [{:name "student_count", :base_type "Integer"}+ {:name "average_score", :base_type "Float"}]}+ :stdout (ok-stdout 1 2)}+ result)))))))++(defn- datetime-equal?+ [expected-iso-str actual-pandas-str]+ (let [expected-dt (t/zoned-date-time expected-iso-str)+ actual-dt (t/zoned-date-time actual-pandas-str)]+ (= (t/to-millis-from-epoch expected-dt)+ (t/to-millis-from-epoch actual-dt))))++(deftest transform-type-test+ (mt/test-drivers #{:postgres}+ (mt/with-empty-db+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS sample_table"])+ _ (jdbc/execute! db-spec ["CREATE TABLE sample_table (+ id BIGSERIAL PRIMARY KEY,+ name VARCHAR(255),+ description TEXT,+ count BIGINT,+ price FLOAT,+ is_active BOOLEAN,+ created_date DATE,+ updated_at TIMESTAMP,+ scheduled_for TIMESTAMP WITH TIME ZONE+ )"])+ _ (jdbc/execute! db-spec ["INSERT INTO sample_table+ (name, description, count, price, is_active, created_date, updated_at, scheduled_for)+ VALUES+ ('Product A', 'A sample product description', 100, 29.99, true, '2024-01-15', '2024-01-15 10:30:00', '2024-01-16 14:00:00+00'),+ ('Product B', 'Another product with longer description text', 50, 15.50, false, '2024-02-01', '2024-02-01 09:15:30', '2024-02-02 16:30:00-05'),+ ('Product C', NULL, 0, 0.0, true, '2024-03-10', '2024-03-10 18:45:15', '2024-03-11 08:00:00+02')"])++ _ (sync/sync-database! (mt/db))++ transform-code (str "import pandas as pd\n"+ "\n"+ "def transform(sample_table):\n"+ " df = sample_table.copy()\n"+ " return df")+ result (execute! {:code transform-code+ :tables {"sample_table" (mt/id :sample_table)}})+ rows (parse-jsonl (:output result))+ headers (map name (keys (first rows)))+ [row1 row2 row3] rows+ get-col (fn [row col-name] (get row (keyword col-name)))+ metadata (:output-manifest result)]++ (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])+ (set headers)))++ (is (= 1 (get-col row1 "id")))+ (is (= "Product A" (get-col row1 "name")))+ (is (datetime-equal? "2024-01-16T14:00:00Z" (get-col row1 "scheduled_for")))++ (is (= 2 (get-col row2 "id")))+ (is (= "Product B" (get-col row2 "name")))+ (is (datetime-equal? "2024-02-02T21:30:00Z" (get-col row2 "scheduled_for")))++ (is (= 3 (get-col row3 "id")))+ (is (= "Product C" (get-col row3 "name")))+ (is (datetime-equal? "2024-03-11T06:00:00Z" (get-col row3 "scheduled_for")))+ (testing "types are preserved correctly"+ (is (= {"id" :type/BigInteger+ "name" :type/Text+ "description" :type/Text+ "count" :type/BigInteger+ "price" :type/Float+ "is_active" :type/Boolean+ ;; Our hack works+ "created_date" :type/Date+ "updated_at" :type/DateTime+ "scheduled_for" :type/DateTimeWithLocalTZ}+ (u/for-map [{:keys [name base_type]} (:fields metadata)]+ [name (python-runner/restricted-insert-type base_type)]))))))))++(deftest transform-function-with-library-test+ (testing "transform function can use libraries"+ (mt/test-drivers #{:postgres}+ (mt/with-temp [:model/PythonLibrary _ {:path "circle"+ :source "import math\n\ndef calculate_circle_area(radius):\n return math.pi * radius ** 2"}+ :model/PythonLibrary _ {:path "utils"+ :source "def format_currency(amount):\n return f\"${amount:,.2f}\""}]+ (let [transform-code (str "import pandas as pd\n"+ "from circle import calculate_circle_area\n"+ "from utils import format_currency\n"+ "\n"+ "def transform():\n"+ " data = [\n"+ " {'radius': 5, 'area': calculate_circle_area(5), 'price': format_currency(78.54)},\n"+ " {'radius': 10, 'area': calculate_circle_area(10), 'price': format_currency(314.16)}\n"+ " ]\n"+ " return pd.DataFrame(data)")+ result (execute! {:code transform-code})]+ (is (=? {:output (jsonl-output [{:radius 5, :area 78.5398163397, :price "$78.54"}+ {:radius 10, :area 314.159265359, :price "$314.16"}])+ :stdout (ok-stdout 2 3)+ #_#_:stderr ""}+ result)))))))++(deftest transform-function-without-libraries-test+ (testing "transform function works when no libraries exist"+ (mt/test-drivers #{:postgres}+ (with-redefs [t2/select-fn->fn (fn [k v model]+ (when (and (= k :path)+ (= v :source)+ (= model :model/PythonLibrary))+ {}))]+ (let [transform-code (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'status': ['ok']})")+ result (execute! {:code transform-code})]+ (is (=? {:output (jsonl-output [{:status "ok"}])+ :stdout (ok-stdout 1 1)}+ result)))))))++(deftest transform-function-library-import-error-test+ (testing "transform function handles missing library gracefully"+ (mt/test-drivers #{:postgres}+ (with-redefs [t2/select-fn->fn (fn [k v model]+ (when (and (= k :path)+ (= v :source)+ (= model :model/PythonLibrary))+ {"utils" "def helper():\n return 42"}))]+ (let [transform-code (str "import pandas as pd\n"+ "from common import some_function # This library doesn't exist\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'value': [some_function()]})")+ result (execute! {:code transform-code})]+ ;; TODO this error message could still be improved a lot+ (is (=? {#_#_:error "Execution failed"+ :exit_code 1+ :stderr #(str/includes? % "No module named 'common'")}+ result)))))))++(deftest transform-type-roundtrip-test+ (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}+ (mt/with-empty-db+ (let [driver driver/*driver*+ db-id (mt/id)+ table-name (if (= :redshift driver)+ (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))+ (mt/random-name))+ schema-name (if (= :clickhouse driver)+ (-> (mt/db) :details :db)+ (sql.tx/session-schema driver))+ qualified-table-name (if schema-name+ (keyword schema-name table-name)+ (keyword table-name))+ table-schema {:name qualified-table-name+ :columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "price" :type :type/Float :nullable? true}+ {:name "active" :type :type/Boolean :nullable? true}+ {:name "created_tz" :type :type/DateTimeWithTZ :nullable? true}+ {:name "created_at" :type :type/DateTime :nullable? true}+ {:name "created_date" :type :type/Date :nullable? true}+ {:name "description" :type :type/Text :nullable? true}]}+ _ (mt/as-admin+ (transforms.util/create-table-from-schema! driver db-id table-schema))++ row-values [[1+ 19.99+ (if (= driver/*driver* :sqlserver) 1 true)+ (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00Z")+ (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00")+ "2024-01-01"+ "Sample product"]]+ _ (driver/insert-from-source! driver db-id table-schema+ {:type :rows :data row-values})++ _ (sync/sync-database! (mt/db) {:scan :schema})++ transform-code (str "import pandas as pd\n"+ "\n"+ "def transform(" table-name "):\n"+ " return " table-name)+ result (execute! {:code transform-code+ :tables {table-name (mt/id qualified-table-name)}})++ metadata (:output-manifest result)]++ (testing "All expected columns are present"+ (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}+ (set (map :name (:fields metadata))))))++ (testing "types are preserved correctly"+ (is (= {"id" (if (= :snowflake driver) :type/Number :type/Integer)+ "price" :type/Float+ "active" :type/Boolean+ "created_tz" (case driver+ :snowflake :type/DateTimeWithTZ+ :sqlserver :type/DateTimeWithZoneOffset+ :type/DateTimeWithLocalTZ)+ "created_at" :type/DateTime+ "created_date" :type/Date+ "description" :type/Text}+ (u/for-map [{:keys [name base_type]} (:fields metadata)]+ [name (python-runner/restricted-insert-type base_type)]))))++ ;; cleanup+ (driver/drop-table! driver db-id qualified-table-name)))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.cljnew file mode 100644index 00000000000..eb3f7539fe6--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj@@ -0,0 +1,97 @@+;; This is a driver test currently to benefit from the localstack S3 setup in driver tests.+;; Perhaps we should create a new testing category?+;; This code otherwise runs in the appdb section, but it does not touch the appdb.+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.s3-test+ "Tests for S3 operations in transforms-python module."+ (:require+ [clj-http.client :as http]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.transforms-python.s3 :as s3]+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase.test :as mt]+ [metabase.util.log :as log])+ (:import+ (java.net ConnectException)+ (java.nio.file Files)+ (java.nio.file.attribute FileAttribute)+ (software.amazon.awssdk.services.s3 S3Client)+ (software.amazon.awssdk.services.s3.model NoSuchKeyException)))++(set! *warn-on-reflection* true)++(comment+ (remove-ns (ns-name *ns*)))++(defn- s3-endpoint-running? []+ (try+ (http/head (transforms-python.settings/python-runner-url) {:throw-exceptions false})+ (catch ConnectException _+ false)))++;; TODO remove this once CI situation is resolved+(use-fixtures :each (fn [thunk]+ (if (s3-endpoint-running?)+ (thunk)+ (log/warn "Skipping S3 tests because localstack isn't running"))))++(deftest s3-read-write-test+ (testing "We can open an s3 connection, and read and write things"+ (with-open [s3-client (s3/create-s3-client)]+ (is (instance? S3Client s3-client))+ (let [bucket (transforms-python.settings/python-storage-s-3-bucket)+ key (str "test-object-" (random-uuid) ".txt")+ body (str "Hello, S3! My secret is:" (random-uuid))]++ (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))+ (is (= :default (s3/read-to-string s3-client bucket key :default)))++ (let [tmp-file (Files/createTempFile "s3-test" ".txt" (into-array FileAttribute []))]+ (try+ (spit (.toFile tmp-file) body)++ (is (s3/upload-file s3-client bucket key (.toFile tmp-file)))++ (is (= body (s3/read-to-string s3-client bucket key)))+ (is (= body (s3/read-to-string s3-client bucket key :default)))++ (s3/delete s3-client bucket key)++ (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))+ (is (= :default (s3/read-to-string s3-client bucket key :default)))++ (finally+ (Files/deleteIfExists tmp-file))))))))++(deftest open-s3-shared-storage-test+ (testing "open-s3-shared-storage! returns closeable derefable"+ (mt/with-premium-features #{:transforms-python}+ (let [table-name->id {"users" 1}+ storage-ref (s3/open-shared-storage! table-name->id)+ {:keys [s3-client bucket-name objects]} @storage-ref]+ (testing "can be dereferenced"+ (doseq [[k {:keys [method path url]}] objects+ :let [url (str/replace url "localstack" "localhost")]]+ (testing (format "%s (%s)" k method)+ (is (= :not-created (s3/read-to-string s3-client bucket-name path :not-created)))+ (let [content (str (random-uuid))]+ (case method+ ;; Files that we should write, and python-runner should fetch+ :get (let [tmp-file (Files/createTempFile "s3-test" ".txt" (into-array FileAttribute []))]+ (try+ (spit (.toFile tmp-file) content)+ (is (s3/upload-file s3-client bucket-name path (.toFile tmp-file)))+ (finally+ (Files/deleteIfExists tmp-file)))+ (is (= content (s3/read-to-string s3-client bucket-name path :not-created)))+ (is (= content (:body (http/get url)))))+ ;; Files that python-runner should post, and we should fetch+ :put (do+ (is (http/put url {:body content}))+ (is (= content (s3/read-to-string s3-client bucket-name path :not-created)))))))))++ (testing "closing the ref deletes the files"+ (.close storage-ref)+ (doseq [[k {:keys [path]}] objects]+ (testing (str k)+ (is (= :not-created (s3/read-to-string s3-client bucket-name path :not-created))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljnew file mode 100644index 00000000000..77e8db1bf51--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -0,0 +1,367 @@+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.transforms-api-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! get-test-schema]]+ [metabase.driver :as driver]+ [metabase.test :as mt]+ [toucan2.core :as t2])+ (:import+ (clojure.lang IDeref)+ (java.io Closeable)+ (java.time Duration)))++(set! *warn-on-reflection* true)++(deftest create-python-transform-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/dataset transforms-dataset/transforms-test+ (letfn [(create-transform! []+ (let [schema (get-test-schema)+ transform-payload {:name "My beautiful python runner"+ :source {:type "python"+ :body "print('hello world')"+ :source-tables {}}+ :target {:type "table"+ :schema schema+ :name "gadget_products"+ :database (mt/id)}}]+ (mt/user-http-request :crowberto :post "ee/transform"+ transform-payload)))]++ (testing "without any feature flags"+ (mt/with-premium-features #{}+ (testing "creating python transform without any features fails"+ (is (= "error-premium-feature-not-available"+ (:status (mt/user-http-request :crowberto :post 402 "ee/transform"+ {:name "My beautiful python runner"+ :source {:type "python"+ :body "print('hello world')"+ :source-tables {}}+ :target {:type "table"+ :schema (get-test-schema)+ :name "gadget_products"+ :database (mt/id)}})))))))++ (testing "with only transforms feature flag (no transforms-python)"+ (mt/with-premium-features #{:transforms}+ (testing "creating python transform without transforms-python feature fails"+ (is (= "Premium features required for this transform type are not enabled."+ (mt/user-http-request :crowberto :post 402 "ee/transform"+ {:name "My beautiful python runner"+ :source {:type "python"+ :body "print('hello world')"+ :source-tables {}}+ :target {:type "table"+ :schema (get-test-schema)+ :name "gadget_products"+ :database (mt/id)}}))))))++ (testing "with transforms-python feature flag"+ (mt/with-premium-features #{:transforms :transforms-python}+ (with-transform-cleanup! [table-name "gadget_products"]+ (let [transform (create-transform!)]+ (is (= "print('hello chris')"+ (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id transform))+ {:name "My beautiful python runner"+ :source {:type "python"+ :body "print('hello chris')"+ :source-tables {}}+ :target {:type "table"+ :schema (get-test-schema)+ :name table-name+ :database (mt/id)}})+ :source :body)))))))))))++(deftest update-python-transform-feature-flag-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "Updating a python transform requires both :transforms and :transforms-python features"+ (mt/with-temp [:model/Transform {id :id+ :as transform} {:name "Original Python Transform"+ :source {:type "python"+ :body "print('original')"+ :source-tables {}}+ :target {:type "table"+ :schema "scheam"+ :name "table"+ :database (mt/id)}}]+ (mt/with-premium-features #{}+ (let [response (mt/user-http-request :crowberto :put+ (format "ee/transform/%d" id)+ (assoc-in transform [:source :body] "print('no features')"))]+ (is (= "error-premium-feature-not-available" (:status response))+ "Should return 403 without any features")))))))++(deftest run-python-transform-feature-flag-test+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (testing "Running a python transform requires both :transforms and :transforms-python features"+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [table-name "test_run_python"]+ (let [schema (get-test-schema)+ transform-payload {:name "Test Run Python Transform"+ :source {:type "python"+ :body "def transform():\n pass"+ :source-tables {}}+ :target {:type "table"+ :schema schema+ :name table-name+ :database (mt/id)}}+ created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]+ (mt/with-premium-features #{}+ (let [response (mt/user-http-request :crowberto :post 402+ (format "ee/transform/%d/run" (:id created)))]+ (is (= "error-premium-feature-not-available" (:status response)))))+ (mt/with-premium-features #{:transforms}+ (let [response (mt/user-http-request :crowberto :post+ (format "ee/transform/%d/run" (:id created)))]+ (is (= "Premium features required for this transform type are not enabled." response)+ "Should return 403 without :transforms-python feature"))))))))))++(deftest execute-python-transform-test+ (testing "transform execution with :transforms/table target"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"+ :schema schema+ :name "target_table"}]+ (let [original {:name "Gadget Products"+ :source {:type "python"+ :source-database (mt/id)+ :source-tables {"transforms_customers" (mt/id :transforms_customers)}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}+ :target (assoc target :database (mt/id))}+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform" original)]+ (transforms.tu/test-run transform-id)+ (transforms.tu/wait-for-table table-name 5000)+ (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))+ (is (= [["Alice" 25] ["Bob" 30]]+ (transforms.tu/table-rows table-name)))))))))))++(defn- subsequence?+ "Returns true if sequence ys is a subsequence of xs:+ every element of ys appears in xs in the same order,+ though not necessarily contiguously."+ [xs ys]+ (cond+ (empty? ys) true+ (empty? xs) false+ (= (first xs) (first ys)) (recur (rest xs) (rest ys))+ :else (recur (rest xs) ys)))++(deftest python-transform-logging-test+ (letfn [(program->source [program]+ (->> (concat ["import pandas as pd"+ "def transform():"]+ (for [s program] (str " " s))+ [" return pd.DataFrame({'x': [42]})"])+ (str/join "\n")))++ (create-transform [{:keys [program]} target]+ {:post [(integer? %)]}+ (:id (mt/user-http-request :crowberto :post 200 "ee/transform"+ {:name "Python logging test"+ :source {:type "python"+ :body (program->source program)+ :source-tables {}}+ :target (assoc target :database (mt/id))})))++ (block-on-run [{:keys [expect-status]} target transform-id]+ (try+ (transforms.tu/test-run transform-id)+ (catch Throwable e+ ;; test-run throws for non success, but we want to test failures too+ (when-not (= expect-status (:status (ex-data e)))+ (throw e))))+ (when (= :succeeded expect-status)+ (transforms.tu/wait-for-table (:name target) 5000)))++ (get-last-run [transform-id]+ (:last_run (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" transform-id))))++ (open-message-value-observer [transform-id]+ (let [states (atom [])+ fut (future+ (try+ (loop []+ (let [{:keys [message]} (get-last-run transform-id)]+ (cond+ (.isInterrupted (Thread/currentThread))+ nil++ ;; same message as last time+ (= message (peek @states))+ (recur)++ ;; new message value+ :else (do (swap! states conj message)+ (recur)))))+ (catch InterruptedException _ nil)))]+ (reify IDeref+ (deref [_] @states)+ Closeable+ (close [_]+ (future-cancel fut)+ (assert (not= :timeout (try (deref fut 1000 :timeout) (catch Throwable _))) "Observation thread did not exit!")))))++ (run-scenario [scenario schema]+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO+ transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]+ (fn [& _] (throw e))+ @#'transforms-python.execute/transfer-file-to-db)]+ (with-transform-cleanup! [target {:type "table"+ :schema schema+ :name "result"}]+ (let [transform-id (create-transform scenario target)+ observed-messages (with-open [observer ^Closeable (open-message-value-observer transform-id)]+ (block-on-run scenario target transform-id)+ @observer)+ last-run (get-last-run transform-id)]+ {:observed-messages observed-messages+ :last-run last-run}))))]+ (let [scenarios [{:desc "stdin"+ :program ["print(\"hello, world\")"]+ :expect-status :succeeded+ :expected ["hello, world"]}+ {:desc "stderr"+ :program ["import sys" "print(\"hello, world\", file=sys.stderr)"]+ :expect-status :succeeded+ :expected ["hello, world"]}+ {:desc "interleaved streams"+ :program ["import sys"+ "print(\"1\", file=sys.stderr)"+ "print(\"2\", file=sys.stderr)"+ "print(\"3\", file=sys.stdout)"+ "print(\"4\", file=sys.stderr)"]+ :expect-status :succeeded+ :expected ["1" "2" "3" "4"]}+ {:desc "syntax error"+ :program ["print(40 + 2)"+ "this is not valid code"]+ :expect-status :failed+ :expected ["SyntaxError: invalid syntax"]}+ {:desc "takes time, early feedback possible"+ :program ["import time"+ "print(\"a\")"+ "time.sleep(0.1)"+ "print(\"b\")"+ "time.sleep(0.1)"+ "print(\"c\")"]+ :expect-early-feedback true+ :expect-status :succeeded+ :expected ["a" "b" "c"]}+ {:desc "crash during writeback"+ :program ["print(42)" "print(\"is the answer\")"]+ :expect-status :failed+ :expected ["42" "is the answer" "Failed to create the resulting table"]+ :writeback-ex (Exception. "Boom!")}]]+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]+ (doseq [{:keys [expected+ expect-early-feedback+ expect-status+ desc] :as scenario}+ scenarios]+ (testing desc+ (let [{:keys [last-run observed-messages]} (run-scenario scenario schema)+ {:keys [status message]} last-run]+ (testing "sanity: status is what we expect"+ (is (= expect-status (some-> status keyword))))+ (testing "sanity: message has a value"+ (is message))+ (when message+ (is (str/starts-with? message "Executing Python transform"))+ (testing "all observed values of message reflect a prefix of the final message (ordering consistent)"+ (doseq [observed-message observed-messages]+ ;; hmmm, sometimes the order seems flipped+ (is (str/starts-with? message observed-message))))+ (when expect-early-feedback+ (testing "scenario takes time, we should see partial messages for immediate feedback"+ (is (< 1 (count observed-messages)))))+ (testing "message includes the expected lines"+ (is (subsequence? (str/split-lines message) expected))))))))))))))++(deftest get-python-transform-with-different-target-database-test+ (testing "GET /api/ee/transform/:id correctly fetches target table from different database"+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/with-temp [:model/Database target-db {:engine :h2+ :details {:db "mem:target-db"}}+ :model/Table target-table {:db_id (:id target-db)+ :schema "PUBLIC"+ :name "python_target_table"+ :active true}+ :model/Transform transform {:name "Python Transform Cross DB"+ :source {:type "python"+ :source-database (mt/id)+ :source-tables {}+ :body "def transform():\n pass"}+ :target {:type "table"+ :schema "PUBLIC"+ :name "python_target_table"+ :database (:id target-db)}}]+ (let [response (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id transform)))]+ (is (=? {:id (:id target-table)+ :name "python_target_table"+ :schema "PUBLIC"+ :db_id (:id target-db)}+ (:table response))))))))++(deftest python-transform-schema-change-integration-test+ (testing "Python transform handles schema changes using appropriate rename strategy"+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/python)+ ;; takes too long on CI+ :bigquery-cloud-sdk)+ (mt/with-premium-features #{:transforms :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)]+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"+ :schema schema+ :name "schema_change_test"}]++ (let [initial-transform {:name "Schema Change Integration Test"+ :source {:type "python"+ :source-tables {}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}+ :target (assoc target :database (mt/id))}+ ;; Create initial transform via API+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform" initial-transform)]++ ;; Run initial transform and validate+ (transforms.tu/test-run transform-id)+ (transforms.tu/wait-for-table table-name 10000)+ (let [initial-rows (transforms.tu/table-rows table-name)]+ (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob with ages"))++ ;; Update transform with different schema via API endpoint+ (let [updated-transform (assoc initial-transform+ :source {:type "python"+ :source-tables {}+ :body (str "import pandas as pd\n"+ "\n"+ "def transform():\n"+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'friend': ['Bob', 'Alice']})")})+ update-response (mt/user-http-request :crowberto :put 200 (format "ee/transform/%d" transform-id)+ updated-transform)]+ (is (some? update-response) "Transform update should succeed"))++ ;; Run updated transform and validate schema change+ (transforms.tu/test-run transform-id)+ (transforms.tu/wait-for-transform-completion transform-id 10000)++ ;; hmmm, looks like QP needs a bit more time to update metadata+ (Thread/sleep 2000)+ (let [updated-rows (transforms.tu/table-rows table-name)]+ (is (= [["Alice" "Bob"] ["Bob" "Alice"]] updated-rows)+ "Updated data should show Alice/Bob with friends instead of ages"))))))))))diff --git a/enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts b/enterprise/frontend/src/metabase-enterprise/api/python-transform-library.tsnew file mode 100644index 00000000000..c367ec2f0e4--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts@@ -0,0 +1,33 @@+import type {+ GetPythonLibraryRequest,+ PythonLibrary,+ UpdatePythonLibraryRequest,+} from "metabase-types/api";++import { EnterpriseApi } from "./api";+import { idTag, invalidateTags, providePythonLibraryTags } from "./tags";++export const pythonLibraryApi = EnterpriseApi.injectEndpoints({+ endpoints: (builder) => ({+ getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({+ query: ({ path }) => ({+ url: `/api/ee/transforms-python/library/${path}`,+ method: "GET",+ }),+ providesTags: (library) =>+ library ? providePythonLibraryTags(library) : [],+ }),+ updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({+ query: ({ path, source }) => ({+ url: `/api/ee/transforms-python/library/${path}`,+ method: "PUT",+ body: { source },+ }),+ invalidatesTags: (_, error, { path }) =>+ invalidateTags(error, [idTag("python-transform-library", path)]),+ }),+ }),+});++export const { useGetPythonLibraryQuery, useUpdatePythonLibraryMutation } =+ pythonLibraryApi;diff --git a/enterprise/frontend/src/metabase-enterprise/api/tags.ts b/enterprise/frontend/src/metabase-enterprise/api/tags.tsindex 14532d0c46e..136bbe53a3b 100644--- a/enterprise/frontend/src/metabase-enterprise/api/tags.ts+++ b/enterprise/frontend/src/metabase-enterprise/api/tags.ts@@ -3,6 +3,7 @@ import type { TagDescription } from "@reduxjs/toolkit/query";import { TAG_TYPES, provideUserTags } from "metabase/api/tags";import type {Comment,+ PythonLibrary,Transform,TransformJob,TransformRun,@@ -23,6 +24,7 @@ export const ENTERPRISE_TAG_TYPES = ["transform-job","transform-job-via-tag","transform-run",+ "python-transform-library",] as const;export type EnterpriseTagType = (typeof ENTERPRISE_TAG_TYPES)[number];@@ -126,3 +128,9 @@ export function provideCommentTags(return [idTag("comment", comment.id)];}++export function providePythonLibraryTags(+ library: PythonLibrary,+): TagDescription<EnterpriseTagType>[] {+ return [idTag("python-transform-library", library.path)];+}diff --git a/enterprise/frontend/src/metabase-enterprise/api/transform-python.ts b/enterprise/frontend/src/metabase-enterprise/api/transform-python.tsnew file mode 100644index 00000000000..cdfe5dde0ac--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/api/transform-python.ts@@ -0,0 +1,34 @@+import type {+ ExecutePythonTransformRequest,+ ExecutePythonTransformResponse,+} from "metabase-types/api";++import { EnterpriseApi } from "./api";++export const pythonRunnerApi = EnterpriseApi.injectEndpoints({+ endpoints: (builder) => ({+ executePython: builder.mutation<+ ExecutePythonTransformResponse,+ ExecutePythonTransformRequest+ >({+ query: ({ code, tables }) => ({+ url: "/api/ee/transform/test-python",+ method: "POST",+ body: {+ code,+ tables,+ },+ }),+ }),+ cancelPython: builder.mutation<void, void>({+ query: () => ({+ url: "/api/ee/transform/test-python/cancel",+ method: "POST",+ }),+ }),+ }),+ overrideExisting: true,+});++export const { useExecutePythonMutation, useCancelPythonMutation } =+ pythonRunnerApi;diff --git a/enterprise/frontend/src/metabase-enterprise/plugins.js b/enterprise/frontend/src/metabase-enterprise/plugins.jsindex 5bfcb01cdf1..0a676e1f976 100644--- a/enterprise/frontend/src/metabase-enterprise/plugins.js+++ b/enterprise/frontend/src/metabase-enterprise/plugins.js@@ -42,3 +42,4 @@ import "./table-editing";import "./documents";import "./semantic_search";import "./transforms";+import "./transforms-python";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.module.cssnew file mode 100644index 00000000000..13da74dc9fe--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.module.css@@ -0,0 +1,4 @@+.editor {+ width: 100%;+ height: 100%;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.tsxnew file mode 100644index 00000000000..bfa7b07b872--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.tsx@@ -0,0 +1,33 @@+import cx from "classnames";++import { CodeEditor } from "metabase/common/components/CodeEditor";++import S from "./PythonEditor.module.css";+import { completion } from "./utils";++export function PythonEditor({+ value,+ onChange,+ withPandasCompletions,+ className,+ readOnly,+ ...rest+}: {+ value: string;+ onChange?: (value: string) => void;+ withPandasCompletions?: boolean;+ className?: string;+ readOnly?: boolean;+}) {+ return (+ <CodeEditor+ className={cx(S.editor, className)}+ value={value}+ onChange={onChange}+ language="python"+ extensions={withPandasCompletions ? completion : undefined}+ readOnly={readOnly}+ {...rest}+ />+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/index.tsnew file mode 100644index 00000000000..2230beee1d6--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/index.ts@@ -0,0 +1 @@+export * from "./PythonEditor";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.tsnew file mode 100644index 00000000000..89064502b18--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.ts@@ -0,0 +1,676 @@+import {+ type Completion,+ type CompletionContext,+ autocompletion,+} from "@codemirror/autocomplete";+import { pythonLanguage } from "@codemirror/lang-python";+import { t } from "ttag";++const pandasDefinitions = (): Record<string, Completion[]> => ({+ pd: [+ {+ label: "DataFrame",+ type: "class",+ info: t`Two-dimensional, size-mutable, potentially heterogeneous tabular data`,+ },+ {+ label: "Series",+ type: "class",+ info: t`One-dimensional ndarray with axis labels`,+ },+ {+ label: "read_csv",+ type: "function",+ info: t`Read a comma-separated values (csv) file into DataFrame`,+ },+ {+ label: "read_excel",+ type: "function",+ info: t`Read an Excel file into a pandas DataFrame`,+ },+ {+ label: "read_json",+ type: "function",+ info: t`Convert a JSON string to pandas object`,+ },+ {+ label: "read_sql",+ type: "function",+ info: t`Read SQL query or database table into a DataFrame`,+ },+ {+ label: "read_parquet",+ type: "function",+ info: t`Load a parquet object from file path`,+ },+ {+ label: "concat",+ type: "function",+ info: t`Concatenate pandas objects along a particular axis`,+ },+ {+ label: "merge",+ type: "function",+ info: t`Merge DataFrame or named Series objects with a database-style join`,+ },+ {+ label: "pivot_table",+ type: "function",+ info: t`Create a spreadsheet-style pivot table as a DataFrame`,+ },+ {+ label: "crosstab",+ type: "function",+ info: t`Compute a simple cross tabulation of two (or more) factors`,+ },+ {+ label: "get_dummies",+ type: "function",+ info: t`Convert categorical variable(s) into dummy/indicator variables`,+ },+ {+ label: "date_range",+ type: "function",+ info: t`Return a fixed frequency DatetimeIndex`,+ },+ {+ label: "to_datetime",+ type: "function",+ info: t`Convert argument to datetime`,+ },+ {+ label: "to_numeric",+ type: "function",+ info: t`Convert argument to a numeric type`,+ },+ {+ label: "cut",+ type: "function",+ info: t`Bin values into discrete intervals`,+ },+ {+ label: "qcut",+ type: "function",+ info: t`Quantile-based discretization function`,+ },+ {+ label: "isna",+ type: "function",+ info: t`Detect missing values for an array-like object`,+ },+ {+ label: "isnull",+ type: "function",+ info: t`Detect missing values for an array-like object`,+ },+ {+ label: "notna",+ type: "function",+ info: t`Detect non-missing values for an array-like object`,+ },+ {+ label: "notnull",+ type: "function",+ info: t`Detect non-missing values for an array-like object`,+ },+ ],++ // DataFrame methods and properties+ DataFrame: [+ // Viewing/Inspection+ { label: "head", type: "method", info: "Return the first n rows" },+ { label: "tail", type: "method", info: "Return the last n rows" },+ {+ label: "info",+ type: "method",+ info: t`Print a concise summary of a DataFrame`,+ },+ {+ label: "describe",+ type: "method",+ info: t`Generate descriptive statistics`,+ },+ {+ label: "shape",+ type: "property",+ info: t`Return a tuple representing the dimensionality`,+ },+ {+ label: "size",+ type: "property",+ info: t`Return an int representing the number of elements`,+ },+ {+ label: "columns",+ type: "property",+ info: t`The column labels of the DataFrame`,+ },+ {+ label: "index",+ type: "property",+ info: t`The index (row labels) of the DataFrame`,+ },+ {+ label: "dtypes",+ type: "property",+ info: t`Return the dtypes in the DataFrame`,+ },+ {+ label: "values",+ type: "property",+ info: t`Return a Numpy representation of the DataFrame`,+ },++ // Selection/Indexing+ {+ label: "loc",+ type: "property",+ info: t`Access a group of rows and columns by label(s)`,+ },+ {+ label: "iloc",+ type: "property",+ info: t`Purely integer-location based indexing`,+ },+ {+ label: "at",+ type: "property",+ info: t`Access a single value for a row/column label pair`,+ },+ {+ label: "iat",+ type: "property",+ info: t`Access a single value for a row/column pair by integer position`,+ },+ {+ label: "query",+ type: "method",+ info: t`Query the columns of a DataFrame with a boolean expression`,+ },+ {+ label: "filter",+ type: "method",+ info: t`Subset the dataframe rows or columns according to labels`,+ },+ {+ label: "where",+ type: "method",+ info: t`Replace values where the condition is False`,+ },+ {+ label: "mask",+ type: "method",+ info: t`Replace values where the condition is True`,+ },++ // Data Cleaning+ {+ label: "drop",+ type: "method",+ info: t`Drop specified labels from rows or columns`,+ },+ {+ label: "drop_duplicates",+ type: "method",+ info: t`Return DataFrame with duplicate rows removed`,+ },+ { label: "dropna", type: "method", info: "Remove missing values" },+ {+ label: "fillna",+ type: "method",+ info: t`Fill NA/NaN values using the specified method`,+ },+ {+ label: "replace",+ type: "method",+ info: t`Replace values given in to_replace with value`,+ },+ {+ label: "interpolate",+ type: "method",+ info: t`Fill NaN values using an interpolation method`,+ },+ {+ label: "duplicated",+ type: "method",+ info: t`Return boolean Series denoting duplicate rows`,+ },+ { label: "isnull", type: "method", info: "Detect missing values" },+ { label: "isna", type: "method", info: "Detect missing values" },+ {+ label: "notnull",+ type: "method",+ info: t`Detect existing (non-missing) values`,+ },+ {+ label: "notna",+ type: "method",+ info: t`Detect existing (non-missing) values`,+ },++ // Transformations+ {+ label: "sort_values",+ type: "method",+ info: t`Sort by the values along either axis`,+ },+ {+ label: "sort_index",+ type: "method",+ info: t`Sort object by labels (along an axis)`,+ },+ {+ label: "reset_index",+ type: "method",+ info: t`Reset the index, or a level of it`,+ },+ {+ label: "set_index",+ type: "method",+ info: t`Set the DataFrame index using existing columns`,+ },+ {+ label: "reindex",+ type: "method",+ info: t`Conform Series/DataFrame to new index`,+ },+ { label: "rename", type: "method", info: "Alter axes labels" },+ {+ label: "assign",+ type: "method",+ info: t`Assign new columns to a DataFrame`,+ },+ {+ label: "pipe",+ type: "method",+ info: t`Apply chainable functions that expect Series or DataFrames`,+ },++ // Aggregation/GroupBy+ {+ label: "groupby",+ type: "method",+ info: t`Group DataFrame using a mapper or by a Series of columns`,+ },+ {+ label: "agg",+ type: "method",+ info: t`Aggregate using one or more operations`,+ },+ {+ label: "aggregate",+ type: "method",+ info: t`Aggregate using one or more operations`,+ },+ {+ label: "apply",+ type: "method",+ info: t`Apply a function along an axis of the DataFrame`,+ },+ {+ label: "applymap",+ type: "method",+ info: t`Apply a function to a Dataframe elementwise`,+ },+ {+ label: "transform",+ type: "method",+ info: t`Call func on self producing a DataFrame with transformed values`,+ },+ {+ label: "rolling",+ type: "method",+ info: t`Provide rolling window calculations`,+ },+ {+ label: "expanding",+ type: "method",+ info: t`Provide expanding transformations`,+ },+ { label: "resample", type: "method", info: "Resample time-series data" },++ // Statistical Methods+ {+ label: "sum",+ type: "method",+ info: t`Return the sum of the values for the requested axis`,+ },+ {+ label: "mean",+ type: "method",+ info: t`Return the mean of the values for the requested axis`,+ },+ {+ label: "median",+ type: "method",+ info: t`Return the median of the values for the requested axis`,+ },+ {+ label: "mode",+ type: "method",+ info: t`Get the mode(s) of each element along the selected axis`,+ },+ {+ label: "std",+ type: "method",+ info: t`Return sample standard deviation over requested axis`,+ },+ {+ label: "var",+ type: "method",+ info: t`Return unbiased variance over requested axis`,+ },+ {+ label: "min",+ type: "method",+ info: t`Return the minimum of the values for the requested axis`,+ },+ {+ label: "max",+ type: "method",+ info: t`Return the maximum of the values for the requested axis`,+ },+ {+ label: "count",+ type: "method",+ info: t`Count non-NA cells for each column or row`,+ },+ {+ label: "nunique",+ type: "method",+ info: t`Count number of distinct elements in specified axis`,+ },+ {+ label: "value_counts",+ type: "method",+ info: t`Return a Series containing counts of unique values`,+ },+ {+ label: "corr",+ type: "method",+ info: t`Compute pairwise correlation of columns`,+ },+ {+ label: "cov",+ type: "method",+ info: t`Compute pairwise covariance of columns`,+ },+ {+ label: "rank",+ type: "method",+ info: t`Compute numerical data ranks (1 through n) along axis`,+ },+ {+ label: "quantile",+ type: "method",+ info: t`Return values at the given quantile over requested axis`,+ },+ {+ label: "clip",+ type: "method",+ info: t`Trim values at input threshold(s)`,+ },++ // Merging/Joining+ {+ label: "merge",+ type: "method",+ info: t`Merge DataFrame or named Series objects`,+ },+ {+ label: "join",+ type: "method",+ info: t`Join columns of another DataFrame`,+ },+ {+ label: "concat",+ type: "method",+ info: t`Concatenate pandas objects along a particular axis`,+ },+ {+ label: "append",+ type: "method",+ info: t`Append rows of other to the end of caller`,+ },+ {+ label: "update",+ type: "method",+ info: t`Modify in place using non-NA values from another DataFrame`,+ },++ // Reshaping+ {+ label: "pivot",+ type: "method",+ info: t`Return reshaped DataFrame organized by given index / column values`,+ },+ {+ label: "pivot_table",+ type: "method",+ info: t`Create a spreadsheet-style pivot table`,+ },+ {+ label: "melt",+ type: "method",+ info: t`Unpivot a DataFrame from wide to long format`,+ },+ {+ label: "stack",+ type: "method",+ info: t`Stack the prescribed level(s) from columns to index`,+ },+ {+ label: "unstack",+ type: "method",+ info: t`Pivot a level of the (necessarily hierarchical) index labels`,+ },+ { label: "transpose", type: "method", info: "Transpose index and columns" },+ { label: "T", type: "property", info: "Transpose index and columns" },++ // I/O+ {+ label: "to_csv",+ type: "method",+ info: t`Write object to a comma-separated values (csv) file`,+ },+ {+ label: "to_excel",+ type: "method",+ info: t`Write object to an Excel sheet`,+ },+ {+ label: "to_json",+ type: "method",+ info: t`Convert the object to a JSON string`,+ },+ {+ label: "to_sql",+ type: "method",+ info: t`Write records stored in a DataFrame to a SQL database`,+ },+ {+ label: "to_parquet",+ type: "method",+ info: t`Write a DataFrame to the binary parquet format`,+ },+ {+ label: "to_dict",+ type: "method",+ info: t`Convert the DataFrame to a dictionary`,+ },+ {+ label: "to_html",+ type: "method",+ info: t`Render a DataFrame as an HTML table`,+ },+ {+ label: "to_string",+ type: "method",+ info: t`Render a DataFrame to a console-friendly tabular output`,+ },+ {+ label: "to_numpy",+ type: "method",+ info: t`Convert the DataFrame to a NumPy array`,+ },++ // Time Series+ {+ label: "shift",+ type: "method",+ info: t`Shift index by desired number of periods`,+ },+ { label: "tshift", type: "method", info: "Shift the time index" },+ {+ label: "diff",+ type: "method",+ info: t`First discrete difference of element`,+ },+ {+ label: "pct_change",+ type: "method",+ info: t`Percentage change between the current and a prior element`,+ },++ // Plotting (if matplotlib available)+ {+ label: "plot",+ type: "method",+ info: t`Make plots of Series or DataFrame`,+ },+ {+ label: "hist",+ type: "method",+ info: t`Make a histogram of the DataFrame's columns`,+ },+ {+ label: "boxplot",+ type: "method",+ info: t`Make a box plot of the DataFrame columns`,+ },+ ],++ // Series methods (subset of DataFrame methods that make sense for Series)+ Series: [+ { label: "head", type: "method", info: "Return the first n values" },+ { label: "tail", type: "method", info: "Return the last n values" },+ {+ label: "describe",+ type: "method",+ info: t`Generate descriptive statistics`,+ },+ {+ label: "value_counts",+ type: "method",+ info: t`Return a Series containing counts of unique values`,+ },+ {+ label: "unique",+ type: "method",+ info: t`Return unique values of Series object`,+ },+ {+ label: "nunique",+ type: "method",+ info: t`Return number of unique elements in the object`,+ },+ { label: "sort_values", type: "method", info: "Sort by the values" },+ {+ label: "sort_index",+ type: "method",+ info: t`Sort Series by index labels`,+ },+ {+ label: "reset_index",+ type: "method",+ info: t`Generate a new DataFrame or Series with the index reset`,+ },+ {+ label: "dropna",+ type: "method",+ info: t`Return a new Series with missing values removed`,+ },+ {+ label: "fillna",+ type: "method",+ info: t`Fill NA/NaN values using the specified method`,+ },+ {+ label: "apply",+ type: "method",+ info: t`Invoke function on values of Series`,+ },+ {+ label: "map",+ type: "method",+ info: t`Map values of Series according to input correspondence`,+ },+ {+ label: "replace",+ type: "method",+ info: t`Replace values given in to_replace with value`,+ },+ {+ label: "str",+ type: "property",+ info: t`Vectorized string functions for Series and Index`,+ },+ {+ label: "dt",+ type: "property",+ info: t`Accessor object for datetimelike properties`,+ },+ {+ label: "cat",+ type: "property",+ info: t`Accessor object for categorical properties`,+ },+ ],+});++function genericPandasCompletions(context: CompletionContext) {+ const definitions = pandasDefinitions();++ const word = context.matchBefore(/[\w.]*/);+ if (!word || (word.from === word.to && !context.explicit)) {+ return null;+ }++ const text = word.text;++ // Match patterns like "pd.", "df.", etc.+ if (text.includes(".")) {+ const parts = text.split(".");+ const obj = parts.at(-2);+ const partial = parts.at(-1) ?? "";++ if (!obj || !definitions[obj]) {+ return null;+ }++ const options = definitions[obj]+ .filter(+ (item) =>+ !partial ||+ item.label.toLowerCase().startsWith(partial.toLowerCase()),+ )+ .map((comp) => ({+ ...comp,+ info: undefined,+ boost: comp.type === "method" ? 2 : comp.type === "function" ? 1 : 0,+ }));++ return {+ filter: false,+ from: word.from + text.length - partial.length,+ options,+ };+ }+}++export const completion = [+ pythonLanguage.data.of({+ autocomplete: genericPandasCompletions,+ }),+ autocompletion(),+];diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/AliasInput.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/AliasInput.tsxnew file mode 100644index 00000000000..933f49dd400--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/AliasInput.tsx@@ -0,0 +1,65 @@+import { useEffect, useState } from "react";+import { t } from "ttag";++import { ActionIcon, Icon, TextInput } from "metabase/ui";+import type { Table } from "metabase-types/api";++import type { TableSelection } from "../types";+import { slugify } from "../utils";++export function AliasInput({+ table,+ selection,+ onChange,+ usedAliases,+ disabled,+}: {+ selection: TableSelection;+ table?: Table;+ onChange: (value: string) => void;+ usedAliases: Set<string>;+ disabled?: boolean;+}) {+ const [value, setValue] = useState(selection.alias);++ useEffect(() => {+ setValue(selection.alias);+ }, [selection.alias]);++ const defaultAlias = table+ ? slugify(table.name, usedAliases, selection.alias)+ : "";+ const isManualAlias = selection.alias !== defaultAlias;+ const showReset = table && isManualAlias;++ function handleChange(evt: React.ChangeEvent<HTMLInputElement>) {+ setValue(evt.target.value);+ }++ function handleBlur() {+ onChange(value);+ }++ return (+ <TextInput+ w="100%"+ value={value}+ onChange={handleChange}+ onBlur={handleBlur}+ placeholder={t`Enter alias`}+ disabled={disabled}+ rightSection={+ showReset && (+ <ActionIcon+ onClick={() => onChange(defaultAlias)}+ aria-label={t`Reset alias to default`}+ color="gray"+ variant="subtle"+ >+ <Icon name="refresh" size={12} />+ </ActionIcon>+ )+ }+ />+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/index.tsnew file mode 100644index 00000000000..b48a6bcf5c2--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/index.ts@@ -0,0 +1 @@+export * from "./AliasInput";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.module.cssnew file mode 100644index 00000000000..4825502a43b--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.module.css@@ -0,0 +1,11 @@+.dataPicker {+ border-right: 1px solid var(--mb-color-border);+ overflow: auto;+}++.databaseSelector {+ padding: var(--mantine-spacing-sm) var(--mantine-spacing-md);+ border: 1px solid var(--mb-color-border);+ border-radius: var(--mantine-radius-xs);+ cursor: pointer;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsxnew file mode 100644index 00000000000..7d5595e44bc--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx@@ -0,0 +1,300 @@+import { useState } from "react";+import { t } from "ttag";++import { hasFeature } from "metabase/admin/databases/utils";+import {+ skipToken,+ useListDatabasesQuery,+ useListTablesQuery,+} from "metabase/api";+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { DatabaseDataSelector } from "metabase/query_builder/components/DataSelector";+import { Box, Button, Icon, Stack, Text } from "metabase/ui";+import type {+ Database,+ DatabaseId,+ PythonTransformTableAliases,+ Table,+} from "metabase-types/api";++import { AliasInput } from "./AliasInput";+import S from "./PythonDataPicker.module.css";+import { TableSelector } from "./TableSelector";+import type { TableSelection } from "./types";+import {+ getInitialTableSelections,+ isConcreteTableId,+ selectionsToTableAliases,+ slugify,+} from "./utils";++type PythonDataPickerProps = {+ database?: DatabaseId;+ tables: PythonTransformTableAliases;+ onChange: (+ database: number,+ tables: PythonTransformTableAliases,+ tableInfo: Table[],+ ) => void;+};++export function PythonDataPicker({+ database,+ tables,+ onChange,+}: PythonDataPickerProps) {+ const [tableSelections, setTableSelections] = useState<TableSelection[]>(+ getInitialTableSelections(tables),+ );++ const {+ data: databases,+ isLoading: isLoadingDatabases,+ error: databaseError,+ } = useListDatabasesQuery();++ const {+ data: tablesData,+ isLoading: isLoadingTables,+ error: tablesError,+ } = useListTablesQuery(+ database+ ? {+ dbId: database,+ include_hidden: false,+ include_editable_data_model: true,+ }+ : skipToken,+ );++ const handleChange = (+ database: DatabaseId | undefined,+ selections: TableSelection[],+ ) => {+ if (database) {+ const tableAliases = selectionsToTableAliases(selections);+ onChange(database, tableAliases, tablesData ?? []);+ }+ };++ if (databaseError || tablesError) {+ return <LoadingAndErrorWrapper error={databaseError || tablesError} />;+ }++ if (isLoadingDatabases) {+ return <LoadingAndErrorWrapper loading />;+ }++ const availableTables = (tablesData || []).filter(+ (tbl: Table) => tbl.db_id === database && tbl.active,+ );++ const usedAliases = new Set(tableSelections.map((s) => s.alias));++ const handleDatabaseChange = (value: string | null) => {+ const newDatabase = value ? parseInt(value) : undefined;++ if (newDatabase === database) {+ return;+ }++ // Clear selections since they won't make sense+ const newSelections = [+ {+ tableId: undefined,+ alias: "",+ },+ ];+ setTableSelections(newSelections);+ handleChange(newDatabase, newSelections);+ };++ const handleSelectionChange = (+ selectionIndex: number,+ selection: TableSelection,+ ) => {+ const newSelections = Array.from(tableSelections);+ newSelections[selectionIndex] = selection;+ setTableSelections(newSelections);+ handleChange(database, newSelections);+ };++ const handleAddTable = () => {+ setTableSelections([+ ...tableSelections,+ {+ tableId: undefined,+ alias: "",+ },+ ]);+ };++ const handleRemoveTable = (selectionIndex: number) => {+ const newSelections = Array.from(tableSelections);+ newSelections.splice(selectionIndex, 1);+ setTableSelections(newSelections);+ handleChange(database, newSelections);+ };++ return (+ <Stack+ p="md"+ gap="md"+ h="100%"+ w="30%"+ miw="300px"+ className={S.dataPicker}+ data-testid="python-data-picker"+ >+ <Box>+ <Text fw="bold">{t`Source database`}</Text>+ <Text size="sm" c="text-light" mb="sm">+ {t`Select the database that contains your source data.`}+ </Text>++ <DatabaseDataSelector+ className={S.databaseSelector}+ selectedDatabaseId={database}+ setDatabaseFn={handleDatabaseChange}+ databases={databases?.data ?? []}+ databaseIsDisabled={(database: Database) =>+ !hasFeature(database, "transforms/python")+ }+ />+ </Box>+ {database && (+ <Box>+ <Text fw="bold">{t`Pick tables and alias them`}</Text>+ <Text size="sm" c="text-light" mb="sm">+ {t`Select tables to use as data sources and provide aliases that can be referenced in your Python script.`}+ </Text>+ <Stack gap="md">+ {tableSelections.map((selection, index) => (+ <SelectionInput+ key={index}+ selection={selection}+ database={database}+ tables={tables}+ usedAliases={usedAliases}+ availableTables={availableTables}+ onChange={(selection) =>+ handleSelectionChange(index, selection)+ }+ onRemove={() => handleRemoveTable(index)}+ disabled={isLoadingTables}+ />+ ))}+ <AddTableButton+ onClick={handleAddTable}+ disabled={availableTables.length === 0}+ />+ </Stack>+ </Box>+ )}+ </Stack>+ );+}++function SelectionInput({+ database,+ tables,+ selection,+ availableTables,+ usedAliases,+ onChange,+ onRemove,+ disabled,+}: {+ database: DatabaseId | undefined;+ tables: PythonTransformTableAliases;+ selection: TableSelection;+ availableTables: Table[];+ usedAliases: Set<string>;+ onChange: (selection: TableSelection) => void;+ onRemove: () => void;+ disabled?: boolean;+}) {+ const table = availableTables.find((table) => table.id === selection.tableId);++ function handleAliasChange(newAlias: string) {+ const newSelection = {+ ...selection,+ alias: slugify(newAlias, usedAliases, selection.alias),+ };++ onChange(newSelection);+ }++ function handleTableChange(table: Table | undefined) {+ if (!table || !isConcreteTableId(table.id)) {+ onChange({+ ...selection,+ tableId: undefined,+ });+ return;+ }++ const oldTable = availableTables.find(+ (table) => table.id === selection?.tableId,+ );++ const wasOldAliasManuallySet =+ selection.alias !== "" &&+ (!oldTable ||+ selection.alias !==+ slugify(oldTable.name, usedAliases, selection.alias));++ const newAlias = wasOldAliasManuallySet+ ? selection.alias+ : slugify(table.name, usedAliases);++ onChange({+ ...selection,+ tableId: table.id,+ alias: newAlias,+ });+ }++ return (+ <Stack gap="xs" align="center" w="100%">+ <TableSelector+ database={database}+ table={table}+ selectedTableIds={Object.values(tables)}+ onChange={handleTableChange}+ onRemove={onRemove}+ availableTables={availableTables}+ disabled={disabled}+ />+ <AliasInput+ selection={selection}+ table={table}+ onChange={handleAliasChange}+ usedAliases={usedAliases}+ disabled={disabled}+ />+ </Stack>+ );+}++function AddTableButton({+ onClick,+ disabled,+}: {+ disabled?: boolean;+ onClick: () => void;+}) {+ return (+ <Box>+ <Button+ leftSection={<Icon name="add_data" />}+ variant="subtle"+ px={0}+ onClick={onClick}+ disabled={disabled}+ >+ {t`Add a table`}+ </Button>+ </Box>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.module.cssnew file mode 100644index 00000000000..ef95488c4d7--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.module.css@@ -0,0 +1,7 @@+.tableSelector {+ border: 1px solid var(--border-color);+}++.tableSelectorButtonInner {+ justify-content: start;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.tsxnew file mode 100644index 00000000000..ebd48a3008f--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.tsx@@ -0,0 +1,130 @@+import { useDisclosure } from "@mantine/hooks";+import { t } from "ttag";++import type { CollectionPickerItem } from "metabase/common/components/Pickers/CollectionPicker";+import {+ type DataPickerItem,+ DataPickerModal,+} from "metabase/common/components/Pickers/DataPicker";+import {+ ActionIcon,+ Box,+ Button,+ Group,+ Icon,+ Stack,+ Tooltip,+} from "metabase/ui";+import type {+ ConcreteTableId,+ DatabaseId,+ RecentItem,+ Table,+ TableId,+} from "metabase-types/api";++import { isConcreteTableId } from "../utils";++import S from "./TableSelector.module.css";++export function TableSelector({+ database,+ availableTables,+ selectedTableIds,+ disabled,+ onChange,+ onRemove,+ table,+}: {+ database: DatabaseId | undefined;+ table: Table | undefined;+ availableTables: Table[];+ selectedTableIds: ConcreteTableId[];+ disabled?: boolean;+ onChange: (table: Table | undefined) => void;+ onRemove: () => void;+}) {+ const [isOpened, { open, close }] = useDisclosure();++ function handleChange(tableId: TableId | undefined) {+ const table = availableTables.find((table) => table.id === tableId);+ if (table) {+ onChange(table);+ }+ }++ function shouldDisableItem(+ item: DataPickerItem | CollectionPickerItem | RecentItem,+ ) {+ if (item.model === "table") {+ // Filter available tables to exclude already selected ones (except current selection)+ return !isConcreteTableId(item.id) || selectedTableIds.includes(item.id);+ }+ if (item.model === "database") {+ return item.id !== database;+ }+ return true;+ }++ return (+ <>+ <Group w="100%" bdrs="xs" gap="xs" className={S.tableSelector}>+ <Button+ flex="1 1 auto"+ onClick={open}+ disabled={disabled}+ classNames={{ inner: S.tableSelectorButtonInner }}+ px="sm"+ py="lg"+ variant="subtle"+ >+ <Stack gap={0} align="start" justify="center">+ {table ? (+ <>+ <Box fz="sm" c="text-medium" fw="normal">+ {table?.db?.name} / {table?.schema}+ </Box>+ <Box c="text-dark">{table?.display_name}</Box>+ </>+ ) : (+ <Box c="text-dark">{t`Select a table…`}</Box>+ )}+ </Stack>+ </Button>++ <Tooltip label={t`Remove this table`}>+ <ActionIcon+ onClick={onRemove}+ pr="sm"+ aria-label={t`Remove this table`}+ >+ <Icon name="close" c="text-dark" />+ </ActionIcon>+ </Tooltip>+ </Group>+ {isOpened && (+ <DataPickerModal+ title={t`Pick a table`}+ value={getDataPickerValue(table)}+ databaseId={database}+ onChange={handleChange}+ onClose={close}+ shouldDisableItem={shouldDisableItem}+ />+ )}+ </>+ );+}++function getDataPickerValue(table: Table | undefined) {+ if (!table) {+ return undefined;+ }+ return {+ model: "table" as const,+ id: table.id,+ name: table.display_name,+ db_id: table.db_id,+ schema: table.schema,+ };+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/index.tsnew file mode 100644index 00000000000..992eb097ecc--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/index.ts@@ -0,0 +1 @@+export * from "./TableSelector";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/index.tsnew file mode 100644index 00000000000..37019632951--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/index.ts@@ -0,0 +1 @@+export * from "./PythonDataPicker";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/types.tsnew file mode 100644index 00000000000..2182b903c8f--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/types.ts@@ -0,0 +1,6 @@+import type { ConcreteTableId } from "metabase-types/api";++export type TableSelection = {+ tableId: ConcreteTableId | undefined;+ alias: string;+};diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.tsnew file mode 100644index 00000000000..4f1c7a670dc--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts@@ -0,0 +1,71 @@+import { slugify as toSlug } from "metabase/lib/formatting";+import type {+ ConcreteTableId,+ PythonTransformTableAliases,+ TableId,+} from "metabase-types/api";++import type { TableSelection } from "./types";++export function getInitialTableSelections(+ tables: PythonTransformTableAliases | undefined,+) {+ if (tables && Object.keys(tables).length > 0) {+ return Object.entries(tables).map(([alias, tableId]) => ({+ tableId,+ alias,+ }));+ }+ // Start with one empty row by default+ return [+ {+ tableId: undefined,+ alias: "",+ },+ ];+}++export function selectionsToTableAliases(+ selections: TableSelection[],+): PythonTransformTableAliases {+ const tableAliases: PythonTransformTableAliases = {};++ for (const selection of selections) {+ const { alias, tableId } = selection;+ if (tableId !== undefined && alias !== "") {+ tableAliases[alias] = tableId;+ }+ }+ return tableAliases;+}++export function slugify(+ name: string,+ usedNames: Set<string> = new Set(),+ allowThisAliasIfItExists?: string,+) {+ if (name === "") {+ return "";+ }++ const plain = toSlug(name).replace(/\./, "_");+ if (plain === allowThisAliasIfItExists) {+ return plain;+ }+ if (!usedNames.has(plain)) {+ return plain;+ }+ for (let index = 1; index <= 100; index++) {+ const alias = `${plain}_${index}`;+ if (!usedNames.has(alias)) {+ return alias;+ }+ }+ return name;+}++export function isConcreteTableId(+ id: TableId | undefined,+): id is ConcreteTableId {+ return typeof id === "number";+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.module.cssnew file mode 100644index 00000000000..d0e2d81f9c8--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.module.css@@ -0,0 +1,10 @@+.root {+ border-bottom: 1px solid var(--mb-color-border);+ position: relative;+}++.libraryActions {+ position: absolute;+ top: 0;+ right: 0;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.tsxnew file mode 100644index 00000000000..1c1ba2f82d2--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.tsx@@ -0,0 +1,146 @@+import type { ReactNode } from "react";+import { ResizableBox } from "react-resizable";+import { t } from "ttag";++import Link from "metabase/common/components/Link";+import RunButtonWithTooltip from "metabase/query_builder/components/RunButtonWithTooltip";+import { Box, Checkbox, Flex, Icon, Stack } from "metabase/ui";+import { SHARED_LIB_IMPORT_PATH } from "metabase-enterprise/transforms-python/constants";+import { getPythonLibraryUrl } from "metabase-enterprise/transforms-python/urls";++import { PythonEditor } from "../../PythonEditor";++import S from "./PythonEditorBody.module.css";+import { ResizableBoxHandle } from "./ResizableBoxHandle";+import { hasImport, insertImport, removeImport } from "./utils";++type PythonEditorBodyProps = {+ source: string;+ isRunnable: boolean;+ onChange: (source: string) => void;+ onRun?: () => void;+ onCancel?: () => void;+ isRunning?: boolean;+ isDirty?: boolean;+ tables?: Record<string, number>;+ withDebugger?: boolean;+};++const EDITOR_HEIGHT = 400;++export function PythonEditorBody({+ source,+ onChange,+ isRunnable,+ onRun,+ onCancel,+ isRunning,+ isDirty,+ withDebugger,+}: PythonEditorBodyProps) {+ return (+ <MaybeResizableBox resizable={withDebugger}>+ <Flex h="100%" align="end" bg="bg-light" pos="relative">+ <PythonEditor+ value={source}+ onChange={onChange}+ withPandasCompletions+ data-testid="python-editor"+ />++ {withDebugger && (+ <Box p="md">+ <RunButtonWithTooltip+ disabled={!isRunnable}+ isRunning={isRunning}+ isDirty={isDirty}+ onRun={onRun}+ onCancel={onCancel}+ getTooltip={() => t`Run Python script`}+ />+ </Box>+ )}+ <SharedLibraryActions source={source} onChange={onChange} />+ </Flex>+ </MaybeResizableBox>+ );+}++function MaybeResizableBox({+ resizable,+ children,+}: {+ resizable?: boolean;+ children?: ReactNode;+}) {+ if (!resizable) {+ return <Box h="100%">{children}</Box>;+ }++ return (+ <ResizableBox+ axis="y"+ height={EDITOR_HEIGHT}+ handle={<ResizableBoxHandle />}+ resizeHandles={["s"]}+ >+ {children}+ </ResizableBox>+ );+}++function SharedLibraryActions({+ source,+ onChange,+}: {+ source: string;+ onChange: (source: string) => void;+}) {+ return (+ <Stack className={S.libraryActions} p="md" gap="sm">+ <SharedLibraryImportToggle source={source} onChange={onChange} />+ <SharedLibraryEditLink />+ </Stack>+ );+}++function SharedLibraryImportToggle({+ source,+ onChange,+}: {+ source: string;+ onChange: (source: string) => void;+}) {+ const hasSharedLib = hasImport(source, SHARED_LIB_IMPORT_PATH);++ function handleToggleSharedLib() {+ if (hasImport(source, SHARED_LIB_IMPORT_PATH)) {+ onChange(removeImport(source, SHARED_LIB_IMPORT_PATH));+ } else {+ onChange(insertImport(source, SHARED_LIB_IMPORT_PATH));+ }+ }++ return (+ <Checkbox+ label={t`Import common library`}+ checked={hasSharedLib}+ onChange={handleToggleSharedLib}+ size="sm"+ />+ );+}++function SharedLibraryEditLink() {+ return (+ <Flex+ component={Link}+ target="_blank"+ to={getPythonLibraryUrl({ path: SHARED_LIB_IMPORT_PATH })}+ gap="sm"+ >+ <Icon name="pencil" />+ {t`Edit common library`}+ </Flex>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.module.cssnew file mode 100644index 00000000000..ac90231539b--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.module.css@@ -0,0 +1,3 @@+.root {+ cursor: row-resize;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.tsxnew file mode 100644index 00000000000..8020e0504fc--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.tsx@@ -0,0 +1,27 @@+import { type HTMLAttributes, type Ref, forwardRef } from "react";+import _ from "underscore";++import { Box, Flex, rem } from "metabase/ui";++import S from "./ResizableBoxHandle.module.css";++export const ResizableBoxHandle = forwardRef(function ResizableBoxHandle(+ props: HTMLAttributes<HTMLDivElement>,+ ref: Ref<HTMLDivElement>,+) {+ return (+ <Flex+ ref={ref}+ className={S.root}+ align="center"+ justify="center"+ w="100%"+ h="sm"+ pos="absolute"+ bottom={rem(-4)}+ {..._.omit(props, "handleAxis")}+ >+ <Box w="6.25rem" h="xs" bg="border" />+ </Flex>+ );+});diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/index.tsnew file mode 100644index 00000000000..3141dc8e2ba--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/index.ts@@ -0,0 +1 @@+export * from "./ResizableBoxHandle";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/index.tsnew file mode 100644index 00000000000..1f7cef4b142--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/index.ts@@ -0,0 +1 @@+export * from "./PythonEditorBody";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/utils.tsnew file mode 100644index 00000000000..577de6df205--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/utils.ts@@ -0,0 +1,33 @@+export function insertImport(source: string, path: string) {+ const lines = source.split("\n");++ // Find the first line that is not a comment+ for (let i = 0; i < lines.length; i++) {+ const line = lines[i];+ if (line.trim().startsWith("#")) {+ continue;+ }+ lines.splice(i, 0, `import ${pathToImportSpecifier(path)}`);+ break;+ }++ return lines.join("\n");+}++function pathToImportSpecifier(path: string) {+ return path.replace(/\.py$/, "").split("/").join(".");+}++function libImportRegex(path: string) {+ return new RegExp(`import ${pathToImportSpecifier(path)}`, "g");+}++export function removeImport(source: string, path: string) {+ const lines = source.split("\n");+ return lines.filter((line) => !libImportRegex(path).test(line)).join("\n");+}++export function hasImport(source: string, path: string) {+ const regex = libImportRegex(path);+ return regex.test(source);+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/ExecutionOutputTable.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/ExecutionOutputTable.tsxnew file mode 100644index 00000000000..035c567ef4c--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/ExecutionOutputTable.tsx@@ -0,0 +1,25 @@+import { useMemo } from "react";++import { DataGrid, useDataGridInstance } from "metabase/data-grid";++import { type Row, parseOutput } from "./utils";++export function ExecutionOutputTable({ output }: { output?: string }) {+ const { headers, rows } = useMemo(() => parseOutput(output ?? ""), [output]);++ const tableProps = useDataGridInstance<Row, unknown>({+ data: rows,+ columnsOptions: headers.map((header) => ({+ id: header,+ name: header,+ accessorFn: (row) => row[header],+ })),+ minGridWidth: 100,+ });++ if (!output || headers.length === 0) {+ return null;+ }++ return <DataGrid {...tableProps} />;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.module.cssnew file mode 100644index 00000000000..6553025e1fe--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.module.css@@ -0,0 +1,56 @@+.logs {+ white-space: pre;+ border: 1px solid var(--mb-color-border);+ font-family: var(--mb-default-monospace-font-family);+}++.visualization {+ flex: 1 0 auto;+}++.results {+ width: 100%;+ border-collapse: collapse;+ border: 1px solid var(--mb-color-border);+ border-radius: var(--mantine-radius-md);+}++.empty {+ border-top: 1px solid var(--mb-color-border);+}++.tableHeader {+ background-color: var(--mb-color-bg-light);+ padding: var(--mantine-spacing-sm);+}++.tableCell {+ border-top: 1px solid var(--mb-color-border);+ padding: var(--mantine-spacing-sm);+}++.error {+ color: var(--mb-color-error);+ background-color: var(--mb-color-bg-error);+ border-bottom: 1px solid var(--mb-color-border);+}++.info {+ color: var(--mb-color-text-dark);+ border: 1px solid var(--mb-color-bg-medium);+ background-color: var(--mb-color-bg-light);+}++.success {+ color: var(--mb-color-success);+ border-bottom: 1px solid var(--mb-color-border);+}++.loading {+ position: absolute;+ top: 0;+ left: 0;+ bottom: 0;+ right: 0;+ background-color: rgba(255, 255, 255, 0.5);+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.tsxnew file mode 100644index 00000000000..94f4f9034b5--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.tsx@@ -0,0 +1,175 @@+import type { ReactNode } from "react";+import { c, t } from "ttag";++import EmptyCodeResult from "assets/img/empty-states/code.svg";+import DebouncedFrame from "metabase/common/components/DebouncedFrame";+import { LoadingSpinner } from "metabase/common/components/MetadataInfo/MetadataInfo.styled";+import { isMac } from "metabase/lib/browser";+import { Box, Flex, Icon, Stack, Text, Title } from "metabase/ui";++import type { ExecutionResult } from "../utils";++import { ExecutionOutputTable } from "./ExecutionOutputTable";+import S from "./PythonEditorResults.module.css";++type PythonEditorProps = {+ isRunning?: boolean;+ executionResult?: ExecutionResult | null;+};++export function PythonEditorResults({+ executionResult,+ isRunning,+}: PythonEditorProps) {+ return (+ <DebouncedFrame className={S.visualization}>+ <ExecutionResultHeader executionResult={executionResult} />+ <ExecutionResults executionResult={executionResult} />+ {isRunning && <LoadingState />}+ </DebouncedFrame>+ );+}++function ExecutionResultHeader({+ executionResult,+}: {+ executionResult?: ExecutionResult | null;+}) {+ const message = getMessageForExecutionResult(executionResult);+ return <Flex w="100%">{message}</Flex>;+}++function getRunQueryShortcut() {+ return isMac() ? t`⌘ + return` : t`Ctrl + enter`;+}++function LoadingState() {+ return (+ <Flex p="md" className={S.loading}>+ <LoadingSpinner />+ </Flex>+ );+}++function EmptyState() {+ const keyboardShortcut = getRunQueryShortcut();++ return (+ <Flex h="100%" align="center" justify="center" className={S.empty}>+ <Stack maw="25rem" gap={0} ta="center" align="center">+ <Box maw="3rem" mb="0.75rem">+ <img src={EmptyCodeResult} alt="Code prompt icon" />+ </Box>+ <Text c="text-medium">+ {c("{0} refers to the keyboard shortcut")+ .jt`To run your code, click on the Run button or type ${(+ <b key="shortcut">({keyboardShortcut})</b>+ )}`}+ </Text>+ </Stack>+ </Flex>+ );+}++function ExecutionResults({+ executionResult,+}: {+ executionResult?: ExecutionResult | null;+}) {+ if (!executionResult) {+ return <EmptyState />;+ }++ return (+ <Stack gap={0}>+ <ExecutionLogs+ label={t`Standard output:`}+ content={executionResult.stdout}+ />+ <ExecutionLogs+ label={t`Standard error:`}+ content={executionResult.stderr}+ />++ <ExecutionOutputTable output={executionResult.output} />+ </Stack>+ );+}++function getMessageForExecutionResult(+ executionResult?: ExecutionResult | null,+) {+ if (!executionResult) {+ return null;+ }+ if (executionResult.error) {+ return (+ <Box className={S.error} p="sm" w="100%">+ <Flex gap="sm">+ <Icon name="warning" />+ <Stack gap={0}>+ <Title order={5} mb="xs">+ {t`An error occurred while executing your Python script`}+ </Title>+ {executionResult.error}+ </Stack>+ </Flex>+ </Box>+ );+ }++ if (!executionResult.output) {+ return (+ <Flex className={S.info} p="sm" gap="sm" align="center" w="100%">+ <Icon name="info" />+ {t`No results to display.`}+ </Flex>+ );+ }++ return (+ <Flex className={S.success} p="sm" gap="sm" align="center" w="100%">+ <Icon name="check" />+ {t`Script executed successfully.`}+ </Flex>+ );+}++function ExecutionLogs({+ label,+ content,+}: {+ label: string;+ content?: string | null;+}) {+ if (!content) {+ return null;+ }++ return (+ <Section title={label}>+ <Box p="sm" bg="bg-light" mah="150px" bdrs="xs" className={S.logs}>+ {content}+ </Box>+ </Section>+ );+}++function Section({+ title,+ children,+}: {+ title: ReactNode;+ children: ReactNode;+}) {+ return (+ <Box>+ {title && (+ <Title order={5} mb="xs">+ {title}+ </Title>+ )}+ {children}+ </Box>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/index.tsnew file mode 100644index 00000000000..b4bca07a82b--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/index.ts@@ -0,0 +1 @@+export * from "./PythonEditorResults";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/utils.tsnew file mode 100644index 00000000000..fb578e81853--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/utils.ts@@ -0,0 +1,38 @@+import type { RowValue } from "metabase-types/api";++export type Row = Record<string, RowValue>;++// Parses the output of a Python transform (given as JSON-lines)+// into a set of headers and rows.+//+// TODO: The server should return a non-stringified JSON array of rows,+// not a string.+export function parseOutput(output: string): {+ headers: string[];+ rows: Row[];+} {+ if (!output?.trim()) {+ return { headers: [], rows: [] };+ }++ const lines = output.trim().split("\n");+ const headers = new Set<string>();+ const rows: Row[] = [];++ for (const line of lines) {+ try {+ const data = JSON.parse(line) as Row;+ for (const key in data) {+ headers.add(key);+ }+ rows.push(data);+ } catch (err) {+ // noop+ }+ }++ return {+ headers: Array.from(headers),+ rows,+ };+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsxnew file mode 100644index 00000000000..567229fea9f--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx@@ -0,0 +1,150 @@+import { useHotkeys } from "@mantine/hooks";+import { useState } from "react";++import { Flex, Stack } from "metabase/ui";+import { EditorHeader } from "metabase-enterprise/transforms/components/QueryEditor/EditorHeader";+import type {+ DatabaseId,+ PythonTransformSource,+ PythonTransformTableAliases,+ Table,+} from "metabase-types/api";++import { PythonDataPicker } from "./PythonDataPicker";+import { PythonEditorBody } from "./PythonEditorBody";+import { PythonEditorResults } from "./PythonEditorResults";+import {+ getValidationResult,+ isPythonTransformSource,+ updateTransformSignature,+ useShouldShowPythonDebugger,+ useTestPythonTransform,+} from "./utils";++export type PythonTransformSourceDraft = {+ type: "python";+ body: string;+ "source-database": DatabaseId | undefined;+ "source-tables": PythonTransformTableAliases;+};++export type PythonTransformEditorProps = {+ initialSource: PythonTransformSourceDraft;+ isNew?: boolean;+ isSaving?: boolean;+ isRunnable?: boolean;+ onSave: (newSource: PythonTransformSource) => void;+ onCancel: () => void;+};++export function PythonTransformEditor({+ initialSource,+ isNew = true,+ isSaving = false,+ isRunnable = true,+ onSave,+ onCancel,+}: PythonTransformEditorProps) {+ const [source, setSource] = useState(initialSource);+ const [isSourceDirty, setIsSourceDirty] = useState(false);++ const { isRunning, isDirty, cancel, run, executionResult } =+ useTestPythonTransform(source);++ const handleScriptChange = (body: string) => {+ const newSource = {+ ...source,+ body,+ };+ setSource(newSource);+ setIsSourceDirty(true);+ };++ const handleDataChange = (+ database: number,+ sourceTables: PythonTransformTableAliases,+ tableInfo: Table[],+ ) => {+ const updatedScript = updateTransformSignature(+ source.body,+ sourceTables,+ tableInfo,+ );++ const newSource = {+ ...source,+ body: updatedScript,+ "source-database": database,+ "source-tables": sourceTables,+ };+ setSource(newSource);+ setIsSourceDirty(true);+ };++ const handleSave = () => {+ if (isPythonTransformSource(source)) {+ onSave(source);+ }+ };++ const showDebugger = useShouldShowPythonDebugger();++ const handleCmdEnter = () => {+ if (!showDebugger) {+ return;+ }+ if (isRunning) {+ cancel();+ } else if (isRunnable && isPythonTransformSource(source)) {+ run();+ }+ };++ useHotkeys([["mod+Enter", handleCmdEnter]], []);++ const validationResult = getValidationResult(source);++ return (+ <Stack+ w="100%"+ h="100%"+ bg="bg-white"+ data-testid="transform-query-editor"+ gap={0}+ >+ <EditorHeader+ isNew={isNew}+ isSaving={isSaving}+ onSave={handleSave}+ onCancel={onCancel}+ validationResult={validationResult}+ isQueryDirty={isSourceDirty}+ />+ <Flex h="100%" w="100%">+ <PythonDataPicker+ database={source["source-database"]}+ tables={source["source-tables"]}+ onChange={handleDataChange}+ />+ <Stack w="100%" h="100%" gap={0}>+ <PythonEditorBody+ isRunnable={isRunnable && isPythonTransformSource(source)}+ isRunning={isRunning}+ isDirty={isDirty}+ onRun={run}+ onCancel={cancel}+ source={source.body}+ onChange={handleScriptChange}+ withDebugger={showDebugger}+ />+ {showDebugger && (+ <PythonEditorResults+ isRunning={isRunning}+ executionResult={executionResult}+ />+ )}+ </Stack>+ </Flex>+ </Stack>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/index.tsnew file mode 100644index 00000000000..6f64105baa1--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/index.ts@@ -0,0 +1 @@+export * from "./PythonTransformEditor";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.tsnew file mode 100644index 00000000000..95ebe40a3f5--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts@@ -0,0 +1,249 @@+import { useRef, useState } from "react";+import { useLocation } from "react-use";+import { t } from "ttag";++import { getErrorMessage } from "metabase/api/utils";+import { useExecutePythonMutation } from "metabase-enterprise/api/transform-python";+import type {+ ExecutePythonTransformResponse,+ PythonTransformSource,+ PythonTransformTableAliases,+ Table,+ TableId,+} from "metabase-types/api";++import type { PythonTransformSourceDraft } from "./PythonTransformEditor";++export function updateTransformSignature(+ script: string,+ tables: PythonTransformTableAliases,+ tableInfo: Table[],+): string {+ const tableAliases = Object.keys(tables);++ const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;++ const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;+ const newSignature =+ signatureOneLine.length > 80 && tableAliases.length > 0+ ? `def transform(\n${tableAliases.map((alias) => ` ${alias}`).join(",\n")},\n):`+ : signatureOneLine;++ function getTableName(tableId: TableId) {+ const table = tableInfo.find((t) => t.id === tableId);+ if (!table) {+ return undefined;+ }+ return [table.db?.name, table.schema, table.name].filter(Boolean).join(".");+ }++ if (transformRegex.test(script)) {+ // Capture the existing indentation after the colon+ const match = script.match(transformRegex);+ const originalIndent = match ? match[1] : " ";++ let updatedScript = script.replace(+ transformRegex,+ newSignature + "\n" + originalIndent,+ );++ const argsRegex =+ /(\s+"""[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;+ const argsMatch = updatedScript.match(argsRegex);++ if (tableAliases.length > 0) {+ const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));+ const newArgsSection = tableAliases+ .map((alias) => {+ const padding = " ".repeat(maxAliasLength - alias.length);+ const tableId = tables[alias];+ const tableName = getTableName(tableId);+ return ` ${alias}:${padding} DataFrame containing the data from the "${tableName}" table`;+ })+ .join("\n");++ if (argsMatch) {+ // Replace existing Args section+ updatedScript = updatedScript.replace(+ argsRegex,+ ` $1++ Args:+${newArgsSection}+$3`,+ );+ } else {+ // No Args section exists, look for the pattern before Returns and insert Args+ // Also remove the instructional text about selecting tables+ const insertRegex =+ /(\s+"""[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;+ const insertMatch = updatedScript.match(insertRegex);+ if (insertMatch) {+ updatedScript = updatedScript.replace(+ insertRegex,+ `$1++ Args:+${newArgsSection}+$2`,+ );+ } else {+ // Fallback: just look for Returns section without the specific text+ // Remove any extra whitespace before Returns and ensure single blank line+ const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;+ const fallbackMatch = updatedScript.match(fallbackRegex);+ if (fallbackMatch) {+ updatedScript = updatedScript.replace(+ fallbackRegex,+ `$1++ Args:+${newArgsSection}+$2`,+ );+ }+ }+ }+ } else if (argsMatch) {+ // Remove existing Args section when no tables+ updatedScript = updatedScript.replace(argsRegex, "$1$3");+ }++ return updatedScript;+ }++ const maxAliasLength =+ tableAliases.length > 0+ ? Math.max(...tableAliases.map((a) => a.length))+ : 0;+ const functionTemplate = `+${newSignature}+ """+ Transform function that processes the input data.+ ${+ tableAliases.length > 0+ ? `++ Args:+${tableAliases+ .map((alias) => {+ const padding = " ".repeat(maxAliasLength - alias.length);+ const tableId = tables[alias];+ const tableName = getTableName(tableId);+ if (alias === tableName) {+ return ` ${alias}:${padding} DataFrame containing the data from the corresponding table`;+ }+ return ` ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;+ })+ .join("\n")}`+ : ""+ }++ Returns:+ DataFrame: The transformed data+ """+ import pandas as pd+ return pd.DataFrame()+`;++ return script + functionTemplate;+}++export type ExecutionResult = {+ output?: string;+ stdout?: string;+ stderr?: string;+ error?: string;+};++type TestPythonScriptState = {+ isRunning: boolean;+ isDirty: boolean;+ executionResult: ExecutionResult | null;+ run: () => void;+ cancel: () => void;+};++export function useTestPythonTransform(+ source: PythonTransformSourceDraft,+): TestPythonScriptState {+ const [executePython, { isLoading: isRunning, originalArgs }] =+ useExecutePythonMutation();+ const abort = useRef<(() => void) | null>(null);+ const [executionResult, setData] =+ useState<ExecutePythonTransformResponse | null>(null);++ const isDirty = originalArgs?.code !== source.body;++ const run = async () => {+ if (source["source-database"] === undefined) {+ return null;+ }+ const request = executePython({+ code: source.body,+ tables: source["source-tables"],+ });+ abort.current = () => request.abort();++ try {+ const data = await request.unwrap();+ setData(data);+ } catch (error) {+ if (typeof error === "object" && error !== null) {+ if ("name" in error && error.name === "AbortError") {+ setData({ error: t`Python script execution was canceled` });+ return;+ }+ }++ const errorMessage = getErrorMessage(error, t`An unknown error occurred`);+ setData({ error: errorMessage });+ }+ };++ const cancel = () => {+ abort.current?.();+ };++ return {+ isRunning,+ isDirty,+ cancel,+ run,+ executionResult,+ };+}++export function isPythonTransformSource(+ source: PythonTransformSourceDraft,+): source is PythonTransformSource {+ return source.type === "python" && source["source-database"] !== undefined;+}++export function useShouldShowPythonDebugger() {+ const location = useLocation();+ const params = new URLSearchParams(location.search);+ return params.get("debugger") === "1";+}++export function getValidationResult(source: PythonTransformSourceDraft) {+ if (!source["source-database"]) {+ return { isValid: false, errorMessage: t`Select a source a database` };+ }++ if (source.body.trim() === "") {+ return {+ isValid: false,+ errorMessage: t`The Python script cannot be empty`,+ };+ }++ if (Object.keys(source["source-tables"]).length === 0) {+ return {+ isValid: false,+ errorMessage: t`Select at least one table to alias`,+ };+ }++ return { isValid: true };+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.module.cssnew file mode 100644index 00000000000..a51a0a3cf90--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.module.css@@ -0,0 +1,10 @@+.grid {+ display: grid;+ grid-template-columns: max-content 1fr;+ gap: var(--mantine-spacing-sm) var(--mantine-spacing-md);+}++.alias {+ font-family: var(--mb-default-monospace-font-family);+ font-size: 0.75rem;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.tsxnew file mode 100644index 00000000000..a98fca724e9--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.tsx@@ -0,0 +1,118 @@+import { t } from "ttag";++import { skipToken, useGetTableQuery } from "metabase/api";+import Link from "metabase/common/components/Link";+import CS from "metabase/css/core/index.css";+import { Box, Group, Icon, type IconName, Loader, Stack } from "metabase/ui";+import { SplitSection } from "metabase-enterprise/transforms/components/SplitSection";+import {+ getBrowseDatabaseUrl,+ getBrowseSchemaUrl,+ getQueryBuilderUrl,+} from "metabase-enterprise/transforms/urls";+import type { Transform } from "metabase-types/api";++import S from "./SourceSection.module.css";++type SourceSectionProps = {+ transform: Transform;+};++export function SourceSection({ transform }: SourceSectionProps) {+ const { source } = transform;+ if (source.type !== "python") {+ return null;+ }++ const { "source-database": sourceDatabaseId, "source-tables": sourceTables } =+ source;++ return (+ <SplitSection+ label={t`Transform source`}+ description={t`The data sources for this Python transform, by alias.`}+ >+ <Stack p="lg" gap="sm" className={S.grid}>+ {Object.entries(sourceTables).map(([alias, tableId]) => (+ <TableWithAlias+ key={alias}+ alias={alias}+ tableId={tableId as number}+ databaseId={sourceDatabaseId as number}+ />+ ))}+ </Stack>+ </SplitSection>+ );+}++type TableWithAliasProps = {+ alias: string;+ tableId: number;+ databaseId: number;+};++function TableWithAlias({ alias, tableId, databaseId }: TableWithAliasProps) {+ const { data: table, isLoading } = useGetTableQuery(+ tableId ? { id: tableId } : skipToken,+ );++ if (isLoading) {+ return <Loader size="xs" data-testid="loader" />;+ }++ return (+ <>+ <Group className={S.alias}>{alias}</Group>+ <Group gap="sm">+ {table?.db && (+ <>+ <SourceItemLink+ label={table.db.name}+ icon="database"+ to={getBrowseDatabaseUrl(table.db.id)}+ data-testid="source-database-link"+ />+ <SourceItemDivider />+ </>+ )}+ {table?.schema && (+ <>+ <SourceItemLink+ label={table.schema}+ icon="folder"+ to={getBrowseSchemaUrl(databaseId, table.schema)}+ />+ <SourceItemDivider />+ </>+ )}+ <SourceItemLink+ label={table?.display_name || table?.name || `Table ${tableId}`}+ icon="table2"+ to={table ? getQueryBuilderUrl(table.id, table.db_id) : undefined}+ />+ </Group>+ </>+ );+}++type SourceItemLinkProps = {+ label: string;+ icon: IconName;+ to?: string;+};++function SourceItemLink({ label, icon, to }: SourceItemLinkProps) {+ return (+ <Link className={CS.link} to={to ?? ""} disabled={to == null}>+ <Group gap="xs" align="center" justify="center">+ <Icon name={icon} />+ <Box>{label}</Box>+ </Group>+ </Link>+ );+}++function SourceItemDivider() {+ return <Icon name="chevronright" size={8} />;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.unit.spec.tsxnew file mode 100644index 00000000000..16a3644593b--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.unit.spec.tsx@@ -0,0 +1,83 @@+import { setupTableEndpoints } from "__support__/server-mocks";+import { renderWithProviders, screen, waitFor } from "__support__/ui";+import type { ConcreteTableId, Table, Transform } from "metabase-types/api";+import { createMockTable } from "metabase-types/api/mocks";+import {+ createMockPythonTransformSource,+ createMockTransform,+} from "metabase-types/api/mocks/transform";++import { SourceSection } from "./SourceSection";++const TABLE_A = createMockTable({+ id: 1,+ name: "table_a",+ display_name: "Table A",+ schema: "schema_a",+});++const TABLE_B = createMockTable({+ id: 2,+ name: "table_b",+ display_name: "Table B",+ schema: "schema_b",+});++type SetupOpts = {+ tables?: Table[];+ transform?: Transform;+};++function setup({+ tables = [TABLE_A, TABLE_B],+ transform = createMockTransform(),+}: SetupOpts = {}) {+ for (const table of tables) {+ setupTableEndpoints(table);+ }+ renderWithProviders(<SourceSection transform={transform} />);+}++describe("SourceSection", () => {+ it("should render nothing for query transform", () => {+ setup({+ transform: createMockTransform({+ source: {+ type: "query",+ query: { type: "query", database: 1, query: { "source-table": 1 } },+ },+ }),+ });+ expect(screen.queryByText("Transform source")).not.toBeInTheDocument();+ });++ it("should render the section with label and description", async () => {+ setup({+ transform: createMockTransform({+ source: createMockPythonTransformSource({+ sourceTables: {+ orders: TABLE_A.id as ConcreteTableId,+ customers: TABLE_B.id as ConcreteTableId,+ },+ }),+ }),+ });++ expect(screen.getByText("Transform source")).toBeInTheDocument();+ expect(+ screen.getByText("The data sources for this Python transform, by alias."),+ ).toBeInTheDocument();++ await waitFor(() => {+ expect(screen.queryAllByTestId("loader")).toHaveLength(0);+ });++ expect(screen.getByText("orders")).toBeInTheDocument();+ expect(screen.getByText("schema_a")).toBeInTheDocument();+ expect(screen.getByText("Table A")).toBeInTheDocument();++ expect(screen.getByText("customers")).toBeInTheDocument();+ expect(screen.getByText("schema_b")).toBeInTheDocument();+ expect(screen.getByText("Table B")).toBeInTheDocument();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/index.tsnew file mode 100644index 00000000000..364b1d65f4c--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/index.ts@@ -0,0 +1 @@+export * from "./SourceSection";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/constants.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/constants.tsnew file mode 100644index 00000000000..f62515eb84f--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/constants.ts@@ -0,0 +1 @@+export const SHARED_LIB_IMPORT_PATH = "common.py";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/index.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/index.tsxnew file mode 100644index 00000000000..29a5028c651--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/index.tsx@@ -0,0 +1,54 @@+import { t } from "ttag";++import { AdminNavItem } from "metabase/admin/components/AdminNav";+import { ForwardRefLink } from "metabase/common/components/Link";+import { Route } from "metabase/hoc/Title";+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";+import { Icon, Menu } from "metabase/ui";+import { hasPremiumFeature } from "metabase-enterprise/settings";+import { trackTransformCreate } from "metabase-enterprise/transforms/analytics";+import { FullWidthTransformPageLayout } from "metabase-enterprise/transforms/pages/TransformPageLayout";+import { getNewTransformFromTypeUrl } from "metabase-enterprise/transforms/urls";++import { PythonTransformEditor } from "./components/PythonTransformEditor";+import { SourceSection } from "./components/SourceSection";+import { SHARED_LIB_IMPORT_PATH } from "./constants";+import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";+import { PythonRunnerSettingsPage } from "./pages/PythonRunnerSettingsPage";+import { getPythonLibraryUrl } from "./urls";++if (hasPremiumFeature("transforms-python")) {+ PLUGIN_TRANSFORMS_PYTHON.PythonRunnerSettingsPage = PythonRunnerSettingsPage;+ PLUGIN_TRANSFORMS_PYTHON.TransformEditor = PythonTransformEditor;+ PLUGIN_TRANSFORMS_PYTHON.SourceSection = SourceSection;++ PLUGIN_TRANSFORMS_PYTHON.getAdminRoutes = () => (+ <Route component={FullWidthTransformPageLayout}>+ <Route path="library/:path" component={PythonLibraryEditorPage} />+ </Route>+ );++ PLUGIN_TRANSFORMS_PYTHON.getTransformsNavLinks = () => (+ <AdminNavItem+ label={t`Python library`}+ path={getPythonLibraryUrl({ path: SHARED_LIB_IMPORT_PATH })}+ icon="code_block"+ />+ );++ PLUGIN_TRANSFORMS_PYTHON.getCreateTransformsMenuItems = () => (+ <Menu.Item+ component={ForwardRefLink}+ to={getNewTransformFromTypeUrl("python")}+ leftSection={<Icon name="code_block" />}+ onClick={() => {+ trackTransformCreate({+ triggeredFrom: "transform-page-create-menu",+ creationType: "python",+ });+ }}+ >+ {t`Python script`}+ </Menu.Item>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.cssnew file mode 100644index 00000000000..7b3022ff2fd--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css@@ -0,0 +1,5 @@+.editor {+ /* HACK: 133px is the height of the headers */+ height: calc(100vh - 133px);+ flex-grow: 1;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsxnew file mode 100644index 00000000000..a6b4e971ea3--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx@@ -0,0 +1,140 @@+import { useEffect, useState } from "react";+import { t } from "ttag";++import Button from "metabase/common/components/Button";+import EditBar from "metabase/common/components/EditBar";+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { isResourceNotFoundError } from "metabase/lib/errors";+import { useMetadataToasts } from "metabase/metadata/hooks";+import { Box, Flex } from "metabase/ui";+import {+ useGetPythonLibraryQuery,+ useUpdatePythonLibraryMutation,+} from "metabase-enterprise/api/python-transform-library";++import { PythonEditor } from "../../components/PythonEditor";+import type { PythonLibraryEditorPageParams } from "../../types";++import S from "./PythonLibraryEditorPage.module.css";++type PythonLibraryEditorPageProps = {+ params: PythonLibraryEditorPageParams;+};++const EMPTY_LIBRARY_SOURCE = `+# This is your Python library.+# You can add functions and classes here that can be reused in Python transforms.+`+ .trim()+ .concat("\n");++export function PythonLibraryEditorPage({+ params,+}: PythonLibraryEditorPageProps) {+ const { path } = params;+ const [source, setSource] = useState("");++ const {+ data: library,+ isLoading,+ error,+ } = useGetPythonLibraryQuery({ path });+ const [updatePythonLibrary, { isLoading: isSaving }] =+ useUpdatePythonLibraryMutation();+ const { sendSuccessToast, sendErrorToast } = useMetadataToasts();++ function handleRevert() {+ if (isLoading) {+ return;+ }++ if (isResourceNotFoundError(error)) {+ setSource(EMPTY_LIBRARY_SOURCE);+ return;+ }++ if (library != null) {+ setSource(library.source || "");+ }+ }++ async function handleSave() {+ try {+ await updatePythonLibrary({ path, source }).unwrap();+ sendSuccessToast(t`Python library saved`);+ } catch (error) {+ sendErrorToast(t`Python library could not be saved`);+ }+ }++ // When the library loads, set the source to the current library source+ useEffect(handleRevert, [isLoading, error, library]);++ const isDirty = source !== (library?.source ?? EMPTY_LIBRARY_SOURCE);++ if (isLoading || (error && !isResourceNotFoundError(error))) {+ return (+ <Box p="md">+ <LoadingAndErrorWrapper loading={isLoading} error={error} />+ </Box>+ );+ }++ return (+ <Flex h="100%" w="100%" bg="bg-light" gap={0} direction="column">+ <LibraryEditorHeader+ onSave={handleSave}+ onRevert={handleRevert}+ isDirty={isDirty}+ isSaving={isSaving}+ />+ <PythonEditor+ value={source}+ onChange={setSource}+ withPandasCompletions+ className={S.editor}+ data-testid="python-editor"+ />+ </Flex>+ );+}++export function LibraryEditorHeader({+ isDirty,+ isSaving,+ onSave,+ onRevert,+}: {+ isDirty?: boolean;+ isSaving?: boolean;+ onSave: () => void;+ onRevert: () => void;+}) {+ return (+ <EditBar+ title={t`You are editing the shared Python library`}+ admin+ data-testid="library-editor-header"+ buttons={[+ <Button+ key="save"+ onClick={onRevert}+ primary+ small+ disabled={!isDirty || isSaving}+ >+ {t`Revert`}+ </Button>,+ <Button+ key="save"+ onClick={onSave}+ primary+ small+ disabled={!isDirty || isSaving}+ >+ {t`Save`}+ </Button>,+ ]}+ />+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/index.tsnew file mode 100644index 00000000000..3b95d285898--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/index.ts@@ -0,0 +1 @@+export * from "./PythonLibraryEditorPage";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsxnew file mode 100644index 00000000000..aa144670cf4--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx@@ -0,0 +1,67 @@+import { t } from "ttag";++import {+ SettingsPageWrapper,+ SettingsSection,+} from "metabase/admin/components/SettingsSection";+import { AdminSettingInput } from "metabase/admin/settings/components/widgets/AdminSettingInput";++export function PythonRunnerSettingsPage() {+ return (+ <SettingsPageWrapper title={t`Python Runner`}>+ <SettingsSection title={t`Service Configuration`}>+ <AdminSettingInput+ name="python-runner-url"+ title={t`Python Execution Server URL`}+ description={t`URL for the Python execution server that runs transform functions.`}+ placeholder="http://localhost:5001"+ inputType="text"+ />+ </SettingsSection>+ <SettingsSection title={t`S3 Storage Configuration`}>+ <AdminSettingInput+ name="python-storage-s-3-endpoint"+ title={t`S3 Endpoint`}+ description={t`S3 endpoint URL for storing Python execution artifacts.`}+ inputType="text"+ />+ <AdminSettingInput+ name="python-storage-s-3-region"+ title={t`S3 Region`}+ description={t`AWS region for S3 storage.`}+ inputType="text"+ />+ <AdminSettingInput+ name="python-storage-s-3-bucket"+ title={t`S3 Bucket`}+ description={t`S3 bucket name for storing Python execution artifacts.`}+ inputType="text"+ />+ <AdminSettingInput+ name="python-storage-s-3-access-key"+ title={t`S3 Access Key ID`}+ description={t`AWS access key ID for S3 authentication.`}+ inputType="password"+ />+ <AdminSettingInput+ name="python-storage-s-3-secret-key"+ title={t`S3 Secret Access Key`}+ description={t`AWS secret access key for S3 authentication.`}+ inputType="password"+ />+ <AdminSettingInput+ name="python-storage-s-3-container-endpoint"+ title={t`S3 Container Endpoint (Optional)`}+ description={t`Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.`}+ inputType="text"+ />+ <AdminSettingInput+ name="python-storage-s-3-path-style-access"+ title={t`Use Path-Style Access`}+ description={t`Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).`}+ inputType="boolean"+ />+ </SettingsSection>+ </SettingsPageWrapper>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/index.tsnew file mode 100644index 00000000000..2d7ac75c267--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/index.ts@@ -0,0 +1 @@+export * from "./PythonRunnerSettingsPage";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/types.tsnew file mode 100644index 00000000000..97f8c0deb69--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/types.ts@@ -0,0 +1,3 @@+export type PythonLibraryEditorPageParams = {+ path: string;+};diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/urls.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/urls.tsnew file mode 100644index 00000000000..6f96c6a70ea--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/urls.ts@@ -0,0 +1,5 @@+import type { PythonLibraryEditorPageParams } from "./types";++export function getPythonLibraryUrl({ path }: PythonLibraryEditorPageParams) {+ return `/admin/transforms/library/${path}`;+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts b/enterprise/frontend/src/metabase-enterprise/transforms/analytics.tsindex d0a9ca94ee0..43cbc9c96e1 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts@@ -34,7 +34,7 @@ export function trackTransformCreate({creationType,}: {triggeredFrom: "transform-page-create-menu";- creationType: "query" | "native" | "saved-question";+ creationType: "query" | "native" | "python" | "saved-question";}) {trackSimpleEvent({event: "transform_create",diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.modules.css b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.modules.cssnew file mode 100644index 00000000000..e2c84ad8653--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.modules.css@@ -0,0 +1,9 @@+.logOutput {+ background-color: var(--mb-color-bg-light);+ font-family: var(--mb-default-monospace-font-family);+ white-space: pre;+ padding: 1rem;+ overflow: auto;+ line-height: 1.4;+ border-top: 1px solid var(--mb-color-border);+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsxnew file mode 100644index 00000000000..a6c440e6413--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx@@ -0,0 +1,67 @@+import * as React from "react";+import { useEffect, useMemo, useRef, useState } from "react";+import reactAnsiStyle from "react-ansi-style";+import { t } from "ttag";++import { AnsiLogs } from "metabase/common/components/AnsiLogs";++import S from "./LogOutput.modules.css";++interface LogOutputProps {+ content?: string;+}++export function LogOutput({ content }: LogOutputProps) {+ const containerRef = useRef<HTMLDivElement>(null);+ const [isUserScrolled, setIsUserScrolled] = useState(false);+ const [shouldAutoScroll, setShouldAutoScroll] = useState(true);++ const displayContent = useMemo(() => {+ if (!content || content.trim().length === 0) {+ return t`No output`;+ }++ return reactAnsiStyle(React, content);+ }, [content]);++ // Auto-scroll to bottom when content changes+ useEffect(() => {+ if (containerRef.current && shouldAutoScroll && !isUserScrolled) {+ containerRef.current.scrollTop = containerRef.current.scrollHeight;+ }+ }, [displayContent, shouldAutoScroll, isUserScrolled]);++ // Handle scroll events to detect user scrolling+ const handleScroll = () => {+ if (!containerRef.current) {+ return;+ }++ const { scrollTop, scrollHeight, clientHeight } = containerRef.current;+ const isAtBottom = Math.abs(scrollHeight - clientHeight - scrollTop) < 5;++ if (isAtBottom) {+ setIsUserScrolled(false);+ setShouldAutoScroll(true);+ } else {+ setIsUserScrolled(true);+ setShouldAutoScroll(false);+ }+ };++ return (+ <AnsiLogs+ ref={containerRef}+ className={S.logOutput}+ onScroll={handleScroll}+ mah="12rem"+ p="sm"+ bg="bg-light"+ fz="xs"+ data-css-specificity-hack=""+ data-testid="log-output"+ >+ {displayContent}+ </AnsiLogs>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.tsnew file mode 100644index 00000000000..a8aa12bfc51--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts@@ -0,0 +1 @@+export { LogOutput } from "./LogOutput";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsxindex 39ecd1f7901..3cd496d6939 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx@@ -9,7 +9,10 @@ import { NativeQueryPreview } from "metabase/querying/notebook/components/Nativeimport { Center, Flex, Loader, Modal, Stack } from "metabase/ui";import * as Lib from "metabase-lib";import type Question from "metabase-lib/v1/Question";-import type { DatasetQuery, NativeQuerySnippet } from "metabase-types/api";+import type {+ NativeQuerySnippet,+ QueryTransformSource,+} from "metabase-types/api";import { useQueryMetadata } from "../../hooks/use-query-metadata";import { useQueryResults } from "../../hooks/use-query-results";@@ -32,21 +35,23 @@ import {} from "./utils";type QueryEditorProps = {- initialQuery: DatasetQuery;+ initialSource: QueryTransformSource;isNew?: boolean;isSaving?: boolean;- onSave: (newQuery: DatasetQuery) => void;+ onSave: (source: QueryTransformSource) => void;onCancel: () => void;};export function QueryEditor({- initialQuery,+ initialSource,isNew = true,isSaving = false,onSave,onCancel,}: QueryEditorProps) {- const { question, isQueryDirty, setQuestion } = useQueryState(initialQuery);+ const { question, isQueryDirty, setQuestion } = useQueryState(+ initialSource.query,+ );const { isInitiallyLoaded } = useQueryMetadata(question);const {result,@@ -67,7 +72,7 @@ export function QueryEditor({};const handleSave = () => {- onSave(question.datasetQuery());+ onSave({ type: "query", query: question.datasetQuery() });};const handleCmdEnter = () => {diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/index.tsdeleted file mode 100644index 19070a74f88..00000000000--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/index.ts+++ /dev/null@@ -1 +0,0 @@-export * from "./RunErrorInfo";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.module.css b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.module.csssimilarity index 100%rename from enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.module.cssrename to enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.module.cssdiff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.tsxsimilarity index 60%rename from enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.tsxrename to enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.tsxindex fee2d7ebed7..753c7476edb 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.tsx@@ -16,15 +16,17 @@ import {Stack,Tooltip,} from "metabase/ui";+import type { TransformRunStatus } from "metabase-types/api";-import S from "./RunErrorInfo.module.css";+import S from "./RunInfo.module.css";-type RunErrorInfoProps = {+type RunInfoProps = {+ status: TransformRunStatus;message: string;endTime: Date | null;};-export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {+export function RunInfo({ status, message, endTime }: RunInfoProps) {const [isOpened, { open, close }] = useDisclosure();const handleIconClick = (event: MouseEvent) => {@@ -38,16 +40,20 @@ export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {event.stopPropagation();};+ if (!message || (status !== "failed" && status !== "succeeded")) {+ return null;+ }+return (<>- <Tooltip label={t`See error`}>+ <Tooltip label={getTooltip(status)}><ActionIcon aria-label={t`See error`} onClick={handleIconClick}><Icon name="document" /></ActionIcon></Tooltip>{isOpened && (<Modal- title={getTitle(endTime)}+ title={getTitle(status, endTime)}size="xl"padding="xl"opened@@ -55,17 +61,34 @@ export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {onClose={close}><FocusTrap.InitialFocus />- <RunErrorModalContent message={message} onClose={close} />+ <RunInfoModalContent message={message} onClose={close} /></Modal>)}</>);}-function getTitle(endTime: Date | null) {- return endTime- ? t`Failed at ${dayjs(endTime).format("lll")}, with this error`- : t`Failed with this error`;+function getTooltip(status: TransformRunStatus) {+ if (status === "failed") {+ return t`See error`;+ }+ if (status === "succeeded") {+ return t`See logs`;+ }+}++function getTitle(status: TransformRunStatus, endTime: Date | null) {+ if (status === "failed") {+ return endTime+ ? t`Failed at ${dayjs(endTime).format("lll")}, with this error`+ : t`Failed with this error`;+ }+ if (status === "succeeded") {+ return endTime+ ? t`Succeeded at ${dayjs(endTime).format("lll")}, with the following logs`+ : t`Succeeded with the following logs`;+ }+ return null;}type RunErrorModalContentProps = {@@ -73,7 +96,7 @@ type RunErrorModalContentProps = {onClose: () => void;};-function RunErrorModalContent({ message, onClose }: RunErrorModalContentProps) {+function RunInfoModalContent({ message, onClose }: RunErrorModalContentProps) {return (<Stack pt="md" gap="lg"><Box className={S.codeContainer} pos="relative" pr="lg">diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.unit.spec.tsxsimilarity index 87%rename from enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.unit.spec.tsxrename to enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.unit.spec.tsxindex ae28491c513..f547e78a858 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.unit.spec.tsx@@ -2,7 +2,7 @@ import userEvent from "@testing-library/user-event";import { renderWithProviders, screen } from "__support__/ui";-import { RunErrorInfo } from "./RunErrorInfo";+import { RunInfo } from "./RunInfo";type SetupOpts = {message?: string;@@ -13,10 +13,12 @@ function setup({message = "Test error message",endTime = null,}: SetupOpts = {}) {- renderWithProviders(<RunErrorInfo message={message} endTime={endTime} />);+ renderWithProviders(+ <RunInfo status="failed" message={message} endTime={endTime} />,+ );}-describe("RunErrorInfo", () => {+describe("RunInfo", () => {it("should show the error message when there is a date", async () => {const message = "Test error with end time";const endTime = new Date("2023-10-15T14:30:00");diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/index.tsnew file mode 100644index 00000000000..cbcab61e33b--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/index.ts@@ -0,0 +1 @@+export * from "./RunInfo";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsxindex 70cdefa20ff..d20531767ce 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsx@@ -5,7 +5,7 @@ import { Box, Group, Icon } from "metabase/ui";import { parseTimestampWithTimezone } from "metabase-enterprise/transforms/utils";import type { TransformRun } from "metabase-types/api";-import { RunErrorInfo } from "../RunErrorInfo";+import { RunInfo } from "../RunInfo";type RunStatusSectionProps = {run: TransformRun | null;@@ -38,7 +38,8 @@ export function RunStatus({const errorInfo =message != null ? (- <RunErrorInfo+ <RunInfo+ status={status}message={message}endTime={endTime ? endTime.toDate() : null}/>diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsxindex e4d9206b3e7..2ef6e1b2c57 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx@@ -3,7 +3,7 @@ import type { Transform, TransformRunStatus } from "metabase-types/api";import { formatStatus } from "../../utils";import { RunCancelButton } from "../RunCancelButton/RunCancelButton";-import { RunErrorInfo } from "../RunErrorInfo";+import { RunInfo } from "../RunInfo";import S from "./RunStatusInfo.module.css";@@ -26,7 +26,7 @@ export function RunStatusInfo({<Group gap="xs" className={S.runStatusInfo} wrap="nowrap"><Box c={isError ? "error" : undefined}>{formatStatus(status)}</Box>{isError && message != null && (- <RunErrorInfo message={message} endTime={endTime} />+ <RunInfo status={status} message={message} endTime={endTime} />)}<RunCancelButton transform={transform} status={status} /></Group>diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/index.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/index.tsxindex 19724eef476..af18a28c5f1 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/index.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/index.tsx@@ -3,13 +3,13 @@ import { t } from "ttag";import { createAdminRouteGuard } from "metabase/admin/utils";import { Route } from "metabase/hoc/Title";-import { PLUGIN_TRANSFORMS } from "metabase/plugins";+import { PLUGIN_TRANSFORMS, PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";import { hasPremiumFeature } from "metabase-enterprise/settings";import { JobListPage } from "./pages/JobListPage";import { JobPage } from "./pages/JobPage";import { NewJobPage } from "./pages/NewJobPage";-import { NewTransformQueryPage } from "./pages/NewTransformQueryPage";+import { NewTransformPage } from "./pages/NewTransformPage";import { RunListPage } from "./pages/RunListPage";import { TransformListPage } from "./pages/TransformListPage";import { TransformPage } from "./pages/TransformPage";@@ -34,8 +34,9 @@ if (hasPremiumFeature("transforms")) {<Route path="jobs/:jobId" component={JobPage} /><Route path=":transformId" component={TransformPage} /></Route>- <Route path="new/:type" component={NewTransformQueryPage} />- <Route path="new/card/:cardId" component={NewTransformQueryPage} />+ {PLUGIN_TRANSFORMS_PYTHON.getAdminRoutes()}+ <Route path="new/:type" component={NewTransformPage} />+ <Route path="new/card/:cardId" component={NewTransformPage} /><Route path=":transformId/query" component={TransformQueryPage} /></Route></Route>diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsxsimilarity index 91%rename from enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsxrename to enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsxindex e040f013a52..a1b9fbfa259 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx@@ -24,18 +24,18 @@ import { trackTransformCreated } from "metabase-enterprise/transforms/analytics"import { SchemaFormSelect } from "metabase-enterprise/transforms/components/SchemaFormSelect";import type {CreateTransformRequest,- DatasetQuery,Transform,+ TransformSource,} from "metabase-types/api";type CreateTransformModalProps = {- query: DatasetQuery;+ source: TransformSource;onCreate: (transform: Transform) => void;onClose: () => void;};export function CreateTransformModal({- query,+ source,onCreate,onClose,}: CreateTransformModalProps) {@@ -43,7 +43,7 @@ export function CreateTransformModal({<Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}><FocusTrap.InitialFocus /><CreateTransformForm- query={query}+ source={source}onCreate={onCreate}onClose={onClose}/>@@ -52,7 +52,7 @@ export function CreateTransformModal({}type CreateTransformFormProps = {- query: DatasetQuery;+ source: TransformSource;onCreate: (transform: Transform) => void;onClose: () => void;};@@ -72,11 +72,12 @@ const NEW_TRANSFORM_SCHEMA = Yup.object({});function CreateTransformForm({- query,+ source,onCreate,onClose,}: CreateTransformFormProps) {- const { database: databaseId } = query;+ const databaseId =+ source.type === "query" ? source.query.database : source["source-database"];const {data: database,@@ -108,7 +109,10 @@ function CreateTransformForm({}const handleSubmit = async (values: NewTransformValues) => {- const request = getCreateRequest(query, values);+ if (!databaseId) {+ throw new Error("Database ID is required");+ }+ const request = getCreateRequest(source, values, databaseId);const transform = await createTransform(request).unwrap();trackTransformCreated({ transformId: transform.id });@@ -171,20 +175,19 @@ function getInitialValues(schemas: string[]): NewTransformValues {}function getCreateRequest(- query: DatasetQuery,+ source: TransformSource,{ name, description, targetName, targetSchema }: NewTransformValues,+ databaseId: number,): CreateTransformRequest {return {name: name,description,- source: {- type: "query",- query,- },+ source,target: {type: "table",name: targetName,schema: targetSchema,+ database: databaseId,},};}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/index.tssimilarity index 100%rename from enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/index.tsrename to enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/index.tsdiff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsxnew file mode 100644index 00000000000..43d04860217--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx@@ -0,0 +1,144 @@+import { useDisclosure } from "@mantine/hooks";+import { useState } from "react";+import { push } from "react-router-redux";++import { skipToken, useGetCardQuery } from "metabase/api";+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { useDispatch } from "metabase/lib/redux";+import * as Urls from "metabase/lib/urls";+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";+import type {+ CardId,+ DatasetQuery,+ Transform,+ TransformSource,+} from "metabase-types/api";++import { QueryEditor } from "../../components/QueryEditor";+import { getTransformListUrl, getTransformUrl } from "../../urls";++import { CreateTransformModal } from "./CreateTransformModal";+import {+ getInitialPythonTransformSource,+ getInitialQueryTransformSource,+} from "./utils";++type NewTransformPageParams = {+ type?: string;+ cardId?: string;+};++type NewTransformPageParsedParams = {+ type: DatasetQuery["type"] | "python";+ cardId?: CardId;+};++type NewTransformPageProps = {+ params: NewTransformPageParams;+};++export function NewTransformPage({ params }: NewTransformPageProps) {+ const { type, cardId } = getParsedParams(params);++ const [source, setSource] = useState<TransformSource | null>(null);+ const [isModalOpened, { open: openModal, close: closeModal }] =+ useDisclosure();+ const dispatch = useDispatch();++ const handleCreate = (transform: Transform) => {+ dispatch(push(getTransformUrl(transform.id)));+ };++ const handleSave = (newSource: TransformSource) => {+ setSource(newSource);+ openModal();+ };++ const handleCancel = () => {+ dispatch(push(getTransformListUrl()));+ };++ return (+ <>+ <NewTransformEditorBody+ type={type}+ cardId={cardId}+ onSave={handleSave}+ onCancel={handleCancel}+ />+ {isModalOpened && source !== null && (+ <CreateTransformModal+ source={source}+ onCreate={handleCreate}+ onClose={closeModal}+ />+ )}+ </>+ );+}++function NewTransformEditorBody(props: {+ type: DatasetQuery["type"] | "python";+ cardId?: CardId;+ onSave: (source: TransformSource) => void;+ onCancel: () => void;+}) {+ const { type, ...rest } = props;+ if (type === "python") {+ return (+ <PLUGIN_TRANSFORMS_PYTHON.TransformEditor+ initialSource={getInitialPythonTransformSource()}+ isNew+ onSave={props.onSave}+ onCancel={props.onCancel}+ />+ );+ }++ return <NewQueryTransformEditorBody {...rest} type={type} />;+}++function NewQueryTransformEditorBody({+ type,+ cardId,+ onSave,+ onCancel,+}: {+ type: DatasetQuery["type"];+ cardId?: CardId;+ onSave: (source: TransformSource) => void;+ onCancel: () => void;+}) {+ const {+ data: card,+ isLoading,+ error,+ } = useGetCardQuery(cardId ? { id: cardId } : skipToken);++ if (isLoading || error) {+ return <LoadingAndErrorWrapper loading={isLoading} error={error} />;+ }++ return (+ <QueryEditor+ initialSource={getInitialQueryTransformSource(card, type)}+ isNew+ onSave={onSave}+ onCancel={onCancel}+ />+ );+}++function getParsedParams({+ type,+ cardId,+}: NewTransformPageParams): NewTransformPageParsedParams {+ if (type === "python") {+ return { type: "python" };+ }++ return {+ type: type === "native" ? "native" : "query",+ cardId: cardId != null ? Urls.extractEntityId(cardId) : undefined,+ };+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/index.tsnew file mode 100644index 00000000000..abaf276bac8--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/index.ts@@ -0,0 +1 @@+export * from "./NewTransformPage";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.tsnew file mode 100644index 00000000000..7c71e2bdd2e--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts@@ -0,0 +1,36 @@+import Question from "metabase-lib/v1/Question";+import type { Card, DatasetQuery } from "metabase-types/api";++export function getInitialQueryTransformSource(+ card: Card | undefined,+ type: DatasetQuery["type"] | undefined,+) {+ const query =+ card != null+ ? card.dataset_query+ : Question.create({ type }).datasetQuery();++ return { type: "query" as const, query };+}++export function getInitialPythonTransformSource() {+ return {+ type: "python" as const,+ "source-database": undefined,+ "source-tables": {},+ body: `# Write your Python transformation script here+import pandas as pd++def transform():+ """+ Your transformation function.++ Select tables above to add them as function parameters.++ Returns:+ DataFrame to write to the destination table+ """+ # Your transformation logic here+ return pd.DataFrame([{"message": "Hello from Python transform!"}])`,+ };+}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsxdeleted file mode 100644index 313c241fa08..00000000000--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx+++ /dev/null@@ -1,105 +0,0 @@-import { useDisclosure } from "@mantine/hooks";-import { useState } from "react";-import { push } from "react-router-redux";--import { skipToken, useGetCardQuery } from "metabase/api";-import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";-import { useDispatch } from "metabase/lib/redux";-import * as Urls from "metabase/lib/urls";-import Question from "metabase-lib/v1/Question";-import type { Card, CardId, DatasetQuery, Transform } from "metabase-types/api";--import { QueryEditor } from "../../components/QueryEditor";-import { getTransformListUrl, getTransformUrl } from "../../urls";--import { CreateTransformModal } from "./CreateTransformModal";--type NewTransformQueryPageParams = {- type?: string;- cardId?: string;-};--type NewTransformQueryPageParsedParams = {- type?: DatasetQuery["type"];- cardId?: CardId;-};--type NewTransformQueryPageProps = {- params: NewTransformQueryPageParams;-};--export function NewTransformQueryPage({ params }: NewTransformQueryPageProps) {- const { type, cardId } = getParsedParams(params);- const {- data: card,- isLoading,- error,- } = useGetCardQuery(cardId ? { id: cardId } : skipToken);-- if (isLoading || error) {- return <LoadingAndErrorWrapper loading={isLoading} error={error} />;- }-- return <NewTransformPageBody initialQuery={getInitialQuery(card, type)} />;-}--type NewTransformPageBodyProps = {- initialQuery: DatasetQuery;-};--function NewTransformPageBody({ initialQuery }: NewTransformPageBodyProps) {- const [query, setQuery] = useState(initialQuery);- const [isModalOpened, { open: openModal, close: closeModal }] =- useDisclosure();- const dispatch = useDispatch();-- const handleCreate = (transform: Transform) => {- dispatch(push(getTransformUrl(transform.id)));- };-- const handleSaveClick = (newQuery: DatasetQuery) => {- setQuery(newQuery);- openModal();- };-- const handleCancelClick = () => {- dispatch(push(getTransformListUrl()));- };-- return (- <>- <QueryEditor- initialQuery={initialQuery}- isNew- onSave={handleSaveClick}- onCancel={handleCancelClick}- />- {isModalOpened && (- <CreateTransformModal- query={query}- onCreate={handleCreate}- onClose={closeModal}- />- )}- </>- );-}--function getParsedParams({- type,- cardId,-}: NewTransformQueryPageParams): NewTransformQueryPageParsedParams {- return {- type: type === "native" ? "native" : "query",- cardId: cardId != null ? Urls.extractEntityId(cardId) : undefined,- };-}--function getInitialQuery(- card: Card | undefined,- type: DatasetQuery["type"] | undefined,-) {- return card != null- ? card.dataset_query- : Question.create({ type }).datasetQuery();-}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/index.tsdeleted file mode 100644index f4f8bafb498..00000000000--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/index.ts+++ /dev/null@@ -1 +0,0 @@-export * from "./NewTransformQueryPage";diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsxindex ce5850d9c0d..4432802e828 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx@@ -10,6 +10,7 @@ import {type QuestionPickerValueItem,} from "metabase/common/components/Pickers/QuestionPicker";import { useDispatch } from "metabase/lib/redux";+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";import { Button, Center, Icon, Loader, Menu } from "metabase/ui";import { trackTransformCreate } from "metabase-enterprise/transforms/analytics";import { doesDatabaseSupportTransforms } from "metabase-enterprise/transforms/utils";@@ -85,8 +86,9 @@ export function CreateTransformMenu() {>{t`SQL query`}</Menu.Item>+ {PLUGIN_TRANSFORMS_PYTHON.getCreateTransformsMenuItems()}<Menu.Item- leftSection={<Icon name="copy" />}+ leftSection={<Icon name="folder" />}onClick={handleSavedQuestionClick}>{t`A copy of a saved question`}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsxindex ce4055a5dce..828205fbbeb 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx@@ -6,6 +6,7 @@ import { t } from "ttag";import { useDispatch } from "metabase/lib/redux";import { useMetadataToasts } from "metabase/metadata/hooks";import { Button, Card, Group, Icon } from "metabase/ui";+import { PythonEditor } from "metabase-enterprise/transforms-python/components/PythonEditor";import type { Transform } from "metabase-types/api";import { QueryView } from "../../../components/QueryView";@@ -20,9 +21,12 @@ type ManageSectionProps = {};export function ManageSection({ transform }: ManageSectionProps) {+ const sectionLabel =+ transform.source.type === "python" ? t`Python script` : t`Query`;+return (<TitleSection- label={t`Query`}+ label={sectionLabel}rightSection={<Group><EditQueryButton transform={transform} />@@ -31,7 +35,12 @@ export function ManageSection({ transform }: ManageSectionProps) {}><Card p={0} shadow="none" withBorder>- <QueryView query={transform.source.query} />+ {transform.source.type === "query" && (+ <QueryView query={transform.source.query} />+ )}+ {transform.source.type === "python" && (+ <PythonEditor value={transform.source.body} readOnly />+ )}</Card></TitleSection>);@@ -43,6 +52,8 @@ type EditQueryButtonProps = {function EditQueryButton({ transform }: EditQueryButtonProps) {const isDisabled = isTransformRunning(transform);+ const buttonText =+ transform.source.type === "python" ? t`Edit script` : t`Edit query`;return (<Button@@ -51,7 +62,7 @@ function EditQueryButton({ transform }: EditQueryButtonProps) {leftSection={<Icon name="pencil_lines" aria-hidden />}disabled={isDisabled}>- {t`Edit query`}+ {buttonText}</Button>);}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsxindex 2d7bad079d2..a415209fac0 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx@@ -13,6 +13,7 @@ import {useUpdateTransformMutation,} from "metabase-enterprise/api";import { trackTransformTriggerManualRun } from "metabase-enterprise/transforms/analytics";+import { LogOutput } from "metabase-enterprise/transforms/components/LogOutput";import type { Transform, TransformTagId } from "metabase-types/api";import { RunButton } from "../../../components/RunButton";@@ -32,10 +33,13 @@ export function RunSection({ transform }: RunSectionProps) {description={t`This transform will be run whenever the jobs it belongs to are scheduled.`}data-testid="run-section">- <Group p="lg" justify="space-between">- <RunStatusSection transform={transform} />- <RunButtonSection transform={transform} />- </Group>+ <Stack>+ <Group p="lg" justify="space-between">+ <RunStatusSection transform={transform} />+ <RunButtonSection transform={transform} />+ </Group>+ <RunOutputSection transform={transform} />+ </Stack><Divider /><Group p="lg" gap="lg"><Stack gap="sm">@@ -138,6 +142,18 @@ function RunButtonSection({ transform }: RunButtonSectionProps) {);}+function RunOutputSection({ transform }: RunSectionProps) {+ if (!transform?.last_run?.message) {+ return null;+ }+ const { status, message } = transform.last_run;+ if (status !== "started" && status !== "succeeded") {+ return null;+ }++ return <LogOutput content={message} />;+}+type TagSectionProps = {transform: Transform;};diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsxindex 114ff238e74..7a9540c47bc 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx@@ -18,6 +18,7 @@ import {Loader,Text,} from "metabase/ui";+import { sourceDatabaseId } from "metabase-enterprise/transforms/utils";import type { Transform } from "metabase-types/api";import { SplitSection } from "../../../components/SplitSection";@@ -59,7 +60,7 @@ type TargetInfoProps = {function TargetInfo({ transform }: TargetInfoProps) {const { source, target, table } = transform;- const { database: databaseId } = source.query;+ const databaseId = sourceDatabaseId(source);const { data: databaseFromApi, isLoading: isDatabaseLoading } =useGetDatabaseQuery(diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsxindex ab1d27cbf43..d4dfc693389 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx@@ -31,6 +31,7 @@ import {useUpdateTransformMutation,} from "metabase-enterprise/api";import { SchemaFormSelect } from "metabase-enterprise/transforms/components/SchemaFormSelect";+import { sourceDatabaseId } from "metabase-enterprise/transforms/utils";import type { Transform, UpdateTransformRequest } from "metabase-types/api";type UpdateTargetModalProps = {@@ -83,7 +84,7 @@ function UpdateTargetForm({onClose,}: UpdateTargetFormProps) {const { source, target, table } = transform;- const { database: databaseId } = source.query;+ const databaseId = sourceDatabaseId(source);const [updateTransform] = useUpdateTransformMutation();const [deleteTransformTarget] = useDeleteTransformTargetMutation();const initialValues = useMemo(() => getInitialValues(transform), [transform]);@@ -112,10 +113,15 @@ function UpdateTargetForm({}const handleSubmit = async (values: EditTransformValues) => {+ if (!databaseId) {+ throw new Error("Database ID is required");+ }if (shouldDeleteTarget) {await deleteTransformTarget(transform.id).unwrap();}- await updateTransform(getUpdateRequest(transform, values)).unwrap();+ await updateTransform(+ getUpdateRequest(transform, values, databaseId),+ ).unwrap();onUpdate();};@@ -198,6 +204,7 @@ function getSubmitButtonColor(shouldDeleteTarget: boolean) {function getUpdateRequest({ id }: Transform,{ name, schema }: EditTransformValues,+ databaseId: number,): UpdateTransformRequest {return {id,@@ -205,6 +212,7 @@ function getUpdateRequest(type: "table",name,schema,+ database: databaseId,},};}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsxindex 10a5c8c96cb..88c00236223 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx@@ -4,6 +4,7 @@ import { t } from "ttag";import { skipToken } from "metabase/api";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";import * as Urls from "metabase/lib/urls";+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";import { Stack } from "metabase/ui";import { useGetTransformQuery } from "metabase-enterprise/api";import type { Transform, TransformId } from "metabase-types/api";@@ -64,6 +65,7 @@ export function TransformPage({ params }: TransformPageProps) {<NameSection transform={transform} /></Stack><RunSection transform={transform} />+ <PLUGIN_TRANSFORMS_PYTHON.SourceSection transform={transform} /><TargetSection transform={transform} /><ManageSection transform={transform} /><DependenciesSection transform={transform} />diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsxindex 79df4aeacc3..ad4d9853f4a 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx@@ -7,6 +7,7 @@ import {} from "metabase/admin/components/AdminNav";import { AdminSettingsLayout } from "metabase/common/components/AdminLayout/AdminSettingsLayout";import { useSelector } from "metabase/lib/redux";+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";import { getLocation } from "metabase/selectors/routing";import { getJobListUrl, getRunListUrl, getTransformListUrl } from "../../urls";@@ -18,6 +19,7 @@ type TransformPageLayoutPropsParams = {type TransformPageLayoutProps = {params: TransformPageLayoutPropsParams;+ fullWidth?: boolean;children?: ReactNode;};@@ -47,18 +49,24 @@ export function DetailsPageLayout({function TransformPageLayout({params,maw,+ fullWidth,children,}: TransformPageLayoutOwnProps) {return (<AdminSettingsLayoutsidebar={<TransformPageSidebar params={params} />}maw={maw}+ fullWidth={fullWidth}>{children}</AdminSettingsLayout>);}+export function FullWidthTransformPageLayout(props: TransformPageLayoutProps) {+ return <TransformPageLayout {...props} fullWidth />;+}+type TransformPageSidebarProps = {params: TransformPageLayoutPropsParams;};@@ -91,6 +99,7 @@ function TransformPageSidebar({ params }: TransformPageSidebarProps) {icon="list"active={pathname === runListUrl}/>+ {PLUGIN_TRANSFORMS_PYTHON.getTransformsNavLinks()}</AdminNavWrapper>);}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsxindex 2d1b29b76de..87fac2e71e8 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx@@ -6,11 +6,12 @@ import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrimport { useDispatch } from "metabase/lib/redux";import * as Urls from "metabase/lib/urls";import { useMetadataToasts } from "metabase/metadata/hooks";+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";import {useGetTransformQuery,useUpdateTransformMutation,} from "metabase-enterprise/api";-import type { DatasetQuery, Transform } from "metabase-types/api";+import type { Transform, TransformSource } from "metabase-types/api";import { QueryEditor } from "../../components/QueryEditor";import { getTransformUrl } from "../../urls";@@ -53,13 +54,10 @@ export function TransformQueryPageBody({const dispatch = useDispatch();const { sendErrorToast, sendSuccessToast } = useMetadataToasts();- const handleSave = async (query: DatasetQuery) => {+ const handleSourceSave = async (source: TransformSource) => {const { error } = await updateTransform({id: transform.id,- source: {- type: "query",- query,- },+ source,});if (error) {@@ -74,12 +72,23 @@ export function TransformQueryPageBody({dispatch(push(getTransformUrl(transform.id)));};+ if (transform.source.type === "python") {+ return (+ <PLUGIN_TRANSFORMS_PYTHON.TransformEditor+ initialSource={transform.source}+ isNew={false}+ isSaving={isLoading}+ onSave={handleSourceSave}+ onCancel={handleCancel}+ />+ );+ }return (<QueryEditor- initialQuery={transform.source.query}+ initialSource={transform.source}isNew={false}isSaving={isLoading}- onSave={handleSave}+ onSave={handleSourceSave}onCancel={handleCancel}/>);diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts b/enterprise/frontend/src/metabase-enterprise/transforms/urls.tsindex 22783c881e9..143828b9c07 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts@@ -15,7 +15,9 @@ export function getTransformListUrl() {return ROOT_URL;}-export function getNewTransformFromTypeUrl(type: DatasetQuery["type"]) {+export function getNewTransformFromTypeUrl(+ type: DatasetQuery["type"] | "python",+) {return `${ROOT_URL}/new/${type}`;}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/utils.tsindex 28cce8088fb..9e93c5ddd7c 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts@@ -4,8 +4,10 @@ import { hasFeature } from "metabase/admin/databases/utils";import { parseTimestamp } from "metabase/lib/time-dayjs";import type {Database,+ DatabaseId,TransformRunMethod,TransformRunStatus,+ TransformSource,} from "metabase-types/api";export function parseTimestampWithTimezone(@@ -62,3 +64,15 @@ export function doesDatabaseSupportTransforms(database?: Database): boolean {hasFeature(database, "transforms/table"));}++export function sourceDatabaseId(source: TransformSource): DatabaseId | null {+ if (source.type === "query") {+ return source.query.database;+ }++ if (source.type === "python") {+ return source["source-database"];+ }++ return null;+}diff --git a/frontend/src/metabase-types/analytics/event.ts b/frontend/src/metabase-types/analytics/event.tsindex 406e71fc769..136f17be399 100644--- a/frontend/src/metabase-types/analytics/event.ts+++ b/frontend/src/metabase-types/analytics/event.ts@@ -297,7 +297,7 @@ export type TransformJobTriggerManualRunEvent = ValidateEvent<{export type TransformCreateEvent = ValidateEvent<{event: "transform_create";triggered_from: "transform-page-create-menu";- event_detail: "query" | "native" | "saved-question";+ event_detail: "query" | "native" | "python" | "saved-question";}>;export type TransformCreatedEvent = ValidateEvent<{diff --git a/frontend/src/metabase-types/api/database.ts b/frontend/src/metabase-types/api/database.tsindex 35d3414f13e..13e19eb58ef 100644--- a/frontend/src/metabase-types/api/database.ts+++ b/frontend/src/metabase-types/api/database.ts@@ -54,6 +54,7 @@ export type DatabaseFeature =| "distinct-where"| "saved-question-sandboxing"| "split-part"+ | "transforms/python"| "transforms/table";export interface Database extends DatabaseData {diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.tsindex de38f9568bf..fc1a8e7b88f 100644--- a/frontend/src/metabase-types/api/mocks/settings.ts+++ b/frontend/src/metabase-types/api/mocks/settings.ts@@ -139,6 +139,7 @@ export const createMockTokenFeatures = (documents: false,semantic_search: false,transforms: false,+ "transforms-python": false,...opts,});diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.tsindex 12b76827841..ef8c1333be4 100644--- a/frontend/src/metabase-types/api/mocks/transform.ts+++ b/frontend/src/metabase-types/api/mocks/transform.ts@@ -1,4 +1,6 @@import type {+ DatabaseId,+ PythonTransformTableAliases,Transform,TransformJob,TransformRun,@@ -9,13 +11,27 @@ import type {import { createMockStructuredDatasetQuery } from "./query";-export function createMockTransformSource(- opts?: Partial<TransformSource>,-): TransformSource {+export function createMockTransformSource(): TransformSource {return {type: "query",query: createMockStructuredDatasetQuery(),- ...opts,+ };+}++export function createMockPythonTransformSource({+ sourceDatabase = 1,+ sourceTables = {},+ body = "# Python script\nprint('Hello, world!')",+}: {+ sourceDatabase?: DatabaseId;+ sourceTables?: PythonTransformTableAliases;+ body?: string;+}): TransformSource {+ return {+ type: "python",+ body,+ "source-database": sourceDatabase,+ "source-tables": sourceTables,};}@@ -26,6 +42,7 @@ export function createMockTransformTarget(type: "table",name: "Table",schema: null,+ database: 1,...opts,};}diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex 103030c0b53..1e87e5ac8e3 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -324,6 +324,7 @@ export const tokenFeatures = ["documents","semantic_search","transforms",+ "transforms-python",] as const;export type TokenFeature = (typeof tokenFeatures)[number];@@ -677,6 +678,15 @@ export interface EnterpriseSettings extends Settings {"database-replication-connections"?: DatabaseReplicationConnections | null;"embedding-hub-test-embed-snippet-created": boolean;"embedding-hub-production-embed-snippet-created": boolean;+ "python-runner-url"?: string | null;+ "python-runner-api-token"?: string | null;+ "python-storage-s-3-endpoint"?: string | null;+ "python-storage-s-3-region"?: string | null;+ "python-storage-s-3-bucket"?: string | null;+ "python-storage-s-3-access-key"?: string | null;+ "python-storage-s-3-secret-key"?: string | null;+ "python-storage-s-3-container-endpoint"?: string | null;+ "python-storage-s-3-path-style-access"?: boolean | null;/*** @deprecated*/diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex 211cf76ad69..4987a03f2e7 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -1,7 +1,8 @@+import type { DatabaseId } from "./database";import type { PaginationRequest, PaginationResponse } from "./pagination";import type { DatasetQuery } from "./query";import type { ScheduleDisplayType } from "./settings";-import type { Table } from "./table";+import type { ConcreteTableId, Table } from "./table";export type TransformId = number;export type TransformTagId = number;@@ -23,17 +24,28 @@ export type Transform = {last_run?: TransformRun | null;};-export type TransformSource = {+export type PythonTransformTableAliases = Record<string, ConcreteTableId>;++export type PythonTransformSource = {+ type: "python";+ body: string;+ "source-database": DatabaseId;+ "source-tables": PythonTransformTableAliases;+};+export type QueryTransformSource = {type: "query";query: DatasetQuery;};+export type TransformSource = QueryTransformSource | PythonTransformSource;+export type TransformTargetType = "table";export type TransformTarget = {type: TransformTargetType;name: string;schema: string | null;+ database: number;};export type TransformRun = {@@ -146,3 +158,31 @@ export type ListTransformRunsRequest = {export type ListTransformRunsResponse = {data: TransformRun[];} & PaginationResponse;++export type ExecutePythonTransformRequest = {+ code: string;+ tables: PythonTransformTableAliases;+};++export type ExecutePythonTransformResponse = {+ output?: string;+ stdout?: string;+ stderr?: string;+ error?: string;+ exit_code?: number;+ timeout?: boolean;+};++export type PythonLibrary = {+ path: string;+ source: string;+};++export type GetPythonLibraryRequest = {+ path: string;+};++export type UpdatePythonLibraryRequest = {+ path: string;+ source: string;+};diff --git a/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx b/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsxindex e8f0ba2c059..5130adab58b 100644--- a/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx+++ b/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx@@ -23,6 +23,7 @@ export function SettingsNav() {const hasSaml = useHasTokenFeature("sso_saml");const hasJwt = useHasTokenFeature("sso_jwt");const hasScim = useHasTokenFeature("scim");+ const hasPythonTransforms = useHasTokenFeature("transforms-python");return (<AdminNavWrapper>@@ -87,6 +88,13 @@ export function SettingsNav() {</SettingsNavItem><NavDivider /><SettingsNavItem path="uploads" label={t`Uploads`} icon="upload" />+ {hasPythonTransforms && (+ <SettingsNavItem+ path="python-runner"+ label={t`Python Runner`}+ icon="snippet"+ />+ )}<SettingsNavItempath="public-sharing"label={t`Public sharing`}diff --git a/frontend/src/metabase/admin/settingsRoutes.tsx b/frontend/src/metabase/admin/settingsRoutes.tsxindex e49f855c47d..4b21388111c 100644--- a/frontend/src/metabase/admin/settingsRoutes.tsx+++ b/frontend/src/metabase/admin/settingsRoutes.tsx@@ -2,7 +2,10 @@ import { IndexRedirect, Route } from "react-router";import { AdminSettingsLayout } from "metabase/common/components/AdminLayout/AdminSettingsLayout";import { NotFound } from "metabase/common/components/ErrorPages";-import { PLUGIN_AUTH_PROVIDERS } from "metabase/plugins";+import {+ PLUGIN_AUTH_PROVIDERS,+ PLUGIN_TRANSFORMS_PYTHON,+} from "metabase/plugins";import { GoogleAuthForm } from "./settings/auth/components/GoogleAuthForm";import { SettingsLdapForm } from "./settings/components/SettingsLdapForm";@@ -58,6 +61,10 @@ export const getSettingsRoutes = () => (<Route path="maps" component={MapsSettingsPage} /><Route path="localization" component={LocalizationSettingsPage} /><Route path="uploads" component={UploadSettingsPage} />+ <Route+ path="python-runner"+ component={PLUGIN_TRANSFORMS_PYTHON.PythonRunnerSettingsPage}+ /><Route path="public-sharing" component={PublicSharingSettingsPage} /><Route path="license" component={LicenseSettingsPage} /><Route path="appearance" component={() => <AppearanceSettingsPage />} />diff --git a/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsx b/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsxindex 57005a4cd0d..8939afa159e 100644--- a/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsx+++ b/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsx@@ -1,24 +1,7 @@// eslint-disable-next-line no-restricted-imports-import { css } from "@emotion/react";-// eslint-disable-next-line no-restricted-importsimport styled from "@emotion/styled";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";-import { color } from "metabase/lib/colors";--const LOG_PREFIX = ".react-ansi-style";--const LOG_COLORS = {- black: color("text-dark"),- white: color("text-white"),- gray: color("text-medium"),- red: color("saturated-red"),- green: color("saturated-green"),- yellow: color("saturated-yellow"),- blue: color("saturated-blue"),- magenta: color("saturated-purple"),- cyan: "cyan",-};export const LogsContainer = styled(LoadingAndErrorWrapper)`height: 100%;@@ -36,41 +19,4 @@ export const LogsContent = styled.div`padding: 1em;overflow: auto;height: 100%;-- ${LOG_PREFIX}-bold {- font-weight: bold;- }-- ${LOG_PREFIX}-dim {- font-weight: lighter;- }-- ${LOG_PREFIX}-italic {- font-style: italic;- }-- ${LOG_PREFIX}-underline {- text-decoration: underline;- }-- ${LOG_PREFIX}-inverse {- color: var(--mb-color-text-dark);- background-color: var(--mb-color-bg-white);- }-- ${LOG_PREFIX}-hidden {- color: transparent;- }-- ${LOG_PREFIX}-strikethrough {- text-decoration: line-through;- }-- ${Object.entries(LOG_COLORS).map(- ([key, color]) => css`- ${LOG_PREFIX}-${key} {- color: ${color};- }- `,- )};`;diff --git a/frontend/src/metabase/admin/tools/components/Logs/Logs.tsx b/frontend/src/metabase/admin/tools/components/Logs/Logs.tsxindex 84ae61914ed..a7d94a441c2 100644--- a/frontend/src/metabase/admin/tools/components/Logs/Logs.tsx+++ b/frontend/src/metabase/admin/tools/components/Logs/Logs.tsx@@ -9,6 +9,7 @@ import {SettingsPageWrapper,SettingsSection,} from "metabase/admin/components/SettingsSection";+import { AnsiLogs } from "metabase/common/components/AnsiLogs";import Select, { Option } from "metabase/common/components/Select";import { useUrlState } from "metabase/common/hooks/use-url-state";import { openSaveDialog } from "metabase/lib/dom";@@ -143,9 +144,14 @@ const LogsBase = ({</Flex></Flex>- <LogsContent id="logs-content" ref={scrollRef} onScroll={onScroll}>+ <AnsiLogs+ id="logs-content"+ ref={scrollRef}+ onScroll={onScroll}+ component={LogsContent}+ >{displayLogs}- </LogsContent>+ </AnsiLogs></LogsContainer></SettingsSection></SettingsPageWrapper>diff --git a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.cssindex 239e47c267c..877200c6645 100644--- a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css+++ b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css@@ -22,7 +22,6 @@display: flex;justify-content: center;width: 100%;- padding: 2rem;position: relative;background-color: var(--mb-color-bg-light);}diff --git a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsxindex 4de4c46c09a..4d06c526281 100644--- a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx+++ b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx@@ -10,11 +10,13 @@ import S from "./AdminSettingsLayout.module.css";export const AdminSettingsLayout = ({sidebar,children,+ fullWidth = false,maw = "50rem",}: {sidebar?: React.ReactNode;children?: React.ReactNode;maw?: string;+ fullWidth?: boolean;}) => {return (<Box className={S.Wrapper}>@@ -24,9 +26,13 @@ export const AdminSettingsLayout = ({{sidebar}</Box>)}- <Box className={S.Content} data-testid="admin-layout-content">- <Box maw={maw} w="100%">- <Box pb="2rem">+ <Box+ className={S.Content}+ data-testid="admin-layout-content"+ p={fullWidth ? 0 : "2rem"}+ >+ <Box maw={fullWidth ? undefined : maw} w="100%">+ <Box pb={fullWidth ? 0 : "2rem"}><ErrorBoundary>{children ?? <NotFound />}</ErrorBoundary></Box></Box>diff --git a/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.module.css b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.module.cssnew file mode 100644index 00000000000..30e417dfb4b--- /dev/null+++ b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.module.css@@ -0,0 +1,64 @@+.ansiLogs {+ :global {+ .react-ansi-style-bold {+ font-weight: bold;+ }++ .react-ansi-style-dim {+ font-weight: lighter;+ }++ .react-ansi-style-italic {+ font-style: italic;+ }++ .react-ansi-style-inverse {+ color: var(--mb-color-text-dark);+ background-color: var(--mb-color-bg-white);+ }++ .react-ansi-style-hidden {+ color: transparent;+ }++ .react-ansi-style-underline {+ text-decoration: underline;+ }++ .react-ansi-style-strikethrough {+ text-decoration: line-through;+ }++ .react-ansi-style-black {+ color: var(--mb-color-text-dark);+ }++ .react-ansi-style-white {+ color: var(--mb-color-text-white);+ }++ .react-ansi-style-gray {+ color: var(--mb-color-text-light);+ }++ .react-ansi-style-red {+ color: var(--mb-color-saturated-red);+ }++ .react-ansi-style-green {+ color: var(--mb-color-saturated-green);+ }++ .react-ansi-style-yellow {+ color: var(--mb-color-saturated-yellow);+ }++ .react-ansi-style-blue {+ color: var(--mb-color-saturated-blue);+ }++ .react-ansi-style-purple {+ color: var(--mb-color-saturated-purple);+ }+ }+}diff --git a/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.tsx b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.tsxnew file mode 100644index 00000000000..4d1daaae9e2--- /dev/null+++ b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.tsx@@ -0,0 +1,36 @@+import cx from "classnames";+import * as React from "react";+import { type Ref, forwardRef, useMemo } from "react";+import reactAnsiStyle from "react-ansi-style";++import {+ Box,+ type BoxComponentProps,+ type BoxProps,+ type PolymorphicComponentProps,+} from "metabase/ui";++import S from "./AnsiLogs.module.css";++type LogOutputProps<C> = PolymorphicComponentProps<C, BoxComponentProps> &+ BoxProps & {+ children?: string;+ };++export const AnsiLogs = forwardRef(function AnsiLogsInner<C = "div">(+ props: LogOutputProps<C>,+ ref: Ref<HTMLDivElement>,+) {+ const { children, className, ...rest } = props;+ const displayContent = useMemo(+ () => reactAnsiStyle(React, children ?? ""),+ [children],+ );++ return (+ // @ts-expect-error: it's very hard to type the ref and component props from mantine+ <Box ref={ref} className={cx(S.ansiLogs, className)} {...rest}>+ {displayContent}+ </Box>+ );+});diff --git a/frontend/src/metabase/common/components/AnsiLogs/index.ts b/frontend/src/metabase/common/components/AnsiLogs/index.tsnew file mode 100644index 00000000000..2587fabd2d1--- /dev/null+++ b/frontend/src/metabase/common/components/AnsiLogs/index.ts@@ -0,0 +1 @@+export * from "./AnsiLogs";diff --git a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.css b/frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.cssdeleted file mode 100644index a5256630201..00000000000--- a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.css+++ /dev/null@@ -1,8 +0,0 @@-.codeEditor {- :global {- .cm-gutter {- background-color: var(--mb-color-brand-light);- color: var(--mb-color-brand);- }- }-}diff --git a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx b/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsxindex fdf7f1d7d18..5741f9fb397 100644--- a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx+++ b/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx@@ -1,9 +1,7 @@import type { Extension } from "@uiw/react-codemirror";-import cx from "classnames";import { CodeMirror } from "metabase/common/components/CodeMirror";-import S from "./CodeEditor.module.css";import type { CodeLanguage } from "./types";import { useExtensions } from "./utils";@@ -16,6 +14,8 @@ type Props = {readOnly?: boolean;value: string;onChange?: (value: string) => void;+ extensions?: Extension[];+ "data-testid"?: string;};export function CodeEditor({@@ -27,8 +27,13 @@ export function CodeEditor({readOnly,value,onChange,+ extensions: externalExtensions,+ ...rest}: Props) {- const extensions = useExtensions({ language });+ const extensions = useExtensions({+ language,+ extensions: externalExtensions,+ });return (<CodeMirror@@ -38,13 +43,14 @@ export function CodeEditor({highlightActiveLine: false,highlightActiveLineGutter: false,}}- className={cx(S.codeEditor, className)}+ className={className}extensions={extensions}id={id}readOnly={readOnly}value={value}onChange={onChange}highlightRanges={highlightRanges}+ {...rest}/>);}diff --git a/frontend/src/metabase/common/components/CodeEditor/utils.ts b/frontend/src/metabase/common/components/CodeEditor/utils.tsindex 74a6a6b2430..6de3a92be40 100644--- a/frontend/src/metabase/common/components/CodeEditor/utils.ts+++ b/frontend/src/metabase/common/components/CodeEditor/utils.ts@@ -3,7 +3,7 @@ import { javascript } from "@codemirror/lang-javascript";import { json } from "@codemirror/lang-json";import { python } from "@codemirror/lang-python";import { sql } from "@codemirror/lang-sql";-import { StreamLanguage } from "@codemirror/language";+import { StreamLanguage, indentUnit } from "@codemirror/language";import { clojure } from "@codemirror/legacy-modes/mode/clojure";import { pug } from "@codemirror/legacy-modes/mode/pug";import { ruby } from "@codemirror/legacy-modes/mode/ruby";@@ -15,12 +15,17 @@ import type { CodeLanguage } from "./types";export function useExtensions({language,+ extensions,}: {language?: CodeLanguage | Extension;+ extensions?: Extension[];}) {return useMemo(- () => [...(language ? [getLanguageExtension(language)] : [])],- [language],+ () => [+ ...(extensions ?? []),+ ...(language ? [getLanguageExtension(language)] : []),+ ],+ [language, extensions],);}@@ -37,7 +42,7 @@ export function getLanguageExtension(language: CodeLanguage | Extension) {case "json":return json();case "python":- return python();+ return [python(), indentUnit.of(" ")];case "mustache":return handlebars;case "pug":diff --git a/frontend/src/metabase/lib/errors/messages.ts b/frontend/src/metabase/lib/errors/messages.tsindex e75e7a0c88f..2455cdcff1a 100644--- a/frontend/src/metabase/lib/errors/messages.ts+++ b/frontend/src/metabase/lib/errors/messages.ts@@ -31,5 +31,5 @@ export function getResponseErrorMessage(error: unknown): string | undefined {}export function isResourceNotFoundError(error: any) {- return error?.status === 404;+ return error instanceof Object && "status" in error && error.status === 404;}diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.tsindex 3223b9bcdfe..a940c7f5731 100644--- a/frontend/src/metabase/plugins/index.ts+++ b/frontend/src/metabase/plugins/index.ts@@ -78,11 +78,14 @@ import type {ModelCacheRefreshStatus,ParameterId,Pulse,+ PythonTransformSource,+ PythonTransformTableAliases,Revision,Series,TableId,Timeline,TimelineEvent,+ Transform,User,VisualizationDisplay,} from "metabase-types/api";@@ -838,3 +841,33 @@ export const PLUGIN_TRANSFORMS: TransformsPlugin = {getAdminPaths: () => [],getAdminRoutes: () => null,};++export type PythonTransformsPlugin = {+ PythonRunnerSettingsPage: ComponentType;+ SourceSection: ComponentType<{ transform: Transform }>;+ TransformEditor: ComponentType<{+ initialSource: {+ type: "python";+ body: string;+ "source-database": DatabaseId | undefined;+ "source-tables": PythonTransformTableAliases;+ };+ isNew?: boolean;+ isSaving?: boolean;+ isRunnable?: boolean;+ onSave: (newSource: PythonTransformSource) => void;+ onCancel: () => void;+ }>;+ getAdminRoutes: () => ReactNode;+ getTransformsNavLinks: () => ReactNode;+ getCreateTransformsMenuItems: () => ReactNode;+};++export const PLUGIN_TRANSFORMS_PYTHON: PythonTransformsPlugin = {+ PythonRunnerSettingsPage: NotFoundPlaceholder,+ TransformEditor: NotFoundPlaceholder,+ SourceSection: PluginPlaceholder,+ getAdminRoutes: () => null,+ getTransformsNavLinks: () => null,+ getCreateTransformsMenuItems: () => null,+};diff --git a/frontend/src/metabase/ui/components/utils/Box/index.ts b/frontend/src/metabase/ui/components/utils/Box/index.tsindex 5a9a65f2b4b..320e0657bce 100644--- a/frontend/src/metabase/ui/components/utils/Box/index.ts+++ b/frontend/src/metabase/ui/components/utils/Box/index.ts@@ -1,2 +1,2 @@export { Box } from "@mantine/core";-export type { BoxProps } from "@mantine/core";+export type { BoxProps, BoxComponentProps } from "@mantine/core";diff --git a/frontend/src/metabase/ui/index.ts b/frontend/src/metabase/ui/index.tsindex 871cf98354d..d27e39618d8 100644--- a/frontend/src/metabase/ui/index.ts+++ b/frontend/src/metabase/ui/index.ts@@ -6,6 +6,7 @@ export type {MantineTheme,MantineThemeOther,MantineThemeOverride,+ PolymorphicComponentProps,} from "@mantine/core";export { useHover } from "@mantine/hooks";export * from "./components";diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex f3a4e491105..d01238d7bb5 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -13,11 +13,13 @@[metabase.driver.bigquery-cloud-sdk.query-processor :as bigquery.qp][metabase.driver.common.table-rows-sample :as table-rows-sample][metabase.driver.sql :as driver.sql]+ [metabase.driver.sql-jdbc :as driver.sql-jdbc][metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.util :as sql.u][metabase.driver.sync :as driver.s][metabase.util :as u]+ [metabase.util.date-2 :as u.date][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]@@ -40,6 +42,8 @@Field$ModeFieldValueFieldValueList+ InsertAllRequest+ InsertAllRequest$RowToInsertQueryJobConfigurationSchemaTable@@ -47,6 +51,7 @@TableIdTableResult)(com.google.common.collect ImmutableMap)+ (com.google.gson JsonParser)(java.util Iterator)))(set! *warn-on-reflection* true)@@ -232,6 +237,33 @@"BIGNUMERIC" :type/Decimal:type/*))+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for BigQuery that dispatches on type."+ {:arglists '([type])}+ identity)++;; we can't recover the parameterized types+(defmethod type->database-type :type/Array [_] [[:raw "JSON"]])+(defmethod type->database-type :type/Dictionary [_] [[:raw "JSON"]])++(defmethod type->database-type :type/Boolean [_] [[:raw "BOOL"]])+(defmethod type->database-type :type/Float [_] [[:raw "FLOAT64"]])+(defmethod type->database-type :type/Integer [_] [[:raw "INT"]])+(defmethod type->database-type :type/Number [_] [[:raw "INT"]])+(defmethod type->database-type :type/Text [_] [[:raw "STRING"]])+(defmethod type->database-type :type/TextLike [_] [[:raw "STRING"]])+(defmethod type->database-type :type/Date [_] [[:raw "DATE"]])+(defmethod type->database-type :type/DateTime [_] [[:raw "DATETIME"]])+(defmethod type->database-type :type/DateTimeWithTZ [_] [[:raw "TIMESTAMP"]])+(defmethod type->database-type :type/Time [_] [[:raw "TIME"]])+(defmethod type->database-type :type/JSON [_] [[:raw "JSON"]])+(defmethod type->database-type :type/SerializedJSON [_] [[:raw "JSON"]])+(defmethod type->database-type :type/Decimal [_] [[:raw "BIGDECIMAL"]])++(defmethod driver/type->database-type :bigquery-cloud-sdk+ [_driver base-type]+ (type->database-type base-type))+(defn- field->database+base-type"Returns a normalized `database-type` and its `base-type` for a type from BigQuery Field type.@@ -737,6 +769,9 @@:expressions true:now true:percentile-aggregations true+ ;; we can't support `alter table .. rename ..` in general+ ;; since it won't work for streaming tables+ :rename false:metadata/key-constraints false:identifiers-with-spaces true:expressions/integer true@@ -752,6 +787,7 @@:expression-literals true:database-routing true:metadata/table-existence-check true+ :transforms/python true:transforms/table true}](defmethod driver/database-supports? [:bigquery-cloud-sdk feature] [_driver _feature _db] supported?))@@ -838,6 +874,85 @@(let [table-str (get-table-str table)][(str "DROP TABLE IF EXISTS " table-str)]))+(defmethod driver/create-table! :bigquery-cloud-sdk+ [driver database-id table-name column-definitions & {:keys [primary-key]}]+ (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)]+ (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))++(defmethod driver/drop-table! :bigquery-cloud-sdk+ [driver database-id table-name]+ (let [sql (driver/compile-drop-table driver table-name)]+ (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))++(defn- convert-value-for-insertion+ [base-type value]+ (condp #(isa? %2 %1) base-type+ :type/JSON+ (.toString (JsonParser/parseString value))++ :type/Dictionary+ (JsonParser/parseString value)++ :type/Array+ (JsonParser/parseString value)++ :type/Integer+ (parse-long value)++ :type/Float+ (parse-double value)++ :type/Boolean+ (parse-boolean value)++ :type/Numeric+ (bigdec value)++ :type/Decimal+ (bigdec value)++ :type/Date+ (u.date/format (u.date/parse value))++ :type/DateTime+ (u.date/format (u.date/parse value))++ :type/DateTimeWithLocalTZ+ (u.date/format (u.date/parse value))++ value))++(defmethod driver/insert-col->val [:bigquery-cloud-sdk :jsonl-file]+ [_driver _ column-def v]+ (if (string? v)+ (convert-value-for-insertion (:type column-def) v)+ v))++(defmethod driver/insert-from-source! [:bigquery-cloud-sdk :rows]+ [_driver db-id {table-name :name :keys [columns]} {:keys [data]}]+ (let [col-names (map :name columns)+ {:keys [details]} (t2/select-one :model/Database db-id)++ client (database-details->client details)+ project-id (get-project-id details)++ dataset-id (namespace table-name)+ table-name (name table-name)++ table-id (.getTableId (get-table client project-id dataset-id table-name))++ prepared-rows (map #(into {} (map vector col-names %)) data)]+ (doseq [chunk (partition-all (or driver/*insert-chunk-rows* 1000) prepared-rows)]+ (let [insert-request-builder (InsertAllRequest/newBuilder table-id)]+ (doseq [^java.util.Map row chunk]+ (.addRow insert-request-builder (InsertAllRequest$RowToInsert/of row)))+ (let [insert-request (.build insert-request-builder)+ response (.insertAll client insert-request)]+ (when (.hasErrors response)+ (let [errors (.getInsertErrors response)]+ (throw (ex-info "BigQuery insert failed"+ {:errors (into [] (map str errors))})))))))))+(defmethod driver/execute-raw-queries! :bigquery-cloud-sdk[_driver connection-details queries];; connection-details is either database details directly (from transforms)@@ -905,3 +1020,8 @@:detailslist-datasets(some #{schema}))))++(defmethod driver/table-name-length-limit :bigquery-cloud-sdk+ [_driver]+ ;; https://cloud.google.com/bigquery/docs/tables+ 1024)diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex 2fe382680f2..e9a381131ab 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -1327,3 +1327,22 @@(is (=? [["2024-12-11T16:23:55.123456Z" #"2024-12-11T16:23:55.123456.*"]](-> (qp/process-query query)mt/rows))))))++(deftest ^:parallel type->database-type-test+ (testing "type->database-type multimethod returns correct BigQuery types"+ (are [base-type expected] (= expected (driver/type->database-type :bigquery-cloud-sdk base-type))+ :type/Array [[:raw "JSON"]]+ :type/Dictionary [[:raw "JSON"]]+ :type/Boolean [[:raw "BOOL"]]+ :type/Float [[:raw "FLOAT64"]]+ :type/Integer [[:raw "INT"]]+ :type/Number [[:raw "INT"]]+ :type/Text [[:raw "STRING"]]+ :type/TextLike [[:raw "STRING"]]+ :type/Date [[:raw "DATE"]]+ :type/DateTime [[:raw "DATETIME"]]+ :type/DateTimeWithTZ [[:raw "TIMESTAMP"]]+ :type/Time [[:raw "TIME"]]+ :type/JSON [[:raw "JSON"]]+ :type/SerializedJSON [[:raw "JSON"]]+ :type/Decimal [[:raw "BIGDECIMAL"]])))diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.cljindex 7d82580335e..354a304ea7d 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj@@ -486,3 +486,11 @@(execute! "select name from `%s`.metabase_test_tracking.datasets order by accessed_at" (project-id))(database-exists?! (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))++(defn ^:private get-test-data-name+ []+ (test-dataset-id+ (tx/get-dataset-definition (or data.impl/*dbdef-used-to-create-db*+ (tx/default-dataset :bigquery-cloud-sdk)))))++(defmethod sql.tx/session-schema :bigquery-cloud-sdk [_driver] (get-test-data-name))diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.cljindex 482e2d3526f..dff992030f4 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj@@ -2,6 +2,7 @@"Driver for ClickHouse databases"(:require[clojure.core.memoize :as memoize]+ [clojure.java.jdbc :as jdbc][clojure.string :as str][metabase.driver :as driver][metabase.driver-api.core :as driver-api]@@ -52,9 +53,11 @@:window-functions/cumulative (not driver-api/is-test?):left-join (not driver-api/is-test?):describe-fks false+ :rename true:actions false:metadata/key-constraints false:database-routing true+ :transforms/python true:transforms/table true}](defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?))@@ -212,6 +215,28 @@:metabase.upload/datetime "Nullable(DateTime64(3))":metabase.upload/offset-datetime nil))+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for ClickHouse that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/Boolean [_] [[:raw "Nullable(Boolean)"]])+(defmethod type->database-type :type/Float [_] [[:raw "Nullable(Float64)"]])+(defmethod type->database-type :type/Integer [_] [[:raw "Nullable(Int32)"]])+(defmethod type->database-type :type/Number [_] [[:raw "Nullable(Int64)"]])+(defmethod type->database-type :type/BigInteger [_] [[:raw "Nullable(Int64)"]])+(defmethod type->database-type :type/Text [_] [[:raw "Nullable(String)"]])+(defmethod type->database-type :type/TextLike [_] [[:raw "Nullable(String)"]])+(defmethod type->database-type :type/Date [_] [[:raw "Nullable(Date32)"]])+(defmethod type->database-type :type/Time [_] [[:raw "Nullable(Time)"]])+(defmethod type->database-type :type/DateTime [_] [[:raw "Nullable(DateTime64(3))"]])+;; we're lossy here+(defmethod type->database-type :type/DateTimeWithTZ [_] [[:raw "Nullable(DateTime64(3, 'UTC'))"]])++(defmethod driver/type->database-type :clickhouse+ [_driver base-type]+ (type->database-type base-type))+(defmethod driver/table-name-length-limit :clickhouse[_driver];; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses@@ -219,7 +244,8 @@206)(defn- quote-name [s]- (let [parts (str/split (name s) #"\.")]+ (let [s (if (and (keyword? s) (namespace s)) (str (namespace s) "." (name s)) s)+ parts (filter identity (str/split (name s) #"\."))](str/join "." (map #(str "`" % "`") parts))))(defn- create-table!-sql@@ -241,7 +267,18 @@{:write? true}(fn [^java.sql.Connection conn](with-open [stmt (.createStatement conn)]- (.execute stmt (create-table!-sql driver table-name column-definitions :primary-key primary-key))))))+ (let [sql (create-table!-sql driver table-name column-definitions :primary-key primary-key)]+ (.execute stmt sql))))))++;; rename-tables!* only supported by the atomic engine+;; https://clickhouse.com/docs/engines/database-engines/atomic#exchange-tables++(defmethod driver/rename-table! :clickhouse+ [_driver db-id old-table-name new-table-name]+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]+ (with-open [stmt (.createStatement ^java.sql.Connection (:connection conn))]+ (let [sql (format "RENAME TABLE %s TO %s" (quote-name old-table-name) (quote-name new-table-name))]+ (.execute stmt sql)))))(defmethod driver/insert-into! :clickhouse[driver db-id table-name column-names values]diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.cljindex 80533a389e6..904cda43911 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj@@ -358,6 +358,19 @@"Failed to get server info""Code: 516. DB::Exception: asdf: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) (version 25.7.4.11 (official build))"]))))+(deftest ^:parallel type->database-type-test+ (testing "type->database-type multimethod returns correct ClickHouse types"+ (are [base-type expected] (= expected (driver/type->database-type :clickhouse base-type))+ :type/Boolean [[:raw "Nullable(Boolean)"]]+ :type/Float [[:raw "Nullable(Float64)"]]+ :type/Integer [[:raw "Nullable(Int32)"]]+ :type/Number [[:raw "Nullable(Int64)"]]+ :type/Text [[:raw "Nullable(String)"]]+ :type/TextLike [[:raw "Nullable(String)"]]+ :type/Date [[:raw "Nullable(Date32)"]]+ :type/DateTime [[:raw "Nullable(DateTime64(3))"]]+ :type/DateTimeWithTZ [[:raw "Nullable(DateTime64(3, 'UTC'))"]])))+(deftest ^:parallel query-with-cte-subquery-and-param-test(mt/test-driver :clickhouse(testing "a query with a CTE in a subquery and a parameter should work correctly"diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.cljindex 3fc954435eb..89b1373c33f 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj@@ -1,6 +1,6 @@(ns metabase.driver.mongo"MongoDB Driver."- (:refer-clojure :exclude [some])+ (:refer-clojure :exclude [some mapv])(:require[clojure.set :as set][clojure.string :as str]@@ -18,9 +18,10 @@[metabase.driver.settings :as driver.settings][metabase.driver.util :as driver.u][metabase.util :as u]+ [metabase.util.date-2 :as u.date][metabase.util.json :as json][metabase.util.log :as log]- [metabase.util.performance :refer [some]]+ [metabase.util.performance :refer [some mapv]][taoensso.nippy :as nippy])(:import(com.mongodb.client MongoClient MongoDatabase)@@ -426,6 +427,7 @@:nested-queries true:set-timezone true:standard-deviation-aggregations true+ :rename true:test/jvm-timezone-setting false:identifiers-with-spaces true:saved-question-sandboxing false@@ -435,6 +437,8 @@:expressions/today true;; Index sync is turned off across the application as it is not used ATM.:index-info false+ :python-transforms true+ :transforms/python true:database-routing true}](defmethod driver/database-supports? [:mongo feature] [_driver _feature _db] supported?))@@ -555,6 +559,146 @@(log/debugf e "Query:\n%s" native-form)native-form)))+(defmethod driver/create-table! :mongo+ [_driver database-id table-name _column-definitions & {:keys [primary-key]}]+ ;; MongoDB collections are created implicitly when first document is inserted+ ;; We can create an empty collection explicitly if needed+ (mongo.connection/with-mongo-database [^MongoDatabase db database-id]+ (.createCollection db (name table-name))+ ;; Create indexes for any primary key fields+ (when primary-key+ (doseq [pk-field primary-key]+ (mongo.util/create-index+ (mongo.util/collection db (name table-name))+ {pk-field 1})))))++(defmethod driver/drop-table! :mongo+ [_driver db-id table-name]+ (mongo.connection/with-mongo-database [^MongoDatabase db db-id]+ (some-> (mongo.util/collection db (name table-name))+ .drop)))++(defmethod driver/rename-table! :mongo+ [_driver db-id old-table-name new-table-name]+ (mongo.connection/with-mongo-database [^MongoDatabase db db-id]+ (let [old-collection (mongo.util/collection db (name old-table-name))]+ (.renameCollection old-collection+ (com.mongodb.MongoNamespace.+ (.getName db)+ (name new-table-name))))))++(defmethod driver/drop-transform-target! [:mongo :table]+ [driver database target]+ (driver/drop-table! driver (:id database) (:name target)))++(defmethod driver/connection-spec :mongo+ [_driver database]+ (:details database))++(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for MongoDB that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/TextLike [_] "string")+(defmethod type->database-type :type/Text [_] "string")+(defmethod type->database-type :type/Number [_] "long")+(defmethod type->database-type :type/Integer [_] "int")+(defmethod type->database-type :type/BigInteger [_] "long")+(defmethod type->database-type :type/Float [_] "double")+(defmethod type->database-type :type/Decimal [_] "decimal")+(defmethod type->database-type :type/Boolean [_] "bool")+(defmethod type->database-type :type/Date [_] "date")+(defmethod type->database-type :type/DateTime [_] "date")+(defmethod type->database-type :type/DateTimeWithTZ [_] "date")+(defmethod type->database-type :type/Time [_] "date")+(defmethod type->database-type :type/TimeWithTZ [_] "date")+(defmethod type->database-type :type/Instant [_] "date")+(defmethod type->database-type :type/UUID [_] "uuid")+(defmethod type->database-type :type/JSON [_] "object")+(defmethod type->database-type :type/SerializedJSON [_] "string")+(defmethod type->database-type :type/Array [_] "array")+(defmethod type->database-type :type/Dictionary [_] "object")+(defmethod type->database-type :type/MongoBSONID [_] "objectId")+(defmethod type->database-type :type/MongoBinData [_] "binData")+(defmethod type->database-type :type/IPAddress [_] "string")+(defmethod type->database-type :default [_] "object")++(defmethod driver/type->database-type :mongo+ [_driver base-type]+ (type->database-type base-type))++(defn- convert-value-for-insertion+ [base-type value]+ (condp #(isa? %2 %1) base-type+ :type/JSON+ (json/decode value)++ :type/Dictionary+ (json/decode value)++ :type/Array+ (json/decode value)++ :type/Integer+ (parse-long value)++ :type/BigInteger+ (bigint value)++ :type/Float+ (parse-double value)++ :type/Decimal+ (bigdec value)++ :type/Number+ (bigint value)++ :type/Boolean+ (parse-boolean value)++ :type/Date+ (u.date/parse value)++ :type/DateTime+ (u.date/parse value)++ :type/DateTimeWithTZ+ (u.date/parse value)++ :type/Time+ (u.date/parse value)++ :type/TimeWithTZ+ (u.date/parse value)++ :type/Instant+ (u.date/parse value)++ :type/UUID+ (parse-uuid value)++ value))++(defmethod driver/insert-col->val [:mongo :jsonl-file]+ [_driver _ column-def v]+ (if (string? v)+ (convert-value-for-insertion (:type column-def) v)+ v))++(defmethod driver/insert-from-source! [:mongo :rows]+ [_driver db-id {table-name :name :keys [columns]} {:keys [data]}]+ (let [col-names (mapv :name columns)]+ (mongo.connection/with-mongo-database [^MongoDatabase db db-id]+ (let [collection (mongo.util/collection db (name table-name))+ documents (map #(into {} (map vector col-names %))+ data)]+ (if (> (bounded-count 2 documents) 1)+ ;; TODO: batch to avoid realizing everything in memory+ (mongo.util/insert-many collection documents)+ (mongo.util/insert-one collection (first documents)))))))+;; Following code is using monger. Leaving it here for a reference as it could be transformed when there is need;; for ssl experiments.#_(comment@@ -618,3 +762,7 @@credentials)](mg/get-db-names unauthenticated-connection))):.)++(defmethod driver/table-name-length-limit :mongo+ [_driver]+ 64)diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj b/modules/drivers/mongo/src/metabase/driver/mongo/execute.cljindex de6307e2678..b0b89eb11a4 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj@@ -215,9 +215,10 @@db-name (mongo.db/db-name database)client-database (mongo.util/database mongo.connection/*mongo-client* db-name)](with-open [session ^ClientSession (mongo.util/start-session! mongo.connection/*mongo-client*)]- (a/go- (when (a/<! (driver-api/canceled-chan))- (mongo.util/kill-session! client-database session)))+ (when-let [cancel-chan (driver-api/canceled-chan)]+ (a/go+ (when (a/<! cancel-chan)+ (mongo.util/kill-session! client-database session))))(let [aggregate ^AggregateIterable (*aggregate* client-databasecollection-namesessiondiff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.cljindex fcd69ba2d14..a677ca0b1ad 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj@@ -1148,3 +1148,29 @@(is (= [[1 "ref1" ref-1 "some_collection" id-1 nil][2 "ref2" ref-2 "some_collection" id-2 "some_db"]](mt/rows (mt/run-mbql-query dbref_coll))))))))++(deftest ^:parallel type->database-type-test+ (testing "type->database-type multimethod returns correct MongoDB types"+ (are [base-type expected] (= expected (driver/type->database-type :mongo base-type))+ :type/TextLike "string"+ :type/Text "string"+ :type/Number "long"+ :type/Integer "int"+ :type/BigInteger "long"+ :type/Float "double"+ :type/Decimal "decimal"+ :type/Boolean "bool"+ :type/Date "date"+ :type/DateTime "date"+ :type/DateTimeWithTZ "date"+ :type/Time "date"+ :type/TimeWithTZ "date"+ :type/Instant "date"+ :type/UUID "uuid"+ :type/JSON "object"+ :type/SerializedJSON "string"+ :type/Array "array"+ :type/Dictionary "object"+ :type/MongoBSONID "objectId"+ :type/MongoBinData "binData"+ :type/IPAddress "string")))diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.cljindex 450d6df4945..80d4ccd411f 100644--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj@@ -37,6 +37,8 @@(doseq [[feature supported?] {:connection-impersonation true:describe-fields true:describe-fks true+ :rename true+ :atomic-renames true:expression-literals true:identifiers-with-spaces false:uuid-type false@@ -44,6 +46,7 @@:test/jvm-timezone-setting false:database-routing true:metadata/table-existence-check true+ :transforms/python true:transforms/table true}](defmethod driver/database-supports? [:redshift feature] [_driver _feat _db] supported?))@@ -539,6 +542,27 @@:metabase.upload/datetime [:timestamp]:metabase.upload/offset-datetime [:timestamp-with-time-zone]))+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for Redshift that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/TextLike [_] [[:varchar 65535]])+(defmethod type->database-type :type/Text [_] [[:varchar 65535]])+(defmethod type->database-type :type/Number [_] [:bigint])+(defmethod type->database-type :type/BigInteger [_] [:bigint])+(defmethod type->database-type :type/Integer [_] [:integer])+(defmethod type->database-type :type/Float [_] [(keyword "double precision")])+(defmethod type->database-type :type/Boolean [_] [:boolean])+(defmethod type->database-type :type/Date [_] [:date])+(defmethod type->database-type :type/DateTime [_] [:timestamp])+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])+(defmethod type->database-type :type/Time [_] [:time])++(defmethod driver/type->database-type :redshift+ [_driver base-type]+ (type->database-type base-type))+(defmethod driver/allowed-promotions :redshift [_] {})(defmethod driver/table-name-length-limit :redshiftdiff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.cljindex 04e793143a9..7be1bf6dd84 100644--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj@@ -519,3 +519,17 @@(testing (format "database-type %s" (pr-str database-type))(is (= exp-base-type(sql-jdbc.sync/database-type->base-type :redshift database-type))))))++(deftest ^:parallel type->database-type-test+ (testing "type->database-type multimethod returns correct Redshift types"+ (are [base-type expected] (= expected (driver/type->database-type :redshift base-type))+ :type/TextLike [[:varchar 65535]]+ :type/Text [[:varchar 65535]]+ :type/Integer [:integer]+ :type/Float [(keyword "double precision")]+ :type/Number [:bigint]+ :type/Boolean [:boolean]+ :type/Date [:date]+ :type/DateTime [:timestamp]+ :type/DateTimeWithTZ [:timestamp-with-time-zone]+ :type/Time [:time])))diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex 0bf79b03bdd..c8eb9ff3311 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -6,6 +6,7 @@[clojure.java.jdbc :as jdbc][clojure.set :as set][clojure.string :as str]+ [honey.sql :as sql][java-time.api :as t][medley.core :as m][metabase.driver :as driver]@@ -55,6 +56,7 @@(doseq [[feature supported?] {:connection-impersonation true:connection-impersonation-requires-role true+ :rename true:convert-timezone true:datetime-diff true:describe-fields false@@ -68,6 +70,7 @@:now true:database-routing true:metadata/table-existence-check true+ :transforms/python true:transforms/table true}](defmethod driver/database-supports? [:snowflake feature] [_driver _feature _db] supported?))@@ -264,7 +267,7 @@:FLOAT4 :type/Float:FLOAT8 :type/Float:DOUBLE :type/Float- (keyword "DOUBLE PRECISON") :type/Float+ (keyword "DOUBLE PRECISION") :type/Float:REAL :type/Float:VARCHAR :type/Text:CHAR :type/Text@@ -292,6 +295,29 @@:OBJECT :type/Dictionary:ARRAY :type/*} base-type))+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for Snowflake that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/Array [_] [:ARRAY])+(defmethod type->database-type :type/Boolean [_] [:BOOLEAN])+(defmethod type->database-type :type/Date [_] [:DATE])+(defmethod type->database-type :type/DateTime [_] [:DATETIME])+(defmethod type->database-type :type/DateTimeWithLocalTZ [_] [:TIMESTAMPTZ])+(defmethod type->database-type :type/DateTimeWithTZ [_] [:TIMESTAMPLTZ])+(defmethod type->database-type :type/Decimal [_] [:DECIMAL])+(defmethod type->database-type :type/Float [_] [:DOUBLE])+(defmethod type->database-type :type/Number [_] [:BIGINT])+(defmethod type->database-type :type/BigInteger [_] [:BIGINT])+(defmethod type->database-type :type/Integer [_] [:INTEGER])+(defmethod type->database-type :type/Text [_] [:TEXT])+(defmethod type->database-type :type/Time [_] [:TIME])++(defmethod driver/type->database-type :snowflake+ [_driver base-type]+ (type->database-type base-type))+(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :seconds] [_ _ expr] [:to_timestamp_tz expr])(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :milliseconds] [_ _ expr] [:to_timestamp_tz expr 3])(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :microseconds] [_ _ expr] [:to_timestamp_tz expr 6])@@ -873,3 +899,17 @@[driver conn-spec schema](let [sql [[(format "CREATE SCHEMA IF NOT EXISTS \"%s\";" schema)]]](driver/execute-raw-queries! driver conn-spec sql)))++(defmethod driver/rename-table! :snowflake+ [driver db-id from-table to-table]+ (let [sql (first (sql/format {:alter-table (keyword from-table)+ :rename-table (keyword to-table)}+ :quoted true+ :dialect (sql.qp/quote-style driver)))]+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]+ (jdbc/execute! conn sql))))++(defmethod driver/table-name-length-limit :snowflake+ [_driver]+ ;; https://docs.snowflake.com/en/sql-reference/identifiers+ 255)diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex c3ce83824b0..8691be2d24a 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -1305,3 +1305,19 @@(is (true? details-changed?))(is (= original-priv-key priv-key-after-update))(is (not= priv-key-after-update priv-key-after-event)))))))))++(deftest ^:parallel type->database-type-test+ (testing "type->database-type multimethod returns correct Snowflake types"+ (are [base-type expected] (= expected (driver/type->database-type :snowflake base-type))+ :type/Array [:ARRAY]+ :type/Boolean [:BOOLEAN]+ :type/Date [:DATE]+ :type/DateTime [:DATETIME]+ :type/DateTimeWithLocalTZ [:TIMESTAMPTZ]+ :type/DateTimeWithTZ [:TIMESTAMPLTZ]+ :type/Decimal [:DECIMAL]+ :type/Float [:DOUBLE]+ :type/Number [:BIGINT]+ :type/Integer [:INTEGER]+ :type/Text [:TEXT]+ :type/Time [:TIME])))diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.cljindex 6dd8e353c9d..a4c486eaf41 100644--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj@@ -379,3 +379,5 @@order by created"](jdbc/reducible-query (no-db-connection-spec))(into [] (map (juxt :database_name :created)))))++(defmethod sql.tx/session-schema :snowflake [_driver] "PUBLIC")diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex d6cd9fc0178..aa50db2166b 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -4,6 +4,7 @@(:require[clojure.data.xml :as xml][clojure.java.io :as io]+ [clojure.java.jdbc :as jdbc][clojure.string :as str][honey.sql :as sql][honey.sql.helpers :as sql.helpers]@@ -24,6 +25,7 @@[metabase.driver.sql.util :as sql.u][metabase.util :as u][metabase.util.honey-sql-2 :as h2x]+ [metabase.util.json :as json][metabase.util.log :as log][metabase.util.performance :as perf :refer [mapv]])(:import@@ -54,6 +56,7 @@:connection-impersonation-requires-role true:uuid-type true:convert-timezone true+ :rename true:datetime-diff true:expression-literals true;; Index sync is turned off across the application as it is not used ATM.@@ -62,6 +65,7 @@:regex false:test/jvm-timezone-setting false:metadata/table-existence-check true+ :transforms/python true:transforms/table true:jdbc/statements false}](defmethod driver/database-supports? [:sqlserver feature] [_driver _feature _db] supported?))@@ -121,6 +125,28 @@:xml :type/*(keyword "int identity") :type/Integer} column-type)) ; auto-incrementing integer (ie pk) field+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for SQL Server that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/Boolean [_] [:bit])+(defmethod type->database-type :type/Date [_] [:date])+(defmethod type->database-type :type/DateTime [_] [:datetime2])+(defmethod type->database-type :type/DateTimeWithTZ [_] [:datetimeoffset])+(defmethod type->database-type :type/Decimal [_] [:decimal])+(defmethod type->database-type :type/Float [_] [:float])+(defmethod type->database-type :type/Integer [_] [:int])+(defmethod type->database-type :type/Number [_] [:bigint])+(defmethod type->database-type :type/BigInteger [_] [:bigint])+(defmethod type->database-type :type/Text [_] [:text])+(defmethod type->database-type :type/Time [_] [:time])+(defmethod type->database-type :type/UUID [_] [:uniqueidentifier])++(defmethod driver/type->database-type :sqlserver+ [_driver base-type]+ (type->database-type base-type))+(defmethod sql-jdbc.conn/connection-details->spec :sqlserver[_ {:keys [user password db host port instance domain ssl]:or {user "dbuser", password "dbpassword", db "", host "localhost"}@@ -938,6 +964,21 @@[_ e](= (sql-jdbc/get-sql-state e) "S0002"))+(defmethod driver/insert-from-source! [:sqlserver :jsonl-file]+ [driver db-id {:keys [columns] :as table-definition} {:keys [file]}]+ (with-open [rdr (io/reader file)]+ (let [lines (line-seq rdr)+ data-rows (map (fn [line]+ (let [m (json/decode line)]+ (mapv (fn [column]+ (let [value (get m (:name column))]+ (if (boolean? value)+ (if value 1 0)+ value)))+ columns)))+ lines)]+ (driver/insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))+(defmethod driver/compile-transform :sqlserver[driver {:keys [query output-table]}](let [^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))@@ -966,3 +1007,15 @@[driver conn-spec schema](let [sql [[(format "IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '%s') EXEC('CREATE SCHEMA [%s];');" schema schema)]]](driver/execute-raw-queries! driver conn-spec sql)))++(defmethod driver/rename-table! :sqlserver+ [_driver db-id old-table-name new-table-name]+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]+ (with-open [stmt (.createStatement ^java.sql.Connection (:connection conn))]+ (let [sql (format "EXEC sp_rename '%s', '%s';" (name old-table-name) (name new-table-name))]+ (.execute stmt sql)))))++(defmethod driver/table-name-length-limit :sqlserver+ [_driver]+ ;; https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms191240(v=sql.105)#sysname+ 128)diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 824599e69de..2e5e6ff1cb7 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -756,3 +756,18 @@{:filter [:expression "NameEquals" {:base-type :type/Booleandriver-api/qp.add.source-table driver-api/qp.add.nonedriver-api/qp.add.desired-alias nil}]})))))))++(deftest ^:parallel type->database-type-test+ (testing "type->database-type multimethod returns correct SQL Server types"+ (are [base-type expected] (= expected (driver/type->database-type :sqlserver base-type))+ :type/Boolean [:bit]+ :type/Date [:date]+ :type/DateTime [:datetime2]+ :type/DateTimeWithTZ [:datetimeoffset]+ :type/Decimal [:decimal]+ :type/Float [:float]+ :type/Integer [:int]+ :type/Number [:bigint]+ :type/Text [:text]+ :type/Time [:time]+ :type/UUID [:uniqueidentifier])))diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 914ade6c8e3..6b9e032c1b0 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -401,7 +401,56 @@(prometheus/counter :metabase-gsheets/connection-deleted{:description "How many times the instance has deleted their Google Sheets connection."})(prometheus/counter :metabase-gsheets/connection-manually-synced- {:description "How many times the instance has manually sync'ed their Google Sheets connection."})])+ {:description "How many times the instance has manually sync'ed their Google Sheets connection."})++ ;; transform metrics+ (prometheus/counter :metabase-transforms/job-runs-total+ {:description "Total number of transform job runs started."+ :labels [:job-id :run-method]})+ (prometheus/counter :metabase-transforms/job-runs-completed+ {:description "Number of transform job runs that completed successfully."+ :labels [:job-id :run-method]})+ (prometheus/counter :metabase-transforms/job-runs-failed+ {:description "Number of transform job runs that failed."+ :labels [:job-id :run-method]})+ (prometheus/histogram :metabase-transforms/job-run-duration-ms+ {:description "Duration in milliseconds of transform job runs."+ :labels [:job-id :run-method]+ ;; 100ms -> 6 hours+ :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000 7200000 14400000 21600000]})+ (prometheus/counter :metabase-transforms/stage-started+ {:description "Number of transform stages started."+ :labels [:job-run-id :stage-type :stage-label]})+ (prometheus/counter :metabase-transforms/stage-completed+ {:description "Number of transform stages completed successfully."+ :labels [:job-run-id :stage-type :stage-label]})+ (prometheus/counter :metabase-transforms/stage-failed+ {:description "Number of transform stages that failed."+ :labels [:job-run-id :stage-type :stage-label]})+ (prometheus/histogram :metabase-transforms/stage-duration-ms+ {:description "Duration in milliseconds of individual transform stages."+ :labels [:job-run-id :stage-type :stage-label]+ ;; 10ms -> 10 minutes+ :buckets [10 100 500 1000 5000 10000 30000 60000 300000 600000]})+ (prometheus/histogram :metabase-transforms/data-transfer-bytes+ {:description "Size in bytes of data transferred during transform stages."+ :labels [:job-run-id :stage-label]+ ;; 1KB -> 10GB+ :buckets [1000 10000 100000 1000000 10000000 100000000 1000000000 10000000000]})+ (prometheus/histogram :metabase-transforms/data-transfer-rows+ {:description "Number of rows transferred during transform stages."+ :labels [:job-run-id :stage-label]+ ;; 10 -> 10M rows+ :buckets [10 100 1000 10000 100000 1000000 10000000]})+ ;; Python-transform specific metrics+ (prometheus/histogram :metabase-transforms/python-api-call-duration-ms+ {:description "Duration of Python runner API calls."+ :labels [:job-run-id]+ ;; 100ms -> 6 hours+ :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000 7200000 14400000 21600000]})+ (prometheus/counter :metabase-transforms/python-api-calls-total+ {:description "Total number of Python runner API calls."+ :labels [:job-run-id :status]})])(defn- quartz-collectors[]diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.cljindex 3bb2e4f34a8..2b8498c3fd9 100644--- a/src/metabase/analytics/stats.clj+++ b/src/metabase/analytics/stats.clj@@ -883,7 +883,10 @@:enabled (premium-features/table-data-editing?)}{:name :transforms:available (premium-features/enable-transforms?)- :enabled (premium-features/enable-transforms?)}])+ :enabled (premium-features/enable-transforms?)}+ {:name :transforms-python+ :available (premium-features/enable-python-transforms?)+ :enabled (premium-features/enable-python-transforms?)}])(defn- snowplow-features[]diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 86b6e8c9fcd..72a26c449e3 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -7,8 +7,10 @@SQL-based drivers can use the `:sql` driver as a parent, and JDBC-based SQL drivers can use `:sql-jdbc`. Both ofthese drivers define additional multimethods that child drivers should implement; see [[metabase.driver.sql]] and[[metabase.driver.sql-jdbc]] for more details."+ (:refer-clojure :exclude [some mapv])#_{:clj-kondo/ignore [:metabase/modules]}(:require+ [clojure.java.io :as io][clojure.set :as set][clojure.string :as str][metabase.auth-provider.core :as auth-provider]@@ -18,7 +20,9 @@[metabase.query-processor.error-type :as qp.error-type][metabase.util :as u][metabase.util.i18n :refer [tru]]+ [metabase.util.json :as json][metabase.util.malli :as mu]+ [metabase.util.performance :refer [mapv]][potemkin :as p]))(set! *warn-on-reflection* true)@@ -628,6 +632,12 @@;; Does the driver support multi-level-schema for e.g. multicatalog support in databricks:multi-level-schema+ ;; Does the driver support table renaming+ :rename++ ;; Does the driver support atomic multi-table renaming+ :atomic-renames+;; Does the driver support custom writeback actions. Drivers that support this must;; implement [[execute-write-query!]]:actions/custom@@ -732,6 +742,9 @@;; Does this driver support transforms with a table as the target?:transforms/table+ ;; Does this driver support executing python transforms?+ :transforms/python+;; Does this driver properly support the table-exists? method for checking table existence?:metadata/table-existence-check@@ -1303,10 +1316,13 @@(update-vals first))](rename-tables!* driver db-id sorted-rename-map)))-#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}-(defn rename-table!+(defmulti rename-table!"Rename a single table."- {:added "0.57.0"}+ {:added "0.57.0", :arglists '([driver db-id old-table-name new-table-name])}+ dispatch-on-initialized-driver+ :hierarchy #'hierarchy)++(defmethod rename-table! ::driver[driver db-id from-table to-table](rename-tables!* driver db-id {from-table to-table}))@@ -1326,6 +1342,57 @@dispatch-on-initialized-driver:hierarchy #'hierarchy)+(defmulti insert-col->val+ "Parse a value for insertion based on driver, data source type and column definition.++ Takes:+ - driver: The database driver+ - data-source-type: The data source type, see [[insert-from-source!]]+ - column-def: Column definition map with `:type` and optionally `:database-type`/`:nullable?`+ - val: The value to parse. Format is data-source-type specific.++ Drivers should implement this when their insertion mechanism needs values converted to proper types.++ Default implementation returns the value unchanged"+ {:added "0.57.0", :arglists '([driver data-source-type column-def string-val])}+ (fn [driver type _ _] [(dispatch-on-initialized-driver driver) type])+ :hierarchy #'hierarchy)++(defmulti insert-from-source!+ "Inserts data from a data source into an existing table. Table must exist. Blocks until completion.++ `table-definition` is a map with `:name` (may be schema-qualified) and `:columns`+ (vector of maps with `:name` and optional `:type`, `:database-type`). Column order must match data row order.++ `data-source` dispatches on `:type`. Built-in types include `:jsonl-file` (with `:file`) and `:rows`+ (with `:data` as reducible of row vectors). Drivers may implement additional types.++ Implementations may leave partial data on failure, the method makes no rollback guarantees.+ Data visibility to other connections is not guaranteed immediately.++ Default implementation for `jsonl-file` data-source is provided, which delegate to a `:rows`+ data-source. Non-jdbc drivers must at least implement a `:rows` datasource."+ {:added "0.57.0", :arglists '([driver database-id table-definition data-source])}+ (fn [driver _ _ data-source]+ [(dispatch-on-initialized-driver driver) (:type data-source)])+ :hierarchy #'hierarchy)++(defmethod insert-col->val [::driver :jsonl-file] [_driver _ _column-def val]+ val)++(defmethod insert-from-source! [::driver :jsonl-file]+ [driver db-id {:keys [columns] :as table-definition} {:keys [file]}]+ (with-open [rdr (io/reader file)]+ (let [lines (line-seq rdr)+ data-rows (map (fn [line]+ (let [m (json/decode line)]+ (mapv (fn [column]+ (let [raw-val (get m (:name column))]+ (insert-col->val driver :jsonl-file column raw-val)))+ columns)))+ lines)]+ (insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))+(defmulti add-columns!"Add columns given by `column-definitions` to a table named `table-name`. If the table doesn't exist it will throw an error.`args` is an optional map with an optional key `primary-key`. The `primary-key` value is a vector of column names@@ -1393,6 +1460,12 @@dispatch-on-initialized-driver:hierarchy #'hierarchy)+(defmulti type->database-type+ "Returns the database type for a given Metabase type as a HoneySQL spec."+ {:added "0.57.0", :arglists '([driver base-type])}+ (fn [driver _base-type] driver)+ :hierarchy #'hierarchy)+(defmulti allowed-promotions"Returns a mapping of which types a column can be implicitly relaxed to, based on the content of appended values.In the context of uploads, this permits certain appends or replacements of an existing csv tablediff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.cljindex 5274c51ac24..d438b5bbf40 100644--- a/src/metabase/driver/h2.clj+++ b/src/metabase/driver/h2.clj@@ -609,6 +609,30 @@:metabase.upload/datetime [:timestamp]:metabase.upload/offset-datetime [:timestamp-with-time-zone]))+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for H2 that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/TextLike [_] [:varchar])+(defmethod type->database-type :type/Text [_] [:varchar])+(defmethod type->database-type :type/Integer [_] [:int])+(defmethod type->database-type :type/Number [_] [:bigint])+(defmethod type->database-type :type/BigInteger [_] [:bigint])+(defmethod type->database-type :type/Float [_] [(keyword "DOUBLE PRECISION")])+(defmethod type->database-type :type/Decimal [_] [:decimal])+(defmethod type->database-type :type/Boolean [_] [:boolean])+(defmethod type->database-type :type/Date [_] [:date])+(defmethod type->database-type :type/DateTime [_] [:timestamp])+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])+(defmethod type->database-type :type/Time [_] [:time])+(defmethod type->database-type :type/TimeWithTZ [_] [:time-with-time-zone])+(defmethod type->database-type :type/UUID [_] [:uuid])++(defmethod driver/type->database-type :h2+ [_driver base-type]+ (type->database-type base-type))+(defmethod driver/create-auto-pk-with-append-csv? :h2 [_driver] true)(defmethod driver/table-name-length-limit :h2diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex fcc8ee548a9..3d85154331c 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -78,6 +78,8 @@:schemas false:uploads true:identifiers-with-spaces true+ :rename true+ :atomic-renames true:expressions/integer true:expressions/float true:expressions/date true@@ -90,6 +92,7 @@:expression-literals true:database-routing true:metadata/table-existence-check true+ :transforms/python true:transforms/table true}](defmethod driver/database-supports? [:mysql feature] [_driver _feature _db] supported?))@@ -793,6 +796,30 @@:metabase.upload/datetime [:datetime]:metabase.upload/offset-datetime [:timestamp]))+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for MySQL that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/TextLike [_] [:text])+(defmethod type->database-type :type/Text [_] [:text])+(defmethod type->database-type :type/Number [_] [:bigint])+(defmethod type->database-type :type/BigInteger [_] [:bigint])+(defmethod type->database-type :type/Integer [_] [:int])+(defmethod type->database-type :type/Float [_] [:double])+(defmethod type->database-type :type/Decimal [_] [:decimal])+(defmethod type->database-type :type/Boolean [_] [:boolean])+(defmethod type->database-type :type/Date [_] [:date])+(defmethod type->database-type :type/DateTime [_] [:datetime])+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp])+(defmethod type->database-type :type/Time [_] [:time])+(defmethod type->database-type :type/JSON [_] [:json])+(defmethod type->database-type :type/SerializedJSON [_] [:json])++(defmethod driver/type->database-type :mysql+ [_driver base-type]+ (type->database-type base-type))+(defmethod driver/allowed-promotions :mysql[_driver]{:metabase.upload/int #{:metabase.upload/float}@@ -894,7 +921,7 @@(if (not= (get-global-variable db-id "local_infile") "ON");; If it isn't turned on, fall back to the generic "INSERT INTO ..." way((get-method driver/insert-into! :sql-jdbc) driver db-id table-name column-names values)- (let [temp-file (File/createTempFile table-name ".tsv")+ (let [temp-file (File/createTempFile (name table-name) ".tsv")file-path (.getAbsolutePath temp-file)](try(let [tsvs (map (partial row->tsv driver (count column-names)) values)@@ -915,6 +942,12 @@(finally(.delete temp-file))))))+(defmethod driver/insert-col->val [:mysql :jsonl-file]+ [_driver _ column-def v]+ (if (and (string? v) (isa? (:type column-def) :type/DateTimeWithTZ))+ (t/offset-date-time v)+ v))+(defn- parse-grant"Parses the contents of a row from the output of a `SHOW GRANTS` statement, to extract the data neededto reconstruct the set of table privileges that the current user has. Returns nil if the grant doesn'tdiff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex 5d5e9541d35..fa810d77a53 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -66,6 +66,8 @@:convert-timezone true:datetime-diff true:now true+ :rename true+ :atomic-renames true:persist-models true:schemas true:identifiers-with-spaces true@@ -79,6 +81,7 @@:expressions/date true:database-routing true:transforms/table true+ :transforms/python true:metadata/table-existence-check true}](defmethod driver/database-supports? [:postgres feature] [_driver _feature _db] supported?))@@ -1035,6 +1038,33 @@:type driver-api/qp.error-type.invalid-query}))(throw e))))))+(defmulti ^:private type->database-type+ "Internal type->database-type multimethod for Postgres that dispatches on type."+ {:arglists '([type])}+ identity)++(defmethod type->database-type :type/TextLike [_] [:text])+(defmethod type->database-type :type/Text [_] [:text])+(defmethod type->database-type :type/Number [_] [:bigint])+(defmethod type->database-type :type/BigInteger [_] [:bigint])+(defmethod type->database-type :type/Integer [_] [:int])+(defmethod type->database-type :type/Float [_] [:float])+(defmethod type->database-type :type/Decimal [_] [:decimal])+(defmethod type->database-type :type/Boolean [_] [:boolean])+(defmethod type->database-type :type/Date [_] [:date])+(defmethod type->database-type :type/DateTime [_] [:timestamp])+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])+(defmethod type->database-type :type/Time [_] [:time])+(defmethod type->database-type :type/TimeWithTZ [_] [:time-with-time-zone])+(defmethod type->database-type :type/UUID [_] [:uuid])+(defmethod type->database-type :type/JSON [_] [:jsonb])+(defmethod type->database-type :type/SerializedJSON [_] [:jsonb])+(defmethod type->database-type :type/IPAddress [_] [:inet])++(defmethod driver/type->database-type :postgres+ [_driver base-type]+ (type->database-type base-type))+(defmethod driver/upload-type->database-type :postgres[_driver upload-type](case upload-typediff --git a/src/metabase/driver/sql_jdbc.clj b/src/metabase/driver/sql_jdbc.cljindex 1110481c56e..9c968638a8b 100644--- a/src/metabase/driver/sql_jdbc.clj+++ b/src/metabase/driver/sql_jdbc.clj@@ -177,8 +177,7 @@(jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)](jdbc/execute! conn sql))))-(defmethod driver/insert-into! :sql-jdbc- [driver db-id table-name column-names values]+(defn- insert-into!-sqls [driver table-name column-names values inline?](let [;; We need to partition the insert into multiple statements for both performance and correctness.;;;; On Postgres with a large file, 100 (3.76m) was significantly faster than 50 (4.03m) and 25 (4.27m). 1,000 was a@@ -192,12 +191,21 @@sqls (map #(sql/format {:insert-into (keyword table-name):columns (quote-columns driver column-names):values %}+ :inline inline?:quoted true:dialect dialect)chunks)]- (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]- (doseq [sql sqls]- (jdbc/execute! conn sql)))))+ sqls))++(defmethod driver/insert-into! :sql-jdbc+ [driver db-id table-name column-names values]+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]+ (doseq [sql (insert-into!-sqls driver table-name column-names values false)]+ (jdbc/execute! conn sql))))++(defmethod driver/insert-from-source! [:sql-jdbc :rows]+ [driver db-id {table-name :name :keys [columns]} {:keys [data]}]+ (driver/insert-into! driver db-id table-name (mapv :name columns) data))(defmethod driver/add-columns! :sql-jdbc[driver db-id table-name column-definitions & {:keys [primary-key]}]diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.cljindex 2e965d304f9..3b69a16f5ca 100644--- a/src/metabase/premium_features/core.clj+++ b/src/metabase/premium_features/core.clj@@ -73,6 +73,7 @@enable-sso-ldap?enable-sso-saml?enable-transforms?+ enable-python-transforms?enable-upload-management?enable-whitelabeling?has-attached-dwh?diff --git a/src/metabase/premium_features/settings.clj b/src/metabase/premium_features/settings.cljindex 319e3957f3d..da840c557c2 100644--- a/src/metabase/premium_features/settings.clj+++ b/src/metabase/premium_features/settings.clj@@ -295,6 +295,10 @@"Should we allow users to use transforms?":transforms)+(define-premium-feature ^{:added "0.57.0"} enable-python-transforms?+ "Should we allow users to use Python transforms?"+ :transforms-python)+(defn- -token-features []{:advanced_permissions (enable-advanced-permissions?):ai_sql_fixer (enable-ai-sql-fixer?)@@ -340,6 +344,7 @@:sso_saml (enable-sso-saml?):table_data_editing (table-data-editing?):transforms (enable-transforms?)+ :transforms-python (enable-python-transforms?):upload_management (enable-upload-management?):whitelabel (enable-whitelabeling?)})diff --git a/test/metabase/driver/common_test.clj b/test/metabase/driver/common_test.cljindex b54625923b0..4e279cd1317 100644--- a/test/metabase/driver/common_test.clj+++ b/test/metabase/driver/common_test.clj@@ -7,7 +7,11 @@[metabase.driver.util :as driver.u][metabase.premium-features.core :as premium-features][metabase.settings.core :as setting]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.test.data.sql :as sql.tx]+ [metabase.util.json :as json]))++(set! *warn-on-reflection* true)(deftest ^:parallel base-type-inference-test(is (= :type/Text@@ -88,3 +92,142 @@[2 456.0 0.789 789.0]](mt/formatted-rows [int double double double](mt/run-mbql-query json-decimals-table)))))))))++(defn- base-type-test-data+ "Base types that all drivers should support with test data."+ [driver]+ {:columns [{:name "id" :type :type/Integer :nullable? false}+ {:name "name" :type :type/Text :nullable? true}+ {:name "price" :type :type/Float :nullable? true}+ {:name "active" :type :type/Boolean :nullable? true}+ {:name "created_date" :type :type/Date :nullable? true}+ {:name "created_at" :type :type/DateTime :nullable? true}]+ :data [[1 "Product A" 19.99 (if (= :sqlserver driver) 1 true) "2024-01-01" "2024-01-01T12:00:00"]+ [2 "Product B" 15.50 (if (= :sqlserver driver) 1 false) "2024-02-01" "2024-02-01T09:15:30"]+ [3 nil nil nil nil nil]]})++(deftest insert-from-source!-test+ (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :mongo :clickhouse}+ (mt/with-empty-db+ (let [driver driver/*driver*+ db-id (mt/id)+ table-name (mt/random-name)+ schema-name (when (get-method sql.tx/session-schema driver) (sql.tx/session-schema driver))+ qualified-table-name (if schema-name+ (keyword schema-name table-name)+ (keyword table-name))+ {:keys [columns data]} (base-type-test-data driver)+ column-definitions (into {} (map (fn [{:keys [name type]}]+ [name (driver/type->database-type driver type)]))+ columns)]+ (mt/as-admin+ (try+ (driver/create-table! driver db-id qualified-table-name column-definitions {})++ (testing "insert-from-source! should insert rows with all basic types correctly"+ (let [data-source {:type :rows :data data}+ _ (driver/insert-from-source! driver db-id+ {:name qualified-table-name+ :columns columns}+ data-source)]+ (when-let [inserted-rows (and (not (#{:bigquery-cloud-sdk :snowflake :mongo} driver)) ;table-rows-seq not implemented+ (driver/table-rows-seq driver/*driver* (mt/db) {:name table-name+ :schema schema-name}))]+ (is (= (count data) (count inserted-rows)) "Should insert all test rows including nulls"))))+ (finally+ (driver/drop-table! driver db-id qualified-table-name))))))))++(deftest insert-from-jsonl-file-test+ (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :mongo :clickhouse}+ (mt/with-empty-db+ (let [driver driver/*driver*+ db-id (mt/id)+ table-name (mt/random-name)+ schema-name (when (get-method sql.tx/session-schema driver) (sql.tx/session-schema driver))+ qualified-table-name (if schema-name+ (keyword schema-name table-name)+ (keyword table-name))+ {:keys [columns data]} (base-type-test-data driver)+ column-definitions (into {} (map (fn [{:keys [name type]}]+ [name (driver/type->database-type driver type)]))+ columns)]+ (mt/as-admin+ (try+ (driver/create-table! driver db-id qualified-table-name column-definitions {})++ (testing "insert-from-source! should insert rows from JSONL file correctly"+ (let [temp-file (java.io.File/createTempFile "test-data" ".jsonl")+ col-names (map :name columns)++ json-rows (map (fn [row]+ (into {} (map vector col-names row)))+ data)]+ (try++ (with-open [writer (java.io.FileWriter. temp-file)]+ (doseq [row json-rows]+ (.write writer (str (json/encode row) "\n"))))++ (let [data-source {:type :jsonl-file :file temp-file}+ _ (driver/insert-from-source! driver db-id+ {:name qualified-table-name+ :columns columns}+ data-source)]+ (when-let [inserted-rows (and (not (#{:bigquery-cloud-sdk :snowflake :mongo} driver)) ;table-rows-seq not implemented+ (driver/table-rows-seq driver/*driver* (mt/db) {:name table-name+ :schema schema-name}))]+ (is (= (count data) (count inserted-rows))+ "Should insert all test rows from JSONL file including nulls")))+ (finally+ (.delete temp-file)))))+ (finally+ (driver/drop-table! driver db-id qualified-table-name))))))))++(deftest ^:parallel type->database-type-h2-test+ (mt/test-driver :h2+ (testing "type->database-type multimethod returns correct H2 types"+ (are [base-type expected] (= expected (driver/type->database-type :h2 base-type))+ :type/Boolean [:boolean]+ :type/Date [:date]+ :type/DateTime [:timestamp]+ :type/DateTimeWithTZ [:timestamp-with-time-zone]+ :type/Float [(keyword "DOUBLE PRECISION")]+ :type/Integer [:int]+ :type/Number [:bigint]+ :type/Text [:varchar]+ :type/Time [:time]+ :type/UUID [:uuid]))))++(deftest ^:parallel type->database-type-mysql-test+ (mt/test-driver :mysql+ (testing "type->database-type multimethod returns correct MySQL types"+ (are [base-type expected] (= expected (driver/type->database-type :mysql base-type))+ :type/Boolean [:boolean]+ :type/Date [:date]+ :type/DateTime [:datetime]+ :type/DateTimeWithTZ [:timestamp]+ :type/Float [:double]+ :type/Decimal [:decimal]+ :type/Integer [:int]+ :type/Number [:bigint]+ :type/Text [:text]+ :type/Time [:time]))))++(deftest ^:parallel type->database-type-postgres-test+ (mt/test-driver :postgres+ (testing "type->database-type multimethod returns correct PostgreSQL types"+ (are [base-type expected] (= expected (driver/type->database-type :postgres base-type))+ :type/Boolean [:boolean]+ :type/Date [:date]+ :type/DateTime [:timestamp]+ :type/DateTimeWithTZ [:timestamp-with-time-zone]+ :type/Decimal [:decimal]+ :type/Float [:float]+ :type/Integer [:int]+ :type/JSON [:jsonb]+ :type/Number [:bigint]+ :type/SerializedJSON [:jsonb]+ :type/Text [:text]+ :type/Time [:time]+ :type/TimeWithTZ [:time-with-time-zone]+ :type/UUID [:uuid]))))diff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.cljindex d3b6f540721..b759d040712 100644--- a/test/metabase/driver/sql_jdbc_test.clj+++ b/test/metabase/driver/sql_jdbc_test.clj@@ -3,7 +3,6 @@[clojure.test :refer :all][metabase.driver :as driver][metabase.driver.common.table-rows-sample :as table-rows-sample]- [metabase.driver.mysql :as mysql][metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.util :as driver.u]@@ -298,80 +297,102 @@(map #(vector (:id %) (:name %)))sort))-#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}(deftest rename-tables-test- (mt/test-drivers #{:postgres :mysql}- (when-not (mysql/mariadb? (mt/db))- (testing "rename-tables should rename multiple tables atomically"- (let [db-id (mt/id)- driver driver/*driver*- schema (sql.tx/session-schema driver)- test-table-1 (mt/random-name)- test-table-2 (mt/random-name)- qualified-table-1 (qualified-table-name schema test-table-1)- qualified-table-2 (qualified-table-name schema test-table-2)- temp-table-1 (str test-table-1 "_temp")- temp-table-2 (str test-table-2 "_temp")- qualified-temp-1 (qualified-table-name schema temp-table-1)- qualified-temp-2 (qualified-table-name schema temp-table-2)- test-data-1 [[1 "Alice"] [2 "Bob"]]- test-data-2 [[1 "Product A"] [2 "Product B"]]]-- ;; so the create-table is using qp and it requires permission setup- ;; TODO we should change the implementation of create-table to use jdbc direclty- (mt/with-current-user (mt/user->id :crowberto)- (driver/create-table! driver db-id qualified-table-1- {"id" "INTEGER", "name" "VARCHAR(255)"} {})- (driver/create-table! driver db-id qualified-table-2- {"id" "INTEGER", "name" "VARCHAR(255)"} {})-- (try- (driver/insert-into! driver db-id qualified-table-1 ["id" "name"] test-data-1)- (driver/insert-into! driver db-id qualified-table-2 ["id" "name"] test-data-2)-- (testing "basic rename operations work correctly"- (driver/rename-tables! driver db-id- {qualified-table-1 qualified-temp-1- qualified-table-2 qualified-temp-2})-- (is (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema}))- (is (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))- (is (not (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema})))- (is (not (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))-- (is (= test-data-1 (table-rows qualified-temp-1)))- (is (= test-data-2 (table-rows qualified-temp-2)))-- (driver/rename-tables! driver db-id- {qualified-temp-1 qualified-table-1- qualified-temp-2 qualified-table-2}))-- (testing "atomicity: all renames fail if any rename fails"- (let [conflict-table (str test-table-2 "_conflict")- qualified-conflict (qualified-table-name schema conflict-table)]- (driver/create-table! driver db-id qualified-conflict {"id" "INTEGER"} {})-- (try- (is (thrown? Exception- (driver/rename-tables! driver db-id- {qualified-table-1 qualified-temp-1- qualified-table-2 qualified-conflict})))-- (testing "original tables should still exist after failed atomic rename"- (is (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema}))- (is (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))-- (testing "temp tables should not exist after failed atomic rename"- (is (not (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema})))- (is (not (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))))-- (testing "original data should be intact after failed atomic rename"- (is (= test-data-1 (table-rows qualified-table-1)))- (is (= test-data-2 (table-rows qualified-table-2))))-- (finally- (driver/drop-table! driver db-id qualified-conflict)))))-- (finally- (driver/drop-table! driver db-id qualified-table-1)- (driver/drop-table! driver db-id qualified-table-2)))))))))+ (mt/test-drivers (mt/normal-drivers-with-feature :atomic-renames)+ (testing "rename-tables should rename multiple tables atomically"+ (let [db-id (mt/id)+ driver driver/*driver*+ schema (sql.tx/session-schema driver)+ test-table-1 (mt/random-name)+ test-table-2 (mt/random-name)+ qualified-table-1 (qualified-table-name schema test-table-1)+ qualified-table-2 (qualified-table-name schema test-table-2)+ temp-table-1 (str test-table-1 "_temp")+ temp-table-2 (str test-table-2 "_temp")+ qualified-temp-1 (qualified-table-name schema temp-table-1)+ qualified-temp-2 (qualified-table-name schema temp-table-2)+ test-data-1 [[1 "Alice"] [2 "Bob"]]+ test-data-2 [[1 "Product A"] [2 "Product B"]]]++ (driver/create-table! driver db-id qualified-table-1+ {"id" "INTEGER", "name" "VARCHAR(255)"} {})+ (driver/create-table! driver db-id qualified-table-2+ {"id" "INTEGER", "name" "VARCHAR(255)"} {})++ (try+ (driver/insert-into! driver db-id qualified-table-1 ["id" "name"] test-data-1)+ (driver/insert-into! driver db-id qualified-table-2 ["id" "name"] test-data-2)++ (testing "basic rename operations work correctly"+ (driver/rename-tables! driver db-id+ {qualified-table-1 qualified-temp-1+ qualified-table-2 qualified-temp-2})++ (is (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema}))+ (is (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))+ (is (not (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema})))+ (is (not (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))++ (is (= test-data-1 (table-rows qualified-temp-1)))+ (is (= test-data-2 (table-rows qualified-temp-2)))++ (driver/rename-tables! driver db-id+ {qualified-temp-1 qualified-table-1+ qualified-temp-2 qualified-table-2}))++ (testing "atomicity: all renames fail if any rename fails"+ (let [conflict-table (str test-table-2 "_conflict")+ qualified-conflict (qualified-table-name schema conflict-table)]+ (driver/create-table! driver db-id qualified-conflict {"id" "INTEGER"} {})++ (try+ (is (thrown? Exception+ (driver/rename-tables! driver db-id+ {qualified-table-1 qualified-temp-1+ qualified-table-2 qualified-conflict})))++ (testing "original tables should still exist after failed atomic rename"+ (is (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema}))+ (is (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))++ (testing "temp tables should not exist after failed atomic rename"+ (is (not (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema})))+ (is (not (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))))++ (testing "original data should be intact after failed atomic rename"+ (is (= test-data-1 (table-rows qualified-table-1)))+ (is (= test-data-2 (table-rows qualified-table-2))))++ (finally+ (driver/drop-table! driver db-id qualified-conflict)))))++ (finally+ (driver/drop-table! driver db-id qualified-table-1)+ (driver/drop-table! driver db-id qualified-table-2)))))))++(deftest rename-table-test+ (mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc+ :+features [:rename]})+ (testing "rename-table! should rename a single table correctly"+ (let [db-id (mt/id)+ driver driver/*driver*+ schema (sql.tx/session-schema driver)+ test-table (mt/random-name)+ renamed-table (str test-table "_renamed")+ qualified-table (qualified-table-name schema test-table)+ qualified-renamed (qualified-table-name schema renamed-table)]+ (driver/create-table! driver db-id qualified-table+ {"id" "INTEGER", "name" "VARCHAR(255)"} {})+ (try+ (testing "single table rename works correctly"+ (driver/rename-table! driver db-id qualified-table qualified-renamed)+ (is (driver/table-exists? driver (mt/db) {:name renamed-table :schema schema})+ "Renamed table should exist")+ (is (not (driver/table-exists? driver (mt/db) {:name test-table :schema schema}))+ "Original table should not exist"))++ (finally+ (when (driver/table-exists? driver (mt/db) {:name renamed-table :schema schema})+ (driver/drop-table! driver db-id qualified-renamed))+ (when (driver/table-exists? driver (mt/db) {:name test-table :schema schema})+ (driver/drop-table! driver db-id qualified-table))))))))diff --git a/test/metabase/driver_test.clj b/test/metabase/driver_test.cljindex d979a278506..fde4af96904 100644--- a/test/metabase/driver_test.clj+++ b/test/metabase/driver_test.clj@@ -393,3 +393,14 @@(qp/process-query success-query)(is (= 3.0 (mt/metric-value system :metabase-query-processor/query {:driver driver/*driver* :status "success"})))(is (= 2.0 (mt/metric-value system :metabase-query-processor/query {:driver driver/*driver* :status "failure"}))))))))++(deftest python-transform-drivers-multimethods-support+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)+ (let [driver driver/*driver*]+ (is (get-method driver/create-table! driver))+ (is (get-method driver/table-name-length-limit driver))+ (is (get-method driver/drop-table! driver))+ (is (let [should-be-supported-by-all #{:type/Number :type/Text :type/Date :type/DateTime :type/DateTimeWithTZ :type/Boolean}]+ (and (get-method driver/type->database-type driver)+ (every? #(driver/type->database-type driver %) should-be-supported-by-all))))+ (is (get-method driver/insert-from-source! [driver :jsonl-file])))))