Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Python Transforms (#63700)

Nicola Mometto7b6fbd6374a1
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 81ce3c877b4..cbbf8a98290 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -173,6 +173,7 @@
metabase.driver.sql-jdbc.execute/execute-prepared-statement!
metabase.notification.models/create-notification!
metabase.pulse.send/send-pulse!
+ metabase-enterprise.transforms-python.python-runner-test/execute!
;; Safe in tests, where we typically use a private, temporary index per thread.
metabase.search.core/reindex!
metabase.util/run-count!
@@ -418,6 +419,7 @@
metabase-enterprise.transforms.api-test/with-transform-tags clojure.core/let
metabase-enterprise.transforms.api-test/with-worker-runs clojure.core/let
metabase-enterprise.transforms.test-util/with-transform-cleanup! clojure.core/let
+ metabase-enterprise.transforms.execute/with-transform-lifecycle clojure.core/let
metabase.actions.test-util/with-actions clojure.core/let
metabase.api.common/let-404 clojure.core/let
metabase.app-db.custom-migrations.util/with-temp-scheduler! clojure.core/fn
@@ -519,6 +521,7 @@
metabase-enterprise.cache.config-test/with-temp-persist-models hooks.common/with-seven-bindings
metabase-enterprise.semantic-search.db.connection/with-migrate-tx hooks.common/let-one-with-optional-value
metabase-enterprise.serialization.test-util/with-temp-dpc hooks.toucan2.tools.with-temp/with-temp
+ metabase-enterprise.transforms-python.drivers-test/with-test-table hooks.common/with-two-bindings
metabase.api.common.internal-test/with-jetty-server hooks.common/with-one-binding
metabase.api.macros/defendpoint hooks.metabase.api.macros/defendpoint
metabase.app-db.schema-migrations-test.impl/test-migrations hooks.metabase.app-db.schema-migrations-test.impl/test-migrations
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 5d92ee979bf..c98b29ea88d 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -894,6 +894,19 @@
enterprise/advanced-permissions
enterprise/sandbox}}
+ python-runner
+ {:team "Workflows"
+ :api #{metabase.python-runner.api
+ metabase.python-runner.core
+ metabase.python-runner.init}
+ :uses #{api
+ driver
+ models
+ query-processor
+ request
+ util
+ enterprise/transforms}}
+
queries
{:team "Querying"
:api #{metabase.queries.api
@@ -1984,6 +1997,7 @@
:api #{metabase-enterprise.transforms.api
metabase-enterprise.transforms.core
metabase-enterprise.transforms.init
+ metabase-enterprise.transforms.instrumentation
metabase-enterprise.transforms.util}
:uses #{analytics
api
@@ -2009,6 +2023,7 @@
enterprise/transforms-python
{:team "Querying"
:api #{metabase-enterprise.transforms-python.api
+ metabase-enterprise.transforms-python.execute
metabase-enterprise.transforms-python.init
metabase-enterprise.transforms-python.settings}
:uses #{api
diff --git a/.github/scripts/setup-python-runner.sh b/.github/scripts/setup-python-runner.sh
new file mode 100755
index 00000000000..518b3cde0fd
--- /dev/null
+++ b/.github/scripts/setup-python-runner.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+# Parse command line arguments
+SKIP_START=false
+
+while [[ $# -gt 0 ]]; do
+ case $1 in
+ --skip-start)
+ SKIP_START=true
+ shift
+ ;;
+ *)
+ echo "Unknown option: $1"
+ echo "Usage: $0 [--skip-start]"
+ exit 1
+ ;;
+ esac
+done
+
+echo "Cloning python-runner repository..."
+git clone --depth 1 https://${METABASE_AUTOMATION_USER_TOKEN}@github.com/metabase/python-runner-container.git python-runner
+
+if [ "$SKIP_START" = false ]; then
+ echo "Starting python-runner..."
+ cd python-runner
+ make run-ci
+ cd ..
+fi
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.yml
index a2587bbbde1..55853b0f845 100644
--- a/.github/workflows/app-db.yml
+++ b/.github/workflows/app-db.yml
@@ -35,21 +35,21 @@ jobs:
build-static-viz: false
test-args: >-
:only '"enterprise/backend/test"'
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- name: EE App DB Tests (Part 1)
build-static-viz: true
test-args: >-
:only '"test"'
:partition/total 2
:partition/index 0
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- name: EE App DB Tests (Part 2)
build-static-viz: true
test-args: >-
:only '"test"'
:partition/total 2
:partition/index 1
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
services:
mysql:
@@ -147,21 +147,21 @@ jobs:
build-static-viz: false
test-args: >-
:only '"enterprise/backend/test"'
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- name: EE App DB Tests (Part 1)
build-static-viz: true
test-args: >-
:only '"test"'
:partition/total 2
:partition/index 0
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- name: EE App DB Tests (Part 2)
build-static-viz: true
test-args: >-
:only '"test"'
:partition/total 2
:partition/index 1
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
services:
mysql:
@@ -259,11 +259,13 @@ jobs:
build-static-viz: false
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ exclude-tag: ':mb/transforms-python-test'
- name: Enterprise Tests
build-static-viz: false
test-args: >-
:only '"enterprise/backend/test"'
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- name: EE App DB Tests (Part 1)
build-static-viz: true
test-args: >-
@@ -271,14 +273,14 @@ jobs:
:exclude-tags [:mb/driver-tests]
:partition/total 2
:partition/index 0
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- name: EE App DB Tests (Part 2)
build-static-viz: true
test-args: >-
:only '"test"'
:partition/total 2
:partition/index 1
- exclude-tag: ':mb/driver-tests'
+ exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
env:
# Only run old migrations tests on pushes to master or release branches. All other branches should skip tests
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index a1f8e7d9d97..c35e0121fb9 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -90,6 +90,7 @@ jobs:
junit-name: 'be-tests-athena-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -106,6 +107,18 @@ jobs:
if: ${{ !inputs.skip }}
runs-on: ubuntu-22.04
timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ job:
+ - name: Driver Tests
+ test-args: >-
+ :only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ - name: Transforms Python Tests
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ needs-python-runner: true
env:
CI: 'true'
DRIVERS: bigquery-cloud-sdk
@@ -116,15 +129,19 @@ jobs:
MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
- name: BigQuery
+ name: BigQuery ${{ matrix.job.name }}
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test BigQuery Cloud SDK driver
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-bigquery-cloud-sdk-ee'
- test-args: >-
- :only-tags [:mb/driver-tests]
+ test-args: ${{ matrix.job.test-args }}
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -141,6 +158,18 @@ jobs:
if: ${{ !inputs.skip }}
runs-on: ubuntu-22.04
timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ job:
+ - name: Driver Tests
+ test-args: >-
+ :only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ - name: Transforms Python Tests
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ needs-python-runner: true
env:
CI: 'true'
DRIVERS: clickhouse
@@ -148,9 +177,14 @@ jobs:
MB_CLICKHOUSE_TEST_PORT: 8123
MB_CLICKHOUSE_TEST_OLD_PORT: 8124
MB_CLICKHOUSE_TEST_NGINX_PORT: 8127
- name: Clickhouse
+ name: Clickhouse ${{ matrix.job.name }}
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Add ClickHouse TLS instance to /etc/hosts
run: |
sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts
@@ -170,8 +204,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-clickhouse-ee'
- test-args: >-
- :only-tags [:mb/driver-tests]
+ test-args: ${{ matrix.job.test-args }}
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -208,6 +241,7 @@ jobs:
junit-name: 'be-tests-druid-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -244,6 +278,7 @@ jobs:
junit-name: 'be-tests-druid-jdbc-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -280,6 +315,7 @@ jobs:
junit-name: 'be-tests-databricks-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -317,6 +353,7 @@ jobs:
junit-name: 'be-tests-databricks-multi-catalog-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -338,12 +375,17 @@ jobs:
name: H2
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test H2 driver
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-h2'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -388,6 +430,11 @@ jobs:
- name: Driver Tests
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ - name: Transforms Python Tests
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ needs-python-runner: true
services:
mysql:
@@ -442,6 +489,11 @@ jobs:
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ #if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test ${{ matrix.version.name }}
uses: ./.github/actions/test-driver
with:
@@ -476,6 +528,15 @@ jobs:
- name: MongoDB Latest
junit-name: be-tests-mongo-latest-ee
image: mongo:latest
+ job:
+ - name: Driver Tests
+ test-args: >-
+ :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
+ :only-tags [:mb/driver-tests]
+ - name: Transforms Python Tests
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ needs-python-runner: true
env:
CI: 'true'
DRIVERS: mongo
@@ -489,16 +550,19 @@ jobs:
env:
MONGO_INITDB_ROOT_USERNAME: metabase
MONGO_INITDB_ROOT_PASSWORD: metasample123
- name: ${{ matrix.version.name }}
+ name: ${{ matrix.version.name }} ${{ matrix.job.name }}
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test ${{ matrix.version.name }}
uses: ./.github/actions/test-driver
with:
junit-name: ${{ matrix.version.junit-name }}
- test-args: >-
- :exclude-tags [:mongo-sharded-cluster-tests]
- :only-tags [:mb/driver-tests]
+ test-args: ${{ matrix.job.test-args }}
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -562,7 +626,7 @@ jobs:
with:
junit-name: ${{ matrix.version.junit-name }}
test-args: >-
- :exclude-tags [:mongo-sharded-cluster-tests]
+ :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
:only-tags [:mb/driver-tests]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
@@ -670,6 +734,7 @@ jobs:
junit-name: ${{ matrix.version.junit-name }}
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -704,6 +769,11 @@ jobs:
- name: Driver Tests
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ - name: Transforms Python Tests
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ needs-python-runner: true
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
env:
# Only run old migrations tests on pushes to master or release branches. All other branches should skip tests
@@ -745,6 +815,11 @@ jobs:
POSTGRES_HOST_AUTH_METHOD: trust
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ #if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
uses: ./.github/actions/test-driver
with:
@@ -821,6 +896,7 @@ jobs:
junit-name: 'be-tests-presto-jdbc-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -850,8 +926,14 @@ jobs:
skip: false
test-args: >-
:only-tags [:mb/driver-tests]
- :exclude-tags [:mb/upload-tests]
+ :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]'
:fail-fast? true
+ - name: Redshift Transforms Python Tests
+ skip: false
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ :fail-fast? true
+ needs-python-runner: true
env:
CI: 'true'
@@ -864,6 +946,11 @@ jobs:
name: ${{ matrix.job.name }}
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test ${{ matrix.job.name }}
if: ${{ !matrix.job.skip }}
uses: ./.github/actions/test-driver
@@ -887,6 +974,18 @@ jobs:
if: ${{ !inputs.skip }}
runs-on: ubuntu-22.04
timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ job:
+ - name: Driver Tests
+ test-args: >-
+ :only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ - name: Transforms Python Tests
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ needs-python-runner: true
env:
CI: 'true'
DRIVERS: snowflake
@@ -905,15 +1004,19 @@ jobs:
MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER
MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE
MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB
- name: Snowflake
+ name: Snowflake ${{ matrix.job.name }}
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test Snowflake driver
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-snowflake-ee'
- test-args: >-
- :only-tags [:mb/driver-tests]
+ test-args: ${{ matrix.job.test-args }}
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -948,6 +1051,7 @@ jobs:
junit-name: 'be-tests-sparksql-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -977,6 +1081,7 @@ jobs:
junit-name: 'be-tests-sqlite-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -1004,6 +1109,15 @@ jobs:
- name: SQL Server 2022
junit-name: be-tests-sqlserver-2022-ee
image: mcr.microsoft.com/mssql/server:2022-latest
+ job:
+ - name: Driver Tests
+ test-args: >-
+ :only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ - name: Transforms Python Tests
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ needs-python-runner: true
env:
CI: 'true'
DRIVERS: sqlserver
@@ -1019,15 +1133,19 @@ jobs:
ACCEPT_EULA: Y
SA_PASSWORD: 'P@ssw0rd'
MSSQL_MEMORY_LIMIT_MB: 1024
- name: ${{ matrix.version.name }}
+ name: ${{ matrix.version.name }} ${{ matrix.job.name }}
steps:
- uses: actions/checkout@v4
+ - name: Setup python-runner
+ if: ${{ matrix.job.needs-python-runner == true }}
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Test ${{ matrix.version.name }}
uses: ./.github/actions/test-driver
with:
junit-name: ${{ matrix.version.junit-name }}
- test-args: >-
- :only-tags [:mb/driver-tests]
+ test-args: ${{ matrix.job.test-args }}
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
@@ -1136,6 +1254,7 @@ jobs:
junit-name: 'be-tests-vertica-ee'
test-args: >-
:only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always()
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index 8cda8b9f24f..196788c485d 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -60,6 +60,14 @@ jobs:
CYPRESS_IS_EMBEDDING_SDK: "${{ inputs.name == 'embedding-sdk' }}"
steps:
- uses: actions/checkout@v4
+ with:
+ submodules: true
+ token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+
+ - name: Setup python-runner
+ env:
+ METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ run: ./.github/scripts/setup-python-runner.sh
- name: Prepare Docker containers
uses: ./.github/actions/e2e-prepare-containers
diff --git a/deps.edn b/deps.edn
index 713c03773e5..b82eabaf744 100644
--- a/deps.edn
+++ b/deps.edn
@@ -198,6 +198,8 @@
org.eclipse.jetty.ee9/jetty-ee9-servlet
org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
slingshot/slingshot {:mvn/version "0.12.2"} ; enhanced throw/catch, used by other deps
+ software.amazon.awssdk/s3 {:mvn/version "2.33.0"} ; AWS SDK v2 for S3
+ software.amazon.awssdk/sts {:mvn/version "2.33.0"} ; Required for S3 client to use service account
stencil/stencil {:mvn/version "0.5.0"} ; Mustache templates for Clojure
user-agent/user-agent {:mvn/version "0.1.1"} ; User-Agent string parser, for Login History page & elsewhere
weavejester/dependency {:mvn/version "1.0.0"} ; Dependency graphs and topological sorting
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index a91cecc05b3..36bd741b38a 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -20,7 +20,22 @@ title: Driver interface changelog
- Added `metabase.driver/rename-tables!*` multimethod for atomic table renaming operations. Takes a map of {from-table to-table}
pairs that has been topologically sorted.
-- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+- Added `metabase.driver/rename-table!` multimethod for table renaming operations. Takes a single from-table to-table pair.
+ Fallbacks to singleton call to `rename-tables!*` if available.
+
+- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!`
+ which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase
+ type (from the type hierarchy) as a HoneySQL spec. This method handles general Metabase base types.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources
+ into existing tables. This multimethod dispatches on both the driver and the data source type
+ (`:rows` or `:jsonl-file`). It allows drivers to optimize based on the data source type and returns the number
+ of rows inserted. Default implementations are provided for both `:rows` and `:jsonl-file` source types.
+
+- Added `metabase.driver/insert-col->val` multimethod to parse values for insertion based on driver, data source type
+ and column definition. Drivers should implement this when their insertion mechanism needs values converted to proper types.
- `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
instead of just the top-level message as a string.
diff --git a/e2e/support/helpers/api/index.ts b/e2e/support/helpers/api/index.ts
index ee10d63d045..0624a12c28d 100644
--- a/e2e/support/helpers/api/index.ts
+++ b/e2e/support/helpers/api/index.ts
@@ -38,4 +38,4 @@ export { getCurrentUser } from "./getCurrentUser";
export { remapDisplayValueToFK } from "./remapDisplayValueToFK";
export { updateComment } from "./updateComment";
export { updateDashboardCards } from "./updateDashboardCards";
-export { updateSetting } from "./updateSetting";
+export { updateSetting, updateEnterpriseSetting } from "./updateSetting";
diff --git a/e2e/support/helpers/api/updateSetting.ts b/e2e/support/helpers/api/updateSetting.ts
index e20773441bb..d7b8bb23851 100644
--- a/e2e/support/helpers/api/updateSetting.ts
+++ b/e2e/support/helpers/api/updateSetting.ts
@@ -1,4 +1,4 @@
-import type { Settings } from "metabase-types/api";
+import type { EnterpriseSettings, Settings } from "metabase-types/api";
export const updateSetting = <
TKey extends keyof Settings,
@@ -9,3 +9,13 @@ export const updateSetting = <
): Cypress.Chainable<Cypress.Response<never>> => {
return cy.request<never>("PUT", `/api/setting/${setting}`, { value });
};
+
+export const updateEnterpriseSetting = <
+ TKey extends keyof EnterpriseSettings,
+ TValue extends EnterpriseSettings[TKey],
+>(
+ setting: TKey,
+ value: TValue,
+): Cypress.Chainable<Cypress.Response<never>> => {
+ return cy.request<never>("PUT", `/api/setting/${setting}`, { value });
+};
diff --git a/e2e/support/helpers/e2e-codemirror-helpers.ts b/e2e/support/helpers/e2e-codemirror-helpers.ts
index 088a6928a83..6da51822fcc 100644
--- a/e2e/support/helpers/e2e-codemirror-helpers.ts
+++ b/e2e/support/helpers/e2e-codemirror-helpers.ts
@@ -1,6 +1,208 @@
+type TypeOptions = {
+ delay?: number;
+ focus?: boolean;
+ allowFastSet?: boolean;
+};
+
+export function codeMirrorHelpers<T extends object>(testId: string, extra: T) {
+ const helpers = {
+ get() {
+ cy.findAllByTestId("loading-indicator").should("not.exist");
+ return cy.get(`[data-testid=${testId}] .cm-content`);
+ },
+ focus() {
+ helpers.get().should("be.visible").click("right");
+ helpers.get().get(".cm-editor").should("have.class", "cm-focused");
+ return helpers;
+ },
+ blur() {
+ helpers.get().get(".cm-content").blur();
+ return helpers;
+ },
+ selectAll() {
+ const isMac = Cypress.platform === "darwin";
+ const metaKey = isMac ? "Meta" : "Control";
+ helpers.focus();
+ cy.realPress([metaKey, "A"]);
+ return helpers;
+ },
+ clear() {
+ helpers.selectAll();
+ cy.realPress(["Backspace"]);
+ return helpers;
+ },
+ type(
+ text: string,
+ { focus = true, delay = 10, allowFastSet = false }: TypeOptions = {},
+ ) {
+ if (focus) {
+ helpers.focus();
+ }
+
+ if (allowFastSet) {
+ // Enter the formula in one go
+ // HACK: we do invoke("text") instead of type() because type() does not work on
+ // CodeMirror elements in Cypress. realType() would work but some of the formulas
+ // contain special characters that are not supported by realType().
+ helpers.get().invoke("text", text);
+
+ // invoke("text") does not trigger the validator, so we need to trigger it manually
+ // by typing something
+ helpers.type(" {backspace}");
+
+ return helpers;
+ }
+
+ const isMac = Cypress.platform === "darwin";
+ const metaKey = isMac ? "Meta" : "Control";
+
+ const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);
+
+ function insert(text: string) {
+ cy.realType(text, { pressDelay: delay });
+ }
+
+ // HACK: realType does not accept a lot of common escape sequences,
+ // so we implement them manually here for the native editor.
+ parts.forEach((part) => {
+ switch (part.toLowerCase()) {
+ case "":
+ return;
+
+ case "{clear}":
+ return helpers.clear();
+
+ case "{selectall}":
+ return helpers.selectAll();
+
+ case "{leftarrow}":
+ return cy.realPress(["ArrowLeft"]);
+
+ case "{rightarrow}":
+ return cy.realPress(["ArrowRight"]);
+
+ case "{downarrow}":
+ return cy.realPress(["ArrowDown"]);
+
+ case "{uparrow}":
+ return cy.realPress(["ArrowUp"]);
+
+ case "{enter}":
+ return cy.realPress(["Enter"]);
+
+ case "{home}":
+ case "{movetostart}":
+ return cy.realPress(["Home"]);
+
+ case "{end}":
+ case "{movetoend}":
+ return cy.realPress(["End"]);
+
+ case "{backspace}":
+ return cy.realPress(["Backspace"]);
+
+ case "{tab}":
+ return cy.realPress(["Tab"]);
+
+ case "{nextcompletion}":
+ cy.wait(50);
+ return cy.realPress([metaKey, "j"]);
+
+ case "{prevcompletion}":
+ cy.wait(50);
+ return cy.realPress([metaKey, "k"]);
+
+ case "{{}":
+ return cy.realType("{");
+ }
+
+ if (part.startsWith("{") && part.endsWith("}")) {
+ // unknown escape sequence, let's try typing it
+ insert("{");
+ insert(part.slice(1));
+ return;
+ }
+
+ // realType does not support → arrow, so we replace it with ->, which the editor
+ // expands into →
+ const unexpanded = part.replaceAll(/→/g, "->");
+ const alphabet =
+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789^[]()-,.;_!@#$%&*+=/<>\" ':;\\\n{}";
+ if (unexpanded.split("").some((char) => !alphabet.includes(char))) {
+ throw new Error(
+ `unknown character in codeMirrorHelpers.type in ${part}`,
+ );
+ }
+
+ insert(unexpanded);
+ });
+
+ return helpers;
+ },
+ textbox() {
+ return helpers.get().get("[role='textbox']");
+ },
+ value() {
+ // Get the multiline text content of the editor
+ return helpers
+ .textbox()
+ .get(".cm-line")
+ .then((lines) => {
+ const text: string[] = [];
+ lines.each((_, line) => {
+ const placeholder = line.querySelector(".cm-placeholder");
+ if (placeholder) {
+ return;
+ }
+ text.push(line.textContent ?? "");
+ });
+ return text.join("\n");
+ });
+ },
+ completions() {
+ return cy.get(".cm-tooltip-autocomplete").should("be.visible");
+ },
+ completion(label: string) {
+ return cy.get(".cm-completionLabel").contains(label).parent();
+ },
+ acceptCompletion(key: "enter" | "tab" = "enter") {
+ helpers.completions().should("be.visible");
+
+ // Avoid flakiness with CodeMirror not accepting the suggestion immediately
+ cy.wait(300);
+ helpers.type(`{${key}}`, { focus: false });
+ },
+ selectCompletion(name: string) {
+ helpers.completions().should("be.visible");
+
+ // Avoid flakiness with CodeMirror not accepting the suggestion immediately
+ cy.wait(300);
+ helpers.completion(name).click();
+ },
+ paste(content: string) {
+ helpers.textbox().then((el) => {
+ const clipboardData = new DataTransfer();
+ clipboardData.setData("text/plain", content);
+
+ const pasteEvent = new ClipboardEvent("paste", {
+ bubbles: true,
+ cancelable: true,
+ clipboardData,
+ });
+
+ el[0].dispatchEvent(pasteEvent);
+ });
+ },
+ ...extra,
+ };
+
+ return helpers;
+}
+
export function codeMirrorEditor() {
return cy.get(".cm-content");
}
+
export function codeMirrorValue() {
// Get the multiline text content of the editor
return codeMirrorEditor()
diff --git a/e2e/support/helpers/e2e-custom-column-helpers.ts b/e2e/support/helpers/e2e-custom-column-helpers.ts
index 66431f6ce19..92f23114ccf 100644
--- a/e2e/support/helpers/e2e-custom-column-helpers.ts
+++ b/e2e/support/helpers/e2e-custom-column-helpers.ts
@@ -1,5 +1,7 @@
import { popover } from "e2e/support/helpers/e2e-ui-elements-helpers";
+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";
+
export function expressionEditorWidget() {
return cy.findByTestId("expression-editor");
}
@@ -8,9 +10,6 @@ export function expressionEditorTextfield() {
return CustomExpressionEditor.get();
}
-const isMac = Cypress.platform === "darwin";
-const metaKey = isMac ? "Meta" : "Control";
-
export function enterCustomColumnDetails({
formula,
name,
@@ -71,224 +70,43 @@ export function enterCustomColumnDetails({
}
}
-export const CustomExpressionEditor = {
- get() {
- return cy.findByTestId("custom-expression-query-editor");
- },
- /**
- * @param {string} text the formula to type
- * @param {Object} options
- * @param {boolean} [options.focus]
- * true by default. Set to false to stop the helper from explicitly focussing the formula input.
- * @param {boolean} [options.allowFastSet]
- * true by default. However, if you need to examine the popover in the test, it should be set to false so the popover is not dismissed
- * Because CodeMirror uses a contenteditable div, and it is not possible to use cy.type() on it, we emulate .type with realPress.
- * This does not always work, since realPress() does not support all characters. Setting this to true will enable an escape hatch
- * that uses cy.invoke('text') under the hood, to allow for formulas that contain unsupported characters.
- * This has some other sideeffects however, so use it sparingly.
- */
- type(
- text: string,
- {
- allowFastSet = false,
- focus = true,
- delay = 0,
- }: {
- focus?: boolean;
- allowFastSet?: boolean;
- delay?: number;
- } = {},
- ) {
- if (focus) {
- CustomExpressionEditor.focus();
- }
-
- if (allowFastSet) {
- // Enter the formula in one go
- // HACK: we do invoke("text") instead of type() because type() does not work on
- // CodeMirror elements in Cypress. realType() would work but some of the formulas
- // contain special characters that are not supported by realType().
- CustomExpressionEditor.get().findByRole("textbox").invoke("text", text);
-
- // invoke("text") does not trigger the validator, so we need to trigger it manually
- // by typing something
- CustomExpressionEditor.type(" {backspace}");
-
+export const CustomExpressionEditor = codeMirrorHelpers(
+ "custom-expression-query-editor",
+ {
+ formatButton() {
+ return cy.findByLabelText("Auto-format");
+ },
+ format() {
+ CustomExpressionEditor.formatButton().should("be.visible");
+ CustomExpressionEditor.formatButton().click();
return CustomExpressionEditor;
- }
-
- const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);
-
- parts.forEach((part) => {
- cy.wait(delay);
- switch (part.toLowerCase()) {
- case "":
- return;
-
- case "{clear}":
- return CustomExpressionEditor.clear();
-
- case "{selectall}":
- return CustomExpressionEditor.selectAll();
-
- case "{leftarrow}":
- return cy.realPress(["ArrowLeft"]);
-
- case "{rightarrow}":
- return cy.realPress(["ArrowRight"]);
-
- case "{downarrow}":
- return cy.realPress(["ArrowDown"]);
-
- case "{uparrow}":
- return cy.realPress(["ArrowUp"]);
-
- case "{enter}":
- return cy.realPress(["Enter"]);
-
- case "{home}":
- case "{movetostart}":
- return cy.realPress(["Home"]);
-
- case "{end}":
- case "{movetoend}":
- return cy.realPress(["End"]);
-
- case "{backspace}":
- return cy.realPress(["Backspace"]);
-
- case "{tab}":
- return cy.realPress(["Tab"]);
-
- case "{nextcompletion}":
- cy.wait(50);
- return cy.realPress([metaKey, "j"]);
-
- case "{prevcompletion}":
- cy.wait(50);
- return cy.realPress([metaKey, "k"]);
-
- case "{{}":
- return cy.realType("{");
- }
-
- if (part.startsWith("{") && part.endsWith("}")) {
- // unknown escape sequence, let's try typing it
- throw new Error(
- `unknown escape sequence in CustomExpressionEditor.type: ${part}`,
- );
- }
-
- // realType does not support → arrow, so we replace it with ->, which the editor
- // expands into →
- const unexpanded = part.replaceAll(/→/g, "->");
-
- const alphabet =
- "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789^[]()-,.;_!@#$%&*+=/<>\" ':;\\\n";
- if (unexpanded.split("").some((char) => !alphabet.includes(char))) {
- throw new Error(
- `unknown character in CustomExpressionEditor.type in ${part}`,
- );
- }
-
- cy.realType(unexpanded, { delay });
- });
- return CustomExpressionEditor;
- },
- focus() {
- CustomExpressionEditor.get().click("right");
- return CustomExpressionEditor;
- },
- blur() {
- // click outside the expression editor
- cy.findByTestId("expression-editor").click("bottomRight", { force: true });
- return CustomExpressionEditor;
- },
- formatButton() {
- return cy.findByLabelText("Auto-format");
- },
- format() {
- CustomExpressionEditor.formatButton().should("be.visible");
- CustomExpressionEditor.formatButton().click();
- return CustomExpressionEditor;
- },
- selectAll() {
- CustomExpressionEditor.focus();
- cy.realPress([metaKey, "A"]);
- return CustomExpressionEditor;
- },
- clear() {
- CustomExpressionEditor.selectAll();
- cy.realPress(["Backspace"]);
- return CustomExpressionEditor;
- },
- textbox() {
- return CustomExpressionEditor.get().get("[role='textbox']");
- },
- value() {
- // Get the multiline text content of the editor
- return CustomExpressionEditor.textbox()
- .get(".cm-line")
- .then((lines) => {
- const text: string[] = [];
- lines.each((_, line) => {
- text.push(line.textContent ?? "");
- });
- const value = text.join("\n");
- const placeholder = "Type your expression, press '[' for columns…";
- if (value === placeholder) {
- return "";
- }
- return value;
+ },
+ helpTextHeader() {
+ return cy.findByTestId("expression-helper-popover-structure");
+ },
+ helpText() {
+ return cy.findByTestId("expression-helper");
+ },
+ nameInput() {
+ return cy.findByTestId("expression-name");
+ },
+ functionBrowser() {
+ return cy.findByTestId("expression-editor-function-browser");
+ },
+ completions() {
+ return cy.findByTestId("custom-expression-editor-suggestions");
+ },
+ completion(name: string) {
+ return CustomExpressionEditor.completions()
+ .findAllByRole("option")
+ .contains(name)
+ .first();
+ },
+ blur() {
+ cy.findByTestId("expression-editor").click("bottomRight", {
+ force: true,
});
+ return CustomExpressionEditor;
+ },
},
- completions() {
- return cy.findByTestId("custom-expression-editor-suggestions");
- },
- completion(name: string) {
- return CustomExpressionEditor.completions()
- .findAllByRole("option")
- .contains(name)
- .first();
- },
- acceptCompletion(key: "enter" | "tab" = "enter") {
- CustomExpressionEditor.completions().should("be.visible");
-
- // Avoid flakiness with CodeMirror not accepting the suggestion immediately
- cy.wait(300);
- CustomExpressionEditor.type(`{${key}}`, { focus: false });
- },
- selectCompletion(name: string) {
- CustomExpressionEditor.completions().should("be.visible");
-
- // Avoid flakiness with CodeMirror not accepting the suggestion immediately
- cy.wait(300);
- CustomExpressionEditor.completion(name).click();
- },
- helpTextHeader() {
- return cy.findByTestId("expression-helper-popover-structure");
- },
- helpText() {
- return cy.findByTestId("expression-helper");
- },
- paste(content: string) {
- CustomExpressionEditor.textbox().then((el) => {
- const clipboardData = new DataTransfer();
- clipboardData.setData("text/plain", content);
-
- const pasteEvent = new ClipboardEvent("paste", {
- bubbles: true,
- cancelable: true,
- clipboardData,
- });
-
- el[0].dispatchEvent(pasteEvent);
- });
- },
- nameInput() {
- return cy.findByTestId("expression-name");
- },
- functionBrowser() {
- return cy.findByTestId("expression-editor-function-browser");
- },
-};
+);
diff --git a/e2e/support/helpers/e2e-native-editor-helpers.ts b/e2e/support/helpers/e2e-native-editor-helpers.ts
index d55a4cc6f3a..33b4458f994 100644
--- a/e2e/support/helpers/e2e-native-editor-helpers.ts
+++ b/e2e/support/helpers/e2e-native-editor-helpers.ts
@@ -1,42 +1,6 @@
+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";
import { popover } from "./e2e-ui-elements-helpers";
-function nativeEditor() {
- cy.findAllByTestId("loading-indicator").should("not.exist");
- return cy.get("[data-testid=native-query-editor] .cm-content");
-}
-
-function focusNativeEditor() {
- nativeEditor().should("be.visible").click();
-
- nativeEditor().get(".cm-editor").should("have.class", "cm-focused");
-
- return nativeEditor();
-}
-
-function blurNativeEditor() {
- nativeEditor().get(".cm-editor").blur();
-}
-
-function nativeEditorCompletions() {
- return cy.get(".cm-tooltip-autocomplete").should("be.visible");
-}
-
-function nativeEditorCompletion(label: string) {
- return cy.get(".cm-completionLabel").contains(label).parent();
-}
-
-function nativeEditorSelectAll() {
- const isMac = Cypress.platform === "darwin";
- const metaKey = isMac ? "Meta" : "Control";
- focusNativeEditor().realPress([metaKey, "A"]);
- cy.get(".cm-selectionBackground").should("exist");
-}
-
-function clearNativeEditor() {
- nativeEditorSelectAll();
- cy.realPress(["Backspace"]);
-}
-
export function nativeEditorDataSource() {
return cy.findAllByTestId("gui-builder-data").first();
}
@@ -46,139 +10,7 @@ export function selectNativeEditorDataSource(name: string) {
popover().findByText(name).click();
}
-type TypeOptions = {
- delay?: number;
- focus?: boolean;
-};
-
-function nativeEditorType(
- text: string,
- { delay = 10, focus = true }: TypeOptions = {},
-) {
- if (focus) {
- focusNativeEditor();
- }
-
- const isMac = Cypress.platform === "darwin";
- const metaKey = isMac ? "Meta" : "Control";
-
- const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);
-
- function type(text: string) {
- cy.realType(text, {
- pressDelay: delay,
- });
- }
-
- // HACK: realType does not accept a lot of common escape sequences,
- // so we implement them manually here for the native editor.
- parts.forEach((part) => {
- switch (part.toLowerCase()) {
- case "":
- return;
-
- case "{clear}":
- return clearNativeEditor();
-
- case "{selectall}":
- return nativeEditorSelectAll();
-
- case "{leftarrow}":
- return cy.realPress(["ArrowLeft"]);
-
- case "{rightarrow}":
- return cy.realPress(["ArrowRight"]);
-
- case "{downarrow}":
- return cy.realPress(["ArrowDown"]);
-
- case "{uparrow}":
- return cy.realPress(["ArrowUp"]);
-
- case "{enter}":
- return cy.realPress(["Enter"]);
-
- case "{home}":
- case "{movetostart}":
- return cy.realPress(["Home"]);
-
- case "{end}":
- case "{movetoend}":
- return cy.realPress(["End"]);
-
- case "{backspace}":
- return cy.realPress(["Backspace"]);
-
- case "{tab}":
- return cy.realPress(["Tab"]);
-
- case "{nextcompletion}":
- cy.wait(50);
- return cy.realPress([metaKey, "j"]);
-
- case "{prevcompletion}":
- cy.wait(50);
- return cy.realPress([metaKey, "k"]);
-
- case "{{}":
- return cy.realType("{");
- }
-
- if (part.startsWith("{") && part.endsWith("}")) {
- // unknown escape sequence, let's try typing it
- type("{");
- type(part.slice(1));
- return;
- }
-
- type(part);
- });
-
- return nativeEditor();
-}
-
-export const NativeEditor = {
- get: nativeEditor,
- type(text: string, options: TypeOptions = {}) {
- nativeEditorType(text, options);
- return NativeEditor;
- },
- focus() {
- focusNativeEditor();
- return NativeEditor;
- },
- blur() {
- blurNativeEditor();
- return NativeEditor;
- },
- selectAll() {
- nativeEditorSelectAll();
- return NativeEditor;
- },
- clear() {
- clearNativeEditor();
- return NativeEditor;
- },
- textbox() {
- return NativeEditor.get().get("[role='textbox']");
- },
- value() {
- // Get the multiline text content of the editor
- return NativeEditor.textbox()
- .get(".cm-line")
- .then((lines) => {
- const text: string[] = [];
- lines.each((_, line) => {
- text.push(line.textContent ?? "");
- });
- const value = text.join("\n");
- const placeholder = "SELECT * FROM TABLE_NAME";
- if (value === placeholder) {
- return "";
- }
- return value;
- });
- },
- completions: nativeEditorCompletions,
- completion: nativeEditorCompletion,
-};
+export const NativeEditor = codeMirrorHelpers("native-query-editor", {
+ dataSource: nativeEditorDataSource,
+ selectDataSource: selectNativeEditorDataSource,
+});
diff --git a/e2e/support/helpers/e2e-python-helpers.ts b/e2e/support/helpers/e2e-python-helpers.ts
new file mode 100644
index 00000000000..6b073adb074
--- /dev/null
+++ b/e2e/support/helpers/e2e-python-helpers.ts
@@ -0,0 +1,3 @@
+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";
+
+export const PythonEditor = codeMirrorHelpers("python-editor", {});
diff --git a/e2e/support/helpers/index.ts b/e2e/support/helpers/index.ts
index dad0c1cd59b..36e00b8db54 100644
--- a/e2e/support/helpers/index.ts
+++ b/e2e/support/helpers/index.ts
@@ -36,6 +36,7 @@ export * from "./e2e-native-editor-helpers";
export * from "./e2e-notebook-helpers";
export * from "./e2e-notification-helpers";
export * from "./e2e-permissions-helpers";
+export * from "./e2e-python-helpers";
export * from "./e2e-qa-databases-helpers";
export * from "./e2e-relative-date-picker-helpers";
export * from "./e2e-request-helpers";
diff --git a/e2e/test/scenarios/admin/transforms.cy.spec.ts b/e2e/test/scenarios/admin/transforms.cy.spec.ts
index f98372f402e..d680d7451a7 100644
--- a/e2e/test/scenarios/admin/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/admin/transforms.cy.spec.ts
@@ -1,10 +1,13 @@
const { H } = cy;
+import dedent from "ts-dedent";
+
import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
import type {
CardType,
ListTransformRunsResponse,
+ PythonTransformTableAliases,
TransformTagId,
} from "metabase-types/api";
@@ -187,6 +190,123 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
testSnippets();
});
+ it(
+ "should be possible to create and run a Python transform",
+ { tags: ["@transforms-python"] },
+ () => {
+ setPythonRunnerSettings();
+ cy.log("create a new transform");
+ visitTransformListPage();
+ getTransformListPage().button("Create a transform").click();
+ H.popover().findByText("Python script").click();
+
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_create",
+ event_detail: "python",
+ triggered_from: "transform-page-create-menu",
+ });
+
+ cy.findByTestId("python-data-picker")
+ .findByText("Select a database")
+ .click();
+
+ cy.log("Unsupported databases should be disabled");
+ H.popover()
+ .findByRole("option", { name: "Sample Database" })
+ .should("have.attr", "aria-disabled", "true");
+
+ cy.log("Select database");
+ H.popover().findByText(DB_NAME).click();
+
+ getPythonDataPicker().button("Select a table…").click();
+ H.entityPickerModal().findByText("Animals").click();
+
+ getPythonDataPicker().within(() => {
+ cy.findByText("Writable Postgres12 / Schema A").should("be.visible");
+ cy.findByText("Animals").should("be.visible");
+ cy.findByPlaceholderText("Enter alias").should(
+ "have.value",
+ "animals",
+ );
+ cy.findByPlaceholderText("Enter alias")
+ .clear()
+ .type("foo bar")
+ .blur();
+ cy.findByPlaceholderText("Enter alias").should(
+ "have.value",
+ "foo_bar",
+ );
+ });
+
+ H.PythonEditor.value()
+ .should("contain", "def transform(foo_bar):")
+ .should(
+ "contain",
+ 'foo_bar: DataFrame containing the data from the "Writable Postgres12.Schema A.Animals" table',
+ );
+
+ getPythonDataPicker().within(() => {
+ cy.findByText("Add a table").click();
+ cy.button("Select a table…").click();
+ });
+
+ H.entityPickerModal().within(() => {
+ cy.log("Selecting the same table should not be possible");
+ cy.findByText("Animals")
+ .parent()
+ .parent()
+ .parent()
+ .should("have.attr", "data-disabled", "true");
+
+ cy.findByText("Schema B").click();
+ cy.findByText("Animals").click();
+ });
+
+ getPythonDataPicker().within(() => {
+ cy.icon("refresh").click();
+ cy.findAllByPlaceholderText("Enter alias")
+ .first()
+ .should("have.value", "animals_1")
+ .clear()
+ .type("foo bar")
+ .blur()
+ .should("have.value", "foo_bar");
+ cy.findAllByPlaceholderText("Enter alias")
+ .eq(1)
+ .should("have.value", "animals")
+ .clear()
+ .type("foo bar")
+ .blur()
+ .should("have.value", "foo_bar_1");
+ });
+
+ getQueryEditor().button("Save").click();
+
+ H.modal().within(() => {
+ cy.findByLabelText("Name").clear().type("Python transform");
+ cy.findByLabelText("Table name").clear().type("python_transform");
+ cy.button("Save").click();
+ });
+
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_created",
+ });
+
+ cy.log("run the transform and make sure its table can be queried");
+ runTransformAndWaitForSuccess();
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_trigger_manual_run",
+ triggered_from: "transform-page",
+ });
+
+ getRunSection().should("contain", "Executing Python transform");
+
+ getTableLink().click();
+ H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
+ H.assertQueryBuilderRowCount(1);
+ },
+ );
+
it("should be able to create and run a transform from a question or a model", () => {
function testCardSource({
type,
@@ -1169,6 +1289,38 @@ LIMIT
H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
H.assertQueryBuilderRowCount(1);
});
+
+ it("should be able to update a Python query", () => {
+ setPythonRunnerSettings();
+ cy.log("create a new transform");
+ H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
+ (id) => {
+ createPythonTransform({
+ body: dedent`
+ import pandas as pd
+
+ def transform(foo):
+ return pd.DataFrame([{"foo": 42 }])
+ `,
+ sourceTables: { foo: id },
+ visitTransform: true,
+ });
+ },
+ );
+
+ cy.log("update the query");
+ getTransformPage().findByRole("link", { name: "Edit script" }).click();
+ H.PythonEditor.type("{backspace}{backspace}{backspace} + 10 }])");
+
+ getQueryEditor().button("Save changes").click();
+ cy.wait("@updateTransform");
+
+ cy.log("run the transform and make sure the query has changed");
+ runTransformAndWaitForSuccess();
+ getTableLink().click();
+ H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
+ H.assertQueryBuilderRowCount(1);
+ });
});
describe("runs", () => {
@@ -1395,6 +1547,136 @@ LIMIT
H.main().findByText("Dependencies").should("not.exist");
});
});
+
+ describe("python > common library", () => {
+ it(
+ "should be possible to edit and save the common library",
+ { tags: ["@transforms-python"] },
+ () => {
+ visitCommonLibrary();
+
+ cy.log("updating the library should be possible");
+ H.PythonEditor.clear().type(
+ dedent`
+ def useful_calculation(a, b):
+ return a + b
+ `,
+ );
+ getLibraryEditorHeader().findByText("Save").click();
+
+ cy.log("the contents should be saved properly");
+ visitCommonLibrary();
+ H.PythonEditor.value().should(
+ "eq",
+ dedent`
+ def useful_calculation(a, b):
+ return a + b
+ `,
+ );
+
+ cy.log("reverting the changes should be possible");
+ H.PythonEditor.clear().type("# oops");
+ getLibraryEditorHeader().findByText("Revert").click();
+ H.PythonEditor.value().should(
+ "eq",
+ dedent`
+ def useful_calculation(a, b):
+ return a + b
+ `,
+ );
+ },
+ );
+
+ it(
+ "should be possible to use the common library",
+ { tags: ["@transforms-python"] },
+ () => {
+ setPythonRunnerSettings();
+ createPythonLibrary(
+ "common.py",
+ dedent`
+ def useful_calculation(a, b):
+ return a + b
+ `,
+ );
+
+ visitTransformListPage();
+ getTransformListPage().button("Create a transform").click();
+ H.popover().findByText("Python script").click();
+
+ H.PythonEditor.clear().type(
+ dedent`
+ import pandas as pd
+
+ def transform():
+ return pd.DataFrame([{"foo": common.useful_calculation(1, 2)}])
+ `,
+ );
+
+ getQueryEditor().findByText("Import common library").click();
+ H.PythonEditor.value().should("contain", "import common");
+
+ cy.findByTestId("python-data-picker")
+ .findByText("Select a database")
+ .click();
+
+ H.popover().findByText(DB_NAME).click();
+
+ cy.findByTestId("python-data-picker")
+ .findByText("Select a table…")
+ .click();
+
+ H.entityPickerModal().within(() => {
+ cy.findByText("Schema a").click();
+ cy.findByText("Animals").click();
+ });
+
+ getQueryEditor().button("Save").click();
+
+ H.modal().within(() => {
+ cy.findByLabelText("Name").clear().type("Python transform");
+ cy.findByLabelText("Table name").clear().type("python_transform");
+ cy.button("Save").click();
+ });
+
+ runTransformAndWaitForSuccess();
+ getTableLink().click();
+ H.queryBuilderHeader()
+ .findByText("Python Transform")
+ .should("be.visible");
+ H.assertQueryBuilderRowCount(1);
+ cy.findByTestId("scalar-value").should("have.text", "3");
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_created",
+ });
+
+ cy.log("update the common library and run the transform again");
+ cy.go("back");
+ createPythonLibrary(
+ "common.py",
+ dedent`
+ def useful_calculation(a, b):
+ return a + b + 40
+ `,
+ );
+ runTransformAndWaitForSuccess();
+ getTableLink().click();
+ H.queryBuilderHeader()
+ .findByText("Python Transform")
+ .should("be.visible");
+ H.assertQueryBuilderRowCount(1);
+ cy.findByTestId("scalar-value").should("have.text", "43");
+ },
+ );
+
+ function visitCommonLibrary(path = "common.py") {
+ cy.visit(`/admin/transforms/library/${path}`);
+ }
+
+ function getLibraryEditorHeader() {
+ return cy.findByTestId("library-editor-header");
+ }
+ });
});
describe("scenarios > admin > transforms > databases without :schemas", () => {
@@ -2476,6 +2758,41 @@ function createSqlTransform({
{ wrapId: true, visitTransform },
);
}
+function createPythonTransform({
+ body,
+ sourceTables,
+ targetTable = TARGET_TABLE,
+ targetSchema = TARGET_SCHEMA,
+ tagIds,
+ visitTransform,
+}: {
+ body: string;
+ sourceTables: PythonTransformTableAliases;
+ targetTable?: string;
+ targetSchema?: string;
+ tagIds?: TransformTagId[];
+ visitTransform?: boolean;
+}) {
+ H.createTransform(
+ {
+ name: "Python transform",
+ source: {
+ type: "python",
+ "source-database": WRITABLE_DB_ID,
+ "source-tables": sourceTables,
+ body,
+ },
+ target: {
+ type: "table",
+ database: WRITABLE_DB_ID,
+ name: targetTable,
+ schema: targetSchema,
+ },
+ tag_ids: tagIds,
+ },
+ { wrapId: true, visitTransform },
+ );
+}
function visitTableQuestion({
targetTable = TARGET_TABLE,
@@ -2531,3 +2848,35 @@ function assertOptionNotSelected(name: string) {
function editorSidebar() {
return cy.findByTestId("editor-sidebar");
}
+
+function getPythonDataPicker() {
+ return cy.findByTestId("python-data-picker");
+}
+
+function createPythonLibrary(path: string, source: string) {
+ cy.request("PUT", `/api/ee/transforms-python/library/${path}`, {
+ source,
+ });
+}
+
+function setPythonRunnerSettings() {
+ H.updateEnterpriseSetting("python-runner-url", "http://localhost:5001");
+ H.updateEnterpriseSetting("python-runner-api-token", "dev-token-12345");
+ H.updateEnterpriseSetting(
+ "python-storage-s-3-endpoint",
+ "http://localhost:4566",
+ );
+ H.updateEnterpriseSetting("python-storage-s-3-region", "us-east-1");
+ H.updateEnterpriseSetting(
+ "python-storage-s-3-bucket",
+ "metabase-python-runner",
+ );
+ H.updateEnterpriseSetting("python-storage-s-3-prefix", "test-prefix");
+ H.updateEnterpriseSetting("python-storage-s-3-access-key", "test");
+ H.updateEnterpriseSetting("python-storage-s-3-secret-key", "test");
+ H.updateEnterpriseSetting(
+ "python-storage-s-3-container-endpoint",
+ "http://localstack:4566",
+ );
+ H.updateEnterpriseSetting("python-storage-s-3-path-style-access", true);
+}
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
index e9522faf3b1..28e174d6cf3 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
@@ -1,7 +1,9 @@
(ns metabase-enterprise.transforms.api
(:require
+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms.api.transform-job]
[metabase-enterprise.transforms.api.transform-tag]
+ [metabase-enterprise.transforms.canceling :as transforms.canceling]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.models.transform :as transform.model]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +31,21 @@
(set! *warn-on-reflection* true)
(mr/def ::transform-source
- [:map
- [:type [:= "query"]]
- [:query [:map [:database :int]]]])
+ [:multi {:dispatch (comp keyword :type)}
+ [:query
+ [:map
+ [:type [:= "query"]]
+ [:query [:map [:database :int]]]]]
+ [:python
+ [:map {:closed true}
+ [:source-database {:optional true} :int]
+ [:source-tables [:map-of :string :int]]
+ [:type [:= "python"]]
+ [:body :string]]]])
(mr/def ::transform-target
[:map
+ [:database {:optional true} :int]
[:type [:enum "table"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:name :string]])
@@ -55,14 +66,10 @@
:schema "transforms"
:name "gadget_products"}}])
-(defn- source-database-id
- [transform]
- (-> transform :source :query :database))
-
(defn- check-database-feature
[transform]
- (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
- (deferred-tru "The source database cannot be found."))
+ (let [database (api/check-400 (t2/select-one :model/Database (transforms.util/target-database-id transform))
+ (deferred-tru "The target database cannot be found."))
feature (transforms.util/required-database-feature transform)]
(api/check-400 (not (:is_sample database))
(deferred-tru "Cannot run transforms on the sample database."))
@@ -73,6 +80,11 @@
(api/check-400 (not (transforms.util/db-routing-enabled? database))
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
+(defn- check-feature-enabled!
+ [transform]
+ (api/check (transforms.util/check-feature-enabled transform)
+ [402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
+
(api.macros/defendpoint :get "/"
"Get a list of transforms."
[_route-params
@@ -95,6 +107,7 @@
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
(api/check-superuser)
(check-database-feature body)
+ (check-feature-enabled! body)
(api/check (not (transforms.util/target-table-exists? body))
403
(deferred-tru "A table with that name already exists."))
@@ -115,8 +128,7 @@
(log/info "get transform" id)
(api/check-superuser)
(let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))
- database-id (source-database-id transform)
- target-table (transforms.util/target-table database-id target :active true)]
+ target-table (transforms.util/target-table (transforms.util/target-database-id transform) target :active true)]
(-> transform
(t2/hydrate :last_run :transform_tag_ids)
(u/update-some :last_run transforms.util/localize-run-timestamps)
@@ -173,10 +185,13 @@
(let [old (t2/select-one :model/Transform id)
new (merge old body)
target-fields #(-> % :target (select-keys [:schema :name]))]
+ ;; we must validate on a full transform object
+ (check-feature-enabled! new)
(check-database-feature new)
- (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
- (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
- {:status-code 400})))
+ (when (transforms.util/query-transform? old)
+ (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+ (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+ {:status-code 400}))))
(api/check (not (and (not= (target-fields old) (target-fields new))
(transforms.util/target-table-exists? new)))
403
@@ -212,8 +227,11 @@
[:id ms/PositiveInt]]]
(log/info "canceling transform " id)
(api/check-superuser)
- (let [run (api/check-404 (transform-run/running-run-for-transform-id id))]
- (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+ (let [transform (api/check-404 (t2/select-one :model/Transform id))
+ run (api/check-404 (transform-run/running-run-for-transform-id id))]
+ (transform-run-cancelation/mark-cancel-started-run! (:id run))
+ (when (transforms.util/python-transform? transform)
+ (transforms.canceling/cancel-run! (:id run))))
nil)
(api.macros/defendpoint :post "/:id/run"
@@ -223,10 +241,15 @@
(log/info "run transform" id)
(api/check-superuser)
(let [transform (api/check-404 (t2/select-one :model/Transform id))
+ _ (check-feature-enabled! transform)
start-promise (promise)]
- (u.jvm/in-virtual-thread*
- (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
- :run-method :manual}))
+ (if (transforms.util/python-transform? transform)
+ (u.jvm/in-virtual-thread*
+ (transforms-python.execute/execute-python-transform! transform {:start-promise start-promise
+ :run-method :manual}))
+ (u.jvm/in-virtual-thread*
+ (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+ :run-method :manual})))
(when (instance? Throwable @start-promise)
(throw @start-promise))
(let [result @start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/canceling.clj b/enterprise/backend/src/metabase_enterprise/transforms/canceling.clj
index 0c62714a8d6..f4a2612ccbc 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/canceling.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/canceling.clj
@@ -49,7 +49,9 @@
(chan-signal-cancel! run-id)
(transform-run/timeout-run! run-id)))
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+ "Cancel a run with id."
+ [run-id]
(when (chan-signal-cancel! run-id)
(transform-run/cancel-run! run-id)))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
index bc1b82fdca0..bfc527b5349 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.transforms.execute
(:require
+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
@@ -54,7 +55,8 @@
(transforms.util/run-cancelable-transform!
run-id driver transform-details
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
- (transforms.util/sync-target! target database run-id)))
+ (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+ (transforms.util/sync-target! target database run-id))))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj
new file mode 100644
index 00000000000..dbc32f09081
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj
@@ -0,0 +1,193 @@
+(ns metabase-enterprise.transforms.instrumentation
+ (:require
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+ [:enum :export :computation :import])
+
+(def ^:private label-to-stage
+ {:dwh-to-file :export
+ :file-to-s3 :export
+ :file-to-dwh :import
+ :python-execution :computation
+ :mbql-query :computation
+ :table-sync :import})
+
+(mr/def ::stage-label
+ [:enum
+ :dwh-to-file
+ :file-to-s3
+ :file-to-dwh
+
+ :python-execution
+ :mbql-query
+ :table-sync])
+
+(mu/defn record-stage-start!
+ "Record the start of a transform stage."
+ [job-run-id :- pos-int?
+ stage-label :- ::stage-label]
+ (let [stage-type (label-to-stage stage-label)]
+ (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+ (prometheus/inc! :metabase-transforms/stage-started
+ {:job-run-id (str job-run-id)
+ :stage-type (name stage-type)
+ :stage-label (name stage-label)})))
+
+(mu/defn record-stage-completion!
+ "Record the successful completion of a transform stage."
+ [job-run-id :- pos-int?
+ stage-label :- ::stage-label
+ duration-ms :- int?]
+ (let [stage-type (label-to-stage stage-label)]
+ (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+ (let [labels {:job-run-id (str job-run-id)
+ :stage-type (name stage-type)
+ :stage-label (name stage-label)}]
+ (prometheus/inc! :metabase-transforms/stage-completed labels)
+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(mu/defn record-stage-failure!
+ "Record the failure of a transform stage."
+ [job-run-id :- pos-int?
+ stage-label :- ::stage-label
+ duration-ms :- int?]
+ (let [stage-type (label-to-stage stage-label)]
+ (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))
+ (let [labels {:job-run-id (str job-run-id)
+ :stage-type (name stage-type)
+ :stage-label (name stage-label)}]
+ (prometheus/inc! :metabase-transforms/stage-failed labels)
+ (when duration-ms
+ (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))))
+
+(defn with-timing
+ "Generic timing function that executes body while recording metrics.
+
+ Takes a map of optional functions:
+ - :start-fn - function to call when starting
+ - :success-fn - function to call on success (receives duration-ms as last arg)
+ - :error-fn - function to call on error (receives duration-ms as last arg)
+
+ Usage:
+ (with-timing {:start-fn #(record-stage-start! %1 %3)
+ :success-fn #(record-stage-completion! %1 %2 %3)
+ :error-fn #(record-stage-failure! %1 %2 %3)}
+ [job-run-id stage-label]
+ #(do-work)) "
+ [callbacks args body-fn]
+ (let [start-time (u/start-timer)]
+ (when-let [start-fn (:start-fn callbacks)]
+ (apply start-fn args))
+ (try
+ (let [result (body-fn)]
+ (when-let [success-fn (:success-fn callbacks)]
+ (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+ result)
+ (catch Throwable t
+ (when-let [error-fn (:error-fn callbacks)]
+ (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+ (throw t)))))
+
+(defmacro with-stage-timing
+ "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+ Usage:
+ (with-stage-timing [job-run-id :dwh-to-file]
+ ;; stage implementation
+ result)"
+ [args & body]
+ `(with-timing {:start-fn record-stage-start!
+ :success-fn record-stage-completion!
+ :error-fn record-stage-failure!}
+ ~args
+ (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+ "Record metrics about data transfer (size and rows)."
+ [job-run-id :- pos-int?
+ stage-label :- ::stage-label
+ bytes :- [:maybe int?]
+ rows :- [:maybe int?]]
+ (if (and (nil? bytes) (nil? rows))
+ (log/warnf "Data transfer recorded: run-id=%d stage=%s but no bytes or rows provided"
+ job-run-id
+ (name stage-label))
+ (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+ job-run-id
+ (name stage-label)
+ (if bytes (str " bytes=" bytes) "")
+ (if rows (str " rows=" rows) "")))
+ (let [labels {:job-run-id (str job-run-id)
+ :stage-label (name stage-label)}]
+ (when bytes
+ (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+ (when rows
+ (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+ "Record the start of a transform job run."
+ [job-id run-method]
+ (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+ (prometheus/inc! :metabase-transforms/job-runs-total
+ {:job-id (str job-id)
+ :run-method (name run-method)}))
+
+(defn record-job-completion!
+ "Record the successful completion of a transform job run."
+ [job-id run-method duration-ms]
+ (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+ (let [labels {:job-id (str job-id)
+ :run-method (name run-method)}]
+ (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+ (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+ "Record the failure of a transform job run."
+ [job-id run-method duration-ms]
+ (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+ (let [labels {:job-id (str job-id)
+ :run-method (name run-method)}]
+ (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+ (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defmacro with-job-timing
+ "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+ Usage:
+ (with-job-timing [job-id run-method]
+ ;; job implementation
+ result)"
+ [[job-id run-method] & body]
+ `(with-timing {:start-fn record-job-start!
+ :success-fn record-job-completion!
+ :error-fn record-job-failure!}
+ [~job-id ~run-method]
+ (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+ "Record metrics about Python API calls."
+ [job-run-id :- pos-int?
+ duration-ms :- int?
+ status :- [:enum :success :error :timeout]]
+ (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+ (let [labels {:job-run-id (str job-run-id)}]
+ (prometheus/inc! :metabase-transforms/python-api-calls-total
+ (assoc labels :status (name status)))
+ (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+ "Execute body while timing a Python API call."
+ [[job-run-id] & body]
+ `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+ (record-python-api-call! job-run-id# duration-ms# :success))
+ :error-fn (fn [job-run-id# duration-ms#]
+ (record-python-api-call! job-run-id# duration-ms# :error))}
+ [~job-run-id]
+ (^:once fn* [] ~@body)))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
index 68ab07b40af..7740701daad 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
@@ -5,11 +5,14 @@
[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
[clojurewerkz.quartzite.triggers :as triggers]
[flatland.ordered.set :as ordered-set]
+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms.execute :as transforms.execute]
+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.models.job-run :as transforms.job-run]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.ordering :as transforms.ordering]
[metabase-enterprise.transforms.settings :as transforms.settings]
+ [metabase-enterprise.transforms.util :as transforms.util]
[metabase.task.core :as task]
[metabase.util.log :as log]
[toucan2.core :as t2]))
@@ -64,18 +67,22 @@
(map transforms-by-id complete)))))
(defn- run-transform! [run-id run-method {transform-id :id :as transform}]
- (when
- (transform-run/running-run-for-transform-id transform-id)
- (log/warn "Transform" (pr-str transform-id) "already running, waiting")
- (loop []
- (Thread/sleep 2000)
+ (if-not (transforms.util/check-feature-enabled transform)
+ (log/warnf "Skip running transform %d due to lacking premium features" transform-id)
+ (do
(when (transform-run/running-run-for-transform-id transform-id)
- (recur))))
-
- (log/info "Executing job transform" (pr-str transform-id))
- (transforms.execute/run-mbql-transform! transform {:run-method run-method})
-
- (transforms.job-run/add-run-activity! run-id))
+ (log/warn "Transform" (pr-str transform-id) "already running, waiting")
+ (loop []
+ (Thread/sleep 2000)
+ (when (transform-run/running-run-for-transform-id transform-id)
+ (recur))))
+ (log/info "Executing job transform" (pr-str transform-id))
+ (if (transforms.util/python-transform? transform)
+ (transforms.instrumentation/with-stage-timing [run-id :python-execution]
+ (transforms-python.execute/execute-python-transform! transform {:run-method run-method}))
+ (transforms.instrumentation/with-stage-timing [run-id :mbql-query]
+ (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+ (transforms.job-run/add-run-activity! run-id))))
(defn run-transforms!
"Run a series of transforms and their dependencies.
@@ -113,12 +120,13 @@
(let [transforms (job-transform-ids job-id)]
(log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
(let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
- (try
- (run-transforms! run-id transforms opts)
- (transforms.job-run/succeed-started-run! run-id)
- (catch Throwable t
- (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
- (throw t)))))))
+ (transforms.instrumentation/with-job-timing [job-id run-method]
+ (try
+ (run-transforms! run-id transforms opts)
+ (transforms.job-run/succeed-started-run! run-id)
+ (catch Throwable t
+ (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+ (throw t))))))))
(def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
index c7bf77c2f2b..ae184e38de3 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
@@ -16,23 +16,29 @@
(set! *warn-on-reflection* true)
(defn- transform-deps [transform]
- (let [query (-> (get-in transform [:source :query])
- transforms.util/massage-sql-query
- qp.preprocess/preprocess
- ;; legacy usage -- don't do things like this going forward
- #_{:clj-kondo/ignore [:discouraged-var]}
- lib/->legacy-MBQL)]
- (case (:type query)
- :native (driver/native-query-deps (-> (qp.store/metadata-provider)
- lib.metadata/database
- :engine)
- (get-in query [:native :query]))
- :query (into #{}
- (keep #(clojure.core.match/match %
- [:source-table source] (when (int? source)
- {:table source})
- _ nil))
- (tree-seq coll? seq query)))))
+ (if (transforms.util/python-transform? transform)
+ ;; Python transforms have dependencies via source-tables mapping
+ (into #{}
+ (map #(hash-map :table %))
+ (vals (get-in transform [:source :source-tables])))
+ ;; Query transforms have dependencies via SQL query analysis
+ (let [query (-> (get-in transform [:source :query])
+ transforms.util/massage-sql-query
+ qp.preprocess/preprocess
+ ;; legacy usage -- don't do things like this going forward
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ lib/->legacy-MBQL)]
+ (case (:type query)
+ :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+ lib.metadata/database
+ :engine)
+ (get-in query [:native :query]))
+ :query (into #{}
+ (keep #(clojure.core.match/match %
+ [:source-table source] (when (int? source)
+ {:table source})
+ _ nil))
+ (tree-seq coll? seq query))))))
(defn- dependency-map [transforms]
(into {}
@@ -56,12 +62,15 @@
The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
- dependency is ignored."
+ dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
[transforms]
- (let [transforms-by-db (->> transforms
+ (let [;; Group all transforms by their database
+ transforms-by-db (->> transforms
(map (fn [transform]
- {(get-in transform [:source :query :database]) [transform]}))
+ (let [db-id (transforms.util/target-database-id transform)]
+ {db-id [transform]})))
(apply merge-with into))
+
transform-ids (into #{} (map :id) transforms)
{:keys [output-tables dependencies]} (->> transforms-by-db
(map (fn [[db-id db-transforms]]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index b22684bd615..ccd5a47bdef 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -7,12 +7,16 @@
[metabase-enterprise.transforms.models.transform-run :as transform-run]
[metabase-enterprise.transforms.settings :as transforms.settings]
[metabase.driver :as driver]
+ [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.premium-features.core :as premium-features]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.sync.core :as sync]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[toucan2.core :as t2])
(:import
(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -27,6 +31,24 @@
(keyword schema name)
(keyword name)))
+(defn query-transform?
+ "Check if this is a query transform: native query / mbql query."
+ [transform]
+ (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+ "Check if this is a Python transform."
+ [transform]
+ (= :python (-> transform :source :type keyword)))
+
+(defn check-feature-enabled
+ "Checking whether we have proper feature flags for using a given transform."
+ [transform]
+ (if (python-transform? transform)
+ (and (premium-features/has-feature? :transforms)
+ (premium-features/has-feature? :transforms-python))
+ (premium-features/has-feature? :transforms)))
+
(defn try-start-unless-already-running
"Start a transform run, throwing an informative error if already running."
[id run-method]
@@ -76,10 +98,18 @@
(log/info "Syncing target" (pr-str target) "for transform")
(activate-table-and-mark-computed! database target)))
+;; TODO this and target-database-id can be transforms multimethods?
+(defn target-database-id
+ "Return the target database id of a transform"
+ [transform]
+ (if (python-transform? transform)
+ (-> transform :target :database)
+ (-> transform :source :query :database)))
+
(defn target-table-exists?
"Test if the target table of a transform already exists."
- [{:keys [source target] :as _transform}]
- (let [db-id (-> source :query :database)
+ [{:keys [target] :as transform}]
+ (let [db-id (target-database-id transform)
{driver :engine :as database} (t2/select-one :model/Database db-id)]
(driver/table-exists? driver database target)))
@@ -118,10 +148,10 @@
(defn delete-target-table!
"Delete the target table of a transform and sync it from the app db."
- [{:keys [id target source], :as _transform}]
+ [{:keys [id target], :as transform}]
(when target
(let [target (update target :type keyword)
- database-id (-> source :query :database)
+ database-id (target-database-id transform)
{driver :engine :as database} (t2/select-one :model/Database database-id)]
(driver/drop-transform-target! driver database target)
(log/info "Deactivating target " (pr-str target) "for transform" id)
@@ -146,8 +176,10 @@
(defn required-database-feature
"Returns the database feature necessary to execute `transform`."
[transform]
- (case (-> transform :target :type)
- "table" :transforms/table))
+ (if (python-transform? transform)
+ :transforms/python
+ (case (-> transform :target :type)
+ "table" :transforms/table)))
(defn ->instant
"Convert a temporal value `t` to an Instant in the system timezone."
@@ -178,6 +210,65 @@
(u/update-some :start_time utc-timestamp-string)
(u/update-some :end_time utc-timestamp-string)))
+(mr/def ::column-definition
+ [:map
+ [:name :string]
+ [:type ::lib.schema.common/base-type]
+ [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+ [:map
+ [:name :keyword]
+ [:columns [:sequential ::column-definition]]
+ [:primary-key {:optional true} [:sequential :string]]])
+
+(mu/defn create-table-from-schema!
+ "Create a table from a table-schema"
+ [driver :- :keyword
+ database-id :- pos-int?
+ table-schema :- ::table-definition]
+ (let [{:keys [columns] table-name :name} table-schema
+ column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+ (let [db-type (if database-type
+ [[:raw database-type]]
+ (try
+ (driver/type->database-type driver type)
+ (catch IllegalArgumentException _
+ (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)
+ (driver/type->database-type driver :type/Text))))]
+ [name db-type])))
+ columns)
+ primary-key-opts (select-keys table-schema [:primary-key])]
+ (log/infof "Creating table %s with %d columns" table-name (count columns))
+ (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn drop-table!
+ "Drop a table in the database."
+ [driver database-id table-name]
+ (log/infof "Dropping table %s" table-name)
+ (driver/drop-table! driver database-id table-name))
+
+(defn temp-table-name
+ "Generate a temporary table name with the given suffix and current timestamp in seconds.
+ If table name would exceed max table name length for the driver, fallback to using a shorter base-name."
+ [driver base-table-name suffix]
+ {:pre [(keyword? base-table-name)]}
+ (let [suffix (str "_" suffix "_" (quot (System/currentTimeMillis) 1000))
+ max-len (or (driver/table-name-length-limit driver) Integer/MAX_VALUE)
+ max-prefix-len (- max-len (count suffix))
+ table-name (name base-table-name)
+ prefix (subs table-name 0 (min (count table-name) max-prefix-len))]
+ (assert (pos? max-prefix-len))
+ (keyword (namespace base-table-name)
+ (str prefix suffix))))
+
+(defn rename-tables!
+ "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.
+ This is a simpler, composable operation that only handles renaming."
+ [driver database-id rename-map]
+ (log/infof "Renaming tables: %s" (pr-str rename-map))
+ (driver/rename-tables! driver database-id rename-map))
+
(defn db-routing-enabled?
"Returns whether or not the given database is either a router or destination database"
[db-or-id]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
new file mode 100644
index 00000000000..07a36320bf3
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -0,0 +1,328 @@
+(ns metabase-enterprise.transforms-python.execute
+ (:require
+ [clojure.core.async :as a]
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabase-enterprise.transforms-python.python-runner :as python-runner]
+ [metabase-enterprise.transforms-python.s3 :as s3]
+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+ [metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.driver :as driver]
+ [metabase.driver.util :as driver.u]
+ [metabase.util :as u]
+ [metabase.util.jvm :as u.jvm]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2])
+ (:import
+ (java.io Closeable File)
+ (java.net SocketException)
+ (java.time Duration)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:const temp-table-suffix-new
+ "Suffix used for temporary tables containing new data during swap operations."
+ "new")
+
+(def ^:const temp-table-suffix-old
+ "Suffix used for temporary tables containing old data during swap operations."
+ "old")
+
+(defn- empty-message-log
+ "Returns a new message log.
+
+ Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.
+ The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.
+
+ Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.
+
+ The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."
+ []
+ (atom {:pre-python [] ; log! outputs previous to the python execution, i.e table read progress
+ :python nil ; events json structured logs from the /logs endpoint
+ :post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target
+
+(defn- log!
+ "Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."
+ [message-log s]
+ (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
+ nil)
+
+(defn- replace-python-logs!
+ "Called when events output is received from a /logs call to copy those events into the message-log."
+ [message-log events]
+ (swap! message-log assoc :python events)
+ nil)
+
+(defn- message-log->transform-run-message
+ "Presents the content of the log as a string suitable for storage in transform_run.message."
+ [message-log]
+ (let [{:keys [pre-python python post-python]} @message-log]
+ (str/join "\n" (concat pre-python
+ (map :message python)
+ post-python))))
+
+(defn- save-log-to-transform-run-message!
+ "Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."
+ [run-id message-log]
+ (t2/update! :model/TransformRun
+ :id run-id
+ {:message (message-log->transform-run-message message-log)}))
+
+(def ^:private ^Duration python-message-loop-sleep-duration
+ (Duration/ofMillis 1000))
+
+(defn- python-message-update-loop!
+ "Block while relevant log data is replicated from the runner into the message log.
+ When new logs are received, the log data will be flushed to the transform_run.message field as a string.
+
+ The loop will exit on error or when interrupted."
+ [run-id message-log]
+ (try
+ (loop []
+ (if (.isInterrupted (Thread/currentThread))
+ (log/debug "Message update loop interrupted")
+ (do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]
+ (when (pos? sleep-ms) (Thread/sleep sleep-ms)))
+ (let [{:keys [status body]} (python-runner/get-logs run-id)]
+ (cond
+ (<= 200 status 299)
+ (let [{:keys [execution_id events]} body]
+ (if-not (= run-id execution_id)
+ (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+ (recur))
+ (do
+ (replace-python-logs! message-log events)
+ (save-log-to-transform-run-message! run-id message-log)
+ (recur))))
+ (= 404 status)
+ (do
+ (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+ (recur))
+ :else
+ (do
+ (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+ (log/debug "Exiting due to poll error")))))))
+ (catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
+ (catch InterruptedException _)
+ (catch Throwable e
+ (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
+
+(defn- open-python-message-update-future! ^Closeable [run-id message-log]
+ (let [cleanup (fn [fut]
+ (future-cancel fut)
+ (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
+ (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
+ (log/debugf "Log polling task done, run-id: %s" run-id)))
+ fut (u.jvm/in-virtual-thread*
+ (python-message-update-loop! run-id message-log))]
+ (reify Closeable
+ (close [_] (cleanup fut)))))
+
+(defn- maybe-retry-with-backoff
+ [driver thunk]
+ ;; bigquery can have eventual consistency hiccups where the table is not yet available for insertion
+ ;; immediately after creation, so we retry a few times with backoff
+ (if (= driver :bigquery-cloud-sdk)
+ (loop [attempt 0]
+ (let [ret (try
+ (thunk)
+ (catch Exception e
+ (if (and (< attempt 5)
+ (or (str/includes? (str e) "not found")
+ (str/includes? (str e) "Not found")))
+ (let [delay-ms (* 150 (Math/pow 2 attempt))]
+ (log/debugf "BigQuery operation failed (attempt %d), retrying in %dms" (inc attempt) delay-ms)
+ (Thread/sleep (long delay-ms))
+ ::recur)
+ (throw e))))]
+ (if (= ::recur ret)
+ (recur (inc attempt))
+ ret)))
+ (thunk)))
+
+(defn- create-table-and-insert-data!
+ "Create a table from metadata and insert data from source."
+ [driver db-id table-name metadata data-source]
+ (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+ :columns (mapv (fn [{:keys [name base_type #_database_type]}]
+ {:name name
+ :type (python-runner/restricted-insert-type base_type)
+ ;; :database-type database_type
+ :nullable? true})
+ (:fields metadata))}
+ data-source (assoc data-source :table-schema table-schema)]
+ (transforms.util/create-table-from-schema! driver db-id table-schema)
+
+ (->> #(driver/insert-from-source! driver db-id table-schema data-source)
+ (maybe-retry-with-backoff driver))))
+
+(defn- transfer-with-rename-tables-strategy!
+ "Transfer data using the rename-tables*! multimethod with atomicity guarantees.
+ Creates new table, then atomically renames target->old and new->target, then drops old."
+ [driver db-id table-name metadata data-source]
+ (let [source-table-name (transforms.util/temp-table-name driver table-name temp-table-suffix-new)
+ temp-table-name (transforms.util/temp-table-name driver table-name temp-table-suffix-old)]
+ (log/info "Using rename-tables strategy with atomicity guarantees")
+ (try
+
+ (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+ (transforms.util/rename-tables! driver db-id {table-name temp-table-name
+ source-table-name table-name})
+ (transforms.util/drop-table! driver db-id temp-table-name)
+
+ (catch Exception e
+ (log/error e "Failed to transfer data using rename-tables strategy")
+ (try
+ (transforms.util/drop-table! driver db-id source-table-name)
+ (catch Exception _))
+ (throw e)))))
+
+(defn- transfer-with-create-drop-rename-strategy!
+ "Transfer data using create + drop + rename to minimize time without data.
+ Creates new table, drops old table, then renames new->target."
+ [driver db-id table-name metadata data-source]
+ (let [source-table-name (transforms.util/temp-table-name driver table-name temp-table-suffix-new)]
+ (log/info "Using create-drop-rename strategy to minimize downtime")
+ (try
+
+ (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+ (transforms.util/drop-table! driver db-id table-name)
+ (driver/rename-table! driver db-id source-table-name table-name)
+
+ (catch Exception e
+ (log/error e "Failed to transfer data using create-drop-rename strategy")
+ (try
+ (transforms.util/drop-table! driver db-id source-table-name)
+ (catch Exception _))
+ (throw e)))))
+
+(defn- transfer-with-drop-create-fallback-strategy!
+ "Transfer data using drop + create fallback strategy.
+ Drops old table, then creates new table directly with target name."
+ [driver db-id table-name metadata data-source]
+ (log/info "Using drop-create fallback strategy")
+ (try
+
+ (transforms.util/drop-table! driver db-id table-name)
+ (create-table-and-insert-data! driver db-id table-name metadata data-source)
+
+ (catch Exception e
+ (log/error e "Failed to transfer data using drop-create fallback strategy")
+ (throw e))))
+
+(defn- transfer-file-to-db [driver {db-id :id :as db} {:keys [target] :as transform} metadata temp-file]
+ (let [table-name (transforms.util/qualified-table-name driver target)
+ table-exists? (transforms.util/target-table-exists? transform)
+ data-source {:type :jsonl-file
+ :file temp-file}]
+ (cond
+ (not table-exists?)
+ (do
+ (log/info "New table")
+ (create-table-and-insert-data! driver db-id table-name metadata data-source))
+
+ (driver.u/supports? driver :atomic-renames db)
+ (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
+
+ (driver.u/supports? driver :rename db)
+ (transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
+
+ :else
+ (transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
+
+(defn- start-cancellation-process!
+ "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
+ Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
+ [server-url run-id cancel-chan]
+ (a/go (when (a/<! cancel-chan)
+ (python-runner/cancel-python-code-http-call! server-url run-id))))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+ ;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
+ (with-open [log-future-ref (open-python-message-update-future! run-id message-log)
+ shared-storage-ref (s3/open-shared-storage! (:source-tables source))]
+ (let [driver (:engine db)
+ server-url (transforms-python.settings/python-runner-url)
+ _ (python-runner/copy-tables-to-s3! {:run-id run-id
+ :shared-storage @shared-storage-ref
+ :table-name->id (:source-tables source)
+ :cancel-chan cancel-chan})
+ _ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+ {:keys [status body] :as response}
+ (python-runner/execute-python-code-http-call!
+ {:server-url server-url
+ :code (:body source)
+ :run-id run-id
+ :table-name->id (:source-tables source)
+ :shared-storage @shared-storage-ref})
+ ;; TODO temporary to keep more code stable while refactoring
+ ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
+ {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+ (.close log-future-ref) ; early close to force any writes to flush
+ (when (seq events)
+ (replace-python-logs! message-log events))
+ (if (not= 200 status)
+ (throw (ex-info "Python runner call failed"
+ {:status-code 400
+ :api-status-code status
+ :body body
+ :events events
+ :transform-run-message (message-log->transform-run-message message-log)}))
+ (try
+ (let [temp-file (File/createTempFile "transform-output-" ".jsonl")]
+ (when-not (seq (:fields output-manifest))
+ (throw (ex-info "No fields in metadata"
+ {:metadata output-manifest
+ :raw-body body
+ :events events
+ :transform-run-message (message-log->transform-run-message message-log)})))
+ (try
+ (with-open [writer (io/writer temp-file)]
+ (.write writer ^String output))
+ (let [file-size (.length temp-file)]
+ (transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]
+ (transfer-file-to-db driver db transform output-manifest temp-file))
+ (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+ (finally
+ (.delete temp-file))))
+ response
+ (catch Exception e
+ (log/error e "Failed to to create resulting table")
+ (throw (ex-info "Failed to create the resulting table"
+ {:transform-run-message (message-log->transform-run-message message-log)}
+ e))))))))
+
+(defn execute-python-transform!
+ "Execute a Python transform by calling the python runner.
+
+ Blocks until the transform returns."
+ [transform {:keys [run-method start-promise]}]
+ (assert (transforms.util/python-transform? transform) "Transform must be a python transform")
+ (try
+ (let [message-log (empty-message-log)
+ {:keys [target] transform-id :id} transform
+ {driver :engine :as db} (t2/select-one :model/Database (:database target))
+ {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method)]
+ (some-> start-promise (deliver [:started run-id]))
+ (log! message-log "Executing Python transform")
+ (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+ (let [start-ms (u/start-timer)
+ transform-details {:db-id (:id db)
+ :transform-type (keyword (:type target))
+ :conn-spec (driver/connection-spec driver db)
+ :output-schema (:schema target)
+ :output-table (transforms.util/qualified-table-name driver target)}
+ run-fn (fn [cancel-chan]
+ (run-python-transform! transform db run-id cancel-chan message-log)
+ (log! message-log (format "Python execution finished successfully in %s" (Duration/ofMillis (u/since-ms start-ms))))
+ (save-log-to-transform-run-message! run-id message-log))
+ result (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn)]
+ (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+ (transforms.util/sync-target! target db run-id))
+ {:run_id run-id
+ :result result}))
+ (catch Throwable t
+ (log/error t "Error executing Python transform")
+ (throw t))))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
new file mode 100644
index 00000000000..60dc6d6e8dc
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -0,0 +1,283 @@
+(ns metabase-enterprise.transforms-python.python-runner
+ (:require
+ [clj-http.client :as http]
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [medley.core :as m]
+ [metabase-enterprise.transforms-python.s3 :as s3]
+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+ [metabase.config.core :as config]
+ [metabase.driver :as driver]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor.pipeline :as qp.pipeline]
+ ;; TODO check that querying team are ok with us accessing this directly, otherwise make another plan
+ ^{:clj-kondo/ignore [:discouraged-namespace]}
+ [metabase.query-processor.store :as qp.store]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2])
+ (:import
+ (clojure.lang PersistentQueue)
+ (java.io BufferedWriter File OutputStream OutputStreamWriter)
+ (java.nio.charset StandardCharsets)))
+
+(set! *warn-on-reflection* true)
+
+(defn- authorization-headers
+ "Returns HTTP headers with Authorization bearer token if configured.
+ Throws configuration error in production if token is not set."
+ []
+ (let [api-token (transforms-python.settings/python-runner-api-token)]
+ (if api-token
+ {"Authorization" (str "Bearer " api-token)}
+ (if config/is-prod?
+ (throw (ex-info "Python runner API token is required in production but not configured"
+ {:error-type :configuration-error}))
+ {}))))
+
+(defn- python-runner-request
+ "Helper function for making HTTP requests to the python runner service."
+ [server-url method endpoint & [request-options]]
+ (let [url (str server-url "/v1" endpoint)
+ base-options {:content-type :json
+ :accept :json
+ :throw-exceptions false
+ :as :json
+ :headers (authorization-headers)}]
+ (http/request (merge base-options request-options {:method method, :url url}))))
+
+(defn- write-jsonl-to-stream! [^OutputStream os col-names reducible-rows]
+ (let [none? (volatile! true)
+ writer (-> os
+ (OutputStreamWriter. StandardCharsets/UTF_8)
+ (BufferedWriter.))]
+
+ (run! (fn [row]
+ (when @none? (vreset! none? false))
+ (let [row-map (zipmap col-names row)]
+ (json/encode-to row-map writer {})
+ (.newLine writer)))
+ reducible-rows)
+
+ ;; Workaround for LocalStack, which doesn't support zero byte files.
+ (when @none?
+ (.write writer " "))
+
+ (doto writer
+ (.flush)
+ (.close))))
+
+(defn- execute-mbql-query
+ [driver db-id query respond cancel-chan]
+ (driver/with-driver driver
+ (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+ query {:database db-id
+ :type :native
+ :native native}]
+ (qp.store/with-metadata-provider db-id
+ (binding [qp.pipeline/*canceled-chan* cancel-chan]
+ (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
+
+(defn root-type
+ "Supported type for roundtrip/insertion"
+ [base-type]
+ (when base-type
+ (some #(when (isa? base-type %) %)
+ [:type/Number
+ :type/Date
+ :type/DateTime
+ :type/Instant
+ :type/DateTimeWithTZ
+ :type/Text
+ :type/Boolean])))
+
+(defn restricted-insert-type
+ "Type for insertion restricted to supported"
+ [base_type]
+ (if base_type
+ (let [base-type-kw (keyword "type" base_type)]
+ (if (root-type base-type-kw)
+ base-type-kw
+ :type/Text))
+ :type/Text))
+
+(defn- closest-ancestor [t pred]
+ (loop [remaining (conj PersistentQueue/EMPTY [t])]
+ (when-let [t (first remaining)]
+ (if (pred t)
+ t
+ (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+ "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+ [{:keys [base_type effective_type semantic_type]}]
+ (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+ "Generate a manifest to communicate schema and metadata information around the table."
+ [table-id cols-meta]
+ {:schema_version 1
+ :data_format "jsonl"
+ :data_version 1
+ :table_metadata {:table_id table-id}
+ :fields (mapv (fn [col-meta]
+ {:name (:name col-meta)
+ :base_type (some-> (:base_type col-meta) name)
+ :database_type (some-> (:database_type col-meta) name)
+ :root_type (some-> (root-type (:base_type col-meta)) name)
+ :semantic_type (some-> (effective-semantic-type-i-think col-meta) name)
+ :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+ :field_id (:id col-meta)})
+ cols-meta)})
+
+(defn- maybe-fixup-value [col v]
+ (cond
+ (nil? (root-type (:base_type col)))
+ ;; we're not a supported base type, so we just stringify it
+ (when v (json/encode v))
+
+ ;; the clickhouse driver returns bigdecimals for int64 values
+ (and (isa? (:base_type col) :type/Integer)
+ (or (instance? BigDecimal v)
+ (float? v)))
+ (bigint v)
+
+ :else
+ v))
+
+(defn- write-table-data-to-file! [{:keys [db-id driver table-id fields-meta temp-file cancel-chan]}]
+ (let [query {:source-table table-id}]
+ (execute-mbql-query driver db-id query
+ (fn [{cols-meta :cols} reducible-rows]
+ (with-open [os (io/output-stream temp-file)]
+ (let [filtered-col-meta (m/index-by :name fields-meta)
+ col-names (map :name cols-meta)
+ filtered-rows (eduction (map (fn [row]
+ (->>
+ (map vector col-names row)
+ (filter (fn [[n _]]
+ (contains? filtered-col-meta n)))
+ (map (fn [[n v]]
+ (maybe-fixup-value (filtered-col-meta n) v))))))
+ reducible-rows)]
+ (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))
+ cancel-chan)))
+
+(defn get-logs
+ "Return the logs of the current running python process"
+ [run-id]
+ (let [server-url (transforms-python.settings/python-runner-url)]
+ (python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))
+
+(defn execute-python-code-http-call!
+ "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+ the response from the server."
+ [{:keys [server-url code run-id table-name->id shared-storage]}]
+ (let [{:keys [objects]} shared-storage
+ {:keys [output output-manifest events]} objects
+
+ url-for-path (fn [path] (:url (get objects path)))
+ table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))
+ table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
+
+ payload {:code code
+ :library (t2/select-fn->fn :path :source :model/PythonLibrary)
+ :timeout (* 30 60) ;; 30 mins harcoded for now
+ :request_id run-id
+ :output_url (:url output)
+ :output_manifest_url (:url output-manifest)
+ :events_url (:url events)
+ :table_mapping table-name->url
+ :manifest_mapping table-name->manifest-url}
+
+ response (transforms.instrumentation/with-python-api-timing [run-id]
+ (python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
+ ;; when a 500 is returned we observe a string in the body (despite the python returning json)
+ ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
+ (update response :body (fn [string-if-error]
+ (if (string? string-if-error)
+ (try
+ (json/decode+kw string-if-error)
+ (catch Exception _
+ {:error string-if-error}))
+ string-if-error)))))
+
+;; temporary, we should not need to realize data/events files into memory longer term
+(defn read-output-objects
+ "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+ [{:keys [s3-client bucket-name objects]}]
+ (let [{:keys [output output-manifest events]} objects
+ output-content (s3/read-to-string s3-client bucket-name (:path output) nil)
+ output-manifest-content (s3/read-to-string s3-client bucket-name (:path output-manifest) "{}")
+ events-content (s3/read-to-string s3-client bucket-name (:path events))]
+ {:output output-content
+ :output-manifest (json/decode+kw output-manifest-content)
+ :events (mapv json/decode+kw (str/split-lines events-content))}))
+
+(defn cancel-python-code-http-call!
+ "Calls the /cancel endpoint of the python runner. Returns immediately."
+ [server-url run-id]
+ (python-runner-request server-url :post "/cancel" {:body (json/encode {:request_id run-id})
+ :async? true}
+ #_success #(log/debug %)
+ #_failure #(log/error %)))
+
+(defn- safe-delete
+ "Safely delete a file."
+ [^File file]
+ (try (.delete file) (catch Exception _)))
+
+(defn- fields-metadata [_driver table-id]
+ (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position]
+ :table_id table-id
+ :active true
+ ;; we are only interested in top-level objects, so filter out nested fields (parent or path)
+ :parent_id nil
+ :nfc_path nil
+ {:order-by [[:database_position :asc]]}))
+
+;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.
+(defn copy-tables-to-s3!
+ "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+ Blocks until all tables are fully written and committed to shared storage."
+ [{:keys [run-id
+ shared-storage
+ table-name->id
+ cancel-chan]}]
+ ;; TODO there's scope for some parallelism here, in particular across different databases
+ (doseq [table-id (vals table-name->id)
+ :let [{:keys [s3-client bucket-name objects]} shared-storage
+ {data-path :path} (get objects [:table table-id :data])
+ {manifest-path :path} (get objects [:table table-id :manifest])]]
+ (let [tmp-data-file (File/createTempFile data-path "")
+ tmp-meta-file (File/createTempFile manifest-path "")]
+ (try
+ (let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
+ driver (t2/select-one-fn :engine :model/Database db-id)
+ fields-meta (fields-metadata driver table-id)
+ manifest (generate-manifest table-id fields-meta)]
+
+ (transforms.instrumentation/with-stage-timing [run-id :dwh-to-file]
+ (write-table-data-to-file!
+ {:db-id db-id
+ :driver driver
+ :table-id table-id
+ :fields-meta fields-meta
+ :temp-file tmp-data-file
+ :cancel-chan cancel-chan}))
+
+ (with-open [writer (io/writer tmp-meta-file)]
+ (json/encode-to manifest writer {}))
+ (let [data-size (.length tmp-data-file)
+ meta-size (.length tmp-meta-file)]
+ (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file data-size nil)
+
+ (transforms.instrumentation/with-stage-timing [run-id :file-to-s3]
+ (s3/upload-file s3-client bucket-name data-path tmp-data-file)
+ (s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))
+
+ (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))
+ (finally
+ (safe-delete tmp-data-file)
+ (safe-delete tmp-meta-file))))))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
new file mode 100644
index 00000000000..75448593736
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj
@@ -0,0 +1,215 @@
+(ns metabase-enterprise.transforms-python.s3
+ (:require
+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase.util.log :as log])
+ (:import
+ (clojure.lang IDeref)
+ (java.io Closeable File)
+ (java.net URI)
+ (java.time Duration)
+ (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+ (software.amazon.awssdk.auth.credentials AwsCredentialsProvider DefaultCredentialsProvider)
+ (software.amazon.awssdk.core.sync RequestBody)
+ (software.amazon.awssdk.regions Region)
+ (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+ (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+ (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+ (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+;; TODO We should make these durations configurable in future.
+
+;; Longer duration for inputs than for outputs, to compensate for the duration of the code execution itself.
+(def ^:private ^Duration presigned-get-duration (Duration/ofMinutes 30))
+(def ^:private ^Duration presigned-put-duration (Duration/ofHours 5))
+
+;; We should consider namespacing these paths, for example, using the run id to simplify cleanup.
+;; Another idea is to have deterministic paths, so that it's easier to resume incomplete runs.
+(defn- working-dir-for-run
+ "The path within the target bucket within which to store all the files for this run."
+ []
+ (let [shared-prefix (some-> (transforms-python.settings/python-storage-s-3-prefix) (str "/"))]
+ (str shared-prefix "run-" (System/nanoTime) "-" (rand-int 10000))))
+
+(defmacro ^:private maybe-with-endpoint* [builder endpoint]
+ `(let [builder# ~builder]
+ (when-let [region# (transforms-python.settings/python-storage-s-3-region)]
+ (.region builder# (Region/of region#)))
+ (when ~endpoint (.endpointOverride builder# (URI/create ~endpoint)))
+ builder#))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+ (maybe-with-endpoint* builder endpoint))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+ (maybe-with-endpoint* builder endpoint))
+
+(defn- s3-configuration ^S3Configuration []
+ (-> (S3Configuration/builder)
+ (.pathStyleAccessEnabled (transforms-python.settings/python-storage-s-3-path-style-access))
+ (.build)))
+
+(defn- put-object-request ^PutObjectRequest [^String bucket-name ^String key]
+ (-> (PutObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- get-object-request ^GetObjectRequest [^String bucket-name ^String key]
+ (-> (GetObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- delete-object-request ^DeleteObjectRequest [^String bucket-name ^String key]
+ (-> (DeleteObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- maybe-with-credentials*
+ [credentials-provider]
+ (let [access-key (transforms-python.settings/python-storage-s-3-access-key)
+ secret-key (transforms-python.settings/python-storage-s-3-secret-key)]
+ (if (or access-key secret-key)
+ (if-not (and access-key secret-key)
+ (do (log/warnf "Ignoring %s because %s is not defined"
+ (if access-key "access-key" "secret-key")
+ (if (not access-key) "access-key" "secret-key"))
+ (credentials-provider (DefaultCredentialsProvider/create)))
+ (credentials-provider
+ (StaticCredentialsProvider/create
+ (AwsBasicCredentials/create access-key secret-key))))
+ (credentials-provider (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+ (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+ (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn create-s3-client
+ "Create S3 client for transferring table data and manifests."
+ ^S3Client []
+ (.build
+ (doto (S3Client/builder)
+ (maybe-with-endpoint-s3-client (transforms-python.settings/python-storage-s-3-endpoint))
+ maybe-with-credentials-s3-client
+ (.serviceConfiguration (s3-configuration)))))
+
+(defn- create-s3-presigner-for-container
+ "Create S3 presigner for container operations (presigned URLs). Uses distinct container-endpoint if relevant."
+ ^S3Presigner []
+ (let [container-endpoint (transforms-python.settings/python-storage-s-3-container-endpoint)
+ endpoint (or container-endpoint (transforms-python.settings/python-storage-s-3-endpoint))]
+ (.build
+ (doto (S3Presigner/builder)
+ (maybe-with-endpoint-s3-presigner endpoint)
+ maybe-with-credentials-s3-presigner
+ (.serviceConfiguration (s3-configuration))))))
+
+(defn- generate-presigned-get-url
+ "Generate GET URL using container presigner"
+ [^S3Presigner presigner ^String bucket-name ^String key]
+ (let [request (-> (GetObjectPresignRequest/builder)
+ (.signatureDuration presigned-get-duration)
+ (.getObjectRequest (get-object-request bucket-name key))
+ (.build))]
+ (.toString (.url (.presignGetObject presigner request)))))
+
+(defn- generate-presigned-put-url
+ "Generate PUT URL using container presigner"
+ [^S3Presigner presigner ^String bucket-name ^String key]
+ (let [request (-> (PutObjectPresignRequest/builder)
+ (.signatureDuration presigned-put-duration)
+ (.putObjectRequest (put-object-request bucket-name key))
+ (.build))]
+ (.toString (.url (.presignPutObject presigner request)))))
+
+(defn- s3-shared-storage [table-name->id]
+ (let [work-dir-name (working-dir-for-run)
+ container-presigner (create-s3-presigner-for-container)
+ bucket-name (transforms-python.settings/python-storage-s-3-bucket)
+ ref (fn [method relative-path]
+ (let [path (str work-dir-name "/" relative-path)]
+ {:path path
+ :method method
+ :url (case method
+ :put (generate-presigned-put-url container-presigner bucket-name path)
+ :get (generate-presigned-get-url container-presigner bucket-name path))}))]
+ ;; It feels dirty mixing the S3 client object with the other pure values, but it is convenient for the caller
+ {:s3-client (create-s3-client)
+ :bucket-name bucket-name
+ :objects
+ (into
+ {:output (ref :put "output.csv")
+ :output-manifest (ref :put "output-manifest.json")
+ :events (ref :put "events.jsonl")}
+ (for [[table-name id] table-name->id]
+ {[:table id :manifest] (ref :get (str "table-" (name table-name) "-" id ".manifest.json"))
+ [:table id :data] (ref :get (str "table-" (name table-name) "-" id ".jsonl"))}))}))
+
+(declare delete-many)
+
+;; TODO this feels like it's a hair's width from abstracting away s3 versus other transfer mechanisms - go all the way
+(defn open-shared-storage!
+ "Returns a map wrapped in a deref-able auto-closable, to be used in a with-open.
+
+ The map contains:
+ - :s3-client - an S3Client that can be used to upload/download files
+ - :bucket-name - the S3 bucket name
+ - :objects - a map of keys to {:path :method :url} maps, where:
+ - :path is the S3 key
+ - :method is either :get or :put, indicating whether the python-runner should fetch or upload the file.
+ - :url is the presigned URL to be used by the python-runner to fetch or upload the file.
+
+ The objects are broken into the following transform inputs:
+
+ - [:table id :manifest] - for each input table, the manifest JSON file
+ - [:table id :data] - for each input table, the data file, in format specified by the manifest.
+
+ And the following transform outputs:
+ - :output-manifest - the output manifest JSON file
+ - :output - the output data file, in format specified by the output manifest.
+ - :events - a JSONL file containing the events logged during python execution.
+
+ When the value is closed, all the relevant keys will be deleted, but this is best-effort only.
+ We rely on the bucket retention policy to ensure any stragglers are eventually deleted."
+ ^Closeable [table-name->id]
+ (let [shared-storage (s3-shared-storage table-name->id)]
+ (reify IDeref
+ (deref [_] shared-storage)
+ Closeable
+ (close [_] (delete-many (:s3-client shared-storage)
+ (:bucket-name shared-storage)
+ (map :path (vals (:objects shared-storage))))))))
+
+(defn upload-file
+ "Upload the given file to s3"
+ [^S3Client s3-client ^String bucket-name ^String key ^File file]
+ (let [^PutObjectRequest request (put-object-request bucket-name key)]
+ (.putObject s3-client request (RequestBody/fromFile file))))
+
+;; TODO optimize our ingestion to stream the data. reading a string will not work in any case for binary files.
+(defn read-to-string
+ "Get back the contents of the given key as a string."
+ ([s3-client bucket-name key] (read-to-string s3-client bucket-name key ::throw))
+ ([^S3Client s3-client ^String bucket-name ^String key not-found]
+ (try
+ (let [^GetObjectRequest request (get-object-request bucket-name key)
+ response (.getObject s3-client request)]
+ (slurp response))
+ (catch NoSuchKeyException e
+ (if (identical? ::throw not-found)
+ (throw e)
+ not-found)))))
+
+(defn delete
+ ;; TODO better error handling
+ "Delete the given key from s3, ignoring any errors for now"
+ [^S3Client s3-client ^String bucket-name ^String key]
+ (try
+ (.deleteObject s3-client (delete-object-request bucket-name key))
+ (catch Exception e
+ (log/debugf e "Error deleting s3 object %s" key)
+ ;; Ignore deletion errors - object might not exist, or we might not have permissions
+ ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
+ nil)))
+
+(defn delete-many
+ "Best effort delete the given s3 keys"
+ [^S3Client s3-client bucket-name s3-keys]
+ (run! (partial delete s3-client bucket-name) s3-keys))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj
index 2ae3fa46444..f451e84890e 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj
@@ -18,7 +18,7 @@
(setting/defsetting python-runner-api-token
(deferred-tru "API token for authorizing with the python-runner service.")
:type :string
- :visibility :internal
+ :visibility :admin
:default (when (not config/is-prod?) "dev-token-12345")
:feature :transforms-python
:doc false
@@ -52,7 +52,7 @@
(deferred-tru "S3 bucket name for storing Python execution artifacts.")
:type :string
:visibility :admin
- :default (if (not config/is-prod?) "metabase-python-runner" "metabase-cloud-python-runner-staging")
+ :default (when (not config/is-prod?) "metabase-python-runner")
:feature :transforms-python
:doc false
:export? false
@@ -64,7 +64,7 @@
:type :string
:visibility :admin
:feature :transforms-python
- :default (if (not config/is-prod?) "test-prefix" "hosting-4b9351bb22b84791")
+ :default (when (not config/is-prod?) "test-prefix")
:doc false
:export? false
:encryption :no
diff --git a/enterprise/backend/test/metabase_enterprise/api/session_test.clj b/enterprise/backend/test/metabase_enterprise/api/session_test.clj
index c7df48de095..e33028e8e31 100644
--- a/enterprise/backend/test/metabase_enterprise/api/session_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/api/session_test.clj
@@ -47,6 +47,7 @@
:sso-ldap
:sso-saml
:transforms
+ :transforms-python
:upload-management
:whitelabel
:collection-cleanup
@@ -91,6 +92,7 @@
:sso_saml true
:table_data_editing false
:transforms true
+ :transforms-python true
:upload_management true
:whitelabel true
:collection_cleanup true
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index c626c008be4..225a25ea430 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -8,7 +8,7 @@
[metabase-enterprise.transforms.models.transform-transform-tag]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
- [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp]]
+ [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp get-test-schema]]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.legacy-mbql.normalize :as mbql.normalize]
@@ -52,12 +52,6 @@
;; Convert to legacy MBQL which the transform API expects
(lib.convert/->legacy-MBQL query)))
-(defn- get-test-schema
- "Get the schema from the products table in the test dataset.
- This is needed for databases like BigQuery that require a schema/dataset."
- []
- (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)))
-
(deftest create-transform-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}
@@ -73,6 +67,80 @@
:schema schema
:name table-name}})))))))
+(deftest create-transform-feature-flag-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (testing "Creating a query transform without :transforms feature returns 403"
+ (mt/with-premium-features #{}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [query (make-query "Gadget")
+ schema (get-test-schema)
+ response (mt/user-http-request :crowberto :post 402 "ee/transform"
+ {:name "Test Transform"
+ :source {:type "query"
+ :query query}
+ :target {:type "table"
+ :schema schema
+ :name "test_transform"}})]
+ (is (= "error-premium-feature-not-available" (:status response)))))))
+
+ (testing "Creating a query transform with :transforms feature succeeds"
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [table-name "test_query_transform"]
+ (let [query (make-query "Gadget")
+ schema (get-test-schema)
+ response (mt/user-http-request :crowberto :post 200 "ee/transform"
+ {:name "Test Transform"
+ :source {:type "query"
+ :query query}
+ :target {:type "table"
+ :schema schema
+ :name table-name}})]
+ (is (some? (:id response))))))))))
+
+(deftest update-transform-feature-flag-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (testing "Updating a query transform requires :transforms feature"
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [table-name "test_update"]
+ (let [query (make-query "Gadget")
+ schema (get-test-schema)
+ transform-payload {:name "Original Transform"
+ :source {:type "query"
+ :query query}
+ :target {:type "table"
+ :schema schema
+ :name table-name}}
+ created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
+ ;; Now test update without feature flag
+ (mt/with-premium-features #{}
+ (let [response (mt/user-http-request :crowberto :put
+ (format "ee/transform/%d" (:id created))
+ (assoc transform-payload :name "Updated Transform"))]
+ (is (= "error-premium-feature-not-available" (:status response))))))))))))
+
+(deftest run-transform-feature-flag-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (testing "Running a query transform requires :transforms feature"
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [table-name "test_run"]
+ (let [query (make-query "Gadget")
+ schema (get-test-schema)
+ transform-payload {:name "Test Run Transform"
+ :source {:type "query"
+ :query query}
+ :target {:type "table"
+ :schema schema
+ :name table-name}}
+ created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
+ ;; Now test run without feature flag
+ (mt/with-premium-features #{}
+ (let [response (mt/user-http-request :crowberto :post
+ (format "ee/transform/%d/run" (:id created)))]
+ (is (= "error-premium-feature-not-available" (:status response))))))))))))
+
(deftest list-transforms-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(testing "Can list without query parameters"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
index 4c3630ca2d8..f7796ce3825 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
@@ -1,11 +1,11 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test
(:require
[clojure.test :refer :all]
- [medley.core :as m]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
- [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup! delete-schema!]]
+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! delete-schema!]]
+ [metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.lib.core :as lib]
@@ -13,6 +13,7 @@
[metabase.query-processor :as qp]
[metabase.sync.core :as sync]
[metabase.test :as mt]
+ [metabase.test.data.sql :as sql.tx]
[toucan2.core :as t2]
[toucan2.util :as u]))
@@ -34,19 +35,6 @@
[mp table-name]
(->> table-name mt/id (lib.metadata/table mp)))
-(defn- wait-for-table
- "Wait for a table to appear in metadata, with timeout.
- Copied from execute_test.clj - will consolidate later."
- [table-name timeout-ms]
- (let [mp (mt/metadata-provider)
- limit (+ (System/currentTimeMillis) timeout-ms)]
- (loop []
- (Thread/sleep 200)
- (when (> (System/currentTimeMillis) limit)
- (throw (ex-info "table has not been created" {:table-name table-name, :timeout-ms timeout-ms})))
- (or (m/find-first (comp #{table-name} :name) (lib.metadata/tables mp))
- (recur)))))
-
(deftest execute-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
@@ -65,14 +53,14 @@
:query t1-query}
:target target1}]
(transforms.execute/run-mbql-transform! t1 {:run-method :manual})
- (let [table1 (wait-for-table table1-name 10000)
+ (let [table1 (transforms.tu/wait-for-table table1-name 10000)
t2-query (make-query (->> table1 :name (table-name->qp-table (mt/metadata-provider))) "category" lib/= "Gizmo")]
(mt/with-temp [:model/Transform t2 {:name "transform2"
:source {:type :query
:query t2-query}
:target target2}]
(transforms.execute/run-mbql-transform! t2 {:run-method :cron})
- (let [table2 (wait-for-table table2-name 10000)
+ (let [table2 (transforms.tu/wait-for-table table2-name 10000)
check-query (lib/aggregate (make-query (->> table2 :name (table-name->qp-table (mt/metadata-provider)))) (lib/count))
query-result (qp/process-query check-query)]
;; The transforms-test dataset has exactly 4 Gizmo products (IDs 6, 8, 12, 14)
@@ -123,7 +111,7 @@
(doseq [transform [transform-no-limit transform-limit]]
(transforms.execute/run-mbql-transform! transform {:run-method :manual})
(let [table-name (-> transform :target :name)
- _ (wait-for-table table-name 10000)
+ _ (transforms.tu/wait-for-table table-name 10000)
table-result (lib.metadata/table mp (mt/id (keyword table-name)))
query-result (->> (lib/query mp table-result)
(qp/process-query)
@@ -137,6 +125,24 @@
["2024-01-23T00:00:00Z" 14.99]]
query-result)))))))))))
+(deftest create-table-from-schema!-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+ (let [driver driver/*driver*
+ db-id (mt/id)
+ table-name (mt/random-name)
+ schema-name (sql.tx/session-schema driver)
+ table-schema {:name (if schema-name
+ (keyword schema-name table-name)
+ (keyword table-name))
+ :columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "name" :type :type/Text :nullable? true}]}]
+ (mt/as-admin
+ (testing "create-table-from-schema! should create the table successfully"
+ (transforms.util/create-table-from-schema! driver db-id table-schema)
+ (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
+ (is (some? table-exists?) "Table should exist in the database schema")
+ (driver/drop-table! driver db-id (:name table-schema))))))))
+
(deftest transform-schema-created-if-needed-test
(mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})
(mt/dataset transforms-dataset/transforms-test
@@ -157,7 +163,8 @@
:query query}
:target target-table}]
(transforms.execute/run-mbql-transform! transform {:run-method :manual})
- (let [table-result (wait-for-table (:name target-table) 10000)
+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
+ table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
query-result (->> (lib/query mp table-result)
(qp/process-query)
(mt/formatted-rows [int str str 2.0 str])
@@ -207,7 +214,7 @@
:query query}
:target target-table}]
(transforms.execute/run-mbql-transform! transform {:run-method :manual})
- (let [_ (wait-for-table (:name target-table) 10000)
+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
query-result (->> (lib/query mp table-result)
(qp/process-query)
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
index a12d860b401..aeee55451f7 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
@@ -1,7 +1,12 @@
(ns metabase-enterprise.transforms.jobs-test
+ #_{:clj-kondo/ignore [:discouraged-namespace]}
(:require
[clojure.test :refer :all]
- [metabase-enterprise.transforms.jobs :as jobs]))
+ [clojure.tools.logging :as log]
+ [metabase-enterprise.transforms.execute :as transforms.execute]
+ [metabase-enterprise.transforms.jobs :as jobs]
+ [metabase-enterprise.transforms.models.transform-run :as transform-run]
+ [metabase.test :as mt]))
(deftest basic-deps-test
(let [ordering {1 #{2 3}
@@ -75,3 +80,64 @@
(-> (#'jobs/next-transform sorted-ordering transforms-by-id #{2 3})
:id)))
(is (nil? (#'jobs/next-transform sorted-ordering transforms-by-id #{1 2 3})))))
+
+(def ^:private query-source {:type "query"})
+(def ^:private python-source {:type "python"})
+
+(deftest run-transform-feature-flag-test
+ (testing "Query transforms are skipped without :transforms feature"
+ (mt/with-premium-features #{}
+ (let [query-transform {:id 1
+ :source query-source
+ :name "Test Query Transform"}
+ run-id 100
+ logged-messages (atom [])]
+ (with-redefs [log/log* (fn [_ level _ message]
+ (swap! logged-messages conj {:level level :message message}))
+ transform-run/running-run-for-transform-id (constantly nil)]
+ (#'jobs/run-transform! run-id :scheduled query-transform)
+ (is (= 1 (count @logged-messages))
+ "Should log exactly one warning")
+ (is (= :warn (:level (first @logged-messages)))
+ "Should log at warn level")
+ (is (re-matches #".*Skip running transform 1 due to lacking premium features.*"
+ (:message (first @logged-messages)))
+ "Warning message should indicate transform was skipped due to missing features")))))
+
+ (testing "Python transforms are skipped without :transforms-python feature"
+ (mt/with-premium-features #{:transforms}
+ (let [python-transform {:id 2
+ :source python-source
+ :name "Test Python Transform"}
+ run-id 101
+ logged-messages (atom [])]
+ (with-redefs [log/log* (fn [_ level _ message]
+ (swap! logged-messages conj {:level level :message message}))
+ transform-run/running-run-for-transform-id (constantly nil)]
+ (#'jobs/run-transform! run-id :scheduled python-transform)
+ (is (= 1 (count @logged-messages))
+ "Should log exactly one warning")
+ (is (= :warn (:level (first @logged-messages)))
+ "Should log at warn level")
+ (is (re-matches #".*Skip running transform 2 due to lacking premium features.*"
+ (:message (first @logged-messages)))
+ "Warning message should indicate transform was skipped due to missing features")))))
+
+ (testing "Query transforms run with :transforms feature"
+ (mt/with-premium-features #{:transforms}
+ (let [query-transform {:id 3
+ :source query-source
+ :name "Test Query Transform"}
+ run-id 102
+ logged-messages (atom [])
+ run-called? (atom false)]
+ (with-redefs [log/log* (fn [_ level _ message]
+ (swap! logged-messages conj {:level level :message message}))
+ transform-run/running-run-for-transform-id (constantly nil)
+ transforms.execute/run-mbql-transform! (fn [_ _]
+ (reset! run-called? true))]
+ (#'jobs/run-transform! run-id :scheduled query-transform)
+ (is (empty? (filter (comp #{:warn} :level) @logged-messages))
+ "Should not log warnings when feature is enabled")
+ (is @run-called?
+ "Should call run-mbql-transform! when feature is enabled"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
index fa74e8c8306..71d110792bc 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
@@ -12,7 +12,8 @@
(defn- make-transform [query & [name schema]]
(let [name (or name (mt/random-name))
- default-schema (some-> driver/*driver* driver.sql/default-schema)
+ default-schema (when (get-method driver.sql/default-schema driver/*driver*)
+ (driver.sql/default-schema driver/*driver*))
schema (or schema default-schema "public")]
{:source {:type :query
:query query}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/test_util.clj b/enterprise/backend/test/metabase_enterprise/transforms/test_util.clj
index 91b825732ca..fa04c43a593 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/test_util.clj
@@ -1,6 +1,7 @@
(ns metabase-enterprise.transforms.test-util
(:require
[clojure.string :as str]
+ [clojure.test :refer :all]
[metabase.api.common :as api]
[metabase.driver :as driver]
[metabase.test :as mt]
@@ -53,6 +54,14 @@
`(mt/with-model-cleanup [:model/Transform]
~@body)))
+(defn table-rows
+ [table-name]
+ (->>
+ (mt/rows (mt/process-query {:database (mt/id)
+ :query {:source-table (t2/select-one-pk :model/Table :name table-name)}
+ :type :query}))
+ (map (fn [x] (if (= :mongo driver/*driver*) (rest x) x)))))
+
(defn parse-timestamp
"Parse a local datetime and convert it to a ZonedDateTime in the default timezone."
^ZonedDateTime [timestamp-string]
@@ -70,6 +79,62 @@
^String [timestamp-string]
(-> timestamp-string parse-instant str))
+(defn wait-for-table
+ "Wait for a table to appear in metadata, with timeout."
+ [^String table-name timeout-ms]
+ (let [timer (u/start-timer)]
+ (loop []
+ (let [table (t2/select-one :model/Table :name table-name)
+ fields (t2/select :model/Field :table_id (:id table))]
+ (cond
+ (and table (seq fields)) table
+ (> (u/since-ms timer) timeout-ms)
+ (throw (ex-info (format "Table %s did not appear after %dms" table-name timeout-ms)
+ {:table-name table-name :timeout-ms timeout-ms}))
+ :else (do (Thread/sleep 100)
+ (recur)))))))
+
+(defn test-run
+ [transform-id]
+ (let [resp (mt/user-http-request :crowberto :post 202 (format "ee/transform/%s/run" transform-id))
+ timeout-s 10 ; 10 seconds is our timeout to finish execution and sync
+ limit (+ (System/currentTimeMillis) (* timeout-s 1000))]
+ (is (=? {:message "Transform run started"}
+ resp))
+ (loop [last-resp nil]
+ (when (> (System/currentTimeMillis) limit)
+ (throw (ex-info (str "Transform run timed out after " timeout-s " seconds") {:resp last-resp})))
+ (let [resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" transform-id))
+ status (some-> resp :last_run :status keyword)]
+ (when-not (contains? #{:started :succeeded} status)
+ (throw (ex-info (str "Transform run failed with status " status) {:resp resp :status status})))
+ (when-not (some? (:table resp))
+ (Thread/sleep 100)
+ (recur resp))))))
+
+(defn wait-for-transform-completion
+ "Wait for a transform run to complete without triggering a new run.
+ Polls the transform status until it succeeds or times out."
+ [transform-id timeout-ms]
+ (let [start-time (u/start-timer)]
+ (loop []
+ (when (> (u/since-ms start-time) timeout-ms)
+ (throw (ex-info (format "Transform %d did not complete after %dms" transform-id timeout-ms)
+ {:transform-id transform-id :timeout-ms timeout-ms})))
+ (let [resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" transform-id))
+ status (some-> resp :last_run :status keyword)]
+ (case status
+ :succeeded resp
+ (:started :running) (do (Thread/sleep 100) (recur))
+ (throw (ex-info (format "Transform run failed with status %s" status)
+ {:resp resp :status status})))))))
+
+(defn get-test-schema
+ "Get the schema from the products table in the test dataset.
+ This is needed for databases like BigQuery that require a schema/dataset."
+ []
+ (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)))
+
(defmulti delete-schema!
"Deletes a schema."
{:arglists '([driver db schema])}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
new file mode 100644
index 00000000000..af55f2d8e8d
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
@@ -0,0 +1,64 @@
+(ns ^:mb/driver-tests metabase-enterprise.transforms.util-test
+ "Tests for transform utility functions."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.driver :as driver]
+ [metabase.test :as mt]
+ [metabase.test.data.sql :as sql.tx]))
+
+(set! *warn-on-reflection* true)
+
+(deftest temp-table-name-test
+ (testing "temp-table-name generates valid table names respecting driver limits"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (let [driver driver/*driver*]
+
+ (testing "Basic table name generation"
+ (let [result (transforms.util/temp-table-name driver :base_table "test")]
+ (is (keyword? result))
+ (is (nil? (namespace result)))
+ (is (str/starts-with? (name result) "base_table_test_"))))
+
+ (testing "Table name with very large base name respects driver limits"
+ (let [very-long-base-name (keyword (apply str (repeat 200 "a")))
+ result (transforms.util/temp-table-name driver very-long-base-name "test")
+ max-len (or (driver/table-name-length-limit driver) Integer/MAX_VALUE)
+ actual-name (name result)]
+ (is (keyword? result))
+ (is (<= (count actual-name) max-len))
+ (is (re-find #"_test_\d+$" actual-name))))
+
+ (testing "Table name preserves namespace when present"
+ (let [result (transforms.util/temp-table-name driver :schema/base_table "test")]
+ (is (= "schema" (namespace result)))
+ (is (str/starts-with? (name result) "base_table_test_"))))))))
+
+(deftest temp-table-name-creates-table-test
+ (testing "temp-table-name produces names that can actually create tables"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (let [driver driver/*driver*
+ db-id (mt/id)
+
+ long-base-name (keyword (apply str (repeat 1000 "test")))
+
+ table-name (transforms.util/temp-table-name driver long-base-name "temp")
+ schema-name (when (get-method sql.tx/session-schema driver)
+ (sql.tx/session-schema driver))
+ qualified-table-name (if schema-name
+ (keyword schema-name (name table-name))
+ table-name)
+ column-definitions {"id" (driver/type->database-type driver :type/Integer)}]
+ (mt/as-admin
+ (try
+ (testing "Can create table with generated temp name"
+ (driver/create-table! driver db-id qualified-table-name column-definitions {})
+ (when-not (= driver :mongo) ;; mongo doesn't actually create tables
+ (is (driver/table-exists? driver (mt/db) {:schema schema-name :name (name table-name)}))))
+ (finally
+ (try
+ (driver/drop-table! driver db-id qualified-table-name)
+ (catch Exception _e
+ ;; Ignore cleanup errors
+ nil)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
new file mode 100644
index 00000000000..0a155546169
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -0,0 +1,528 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.drivers-test
+ "Comprehensive tests for Python transforms across all supported drivers with all base and exotic types."
+ (:require
+ [clojure.core.async :as a]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms-python.execute :as transforms.execute]
+ [metabase-enterprise.transforms-python.python-runner :as python-runner]
+ [metabase-enterprise.transforms-python.s3 :as s3]
+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+ [metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.driver :as driver]
+ [metabase.driver.mysql :as mysql]
+ [metabase.sync.core :as sync]
+ [metabase.test :as mt]
+ [metabase.test.data.interface :as tx]
+ [metabase.test.data.sql :as sql.tx]
+ [metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def test-id 42)
+
+(defn- execute!
+ "Execute a Python transform with the given code and tables"
+ [{:keys [code tables]}]
+ (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
+ (let [server-url (transforms-python.settings/python-runner-url)
+ cancel-chan (a/promise-chan)
+ table-name->id (or tables {})
+ _ (python-runner/copy-tables-to-s3! {:run-id test-id
+ :shared-storage @shared-storage-ref
+ :table-name->id table-name->id
+ :cancel-chan cancel-chan})
+ response (python-runner/execute-python-code-http-call! {:server-url server-url
+ :code code
+ :run-id test-id
+ :table-name->id table-name->id
+ :shared-storage @shared-storage-ref})
+ {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+ (merge (:body response)
+ {:output output
+ :output-manifest output-manifest
+ :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+ :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+
+(defn- execute-e2e-transform!
+ "Execute an e2e Python transform test using execute-python-transform!"
+ [table-name transform-code source-tables]
+ (let [schema (when (#{:postgres :bigquery-cloud-sdk} driver/*driver*)
+ (sql.tx/session-schema driver/*driver*))
+ target {:type "table"
+ :schema schema
+ :name table-name
+ :database (mt/id)}
+ transform-def {:name (str "E2E Transform Test " table-name)
+ :source {:type "python"
+ :source-tables source-tables
+ :body transform-code}
+ :target target}]
+ (with-transform-cleanup! [_target target]
+ (mt/with-temp [:model/Transform transform transform-def]
+ (transforms.execute/execute-python-transform! transform {:run-method :manual})
+ (let [table (transforms.tu/wait-for-table table-name 10000)
+ columns (t2/select :model/Field :table_id (:id table) {:order-by [:position]})
+ column-names (filterv (fn [x] (not= x "_id")) ;; for mongo
+ (map :name columns))
+ rows (transforms.tu/table-rows table-name)]
+ {:columns column-names
+ :rows rows})))))
+
+(def ^:private base-type-test-data
+ "Base types that all drivers should support with test data."
+ {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "name" :type :type/Text :nullable? true}
+ {:name "price" :type :type/Float :nullable? true}
+ {:name "active" :type :type/Boolean :nullable? true}
+ {:name "created_date" :type :type/Date :nullable? true}
+ {:name "created_at" :type :type/DateTime :nullable? true}]
+ :data [[1 "Product A" 19.99 true "2024-01-01" "2024-01-01T12:00:00"]
+ [2 "Product B" 15.50 false "2024-02-01" "2024-02-01T09:15:30"]
+ [3 nil nil nil nil nil]]})
+
+(def ^:private driver-exotic-types
+ "Driver-specific exotic types with test data."
+ {:postgres {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "bigint" :type :type/BigInteger :nullable? false}
+ {:name "created_tz" :type :type/DateTimeWithTZ :nullable? true}
+ {:name "uuid_field" :type :type/UUID :nullable? true}
+ {:name "json_field" :type :type/JSON :nullable? true}
+ {:name "ip_field" :type :type/IPAddress :nullable? true}
+ {:name "int_array" :type :type/* :nullable? true :database-type "integer[]"}
+ {:name "text_array" :type :type/* :nullable? true :database-type "text[]"}
+ {:name "uuid_array" :type :type/* :nullable? true :database-type "uuid[]"}]
+ :data [[1 -9223372036854775808 "2024-01-01T12:00:00Z" "550e8400-e29b-41d4-a716-446655440000" "{\"key\": \"value\"}" "192.168.1.1"
+ "{1,2,3,4,5}" "{\"hello\",\"world\",\"test\"}" "{550e8400-e29b-41d4-a716-446655440000,123e4567-e89b-12d3-a456-426614174000}"]
+ [2 0 nil nil nil nil nil nil nil]
+ [3 nil "2024-02-01T09:15:30-05:00" nil nil nil "{}" "{}" "{}"]]}
+
+ :mysql {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "json_field" :type :type/JSON :nullable? true}
+ {:name "timestamp" :type :type/DateTimeWithLocalTZ :nullable? true :database-type "timestamp"}]
+ :data [[1 "{\"key\": \"value\"}" "2024-01-01 12:00:00"]
+ [2 nil
+ nil]]}
+ :mariadb {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "json_field" :type :type/JSON :nullable? true}]
+ :data [[1 "{\"key\": \"mariadb_value\"}"]
+ [2 nil]]}
+
+ :bigquery-cloud-sdk {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "json_field" :type :type/JSON :nullable? true}
+ {:name "dict_field" :type :type/Dictionary :nullable? true :database-type "STRUCT<key STRING, value INT64>"}]
+ :data [[1 "{\"key\": \"value\"}" {"key" "test", "value" 42}]
+ [2 nil nil]]}
+ :sqlserver {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "uuid_field" :type :type/UUID :nullable? true}
+ {:name "datetimeoffset_field" :type :type/DateTimeWithTZ :nullable? true :database-type "datetimeoffset"}]
+ :data [[1 "550e8400-e29b-41d4-a716-446655440000" "2024-01-01 12:00:00 -05:00"]
+ [2 nil nil]]}
+ :mongo {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "uuid_field" :type :type/UUID :nullable? true}
+ {:name "array_field" :type :type/Array :nullable? true :database-type "array"}
+ {:name "dict_field" :type :type/Dictionary :nullable? true :database-type "object"}]
+ :data [[1 #uuid "550e8400-e29b-41d4-a716-446655440000" [1, 2, 3] {"nested" "object"}]
+ [2 nil nil nil]]}})
+
+(defn- create-test-table-with-data!
+ "Create a test table with the given schema and data for the current driver."
+ [table-name schema data]
+ (let [driver driver/*driver*
+ db-id (mt/id)
+ schema-name (case driver
+
+ :clickhouse (-> (mt/db) :details :db)
+
+ :mongo nil
+
+ (sql.tx/session-schema driver))
+ qualified-table-name (if schema-name
+ (keyword schema-name table-name)
+ (keyword table-name))
+ table-schema {:name qualified-table-name
+ :columns (:columns schema)}]
+ (mt/as-admin
+ (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+ (when (seq data)
+ (driver/insert-from-source! driver db-id table-schema
+ {:type :rows
+ :data (map (fn [row]
+ (map #(cond
+ (and (= :sqlserver driver) (boolean? %))
+ (if % 1 0)
+
+ (and (= :mongo driver) (string? %) (try (u.date/parse %) (catch Exception _)))
+ (u.date/parse %)
+
+ :else
+ %) row))
+ data)}))
+
+ (sync/sync-database! (mt/db) {:scan :schema})
+
+ (t2/select-one-pk :model/Table :name (name qualified-table-name) :db_id db-id)))
+
+(defn- cleanup-table!
+ "Drop the test table by table ID."
+ [table-id]
+ (try
+ (when-let [table (t2/select-one :model/Table :id table-id)]
+ (let [table-name (:name table)
+ qualified-name (case driver/*driver*
+ :bigquery-cloud-sdk
+ (let [schema (sql.tx/session-schema driver/*driver*)]
+ (if schema
+ (keyword (str schema "." table-name))
+ (keyword table-name)))
+
+ (let [schema (sql.tx/session-schema driver/*driver*)]
+ (if schema
+ (keyword schema table-name)
+ (keyword table-name))))]
+ (driver/drop-table! driver/*driver* (mt/id) qualified-name)))
+ (catch Exception _e
+ nil)))
+
+(defn- validate-transform-output
+ "Validate that the Python transform output preserves types and data correctly,
+ assuming output is in JSONL format."
+ [result expected-columns expected-row-count]
+ (testing "Transform execution succeeded"
+ (is (some? result))
+ (is (contains? result :output))
+ (is (contains? result :output-manifest)))
+
+ (when result
+ (let [lines (str/split-lines (:output result))
+ rows (map json/decode lines)
+ metadata (:output-manifest result)
+ headers (map :name (:fields metadata))]
+
+ (testing "Column headers are correct"
+ (is (= (set expected-columns) (disj (set headers) "_id"))))
+
+ (testing "Row count is correct"
+ (is (= expected-row-count (count rows))))
+
+ (testing "Metadata contains all expected columns"
+ (is (= (set expected-columns)
+ (disj (set (map :name (:fields metadata))) "_id"))))
+
+ {:headers headers
+ :rows rows
+ :metadata metadata})))
+
+#_:clj-kondo/ignore
+(defmacro with-test-table
+ [[table-id table-name] [schema data] & body]
+ `(let [table-name# (if (= :redshift driver/*driver*)
+ (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+ (mt/random-name))
+ table-id# (create-test-table-with-data! table-name# ~schema ~data)]
+ (try
+ (let [~table-id table-id#
+ ~table-name table-name#]
+ ~@body)
+ (finally
+ (cleanup-table! table-id#)))))
+
+(defn- simple-identity-transform-code
+ "Generate Python code for a simple identity transform."
+ [table-name]
+ (str "import pandas as pd\n"
+ "\n"
+ "def transform(" table-name "):\n"
+ " df = " table-name ".copy()\n"
+ " return df"))
+
+(defn- execute-and-validate-transform!
+ "Execute a Python transform and validate the output, returning validation results."
+ [transform-code table-name table-id expected-columns expected-row-count]
+ (let [result (execute! {:code transform-code
+ :tables {table-name table-id}})]
+ (validate-transform-output result expected-columns expected-row-count)))
+
+(defn- test-exotic-types-for-driver!
+ "Helper to test exotic types for the current driver."
+ [driver-key]
+ (when-let [exotic-config (get driver-exotic-types (if (mysql/mariadb? (mt/db)) :mariadb driver-key))]
+ (with-test-table [table-id table-name] [exotic-config (:data exotic-config)]
+ (let [transform-code (simple-identity-transform-code table-name)
+ expected-columns (map :name (:columns exotic-config))
+ expected-row-count (count (:data exotic-config))
+ validation (execute-and-validate-transform!
+ transform-code table-name table-id
+ expected-columns expected-row-count)]
+
+ (when validation
+ (testing (str "Exotic types for " driver-key)
+ (let [{:keys [metadata]} validation
+ type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+ [name (python-runner/restricted-insert-type base_type)])]
+
+ (is (isa? :type/Integer (type-map "id")))
+
+ (case driver-key
+ :postgres (do
+ (is (isa? (type-map "bigint") :type/BigInteger))
+ (is (isa? (type-map "uuid_field") :type/Text #_:type/UUID))
+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+ (is (isa? (type-map "ip_field") :type/Text #_:type/IPAddress)))
+ :mysql (if (mysql/mariadb? (mt/db))
+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+ (do
+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+ (is (isa? (type-map "timestamp") :type/DateTimeWithLocalTZ))))
+
+ :bigquery-cloud-sdk (do
+ (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+ (is (isa? (type-map "dict_field") :type/Text #_:type/JSON)))
+
+ :mongo (do
+ (is (isa? (type-map "array_field") :type/Text #_:type/Array))
+ (is (isa? (type-map "dict_field") :type/Text #_:type/Dictionary)))
+
+ :sqlserver (do
+ (is (isa? (type-map "uuid_field") :type/Text #_:type/UUID))
+ (is (isa? (type-map "datetimeoffset_field") :type/DateTimeWithTZ)))))))
+ validation))))
+
+(deftest create-table-test
+ (testing "Test we can create base table"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (mt/with-empty-db
+ (with-test-table [table-id _table-name] [base-type-test-data (:data base-type-test-data)]
+ (is table-id "Table should be created and have an ID"))))))
+
+(deftest base-types-python-transform-test
+ (testing "Test Python transforms with base types across all supported drivers"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (mt/with-empty-db
+ (with-test-table [table-id table-name] [base-type-test-data (:data base-type-test-data)]
+ (let [transform-code (simple-identity-transform-code table-name)
+ expected-columns ["id" "name" "price" "active" "created_date" "created_at"]
+ validation (execute-and-validate-transform!
+ transform-code table-name table-id expected-columns 3)]
+
+ (when validation
+ (let [{:keys [metadata]} validation]
+ (testing "Base type preservation"
+ (let [type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+ [name (python-runner/restricted-insert-type base_type)])]
+ (is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
+ (is (isa? (type-map "name") :type/Text))
+ (is (isa? (type-map "price") :type/Float))
+ (is (isa? (type-map "active") :type/Boolean))
+ (is (isa? (type-map "created_date") (if (= driver/*driver* :mongo) :type/Instant :type/Date)))
+ (is (isa? (type-map "created_at") :type/DateTime))))))))))))
+
+(deftest exotic-types-python-transform-test
+ (testing "Test Python transforms with driver-specific exotic types"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (mt/with-empty-db
+ (test-exotic-types-for-driver! driver/*driver*)))))
+
+(deftest edge-cases-python-transform-test
+ (testing "Test Python transforms with edge cases: null values, empty strings, extreme values"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (mt/with-empty-db
+ (let [edge-case-schema {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "text_field" :type :type/Text :nullable? true}
+ {:name "int_field" :type :type/Integer :nullable? true}
+ {:name "float_field" :type :type/Float :nullable? true}
+ {:name "bool_field" :type :type/Boolean :nullable? true}
+ {:name "date_field" :type :type/Date :nullable? true}]
+ :data [[1 "" 0 0.0 false "2024-01-01"]
+ [2 "Very long text with special chars: !@#$%^&*(){}[]|\\:;\"'<>,.?/~`"
+ 2147483647 1.79769313486 true "2222-12-31"]
+ [3 nil nil nil nil nil]]}]
+ (with-test-table [table-id table-name] [edge-case-schema (:data edge-case-schema)]
+ (let [transform-code (str "import pandas as pd\n"
+ "import numpy as np\n"
+ "\n"
+ "def transform(" table-name "):\n"
+ " df = " table-name ".copy()\n"
+ " \n"
+ " # Handle text operations safely\n"
+ " df['text_length'] = df['text_field'].fillna(\"\").astype(str).str.len()"
+ " \n"
+ " # Handle numeric operations with null safety\n"
+ " df['int_doubled'] = df['int_field'] * 2\n"
+ " df['float_squared'] = df['float_field'] ** 2\n"
+ " \n"
+ " # Boolean operations\n"
+ " df['bool_inverted'] = ~df['bool_field'].fillna(False)\n"
+ " \n"
+ " return df")
+ expected-columns ["id" "text_field" "int_field" "float_field" "bool_field" "date_field"
+ "text_length" "int_doubled" "float_squared" "bool_inverted"]
+ validation (execute-and-validate-transform!
+ transform-code table-name table-id expected-columns 3)]
+
+ (when validation
+ (let [{:keys [rows metadata]} validation
+ type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+ [name (python-runner/restricted-insert-type base_type)])]
+
+ (testing "Original columns preserved"
+ (is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
+ (is (isa? (type-map "text_field") :type/Text))
+ (is (isa? (type-map "int_field") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
+ (is (isa? (type-map "float_field") :type/Float))
+ (is (isa? (type-map "bool_field") :type/Boolean))
+ (is (isa? (type-map "date_field") (if (= driver/*driver* :mongo) :type/Instant :type/Date))))
+
+ (testing "Computed columns have correct types"
+ (is (isa? (type-map "text_length") :type/Integer))
+ (is (isa? (type-map "int_doubled") :type/Integer))
+ (is (isa? (type-map "float_squared") :type/Float))
+ (is (isa? (type-map "bool_inverted") :type/Boolean)))
+
+ (testing "Edge case data handling"
+ (let [[row1 row2 row3] rows]
+ (is (= 1 (get row1 "id")))
+ (is (= 0 (get row1 "text_length"))) ; empty string length
+ (is (= 0 (get row1 "int_doubled"))) ; 0 * 2
+
+ (is (= 2 (get row2 "id")))
+ (is (not= "" (get row2 "text_length"))) ; long string has length
+
+ (is (= 3 (get row3 "id"))))))))))))))
+
+(deftest idempotent-transform-test
+ (testing "Test that running the same transform multiple times produces identical results"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (mt/with-empty-db
+ (with-test-table [table-id table-name] [base-type-test-data (:data base-type-test-data)]
+ (let [transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform(" table-name "):\n"
+ " df = " table-name ".copy()\n"
+ " df['computed_field'] = df['price'] * 1.1 # 10% markup\n"
+ " df['name_upper'] = df['name'].str.upper()\n"
+ " return df")
+ ;; Run the transform twice
+ result1 (execute!
+ {:code transform-code
+ :tables {table-name table-id}})
+
+ result2 (execute!
+ {:code transform-code
+ :tables {table-name table-id}})]
+ (testing "Both transforms succeeded"
+ (is (some? result1))
+ (is (some? result2)))
+
+ (when (and result1 result2)
+ (testing "Results are identical"
+ (is (= (:output result1) (:output result2)))
+ (is (= (count (:fields (:output-manifest result1)))
+ (count (:fields (:output-manifest result2))))))
+
+ (let [expected-columns ["id" "name" "price" "active" "created_date" "created_at"
+ "computed_field" "name_upper"]
+ validation (validate-transform-output result1 expected-columns 3)]
+
+ (when validation
+ (testing "Computed columns are added correctly"
+ (let [{:keys [metadata]} validation
+ type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+ [name (python-runner/restricted-insert-type base_type)])]
+ (is (= :type/Float (type-map "computed_field")))
+ (is (= :type/Text (type-map "name_upper"))))))))))))))
+
+(deftest comprehensive-e2e-python-transform-test
+ (testing "End-to-end test using execute-python-transform! across all supported drivers with comprehensive type coverage"
+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/python)
+ ;; we sometimes get I/O error in CI due to it taking too long. it's too slow, too flakey to keep enabled
+ :redshift)
+ (mt/with-empty-db
+ (mt/with-premium-features #{:transforms-python}
+ (with-test-table [source-table-id source-table-name] [base-type-test-data (:data base-type-test-data)]
+ (let [table-name (mt/random-name)
+ exotic-config (get driver-exotic-types driver/*driver*)
+ exotic-table-id (when exotic-config
+ (create-test-table-with-data!
+ (str source-table-name "_exotic")
+ exotic-config
+ (:data exotic-config)))]
+
+ (try
+ (let [transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform(" source-table-name
+ (when exotic-config (str ", " source-table-name "_exotic"))
+ "):\n"
+ " # Start with base types table\n"
+ " df = " source-table-name ".copy()\n"
+ " \n"
+ " # Add computed columns to test type handling\n"
+ " df['price_with_tax'] = df['price'] * 1.08 # Float computation\n"
+ " df['name_length'] = df['name'].str.len() # String operations\n"
+ " df['is_expensive'] = df['price'] > 18.0 # Boolean computation\n"
+ " \n"
+ " # Date operations\n"
+ " df['created_year'] = pd.to_datetime(df['created_date']).dt.year\n"
+ " \n"
+ (when exotic-config
+ (str " # Merge with exotic types if available\n"
+ " exotic_df = " source-table-name "_exotic.copy()\n"
+ " df = df.merge(exotic_df, on='id', how='left', suffixes=('', '_exotic'))\n"
+ " df.attrs[\"source_metadata\"] = {**df.attrs.get(\"source_metadata\", {}), **exotic_df.attrs.get(\"source_metadata\", {})}"
+ " \n"))
+ " # Return processed dataframe\n"
+ " return df")
+
+ source-tables (cond-> {source-table-name source-table-id}
+ exotic-table-id
+ (assoc (str source-table-name "_exotic") exotic-table-id))
+
+ result (execute-e2e-transform! table-name transform-code source-tables)
+ {:keys [columns] result-rows :rows} result]
+
+ (testing "E2E transform execution succeeded"
+ (is (seq result-rows) "Transform should produce results"))
+
+ (testing "Expected data transformations"
+ (let [first-row (first result-rows)
+ second-row (second result-rows)
+ third-row (nth result-rows 2)
+ get-column (fn [row col-name] (get (zipmap columns row) col-name))]
+ (testing "Computed columns are present and have reasonable values"
+ (is (> (count columns) 7) "Should have additional computed columns")
+ (is (= 3 (count result-rows)) "Should have 3 rows from source data"))
+
+ (testing "Computed column values are mathematically correct"
+ (testing "Row 1 price_with_tax calculation"
+ (let [price-with-tax (get-column first-row "price_with_tax")]
+ (is (and (number? price-with-tax) (> price-with-tax 21.5) (< price-with-tax 21.7))
+ "First row price_with_tax should be ~21.59")))
+
+ (testing "Row 2 price_with_tax calculation"
+ (let [price-with-tax (get-column second-row "price_with_tax")]
+ (is (and (number? price-with-tax) (> price-with-tax 16.7) (< price-with-tax 16.8))
+ "Second row price_with_tax should be ~16.74")))
+
+ (testing "Name length calculations"
+ (is (== 9 (get-column first-row "name_length")) "First row name_length should be 9")
+ (is (== 9 (get-column second-row "name_length")) "Second row name_length should be 9"))
+
+ (testing "Boolean expense calculations"
+ (is (true? (get-column first-row "is_expensive")) "First row is_expensive should be true")
+ (is (false? (get-column second-row "is_expensive")) "Second row is_expensive should be false"))
+
+ (testing "Date year extraction"
+ (is (== 2024 (get-column first-row "created_year")) "First row created_year should be 2024")
+ (is (== 2024 (get-column second-row "created_year")) "Second row created_year should be 2024"))
+
+ (testing "Null value handling"
+ (is (seq third-row) "Third row should contain values (nulls handled gracefully)"))))))
+ (finally
+ (when exotic-table-id
+ (cleanup-table! exotic-table-id)))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
new file mode 100644
index 00000000000..55a55151371
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj
@@ -0,0 +1,86 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.e2e-test
+ "End-to-end tests for Python library functionality."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms-python.models.python-library :as python-library]
+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+ [metabase.driver :as driver]
+ [metabase.test :as mt]
+ [metabase.test.util :as mt.util]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest transforms-python-with-library-test
+ (testing "Python transform execution with common library"
+ (mt/test-drivers #{:postgres}
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (mt.util/with-discard-model-updates! [:model/PythonLibrary]
+ ;; Create or update the python library
+ (python-library/update-python-library-source! "common"
+ (str "def multiply_by_two(x):\n"
+ " return x * 2\n"
+ "\n"
+ "def add_greeting(name):\n"
+ " return f'Hello, {name}!'\n"))
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+ ;; Create and run a transform that uses the library
+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"
+ :schema schema
+ :name "library_test_output"}]
+ (let [transform-body (str "import pandas as pd\n"
+ "from common import multiply_by_two, add_greeting\n"
+ "\n"
+ "def transform():\n"
+ " data = {\n"
+ " 'number': [1, 2, 3, 4, 5],\n"
+ " 'doubled': [multiply_by_two(x) for x in [1, 2, 3, 4, 5]],\n"
+ " 'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],\n"
+ " 'greeting': [add_greeting(name) for name in ['Alice', 'Bob', 'Charlie', 'David', 'Eve']]\n"
+ " }\n"
+ " return pd.DataFrame(data)")
+ transform-payload {:name "Library Test Transform"
+ :source {:type "python"
+ :source-database (mt/id)
+ :source-tables {}
+ :body transform-body}
+ :target (assoc target :database (mt/id))}
+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+ transform-payload)]
+ (transforms.tu/test-run transform-id)
+ (transforms.tu/wait-for-table table-name 5000)
+ (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+
+ (let [rows (transforms.tu/table-rows table-name)]
+ (is (= 5 (count rows)))
+ (is (every? #(= (* 2 (first %)) (second %)) rows))
+ (is (= ["Hello, Alice!" "Hello, Bob!" "Hello, Charlie!" "Hello, David!" "Hello, Eve!"]
+ (map #(nth % 3) rows)))))))))))))
+
+(deftest execute-transforms-python-test
+ (testing "transform execution with :transforms/table target"
+ (mt/test-drivers #{:postgres}
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"
+ :schema schema
+ :name "target_table"}]
+ (let [original {:name "Gadget Products"
+ :source {:type "python"
+ :source-database (mt/id)
+ :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+ :target (assoc target :database (mt/id))}
+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+ original)]
+ (transforms.tu/test-run transform-id)
+ (transforms.tu/wait-for-table table-name 5000)
+ (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+ (is (= [["Alice" 25] ["Bob" 30]]
+ (transforms.tu/table-rows table-name)))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
new file mode 100644
index 00000000000..4da89bb4cb4
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
@@ -0,0 +1,97 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.execute-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+ [metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
+ (:import
+ (java.util.concurrent CountDownLatch)))
+
+(set! *warn-on-reflection* true)
+
+(deftest atomic-python-transform-swap-test
+ (testing "Python transform execution with atomic table swap"
+ (mt/test-drivers #{:mysql :postgres}
+ (mt/with-premium-features #{:transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"
+ :schema schema
+ :name "swap_tbl"}]
+ (let [initial-transform {:name "Python Transform Initial"
+ :source {:type "python"
+ :source-tables {}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+ :target (assoc target :database (mt/id))}]
+ (mt/with-temp [:model/Transform transform initial-transform]
+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
+ (transforms.tu/wait-for-table table-name 10000)
+
+ (let [initial-rows (transforms.tu/table-rows table-name)]
+ (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
+
+ (t2/update! :model/Transform (:id transform)
+ {:source {:type "python"
+ :source-tables {}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
+
+ (let [swap-latch (CountDownLatch. 1)
+ original-rename-tables-atomic! transforms.util/rename-tables!]
+ (with-redefs [transforms.util/rename-tables! (fn [driver db-id rename-pairs]
+ (.await swap-latch)
+ (original-rename-tables-atomic! driver db-id rename-pairs))]
+ (let [transform-future (future
+ (transforms-python.execute/execute-python-transform!
+ (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual}))]
+ (is (= [["Alice" 25] ["Bob" 30]]
+ (transforms.tu/table-rows table-name))
+ "Original data should still be accessible during transform")
+ (.countDown swap-latch)
+ @transform-future
+ (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
+ (transforms.tu/table-rows table-name))
+ "Table should now contain Charlie, Diana, and Eve")))))))))))))
+
+(deftest python-transform-temp-table-cleanup-test
+ (testing "Python transform cleans up temp tables on success"
+ (mt/test-drivers #{:mysql :postgres}
+ (mt/with-premium-features #{:transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"
+ :schema schema
+ :name "cleanup_"}]
+ (let [transform-def {:name "Python Transform Cleanup"
+ :source {:type "python"
+ :source-tables {}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
+ :target (assoc target :database (mt/id))}]
+ (mt/with-temp [:model/Transform transform transform-def]
+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
+ (transforms.tu/wait-for-table table-name 10000)
+
+ (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
+
+ (let [db-id (mt/id)
+ tables (t2/select :model/Table :db_id db-id :active true)
+ new-table-pattern (re-pattern (str ".*" table-name "_" transforms-python.execute/temp-table-suffix-new "_.*"))
+ old-table-pattern (re-pattern (str ".*" table-name "_" transforms-python.execute/temp-table-suffix-old "_.*"))]
+ (is (not-any? #(or (re-matches new-table-pattern (:name %))
+ (re-matches old-table-pattern (:name %))) tables)
+ (str "No temp tables (_" transforms-python.execute/temp-table-suffix-new "_ or _" transforms-python.execute/temp-table-suffix-old "_) should remain after successful Python transform"))
+
+ (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
+ "Table should contain the expected data after swap")))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
new file mode 100644
index 00000000000..6bca42e7ddd
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
@@ -0,0 +1,46 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.job-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms.schedule :as transforms.schedule]
+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+ [metabase-enterprise.transforms.test-util :as transforms.tu]
+ [metabase.driver :as driver]
+ [metabase.task.core :as task]
+ [metabase.test :as mt]
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
+
+(deftest python-transform-scheduled-job-test
+ (mt/test-helpers-set-global-values!
+ (mt/with-temp-scheduler!
+ (task/init! ::transforms.schedule/RunTransform)
+ (mt/test-drivers #{:postgres}
+ (mt/with-premium-features #{:transforms-python :transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (transforms.tu/with-transform-cleanup! [target {:type "table"
+ :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ :name "target_table"}]
+ (mt/with-temp
+ [:model/TransformTag {tag-id :id} {:name "every second"}
+ :model/Transform {transform-id :id} {:name "Gadget Products"
+ :source {:type "python"
+ :source-database (mt/id)
+ :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+ :target (assoc target :database (mt/id))}
+ :model/TransformTransformTag _ {:transform_id transform-id :tag_id tag-id :position 0}
+ :model/TransformJob {job-id :id :as job} {:schedule "* * * * * ? *"}
+ :model/TransformJobTransformTag _ {:job_id job-id :tag_id tag-id :position 0}]
+ (transforms.schedule/initialize-job! job)
+ (transforms.schedule/update-job! job-id "* * * * * ? *")
+ (is (true? (u/poll {:thunk (fn [] (driver/table-exists? driver/*driver* (mt/db) target))
+ :done? true?
+ :timeout-ms 20000
+ :interval-ms 1000})))
+ (is (true? (u/poll {:thunk (fn [] (t2/exists? :model/Table :name (:name target)))
+ :done? true?
+ :timeout-ms 10000
+ :interval-ms 1000})))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
new file mode 100644
index 00000000000..fabddc94569
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -0,0 +1,110 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.ordering-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms.ordering :as ordering]
+ [metabase.driver :as driver]
+ [metabase.driver.sql :as driver.sql]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(defn- make-transform [query & [name schema]]
+ (let [name (or name (mt/random-name))
+ default-schema (when (get-method driver.sql/default-schema driver/*driver*)
+ (driver.sql/default-schema driver/*driver*))
+ schema (or schema default-schema "public")]
+ {:source {:type :query
+ :query query}
+ :name (str "transform_" name)
+ :target {:schema schema
+ :name name
+ :type :table}}))
+
+(defn- make-python-transform [source-tables & [name schema target-db]]
+ (let [name (or name (mt/random-name))
+ default-schema (when (get-method driver.sql/default-schema driver/*driver*)
+ (driver.sql/default-schema driver/*driver*))
+ schema (or schema default-schema "public")
+ target-db (or target-db (mt/id))]
+ {:source {:type "python"
+ :source-tables source-tables
+ :body "# Python code here\nresult = df"}
+ :name (str "python_transform_" name)
+ :target {:database target-db
+ :schema schema
+ :name name
+ :type "table"}}))
+
+(defn- transform-deps-for-db [transform]
+ (mt/with-metadata-provider (mt/id)
+ (#'ordering/transform-deps transform)))
+
+(deftest python-transform-basic-dependencies-test
+ (testing "Python transforms with source-tables dependencies are extracted correctly"
+ (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
+ "products" (mt/id :products)})]
+ (is (= #{{:table (mt/id :orders)}
+ {:table (mt/id :products)}}
+ (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
+
+(deftest python-transform-ordering-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (testing "Python transforms are ordered correctly based on source-tables dependencies"
+ (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+ :name "output_1"}
+ :model/Field _ {:table_id table1
+ :name "foo"}
+ :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+ :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+ (is (= {t1 #{}
+ t2 #{t1}}
+ (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))
+
+(deftest python-transform-multiple-dependencies-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (testing "Python transforms with multiple table dependencies"
+ (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+ :name "output_1"}
+ :model/Field _ {:table_id table1
+ :name "foo"}
+ :model/Table {table2 :id} {:schema "public"
+ :name "output_2"}
+ :model/Field _ {:table_id table2
+ :name "bar"}
+ :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+ :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
+ :model/Transform {t3 :id} (make-python-transform {"output_1" table1
+ "output_2" table2} "final_output")]
+ (is (= {t1 #{}
+ t2 #{}
+ t3 #{t1 t2}}
+ (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
+
+(deftest mixed-transform-ordering-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python :transforms/table)
+ (testing "Python and query transforms are ordered together correctly"
+ (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+ :name "sql_output"}
+ :model/Field _ {:table_id table1
+ :name "foo"}
+ :model/Table {table2 :id} {:schema "public"
+ :name "python_output"}
+ :model/Field _ {:table_id table2
+ :name "bar"}
+ ;; SQL transform that depends on orders table
+ :model/Transform {t1 :id} (make-transform
+ {:database (mt/id),
+ :type "query",
+ :query {:source-table (mt/id :orders)}}
+ "sql_output")
+ ;; Python transform that depends on the SQL transform's output
+ :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+ ;; Another SQL transform that depends on the Python transform's output
+ :model/Transform {t3 :id} (make-transform
+ {:database (mt/id)
+ :type "query"
+ :query {:source-table table2}}
+ "final_output")]
+ (is (= {t1 #{}
+ t2 #{t1}
+ t3 #{t2}}
+ (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
new file mode 100644
index 00000000000..52c8e04a810
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
@@ -0,0 +1,495 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.python-runner-test
+ (:require
+ [clojure.core.async :as a]
+ [clojure.java.jdbc :as jdbc]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase-enterprise.transforms-python.python-runner :as python-runner]
+ [metabase-enterprise.transforms-python.s3 :as s3]
+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.driver :as driver]
+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [metabase.sync.core :as sync]
+ [metabase.test :as mt]
+ [metabase.test.data.interface :as tx]
+ [metabase.test.data.sql :as sql.tx]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private last-job-run-id (atom 0))
+
+(defn next-job-run-id [] (swap! last-job-run-id inc))
+
+(defn- parse-jsonl [s] (map json/decode+kw (str/split-lines s)))
+
+(defn template->regex
+ "Convert a template string with $var$ placeholders to a regex pattern.
+ Example: template->regex 'File ___PATH___/script.py, line ___LINE___'
+ will match 'File /tmp/unique-path/script.py, line 123'"
+ [template]
+ (-> template
+ (str/replace #"(?m)^\d{13}\s*" "")
+ (str/replace "\\" "\\\\")
+ (str/replace "." "\\.")
+ (str/replace "*" "\\*")
+ (str/replace "+" "\\+")
+ (str/replace "?" "\\?")
+ (str/replace "^" "\\^")
+ (str/replace "$" "\\$")
+ (str/replace "{" "\\{")
+ (str/replace "}" "\\}")
+ (str/replace "(" "\\(")
+ (str/replace ")" "\\)")
+ (str/replace "|" "\\|")
+ (str/replace "[" "\\[")
+ (str/replace "]" "\\]")
+ (str/replace #"___PATH___" "(/[\\\\w-]+)+")
+ (str/replace #"___LINE___" "\\\\d+")
+ re-pattern))
+
+(defn- jsonl-output [expected] #(= expected (parse-jsonl %)))
+
+(defn- execute! [{:keys [code tables]}]
+ (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
+ (let [server-url (transforms-python.settings/python-runner-url)
+ cancel-chan (a/promise-chan)
+ table-name->id (or tables {})
+ test-id (next-job-run-id)
+ _ (python-runner/copy-tables-to-s3! {:run-id test-id
+ :shared-storage @shared-storage-ref
+ :table-name->id table-name->id
+ :cancel-chan cancel-chan})
+ response (python-runner/execute-python-code-http-call! {:server-url server-url
+ :code code
+ :run-id test-id
+ :table-name->id table-name->id
+ :shared-storage @shared-storage-ref})
+ {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+ ;; not sure about munging this all together but its what tests expect for now
+ (merge (:body response)
+ {:output output
+ :output-manifest output-manifest
+ :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+ :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+
+(defn ok-stdout [num-rows num-cols]
+ (format (str "Successfully saved %d rows to S3\n"
+ "Successfully saved output manifest with %d fields to S3")
+ num-rows num-cols))
+
+(deftest ^:parallel transform-function-basic-test
+ (testing "executes transform function and returns CSV output"
+ (let [transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
+ result (execute! {:code transform-code})]
+ (is (=? {:output (jsonl-output [{:name "Alice", :age 25}
+ {:name "Bob", :age 30}])
+ :stdout (ok-stdout 2 2)}
+ result)))))
+
+(deftest ^:parallel transform-function-missing-test
+ (testing "handles missing transform function"
+ (let [result (execute! {:code (str "import pandas as pd\n"
+ "\n"
+ "# No transform function defined")})]
+ (is (=? {:exit_code 1
+ :stderr "ERROR: User script must define a 'transform()' function"
+ :stdout ""}
+ result)))))
+
+(deftest ^:parallel transform-function-wrong-return-type-test
+ (testing "handles transform function returning non-DataFrame"
+ (let [result (execute! {:code (str "def transform():\n"
+ " return 'not a dataframe'")})]
+ (is (=? {:exit_code 1
+ :stderr "ERROR: Transform function must return a pandas DataFrame, got <class 'str'>"
+ :stdout ""}
+ result)))))
+
+(deftest ^:parallel transform-function-error-test
+ (testing "handles transform function with error"
+ (let [result (execute! {:code (str "def transform():\n"
+ " raise ValueError('Something went wrong')")})
+ expected-template (str "ERROR: Transform function failed: Something went wrong\n"
+ "Traceback (most recent call last):\n"
+ " File \"___PATH___/transform_runner.py\", line ___LINE___, in main\n"
+ " result = script.transform()\n"
+ " ^^^^^^^^^^^^^^^^^^\n"
+ " File \"___PATH___/script.py\", line 2, in transform\n"
+ " raise ValueError('Something went wrong')\n"
+ "ValueError: Something went wrong")
+ stderr-pattern (template->regex expected-template)]
+ (is (=? {:exit_code 1
+ :stderr #(re-matches stderr-pattern %)
+ :stdout ""
+ :timeout false}
+ result)))))
+
+(deftest ^:parallel transform-function-complex-dataframe-test
+ (testing "can create complex DataFrames with transform"
+ (let [transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " data = {'x': [1, 2, 3], 'y': [10, 20, 30], 'z': ['a', 'b', 'c']}\n"
+ " return pd.DataFrame(data)")
+ result (execute! {:code transform-code})]
+ (is (=? {:output (jsonl-output [{:x 1, :y 10, :z "a"}
+ {:x 2, :y 20, :z "b"}
+ {:x 3, :y 30, :z "c"}])
+ :stdout (ok-stdout 3 3)}
+ result)))))
+
+(deftest ^:parallel transform-function-with-db-parameter-test
+ (mt/test-drivers #{:postgres}
+ (testing "transform function can accept db parameter for forward compatibility"
+ (let [transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " data = {'name': ['Charlie', 'Dana'], 'score': [85, 92]}\n"
+ " return pd.DataFrame(data)")
+ result (execute! {:code transform-code})]
+ (is (=? {:output (jsonl-output [{:name "Charlie", :score 85}
+ {:name "Dana", :score 92}])
+ :stdout (ok-stdout 2 2)}
+ result))))))
+
+(deftest transform-function-with-pass-thru
+ (testing "transform function successfully connects to PostgreSQL database and reads data"
+ (mt/test-drivers #{:postgres}
+ (mt/with-empty-db
+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+ _ (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+ _ (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+ _ (sync/sync-database! (mt/db))
+
+ transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform(students):\n"
+ " return students")
+ result (execute! {:code transform-code
+ :tables {"students" (mt/id :students)}})]
+
+ (is (=? {:output (jsonl-output [{:id 1 :name "Alice" :score 85}
+ {:id 2 :name "Bob" :score 92}
+ {:id 3 :name "Charlie" :score 88}
+ {:id 4 :name "Dana" :score 90}])
+ :output-manifest {:fields [{:base_type "Integer",
+ :database_type "int4",
+ :effective_type "Integer",
+ :name "id",
+ :root_type "Number",
+ :semantic_type "PK"}
+ {:base_type "Text",
+ :database_type "varchar",
+ :effective_type "Text",
+ :name "name",
+ :root_type "Text",
+ :semantic_type "Name"}
+ {:base_type "Integer",
+ :database_type "int4",
+ :effective_type "Integer",
+ :name "score",
+ :root_type "Number",
+ :semantic_type "Score"}],
+ :source_metadata {:fields [{:base_type "Integer",
+ :database_type "int4",
+ :effective_type "Integer",
+ :name "id",
+ :root_type "Number",
+ :semantic_type "PK"}
+ {:base_type "Text",
+ :database_type "varchar",
+ :effective_type "Text",
+ :name "name",
+ :root_type "Text",
+ :semantic_type "Name"}
+ {:base_type "Integer",
+ :database_type "int4",
+ :effective_type "Integer",
+ :name "score",
+ :root_type "Number",
+ :semantic_type "Score"}],
+ :table_metadata {:table_id (mt/malli=? int?)},
+ :schema_version 1
+ :data_format "jsonl"
+ :data_version 1}
+ :schema_version 1
+ :data_format "jsonl"
+ :data_version 1}
+ :stdout (ok-stdout 4 3)}
+ result)))))))
+
+(deftest transform-function-with-empty-table-test
+ (testing "transform function successfully connects to PostgreSQL database and reads data"
+ (mt/test-drivers #{:postgres}
+ (mt/with-empty-db
+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+ _ (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+
+ _ (sync/sync-database! (mt/db))
+
+ transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform(students):\n"
+ " # Calculate average score\n"
+ " avg_score = students['score'].mean()\n"
+ " result = pd.DataFrame({\n"
+ " 'student_count': [len(students)],\n"
+ " 'average_score': [round(avg_score, 2)]\n"
+ " })\n"
+ " return result")
+ result (execute! {:code transform-code
+ :tables {"students" (mt/id :students)}})]
+
+ (is (=? {:output "{\"student_count\":0,\"average_score\":null}\n"
+ :output-manifest {:schema_version 1
+ :data_format "jsonl"
+ :data_version 1
+ :fields [{:name "student_count", :base_type "Integer"}
+ {:name "average_score", :base_type "Float"}]}
+ :stdout (ok-stdout 1 2)}
+ result)))))))
+
+(deftest transform-function-with-working-database-test
+ (testing "transform function successfully connects to PostgreSQL database and reads data"
+ (mt/test-drivers #{:postgres}
+ (mt/with-empty-db
+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+ _ (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+ _ (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+ _ (sync/sync-database! (mt/db))
+
+ transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform(students):\n"
+ " # Calculate average score\n"
+ " avg_score = students['score'].mean()\n"
+ " result = pd.DataFrame({\n"
+ " 'student_count': [len(students)],\n"
+ " 'average_score': [round(avg_score, 2)]\n"
+ " })\n"
+ " return result")
+ result (execute! {:code transform-code
+ :tables {"students" (mt/id :students)}})]
+
+ (is (=? {:output "{\"student_count\":4,\"average_score\":88.75}\n"
+ :output-manifest {:schema_version 1
+ :data_format "jsonl"
+ :data_version 1
+ :fields [{:name "student_count", :base_type "Integer"}
+ {:name "average_score", :base_type "Float"}]}
+ :stdout (ok-stdout 1 2)}
+ result)))))))
+
+(defn- datetime-equal?
+ [expected-iso-str actual-pandas-str]
+ (let [expected-dt (t/zoned-date-time expected-iso-str)
+ actual-dt (t/zoned-date-time actual-pandas-str)]
+ (= (t/to-millis-from-epoch expected-dt)
+ (t/to-millis-from-epoch actual-dt))))
+
+(deftest transform-type-test
+ (mt/test-drivers #{:postgres}
+ (mt/with-empty-db
+ (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+ _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS sample_table"])
+ _ (jdbc/execute! db-spec ["CREATE TABLE sample_table (
+ id BIGSERIAL PRIMARY KEY,
+ name VARCHAR(255),
+ description TEXT,
+ count BIGINT,
+ price FLOAT,
+ is_active BOOLEAN,
+ created_date DATE,
+ updated_at TIMESTAMP,
+ scheduled_for TIMESTAMP WITH TIME ZONE
+ )"])
+ _ (jdbc/execute! db-spec ["INSERT INTO sample_table
+ (name, description, count, price, is_active, created_date, updated_at, scheduled_for)
+ VALUES
+ ('Product A', 'A sample product description', 100, 29.99, true, '2024-01-15', '2024-01-15 10:30:00', '2024-01-16 14:00:00+00'),
+ ('Product B', 'Another product with longer description text', 50, 15.50, false, '2024-02-01', '2024-02-01 09:15:30', '2024-02-02 16:30:00-05'),
+ ('Product C', NULL, 0, 0.0, true, '2024-03-10', '2024-03-10 18:45:15', '2024-03-11 08:00:00+02')"])
+
+ _ (sync/sync-database! (mt/db))
+
+ transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform(sample_table):\n"
+ " df = sample_table.copy()\n"
+ " return df")
+ result (execute! {:code transform-code
+ :tables {"sample_table" (mt/id :sample_table)}})
+ rows (parse-jsonl (:output result))
+ headers (map name (keys (first rows)))
+ [row1 row2 row3] rows
+ get-col (fn [row col-name] (get row (keyword col-name)))
+ metadata (:output-manifest result)]
+
+ (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
+ (set headers)))
+
+ (is (= 1 (get-col row1 "id")))
+ (is (= "Product A" (get-col row1 "name")))
+ (is (datetime-equal? "2024-01-16T14:00:00Z" (get-col row1 "scheduled_for")))
+
+ (is (= 2 (get-col row2 "id")))
+ (is (= "Product B" (get-col row2 "name")))
+ (is (datetime-equal? "2024-02-02T21:30:00Z" (get-col row2 "scheduled_for")))
+
+ (is (= 3 (get-col row3 "id")))
+ (is (= "Product C" (get-col row3 "name")))
+ (is (datetime-equal? "2024-03-11T06:00:00Z" (get-col row3 "scheduled_for")))
+ (testing "types are preserved correctly"
+ (is (= {"id" :type/BigInteger
+ "name" :type/Text
+ "description" :type/Text
+ "count" :type/BigInteger
+ "price" :type/Float
+ "is_active" :type/Boolean
+ ;; Our hack works
+ "created_date" :type/Date
+ "updated_at" :type/DateTime
+ "scheduled_for" :type/DateTimeWithLocalTZ}
+ (u/for-map [{:keys [name base_type]} (:fields metadata)]
+ [name (python-runner/restricted-insert-type base_type)]))))))))
+
+(deftest transform-function-with-library-test
+ (testing "transform function can use libraries"
+ (mt/test-drivers #{:postgres}
+ (mt/with-temp [:model/PythonLibrary _ {:path "circle"
+ :source "import math\n\ndef calculate_circle_area(radius):\n return math.pi * radius ** 2"}
+ :model/PythonLibrary _ {:path "utils"
+ :source "def format_currency(amount):\n return f\"${amount:,.2f}\""}]
+ (let [transform-code (str "import pandas as pd\n"
+ "from circle import calculate_circle_area\n"
+ "from utils import format_currency\n"
+ "\n"
+ "def transform():\n"
+ " data = [\n"
+ " {'radius': 5, 'area': calculate_circle_area(5), 'price': format_currency(78.54)},\n"
+ " {'radius': 10, 'area': calculate_circle_area(10), 'price': format_currency(314.16)}\n"
+ " ]\n"
+ " return pd.DataFrame(data)")
+ result (execute! {:code transform-code})]
+ (is (=? {:output (jsonl-output [{:radius 5, :area 78.5398163397, :price "$78.54"}
+ {:radius 10, :area 314.159265359, :price "$314.16"}])
+ :stdout (ok-stdout 2 3)
+ #_#_:stderr ""}
+ result)))))))
+
+(deftest transform-function-without-libraries-test
+ (testing "transform function works when no libraries exist"
+ (mt/test-drivers #{:postgres}
+ (with-redefs [t2/select-fn->fn (fn [k v model]
+ (when (and (= k :path)
+ (= v :source)
+ (= model :model/PythonLibrary))
+ {}))]
+ (let [transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'status': ['ok']})")
+ result (execute! {:code transform-code})]
+ (is (=? {:output (jsonl-output [{:status "ok"}])
+ :stdout (ok-stdout 1 1)}
+ result)))))))
+
+(deftest transform-function-library-import-error-test
+ (testing "transform function handles missing library gracefully"
+ (mt/test-drivers #{:postgres}
+ (with-redefs [t2/select-fn->fn (fn [k v model]
+ (when (and (= k :path)
+ (= v :source)
+ (= model :model/PythonLibrary))
+ {"utils" "def helper():\n return 42"}))]
+ (let [transform-code (str "import pandas as pd\n"
+ "from common import some_function # This library doesn't exist\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'value': [some_function()]})")
+ result (execute! {:code transform-code})]
+ ;; TODO this error message could still be improved a lot
+ (is (=? {#_#_:error "Execution failed"
+ :exit_code 1
+ :stderr #(str/includes? % "No module named 'common'")}
+ result)))))))
+
+(deftest transform-type-roundtrip-test
+ (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
+ (mt/with-empty-db
+ (let [driver driver/*driver*
+ db-id (mt/id)
+ table-name (if (= :redshift driver)
+ (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+ (mt/random-name))
+ schema-name (if (= :clickhouse driver)
+ (-> (mt/db) :details :db)
+ (sql.tx/session-schema driver))
+ qualified-table-name (if schema-name
+ (keyword schema-name table-name)
+ (keyword table-name))
+ table-schema {:name qualified-table-name
+ :columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "price" :type :type/Float :nullable? true}
+ {:name "active" :type :type/Boolean :nullable? true}
+ {:name "created_tz" :type :type/DateTimeWithTZ :nullable? true}
+ {:name "created_at" :type :type/DateTime :nullable? true}
+ {:name "created_date" :type :type/Date :nullable? true}
+ {:name "description" :type :type/Text :nullable? true}]}
+ _ (mt/as-admin
+ (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+ row-values [[1
+ 19.99
+ (if (= driver/*driver* :sqlserver) 1 true)
+ (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00Z")
+ (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00")
+ "2024-01-01"
+ "Sample product"]]
+ _ (driver/insert-from-source! driver db-id table-schema
+ {:type :rows :data row-values})
+
+ _ (sync/sync-database! (mt/db) {:scan :schema})
+
+ transform-code (str "import pandas as pd\n"
+ "\n"
+ "def transform(" table-name "):\n"
+ " return " table-name)
+ result (execute! {:code transform-code
+ :tables {table-name (mt/id qualified-table-name)}})
+
+ metadata (:output-manifest result)]
+
+ (testing "All expected columns are present"
+ (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
+ (set (map :name (:fields metadata))))))
+
+ (testing "types are preserved correctly"
+ (is (= {"id" (if (= :snowflake driver) :type/Number :type/Integer)
+ "price" :type/Float
+ "active" :type/Boolean
+ "created_tz" (case driver
+ :snowflake :type/DateTimeWithTZ
+ :sqlserver :type/DateTimeWithZoneOffset
+ :type/DateTimeWithLocalTZ)
+ "created_at" :type/DateTime
+ "created_date" :type/Date
+ "description" :type/Text}
+ (u/for-map [{:keys [name base_type]} (:fields metadata)]
+ [name (python-runner/restricted-insert-type base_type)]))))
+
+ ;; cleanup
+ (driver/drop-table! driver db-id qualified-table-name)))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
new file mode 100644
index 00000000000..eb3f7539fe6
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj
@@ -0,0 +1,97 @@
+;; This is a driver test currently to benefit from the localstack S3 setup in driver tests.
+;; Perhaps we should create a new testing category?
+;; This code otherwise runs in the appdb section, but it does not touch the appdb.
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.s3-test
+ "Tests for S3 operations in transforms-python module."
+ (:require
+ [clj-http.client :as http]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms-python.s3 :as s3]
+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase.test :as mt]
+ [metabase.util.log :as log])
+ (:import
+ (java.net ConnectException)
+ (java.nio.file Files)
+ (java.nio.file.attribute FileAttribute)
+ (software.amazon.awssdk.services.s3 S3Client)
+ (software.amazon.awssdk.services.s3.model NoSuchKeyException)))
+
+(set! *warn-on-reflection* true)
+
+(comment
+ (remove-ns (ns-name *ns*)))
+
+(defn- s3-endpoint-running? []
+ (try
+ (http/head (transforms-python.settings/python-runner-url) {:throw-exceptions false})
+ (catch ConnectException _
+ false)))
+
+;; TODO remove this once CI situation is resolved
+(use-fixtures :each (fn [thunk]
+ (if (s3-endpoint-running?)
+ (thunk)
+ (log/warn "Skipping S3 tests because localstack isn't running"))))
+
+(deftest s3-read-write-test
+ (testing "We can open an s3 connection, and read and write things"
+ (with-open [s3-client (s3/create-s3-client)]
+ (is (instance? S3Client s3-client))
+ (let [bucket (transforms-python.settings/python-storage-s-3-bucket)
+ key (str "test-object-" (random-uuid) ".txt")
+ body (str "Hello, S3! My secret is:" (random-uuid))]
+
+ (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))
+ (is (= :default (s3/read-to-string s3-client bucket key :default)))
+
+ (let [tmp-file (Files/createTempFile "s3-test" ".txt" (into-array FileAttribute []))]
+ (try
+ (spit (.toFile tmp-file) body)
+
+ (is (s3/upload-file s3-client bucket key (.toFile tmp-file)))
+
+ (is (= body (s3/read-to-string s3-client bucket key)))
+ (is (= body (s3/read-to-string s3-client bucket key :default)))
+
+ (s3/delete s3-client bucket key)
+
+ (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))
+ (is (= :default (s3/read-to-string s3-client bucket key :default)))
+
+ (finally
+ (Files/deleteIfExists tmp-file))))))))
+
+(deftest open-s3-shared-storage-test
+ (testing "open-s3-shared-storage! returns closeable derefable"
+ (mt/with-premium-features #{:transforms-python}
+ (let [table-name->id {"users" 1}
+ storage-ref (s3/open-shared-storage! table-name->id)
+ {:keys [s3-client bucket-name objects]} @storage-ref]
+ (testing "can be dereferenced"
+ (doseq [[k {:keys [method path url]}] objects
+ :let [url (str/replace url "localstack" "localhost")]]
+ (testing (format "%s (%s)" k method)
+ (is (= :not-created (s3/read-to-string s3-client bucket-name path :not-created)))
+ (let [content (str (random-uuid))]
+ (case method
+ ;; Files that we should write, and python-runner should fetch
+ :get (let [tmp-file (Files/createTempFile "s3-test" ".txt" (into-array FileAttribute []))]
+ (try
+ (spit (.toFile tmp-file) content)
+ (is (s3/upload-file s3-client bucket-name path (.toFile tmp-file)))
+ (finally
+ (Files/deleteIfExists tmp-file)))
+ (is (= content (s3/read-to-string s3-client bucket-name path :not-created)))
+ (is (= content (:body (http/get url)))))
+ ;; Files that python-runner should post, and we should fetch
+ :put (do
+ (is (http/put url {:body content}))
+ (is (= content (s3/read-to-string s3-client bucket-name path :not-created)))))))))
+
+ (testing "closing the ref deletes the files"
+ (.close storage-ref)
+ (doseq [[k {:keys [path]}] objects]
+ (testing (str k)
+ (is (= :not-created (s3/read-to-string s3-client bucket-name path :not-created))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
new file mode 100644
index 00000000000..77e8db1bf51
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -0,0 +1,367 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.transforms-api-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+ [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! get-test-schema]]
+ [metabase.driver :as driver]
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
+ (:import
+ (clojure.lang IDeref)
+ (java.io Closeable)
+ (java.time Duration)))
+
+(set! *warn-on-reflection* true)
+
+(deftest create-python-transform-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (letfn [(create-transform! []
+ (let [schema (get-test-schema)
+ transform-payload {:name "My beautiful python runner"
+ :source {:type "python"
+ :body "print('hello world')"
+ :source-tables {}}
+ :target {:type "table"
+ :schema schema
+ :name "gadget_products"
+ :database (mt/id)}}]
+ (mt/user-http-request :crowberto :post "ee/transform"
+ transform-payload)))]
+
+ (testing "without any feature flags"
+ (mt/with-premium-features #{}
+ (testing "creating python transform without any features fails"
+ (is (= "error-premium-feature-not-available"
+ (:status (mt/user-http-request :crowberto :post 402 "ee/transform"
+ {:name "My beautiful python runner"
+ :source {:type "python"
+ :body "print('hello world')"
+ :source-tables {}}
+ :target {:type "table"
+ :schema (get-test-schema)
+ :name "gadget_products"
+ :database (mt/id)}})))))))
+
+ (testing "with only transforms feature flag (no transforms-python)"
+ (mt/with-premium-features #{:transforms}
+ (testing "creating python transform without transforms-python feature fails"
+ (is (= "Premium features required for this transform type are not enabled."
+ (mt/user-http-request :crowberto :post 402 "ee/transform"
+ {:name "My beautiful python runner"
+ :source {:type "python"
+ :body "print('hello world')"
+ :source-tables {}}
+ :target {:type "table"
+ :schema (get-test-schema)
+ :name "gadget_products"
+ :database (mt/id)}}))))))
+
+ (testing "with transforms-python feature flag"
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (with-transform-cleanup! [table-name "gadget_products"]
+ (let [transform (create-transform!)]
+ (is (= "print('hello chris')"
+ (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id transform))
+ {:name "My beautiful python runner"
+ :source {:type "python"
+ :body "print('hello chris')"
+ :source-tables {}}
+ :target {:type "table"
+ :schema (get-test-schema)
+ :name table-name
+ :database (mt/id)}})
+ :source :body)))))))))))
+
+(deftest update-python-transform-feature-flag-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (testing "Updating a python transform requires both :transforms and :transforms-python features"
+ (mt/with-temp [:model/Transform {id :id
+ :as transform} {:name "Original Python Transform"
+ :source {:type "python"
+ :body "print('original')"
+ :source-tables {}}
+ :target {:type "table"
+ :schema "scheam"
+ :name "table"
+ :database (mt/id)}}]
+ (mt/with-premium-features #{}
+ (let [response (mt/user-http-request :crowberto :put
+ (format "ee/transform/%d" id)
+ (assoc-in transform [:source :body] "print('no features')"))]
+ (is (= "error-premium-feature-not-available" (:status response))
+ "Should return 403 without any features")))))))
+
+(deftest run-python-transform-feature-flag-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (testing "Running a python transform requires both :transforms and :transforms-python features"
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [table-name "test_run_python"]
+ (let [schema (get-test-schema)
+ transform-payload {:name "Test Run Python Transform"
+ :source {:type "python"
+ :body "def transform():\n pass"
+ :source-tables {}}
+ :target {:type "table"
+ :schema schema
+ :name table-name
+ :database (mt/id)}}
+ created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
+ (mt/with-premium-features #{}
+ (let [response (mt/user-http-request :crowberto :post 402
+ (format "ee/transform/%d/run" (:id created)))]
+ (is (= "error-premium-feature-not-available" (:status response)))))
+ (mt/with-premium-features #{:transforms}
+ (let [response (mt/user-http-request :crowberto :post
+ (format "ee/transform/%d/run" (:id created)))]
+ (is (= "Premium features required for this transform type are not enabled." response)
+ "Should return 403 without :transforms-python feature"))))))))))
+
+(deftest execute-python-transform-test
+ (testing "transform execution with :transforms/table target"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"
+ :schema schema
+ :name "target_table"}]
+ (let [original {:name "Gadget Products"
+ :source {:type "python"
+ :source-database (mt/id)
+ :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+ :target (assoc target :database (mt/id))}
+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform" original)]
+ (transforms.tu/test-run transform-id)
+ (transforms.tu/wait-for-table table-name 5000)
+ (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+ (is (= [["Alice" 25] ["Bob" 30]]
+ (transforms.tu/table-rows table-name)))))))))))
+
+(defn- subsequence?
+ "Returns true if sequence ys is a subsequence of xs:
+ every element of ys appears in xs in the same order,
+ though not necessarily contiguously."
+ [xs ys]
+ (cond
+ (empty? ys) true
+ (empty? xs) false
+ (= (first xs) (first ys)) (recur (rest xs) (rest ys))
+ :else (recur (rest xs) ys)))
+
+(deftest python-transform-logging-test
+ (letfn [(program->source [program]
+ (->> (concat ["import pandas as pd"
+ "def transform():"]
+ (for [s program] (str " " s))
+ [" return pd.DataFrame({'x': [42]})"])
+ (str/join "\n")))
+
+ (create-transform [{:keys [program]} target]
+ {:post [(integer? %)]}
+ (:id (mt/user-http-request :crowberto :post 200 "ee/transform"
+ {:name "Python logging test"
+ :source {:type "python"
+ :body (program->source program)
+ :source-tables {}}
+ :target (assoc target :database (mt/id))})))
+
+ (block-on-run [{:keys [expect-status]} target transform-id]
+ (try
+ (transforms.tu/test-run transform-id)
+ (catch Throwable e
+ ;; test-run throws for non success, but we want to test failures too
+ (when-not (= expect-status (:status (ex-data e)))
+ (throw e))))
+ (when (= :succeeded expect-status)
+ (transforms.tu/wait-for-table (:name target) 5000)))
+
+ (get-last-run [transform-id]
+ (:last_run (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" transform-id))))
+
+ (open-message-value-observer [transform-id]
+ (let [states (atom [])
+ fut (future
+ (try
+ (loop []
+ (let [{:keys [message]} (get-last-run transform-id)]
+ (cond
+ (.isInterrupted (Thread/currentThread))
+ nil
+
+ ;; same message as last time
+ (= message (peek @states))
+ (recur)
+
+ ;; new message value
+ :else (do (swap! states conj message)
+ (recur)))))
+ (catch InterruptedException _ nil)))]
+ (reify IDeref
+ (deref [_] @states)
+ Closeable
+ (close [_]
+ (future-cancel fut)
+ (assert (not= :timeout (try (deref fut 1000 :timeout) (catch Throwable _))) "Observation thread did not exit!")))))
+
+ (run-scenario [scenario schema]
+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO
+ transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
+ (fn [& _] (throw e))
+ @#'transforms-python.execute/transfer-file-to-db)]
+ (with-transform-cleanup! [target {:type "table"
+ :schema schema
+ :name "result"}]
+ (let [transform-id (create-transform scenario target)
+ observed-messages (with-open [observer ^Closeable (open-message-value-observer transform-id)]
+ (block-on-run scenario target transform-id)
+ @observer)
+ last-run (get-last-run transform-id)]
+ {:observed-messages observed-messages
+ :last-run last-run}))))]
+ (let [scenarios [{:desc "stdin"
+ :program ["print(\"hello, world\")"]
+ :expect-status :succeeded
+ :expected ["hello, world"]}
+ {:desc "stderr"
+ :program ["import sys" "print(\"hello, world\", file=sys.stderr)"]
+ :expect-status :succeeded
+ :expected ["hello, world"]}
+ {:desc "interleaved streams"
+ :program ["import sys"
+ "print(\"1\", file=sys.stderr)"
+ "print(\"2\", file=sys.stderr)"
+ "print(\"3\", file=sys.stdout)"
+ "print(\"4\", file=sys.stderr)"]
+ :expect-status :succeeded
+ :expected ["1" "2" "3" "4"]}
+ {:desc "syntax error"
+ :program ["print(40 + 2)"
+ "this is not valid code"]
+ :expect-status :failed
+ :expected ["SyntaxError: invalid syntax"]}
+ {:desc "takes time, early feedback possible"
+ :program ["import time"
+ "print(\"a\")"
+ "time.sleep(0.1)"
+ "print(\"b\")"
+ "time.sleep(0.1)"
+ "print(\"c\")"]
+ :expect-early-feedback true
+ :expect-status :succeeded
+ :expected ["a" "b" "c"]}
+ {:desc "crash during writeback"
+ :program ["print(42)" "print(\"is the answer\")"]
+ :expect-status :failed
+ :expected ["42" "is the answer" "Failed to create the resulting table"]
+ :writeback-ex (Exception. "Boom!")}]]
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+ (doseq [{:keys [expected
+ expect-early-feedback
+ expect-status
+ desc] :as scenario}
+ scenarios]
+ (testing desc
+ (let [{:keys [last-run observed-messages]} (run-scenario scenario schema)
+ {:keys [status message]} last-run]
+ (testing "sanity: status is what we expect"
+ (is (= expect-status (some-> status keyword))))
+ (testing "sanity: message has a value"
+ (is message))
+ (when message
+ (is (str/starts-with? message "Executing Python transform"))
+ (testing "all observed values of message reflect a prefix of the final message (ordering consistent)"
+ (doseq [observed-message observed-messages]
+ ;; hmmm, sometimes the order seems flipped
+ (is (str/starts-with? message observed-message))))
+ (when expect-early-feedback
+ (testing "scenario takes time, we should see partial messages for immediate feedback"
+ (is (< 1 (count observed-messages)))))
+ (testing "message includes the expected lines"
+ (is (subsequence? (str/split-lines message) expected))))))))))))))
+
+(deftest get-python-transform-with-different-target-database-test
+ (testing "GET /api/ee/transform/:id correctly fetches target table from different database"
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/with-temp [:model/Database target-db {:engine :h2
+ :details {:db "mem:target-db"}}
+ :model/Table target-table {:db_id (:id target-db)
+ :schema "PUBLIC"
+ :name "python_target_table"
+ :active true}
+ :model/Transform transform {:name "Python Transform Cross DB"
+ :source {:type "python"
+ :source-database (mt/id)
+ :source-tables {}
+ :body "def transform():\n pass"}
+ :target {:type "table"
+ :schema "PUBLIC"
+ :name "python_target_table"
+ :database (:id target-db)}}]
+ (let [response (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id transform)))]
+ (is (=? {:id (:id target-table)
+ :name "python_target_table"
+ :schema "PUBLIC"
+ :db_id (:id target-db)}
+ (:table response))))))))
+
+(deftest python-transform-schema-change-integration-test
+ (testing "Python transform handles schema changes using appropriate rename strategy"
+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/python)
+ ;; takes too long on CI
+ :bigquery-cloud-sdk)
+ (mt/with-premium-features #{:transforms :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)]
+ (with-transform-cleanup! [{table-name :name :as target} {:type "table"
+ :schema schema
+ :name "schema_change_test"}]
+
+ (let [initial-transform {:name "Schema Change Integration Test"
+ :source {:type "python"
+ :source-tables {}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+ :target (assoc target :database (mt/id))}
+ ;; Create initial transform via API
+ {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform" initial-transform)]
+
+ ;; Run initial transform and validate
+ (transforms.tu/test-run transform-id)
+ (transforms.tu/wait-for-table table-name 10000)
+ (let [initial-rows (transforms.tu/table-rows table-name)]
+ (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob with ages"))
+
+ ;; Update transform with different schema via API endpoint
+ (let [updated-transform (assoc initial-transform
+ :source {:type "python"
+ :source-tables {}
+ :body (str "import pandas as pd\n"
+ "\n"
+ "def transform():\n"
+ " return pd.DataFrame({'name': ['Alice', 'Bob'], 'friend': ['Bob', 'Alice']})")})
+ update-response (mt/user-http-request :crowberto :put 200 (format "ee/transform/%d" transform-id)
+ updated-transform)]
+ (is (some? update-response) "Transform update should succeed"))
+
+ ;; Run updated transform and validate schema change
+ (transforms.tu/test-run transform-id)
+ (transforms.tu/wait-for-transform-completion transform-id 10000)
+
+ ;; hmmm, looks like QP needs a bit more time to update metadata
+ (Thread/sleep 2000)
+ (let [updated-rows (transforms.tu/table-rows table-name)]
+ (is (= [["Alice" "Bob"] ["Bob" "Alice"]] updated-rows)
+ "Updated data should show Alice/Bob with friends instead of ages"))))))))))
diff --git a/enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts b/enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts
new file mode 100644
index 00000000000..c367ec2f0e4
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts
@@ -0,0 +1,33 @@
+import type {
+ GetPythonLibraryRequest,
+ PythonLibrary,
+ UpdatePythonLibraryRequest,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import { idTag, invalidateTags, providePythonLibraryTags } from "./tags";
+
+export const pythonLibraryApi = EnterpriseApi.injectEndpoints({
+ endpoints: (builder) => ({
+ getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({
+ query: ({ path }) => ({
+ url: `/api/ee/transforms-python/library/${path}`,
+ method: "GET",
+ }),
+ providesTags: (library) =>
+ library ? providePythonLibraryTags(library) : [],
+ }),
+ updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({
+ query: ({ path, source }) => ({
+ url: `/api/ee/transforms-python/library/${path}`,
+ method: "PUT",
+ body: { source },
+ }),
+ invalidatesTags: (_, error, { path }) =>
+ invalidateTags(error, [idTag("python-transform-library", path)]),
+ }),
+ }),
+});
+
+export const { useGetPythonLibraryQuery, useUpdatePythonLibraryMutation } =
+ pythonLibraryApi;
diff --git a/enterprise/frontend/src/metabase-enterprise/api/tags.ts b/enterprise/frontend/src/metabase-enterprise/api/tags.ts
index 14532d0c46e..136bbe53a3b 100644
--- a/enterprise/frontend/src/metabase-enterprise/api/tags.ts
+++ b/enterprise/frontend/src/metabase-enterprise/api/tags.ts
@@ -3,6 +3,7 @@ import type { TagDescription } from "@reduxjs/toolkit/query";
import { TAG_TYPES, provideUserTags } from "metabase/api/tags";
import type {
Comment,
+ PythonLibrary,
Transform,
TransformJob,
TransformRun,
@@ -23,6 +24,7 @@ export const ENTERPRISE_TAG_TYPES = [
"transform-job",
"transform-job-via-tag",
"transform-run",
+ "python-transform-library",
] as const;
export type EnterpriseTagType = (typeof ENTERPRISE_TAG_TYPES)[number];
@@ -126,3 +128,9 @@ export function provideCommentTags(
return [idTag("comment", comment.id)];
}
+
+export function providePythonLibraryTags(
+ library: PythonLibrary,
+): TagDescription<EnterpriseTagType>[] {
+ return [idTag("python-transform-library", library.path)];
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/api/transform-python.ts b/enterprise/frontend/src/metabase-enterprise/api/transform-python.ts
new file mode 100644
index 00000000000..cdfe5dde0ac
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/api/transform-python.ts
@@ -0,0 +1,34 @@
+import type {
+ ExecutePythonTransformRequest,
+ ExecutePythonTransformResponse,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+
+export const pythonRunnerApi = EnterpriseApi.injectEndpoints({
+ endpoints: (builder) => ({
+ executePython: builder.mutation<
+ ExecutePythonTransformResponse,
+ ExecutePythonTransformRequest
+ >({
+ query: ({ code, tables }) => ({
+ url: "/api/ee/transform/test-python",
+ method: "POST",
+ body: {
+ code,
+ tables,
+ },
+ }),
+ }),
+ cancelPython: builder.mutation<void, void>({
+ query: () => ({
+ url: "/api/ee/transform/test-python/cancel",
+ method: "POST",
+ }),
+ }),
+ }),
+ overrideExisting: true,
+});
+
+export const { useExecutePythonMutation, useCancelPythonMutation } =
+ pythonRunnerApi;
diff --git a/enterprise/frontend/src/metabase-enterprise/plugins.js b/enterprise/frontend/src/metabase-enterprise/plugins.js
index 5bfcb01cdf1..0a676e1f976 100644
--- a/enterprise/frontend/src/metabase-enterprise/plugins.js
+++ b/enterprise/frontend/src/metabase-enterprise/plugins.js
@@ -42,3 +42,4 @@ import "./table-editing";
import "./documents";
import "./semantic_search";
import "./transforms";
+import "./transforms-python";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.module.css
new file mode 100644
index 00000000000..13da74dc9fe
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.module.css
@@ -0,0 +1,4 @@
+.editor {
+ width: 100%;
+ height: 100%;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.tsx
new file mode 100644
index 00000000000..bfa7b07b872
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.tsx
@@ -0,0 +1,33 @@
+import cx from "classnames";
+
+import { CodeEditor } from "metabase/common/components/CodeEditor";
+
+import S from "./PythonEditor.module.css";
+import { completion } from "./utils";
+
+export function PythonEditor({
+ value,
+ onChange,
+ withPandasCompletions,
+ className,
+ readOnly,
+ ...rest
+}: {
+ value: string;
+ onChange?: (value: string) => void;
+ withPandasCompletions?: boolean;
+ className?: string;
+ readOnly?: boolean;
+}) {
+ return (
+ <CodeEditor
+ className={cx(S.editor, className)}
+ value={value}
+ onChange={onChange}
+ language="python"
+ extensions={withPandasCompletions ? completion : undefined}
+ readOnly={readOnly}
+ {...rest}
+ />
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/index.ts
new file mode 100644
index 00000000000..2230beee1d6
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/index.ts
@@ -0,0 +1 @@
+export * from "./PythonEditor";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.ts
new file mode 100644
index 00000000000..89064502b18
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.ts
@@ -0,0 +1,676 @@
+import {
+ type Completion,
+ type CompletionContext,
+ autocompletion,
+} from "@codemirror/autocomplete";
+import { pythonLanguage } from "@codemirror/lang-python";
+import { t } from "ttag";
+
+const pandasDefinitions = (): Record<string, Completion[]> => ({
+ pd: [
+ {
+ label: "DataFrame",
+ type: "class",
+ info: t`Two-dimensional, size-mutable, potentially heterogeneous tabular data`,
+ },
+ {
+ label: "Series",
+ type: "class",
+ info: t`One-dimensional ndarray with axis labels`,
+ },
+ {
+ label: "read_csv",
+ type: "function",
+ info: t`Read a comma-separated values (csv) file into DataFrame`,
+ },
+ {
+ label: "read_excel",
+ type: "function",
+ info: t`Read an Excel file into a pandas DataFrame`,
+ },
+ {
+ label: "read_json",
+ type: "function",
+ info: t`Convert a JSON string to pandas object`,
+ },
+ {
+ label: "read_sql",
+ type: "function",
+ info: t`Read SQL query or database table into a DataFrame`,
+ },
+ {
+ label: "read_parquet",
+ type: "function",
+ info: t`Load a parquet object from file path`,
+ },
+ {
+ label: "concat",
+ type: "function",
+ info: t`Concatenate pandas objects along a particular axis`,
+ },
+ {
+ label: "merge",
+ type: "function",
+ info: t`Merge DataFrame or named Series objects with a database-style join`,
+ },
+ {
+ label: "pivot_table",
+ type: "function",
+ info: t`Create a spreadsheet-style pivot table as a DataFrame`,
+ },
+ {
+ label: "crosstab",
+ type: "function",
+ info: t`Compute a simple cross tabulation of two (or more) factors`,
+ },
+ {
+ label: "get_dummies",
+ type: "function",
+ info: t`Convert categorical variable(s) into dummy/indicator variables`,
+ },
+ {
+ label: "date_range",
+ type: "function",
+ info: t`Return a fixed frequency DatetimeIndex`,
+ },
+ {
+ label: "to_datetime",
+ type: "function",
+ info: t`Convert argument to datetime`,
+ },
+ {
+ label: "to_numeric",
+ type: "function",
+ info: t`Convert argument to a numeric type`,
+ },
+ {
+ label: "cut",
+ type: "function",
+ info: t`Bin values into discrete intervals`,
+ },
+ {
+ label: "qcut",
+ type: "function",
+ info: t`Quantile-based discretization function`,
+ },
+ {
+ label: "isna",
+ type: "function",
+ info: t`Detect missing values for an array-like object`,
+ },
+ {
+ label: "isnull",
+ type: "function",
+ info: t`Detect missing values for an array-like object`,
+ },
+ {
+ label: "notna",
+ type: "function",
+ info: t`Detect non-missing values for an array-like object`,
+ },
+ {
+ label: "notnull",
+ type: "function",
+ info: t`Detect non-missing values for an array-like object`,
+ },
+ ],
+
+ // DataFrame methods and properties
+ DataFrame: [
+ // Viewing/Inspection
+ { label: "head", type: "method", info: "Return the first n rows" },
+ { label: "tail", type: "method", info: "Return the last n rows" },
+ {
+ label: "info",
+ type: "method",
+ info: t`Print a concise summary of a DataFrame`,
+ },
+ {
+ label: "describe",
+ type: "method",
+ info: t`Generate descriptive statistics`,
+ },
+ {
+ label: "shape",
+ type: "property",
+ info: t`Return a tuple representing the dimensionality`,
+ },
+ {
+ label: "size",
+ type: "property",
+ info: t`Return an int representing the number of elements`,
+ },
+ {
+ label: "columns",
+ type: "property",
+ info: t`The column labels of the DataFrame`,
+ },
+ {
+ label: "index",
+ type: "property",
+ info: t`The index (row labels) of the DataFrame`,
+ },
+ {
+ label: "dtypes",
+ type: "property",
+ info: t`Return the dtypes in the DataFrame`,
+ },
+ {
+ label: "values",
+ type: "property",
+ info: t`Return a Numpy representation of the DataFrame`,
+ },
+
+ // Selection/Indexing
+ {
+ label: "loc",
+ type: "property",
+ info: t`Access a group of rows and columns by label(s)`,
+ },
+ {
+ label: "iloc",
+ type: "property",
+ info: t`Purely integer-location based indexing`,
+ },
+ {
+ label: "at",
+ type: "property",
+ info: t`Access a single value for a row/column label pair`,
+ },
+ {
+ label: "iat",
+ type: "property",
+ info: t`Access a single value for a row/column pair by integer position`,
+ },
+ {
+ label: "query",
+ type: "method",
+ info: t`Query the columns of a DataFrame with a boolean expression`,
+ },
+ {
+ label: "filter",
+ type: "method",
+ info: t`Subset the dataframe rows or columns according to labels`,
+ },
+ {
+ label: "where",
+ type: "method",
+ info: t`Replace values where the condition is False`,
+ },
+ {
+ label: "mask",
+ type: "method",
+ info: t`Replace values where the condition is True`,
+ },
+
+ // Data Cleaning
+ {
+ label: "drop",
+ type: "method",
+ info: t`Drop specified labels from rows or columns`,
+ },
+ {
+ label: "drop_duplicates",
+ type: "method",
+ info: t`Return DataFrame with duplicate rows removed`,
+ },
+ { label: "dropna", type: "method", info: "Remove missing values" },
+ {
+ label: "fillna",
+ type: "method",
+ info: t`Fill NA/NaN values using the specified method`,
+ },
+ {
+ label: "replace",
+ type: "method",
+ info: t`Replace values given in to_replace with value`,
+ },
+ {
+ label: "interpolate",
+ type: "method",
+ info: t`Fill NaN values using an interpolation method`,
+ },
+ {
+ label: "duplicated",
+ type: "method",
+ info: t`Return boolean Series denoting duplicate rows`,
+ },
+ { label: "isnull", type: "method", info: "Detect missing values" },
+ { label: "isna", type: "method", info: "Detect missing values" },
+ {
+ label: "notnull",
+ type: "method",
+ info: t`Detect existing (non-missing) values`,
+ },
+ {
+ label: "notna",
+ type: "method",
+ info: t`Detect existing (non-missing) values`,
+ },
+
+ // Transformations
+ {
+ label: "sort_values",
+ type: "method",
+ info: t`Sort by the values along either axis`,
+ },
+ {
+ label: "sort_index",
+ type: "method",
+ info: t`Sort object by labels (along an axis)`,
+ },
+ {
+ label: "reset_index",
+ type: "method",
+ info: t`Reset the index, or a level of it`,
+ },
+ {
+ label: "set_index",
+ type: "method",
+ info: t`Set the DataFrame index using existing columns`,
+ },
+ {
+ label: "reindex",
+ type: "method",
+ info: t`Conform Series/DataFrame to new index`,
+ },
+ { label: "rename", type: "method", info: "Alter axes labels" },
+ {
+ label: "assign",
+ type: "method",
+ info: t`Assign new columns to a DataFrame`,
+ },
+ {
+ label: "pipe",
+ type: "method",
+ info: t`Apply chainable functions that expect Series or DataFrames`,
+ },
+
+ // Aggregation/GroupBy
+ {
+ label: "groupby",
+ type: "method",
+ info: t`Group DataFrame using a mapper or by a Series of columns`,
+ },
+ {
+ label: "agg",
+ type: "method",
+ info: t`Aggregate using one or more operations`,
+ },
+ {
+ label: "aggregate",
+ type: "method",
+ info: t`Aggregate using one or more operations`,
+ },
+ {
+ label: "apply",
+ type: "method",
+ info: t`Apply a function along an axis of the DataFrame`,
+ },
+ {
+ label: "applymap",
+ type: "method",
+ info: t`Apply a function to a Dataframe elementwise`,
+ },
+ {
+ label: "transform",
+ type: "method",
+ info: t`Call func on self producing a DataFrame with transformed values`,
+ },
+ {
+ label: "rolling",
+ type: "method",
+ info: t`Provide rolling window calculations`,
+ },
+ {
+ label: "expanding",
+ type: "method",
+ info: t`Provide expanding transformations`,
+ },
+ { label: "resample", type: "method", info: "Resample time-series data" },
+
+ // Statistical Methods
+ {
+ label: "sum",
+ type: "method",
+ info: t`Return the sum of the values for the requested axis`,
+ },
+ {
+ label: "mean",
+ type: "method",
+ info: t`Return the mean of the values for the requested axis`,
+ },
+ {
+ label: "median",
+ type: "method",
+ info: t`Return the median of the values for the requested axis`,
+ },
+ {
+ label: "mode",
+ type: "method",
+ info: t`Get the mode(s) of each element along the selected axis`,
+ },
+ {
+ label: "std",
+ type: "method",
+ info: t`Return sample standard deviation over requested axis`,
+ },
+ {
+ label: "var",
+ type: "method",
+ info: t`Return unbiased variance over requested axis`,
+ },
+ {
+ label: "min",
+ type: "method",
+ info: t`Return the minimum of the values for the requested axis`,
+ },
+ {
+ label: "max",
+ type: "method",
+ info: t`Return the maximum of the values for the requested axis`,
+ },
+ {
+ label: "count",
+ type: "method",
+ info: t`Count non-NA cells for each column or row`,
+ },
+ {
+ label: "nunique",
+ type: "method",
+ info: t`Count number of distinct elements in specified axis`,
+ },
+ {
+ label: "value_counts",
+ type: "method",
+ info: t`Return a Series containing counts of unique values`,
+ },
+ {
+ label: "corr",
+ type: "method",
+ info: t`Compute pairwise correlation of columns`,
+ },
+ {
+ label: "cov",
+ type: "method",
+ info: t`Compute pairwise covariance of columns`,
+ },
+ {
+ label: "rank",
+ type: "method",
+ info: t`Compute numerical data ranks (1 through n) along axis`,
+ },
+ {
+ label: "quantile",
+ type: "method",
+ info: t`Return values at the given quantile over requested axis`,
+ },
+ {
+ label: "clip",
+ type: "method",
+ info: t`Trim values at input threshold(s)`,
+ },
+
+ // Merging/Joining
+ {
+ label: "merge",
+ type: "method",
+ info: t`Merge DataFrame or named Series objects`,
+ },
+ {
+ label: "join",
+ type: "method",
+ info: t`Join columns of another DataFrame`,
+ },
+ {
+ label: "concat",
+ type: "method",
+ info: t`Concatenate pandas objects along a particular axis`,
+ },
+ {
+ label: "append",
+ type: "method",
+ info: t`Append rows of other to the end of caller`,
+ },
+ {
+ label: "update",
+ type: "method",
+ info: t`Modify in place using non-NA values from another DataFrame`,
+ },
+
+ // Reshaping
+ {
+ label: "pivot",
+ type: "method",
+ info: t`Return reshaped DataFrame organized by given index / column values`,
+ },
+ {
+ label: "pivot_table",
+ type: "method",
+ info: t`Create a spreadsheet-style pivot table`,
+ },
+ {
+ label: "melt",
+ type: "method",
+ info: t`Unpivot a DataFrame from wide to long format`,
+ },
+ {
+ label: "stack",
+ type: "method",
+ info: t`Stack the prescribed level(s) from columns to index`,
+ },
+ {
+ label: "unstack",
+ type: "method",
+ info: t`Pivot a level of the (necessarily hierarchical) index labels`,
+ },
+ { label: "transpose", type: "method", info: "Transpose index and columns" },
+ { label: "T", type: "property", info: "Transpose index and columns" },
+
+ // I/O
+ {
+ label: "to_csv",
+ type: "method",
+ info: t`Write object to a comma-separated values (csv) file`,
+ },
+ {
+ label: "to_excel",
+ type: "method",
+ info: t`Write object to an Excel sheet`,
+ },
+ {
+ label: "to_json",
+ type: "method",
+ info: t`Convert the object to a JSON string`,
+ },
+ {
+ label: "to_sql",
+ type: "method",
+ info: t`Write records stored in a DataFrame to a SQL database`,
+ },
+ {
+ label: "to_parquet",
+ type: "method",
+ info: t`Write a DataFrame to the binary parquet format`,
+ },
+ {
+ label: "to_dict",
+ type: "method",
+ info: t`Convert the DataFrame to a dictionary`,
+ },
+ {
+ label: "to_html",
+ type: "method",
+ info: t`Render a DataFrame as an HTML table`,
+ },
+ {
+ label: "to_string",
+ type: "method",
+ info: t`Render a DataFrame to a console-friendly tabular output`,
+ },
+ {
+ label: "to_numpy",
+ type: "method",
+ info: t`Convert the DataFrame to a NumPy array`,
+ },
+
+ // Time Series
+ {
+ label: "shift",
+ type: "method",
+ info: t`Shift index by desired number of periods`,
+ },
+ { label: "tshift", type: "method", info: "Shift the time index" },
+ {
+ label: "diff",
+ type: "method",
+ info: t`First discrete difference of element`,
+ },
+ {
+ label: "pct_change",
+ type: "method",
+ info: t`Percentage change between the current and a prior element`,
+ },
+
+ // Plotting (if matplotlib available)
+ {
+ label: "plot",
+ type: "method",
+ info: t`Make plots of Series or DataFrame`,
+ },
+ {
+ label: "hist",
+ type: "method",
+ info: t`Make a histogram of the DataFrame's columns`,
+ },
+ {
+ label: "boxplot",
+ type: "method",
+ info: t`Make a box plot of the DataFrame columns`,
+ },
+ ],
+
+ // Series methods (subset of DataFrame methods that make sense for Series)
+ Series: [
+ { label: "head", type: "method", info: "Return the first n values" },
+ { label: "tail", type: "method", info: "Return the last n values" },
+ {
+ label: "describe",
+ type: "method",
+ info: t`Generate descriptive statistics`,
+ },
+ {
+ label: "value_counts",
+ type: "method",
+ info: t`Return a Series containing counts of unique values`,
+ },
+ {
+ label: "unique",
+ type: "method",
+ info: t`Return unique values of Series object`,
+ },
+ {
+ label: "nunique",
+ type: "method",
+ info: t`Return number of unique elements in the object`,
+ },
+ { label: "sort_values", type: "method", info: "Sort by the values" },
+ {
+ label: "sort_index",
+ type: "method",
+ info: t`Sort Series by index labels`,
+ },
+ {
+ label: "reset_index",
+ type: "method",
+ info: t`Generate a new DataFrame or Series with the index reset`,
+ },
+ {
+ label: "dropna",
+ type: "method",
+ info: t`Return a new Series with missing values removed`,
+ },
+ {
+ label: "fillna",
+ type: "method",
+ info: t`Fill NA/NaN values using the specified method`,
+ },
+ {
+ label: "apply",
+ type: "method",
+ info: t`Invoke function on values of Series`,
+ },
+ {
+ label: "map",
+ type: "method",
+ info: t`Map values of Series according to input correspondence`,
+ },
+ {
+ label: "replace",
+ type: "method",
+ info: t`Replace values given in to_replace with value`,
+ },
+ {
+ label: "str",
+ type: "property",
+ info: t`Vectorized string functions for Series and Index`,
+ },
+ {
+ label: "dt",
+ type: "property",
+ info: t`Accessor object for datetimelike properties`,
+ },
+ {
+ label: "cat",
+ type: "property",
+ info: t`Accessor object for categorical properties`,
+ },
+ ],
+});
+
+function genericPandasCompletions(context: CompletionContext) {
+ const definitions = pandasDefinitions();
+
+ const word = context.matchBefore(/[\w.]*/);
+ if (!word || (word.from === word.to && !context.explicit)) {
+ return null;
+ }
+
+ const text = word.text;
+
+ // Match patterns like "pd.", "df.", etc.
+ if (text.includes(".")) {
+ const parts = text.split(".");
+ const obj = parts.at(-2);
+ const partial = parts.at(-1) ?? "";
+
+ if (!obj || !definitions[obj]) {
+ return null;
+ }
+
+ const options = definitions[obj]
+ .filter(
+ (item) =>
+ !partial ||
+ item.label.toLowerCase().startsWith(partial.toLowerCase()),
+ )
+ .map((comp) => ({
+ ...comp,
+ info: undefined,
+ boost: comp.type === "method" ? 2 : comp.type === "function" ? 1 : 0,
+ }));
+
+ return {
+ filter: false,
+ from: word.from + text.length - partial.length,
+ options,
+ };
+ }
+}
+
+export const completion = [
+ pythonLanguage.data.of({
+ autocomplete: genericPandasCompletions,
+ }),
+ autocompletion(),
+];
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/AliasInput.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/AliasInput.tsx
new file mode 100644
index 00000000000..933f49dd400
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/AliasInput.tsx
@@ -0,0 +1,65 @@
+import { useEffect, useState } from "react";
+import { t } from "ttag";
+
+import { ActionIcon, Icon, TextInput } from "metabase/ui";
+import type { Table } from "metabase-types/api";
+
+import type { TableSelection } from "../types";
+import { slugify } from "../utils";
+
+export function AliasInput({
+ table,
+ selection,
+ onChange,
+ usedAliases,
+ disabled,
+}: {
+ selection: TableSelection;
+ table?: Table;
+ onChange: (value: string) => void;
+ usedAliases: Set<string>;
+ disabled?: boolean;
+}) {
+ const [value, setValue] = useState(selection.alias);
+
+ useEffect(() => {
+ setValue(selection.alias);
+ }, [selection.alias]);
+
+ const defaultAlias = table
+ ? slugify(table.name, usedAliases, selection.alias)
+ : "";
+ const isManualAlias = selection.alias !== defaultAlias;
+ const showReset = table && isManualAlias;
+
+ function handleChange(evt: React.ChangeEvent<HTMLInputElement>) {
+ setValue(evt.target.value);
+ }
+
+ function handleBlur() {
+ onChange(value);
+ }
+
+ return (
+ <TextInput
+ w="100%"
+ value={value}
+ onChange={handleChange}
+ onBlur={handleBlur}
+ placeholder={t`Enter alias`}
+ disabled={disabled}
+ rightSection={
+ showReset && (
+ <ActionIcon
+ onClick={() => onChange(defaultAlias)}
+ aria-label={t`Reset alias to default`}
+ color="gray"
+ variant="subtle"
+ >
+ <Icon name="refresh" size={12} />
+ </ActionIcon>
+ )
+ }
+ />
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/index.ts
new file mode 100644
index 00000000000..b48a6bcf5c2
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/index.ts
@@ -0,0 +1 @@
+export * from "./AliasInput";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.module.css
new file mode 100644
index 00000000000..4825502a43b
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.module.css
@@ -0,0 +1,11 @@
+.dataPicker {
+ border-right: 1px solid var(--mb-color-border);
+ overflow: auto;
+}
+
+.databaseSelector {
+ padding: var(--mantine-spacing-sm) var(--mantine-spacing-md);
+ border: 1px solid var(--mb-color-border);
+ border-radius: var(--mantine-radius-xs);
+ cursor: pointer;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
new file mode 100644
index 00000000000..7d5595e44bc
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
@@ -0,0 +1,300 @@
+import { useState } from "react";
+import { t } from "ttag";
+
+import { hasFeature } from "metabase/admin/databases/utils";
+import {
+ skipToken,
+ useListDatabasesQuery,
+ useListTablesQuery,
+} from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { DatabaseDataSelector } from "metabase/query_builder/components/DataSelector";
+import { Box, Button, Icon, Stack, Text } from "metabase/ui";
+import type {
+ Database,
+ DatabaseId,
+ PythonTransformTableAliases,
+ Table,
+} from "metabase-types/api";
+
+import { AliasInput } from "./AliasInput";
+import S from "./PythonDataPicker.module.css";
+import { TableSelector } from "./TableSelector";
+import type { TableSelection } from "./types";
+import {
+ getInitialTableSelections,
+ isConcreteTableId,
+ selectionsToTableAliases,
+ slugify,
+} from "./utils";
+
+type PythonDataPickerProps = {
+ database?: DatabaseId;
+ tables: PythonTransformTableAliases;
+ onChange: (
+ database: number,
+ tables: PythonTransformTableAliases,
+ tableInfo: Table[],
+ ) => void;
+};
+
+export function PythonDataPicker({
+ database,
+ tables,
+ onChange,
+}: PythonDataPickerProps) {
+ const [tableSelections, setTableSelections] = useState<TableSelection[]>(
+ getInitialTableSelections(tables),
+ );
+
+ const {
+ data: databases,
+ isLoading: isLoadingDatabases,
+ error: databaseError,
+ } = useListDatabasesQuery();
+
+ const {
+ data: tablesData,
+ isLoading: isLoadingTables,
+ error: tablesError,
+ } = useListTablesQuery(
+ database
+ ? {
+ dbId: database,
+ include_hidden: false,
+ include_editable_data_model: true,
+ }
+ : skipToken,
+ );
+
+ const handleChange = (
+ database: DatabaseId | undefined,
+ selections: TableSelection[],
+ ) => {
+ if (database) {
+ const tableAliases = selectionsToTableAliases(selections);
+ onChange(database, tableAliases, tablesData ?? []);
+ }
+ };
+
+ if (databaseError || tablesError) {
+ return <LoadingAndErrorWrapper error={databaseError || tablesError} />;
+ }
+
+ if (isLoadingDatabases) {
+ return <LoadingAndErrorWrapper loading />;
+ }
+
+ const availableTables = (tablesData || []).filter(
+ (tbl: Table) => tbl.db_id === database && tbl.active,
+ );
+
+ const usedAliases = new Set(tableSelections.map((s) => s.alias));
+
+ const handleDatabaseChange = (value: string | null) => {
+ const newDatabase = value ? parseInt(value) : undefined;
+
+ if (newDatabase === database) {
+ return;
+ }
+
+ // Clear selections since they won't make sense
+ const newSelections = [
+ {
+ tableId: undefined,
+ alias: "",
+ },
+ ];
+ setTableSelections(newSelections);
+ handleChange(newDatabase, newSelections);
+ };
+
+ const handleSelectionChange = (
+ selectionIndex: number,
+ selection: TableSelection,
+ ) => {
+ const newSelections = Array.from(tableSelections);
+ newSelections[selectionIndex] = selection;
+ setTableSelections(newSelections);
+ handleChange(database, newSelections);
+ };
+
+ const handleAddTable = () => {
+ setTableSelections([
+ ...tableSelections,
+ {
+ tableId: undefined,
+ alias: "",
+ },
+ ]);
+ };
+
+ const handleRemoveTable = (selectionIndex: number) => {
+ const newSelections = Array.from(tableSelections);
+ newSelections.splice(selectionIndex, 1);
+ setTableSelections(newSelections);
+ handleChange(database, newSelections);
+ };
+
+ return (
+ <Stack
+ p="md"
+ gap="md"
+ h="100%"
+ w="30%"
+ miw="300px"
+ className={S.dataPicker}
+ data-testid="python-data-picker"
+ >
+ <Box>
+ <Text fw="bold">{t`Source database`}</Text>
+ <Text size="sm" c="text-light" mb="sm">
+ {t`Select the database that contains your source data.`}
+ </Text>
+
+ <DatabaseDataSelector
+ className={S.databaseSelector}
+ selectedDatabaseId={database}
+ setDatabaseFn={handleDatabaseChange}
+ databases={databases?.data ?? []}
+ databaseIsDisabled={(database: Database) =>
+ !hasFeature(database, "transforms/python")
+ }
+ />
+ </Box>
+ {database && (
+ <Box>
+ <Text fw="bold">{t`Pick tables and alias them`}</Text>
+ <Text size="sm" c="text-light" mb="sm">
+ {t`Select tables to use as data sources and provide aliases that can be referenced in your Python script.`}
+ </Text>
+ <Stack gap="md">
+ {tableSelections.map((selection, index) => (
+ <SelectionInput
+ key={index}
+ selection={selection}
+ database={database}
+ tables={tables}
+ usedAliases={usedAliases}
+ availableTables={availableTables}
+ onChange={(selection) =>
+ handleSelectionChange(index, selection)
+ }
+ onRemove={() => handleRemoveTable(index)}
+ disabled={isLoadingTables}
+ />
+ ))}
+ <AddTableButton
+ onClick={handleAddTable}
+ disabled={availableTables.length === 0}
+ />
+ </Stack>
+ </Box>
+ )}
+ </Stack>
+ );
+}
+
+function SelectionInput({
+ database,
+ tables,
+ selection,
+ availableTables,
+ usedAliases,
+ onChange,
+ onRemove,
+ disabled,
+}: {
+ database: DatabaseId | undefined;
+ tables: PythonTransformTableAliases;
+ selection: TableSelection;
+ availableTables: Table[];
+ usedAliases: Set<string>;
+ onChange: (selection: TableSelection) => void;
+ onRemove: () => void;
+ disabled?: boolean;
+}) {
+ const table = availableTables.find((table) => table.id === selection.tableId);
+
+ function handleAliasChange(newAlias: string) {
+ const newSelection = {
+ ...selection,
+ alias: slugify(newAlias, usedAliases, selection.alias),
+ };
+
+ onChange(newSelection);
+ }
+
+ function handleTableChange(table: Table | undefined) {
+ if (!table || !isConcreteTableId(table.id)) {
+ onChange({
+ ...selection,
+ tableId: undefined,
+ });
+ return;
+ }
+
+ const oldTable = availableTables.find(
+ (table) => table.id === selection?.tableId,
+ );
+
+ const wasOldAliasManuallySet =
+ selection.alias !== "" &&
+ (!oldTable ||
+ selection.alias !==
+ slugify(oldTable.name, usedAliases, selection.alias));
+
+ const newAlias = wasOldAliasManuallySet
+ ? selection.alias
+ : slugify(table.name, usedAliases);
+
+ onChange({
+ ...selection,
+ tableId: table.id,
+ alias: newAlias,
+ });
+ }
+
+ return (
+ <Stack gap="xs" align="center" w="100%">
+ <TableSelector
+ database={database}
+ table={table}
+ selectedTableIds={Object.values(tables)}
+ onChange={handleTableChange}
+ onRemove={onRemove}
+ availableTables={availableTables}
+ disabled={disabled}
+ />
+ <AliasInput
+ selection={selection}
+ table={table}
+ onChange={handleAliasChange}
+ usedAliases={usedAliases}
+ disabled={disabled}
+ />
+ </Stack>
+ );
+}
+
+function AddTableButton({
+ onClick,
+ disabled,
+}: {
+ disabled?: boolean;
+ onClick: () => void;
+}) {
+ return (
+ <Box>
+ <Button
+ leftSection={<Icon name="add_data" />}
+ variant="subtle"
+ px={0}
+ onClick={onClick}
+ disabled={disabled}
+ >
+ {t`Add a table`}
+ </Button>
+ </Box>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.module.css
new file mode 100644
index 00000000000..ef95488c4d7
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.module.css
@@ -0,0 +1,7 @@
+.tableSelector {
+ border: 1px solid var(--border-color);
+}
+
+.tableSelectorButtonInner {
+ justify-content: start;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.tsx
new file mode 100644
index 00000000000..ebd48a3008f
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.tsx
@@ -0,0 +1,130 @@
+import { useDisclosure } from "@mantine/hooks";
+import { t } from "ttag";
+
+import type { CollectionPickerItem } from "metabase/common/components/Pickers/CollectionPicker";
+import {
+ type DataPickerItem,
+ DataPickerModal,
+} from "metabase/common/components/Pickers/DataPicker";
+import {
+ ActionIcon,
+ Box,
+ Button,
+ Group,
+ Icon,
+ Stack,
+ Tooltip,
+} from "metabase/ui";
+import type {
+ ConcreteTableId,
+ DatabaseId,
+ RecentItem,
+ Table,
+ TableId,
+} from "metabase-types/api";
+
+import { isConcreteTableId } from "../utils";
+
+import S from "./TableSelector.module.css";
+
+export function TableSelector({
+ database,
+ availableTables,
+ selectedTableIds,
+ disabled,
+ onChange,
+ onRemove,
+ table,
+}: {
+ database: DatabaseId | undefined;
+ table: Table | undefined;
+ availableTables: Table[];
+ selectedTableIds: ConcreteTableId[];
+ disabled?: boolean;
+ onChange: (table: Table | undefined) => void;
+ onRemove: () => void;
+}) {
+ const [isOpened, { open, close }] = useDisclosure();
+
+ function handleChange(tableId: TableId | undefined) {
+ const table = availableTables.find((table) => table.id === tableId);
+ if (table) {
+ onChange(table);
+ }
+ }
+
+ function shouldDisableItem(
+ item: DataPickerItem | CollectionPickerItem | RecentItem,
+ ) {
+ if (item.model === "table") {
+ // Filter available tables to exclude already selected ones (except current selection)
+ return !isConcreteTableId(item.id) || selectedTableIds.includes(item.id);
+ }
+ if (item.model === "database") {
+ return item.id !== database;
+ }
+ return true;
+ }
+
+ return (
+ <>
+ <Group w="100%" bdrs="xs" gap="xs" className={S.tableSelector}>
+ <Button
+ flex="1 1 auto"
+ onClick={open}
+ disabled={disabled}
+ classNames={{ inner: S.tableSelectorButtonInner }}
+ px="sm"
+ py="lg"
+ variant="subtle"
+ >
+ <Stack gap={0} align="start" justify="center">
+ {table ? (
+ <>
+ <Box fz="sm" c="text-medium" fw="normal">
+ {table?.db?.name} / {table?.schema}
+ </Box>
+ <Box c="text-dark">{table?.display_name}</Box>
+ </>
+ ) : (
+ <Box c="text-dark">{t`Select a table…`}</Box>
+ )}
+ </Stack>
+ </Button>
+
+ <Tooltip label={t`Remove this table`}>
+ <ActionIcon
+ onClick={onRemove}
+ pr="sm"
+ aria-label={t`Remove this table`}
+ >
+ <Icon name="close" c="text-dark" />
+ </ActionIcon>
+ </Tooltip>
+ </Group>
+ {isOpened && (
+ <DataPickerModal
+ title={t`Pick a table`}
+ value={getDataPickerValue(table)}
+ databaseId={database}
+ onChange={handleChange}
+ onClose={close}
+ shouldDisableItem={shouldDisableItem}
+ />
+ )}
+ </>
+ );
+}
+
+function getDataPickerValue(table: Table | undefined) {
+ if (!table) {
+ return undefined;
+ }
+ return {
+ model: "table" as const,
+ id: table.id,
+ name: table.display_name,
+ db_id: table.db_id,
+ schema: table.schema,
+ };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/index.ts
new file mode 100644
index 00000000000..992eb097ecc
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/index.ts
@@ -0,0 +1 @@
+export * from "./TableSelector";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/index.ts
new file mode 100644
index 00000000000..37019632951
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/index.ts
@@ -0,0 +1 @@
+export * from "./PythonDataPicker";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/types.ts
new file mode 100644
index 00000000000..2182b903c8f
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/types.ts
@@ -0,0 +1,6 @@
+import type { ConcreteTableId } from "metabase-types/api";
+
+export type TableSelection = {
+ tableId: ConcreteTableId | undefined;
+ alias: string;
+};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
new file mode 100644
index 00000000000..4f1c7a670dc
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
@@ -0,0 +1,71 @@
+import { slugify as toSlug } from "metabase/lib/formatting";
+import type {
+ ConcreteTableId,
+ PythonTransformTableAliases,
+ TableId,
+} from "metabase-types/api";
+
+import type { TableSelection } from "./types";
+
+export function getInitialTableSelections(
+ tables: PythonTransformTableAliases | undefined,
+) {
+ if (tables && Object.keys(tables).length > 0) {
+ return Object.entries(tables).map(([alias, tableId]) => ({
+ tableId,
+ alias,
+ }));
+ }
+ // Start with one empty row by default
+ return [
+ {
+ tableId: undefined,
+ alias: "",
+ },
+ ];
+}
+
+export function selectionsToTableAliases(
+ selections: TableSelection[],
+): PythonTransformTableAliases {
+ const tableAliases: PythonTransformTableAliases = {};
+
+ for (const selection of selections) {
+ const { alias, tableId } = selection;
+ if (tableId !== undefined && alias !== "") {
+ tableAliases[alias] = tableId;
+ }
+ }
+ return tableAliases;
+}
+
+export function slugify(
+ name: string,
+ usedNames: Set<string> = new Set(),
+ allowThisAliasIfItExists?: string,
+) {
+ if (name === "") {
+ return "";
+ }
+
+ const plain = toSlug(name).replace(/\./, "_");
+ if (plain === allowThisAliasIfItExists) {
+ return plain;
+ }
+ if (!usedNames.has(plain)) {
+ return plain;
+ }
+ for (let index = 1; index <= 100; index++) {
+ const alias = `${plain}_${index}`;
+ if (!usedNames.has(alias)) {
+ return alias;
+ }
+ }
+ return name;
+}
+
+export function isConcreteTableId(
+ id: TableId | undefined,
+): id is ConcreteTableId {
+ return typeof id === "number";
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.module.css
new file mode 100644
index 00000000000..d0e2d81f9c8
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.module.css
@@ -0,0 +1,10 @@
+.root {
+ border-bottom: 1px solid var(--mb-color-border);
+ position: relative;
+}
+
+.libraryActions {
+ position: absolute;
+ top: 0;
+ right: 0;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.tsx
new file mode 100644
index 00000000000..1c1ba2f82d2
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.tsx
@@ -0,0 +1,146 @@
+import type { ReactNode } from "react";
+import { ResizableBox } from "react-resizable";
+import { t } from "ttag";
+
+import Link from "metabase/common/components/Link";
+import RunButtonWithTooltip from "metabase/query_builder/components/RunButtonWithTooltip";
+import { Box, Checkbox, Flex, Icon, Stack } from "metabase/ui";
+import { SHARED_LIB_IMPORT_PATH } from "metabase-enterprise/transforms-python/constants";
+import { getPythonLibraryUrl } from "metabase-enterprise/transforms-python/urls";
+
+import { PythonEditor } from "../../PythonEditor";
+
+import S from "./PythonEditorBody.module.css";
+import { ResizableBoxHandle } from "./ResizableBoxHandle";
+import { hasImport, insertImport, removeImport } from "./utils";
+
+type PythonEditorBodyProps = {
+ source: string;
+ isRunnable: boolean;
+ onChange: (source: string) => void;
+ onRun?: () => void;
+ onCancel?: () => void;
+ isRunning?: boolean;
+ isDirty?: boolean;
+ tables?: Record<string, number>;
+ withDebugger?: boolean;
+};
+
+const EDITOR_HEIGHT = 400;
+
+export function PythonEditorBody({
+ source,
+ onChange,
+ isRunnable,
+ onRun,
+ onCancel,
+ isRunning,
+ isDirty,
+ withDebugger,
+}: PythonEditorBodyProps) {
+ return (
+ <MaybeResizableBox resizable={withDebugger}>
+ <Flex h="100%" align="end" bg="bg-light" pos="relative">
+ <PythonEditor
+ value={source}
+ onChange={onChange}
+ withPandasCompletions
+ data-testid="python-editor"
+ />
+
+ {withDebugger && (
+ <Box p="md">
+ <RunButtonWithTooltip
+ disabled={!isRunnable}
+ isRunning={isRunning}
+ isDirty={isDirty}
+ onRun={onRun}
+ onCancel={onCancel}
+ getTooltip={() => t`Run Python script`}
+ />
+ </Box>
+ )}
+ <SharedLibraryActions source={source} onChange={onChange} />
+ </Flex>
+ </MaybeResizableBox>
+ );
+}
+
+function MaybeResizableBox({
+ resizable,
+ children,
+}: {
+ resizable?: boolean;
+ children?: ReactNode;
+}) {
+ if (!resizable) {
+ return <Box h="100%">{children}</Box>;
+ }
+
+ return (
+ <ResizableBox
+ axis="y"
+ height={EDITOR_HEIGHT}
+ handle={<ResizableBoxHandle />}
+ resizeHandles={["s"]}
+ >
+ {children}
+ </ResizableBox>
+ );
+}
+
+function SharedLibraryActions({
+ source,
+ onChange,
+}: {
+ source: string;
+ onChange: (source: string) => void;
+}) {
+ return (
+ <Stack className={S.libraryActions} p="md" gap="sm">
+ <SharedLibraryImportToggle source={source} onChange={onChange} />
+ <SharedLibraryEditLink />
+ </Stack>
+ );
+}
+
+function SharedLibraryImportToggle({
+ source,
+ onChange,
+}: {
+ source: string;
+ onChange: (source: string) => void;
+}) {
+ const hasSharedLib = hasImport(source, SHARED_LIB_IMPORT_PATH);
+
+ function handleToggleSharedLib() {
+ if (hasImport(source, SHARED_LIB_IMPORT_PATH)) {
+ onChange(removeImport(source, SHARED_LIB_IMPORT_PATH));
+ } else {
+ onChange(insertImport(source, SHARED_LIB_IMPORT_PATH));
+ }
+ }
+
+ return (
+ <Checkbox
+ label={t`Import common library`}
+ checked={hasSharedLib}
+ onChange={handleToggleSharedLib}
+ size="sm"
+ />
+ );
+}
+
+function SharedLibraryEditLink() {
+ return (
+ <Flex
+ component={Link}
+ target="_blank"
+ to={getPythonLibraryUrl({ path: SHARED_LIB_IMPORT_PATH })}
+ gap="sm"
+ >
+ <Icon name="pencil" />
+ {t`Edit common library`}
+ </Flex>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.module.css
new file mode 100644
index 00000000000..ac90231539b
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.module.css
@@ -0,0 +1,3 @@
+.root {
+ cursor: row-resize;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.tsx
new file mode 100644
index 00000000000..8020e0504fc
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.tsx
@@ -0,0 +1,27 @@
+import { type HTMLAttributes, type Ref, forwardRef } from "react";
+import _ from "underscore";
+
+import { Box, Flex, rem } from "metabase/ui";
+
+import S from "./ResizableBoxHandle.module.css";
+
+export const ResizableBoxHandle = forwardRef(function ResizableBoxHandle(
+ props: HTMLAttributes<HTMLDivElement>,
+ ref: Ref<HTMLDivElement>,
+) {
+ return (
+ <Flex
+ ref={ref}
+ className={S.root}
+ align="center"
+ justify="center"
+ w="100%"
+ h="sm"
+ pos="absolute"
+ bottom={rem(-4)}
+ {..._.omit(props, "handleAxis")}
+ >
+ <Box w="6.25rem" h="xs" bg="border" />
+ </Flex>
+ );
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/index.ts
new file mode 100644
index 00000000000..3141dc8e2ba
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/index.ts
@@ -0,0 +1 @@
+export * from "./ResizableBoxHandle";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/index.ts
new file mode 100644
index 00000000000..1f7cef4b142
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/index.ts
@@ -0,0 +1 @@
+export * from "./PythonEditorBody";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/utils.ts
new file mode 100644
index 00000000000..577de6df205
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/utils.ts
@@ -0,0 +1,33 @@
+export function insertImport(source: string, path: string) {
+ const lines = source.split("\n");
+
+ // Find the first line that is not a comment
+ for (let i = 0; i < lines.length; i++) {
+ const line = lines[i];
+ if (line.trim().startsWith("#")) {
+ continue;
+ }
+ lines.splice(i, 0, `import ${pathToImportSpecifier(path)}`);
+ break;
+ }
+
+ return lines.join("\n");
+}
+
+function pathToImportSpecifier(path: string) {
+ return path.replace(/\.py$/, "").split("/").join(".");
+}
+
+function libImportRegex(path: string) {
+ return new RegExp(`import ${pathToImportSpecifier(path)}`, "g");
+}
+
+export function removeImport(source: string, path: string) {
+ const lines = source.split("\n");
+ return lines.filter((line) => !libImportRegex(path).test(line)).join("\n");
+}
+
+export function hasImport(source: string, path: string) {
+ const regex = libImportRegex(path);
+ return regex.test(source);
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/ExecutionOutputTable.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/ExecutionOutputTable.tsx
new file mode 100644
index 00000000000..035c567ef4c
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/ExecutionOutputTable.tsx
@@ -0,0 +1,25 @@
+import { useMemo } from "react";
+
+import { DataGrid, useDataGridInstance } from "metabase/data-grid";
+
+import { type Row, parseOutput } from "./utils";
+
+export function ExecutionOutputTable({ output }: { output?: string }) {
+ const { headers, rows } = useMemo(() => parseOutput(output ?? ""), [output]);
+
+ const tableProps = useDataGridInstance<Row, unknown>({
+ data: rows,
+ columnsOptions: headers.map((header) => ({
+ id: header,
+ name: header,
+ accessorFn: (row) => row[header],
+ })),
+ minGridWidth: 100,
+ });
+
+ if (!output || headers.length === 0) {
+ return null;
+ }
+
+ return <DataGrid {...tableProps} />;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.module.css
new file mode 100644
index 00000000000..6553025e1fe
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.module.css
@@ -0,0 +1,56 @@
+.logs {
+ white-space: pre;
+ border: 1px solid var(--mb-color-border);
+ font-family: var(--mb-default-monospace-font-family);
+}
+
+.visualization {
+ flex: 1 0 auto;
+}
+
+.results {
+ width: 100%;
+ border-collapse: collapse;
+ border: 1px solid var(--mb-color-border);
+ border-radius: var(--mantine-radius-md);
+}
+
+.empty {
+ border-top: 1px solid var(--mb-color-border);
+}
+
+.tableHeader {
+ background-color: var(--mb-color-bg-light);
+ padding: var(--mantine-spacing-sm);
+}
+
+.tableCell {
+ border-top: 1px solid var(--mb-color-border);
+ padding: var(--mantine-spacing-sm);
+}
+
+.error {
+ color: var(--mb-color-error);
+ background-color: var(--mb-color-bg-error);
+ border-bottom: 1px solid var(--mb-color-border);
+}
+
+.info {
+ color: var(--mb-color-text-dark);
+ border: 1px solid var(--mb-color-bg-medium);
+ background-color: var(--mb-color-bg-light);
+}
+
+.success {
+ color: var(--mb-color-success);
+ border-bottom: 1px solid var(--mb-color-border);
+}
+
+.loading {
+ position: absolute;
+ top: 0;
+ left: 0;
+ bottom: 0;
+ right: 0;
+ background-color: rgba(255, 255, 255, 0.5);
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.tsx
new file mode 100644
index 00000000000..94f4f9034b5
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.tsx
@@ -0,0 +1,175 @@
+import type { ReactNode } from "react";
+import { c, t } from "ttag";
+
+import EmptyCodeResult from "assets/img/empty-states/code.svg";
+import DebouncedFrame from "metabase/common/components/DebouncedFrame";
+import { LoadingSpinner } from "metabase/common/components/MetadataInfo/MetadataInfo.styled";
+import { isMac } from "metabase/lib/browser";
+import { Box, Flex, Icon, Stack, Text, Title } from "metabase/ui";
+
+import type { ExecutionResult } from "../utils";
+
+import { ExecutionOutputTable } from "./ExecutionOutputTable";
+import S from "./PythonEditorResults.module.css";
+
+type PythonEditorProps = {
+ isRunning?: boolean;
+ executionResult?: ExecutionResult | null;
+};
+
+export function PythonEditorResults({
+ executionResult,
+ isRunning,
+}: PythonEditorProps) {
+ return (
+ <DebouncedFrame className={S.visualization}>
+ <ExecutionResultHeader executionResult={executionResult} />
+ <ExecutionResults executionResult={executionResult} />
+ {isRunning && <LoadingState />}
+ </DebouncedFrame>
+ );
+}
+
+function ExecutionResultHeader({
+ executionResult,
+}: {
+ executionResult?: ExecutionResult | null;
+}) {
+ const message = getMessageForExecutionResult(executionResult);
+ return <Flex w="100%">{message}</Flex>;
+}
+
+function getRunQueryShortcut() {
+ return isMac() ? t`⌘ + return` : t`Ctrl + enter`;
+}
+
+function LoadingState() {
+ return (
+ <Flex p="md" className={S.loading}>
+ <LoadingSpinner />
+ </Flex>
+ );
+}
+
+function EmptyState() {
+ const keyboardShortcut = getRunQueryShortcut();
+
+ return (
+ <Flex h="100%" align="center" justify="center" className={S.empty}>
+ <Stack maw="25rem" gap={0} ta="center" align="center">
+ <Box maw="3rem" mb="0.75rem">
+ <img src={EmptyCodeResult} alt="Code prompt icon" />
+ </Box>
+ <Text c="text-medium">
+ {c("{0} refers to the keyboard shortcut")
+ .jt`To run your code, click on the Run button or type ${(
+ <b key="shortcut">({keyboardShortcut})</b>
+ )}`}
+ </Text>
+ </Stack>
+ </Flex>
+ );
+}
+
+function ExecutionResults({
+ executionResult,
+}: {
+ executionResult?: ExecutionResult | null;
+}) {
+ if (!executionResult) {
+ return <EmptyState />;
+ }
+
+ return (
+ <Stack gap={0}>
+ <ExecutionLogs
+ label={t`Standard output:`}
+ content={executionResult.stdout}
+ />
+ <ExecutionLogs
+ label={t`Standard error:`}
+ content={executionResult.stderr}
+ />
+
+ <ExecutionOutputTable output={executionResult.output} />
+ </Stack>
+ );
+}
+
+function getMessageForExecutionResult(
+ executionResult?: ExecutionResult | null,
+) {
+ if (!executionResult) {
+ return null;
+ }
+ if (executionResult.error) {
+ return (
+ <Box className={S.error} p="sm" w="100%">
+ <Flex gap="sm">
+ <Icon name="warning" />
+ <Stack gap={0}>
+ <Title order={5} mb="xs">
+ {t`An error occurred while executing your Python script`}
+ </Title>
+ {executionResult.error}
+ </Stack>
+ </Flex>
+ </Box>
+ );
+ }
+
+ if (!executionResult.output) {
+ return (
+ <Flex className={S.info} p="sm" gap="sm" align="center" w="100%">
+ <Icon name="info" />
+ {t`No results to display.`}
+ </Flex>
+ );
+ }
+
+ return (
+ <Flex className={S.success} p="sm" gap="sm" align="center" w="100%">
+ <Icon name="check" />
+ {t`Script executed successfully.`}
+ </Flex>
+ );
+}
+
+function ExecutionLogs({
+ label,
+ content,
+}: {
+ label: string;
+ content?: string | null;
+}) {
+ if (!content) {
+ return null;
+ }
+
+ return (
+ <Section title={label}>
+ <Box p="sm" bg="bg-light" mah="150px" bdrs="xs" className={S.logs}>
+ {content}
+ </Box>
+ </Section>
+ );
+}
+
+function Section({
+ title,
+ children,
+}: {
+ title: ReactNode;
+ children: ReactNode;
+}) {
+ return (
+ <Box>
+ {title && (
+ <Title order={5} mb="xs">
+ {title}
+ </Title>
+ )}
+ {children}
+ </Box>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/index.ts
new file mode 100644
index 00000000000..b4bca07a82b
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/index.ts
@@ -0,0 +1 @@
+export * from "./PythonEditorResults";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/utils.ts
new file mode 100644
index 00000000000..fb578e81853
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/utils.ts
@@ -0,0 +1,38 @@
+import type { RowValue } from "metabase-types/api";
+
+export type Row = Record<string, RowValue>;
+
+// Parses the output of a Python transform (given as JSON-lines)
+// into a set of headers and rows.
+//
+// TODO: The server should return a non-stringified JSON array of rows,
+// not a string.
+export function parseOutput(output: string): {
+ headers: string[];
+ rows: Row[];
+} {
+ if (!output?.trim()) {
+ return { headers: [], rows: [] };
+ }
+
+ const lines = output.trim().split("\n");
+ const headers = new Set<string>();
+ const rows: Row[] = [];
+
+ for (const line of lines) {
+ try {
+ const data = JSON.parse(line) as Row;
+ for (const key in data) {
+ headers.add(key);
+ }
+ rows.push(data);
+ } catch (err) {
+ // noop
+ }
+ }
+
+ return {
+ headers: Array.from(headers),
+ rows,
+ };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx
new file mode 100644
index 00000000000..567229fea9f
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx
@@ -0,0 +1,150 @@
+import { useHotkeys } from "@mantine/hooks";
+import { useState } from "react";
+
+import { Flex, Stack } from "metabase/ui";
+import { EditorHeader } from "metabase-enterprise/transforms/components/QueryEditor/EditorHeader";
+import type {
+ DatabaseId,
+ PythonTransformSource,
+ PythonTransformTableAliases,
+ Table,
+} from "metabase-types/api";
+
+import { PythonDataPicker } from "./PythonDataPicker";
+import { PythonEditorBody } from "./PythonEditorBody";
+import { PythonEditorResults } from "./PythonEditorResults";
+import {
+ getValidationResult,
+ isPythonTransformSource,
+ updateTransformSignature,
+ useShouldShowPythonDebugger,
+ useTestPythonTransform,
+} from "./utils";
+
+export type PythonTransformSourceDraft = {
+ type: "python";
+ body: string;
+ "source-database": DatabaseId | undefined;
+ "source-tables": PythonTransformTableAliases;
+};
+
+export type PythonTransformEditorProps = {
+ initialSource: PythonTransformSourceDraft;
+ isNew?: boolean;
+ isSaving?: boolean;
+ isRunnable?: boolean;
+ onSave: (newSource: PythonTransformSource) => void;
+ onCancel: () => void;
+};
+
+export function PythonTransformEditor({
+ initialSource,
+ isNew = true,
+ isSaving = false,
+ isRunnable = true,
+ onSave,
+ onCancel,
+}: PythonTransformEditorProps) {
+ const [source, setSource] = useState(initialSource);
+ const [isSourceDirty, setIsSourceDirty] = useState(false);
+
+ const { isRunning, isDirty, cancel, run, executionResult } =
+ useTestPythonTransform(source);
+
+ const handleScriptChange = (body: string) => {
+ const newSource = {
+ ...source,
+ body,
+ };
+ setSource(newSource);
+ setIsSourceDirty(true);
+ };
+
+ const handleDataChange = (
+ database: number,
+ sourceTables: PythonTransformTableAliases,
+ tableInfo: Table[],
+ ) => {
+ const updatedScript = updateTransformSignature(
+ source.body,
+ sourceTables,
+ tableInfo,
+ );
+
+ const newSource = {
+ ...source,
+ body: updatedScript,
+ "source-database": database,
+ "source-tables": sourceTables,
+ };
+ setSource(newSource);
+ setIsSourceDirty(true);
+ };
+
+ const handleSave = () => {
+ if (isPythonTransformSource(source)) {
+ onSave(source);
+ }
+ };
+
+ const showDebugger = useShouldShowPythonDebugger();
+
+ const handleCmdEnter = () => {
+ if (!showDebugger) {
+ return;
+ }
+ if (isRunning) {
+ cancel();
+ } else if (isRunnable && isPythonTransformSource(source)) {
+ run();
+ }
+ };
+
+ useHotkeys([["mod+Enter", handleCmdEnter]], []);
+
+ const validationResult = getValidationResult(source);
+
+ return (
+ <Stack
+ w="100%"
+ h="100%"
+ bg="bg-white"
+ data-testid="transform-query-editor"
+ gap={0}
+ >
+ <EditorHeader
+ isNew={isNew}
+ isSaving={isSaving}
+ onSave={handleSave}
+ onCancel={onCancel}
+ validationResult={validationResult}
+ isQueryDirty={isSourceDirty}
+ />
+ <Flex h="100%" w="100%">
+ <PythonDataPicker
+ database={source["source-database"]}
+ tables={source["source-tables"]}
+ onChange={handleDataChange}
+ />
+ <Stack w="100%" h="100%" gap={0}>
+ <PythonEditorBody
+ isRunnable={isRunnable && isPythonTransformSource(source)}
+ isRunning={isRunning}
+ isDirty={isDirty}
+ onRun={run}
+ onCancel={cancel}
+ source={source.body}
+ onChange={handleScriptChange}
+ withDebugger={showDebugger}
+ />
+ {showDebugger && (
+ <PythonEditorResults
+ isRunning={isRunning}
+ executionResult={executionResult}
+ />
+ )}
+ </Stack>
+ </Flex>
+ </Stack>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/index.ts
new file mode 100644
index 00000000000..6f64105baa1
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/index.ts
@@ -0,0 +1 @@
+export * from "./PythonTransformEditor";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts
new file mode 100644
index 00000000000..95ebe40a3f5
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts
@@ -0,0 +1,249 @@
+import { useRef, useState } from "react";
+import { useLocation } from "react-use";
+import { t } from "ttag";
+
+import { getErrorMessage } from "metabase/api/utils";
+import { useExecutePythonMutation } from "metabase-enterprise/api/transform-python";
+import type {
+ ExecutePythonTransformResponse,
+ PythonTransformSource,
+ PythonTransformTableAliases,
+ Table,
+ TableId,
+} from "metabase-types/api";
+
+import type { PythonTransformSourceDraft } from "./PythonTransformEditor";
+
+export function updateTransformSignature(
+ script: string,
+ tables: PythonTransformTableAliases,
+ tableInfo: Table[],
+): string {
+ const tableAliases = Object.keys(tables);
+
+ const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
+
+ const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;
+ const newSignature =
+ signatureOneLine.length > 80 && tableAliases.length > 0
+ ? `def transform(\n${tableAliases.map((alias) => ` ${alias}`).join(",\n")},\n):`
+ : signatureOneLine;
+
+ function getTableName(tableId: TableId) {
+ const table = tableInfo.find((t) => t.id === tableId);
+ if (!table) {
+ return undefined;
+ }
+ return [table.db?.name, table.schema, table.name].filter(Boolean).join(".");
+ }
+
+ if (transformRegex.test(script)) {
+ // Capture the existing indentation after the colon
+ const match = script.match(transformRegex);
+ const originalIndent = match ? match[1] : " ";
+
+ let updatedScript = script.replace(
+ transformRegex,
+ newSignature + "\n" + originalIndent,
+ );
+
+ const argsRegex =
+ /(\s+"""[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+ const argsMatch = updatedScript.match(argsRegex);
+
+ if (tableAliases.length > 0) {
+ const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));
+ const newArgsSection = tableAliases
+ .map((alias) => {
+ const padding = " ".repeat(maxAliasLength - alias.length);
+ const tableId = tables[alias];
+ const tableName = getTableName(tableId);
+ return ` ${alias}:${padding} DataFrame containing the data from the "${tableName}" table`;
+ })
+ .join("\n");
+
+ if (argsMatch) {
+ // Replace existing Args section
+ updatedScript = updatedScript.replace(
+ argsRegex,
+ ` $1
+
+ Args:
+${newArgsSection}
+$3`,
+ );
+ } else {
+ // No Args section exists, look for the pattern before Returns and insert Args
+ // Also remove the instructional text about selecting tables
+ const insertRegex =
+ /(\s+"""[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;
+ const insertMatch = updatedScript.match(insertRegex);
+ if (insertMatch) {
+ updatedScript = updatedScript.replace(
+ insertRegex,
+ `$1
+
+ Args:
+${newArgsSection}
+$2`,
+ );
+ } else {
+ // Fallback: just look for Returns section without the specific text
+ // Remove any extra whitespace before Returns and ensure single blank line
+ const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+ const fallbackMatch = updatedScript.match(fallbackRegex);
+ if (fallbackMatch) {
+ updatedScript = updatedScript.replace(
+ fallbackRegex,
+ `$1
+
+ Args:
+${newArgsSection}
+$2`,
+ );
+ }
+ }
+ }
+ } else if (argsMatch) {
+ // Remove existing Args section when no tables
+ updatedScript = updatedScript.replace(argsRegex, "$1$3");
+ }
+
+ return updatedScript;
+ }
+
+ const maxAliasLength =
+ tableAliases.length > 0
+ ? Math.max(...tableAliases.map((a) => a.length))
+ : 0;
+ const functionTemplate = `
+${newSignature}
+ """
+ Transform function that processes the input data.
+ ${
+ tableAliases.length > 0
+ ? `
+
+ Args:
+${tableAliases
+ .map((alias) => {
+ const padding = " ".repeat(maxAliasLength - alias.length);
+ const tableId = tables[alias];
+ const tableName = getTableName(tableId);
+ if (alias === tableName) {
+ return ` ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+ }
+ return ` ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+ })
+ .join("\n")}`
+ : ""
+ }
+
+ Returns:
+ DataFrame: The transformed data
+ """
+ import pandas as pd
+ return pd.DataFrame()
+`;
+
+ return script + functionTemplate;
+}
+
+export type ExecutionResult = {
+ output?: string;
+ stdout?: string;
+ stderr?: string;
+ error?: string;
+};
+
+type TestPythonScriptState = {
+ isRunning: boolean;
+ isDirty: boolean;
+ executionResult: ExecutionResult | null;
+ run: () => void;
+ cancel: () => void;
+};
+
+export function useTestPythonTransform(
+ source: PythonTransformSourceDraft,
+): TestPythonScriptState {
+ const [executePython, { isLoading: isRunning, originalArgs }] =
+ useExecutePythonMutation();
+ const abort = useRef<(() => void) | null>(null);
+ const [executionResult, setData] =
+ useState<ExecutePythonTransformResponse | null>(null);
+
+ const isDirty = originalArgs?.code !== source.body;
+
+ const run = async () => {
+ if (source["source-database"] === undefined) {
+ return null;
+ }
+ const request = executePython({
+ code: source.body,
+ tables: source["source-tables"],
+ });
+ abort.current = () => request.abort();
+
+ try {
+ const data = await request.unwrap();
+ setData(data);
+ } catch (error) {
+ if (typeof error === "object" && error !== null) {
+ if ("name" in error && error.name === "AbortError") {
+ setData({ error: t`Python script execution was canceled` });
+ return;
+ }
+ }
+
+ const errorMessage = getErrorMessage(error, t`An unknown error occurred`);
+ setData({ error: errorMessage });
+ }
+ };
+
+ const cancel = () => {
+ abort.current?.();
+ };
+
+ return {
+ isRunning,
+ isDirty,
+ cancel,
+ run,
+ executionResult,
+ };
+}
+
+export function isPythonTransformSource(
+ source: PythonTransformSourceDraft,
+): source is PythonTransformSource {
+ return source.type === "python" && source["source-database"] !== undefined;
+}
+
+export function useShouldShowPythonDebugger() {
+ const location = useLocation();
+ const params = new URLSearchParams(location.search);
+ return params.get("debugger") === "1";
+}
+
+export function getValidationResult(source: PythonTransformSourceDraft) {
+ if (!source["source-database"]) {
+ return { isValid: false, errorMessage: t`Select a source a database` };
+ }
+
+ if (source.body.trim() === "") {
+ return {
+ isValid: false,
+ errorMessage: t`The Python script cannot be empty`,
+ };
+ }
+
+ if (Object.keys(source["source-tables"]).length === 0) {
+ return {
+ isValid: false,
+ errorMessage: t`Select at least one table to alias`,
+ };
+ }
+
+ return { isValid: true };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.module.css
new file mode 100644
index 00000000000..a51a0a3cf90
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.module.css
@@ -0,0 +1,10 @@
+.grid {
+ display: grid;
+ grid-template-columns: max-content 1fr;
+ gap: var(--mantine-spacing-sm) var(--mantine-spacing-md);
+}
+
+.alias {
+ font-family: var(--mb-default-monospace-font-family);
+ font-size: 0.75rem;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.tsx
new file mode 100644
index 00000000000..a98fca724e9
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.tsx
@@ -0,0 +1,118 @@
+import { t } from "ttag";
+
+import { skipToken, useGetTableQuery } from "metabase/api";
+import Link from "metabase/common/components/Link";
+import CS from "metabase/css/core/index.css";
+import { Box, Group, Icon, type IconName, Loader, Stack } from "metabase/ui";
+import { SplitSection } from "metabase-enterprise/transforms/components/SplitSection";
+import {
+ getBrowseDatabaseUrl,
+ getBrowseSchemaUrl,
+ getQueryBuilderUrl,
+} from "metabase-enterprise/transforms/urls";
+import type { Transform } from "metabase-types/api";
+
+import S from "./SourceSection.module.css";
+
+type SourceSectionProps = {
+ transform: Transform;
+};
+
+export function SourceSection({ transform }: SourceSectionProps) {
+ const { source } = transform;
+ if (source.type !== "python") {
+ return null;
+ }
+
+ const { "source-database": sourceDatabaseId, "source-tables": sourceTables } =
+ source;
+
+ return (
+ <SplitSection
+ label={t`Transform source`}
+ description={t`The data sources for this Python transform, by alias.`}
+ >
+ <Stack p="lg" gap="sm" className={S.grid}>
+ {Object.entries(sourceTables).map(([alias, tableId]) => (
+ <TableWithAlias
+ key={alias}
+ alias={alias}
+ tableId={tableId as number}
+ databaseId={sourceDatabaseId as number}
+ />
+ ))}
+ </Stack>
+ </SplitSection>
+ );
+}
+
+type TableWithAliasProps = {
+ alias: string;
+ tableId: number;
+ databaseId: number;
+};
+
+function TableWithAlias({ alias, tableId, databaseId }: TableWithAliasProps) {
+ const { data: table, isLoading } = useGetTableQuery(
+ tableId ? { id: tableId } : skipToken,
+ );
+
+ if (isLoading) {
+ return <Loader size="xs" data-testid="loader" />;
+ }
+
+ return (
+ <>
+ <Group className={S.alias}>{alias}</Group>
+ <Group gap="sm">
+ {table?.db && (
+ <>
+ <SourceItemLink
+ label={table.db.name}
+ icon="database"
+ to={getBrowseDatabaseUrl(table.db.id)}
+ data-testid="source-database-link"
+ />
+ <SourceItemDivider />
+ </>
+ )}
+ {table?.schema && (
+ <>
+ <SourceItemLink
+ label={table.schema}
+ icon="folder"
+ to={getBrowseSchemaUrl(databaseId, table.schema)}
+ />
+ <SourceItemDivider />
+ </>
+ )}
+ <SourceItemLink
+ label={table?.display_name || table?.name || `Table ${tableId}`}
+ icon="table2"
+ to={table ? getQueryBuilderUrl(table.id, table.db_id) : undefined}
+ />
+ </Group>
+ </>
+ );
+}
+
+type SourceItemLinkProps = {
+ label: string;
+ icon: IconName;
+ to?: string;
+};
+
+function SourceItemLink({ label, icon, to }: SourceItemLinkProps) {
+ return (
+ <Link className={CS.link} to={to ?? ""} disabled={to == null}>
+ <Group gap="xs" align="center" justify="center">
+ <Icon name={icon} />
+ <Box>{label}</Box>
+ </Group>
+ </Link>
+ );
+}
+
+function SourceItemDivider() {
+ return <Icon name="chevronright" size={8} />;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.unit.spec.tsx
new file mode 100644
index 00000000000..16a3644593b
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/SourceSection.unit.spec.tsx
@@ -0,0 +1,83 @@
+import { setupTableEndpoints } from "__support__/server-mocks";
+import { renderWithProviders, screen, waitFor } from "__support__/ui";
+import type { ConcreteTableId, Table, Transform } from "metabase-types/api";
+import { createMockTable } from "metabase-types/api/mocks";
+import {
+ createMockPythonTransformSource,
+ createMockTransform,
+} from "metabase-types/api/mocks/transform";
+
+import { SourceSection } from "./SourceSection";
+
+const TABLE_A = createMockTable({
+ id: 1,
+ name: "table_a",
+ display_name: "Table A",
+ schema: "schema_a",
+});
+
+const TABLE_B = createMockTable({
+ id: 2,
+ name: "table_b",
+ display_name: "Table B",
+ schema: "schema_b",
+});
+
+type SetupOpts = {
+ tables?: Table[];
+ transform?: Transform;
+};
+
+function setup({
+ tables = [TABLE_A, TABLE_B],
+ transform = createMockTransform(),
+}: SetupOpts = {}) {
+ for (const table of tables) {
+ setupTableEndpoints(table);
+ }
+ renderWithProviders(<SourceSection transform={transform} />);
+}
+
+describe("SourceSection", () => {
+ it("should render nothing for query transform", () => {
+ setup({
+ transform: createMockTransform({
+ source: {
+ type: "query",
+ query: { type: "query", database: 1, query: { "source-table": 1 } },
+ },
+ }),
+ });
+ expect(screen.queryByText("Transform source")).not.toBeInTheDocument();
+ });
+
+ it("should render the section with label and description", async () => {
+ setup({
+ transform: createMockTransform({
+ source: createMockPythonTransformSource({
+ sourceTables: {
+ orders: TABLE_A.id as ConcreteTableId,
+ customers: TABLE_B.id as ConcreteTableId,
+ },
+ }),
+ }),
+ });
+
+ expect(screen.getByText("Transform source")).toBeInTheDocument();
+ expect(
+ screen.getByText("The data sources for this Python transform, by alias."),
+ ).toBeInTheDocument();
+
+ await waitFor(() => {
+ expect(screen.queryAllByTestId("loader")).toHaveLength(0);
+ });
+
+ expect(screen.getByText("orders")).toBeInTheDocument();
+ expect(screen.getByText("schema_a")).toBeInTheDocument();
+ expect(screen.getByText("Table A")).toBeInTheDocument();
+
+ expect(screen.getByText("customers")).toBeInTheDocument();
+ expect(screen.getByText("schema_b")).toBeInTheDocument();
+ expect(screen.getByText("Table B")).toBeInTheDocument();
+ });
+});
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/index.ts
new file mode 100644
index 00000000000..364b1d65f4c
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/SourceSection/index.ts
@@ -0,0 +1 @@
+export * from "./SourceSection";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/constants.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/constants.ts
new file mode 100644
index 00000000000..f62515eb84f
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/constants.ts
@@ -0,0 +1 @@
+export const SHARED_LIB_IMPORT_PATH = "common.py";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/index.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/index.tsx
new file mode 100644
index 00000000000..29a5028c651
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/index.tsx
@@ -0,0 +1,54 @@
+import { t } from "ttag";
+
+import { AdminNavItem } from "metabase/admin/components/AdminNav";
+import { ForwardRefLink } from "metabase/common/components/Link";
+import { Route } from "metabase/hoc/Title";
+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
+import { Icon, Menu } from "metabase/ui";
+import { hasPremiumFeature } from "metabase-enterprise/settings";
+import { trackTransformCreate } from "metabase-enterprise/transforms/analytics";
+import { FullWidthTransformPageLayout } from "metabase-enterprise/transforms/pages/TransformPageLayout";
+import { getNewTransformFromTypeUrl } from "metabase-enterprise/transforms/urls";
+
+import { PythonTransformEditor } from "./components/PythonTransformEditor";
+import { SourceSection } from "./components/SourceSection";
+import { SHARED_LIB_IMPORT_PATH } from "./constants";
+import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";
+import { PythonRunnerSettingsPage } from "./pages/PythonRunnerSettingsPage";
+import { getPythonLibraryUrl } from "./urls";
+
+if (hasPremiumFeature("transforms-python")) {
+ PLUGIN_TRANSFORMS_PYTHON.PythonRunnerSettingsPage = PythonRunnerSettingsPage;
+ PLUGIN_TRANSFORMS_PYTHON.TransformEditor = PythonTransformEditor;
+ PLUGIN_TRANSFORMS_PYTHON.SourceSection = SourceSection;
+
+ PLUGIN_TRANSFORMS_PYTHON.getAdminRoutes = () => (
+ <Route component={FullWidthTransformPageLayout}>
+ <Route path="library/:path" component={PythonLibraryEditorPage} />
+ </Route>
+ );
+
+ PLUGIN_TRANSFORMS_PYTHON.getTransformsNavLinks = () => (
+ <AdminNavItem
+ label={t`Python library`}
+ path={getPythonLibraryUrl({ path: SHARED_LIB_IMPORT_PATH })}
+ icon="code_block"
+ />
+ );
+
+ PLUGIN_TRANSFORMS_PYTHON.getCreateTransformsMenuItems = () => (
+ <Menu.Item
+ component={ForwardRefLink}
+ to={getNewTransformFromTypeUrl("python")}
+ leftSection={<Icon name="code_block" />}
+ onClick={() => {
+ trackTransformCreate({
+ triggeredFrom: "transform-page-create-menu",
+ creationType: "python",
+ });
+ }}
+ >
+ {t`Python script`}
+ </Menu.Item>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css
new file mode 100644
index 00000000000..7b3022ff2fd
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css
@@ -0,0 +1,5 @@
+.editor {
+ /* HACK: 133px is the height of the headers */
+ height: calc(100vh - 133px);
+ flex-grow: 1;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx
new file mode 100644
index 00000000000..a6b4e971ea3
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx
@@ -0,0 +1,140 @@
+import { useEffect, useState } from "react";
+import { t } from "ttag";
+
+import Button from "metabase/common/components/Button";
+import EditBar from "metabase/common/components/EditBar";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { isResourceNotFoundError } from "metabase/lib/errors";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import { Box, Flex } from "metabase/ui";
+import {
+ useGetPythonLibraryQuery,
+ useUpdatePythonLibraryMutation,
+} from "metabase-enterprise/api/python-transform-library";
+
+import { PythonEditor } from "../../components/PythonEditor";
+import type { PythonLibraryEditorPageParams } from "../../types";
+
+import S from "./PythonLibraryEditorPage.module.css";
+
+type PythonLibraryEditorPageProps = {
+ params: PythonLibraryEditorPageParams;
+};
+
+const EMPTY_LIBRARY_SOURCE = `
+# This is your Python library.
+# You can add functions and classes here that can be reused in Python transforms.
+`
+ .trim()
+ .concat("\n");
+
+export function PythonLibraryEditorPage({
+ params,
+}: PythonLibraryEditorPageProps) {
+ const { path } = params;
+ const [source, setSource] = useState("");
+
+ const {
+ data: library,
+ isLoading,
+ error,
+ } = useGetPythonLibraryQuery({ path });
+ const [updatePythonLibrary, { isLoading: isSaving }] =
+ useUpdatePythonLibraryMutation();
+ const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+
+ function handleRevert() {
+ if (isLoading) {
+ return;
+ }
+
+ if (isResourceNotFoundError(error)) {
+ setSource(EMPTY_LIBRARY_SOURCE);
+ return;
+ }
+
+ if (library != null) {
+ setSource(library.source || "");
+ }
+ }
+
+ async function handleSave() {
+ try {
+ await updatePythonLibrary({ path, source }).unwrap();
+ sendSuccessToast(t`Python library saved`);
+ } catch (error) {
+ sendErrorToast(t`Python library could not be saved`);
+ }
+ }
+
+ // When the library loads, set the source to the current library source
+ useEffect(handleRevert, [isLoading, error, library]);
+
+ const isDirty = source !== (library?.source ?? EMPTY_LIBRARY_SOURCE);
+
+ if (isLoading || (error && !isResourceNotFoundError(error))) {
+ return (
+ <Box p="md">
+ <LoadingAndErrorWrapper loading={isLoading} error={error} />
+ </Box>
+ );
+ }
+
+ return (
+ <Flex h="100%" w="100%" bg="bg-light" gap={0} direction="column">
+ <LibraryEditorHeader
+ onSave={handleSave}
+ onRevert={handleRevert}
+ isDirty={isDirty}
+ isSaving={isSaving}
+ />
+ <PythonEditor
+ value={source}
+ onChange={setSource}
+ withPandasCompletions
+ className={S.editor}
+ data-testid="python-editor"
+ />
+ </Flex>
+ );
+}
+
+export function LibraryEditorHeader({
+ isDirty,
+ isSaving,
+ onSave,
+ onRevert,
+}: {
+ isDirty?: boolean;
+ isSaving?: boolean;
+ onSave: () => void;
+ onRevert: () => void;
+}) {
+ return (
+ <EditBar
+ title={t`You are editing the shared Python library`}
+ admin
+ data-testid="library-editor-header"
+ buttons={[
+ <Button
+ key="save"
+ onClick={onRevert}
+ primary
+ small
+ disabled={!isDirty || isSaving}
+ >
+ {t`Revert`}
+ </Button>,
+ <Button
+ key="save"
+ onClick={onSave}
+ primary
+ small
+ disabled={!isDirty || isSaving}
+ >
+ {t`Save`}
+ </Button>,
+ ]}
+ />
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/index.ts
new file mode 100644
index 00000000000..3b95d285898
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/index.ts
@@ -0,0 +1 @@
+export * from "./PythonLibraryEditorPage";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx
new file mode 100644
index 00000000000..aa144670cf4
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx
@@ -0,0 +1,67 @@
+import { t } from "ttag";
+
+import {
+ SettingsPageWrapper,
+ SettingsSection,
+} from "metabase/admin/components/SettingsSection";
+import { AdminSettingInput } from "metabase/admin/settings/components/widgets/AdminSettingInput";
+
+export function PythonRunnerSettingsPage() {
+ return (
+ <SettingsPageWrapper title={t`Python Runner`}>
+ <SettingsSection title={t`Service Configuration`}>
+ <AdminSettingInput
+ name="python-runner-url"
+ title={t`Python Execution Server URL`}
+ description={t`URL for the Python execution server that runs transform functions.`}
+ placeholder="http://localhost:5001"
+ inputType="text"
+ />
+ </SettingsSection>
+ <SettingsSection title={t`S3 Storage Configuration`}>
+ <AdminSettingInput
+ name="python-storage-s-3-endpoint"
+ title={t`S3 Endpoint`}
+ description={t`S3 endpoint URL for storing Python execution artifacts.`}
+ inputType="text"
+ />
+ <AdminSettingInput
+ name="python-storage-s-3-region"
+ title={t`S3 Region`}
+ description={t`AWS region for S3 storage.`}
+ inputType="text"
+ />
+ <AdminSettingInput
+ name="python-storage-s-3-bucket"
+ title={t`S3 Bucket`}
+ description={t`S3 bucket name for storing Python execution artifacts.`}
+ inputType="text"
+ />
+ <AdminSettingInput
+ name="python-storage-s-3-access-key"
+ title={t`S3 Access Key ID`}
+ description={t`AWS access key ID for S3 authentication.`}
+ inputType="password"
+ />
+ <AdminSettingInput
+ name="python-storage-s-3-secret-key"
+ title={t`S3 Secret Access Key`}
+ description={t`AWS secret access key for S3 authentication.`}
+ inputType="password"
+ />
+ <AdminSettingInput
+ name="python-storage-s-3-container-endpoint"
+ title={t`S3 Container Endpoint (Optional)`}
+ description={t`Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.`}
+ inputType="text"
+ />
+ <AdminSettingInput
+ name="python-storage-s-3-path-style-access"
+ title={t`Use Path-Style Access`}
+ description={t`Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).`}
+ inputType="boolean"
+ />
+ </SettingsSection>
+ </SettingsPageWrapper>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/index.ts
new file mode 100644
index 00000000000..2d7ac75c267
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/index.ts
@@ -0,0 +1 @@
+export * from "./PythonRunnerSettingsPage";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/types.ts
new file mode 100644
index 00000000000..97f8c0deb69
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/types.ts
@@ -0,0 +1,3 @@
+export type PythonLibraryEditorPageParams = {
+ path: string;
+};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/urls.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/urls.ts
new file mode 100644
index 00000000000..6f96c6a70ea
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/urls.ts
@@ -0,0 +1,5 @@
+import type { PythonLibraryEditorPageParams } from "./types";
+
+export function getPythonLibraryUrl({ path }: PythonLibraryEditorPageParams) {
+ return `/admin/transforms/library/${path}`;
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts b/enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts
index d0a9ca94ee0..43cbc9c96e1 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts
@@ -34,7 +34,7 @@ export function trackTransformCreate({
creationType,
}: {
triggeredFrom: "transform-page-create-menu";
- creationType: "query" | "native" | "saved-question";
+ creationType: "query" | "native" | "python" | "saved-question";
}) {
trackSimpleEvent({
event: "transform_create",
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.modules.css b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.modules.css
new file mode 100644
index 00000000000..e2c84ad8653
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.modules.css
@@ -0,0 +1,9 @@
+.logOutput {
+ background-color: var(--mb-color-bg-light);
+ font-family: var(--mb-default-monospace-font-family);
+ white-space: pre;
+ padding: 1rem;
+ overflow: auto;
+ line-height: 1.4;
+ border-top: 1px solid var(--mb-color-border);
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx
new file mode 100644
index 00000000000..a6c440e6413
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx
@@ -0,0 +1,67 @@
+import * as React from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
+import reactAnsiStyle from "react-ansi-style";
+import { t } from "ttag";
+
+import { AnsiLogs } from "metabase/common/components/AnsiLogs";
+
+import S from "./LogOutput.modules.css";
+
+interface LogOutputProps {
+ content?: string;
+}
+
+export function LogOutput({ content }: LogOutputProps) {
+ const containerRef = useRef<HTMLDivElement>(null);
+ const [isUserScrolled, setIsUserScrolled] = useState(false);
+ const [shouldAutoScroll, setShouldAutoScroll] = useState(true);
+
+ const displayContent = useMemo(() => {
+ if (!content || content.trim().length === 0) {
+ return t`No output`;
+ }
+
+ return reactAnsiStyle(React, content);
+ }, [content]);
+
+ // Auto-scroll to bottom when content changes
+ useEffect(() => {
+ if (containerRef.current && shouldAutoScroll && !isUserScrolled) {
+ containerRef.current.scrollTop = containerRef.current.scrollHeight;
+ }
+ }, [displayContent, shouldAutoScroll, isUserScrolled]);
+
+ // Handle scroll events to detect user scrolling
+ const handleScroll = () => {
+ if (!containerRef.current) {
+ return;
+ }
+
+ const { scrollTop, scrollHeight, clientHeight } = containerRef.current;
+ const isAtBottom = Math.abs(scrollHeight - clientHeight - scrollTop) < 5;
+
+ if (isAtBottom) {
+ setIsUserScrolled(false);
+ setShouldAutoScroll(true);
+ } else {
+ setIsUserScrolled(true);
+ setShouldAutoScroll(false);
+ }
+ };
+
+ return (
+ <AnsiLogs
+ ref={containerRef}
+ className={S.logOutput}
+ onScroll={handleScroll}
+ mah="12rem"
+ p="sm"
+ bg="bg-light"
+ fz="xs"
+ data-css-specificity-hack=""
+ data-testid="log-output"
+ >
+ {displayContent}
+ </AnsiLogs>
+ );
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts
new file mode 100644
index 00000000000..a8aa12bfc51
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts
@@ -0,0 +1 @@
+export { LogOutput } from "./LogOutput";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx
index 39ecd1f7901..3cd496d6939 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx
@@ -9,7 +9,10 @@ import { NativeQueryPreview } from "metabase/querying/notebook/components/Native
import { Center, Flex, Loader, Modal, Stack } from "metabase/ui";
import * as Lib from "metabase-lib";
import type Question from "metabase-lib/v1/Question";
-import type { DatasetQuery, NativeQuerySnippet } from "metabase-types/api";
+import type {
+ NativeQuerySnippet,
+ QueryTransformSource,
+} from "metabase-types/api";
import { useQueryMetadata } from "../../hooks/use-query-metadata";
import { useQueryResults } from "../../hooks/use-query-results";
@@ -32,21 +35,23 @@ import {
} from "./utils";
type QueryEditorProps = {
- initialQuery: DatasetQuery;
+ initialSource: QueryTransformSource;
isNew?: boolean;
isSaving?: boolean;
- onSave: (newQuery: DatasetQuery) => void;
+ onSave: (source: QueryTransformSource) => void;
onCancel: () => void;
};
export function QueryEditor({
- initialQuery,
+ initialSource,
isNew = true,
isSaving = false,
onSave,
onCancel,
}: QueryEditorProps) {
- const { question, isQueryDirty, setQuestion } = useQueryState(initialQuery);
+ const { question, isQueryDirty, setQuestion } = useQueryState(
+ initialSource.query,
+ );
const { isInitiallyLoaded } = useQueryMetadata(question);
const {
result,
@@ -67,7 +72,7 @@ export function QueryEditor({
};
const handleSave = () => {
- onSave(question.datasetQuery());
+ onSave({ type: "query", query: question.datasetQuery() });
};
const handleCmdEnter = () => {
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/index.ts
deleted file mode 100644
index 19070a74f88..00000000000
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from "./RunErrorInfo";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.module.css b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.module.css
similarity index 100%
rename from enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.module.css
rename to enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.module.css
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.tsx
similarity index 60%
rename from enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.tsx
rename to enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.tsx
index fee2d7ebed7..753c7476edb 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.tsx
@@ -16,15 +16,17 @@ import {
Stack,
Tooltip,
} from "metabase/ui";
+import type { TransformRunStatus } from "metabase-types/api";
-import S from "./RunErrorInfo.module.css";
+import S from "./RunInfo.module.css";
-type RunErrorInfoProps = {
+type RunInfoProps = {
+ status: TransformRunStatus;
message: string;
endTime: Date | null;
};
-export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {
+export function RunInfo({ status, message, endTime }: RunInfoProps) {
const [isOpened, { open, close }] = useDisclosure();
const handleIconClick = (event: MouseEvent) => {
@@ -38,16 +40,20 @@ export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {
event.stopPropagation();
};
+ if (!message || (status !== "failed" && status !== "succeeded")) {
+ return null;
+ }
+
return (
<>
- <Tooltip label={t`See error`}>
+ <Tooltip label={getTooltip(status)}>
<ActionIcon aria-label={t`See error`} onClick={handleIconClick}>
<Icon name="document" />
</ActionIcon>
</Tooltip>
{isOpened && (
<Modal
- title={getTitle(endTime)}
+ title={getTitle(status, endTime)}
size="xl"
padding="xl"
opened
@@ -55,17 +61,34 @@ export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {
onClose={close}
>
<FocusTrap.InitialFocus />
- <RunErrorModalContent message={message} onClose={close} />
+ <RunInfoModalContent message={message} onClose={close} />
</Modal>
)}
</>
);
}
-function getTitle(endTime: Date | null) {
- return endTime
- ? t`Failed at ${dayjs(endTime).format("lll")}, with this error`
- : t`Failed with this error`;
+function getTooltip(status: TransformRunStatus) {
+ if (status === "failed") {
+ return t`See error`;
+ }
+ if (status === "succeeded") {
+ return t`See logs`;
+ }
+}
+
+function getTitle(status: TransformRunStatus, endTime: Date | null) {
+ if (status === "failed") {
+ return endTime
+ ? t`Failed at ${dayjs(endTime).format("lll")}, with this error`
+ : t`Failed with this error`;
+ }
+ if (status === "succeeded") {
+ return endTime
+ ? t`Succeeded at ${dayjs(endTime).format("lll")}, with the following logs`
+ : t`Succeeded with the following logs`;
+ }
+ return null;
}
type RunErrorModalContentProps = {
@@ -73,7 +96,7 @@ type RunErrorModalContentProps = {
onClose: () => void;
};
-function RunErrorModalContent({ message, onClose }: RunErrorModalContentProps) {
+function RunInfoModalContent({ message, onClose }: RunErrorModalContentProps) {
return (
<Stack pt="md" gap="lg">
<Box className={S.codeContainer} pos="relative" pr="lg">
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.unit.spec.tsx
similarity index 87%
rename from enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.unit.spec.tsx
rename to enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.unit.spec.tsx
index ae28491c513..f547e78a858 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/RunErrorInfo.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.unit.spec.tsx
@@ -2,7 +2,7 @@ import userEvent from "@testing-library/user-event";
import { renderWithProviders, screen } from "__support__/ui";
-import { RunErrorInfo } from "./RunErrorInfo";
+import { RunInfo } from "./RunInfo";
type SetupOpts = {
message?: string;
@@ -13,10 +13,12 @@ function setup({
message = "Test error message",
endTime = null,
}: SetupOpts = {}) {
- renderWithProviders(<RunErrorInfo message={message} endTime={endTime} />);
+ renderWithProviders(
+ <RunInfo status="failed" message={message} endTime={endTime} />,
+ );
}
-describe("RunErrorInfo", () => {
+describe("RunInfo", () => {
it("should show the error message when there is a date", async () => {
const message = "Test error with end time";
const endTime = new Date("2023-10-15T14:30:00");
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/index.ts
new file mode 100644
index 00000000000..cbcab61e33b
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/index.ts
@@ -0,0 +1 @@
+export * from "./RunInfo";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsx
index 70cdefa20ff..d20531767ce 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatus/RunStatus.tsx
@@ -5,7 +5,7 @@ import { Box, Group, Icon } from "metabase/ui";
import { parseTimestampWithTimezone } from "metabase-enterprise/transforms/utils";
import type { TransformRun } from "metabase-types/api";
-import { RunErrorInfo } from "../RunErrorInfo";
+import { RunInfo } from "../RunInfo";
type RunStatusSectionProps = {
run: TransformRun | null;
@@ -38,7 +38,8 @@ export function RunStatus({
const errorInfo =
message != null ? (
- <RunErrorInfo
+ <RunInfo
+ status={status}
message={message}
endTime={endTime ? endTime.toDate() : null}
/>
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx
index e4d9206b3e7..2ef6e1b2c57 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx
@@ -3,7 +3,7 @@ import type { Transform, TransformRunStatus } from "metabase-types/api";
import { formatStatus } from "../../utils";
import { RunCancelButton } from "../RunCancelButton/RunCancelButton";
-import { RunErrorInfo } from "../RunErrorInfo";
+import { RunInfo } from "../RunInfo";
import S from "./RunStatusInfo.module.css";
@@ -26,7 +26,7 @@ export function RunStatusInfo({
<Group gap="xs" className={S.runStatusInfo} wrap="nowrap">
<Box c={isError ? "error" : undefined}>{formatStatus(status)}</Box>
{isError && message != null && (
- <RunErrorInfo message={message} endTime={endTime} />
+ <RunInfo status={status} message={message} endTime={endTime} />
)}
<RunCancelButton transform={transform} status={status} />
</Group>
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/index.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/index.tsx
index 19724eef476..af18a28c5f1 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/index.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/index.tsx
@@ -3,13 +3,13 @@ import { t } from "ttag";
import { createAdminRouteGuard } from "metabase/admin/utils";
import { Route } from "metabase/hoc/Title";
-import { PLUGIN_TRANSFORMS } from "metabase/plugins";
+import { PLUGIN_TRANSFORMS, PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
import { hasPremiumFeature } from "metabase-enterprise/settings";
import { JobListPage } from "./pages/JobListPage";
import { JobPage } from "./pages/JobPage";
import { NewJobPage } from "./pages/NewJobPage";
-import { NewTransformQueryPage } from "./pages/NewTransformQueryPage";
+import { NewTransformPage } from "./pages/NewTransformPage";
import { RunListPage } from "./pages/RunListPage";
import { TransformListPage } from "./pages/TransformListPage";
import { TransformPage } from "./pages/TransformPage";
@@ -34,8 +34,9 @@ if (hasPremiumFeature("transforms")) {
<Route path="jobs/:jobId" component={JobPage} />
<Route path=":transformId" component={TransformPage} />
</Route>
- <Route path="new/:type" component={NewTransformQueryPage} />
- <Route path="new/card/:cardId" component={NewTransformQueryPage} />
+ {PLUGIN_TRANSFORMS_PYTHON.getAdminRoutes()}
+ <Route path="new/:type" component={NewTransformPage} />
+ <Route path="new/card/:cardId" component={NewTransformPage} />
<Route path=":transformId/query" component={TransformQueryPage} />
</Route>
</Route>
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
similarity index 91%
rename from enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx
rename to enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
index e040f013a52..a1b9fbfa259 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
@@ -24,18 +24,18 @@ import { trackTransformCreated } from "metabase-enterprise/transforms/analytics"
import { SchemaFormSelect } from "metabase-enterprise/transforms/components/SchemaFormSelect";
import type {
CreateTransformRequest,
- DatasetQuery,
Transform,
+ TransformSource,
} from "metabase-types/api";
type CreateTransformModalProps = {
- query: DatasetQuery;
+ source: TransformSource;
onCreate: (transform: Transform) => void;
onClose: () => void;
};
export function CreateTransformModal({
- query,
+ source,
onCreate,
onClose,
}: CreateTransformModalProps) {
@@ -43,7 +43,7 @@ export function CreateTransformModal({
<Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
<FocusTrap.InitialFocus />
<CreateTransformForm
- query={query}
+ source={source}
onCreate={onCreate}
onClose={onClose}
/>
@@ -52,7 +52,7 @@ export function CreateTransformModal({
}
type CreateTransformFormProps = {
- query: DatasetQuery;
+ source: TransformSource;
onCreate: (transform: Transform) => void;
onClose: () => void;
};
@@ -72,11 +72,12 @@ const NEW_TRANSFORM_SCHEMA = Yup.object({
});
function CreateTransformForm({
- query,
+ source,
onCreate,
onClose,
}: CreateTransformFormProps) {
- const { database: databaseId } = query;
+ const databaseId =
+ source.type === "query" ? source.query.database : source["source-database"];
const {
data: database,
@@ -108,7 +109,10 @@ function CreateTransformForm({
}
const handleSubmit = async (values: NewTransformValues) => {
- const request = getCreateRequest(query, values);
+ if (!databaseId) {
+ throw new Error("Database ID is required");
+ }
+ const request = getCreateRequest(source, values, databaseId);
const transform = await createTransform(request).unwrap();
trackTransformCreated({ transformId: transform.id });
@@ -171,20 +175,19 @@ function getInitialValues(schemas: string[]): NewTransformValues {
}
function getCreateRequest(
- query: DatasetQuery,
+ source: TransformSource,
{ name, description, targetName, targetSchema }: NewTransformValues,
+ databaseId: number,
): CreateTransformRequest {
return {
name: name,
description,
- source: {
- type: "query",
- query,
- },
+ source,
target: {
type: "table",
name: targetName,
schema: targetSchema,
+ database: databaseId,
},
};
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/index.ts
similarity index 100%
rename from enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/index.ts
rename to enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/index.ts
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx
new file mode 100644
index 00000000000..43d04860217
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx
@@ -0,0 +1,144 @@
+import { useDisclosure } from "@mantine/hooks";
+import { useState } from "react";
+import { push } from "react-router-redux";
+
+import { skipToken, useGetCardQuery } from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useDispatch } from "metabase/lib/redux";
+import * as Urls from "metabase/lib/urls";
+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
+import type {
+ CardId,
+ DatasetQuery,
+ Transform,
+ TransformSource,
+} from "metabase-types/api";
+
+import { QueryEditor } from "../../components/QueryEditor";
+import { getTransformListUrl, getTransformUrl } from "../../urls";
+
+import { CreateTransformModal } from "./CreateTransformModal";
+import {
+ getInitialPythonTransformSource,
+ getInitialQueryTransformSource,
+} from "./utils";
+
+type NewTransformPageParams = {
+ type?: string;
+ cardId?: string;
+};
+
+type NewTransformPageParsedParams = {
+ type: DatasetQuery["type"] | "python";
+ cardId?: CardId;
+};
+
+type NewTransformPageProps = {
+ params: NewTransformPageParams;
+};
+
+export function NewTransformPage({ params }: NewTransformPageProps) {
+ const { type, cardId } = getParsedParams(params);
+
+ const [source, setSource] = useState<TransformSource | null>(null);
+ const [isModalOpened, { open: openModal, close: closeModal }] =
+ useDisclosure();
+ const dispatch = useDispatch();
+
+ const handleCreate = (transform: Transform) => {
+ dispatch(push(getTransformUrl(transform.id)));
+ };
+
+ const handleSave = (newSource: TransformSource) => {
+ setSource(newSource);
+ openModal();
+ };
+
+ const handleCancel = () => {
+ dispatch(push(getTransformListUrl()));
+ };
+
+ return (
+ <>
+ <NewTransformEditorBody
+ type={type}
+ cardId={cardId}
+ onSave={handleSave}
+ onCancel={handleCancel}
+ />
+ {isModalOpened && source !== null && (
+ <CreateTransformModal
+ source={source}
+ onCreate={handleCreate}
+ onClose={closeModal}
+ />
+ )}
+ </>
+ );
+}
+
+function NewTransformEditorBody(props: {
+ type: DatasetQuery["type"] | "python";
+ cardId?: CardId;
+ onSave: (source: TransformSource) => void;
+ onCancel: () => void;
+}) {
+ const { type, ...rest } = props;
+ if (type === "python") {
+ return (
+ <PLUGIN_TRANSFORMS_PYTHON.TransformEditor
+ initialSource={getInitialPythonTransformSource()}
+ isNew
+ onSave={props.onSave}
+ onCancel={props.onCancel}
+ />
+ );
+ }
+
+ return <NewQueryTransformEditorBody {...rest} type={type} />;
+}
+
+function NewQueryTransformEditorBody({
+ type,
+ cardId,
+ onSave,
+ onCancel,
+}: {
+ type: DatasetQuery["type"];
+ cardId?: CardId;
+ onSave: (source: TransformSource) => void;
+ onCancel: () => void;
+}) {
+ const {
+ data: card,
+ isLoading,
+ error,
+ } = useGetCardQuery(cardId ? { id: cardId } : skipToken);
+
+ if (isLoading || error) {
+ return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
+ }
+
+ return (
+ <QueryEditor
+ initialSource={getInitialQueryTransformSource(card, type)}
+ isNew
+ onSave={onSave}
+ onCancel={onCancel}
+ />
+ );
+}
+
+function getParsedParams({
+ type,
+ cardId,
+}: NewTransformPageParams): NewTransformPageParsedParams {
+ if (type === "python") {
+ return { type: "python" };
+ }
+
+ return {
+ type: type === "native" ? "native" : "query",
+ cardId: cardId != null ? Urls.extractEntityId(cardId) : undefined,
+ };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/index.ts
new file mode 100644
index 00000000000..abaf276bac8
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/index.ts
@@ -0,0 +1 @@
+export * from "./NewTransformPage";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts
new file mode 100644
index 00000000000..7c71e2bdd2e
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts
@@ -0,0 +1,36 @@
+import Question from "metabase-lib/v1/Question";
+import type { Card, DatasetQuery } from "metabase-types/api";
+
+export function getInitialQueryTransformSource(
+ card: Card | undefined,
+ type: DatasetQuery["type"] | undefined,
+) {
+ const query =
+ card != null
+ ? card.dataset_query
+ : Question.create({ type }).datasetQuery();
+
+ return { type: "query" as const, query };
+}
+
+export function getInitialPythonTransformSource() {
+ return {
+ type: "python" as const,
+ "source-database": undefined,
+ "source-tables": {},
+ body: `# Write your Python transformation script here
+import pandas as pd
+
+def transform():
+ """
+ Your transformation function.
+
+ Select tables above to add them as function parameters.
+
+ Returns:
+ DataFrame to write to the destination table
+ """
+ # Your transformation logic here
+ return pd.DataFrame([{"message": "Hello from Python transform!"}])`,
+ };
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx
deleted file mode 100644
index 313c241fa08..00000000000
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx
+++ /dev/null
@@ -1,105 +0,0 @@
-import { useDisclosure } from "@mantine/hooks";
-import { useState } from "react";
-import { push } from "react-router-redux";
-
-import { skipToken, useGetCardQuery } from "metabase/api";
-import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
-import { useDispatch } from "metabase/lib/redux";
-import * as Urls from "metabase/lib/urls";
-import Question from "metabase-lib/v1/Question";
-import type { Card, CardId, DatasetQuery, Transform } from "metabase-types/api";
-
-import { QueryEditor } from "../../components/QueryEditor";
-import { getTransformListUrl, getTransformUrl } from "../../urls";
-
-import { CreateTransformModal } from "./CreateTransformModal";
-
-type NewTransformQueryPageParams = {
- type?: string;
- cardId?: string;
-};
-
-type NewTransformQueryPageParsedParams = {
- type?: DatasetQuery["type"];
- cardId?: CardId;
-};
-
-type NewTransformQueryPageProps = {
- params: NewTransformQueryPageParams;
-};
-
-export function NewTransformQueryPage({ params }: NewTransformQueryPageProps) {
- const { type, cardId } = getParsedParams(params);
- const {
- data: card,
- isLoading,
- error,
- } = useGetCardQuery(cardId ? { id: cardId } : skipToken);
-
- if (isLoading || error) {
- return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
- }
-
- return <NewTransformPageBody initialQuery={getInitialQuery(card, type)} />;
-}
-
-type NewTransformPageBodyProps = {
- initialQuery: DatasetQuery;
-};
-
-function NewTransformPageBody({ initialQuery }: NewTransformPageBodyProps) {
- const [query, setQuery] = useState(initialQuery);
- const [isModalOpened, { open: openModal, close: closeModal }] =
- useDisclosure();
- const dispatch = useDispatch();
-
- const handleCreate = (transform: Transform) => {
- dispatch(push(getTransformUrl(transform.id)));
- };
-
- const handleSaveClick = (newQuery: DatasetQuery) => {
- setQuery(newQuery);
- openModal();
- };
-
- const handleCancelClick = () => {
- dispatch(push(getTransformListUrl()));
- };
-
- return (
- <>
- <QueryEditor
- initialQuery={initialQuery}
- isNew
- onSave={handleSaveClick}
- onCancel={handleCancelClick}
- />
- {isModalOpened && (
- <CreateTransformModal
- query={query}
- onCreate={handleCreate}
- onClose={closeModal}
- />
- )}
- </>
- );
-}
-
-function getParsedParams({
- type,
- cardId,
-}: NewTransformQueryPageParams): NewTransformQueryPageParsedParams {
- return {
- type: type === "native" ? "native" : "query",
- cardId: cardId != null ? Urls.extractEntityId(cardId) : undefined,
- };
-}
-
-function getInitialQuery(
- card: Card | undefined,
- type: DatasetQuery["type"] | undefined,
-) {
- return card != null
- ? card.dataset_query
- : Question.create({ type }).datasetQuery();
-}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/index.ts
deleted file mode 100644
index f4f8bafb498..00000000000
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from "./NewTransformQueryPage";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx
index ce5850d9c0d..4432802e828 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx
@@ -10,6 +10,7 @@ import {
type QuestionPickerValueItem,
} from "metabase/common/components/Pickers/QuestionPicker";
import { useDispatch } from "metabase/lib/redux";
+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
import { Button, Center, Icon, Loader, Menu } from "metabase/ui";
import { trackTransformCreate } from "metabase-enterprise/transforms/analytics";
import { doesDatabaseSupportTransforms } from "metabase-enterprise/transforms/utils";
@@ -85,8 +86,9 @@ export function CreateTransformMenu() {
>
{t`SQL query`}
</Menu.Item>
+ {PLUGIN_TRANSFORMS_PYTHON.getCreateTransformsMenuItems()}
<Menu.Item
- leftSection={<Icon name="copy" />}
+ leftSection={<Icon name="folder" />}
onClick={handleSavedQuestionClick}
>
{t`A copy of a saved question`}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx
index ce4055a5dce..828205fbbeb 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx
@@ -6,6 +6,7 @@ import { t } from "ttag";
import { useDispatch } from "metabase/lib/redux";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { Button, Card, Group, Icon } from "metabase/ui";
+import { PythonEditor } from "metabase-enterprise/transforms-python/components/PythonEditor";
import type { Transform } from "metabase-types/api";
import { QueryView } from "../../../components/QueryView";
@@ -20,9 +21,12 @@ type ManageSectionProps = {
};
export function ManageSection({ transform }: ManageSectionProps) {
+ const sectionLabel =
+ transform.source.type === "python" ? t`Python script` : t`Query`;
+
return (
<TitleSection
- label={t`Query`}
+ label={sectionLabel}
rightSection={
<Group>
<EditQueryButton transform={transform} />
@@ -31,7 +35,12 @@ export function ManageSection({ transform }: ManageSectionProps) {
}
>
<Card p={0} shadow="none" withBorder>
- <QueryView query={transform.source.query} />
+ {transform.source.type === "query" && (
+ <QueryView query={transform.source.query} />
+ )}
+ {transform.source.type === "python" && (
+ <PythonEditor value={transform.source.body} readOnly />
+ )}
</Card>
</TitleSection>
);
@@ -43,6 +52,8 @@ type EditQueryButtonProps = {
function EditQueryButton({ transform }: EditQueryButtonProps) {
const isDisabled = isTransformRunning(transform);
+ const buttonText =
+ transform.source.type === "python" ? t`Edit script` : t`Edit query`;
return (
<Button
@@ -51,7 +62,7 @@ function EditQueryButton({ transform }: EditQueryButtonProps) {
leftSection={<Icon name="pencil_lines" aria-hidden />}
disabled={isDisabled}
>
- {t`Edit query`}
+ {buttonText}
</Button>
);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx
index 2d7bad079d2..a415209fac0 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx
@@ -13,6 +13,7 @@ import {
useUpdateTransformMutation,
} from "metabase-enterprise/api";
import { trackTransformTriggerManualRun } from "metabase-enterprise/transforms/analytics";
+import { LogOutput } from "metabase-enterprise/transforms/components/LogOutput";
import type { Transform, TransformTagId } from "metabase-types/api";
import { RunButton } from "../../../components/RunButton";
@@ -32,10 +33,13 @@ export function RunSection({ transform }: RunSectionProps) {
description={t`This transform will be run whenever the jobs it belongs to are scheduled.`}
data-testid="run-section"
>
- <Group p="lg" justify="space-between">
- <RunStatusSection transform={transform} />
- <RunButtonSection transform={transform} />
- </Group>
+ <Stack>
+ <Group p="lg" justify="space-between">
+ <RunStatusSection transform={transform} />
+ <RunButtonSection transform={transform} />
+ </Group>
+ <RunOutputSection transform={transform} />
+ </Stack>
<Divider />
<Group p="lg" gap="lg">
<Stack gap="sm">
@@ -138,6 +142,18 @@ function RunButtonSection({ transform }: RunButtonSectionProps) {
);
}
+function RunOutputSection({ transform }: RunSectionProps) {
+ if (!transform?.last_run?.message) {
+ return null;
+ }
+ const { status, message } = transform.last_run;
+ if (status !== "started" && status !== "succeeded") {
+ return null;
+ }
+
+ return <LogOutput content={message} />;
+}
+
type TagSectionProps = {
transform: Transform;
};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx
index 114ff238e74..7a9540c47bc 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx
@@ -18,6 +18,7 @@ import {
Loader,
Text,
} from "metabase/ui";
+import { sourceDatabaseId } from "metabase-enterprise/transforms/utils";
import type { Transform } from "metabase-types/api";
import { SplitSection } from "../../../components/SplitSection";
@@ -59,7 +60,7 @@ type TargetInfoProps = {
function TargetInfo({ transform }: TargetInfoProps) {
const { source, target, table } = transform;
- const { database: databaseId } = source.query;
+ const databaseId = sourceDatabaseId(source);
const { data: databaseFromApi, isLoading: isDatabaseLoading } =
useGetDatabaseQuery(
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx
index ab1d27cbf43..d4dfc693389 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx
@@ -31,6 +31,7 @@ import {
useUpdateTransformMutation,
} from "metabase-enterprise/api";
import { SchemaFormSelect } from "metabase-enterprise/transforms/components/SchemaFormSelect";
+import { sourceDatabaseId } from "metabase-enterprise/transforms/utils";
import type { Transform, UpdateTransformRequest } from "metabase-types/api";
type UpdateTargetModalProps = {
@@ -83,7 +84,7 @@ function UpdateTargetForm({
onClose,
}: UpdateTargetFormProps) {
const { source, target, table } = transform;
- const { database: databaseId } = source.query;
+ const databaseId = sourceDatabaseId(source);
const [updateTransform] = useUpdateTransformMutation();
const [deleteTransformTarget] = useDeleteTransformTargetMutation();
const initialValues = useMemo(() => getInitialValues(transform), [transform]);
@@ -112,10 +113,15 @@ function UpdateTargetForm({
}
const handleSubmit = async (values: EditTransformValues) => {
+ if (!databaseId) {
+ throw new Error("Database ID is required");
+ }
if (shouldDeleteTarget) {
await deleteTransformTarget(transform.id).unwrap();
}
- await updateTransform(getUpdateRequest(transform, values)).unwrap();
+ await updateTransform(
+ getUpdateRequest(transform, values, databaseId),
+ ).unwrap();
onUpdate();
};
@@ -198,6 +204,7 @@ function getSubmitButtonColor(shouldDeleteTarget: boolean) {
function getUpdateRequest(
{ id }: Transform,
{ name, schema }: EditTransformValues,
+ databaseId: number,
): UpdateTransformRequest {
return {
id,
@@ -205,6 +212,7 @@ function getUpdateRequest(
type: "table",
name,
schema,
+ database: databaseId,
},
};
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx
index 10a5c8c96cb..88c00236223 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx
@@ -4,6 +4,7 @@ import { t } from "ttag";
import { skipToken } from "metabase/api";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import * as Urls from "metabase/lib/urls";
+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
import { Stack } from "metabase/ui";
import { useGetTransformQuery } from "metabase-enterprise/api";
import type { Transform, TransformId } from "metabase-types/api";
@@ -64,6 +65,7 @@ export function TransformPage({ params }: TransformPageProps) {
<NameSection transform={transform} />
</Stack>
<RunSection transform={transform} />
+ <PLUGIN_TRANSFORMS_PYTHON.SourceSection transform={transform} />
<TargetSection transform={transform} />
<ManageSection transform={transform} />
<DependenciesSection transform={transform} />
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx
index 79df4aeacc3..ad4d9853f4a 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx
@@ -7,6 +7,7 @@ import {
} from "metabase/admin/components/AdminNav";
import { AdminSettingsLayout } from "metabase/common/components/AdminLayout/AdminSettingsLayout";
import { useSelector } from "metabase/lib/redux";
+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
import { getLocation } from "metabase/selectors/routing";
import { getJobListUrl, getRunListUrl, getTransformListUrl } from "../../urls";
@@ -18,6 +19,7 @@ type TransformPageLayoutPropsParams = {
type TransformPageLayoutProps = {
params: TransformPageLayoutPropsParams;
+ fullWidth?: boolean;
children?: ReactNode;
};
@@ -47,18 +49,24 @@ export function DetailsPageLayout({
function TransformPageLayout({
params,
maw,
+ fullWidth,
children,
}: TransformPageLayoutOwnProps) {
return (
<AdminSettingsLayout
sidebar={<TransformPageSidebar params={params} />}
maw={maw}
+ fullWidth={fullWidth}
>
{children}
</AdminSettingsLayout>
);
}
+export function FullWidthTransformPageLayout(props: TransformPageLayoutProps) {
+ return <TransformPageLayout {...props} fullWidth />;
+}
+
type TransformPageSidebarProps = {
params: TransformPageLayoutPropsParams;
};
@@ -91,6 +99,7 @@ function TransformPageSidebar({ params }: TransformPageSidebarProps) {
icon="list"
active={pathname === runListUrl}
/>
+ {PLUGIN_TRANSFORMS_PYTHON.getTransformsNavLinks()}
</AdminNavWrapper>
);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
index 2d1b29b76de..87fac2e71e8 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
@@ -6,11 +6,12 @@ import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErr
import { useDispatch } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { useMetadataToasts } from "metabase/metadata/hooks";
+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
import {
useGetTransformQuery,
useUpdateTransformMutation,
} from "metabase-enterprise/api";
-import type { DatasetQuery, Transform } from "metabase-types/api";
+import type { Transform, TransformSource } from "metabase-types/api";
import { QueryEditor } from "../../components/QueryEditor";
import { getTransformUrl } from "../../urls";
@@ -53,13 +54,10 @@ export function TransformQueryPageBody({
const dispatch = useDispatch();
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
- const handleSave = async (query: DatasetQuery) => {
+ const handleSourceSave = async (source: TransformSource) => {
const { error } = await updateTransform({
id: transform.id,
- source: {
- type: "query",
- query,
- },
+ source,
});
if (error) {
@@ -74,12 +72,23 @@ export function TransformQueryPageBody({
dispatch(push(getTransformUrl(transform.id)));
};
+ if (transform.source.type === "python") {
+ return (
+ <PLUGIN_TRANSFORMS_PYTHON.TransformEditor
+ initialSource={transform.source}
+ isNew={false}
+ isSaving={isLoading}
+ onSave={handleSourceSave}
+ onCancel={handleCancel}
+ />
+ );
+ }
return (
<QueryEditor
- initialQuery={transform.source.query}
+ initialSource={transform.source}
isNew={false}
isSaving={isLoading}
- onSave={handleSave}
+ onSave={handleSourceSave}
onCancel={handleCancel}
/>
);
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts b/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts
index 22783c881e9..143828b9c07 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts
@@ -15,7 +15,9 @@ export function getTransformListUrl() {
return ROOT_URL;
}
-export function getNewTransformFromTypeUrl(type: DatasetQuery["type"]) {
+export function getNewTransformFromTypeUrl(
+ type: DatasetQuery["type"] | "python",
+) {
return `${ROOT_URL}/new/${type}`;
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
index 28cce8088fb..9e93c5ddd7c 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
@@ -4,8 +4,10 @@ import { hasFeature } from "metabase/admin/databases/utils";
import { parseTimestamp } from "metabase/lib/time-dayjs";
import type {
Database,
+ DatabaseId,
TransformRunMethod,
TransformRunStatus,
+ TransformSource,
} from "metabase-types/api";
export function parseTimestampWithTimezone(
@@ -62,3 +64,15 @@ export function doesDatabaseSupportTransforms(database?: Database): boolean {
hasFeature(database, "transforms/table")
);
}
+
+export function sourceDatabaseId(source: TransformSource): DatabaseId | null {
+ if (source.type === "query") {
+ return source.query.database;
+ }
+
+ if (source.type === "python") {
+ return source["source-database"];
+ }
+
+ return null;
+}
diff --git a/frontend/src/metabase-types/analytics/event.ts b/frontend/src/metabase-types/analytics/event.ts
index 406e71fc769..136f17be399 100644
--- a/frontend/src/metabase-types/analytics/event.ts
+++ b/frontend/src/metabase-types/analytics/event.ts
@@ -297,7 +297,7 @@ export type TransformJobTriggerManualRunEvent = ValidateEvent<{
export type TransformCreateEvent = ValidateEvent<{
event: "transform_create";
triggered_from: "transform-page-create-menu";
- event_detail: "query" | "native" | "saved-question";
+ event_detail: "query" | "native" | "python" | "saved-question";
}>;
export type TransformCreatedEvent = ValidateEvent<{
diff --git a/frontend/src/metabase-types/api/database.ts b/frontend/src/metabase-types/api/database.ts
index 35d3414f13e..13e19eb58ef 100644
--- a/frontend/src/metabase-types/api/database.ts
+++ b/frontend/src/metabase-types/api/database.ts
@@ -54,6 +54,7 @@ export type DatabaseFeature =
| "distinct-where"
| "saved-question-sandboxing"
| "split-part"
+ | "transforms/python"
| "transforms/table";
export interface Database extends DatabaseData {
diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.ts
index de38f9568bf..fc1a8e7b88f 100644
--- a/frontend/src/metabase-types/api/mocks/settings.ts
+++ b/frontend/src/metabase-types/api/mocks/settings.ts
@@ -139,6 +139,7 @@ export const createMockTokenFeatures = (
documents: false,
semantic_search: false,
transforms: false,
+ "transforms-python": false,
...opts,
});
diff --git a/frontend/src/metabase-types/api/mocks/transform.ts b/frontend/src/metabase-types/api/mocks/transform.ts
index 12b76827841..ef8c1333be4 100644
--- a/frontend/src/metabase-types/api/mocks/transform.ts
+++ b/frontend/src/metabase-types/api/mocks/transform.ts
@@ -1,4 +1,6 @@
import type {
+ DatabaseId,
+ PythonTransformTableAliases,
Transform,
TransformJob,
TransformRun,
@@ -9,13 +11,27 @@ import type {
import { createMockStructuredDatasetQuery } from "./query";
-export function createMockTransformSource(
- opts?: Partial<TransformSource>,
-): TransformSource {
+export function createMockTransformSource(): TransformSource {
return {
type: "query",
query: createMockStructuredDatasetQuery(),
- ...opts,
+ };
+}
+
+export function createMockPythonTransformSource({
+ sourceDatabase = 1,
+ sourceTables = {},
+ body = "# Python script\nprint('Hello, world!')",
+}: {
+ sourceDatabase?: DatabaseId;
+ sourceTables?: PythonTransformTableAliases;
+ body?: string;
+}): TransformSource {
+ return {
+ type: "python",
+ body,
+ "source-database": sourceDatabase,
+ "source-tables": sourceTables,
};
}
@@ -26,6 +42,7 @@ export function createMockTransformTarget(
type: "table",
name: "Table",
schema: null,
+ database: 1,
...opts,
};
}
diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.ts
index 103030c0b53..1e87e5ac8e3 100644
--- a/frontend/src/metabase-types/api/settings.ts
+++ b/frontend/src/metabase-types/api/settings.ts
@@ -324,6 +324,7 @@ export const tokenFeatures = [
"documents",
"semantic_search",
"transforms",
+ "transforms-python",
] as const;
export type TokenFeature = (typeof tokenFeatures)[number];
@@ -677,6 +678,15 @@ export interface EnterpriseSettings extends Settings {
"database-replication-connections"?: DatabaseReplicationConnections | null;
"embedding-hub-test-embed-snippet-created": boolean;
"embedding-hub-production-embed-snippet-created": boolean;
+ "python-runner-url"?: string | null;
+ "python-runner-api-token"?: string | null;
+ "python-storage-s-3-endpoint"?: string | null;
+ "python-storage-s-3-region"?: string | null;
+ "python-storage-s-3-bucket"?: string | null;
+ "python-storage-s-3-access-key"?: string | null;
+ "python-storage-s-3-secret-key"?: string | null;
+ "python-storage-s-3-container-endpoint"?: string | null;
+ "python-storage-s-3-path-style-access"?: boolean | null;
/**
* @deprecated
*/
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index 211cf76ad69..4987a03f2e7 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -1,7 +1,8 @@
+import type { DatabaseId } from "./database";
import type { PaginationRequest, PaginationResponse } from "./pagination";
import type { DatasetQuery } from "./query";
import type { ScheduleDisplayType } from "./settings";
-import type { Table } from "./table";
+import type { ConcreteTableId, Table } from "./table";
export type TransformId = number;
export type TransformTagId = number;
@@ -23,17 +24,28 @@ export type Transform = {
last_run?: TransformRun | null;
};
-export type TransformSource = {
+export type PythonTransformTableAliases = Record<string, ConcreteTableId>;
+
+export type PythonTransformSource = {
+ type: "python";
+ body: string;
+ "source-database": DatabaseId;
+ "source-tables": PythonTransformTableAliases;
+};
+export type QueryTransformSource = {
type: "query";
query: DatasetQuery;
};
+export type TransformSource = QueryTransformSource | PythonTransformSource;
+
export type TransformTargetType = "table";
export type TransformTarget = {
type: TransformTargetType;
name: string;
schema: string | null;
+ database: number;
};
export type TransformRun = {
@@ -146,3 +158,31 @@ export type ListTransformRunsRequest = {
export type ListTransformRunsResponse = {
data: TransformRun[];
} & PaginationResponse;
+
+export type ExecutePythonTransformRequest = {
+ code: string;
+ tables: PythonTransformTableAliases;
+};
+
+export type ExecutePythonTransformResponse = {
+ output?: string;
+ stdout?: string;
+ stderr?: string;
+ error?: string;
+ exit_code?: number;
+ timeout?: boolean;
+};
+
+export type PythonLibrary = {
+ path: string;
+ source: string;
+};
+
+export type GetPythonLibraryRequest = {
+ path: string;
+};
+
+export type UpdatePythonLibraryRequest = {
+ path: string;
+ source: string;
+};
diff --git a/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx b/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx
index e8f0ba2c059..5130adab58b 100644
--- a/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx
+++ b/frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx
@@ -23,6 +23,7 @@ export function SettingsNav() {
const hasSaml = useHasTokenFeature("sso_saml");
const hasJwt = useHasTokenFeature("sso_jwt");
const hasScim = useHasTokenFeature("scim");
+ const hasPythonTransforms = useHasTokenFeature("transforms-python");
return (
<AdminNavWrapper>
@@ -87,6 +88,13 @@ export function SettingsNav() {
</SettingsNavItem>
<NavDivider />
<SettingsNavItem path="uploads" label={t`Uploads`} icon="upload" />
+ {hasPythonTransforms && (
+ <SettingsNavItem
+ path="python-runner"
+ label={t`Python Runner`}
+ icon="snippet"
+ />
+ )}
<SettingsNavItem
path="public-sharing"
label={t`Public sharing`}
diff --git a/frontend/src/metabase/admin/settingsRoutes.tsx b/frontend/src/metabase/admin/settingsRoutes.tsx
index e49f855c47d..4b21388111c 100644
--- a/frontend/src/metabase/admin/settingsRoutes.tsx
+++ b/frontend/src/metabase/admin/settingsRoutes.tsx
@@ -2,7 +2,10 @@ import { IndexRedirect, Route } from "react-router";
import { AdminSettingsLayout } from "metabase/common/components/AdminLayout/AdminSettingsLayout";
import { NotFound } from "metabase/common/components/ErrorPages";
-import { PLUGIN_AUTH_PROVIDERS } from "metabase/plugins";
+import {
+ PLUGIN_AUTH_PROVIDERS,
+ PLUGIN_TRANSFORMS_PYTHON,
+} from "metabase/plugins";
import { GoogleAuthForm } from "./settings/auth/components/GoogleAuthForm";
import { SettingsLdapForm } from "./settings/components/SettingsLdapForm";
@@ -58,6 +61,10 @@ export const getSettingsRoutes = () => (
<Route path="maps" component={MapsSettingsPage} />
<Route path="localization" component={LocalizationSettingsPage} />
<Route path="uploads" component={UploadSettingsPage} />
+ <Route
+ path="python-runner"
+ component={PLUGIN_TRANSFORMS_PYTHON.PythonRunnerSettingsPage}
+ />
<Route path="public-sharing" component={PublicSharingSettingsPage} />
<Route path="license" component={LicenseSettingsPage} />
<Route path="appearance" component={() => <AppearanceSettingsPage />} />
diff --git a/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsx b/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsx
index 57005a4cd0d..8939afa159e 100644
--- a/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsx
+++ b/frontend/src/metabase/admin/tools/components/Logs/Logs.styled.tsx
@@ -1,24 +1,7 @@
// eslint-disable-next-line no-restricted-imports
-import { css } from "@emotion/react";
-// eslint-disable-next-line no-restricted-imports
import styled from "@emotion/styled";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
-import { color } from "metabase/lib/colors";
-
-const LOG_PREFIX = ".react-ansi-style";
-
-const LOG_COLORS = {
- black: color("text-dark"),
- white: color("text-white"),
- gray: color("text-medium"),
- red: color("saturated-red"),
- green: color("saturated-green"),
- yellow: color("saturated-yellow"),
- blue: color("saturated-blue"),
- magenta: color("saturated-purple"),
- cyan: "cyan",
-};
export const LogsContainer = styled(LoadingAndErrorWrapper)`
height: 100%;
@@ -36,41 +19,4 @@ export const LogsContent = styled.div`
padding: 1em;
overflow: auto;
height: 100%;
-
- ${LOG_PREFIX}-bold {
- font-weight: bold;
- }
-
- ${LOG_PREFIX}-dim {
- font-weight: lighter;
- }
-
- ${LOG_PREFIX}-italic {
- font-style: italic;
- }
-
- ${LOG_PREFIX}-underline {
- text-decoration: underline;
- }
-
- ${LOG_PREFIX}-inverse {
- color: var(--mb-color-text-dark);
- background-color: var(--mb-color-bg-white);
- }
-
- ${LOG_PREFIX}-hidden {
- color: transparent;
- }
-
- ${LOG_PREFIX}-strikethrough {
- text-decoration: line-through;
- }
-
- ${Object.entries(LOG_COLORS).map(
- ([key, color]) => css`
- ${LOG_PREFIX}-${key} {
- color: ${color};
- }
- `,
- )};
`;
diff --git a/frontend/src/metabase/admin/tools/components/Logs/Logs.tsx b/frontend/src/metabase/admin/tools/components/Logs/Logs.tsx
index 84ae61914ed..a7d94a441c2 100644
--- a/frontend/src/metabase/admin/tools/components/Logs/Logs.tsx
+++ b/frontend/src/metabase/admin/tools/components/Logs/Logs.tsx
@@ -9,6 +9,7 @@ import {
SettingsPageWrapper,
SettingsSection,
} from "metabase/admin/components/SettingsSection";
+import { AnsiLogs } from "metabase/common/components/AnsiLogs";
import Select, { Option } from "metabase/common/components/Select";
import { useUrlState } from "metabase/common/hooks/use-url-state";
import { openSaveDialog } from "metabase/lib/dom";
@@ -143,9 +144,14 @@ const LogsBase = ({
</Flex>
</Flex>
- <LogsContent id="logs-content" ref={scrollRef} onScroll={onScroll}>
+ <AnsiLogs
+ id="logs-content"
+ ref={scrollRef}
+ onScroll={onScroll}
+ component={LogsContent}
+ >
{displayLogs}
- </LogsContent>
+ </AnsiLogs>
</LogsContainer>
</SettingsSection>
</SettingsPageWrapper>
diff --git a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css
index 239e47c267c..877200c6645 100644
--- a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css
+++ b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css
@@ -22,7 +22,6 @@
display: flex;
justify-content: center;
width: 100%;
- padding: 2rem;
position: relative;
background-color: var(--mb-color-bg-light);
}
diff --git a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx
index 4de4c46c09a..4d06c526281 100644
--- a/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx
+++ b/frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx
@@ -10,11 +10,13 @@ import S from "./AdminSettingsLayout.module.css";
export const AdminSettingsLayout = ({
sidebar,
children,
+ fullWidth = false,
maw = "50rem",
}: {
sidebar?: React.ReactNode;
children?: React.ReactNode;
maw?: string;
+ fullWidth?: boolean;
}) => {
return (
<Box className={S.Wrapper}>
@@ -24,9 +26,13 @@ export const AdminSettingsLayout = ({
{sidebar}
</Box>
)}
- <Box className={S.Content} data-testid="admin-layout-content">
- <Box maw={maw} w="100%">
- <Box pb="2rem">
+ <Box
+ className={S.Content}
+ data-testid="admin-layout-content"
+ p={fullWidth ? 0 : "2rem"}
+ >
+ <Box maw={fullWidth ? undefined : maw} w="100%">
+ <Box pb={fullWidth ? 0 : "2rem"}>
<ErrorBoundary>{children ?? <NotFound />}</ErrorBoundary>
</Box>
</Box>
diff --git a/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.module.css b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.module.css
new file mode 100644
index 00000000000..30e417dfb4b
--- /dev/null
+++ b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.module.css
@@ -0,0 +1,64 @@
+.ansiLogs {
+ :global {
+ .react-ansi-style-bold {
+ font-weight: bold;
+ }
+
+ .react-ansi-style-dim {
+ font-weight: lighter;
+ }
+
+ .react-ansi-style-italic {
+ font-style: italic;
+ }
+
+ .react-ansi-style-inverse {
+ color: var(--mb-color-text-dark);
+ background-color: var(--mb-color-bg-white);
+ }
+
+ .react-ansi-style-hidden {
+ color: transparent;
+ }
+
+ .react-ansi-style-underline {
+ text-decoration: underline;
+ }
+
+ .react-ansi-style-strikethrough {
+ text-decoration: line-through;
+ }
+
+ .react-ansi-style-black {
+ color: var(--mb-color-text-dark);
+ }
+
+ .react-ansi-style-white {
+ color: var(--mb-color-text-white);
+ }
+
+ .react-ansi-style-gray {
+ color: var(--mb-color-text-light);
+ }
+
+ .react-ansi-style-red {
+ color: var(--mb-color-saturated-red);
+ }
+
+ .react-ansi-style-green {
+ color: var(--mb-color-saturated-green);
+ }
+
+ .react-ansi-style-yellow {
+ color: var(--mb-color-saturated-yellow);
+ }
+
+ .react-ansi-style-blue {
+ color: var(--mb-color-saturated-blue);
+ }
+
+ .react-ansi-style-purple {
+ color: var(--mb-color-saturated-purple);
+ }
+ }
+}
diff --git a/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.tsx b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.tsx
new file mode 100644
index 00000000000..4d1daaae9e2
--- /dev/null
+++ b/frontend/src/metabase/common/components/AnsiLogs/AnsiLogs.tsx
@@ -0,0 +1,36 @@
+import cx from "classnames";
+import * as React from "react";
+import { type Ref, forwardRef, useMemo } from "react";
+import reactAnsiStyle from "react-ansi-style";
+
+import {
+ Box,
+ type BoxComponentProps,
+ type BoxProps,
+ type PolymorphicComponentProps,
+} from "metabase/ui";
+
+import S from "./AnsiLogs.module.css";
+
+type LogOutputProps<C> = PolymorphicComponentProps<C, BoxComponentProps> &
+ BoxProps & {
+ children?: string;
+ };
+
+export const AnsiLogs = forwardRef(function AnsiLogsInner<C = "div">(
+ props: LogOutputProps<C>,
+ ref: Ref<HTMLDivElement>,
+) {
+ const { children, className, ...rest } = props;
+ const displayContent = useMemo(
+ () => reactAnsiStyle(React, children ?? ""),
+ [children],
+ );
+
+ return (
+ // @ts-expect-error: it's very hard to type the ref and component props from mantine
+ <Box ref={ref} className={cx(S.ansiLogs, className)} {...rest}>
+ {displayContent}
+ </Box>
+ );
+});
diff --git a/frontend/src/metabase/common/components/AnsiLogs/index.ts b/frontend/src/metabase/common/components/AnsiLogs/index.ts
new file mode 100644
index 00000000000..2587fabd2d1
--- /dev/null
+++ b/frontend/src/metabase/common/components/AnsiLogs/index.ts
@@ -0,0 +1 @@
+export * from "./AnsiLogs";
diff --git a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.css b/frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.css
deleted file mode 100644
index a5256630201..00000000000
--- a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.css
+++ /dev/null
@@ -1,8 +0,0 @@
-.codeEditor {
- :global {
- .cm-gutter {
- background-color: var(--mb-color-brand-light);
- color: var(--mb-color-brand);
- }
- }
-}
diff --git a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx b/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx
index fdf7f1d7d18..5741f9fb397 100644
--- a/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx
+++ b/frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx
@@ -1,9 +1,7 @@
import type { Extension } from "@uiw/react-codemirror";
-import cx from "classnames";
import { CodeMirror } from "metabase/common/components/CodeMirror";
-import S from "./CodeEditor.module.css";
import type { CodeLanguage } from "./types";
import { useExtensions } from "./utils";
@@ -16,6 +14,8 @@ type Props = {
readOnly?: boolean;
value: string;
onChange?: (value: string) => void;
+ extensions?: Extension[];
+ "data-testid"?: string;
};
export function CodeEditor({
@@ -27,8 +27,13 @@ export function CodeEditor({
readOnly,
value,
onChange,
+ extensions: externalExtensions,
+ ...rest
}: Props) {
- const extensions = useExtensions({ language });
+ const extensions = useExtensions({
+ language,
+ extensions: externalExtensions,
+ });
return (
<CodeMirror
@@ -38,13 +43,14 @@ export function CodeEditor({
highlightActiveLine: false,
highlightActiveLineGutter: false,
}}
- className={cx(S.codeEditor, className)}
+ className={className}
extensions={extensions}
id={id}
readOnly={readOnly}
value={value}
onChange={onChange}
highlightRanges={highlightRanges}
+ {...rest}
/>
);
}
diff --git a/frontend/src/metabase/common/components/CodeEditor/utils.ts b/frontend/src/metabase/common/components/CodeEditor/utils.ts
index 74a6a6b2430..6de3a92be40 100644
--- a/frontend/src/metabase/common/components/CodeEditor/utils.ts
+++ b/frontend/src/metabase/common/components/CodeEditor/utils.ts
@@ -3,7 +3,7 @@ import { javascript } from "@codemirror/lang-javascript";
import { json } from "@codemirror/lang-json";
import { python } from "@codemirror/lang-python";
import { sql } from "@codemirror/lang-sql";
-import { StreamLanguage } from "@codemirror/language";
+import { StreamLanguage, indentUnit } from "@codemirror/language";
import { clojure } from "@codemirror/legacy-modes/mode/clojure";
import { pug } from "@codemirror/legacy-modes/mode/pug";
import { ruby } from "@codemirror/legacy-modes/mode/ruby";
@@ -15,12 +15,17 @@ import type { CodeLanguage } from "./types";
export function useExtensions({
language,
+ extensions,
}: {
language?: CodeLanguage | Extension;
+ extensions?: Extension[];
}) {
return useMemo(
- () => [...(language ? [getLanguageExtension(language)] : [])],
- [language],
+ () => [
+ ...(extensions ?? []),
+ ...(language ? [getLanguageExtension(language)] : []),
+ ],
+ [language, extensions],
);
}
@@ -37,7 +42,7 @@ export function getLanguageExtension(language: CodeLanguage | Extension) {
case "json":
return json();
case "python":
- return python();
+ return [python(), indentUnit.of(" ")];
case "mustache":
return handlebars;
case "pug":
diff --git a/frontend/src/metabase/lib/errors/messages.ts b/frontend/src/metabase/lib/errors/messages.ts
index e75e7a0c88f..2455cdcff1a 100644
--- a/frontend/src/metabase/lib/errors/messages.ts
+++ b/frontend/src/metabase/lib/errors/messages.ts
@@ -31,5 +31,5 @@ export function getResponseErrorMessage(error: unknown): string | undefined {
}
export function isResourceNotFoundError(error: any) {
- return error?.status === 404;
+ return error instanceof Object && "status" in error && error.status === 404;
}
diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.ts
index 3223b9bcdfe..a940c7f5731 100644
--- a/frontend/src/metabase/plugins/index.ts
+++ b/frontend/src/metabase/plugins/index.ts
@@ -78,11 +78,14 @@ import type {
ModelCacheRefreshStatus,
ParameterId,
Pulse,
+ PythonTransformSource,
+ PythonTransformTableAliases,
Revision,
Series,
TableId,
Timeline,
TimelineEvent,
+ Transform,
User,
VisualizationDisplay,
} from "metabase-types/api";
@@ -838,3 +841,33 @@ export const PLUGIN_TRANSFORMS: TransformsPlugin = {
getAdminPaths: () => [],
getAdminRoutes: () => null,
};
+
+export type PythonTransformsPlugin = {
+ PythonRunnerSettingsPage: ComponentType;
+ SourceSection: ComponentType<{ transform: Transform }>;
+ TransformEditor: ComponentType<{
+ initialSource: {
+ type: "python";
+ body: string;
+ "source-database": DatabaseId | undefined;
+ "source-tables": PythonTransformTableAliases;
+ };
+ isNew?: boolean;
+ isSaving?: boolean;
+ isRunnable?: boolean;
+ onSave: (newSource: PythonTransformSource) => void;
+ onCancel: () => void;
+ }>;
+ getAdminRoutes: () => ReactNode;
+ getTransformsNavLinks: () => ReactNode;
+ getCreateTransformsMenuItems: () => ReactNode;
+};
+
+export const PLUGIN_TRANSFORMS_PYTHON: PythonTransformsPlugin = {
+ PythonRunnerSettingsPage: NotFoundPlaceholder,
+ TransformEditor: NotFoundPlaceholder,
+ SourceSection: PluginPlaceholder,
+ getAdminRoutes: () => null,
+ getTransformsNavLinks: () => null,
+ getCreateTransformsMenuItems: () => null,
+};
diff --git a/frontend/src/metabase/ui/components/utils/Box/index.ts b/frontend/src/metabase/ui/components/utils/Box/index.ts
index 5a9a65f2b4b..320e0657bce 100644
--- a/frontend/src/metabase/ui/components/utils/Box/index.ts
+++ b/frontend/src/metabase/ui/components/utils/Box/index.ts
@@ -1,2 +1,2 @@
export { Box } from "@mantine/core";
-export type { BoxProps } from "@mantine/core";
+export type { BoxProps, BoxComponentProps } from "@mantine/core";
diff --git a/frontend/src/metabase/ui/index.ts b/frontend/src/metabase/ui/index.ts
index 871cf98354d..d27e39618d8 100644
--- a/frontend/src/metabase/ui/index.ts
+++ b/frontend/src/metabase/ui/index.ts
@@ -6,6 +6,7 @@ export type {
MantineTheme,
MantineThemeOther,
MantineThemeOverride,
+ PolymorphicComponentProps,
} from "@mantine/core";
export { useHover } from "@mantine/hooks";
export * from "./components";
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index f3a4e491105..d01238d7bb5 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -13,11 +13,13 @@
[metabase.driver.bigquery-cloud-sdk.query-processor :as bigquery.qp]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
[metabase.driver.sql :as driver.sql]
+ [metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.util :as sql.u]
[metabase.driver.sync :as driver.s]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -40,6 +42,8 @@
Field$Mode
FieldValue
FieldValueList
+ InsertAllRequest
+ InsertAllRequest$RowToInsert
QueryJobConfiguration
Schema
Table
@@ -47,6 +51,7 @@
TableId
TableResult)
(com.google.common.collect ImmutableMap)
+ (com.google.gson JsonParser)
(java.util Iterator)))
(set! *warn-on-reflection* true)
@@ -232,6 +237,33 @@
"BIGNUMERIC" :type/Decimal
:type/*))
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for BigQuery that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+;; we can't recover the parameterized types
+(defmethod type->database-type :type/Array [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/Dictionary [_] [[:raw "JSON"]])
+
+(defmethod type->database-type :type/Boolean [_] [[:raw "BOOL"]])
+(defmethod type->database-type :type/Float [_] [[:raw "FLOAT64"]])
+(defmethod type->database-type :type/Integer [_] [[:raw "INT"]])
+(defmethod type->database-type :type/Number [_] [[:raw "INT"]])
+(defmethod type->database-type :type/Text [_] [[:raw "STRING"]])
+(defmethod type->database-type :type/TextLike [_] [[:raw "STRING"]])
+(defmethod type->database-type :type/Date [_] [[:raw "DATE"]])
+(defmethod type->database-type :type/DateTime [_] [[:raw "DATETIME"]])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [[:raw "TIMESTAMP"]])
+(defmethod type->database-type :type/Time [_] [[:raw "TIME"]])
+(defmethod type->database-type :type/JSON [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/SerializedJSON [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/Decimal [_] [[:raw "BIGDECIMAL"]])
+
+(defmethod driver/type->database-type :bigquery-cloud-sdk
+ [_driver base-type]
+ (type->database-type base-type))
+
(defn- field->database+base-type
"Returns a normalized `database-type` and its `base-type` for a type from BigQuery Field type.
@@ -737,6 +769,9 @@
:expressions true
:now true
:percentile-aggregations true
+ ;; we can't support `alter table .. rename ..` in general
+ ;; since it won't work for streaming tables
+ :rename false
:metadata/key-constraints false
:identifiers-with-spaces true
:expressions/integer true
@@ -752,6 +787,7 @@
:expression-literals true
:database-routing true
:metadata/table-existence-check true
+ :transforms/python true
:transforms/table true}]
(defmethod driver/database-supports? [:bigquery-cloud-sdk feature] [_driver _feature _db] supported?))
@@ -838,6 +874,85 @@
(let [table-str (get-table-str table)]
[(str "DROP TABLE IF EXISTS " table-str)]))
+(defmethod driver/create-table! :bigquery-cloud-sdk
+ [driver database-id table-name column-definitions & {:keys [primary-key]}]
+ (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)]
+ (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+
+(defmethod driver/drop-table! :bigquery-cloud-sdk
+ [driver database-id table-name]
+ (let [sql (driver/compile-drop-table driver table-name)]
+ (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+
+(defn- convert-value-for-insertion
+ [base-type value]
+ (condp #(isa? %2 %1) base-type
+ :type/JSON
+ (.toString (JsonParser/parseString value))
+
+ :type/Dictionary
+ (JsonParser/parseString value)
+
+ :type/Array
+ (JsonParser/parseString value)
+
+ :type/Integer
+ (parse-long value)
+
+ :type/Float
+ (parse-double value)
+
+ :type/Boolean
+ (parse-boolean value)
+
+ :type/Numeric
+ (bigdec value)
+
+ :type/Decimal
+ (bigdec value)
+
+ :type/Date
+ (u.date/format (u.date/parse value))
+
+ :type/DateTime
+ (u.date/format (u.date/parse value))
+
+ :type/DateTimeWithLocalTZ
+ (u.date/format (u.date/parse value))
+
+ value))
+
+(defmethod driver/insert-col->val [:bigquery-cloud-sdk :jsonl-file]
+ [_driver _ column-def v]
+ (if (string? v)
+ (convert-value-for-insertion (:type column-def) v)
+ v))
+
+(defmethod driver/insert-from-source! [:bigquery-cloud-sdk :rows]
+ [_driver db-id {table-name :name :keys [columns]} {:keys [data]}]
+ (let [col-names (map :name columns)
+ {:keys [details]} (t2/select-one :model/Database db-id)
+
+ client (database-details->client details)
+ project-id (get-project-id details)
+
+ dataset-id (namespace table-name)
+ table-name (name table-name)
+
+ table-id (.getTableId (get-table client project-id dataset-id table-name))
+
+ prepared-rows (map #(into {} (map vector col-names %)) data)]
+ (doseq [chunk (partition-all (or driver/*insert-chunk-rows* 1000) prepared-rows)]
+ (let [insert-request-builder (InsertAllRequest/newBuilder table-id)]
+ (doseq [^java.util.Map row chunk]
+ (.addRow insert-request-builder (InsertAllRequest$RowToInsert/of row)))
+ (let [insert-request (.build insert-request-builder)
+ response (.insertAll client insert-request)]
+ (when (.hasErrors response)
+ (let [errors (.getInsertErrors response)]
+ (throw (ex-info "BigQuery insert failed"
+ {:errors (into [] (map str errors))})))))))))
+
(defmethod driver/execute-raw-queries! :bigquery-cloud-sdk
[_driver connection-details queries]
;; connection-details is either database details directly (from transforms)
@@ -905,3 +1020,8 @@
:details
list-datasets
(some #{schema}))))
+
+(defmethod driver/table-name-length-limit :bigquery-cloud-sdk
+ [_driver]
+ ;; https://cloud.google.com/bigquery/docs/tables
+ 1024)
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index 2fe382680f2..e9a381131ab 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -1327,3 +1327,22 @@
(is (=? [["2024-12-11T16:23:55.123456Z" #"2024-12-11T16:23:55.123456.*"]]
(-> (qp/process-query query)
mt/rows))))))
+
+(deftest ^:parallel type->database-type-test
+ (testing "type->database-type multimethod returns correct BigQuery types"
+ (are [base-type expected] (= expected (driver/type->database-type :bigquery-cloud-sdk base-type))
+ :type/Array [[:raw "JSON"]]
+ :type/Dictionary [[:raw "JSON"]]
+ :type/Boolean [[:raw "BOOL"]]
+ :type/Float [[:raw "FLOAT64"]]
+ :type/Integer [[:raw "INT"]]
+ :type/Number [[:raw "INT"]]
+ :type/Text [[:raw "STRING"]]
+ :type/TextLike [[:raw "STRING"]]
+ :type/Date [[:raw "DATE"]]
+ :type/DateTime [[:raw "DATETIME"]]
+ :type/DateTimeWithTZ [[:raw "TIMESTAMP"]]
+ :type/Time [[:raw "TIME"]]
+ :type/JSON [[:raw "JSON"]]
+ :type/SerializedJSON [[:raw "JSON"]]
+ :type/Decimal [[:raw "BIGDECIMAL"]])))
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
index 7d82580335e..354a304ea7d 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj
@@ -486,3 +486,11 @@
(execute! "select name from `%s`.metabase_test_tracking.datasets order by accessed_at" (project-id))
(database-exists?! (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))
+
+(defn ^:private get-test-data-name
+ []
+ (test-dataset-id
+ (tx/get-dataset-definition (or data.impl/*dbdef-used-to-create-db*
+ (tx/default-dataset :bigquery-cloud-sdk)))))
+
+(defmethod sql.tx/session-schema :bigquery-cloud-sdk [_driver] (get-test-data-name))
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index 482e2d3526f..dff992030f4 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -2,6 +2,7 @@
"Driver for ClickHouse databases"
(:require
[clojure.core.memoize :as memoize]
+ [clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
@@ -52,9 +53,11 @@
:window-functions/cumulative (not driver-api/is-test?)
:left-join (not driver-api/is-test?)
:describe-fks false
+ :rename true
:actions false
:metadata/key-constraints false
:database-routing true
+ :transforms/python true
:transforms/table true}]
(defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?))
@@ -212,6 +215,28 @@
:metabase.upload/datetime "Nullable(DateTime64(3))"
:metabase.upload/offset-datetime nil))
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for ClickHouse that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/Boolean [_] [[:raw "Nullable(Boolean)"]])
+(defmethod type->database-type :type/Float [_] [[:raw "Nullable(Float64)"]])
+(defmethod type->database-type :type/Integer [_] [[:raw "Nullable(Int32)"]])
+(defmethod type->database-type :type/Number [_] [[:raw "Nullable(Int64)"]])
+(defmethod type->database-type :type/BigInteger [_] [[:raw "Nullable(Int64)"]])
+(defmethod type->database-type :type/Text [_] [[:raw "Nullable(String)"]])
+(defmethod type->database-type :type/TextLike [_] [[:raw "Nullable(String)"]])
+(defmethod type->database-type :type/Date [_] [[:raw "Nullable(Date32)"]])
+(defmethod type->database-type :type/Time [_] [[:raw "Nullable(Time)"]])
+(defmethod type->database-type :type/DateTime [_] [[:raw "Nullable(DateTime64(3))"]])
+;; we're lossy here
+(defmethod type->database-type :type/DateTimeWithTZ [_] [[:raw "Nullable(DateTime64(3, 'UTC'))"]])
+
+(defmethod driver/type->database-type :clickhouse
+ [_driver base-type]
+ (type->database-type base-type))
+
(defmethod driver/table-name-length-limit :clickhouse
[_driver]
;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses
@@ -219,7 +244,8 @@
206)
(defn- quote-name [s]
- (let [parts (str/split (name s) #"\.")]
+ (let [s (if (and (keyword? s) (namespace s)) (str (namespace s) "." (name s)) s)
+ parts (filter identity (str/split (name s) #"\."))]
(str/join "." (map #(str "`" % "`") parts))))
(defn- create-table!-sql
@@ -241,7 +267,18 @@
{:write? true}
(fn [^java.sql.Connection conn]
(with-open [stmt (.createStatement conn)]
- (.execute stmt (create-table!-sql driver table-name column-definitions :primary-key primary-key))))))
+ (let [sql (create-table!-sql driver table-name column-definitions :primary-key primary-key)]
+ (.execute stmt sql))))))
+
+;; rename-tables!* only supported by the atomic engine
+;; https://clickhouse.com/docs/engines/database-engines/atomic#exchange-tables
+
+(defmethod driver/rename-table! :clickhouse
+ [_driver db-id old-table-name new-table-name]
+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+ (with-open [stmt (.createStatement ^java.sql.Connection (:connection conn))]
+ (let [sql (format "RENAME TABLE %s TO %s" (quote-name old-table-name) (quote-name new-table-name))]
+ (.execute stmt sql)))))
(defmethod driver/insert-into! :clickhouse
[driver db-id table-name column-names values]
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 80533a389e6..904cda43911 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -358,6 +358,19 @@
"Failed to get server info"
"Code: 516. DB::Exception: asdf: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) (version 25.7.4.11 (official build))"]))))
+(deftest ^:parallel type->database-type-test
+ (testing "type->database-type multimethod returns correct ClickHouse types"
+ (are [base-type expected] (= expected (driver/type->database-type :clickhouse base-type))
+ :type/Boolean [[:raw "Nullable(Boolean)"]]
+ :type/Float [[:raw "Nullable(Float64)"]]
+ :type/Integer [[:raw "Nullable(Int32)"]]
+ :type/Number [[:raw "Nullable(Int64)"]]
+ :type/Text [[:raw "Nullable(String)"]]
+ :type/TextLike [[:raw "Nullable(String)"]]
+ :type/Date [[:raw "Nullable(Date32)"]]
+ :type/DateTime [[:raw "Nullable(DateTime64(3))"]]
+ :type/DateTimeWithTZ [[:raw "Nullable(DateTime64(3, 'UTC'))"]])))
+
(deftest ^:parallel query-with-cte-subquery-and-param-test
(mt/test-driver :clickhouse
(testing "a query with a CTE in a subquery and a parameter should work correctly"
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 3fc954435eb..89b1373c33f 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -1,6 +1,6 @@
(ns metabase.driver.mongo
"MongoDB Driver."
- (:refer-clojure :exclude [some])
+ (:refer-clojure :exclude [some mapv])
(:require
[clojure.set :as set]
[clojure.string :as str]
@@ -18,9 +18,10 @@
[metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.json :as json]
[metabase.util.log :as log]
- [metabase.util.performance :refer [some]]
+ [metabase.util.performance :refer [some mapv]]
[taoensso.nippy :as nippy])
(:import
(com.mongodb.client MongoClient MongoDatabase)
@@ -426,6 +427,7 @@
:nested-queries true
:set-timezone true
:standard-deviation-aggregations true
+ :rename true
:test/jvm-timezone-setting false
:identifiers-with-spaces true
:saved-question-sandboxing false
@@ -435,6 +437,8 @@
:expressions/today true
;; Index sync is turned off across the application as it is not used ATM.
:index-info false
+ :python-transforms true
+ :transforms/python true
:database-routing true}]
(defmethod driver/database-supports? [:mongo feature] [_driver _feature _db] supported?))
@@ -555,6 +559,146 @@
(log/debugf e "Query:\n%s" native-form)
native-form)))
+(defmethod driver/create-table! :mongo
+ [_driver database-id table-name _column-definitions & {:keys [primary-key]}]
+ ;; MongoDB collections are created implicitly when first document is inserted
+ ;; We can create an empty collection explicitly if needed
+ (mongo.connection/with-mongo-database [^MongoDatabase db database-id]
+ (.createCollection db (name table-name))
+ ;; Create indexes for any primary key fields
+ (when primary-key
+ (doseq [pk-field primary-key]
+ (mongo.util/create-index
+ (mongo.util/collection db (name table-name))
+ {pk-field 1})))))
+
+(defmethod driver/drop-table! :mongo
+ [_driver db-id table-name]
+ (mongo.connection/with-mongo-database [^MongoDatabase db db-id]
+ (some-> (mongo.util/collection db (name table-name))
+ .drop)))
+
+(defmethod driver/rename-table! :mongo
+ [_driver db-id old-table-name new-table-name]
+ (mongo.connection/with-mongo-database [^MongoDatabase db db-id]
+ (let [old-collection (mongo.util/collection db (name old-table-name))]
+ (.renameCollection old-collection
+ (com.mongodb.MongoNamespace.
+ (.getName db)
+ (name new-table-name))))))
+
+(defmethod driver/drop-transform-target! [:mongo :table]
+ [driver database target]
+ (driver/drop-table! driver (:id database) (:name target)))
+
+(defmethod driver/connection-spec :mongo
+ [_driver database]
+ (:details database))
+
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for MongoDB that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/TextLike [_] "string")
+(defmethod type->database-type :type/Text [_] "string")
+(defmethod type->database-type :type/Number [_] "long")
+(defmethod type->database-type :type/Integer [_] "int")
+(defmethod type->database-type :type/BigInteger [_] "long")
+(defmethod type->database-type :type/Float [_] "double")
+(defmethod type->database-type :type/Decimal [_] "decimal")
+(defmethod type->database-type :type/Boolean [_] "bool")
+(defmethod type->database-type :type/Date [_] "date")
+(defmethod type->database-type :type/DateTime [_] "date")
+(defmethod type->database-type :type/DateTimeWithTZ [_] "date")
+(defmethod type->database-type :type/Time [_] "date")
+(defmethod type->database-type :type/TimeWithTZ [_] "date")
+(defmethod type->database-type :type/Instant [_] "date")
+(defmethod type->database-type :type/UUID [_] "uuid")
+(defmethod type->database-type :type/JSON [_] "object")
+(defmethod type->database-type :type/SerializedJSON [_] "string")
+(defmethod type->database-type :type/Array [_] "array")
+(defmethod type->database-type :type/Dictionary [_] "object")
+(defmethod type->database-type :type/MongoBSONID [_] "objectId")
+(defmethod type->database-type :type/MongoBinData [_] "binData")
+(defmethod type->database-type :type/IPAddress [_] "string")
+(defmethod type->database-type :default [_] "object")
+
+(defmethod driver/type->database-type :mongo
+ [_driver base-type]
+ (type->database-type base-type))
+
+(defn- convert-value-for-insertion
+ [base-type value]
+ (condp #(isa? %2 %1) base-type
+ :type/JSON
+ (json/decode value)
+
+ :type/Dictionary
+ (json/decode value)
+
+ :type/Array
+ (json/decode value)
+
+ :type/Integer
+ (parse-long value)
+
+ :type/BigInteger
+ (bigint value)
+
+ :type/Float
+ (parse-double value)
+
+ :type/Decimal
+ (bigdec value)
+
+ :type/Number
+ (bigint value)
+
+ :type/Boolean
+ (parse-boolean value)
+
+ :type/Date
+ (u.date/parse value)
+
+ :type/DateTime
+ (u.date/parse value)
+
+ :type/DateTimeWithTZ
+ (u.date/parse value)
+
+ :type/Time
+ (u.date/parse value)
+
+ :type/TimeWithTZ
+ (u.date/parse value)
+
+ :type/Instant
+ (u.date/parse value)
+
+ :type/UUID
+ (parse-uuid value)
+
+ value))
+
+(defmethod driver/insert-col->val [:mongo :jsonl-file]
+ [_driver _ column-def v]
+ (if (string? v)
+ (convert-value-for-insertion (:type column-def) v)
+ v))
+
+(defmethod driver/insert-from-source! [:mongo :rows]
+ [_driver db-id {table-name :name :keys [columns]} {:keys [data]}]
+ (let [col-names (mapv :name columns)]
+ (mongo.connection/with-mongo-database [^MongoDatabase db db-id]
+ (let [collection (mongo.util/collection db (name table-name))
+ documents (map #(into {} (map vector col-names %))
+ data)]
+ (if (> (bounded-count 2 documents) 1)
+ ;; TODO: batch to avoid realizing everything in memory
+ (mongo.util/insert-many collection documents)
+ (mongo.util/insert-one collection (first documents)))))))
+
;; Following code is using monger. Leaving it here for a reference as it could be transformed when there is need
;; for ssl experiments.
#_(comment
@@ -618,3 +762,7 @@
credentials)]
(mg/get-db-names unauthenticated-connection)))
:.)
+
+(defmethod driver/table-name-length-limit :mongo
+ [_driver]
+ 64)
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
index de6307e2678..b0b89eb11a4 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
@@ -215,9 +215,10 @@
db-name (mongo.db/db-name database)
client-database (mongo.util/database mongo.connection/*mongo-client* db-name)]
(with-open [session ^ClientSession (mongo.util/start-session! mongo.connection/*mongo-client*)]
- (a/go
- (when (a/<! (driver-api/canceled-chan))
- (mongo.util/kill-session! client-database session)))
+ (when-let [cancel-chan (driver-api/canceled-chan)]
+ (a/go
+ (when (a/<! cancel-chan)
+ (mongo.util/kill-session! client-database session))))
(let [aggregate ^AggregateIterable (*aggregate* client-database
collection-name
session
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
index fcd69ba2d14..a677ca0b1ad 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
@@ -1148,3 +1148,29 @@
(is (= [[1 "ref1" ref-1 "some_collection" id-1 nil]
[2 "ref2" ref-2 "some_collection" id-2 "some_db"]]
(mt/rows (mt/run-mbql-query dbref_coll))))))))
+
+(deftest ^:parallel type->database-type-test
+ (testing "type->database-type multimethod returns correct MongoDB types"
+ (are [base-type expected] (= expected (driver/type->database-type :mongo base-type))
+ :type/TextLike "string"
+ :type/Text "string"
+ :type/Number "long"
+ :type/Integer "int"
+ :type/BigInteger "long"
+ :type/Float "double"
+ :type/Decimal "decimal"
+ :type/Boolean "bool"
+ :type/Date "date"
+ :type/DateTime "date"
+ :type/DateTimeWithTZ "date"
+ :type/Time "date"
+ :type/TimeWithTZ "date"
+ :type/Instant "date"
+ :type/UUID "uuid"
+ :type/JSON "object"
+ :type/SerializedJSON "string"
+ :type/Array "array"
+ :type/Dictionary "object"
+ :type/MongoBSONID "objectId"
+ :type/MongoBinData "binData"
+ :type/IPAddress "string")))
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.clj
index 450d6df4945..80d4ccd411f 100644
--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj
+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj
@@ -37,6 +37,8 @@
(doseq [[feature supported?] {:connection-impersonation true
:describe-fields true
:describe-fks true
+ :rename true
+ :atomic-renames true
:expression-literals true
:identifiers-with-spaces false
:uuid-type false
@@ -44,6 +46,7 @@
:test/jvm-timezone-setting false
:database-routing true
:metadata/table-existence-check true
+ :transforms/python true
:transforms/table true}]
(defmethod driver/database-supports? [:redshift feature] [_driver _feat _db] supported?))
@@ -539,6 +542,27 @@
:metabase.upload/datetime [:timestamp]
:metabase.upload/offset-datetime [:timestamp-with-time-zone]))
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for Redshift that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/TextLike [_] [[:varchar 65535]])
+(defmethod type->database-type :type/Text [_] [[:varchar 65535]])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/BigInteger [_] [:bigint])
+(defmethod type->database-type :type/Integer [_] [:integer])
+(defmethod type->database-type :type/Float [_] [(keyword "double precision")])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+
+(defmethod driver/type->database-type :redshift
+ [_driver base-type]
+ (type->database-type base-type))
+
(defmethod driver/allowed-promotions :redshift [_] {})
(defmethod driver/table-name-length-limit :redshift
diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
index 04e793143a9..7be1bf6dd84 100644
--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
@@ -519,3 +519,17 @@
(testing (format "database-type %s" (pr-str database-type))
(is (= exp-base-type
(sql-jdbc.sync/database-type->base-type :redshift database-type))))))
+
+(deftest ^:parallel type->database-type-test
+ (testing "type->database-type multimethod returns correct Redshift types"
+ (are [base-type expected] (= expected (driver/type->database-type :redshift base-type))
+ :type/TextLike [[:varchar 65535]]
+ :type/Text [[:varchar 65535]]
+ :type/Integer [:integer]
+ :type/Float [(keyword "double precision")]
+ :type/Number [:bigint]
+ :type/Boolean [:boolean]
+ :type/Date [:date]
+ :type/DateTime [:timestamp]
+ :type/DateTimeWithTZ [:timestamp-with-time-zone]
+ :type/Time [:time])))
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index 0bf79b03bdd..c8eb9ff3311 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -6,6 +6,7 @@
[clojure.java.jdbc :as jdbc]
[clojure.set :as set]
[clojure.string :as str]
+ [honey.sql :as sql]
[java-time.api :as t]
[medley.core :as m]
[metabase.driver :as driver]
@@ -55,6 +56,7 @@
(doseq [[feature supported?] {:connection-impersonation true
:connection-impersonation-requires-role true
+ :rename true
:convert-timezone true
:datetime-diff true
:describe-fields false
@@ -68,6 +70,7 @@
:now true
:database-routing true
:metadata/table-existence-check true
+ :transforms/python true
:transforms/table true}]
(defmethod driver/database-supports? [:snowflake feature] [_driver _feature _db] supported?))
@@ -264,7 +267,7 @@
:FLOAT4 :type/Float
:FLOAT8 :type/Float
:DOUBLE :type/Float
- (keyword "DOUBLE PRECISON") :type/Float
+ (keyword "DOUBLE PRECISION") :type/Float
:REAL :type/Float
:VARCHAR :type/Text
:CHAR :type/Text
@@ -292,6 +295,29 @@
:OBJECT :type/Dictionary
:ARRAY :type/*} base-type))
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for Snowflake that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/Array [_] [:ARRAY])
+(defmethod type->database-type :type/Boolean [_] [:BOOLEAN])
+(defmethod type->database-type :type/Date [_] [:DATE])
+(defmethod type->database-type :type/DateTime [_] [:DATETIME])
+(defmethod type->database-type :type/DateTimeWithLocalTZ [_] [:TIMESTAMPTZ])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:TIMESTAMPLTZ])
+(defmethod type->database-type :type/Decimal [_] [:DECIMAL])
+(defmethod type->database-type :type/Float [_] [:DOUBLE])
+(defmethod type->database-type :type/Number [_] [:BIGINT])
+(defmethod type->database-type :type/BigInteger [_] [:BIGINT])
+(defmethod type->database-type :type/Integer [_] [:INTEGER])
+(defmethod type->database-type :type/Text [_] [:TEXT])
+(defmethod type->database-type :type/Time [_] [:TIME])
+
+(defmethod driver/type->database-type :snowflake
+ [_driver base-type]
+ (type->database-type base-type))
+
(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :seconds] [_ _ expr] [:to_timestamp_tz expr])
(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :milliseconds] [_ _ expr] [:to_timestamp_tz expr 3])
(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :microseconds] [_ _ expr] [:to_timestamp_tz expr 6])
@@ -873,3 +899,17 @@
[driver conn-spec schema]
(let [sql [[(format "CREATE SCHEMA IF NOT EXISTS \"%s\";" schema)]]]
(driver/execute-raw-queries! driver conn-spec sql)))
+
+(defmethod driver/rename-table! :snowflake
+ [driver db-id from-table to-table]
+ (let [sql (first (sql/format {:alter-table (keyword from-table)
+ :rename-table (keyword to-table)}
+ :quoted true
+ :dialect (sql.qp/quote-style driver)))]
+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+ (jdbc/execute! conn sql))))
+
+(defmethod driver/table-name-length-limit :snowflake
+ [_driver]
+ ;; https://docs.snowflake.com/en/sql-reference/identifiers
+ 255)
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index c3ce83824b0..8691be2d24a 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -1305,3 +1305,19 @@
(is (true? details-changed?))
(is (= original-priv-key priv-key-after-update))
(is (not= priv-key-after-update priv-key-after-event)))))))))
+
+(deftest ^:parallel type->database-type-test
+ (testing "type->database-type multimethod returns correct Snowflake types"
+ (are [base-type expected] (= expected (driver/type->database-type :snowflake base-type))
+ :type/Array [:ARRAY]
+ :type/Boolean [:BOOLEAN]
+ :type/Date [:DATE]
+ :type/DateTime [:DATETIME]
+ :type/DateTimeWithLocalTZ [:TIMESTAMPTZ]
+ :type/DateTimeWithTZ [:TIMESTAMPLTZ]
+ :type/Decimal [:DECIMAL]
+ :type/Float [:DOUBLE]
+ :type/Number [:BIGINT]
+ :type/Integer [:INTEGER]
+ :type/Text [:TEXT]
+ :type/Time [:TIME])))
diff --git a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
index 6dd8e353c9d..a4c486eaf41 100644
--- a/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
+++ b/modules/drivers/snowflake/test/metabase/test/data/snowflake.clj
@@ -379,3 +379,5 @@
order by created"]
(jdbc/reducible-query (no-db-connection-spec))
(into [] (map (juxt :database_name :created)))))
+
+(defmethod sql.tx/session-schema :snowflake [_driver] "PUBLIC")
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index d6cd9fc0178..aa50db2166b 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -4,6 +4,7 @@
(:require
[clojure.data.xml :as xml]
[clojure.java.io :as io]
+ [clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[honey.sql :as sql]
[honey.sql.helpers :as sql.helpers]
@@ -24,6 +25,7 @@
[metabase.driver.sql.util :as sql.u]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
+ [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.performance :as perf :refer [mapv]])
(:import
@@ -54,6 +56,7 @@
:connection-impersonation-requires-role true
:uuid-type true
:convert-timezone true
+ :rename true
:datetime-diff true
:expression-literals true
;; Index sync is turned off across the application as it is not used ATM.
@@ -62,6 +65,7 @@
:regex false
:test/jvm-timezone-setting false
:metadata/table-existence-check true
+ :transforms/python true
:transforms/table true
:jdbc/statements false}]
(defmethod driver/database-supports? [:sqlserver feature] [_driver _feature _db] supported?))
@@ -121,6 +125,28 @@
:xml :type/*
(keyword "int identity") :type/Integer} column-type)) ; auto-incrementing integer (ie pk) field
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for SQL Server that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/Boolean [_] [:bit])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:datetime2])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:datetimeoffset])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Float [_] [:float])
+(defmethod type->database-type :type/Integer [_] [:int])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/BigInteger [_] [:bigint])
+(defmethod type->database-type :type/Text [_] [:text])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/UUID [_] [:uniqueidentifier])
+
+(defmethod driver/type->database-type :sqlserver
+ [_driver base-type]
+ (type->database-type base-type))
+
(defmethod sql-jdbc.conn/connection-details->spec :sqlserver
[_ {:keys [user password db host port instance domain ssl]
:or {user "dbuser", password "dbpassword", db "", host "localhost"}
@@ -938,6 +964,21 @@
[_ e]
(= (sql-jdbc/get-sql-state e) "S0002"))
+(defmethod driver/insert-from-source! [:sqlserver :jsonl-file]
+ [driver db-id {:keys [columns] :as table-definition} {:keys [file]}]
+ (with-open [rdr (io/reader file)]
+ (let [lines (line-seq rdr)
+ data-rows (map (fn [line]
+ (let [m (json/decode line)]
+ (mapv (fn [column]
+ (let [value (get m (:name column))]
+ (if (boolean? value)
+ (if value 1 0)
+ value)))
+ columns)))
+ lines)]
+ (driver/insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))
+
(defmethod driver/compile-transform :sqlserver
[driver {:keys [query output-table]}]
(let [^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))
@@ -966,3 +1007,15 @@
[driver conn-spec schema]
(let [sql [[(format "IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '%s') EXEC('CREATE SCHEMA [%s];');" schema schema)]]]
(driver/execute-raw-queries! driver conn-spec sql)))
+
+(defmethod driver/rename-table! :sqlserver
+ [_driver db-id old-table-name new-table-name]
+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+ (with-open [stmt (.createStatement ^java.sql.Connection (:connection conn))]
+ (let [sql (format "EXEC sp_rename '%s', '%s';" (name old-table-name) (name new-table-name))]
+ (.execute stmt sql)))))
+
+(defmethod driver/table-name-length-limit :sqlserver
+ [_driver]
+ ;; https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms191240(v=sql.105)#sysname
+ 128)
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index 824599e69de..2e5e6ff1cb7 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -756,3 +756,18 @@
{:filter [:expression "NameEquals" {:base-type :type/Boolean
driver-api/qp.add.source-table driver-api/qp.add.none
driver-api/qp.add.desired-alias nil}]})))))))
+
+(deftest ^:parallel type->database-type-test
+ (testing "type->database-type multimethod returns correct SQL Server types"
+ (are [base-type expected] (= expected (driver/type->database-type :sqlserver base-type))
+ :type/Boolean [:bit]
+ :type/Date [:date]
+ :type/DateTime [:datetime2]
+ :type/DateTimeWithTZ [:datetimeoffset]
+ :type/Decimal [:decimal]
+ :type/Float [:float]
+ :type/Integer [:int]
+ :type/Number [:bigint]
+ :type/Text [:text]
+ :type/Time [:time]
+ :type/UUID [:uniqueidentifier])))
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 914ade6c8e3..6b9e032c1b0 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -401,7 +401,56 @@
(prometheus/counter :metabase-gsheets/connection-deleted
{:description "How many times the instance has deleted their Google Sheets connection."})
(prometheus/counter :metabase-gsheets/connection-manually-synced
- {:description "How many times the instance has manually sync'ed their Google Sheets connection."})])
+ {:description "How many times the instance has manually sync'ed their Google Sheets connection."})
+
+ ;; transform metrics
+ (prometheus/counter :metabase-transforms/job-runs-total
+ {:description "Total number of transform job runs started."
+ :labels [:job-id :run-method]})
+ (prometheus/counter :metabase-transforms/job-runs-completed
+ {:description "Number of transform job runs that completed successfully."
+ :labels [:job-id :run-method]})
+ (prometheus/counter :metabase-transforms/job-runs-failed
+ {:description "Number of transform job runs that failed."
+ :labels [:job-id :run-method]})
+ (prometheus/histogram :metabase-transforms/job-run-duration-ms
+ {:description "Duration in milliseconds of transform job runs."
+ :labels [:job-id :run-method]
+ ;; 100ms -> 6 hours
+ :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000 7200000 14400000 21600000]})
+ (prometheus/counter :metabase-transforms/stage-started
+ {:description "Number of transform stages started."
+ :labels [:job-run-id :stage-type :stage-label]})
+ (prometheus/counter :metabase-transforms/stage-completed
+ {:description "Number of transform stages completed successfully."
+ :labels [:job-run-id :stage-type :stage-label]})
+ (prometheus/counter :metabase-transforms/stage-failed
+ {:description "Number of transform stages that failed."
+ :labels [:job-run-id :stage-type :stage-label]})
+ (prometheus/histogram :metabase-transforms/stage-duration-ms
+ {:description "Duration in milliseconds of individual transform stages."
+ :labels [:job-run-id :stage-type :stage-label]
+ ;; 10ms -> 10 minutes
+ :buckets [10 100 500 1000 5000 10000 30000 60000 300000 600000]})
+ (prometheus/histogram :metabase-transforms/data-transfer-bytes
+ {:description "Size in bytes of data transferred during transform stages."
+ :labels [:job-run-id :stage-label]
+ ;; 1KB -> 10GB
+ :buckets [1000 10000 100000 1000000 10000000 100000000 1000000000 10000000000]})
+ (prometheus/histogram :metabase-transforms/data-transfer-rows
+ {:description "Number of rows transferred during transform stages."
+ :labels [:job-run-id :stage-label]
+ ;; 10 -> 10M rows
+ :buckets [10 100 1000 10000 100000 1000000 10000000]})
+ ;; Python-transform specific metrics
+ (prometheus/histogram :metabase-transforms/python-api-call-duration-ms
+ {:description "Duration of Python runner API calls."
+ :labels [:job-run-id]
+ ;; 100ms -> 6 hours
+ :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000 7200000 14400000 21600000]})
+ (prometheus/counter :metabase-transforms/python-api-calls-total
+ {:description "Total number of Python runner API calls."
+ :labels [:job-run-id :status]})])
(defn- quartz-collectors
[]
diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.clj
index 3bb2e4f34a8..2b8498c3fd9 100644
--- a/src/metabase/analytics/stats.clj
+++ b/src/metabase/analytics/stats.clj
@@ -883,7 +883,10 @@
:enabled (premium-features/table-data-editing?)}
{:name :transforms
:available (premium-features/enable-transforms?)
- :enabled (premium-features/enable-transforms?)}])
+ :enabled (premium-features/enable-transforms?)}
+ {:name :transforms-python
+ :available (premium-features/enable-python-transforms?)
+ :enabled (premium-features/enable-python-transforms?)}])
(defn- snowplow-features
[]
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 86b6e8c9fcd..72a26c449e3 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -7,8 +7,10 @@
SQL-based drivers can use the `:sql` driver as a parent, and JDBC-based SQL drivers can use `:sql-jdbc`. Both of
these drivers define additional multimethods that child drivers should implement; see [[metabase.driver.sql]] and
[[metabase.driver.sql-jdbc]] for more details."
+ (:refer-clojure :exclude [some mapv])
#_{:clj-kondo/ignore [:metabase/modules]}
(:require
+ [clojure.java.io :as io]
[clojure.set :as set]
[clojure.string :as str]
[metabase.auth-provider.core :as auth-provider]
@@ -18,7 +20,9 @@
[metabase.query-processor.error-type :as qp.error-type]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
+ [metabase.util.json :as json]
[metabase.util.malli :as mu]
+ [metabase.util.performance :refer [mapv]]
[potemkin :as p]))
(set! *warn-on-reflection* true)
@@ -628,6 +632,12 @@
;; Does the driver support multi-level-schema for e.g. multicatalog support in databricks
:multi-level-schema
+ ;; Does the driver support table renaming
+ :rename
+
+ ;; Does the driver support atomic multi-table renaming
+ :atomic-renames
+
;; Does the driver support custom writeback actions. Drivers that support this must
;; implement [[execute-write-query!]]
:actions/custom
@@ -732,6 +742,9 @@
;; Does this driver support transforms with a table as the target?
:transforms/table
+ ;; Does this driver support executing python transforms?
+ :transforms/python
+
;; Does this driver properly support the table-exists? method for checking table existence?
:metadata/table-existence-check
@@ -1303,10 +1316,13 @@
(update-vals first))]
(rename-tables!* driver db-id sorted-rename-map)))
-#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
-(defn rename-table!
+(defmulti rename-table!
"Rename a single table."
- {:added "0.57.0"}
+ {:added "0.57.0", :arglists '([driver db-id old-table-name new-table-name])}
+ dispatch-on-initialized-driver
+ :hierarchy #'hierarchy)
+
+(defmethod rename-table! ::driver
[driver db-id from-table to-table]
(rename-tables!* driver db-id {from-table to-table}))
@@ -1326,6 +1342,57 @@
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
+(defmulti insert-col->val
+ "Parse a value for insertion based on driver, data source type and column definition.
+
+ Takes:
+ - driver: The database driver
+ - data-source-type: The data source type, see [[insert-from-source!]]
+ - column-def: Column definition map with `:type` and optionally `:database-type`/`:nullable?`
+ - val: The value to parse. Format is data-source-type specific.
+
+ Drivers should implement this when their insertion mechanism needs values converted to proper types.
+
+ Default implementation returns the value unchanged"
+ {:added "0.57.0", :arglists '([driver data-source-type column-def string-val])}
+ (fn [driver type _ _] [(dispatch-on-initialized-driver driver) type])
+ :hierarchy #'hierarchy)
+
+(defmulti insert-from-source!
+ "Inserts data from a data source into an existing table. Table must exist. Blocks until completion.
+
+ `table-definition` is a map with `:name` (may be schema-qualified) and `:columns`
+ (vector of maps with `:name` and optional `:type`, `:database-type`). Column order must match data row order.
+
+ `data-source` dispatches on `:type`. Built-in types include `:jsonl-file` (with `:file`) and `:rows`
+ (with `:data` as reducible of row vectors). Drivers may implement additional types.
+
+ Implementations may leave partial data on failure, the method makes no rollback guarantees.
+ Data visibility to other connections is not guaranteed immediately.
+
+ Default implementation for `jsonl-file` data-source is provided, which delegate to a `:rows`
+ data-source. Non-jdbc drivers must at least implement a `:rows` datasource."
+ {:added "0.57.0", :arglists '([driver database-id table-definition data-source])}
+ (fn [driver _ _ data-source]
+ [(dispatch-on-initialized-driver driver) (:type data-source)])
+ :hierarchy #'hierarchy)
+
+(defmethod insert-col->val [::driver :jsonl-file] [_driver _ _column-def val]
+ val)
+
+(defmethod insert-from-source! [::driver :jsonl-file]
+ [driver db-id {:keys [columns] :as table-definition} {:keys [file]}]
+ (with-open [rdr (io/reader file)]
+ (let [lines (line-seq rdr)
+ data-rows (map (fn [line]
+ (let [m (json/decode line)]
+ (mapv (fn [column]
+ (let [raw-val (get m (:name column))]
+ (insert-col->val driver :jsonl-file column raw-val)))
+ columns)))
+ lines)]
+ (insert-from-source! driver db-id table-definition {:type :rows :data data-rows}))))
+
(defmulti add-columns!
"Add columns given by `column-definitions` to a table named `table-name`. If the table doesn't exist it will throw an error.
`args` is an optional map with an optional key `primary-key`. The `primary-key` value is a vector of column names
@@ -1393,6 +1460,12 @@
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
+(defmulti type->database-type
+ "Returns the database type for a given Metabase type as a HoneySQL spec."
+ {:added "0.57.0", :arglists '([driver base-type])}
+ (fn [driver _base-type] driver)
+ :hierarchy #'hierarchy)
+
(defmulti allowed-promotions
"Returns a mapping of which types a column can be implicitly relaxed to, based on the content of appended values.
In the context of uploads, this permits certain appends or replacements of an existing csv table
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj
index 5274c51ac24..d438b5bbf40 100644
--- a/src/metabase/driver/h2.clj
+++ b/src/metabase/driver/h2.clj
@@ -609,6 +609,30 @@
:metabase.upload/datetime [:timestamp]
:metabase.upload/offset-datetime [:timestamp-with-time-zone]))
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for H2 that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/TextLike [_] [:varchar])
+(defmethod type->database-type :type/Text [_] [:varchar])
+(defmethod type->database-type :type/Integer [_] [:int])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/BigInteger [_] [:bigint])
+(defmethod type->database-type :type/Float [_] [(keyword "DOUBLE PRECISION")])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/TimeWithTZ [_] [:time-with-time-zone])
+(defmethod type->database-type :type/UUID [_] [:uuid])
+
+(defmethod driver/type->database-type :h2
+ [_driver base-type]
+ (type->database-type base-type))
+
(defmethod driver/create-auto-pk-with-append-csv? :h2 [_driver] true)
(defmethod driver/table-name-length-limit :h2
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index fcc8ee548a9..3d85154331c 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -78,6 +78,8 @@
:schemas false
:uploads true
:identifiers-with-spaces true
+ :rename true
+ :atomic-renames true
:expressions/integer true
:expressions/float true
:expressions/date true
@@ -90,6 +92,7 @@
:expression-literals true
:database-routing true
:metadata/table-existence-check true
+ :transforms/python true
:transforms/table true}]
(defmethod driver/database-supports? [:mysql feature] [_driver _feature _db] supported?))
@@ -793,6 +796,30 @@
:metabase.upload/datetime [:datetime]
:metabase.upload/offset-datetime [:timestamp]))
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for MySQL that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/TextLike [_] [:text])
+(defmethod type->database-type :type/Text [_] [:text])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/BigInteger [_] [:bigint])
+(defmethod type->database-type :type/Integer [_] [:int])
+(defmethod type->database-type :type/Float [_] [:double])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:datetime])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/JSON [_] [:json])
+(defmethod type->database-type :type/SerializedJSON [_] [:json])
+
+(defmethod driver/type->database-type :mysql
+ [_driver base-type]
+ (type->database-type base-type))
+
(defmethod driver/allowed-promotions :mysql
[_driver]
{:metabase.upload/int #{:metabase.upload/float}
@@ -894,7 +921,7 @@
(if (not= (get-global-variable db-id "local_infile") "ON")
;; If it isn't turned on, fall back to the generic "INSERT INTO ..." way
((get-method driver/insert-into! :sql-jdbc) driver db-id table-name column-names values)
- (let [temp-file (File/createTempFile table-name ".tsv")
+ (let [temp-file (File/createTempFile (name table-name) ".tsv")
file-path (.getAbsolutePath temp-file)]
(try
(let [tsvs (map (partial row->tsv driver (count column-names)) values)
@@ -915,6 +942,12 @@
(finally
(.delete temp-file))))))
+(defmethod driver/insert-col->val [:mysql :jsonl-file]
+ [_driver _ column-def v]
+ (if (and (string? v) (isa? (:type column-def) :type/DateTimeWithTZ))
+ (t/offset-date-time v)
+ v))
+
(defn- parse-grant
"Parses the contents of a row from the output of a `SHOW GRANTS` statement, to extract the data needed
to reconstruct the set of table privileges that the current user has. Returns nil if the grant doesn't
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index 5d5e9541d35..fa810d77a53 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -66,6 +66,8 @@
:convert-timezone true
:datetime-diff true
:now true
+ :rename true
+ :atomic-renames true
:persist-models true
:schemas true
:identifiers-with-spaces true
@@ -79,6 +81,7 @@
:expressions/date true
:database-routing true
:transforms/table true
+ :transforms/python true
:metadata/table-existence-check true}]
(defmethod driver/database-supports? [:postgres feature] [_driver _feature _db] supported?))
@@ -1035,6 +1038,33 @@
:type driver-api/qp.error-type.invalid-query}))
(throw e))))))
+(defmulti ^:private type->database-type
+ "Internal type->database-type multimethod for Postgres that dispatches on type."
+ {:arglists '([type])}
+ identity)
+
+(defmethod type->database-type :type/TextLike [_] [:text])
+(defmethod type->database-type :type/Text [_] [:text])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/BigInteger [_] [:bigint])
+(defmethod type->database-type :type/Integer [_] [:int])
+(defmethod type->database-type :type/Float [_] [:float])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/TimeWithTZ [_] [:time-with-time-zone])
+(defmethod type->database-type :type/UUID [_] [:uuid])
+(defmethod type->database-type :type/JSON [_] [:jsonb])
+(defmethod type->database-type :type/SerializedJSON [_] [:jsonb])
+(defmethod type->database-type :type/IPAddress [_] [:inet])
+
+(defmethod driver/type->database-type :postgres
+ [_driver base-type]
+ (type->database-type base-type))
+
(defmethod driver/upload-type->database-type :postgres
[_driver upload-type]
(case upload-type
diff --git a/src/metabase/driver/sql_jdbc.clj b/src/metabase/driver/sql_jdbc.clj
index 1110481c56e..9c968638a8b 100644
--- a/src/metabase/driver/sql_jdbc.clj
+++ b/src/metabase/driver/sql_jdbc.clj
@@ -177,8 +177,7 @@
(jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
(jdbc/execute! conn sql))))
-(defmethod driver/insert-into! :sql-jdbc
- [driver db-id table-name column-names values]
+(defn- insert-into!-sqls [driver table-name column-names values inline?]
(let [;; We need to partition the insert into multiple statements for both performance and correctness.
;;
;; On Postgres with a large file, 100 (3.76m) was significantly faster than 50 (4.03m) and 25 (4.27m). 1,000 was a
@@ -192,12 +191,21 @@
sqls (map #(sql/format {:insert-into (keyword table-name)
:columns (quote-columns driver column-names)
:values %}
+ :inline inline?
:quoted true
:dialect dialect)
chunks)]
- (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
- (doseq [sql sqls]
- (jdbc/execute! conn sql)))))
+ sqls))
+
+(defmethod driver/insert-into! :sql-jdbc
+ [driver db-id table-name column-names values]
+ (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+ (doseq [sql (insert-into!-sqls driver table-name column-names values false)]
+ (jdbc/execute! conn sql))))
+
+(defmethod driver/insert-from-source! [:sql-jdbc :rows]
+ [driver db-id {table-name :name :keys [columns]} {:keys [data]}]
+ (driver/insert-into! driver db-id table-name (mapv :name columns) data))
(defmethod driver/add-columns! :sql-jdbc
[driver db-id table-name column-definitions & {:keys [primary-key]}]
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index 2e965d304f9..3b69a16f5ca 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -73,6 +73,7 @@
enable-sso-ldap?
enable-sso-saml?
enable-transforms?
+ enable-python-transforms?
enable-upload-management?
enable-whitelabeling?
has-attached-dwh?
diff --git a/src/metabase/premium_features/settings.clj b/src/metabase/premium_features/settings.clj
index 319e3957f3d..da840c557c2 100644
--- a/src/metabase/premium_features/settings.clj
+++ b/src/metabase/premium_features/settings.clj
@@ -295,6 +295,10 @@
"Should we allow users to use transforms?"
:transforms)
+(define-premium-feature ^{:added "0.57.0"} enable-python-transforms?
+ "Should we allow users to use Python transforms?"
+ :transforms-python)
+
(defn- -token-features []
{:advanced_permissions (enable-advanced-permissions?)
:ai_sql_fixer (enable-ai-sql-fixer?)
@@ -340,6 +344,7 @@
:sso_saml (enable-sso-saml?)
:table_data_editing (table-data-editing?)
:transforms (enable-transforms?)
+ :transforms-python (enable-python-transforms?)
:upload_management (enable-upload-management?)
:whitelabel (enable-whitelabeling?)})
diff --git a/test/metabase/driver/common_test.clj b/test/metabase/driver/common_test.clj
index b54625923b0..4e279cd1317 100644
--- a/test/metabase/driver/common_test.clj
+++ b/test/metabase/driver/common_test.clj
@@ -7,7 +7,11 @@
[metabase.driver.util :as driver.u]
[metabase.premium-features.core :as premium-features]
[metabase.settings.core :as setting]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [metabase.test.data.sql :as sql.tx]
+ [metabase.util.json :as json]))
+
+(set! *warn-on-reflection* true)
(deftest ^:parallel base-type-inference-test
(is (= :type/Text
@@ -88,3 +92,142 @@
[2 456.0 0.789 789.0]]
(mt/formatted-rows [int double double double]
(mt/run-mbql-query json-decimals-table)))))))))
+
+(defn- base-type-test-data
+ "Base types that all drivers should support with test data."
+ [driver]
+ {:columns [{:name "id" :type :type/Integer :nullable? false}
+ {:name "name" :type :type/Text :nullable? true}
+ {:name "price" :type :type/Float :nullable? true}
+ {:name "active" :type :type/Boolean :nullable? true}
+ {:name "created_date" :type :type/Date :nullable? true}
+ {:name "created_at" :type :type/DateTime :nullable? true}]
+ :data [[1 "Product A" 19.99 (if (= :sqlserver driver) 1 true) "2024-01-01" "2024-01-01T12:00:00"]
+ [2 "Product B" 15.50 (if (= :sqlserver driver) 1 false) "2024-02-01" "2024-02-01T09:15:30"]
+ [3 nil nil nil nil nil]]})
+
+(deftest insert-from-source!-test
+ (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :mongo :clickhouse}
+ (mt/with-empty-db
+ (let [driver driver/*driver*
+ db-id (mt/id)
+ table-name (mt/random-name)
+ schema-name (when (get-method sql.tx/session-schema driver) (sql.tx/session-schema driver))
+ qualified-table-name (if schema-name
+ (keyword schema-name table-name)
+ (keyword table-name))
+ {:keys [columns data]} (base-type-test-data driver)
+ column-definitions (into {} (map (fn [{:keys [name type]}]
+ [name (driver/type->database-type driver type)]))
+ columns)]
+ (mt/as-admin
+ (try
+ (driver/create-table! driver db-id qualified-table-name column-definitions {})
+
+ (testing "insert-from-source! should insert rows with all basic types correctly"
+ (let [data-source {:type :rows :data data}
+ _ (driver/insert-from-source! driver db-id
+ {:name qualified-table-name
+ :columns columns}
+ data-source)]
+ (when-let [inserted-rows (and (not (#{:bigquery-cloud-sdk :snowflake :mongo} driver)) ;table-rows-seq not implemented
+ (driver/table-rows-seq driver/*driver* (mt/db) {:name table-name
+ :schema schema-name}))]
+ (is (= (count data) (count inserted-rows)) "Should insert all test rows including nulls"))))
+ (finally
+ (driver/drop-table! driver db-id qualified-table-name))))))))
+
+(deftest insert-from-jsonl-file-test
+ (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :mongo :clickhouse}
+ (mt/with-empty-db
+ (let [driver driver/*driver*
+ db-id (mt/id)
+ table-name (mt/random-name)
+ schema-name (when (get-method sql.tx/session-schema driver) (sql.tx/session-schema driver))
+ qualified-table-name (if schema-name
+ (keyword schema-name table-name)
+ (keyword table-name))
+ {:keys [columns data]} (base-type-test-data driver)
+ column-definitions (into {} (map (fn [{:keys [name type]}]
+ [name (driver/type->database-type driver type)]))
+ columns)]
+ (mt/as-admin
+ (try
+ (driver/create-table! driver db-id qualified-table-name column-definitions {})
+
+ (testing "insert-from-source! should insert rows from JSONL file correctly"
+ (let [temp-file (java.io.File/createTempFile "test-data" ".jsonl")
+ col-names (map :name columns)
+
+ json-rows (map (fn [row]
+ (into {} (map vector col-names row)))
+ data)]
+ (try
+
+ (with-open [writer (java.io.FileWriter. temp-file)]
+ (doseq [row json-rows]
+ (.write writer (str (json/encode row) "\n"))))
+
+ (let [data-source {:type :jsonl-file :file temp-file}
+ _ (driver/insert-from-source! driver db-id
+ {:name qualified-table-name
+ :columns columns}
+ data-source)]
+ (when-let [inserted-rows (and (not (#{:bigquery-cloud-sdk :snowflake :mongo} driver)) ;table-rows-seq not implemented
+ (driver/table-rows-seq driver/*driver* (mt/db) {:name table-name
+ :schema schema-name}))]
+ (is (= (count data) (count inserted-rows))
+ "Should insert all test rows from JSONL file including nulls")))
+ (finally
+ (.delete temp-file)))))
+ (finally
+ (driver/drop-table! driver db-id qualified-table-name))))))))
+
+(deftest ^:parallel type->database-type-h2-test
+ (mt/test-driver :h2
+ (testing "type->database-type multimethod returns correct H2 types"
+ (are [base-type expected] (= expected (driver/type->database-type :h2 base-type))
+ :type/Boolean [:boolean]
+ :type/Date [:date]
+ :type/DateTime [:timestamp]
+ :type/DateTimeWithTZ [:timestamp-with-time-zone]
+ :type/Float [(keyword "DOUBLE PRECISION")]
+ :type/Integer [:int]
+ :type/Number [:bigint]
+ :type/Text [:varchar]
+ :type/Time [:time]
+ :type/UUID [:uuid]))))
+
+(deftest ^:parallel type->database-type-mysql-test
+ (mt/test-driver :mysql
+ (testing "type->database-type multimethod returns correct MySQL types"
+ (are [base-type expected] (= expected (driver/type->database-type :mysql base-type))
+ :type/Boolean [:boolean]
+ :type/Date [:date]
+ :type/DateTime [:datetime]
+ :type/DateTimeWithTZ [:timestamp]
+ :type/Float [:double]
+ :type/Decimal [:decimal]
+ :type/Integer [:int]
+ :type/Number [:bigint]
+ :type/Text [:text]
+ :type/Time [:time]))))
+
+(deftest ^:parallel type->database-type-postgres-test
+ (mt/test-driver :postgres
+ (testing "type->database-type multimethod returns correct PostgreSQL types"
+ (are [base-type expected] (= expected (driver/type->database-type :postgres base-type))
+ :type/Boolean [:boolean]
+ :type/Date [:date]
+ :type/DateTime [:timestamp]
+ :type/DateTimeWithTZ [:timestamp-with-time-zone]
+ :type/Decimal [:decimal]
+ :type/Float [:float]
+ :type/Integer [:int]
+ :type/JSON [:jsonb]
+ :type/Number [:bigint]
+ :type/SerializedJSON [:jsonb]
+ :type/Text [:text]
+ :type/Time [:time]
+ :type/TimeWithTZ [:time-with-time-zone]
+ :type/UUID [:uuid]))))
diff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.clj
index d3b6f540721..b759d040712 100644
--- a/test/metabase/driver/sql_jdbc_test.clj
+++ b/test/metabase/driver/sql_jdbc_test.clj
@@ -3,7 +3,6 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
- [metabase.driver.mysql :as mysql]
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.util :as driver.u]
@@ -298,80 +297,102 @@
(map #(vector (:id %) (:name %)))
sort))
-#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
(deftest rename-tables-test
- (mt/test-drivers #{:postgres :mysql}
- (when-not (mysql/mariadb? (mt/db))
- (testing "rename-tables should rename multiple tables atomically"
- (let [db-id (mt/id)
- driver driver/*driver*
- schema (sql.tx/session-schema driver)
- test-table-1 (mt/random-name)
- test-table-2 (mt/random-name)
- qualified-table-1 (qualified-table-name schema test-table-1)
- qualified-table-2 (qualified-table-name schema test-table-2)
- temp-table-1 (str test-table-1 "_temp")
- temp-table-2 (str test-table-2 "_temp")
- qualified-temp-1 (qualified-table-name schema temp-table-1)
- qualified-temp-2 (qualified-table-name schema temp-table-2)
- test-data-1 [[1 "Alice"] [2 "Bob"]]
- test-data-2 [[1 "Product A"] [2 "Product B"]]]
-
- ;; so the create-table is using qp and it requires permission setup
- ;; TODO we should change the implementation of create-table to use jdbc direclty
- (mt/with-current-user (mt/user->id :crowberto)
- (driver/create-table! driver db-id qualified-table-1
- {"id" "INTEGER", "name" "VARCHAR(255)"} {})
- (driver/create-table! driver db-id qualified-table-2
- {"id" "INTEGER", "name" "VARCHAR(255)"} {})
-
- (try
- (driver/insert-into! driver db-id qualified-table-1 ["id" "name"] test-data-1)
- (driver/insert-into! driver db-id qualified-table-2 ["id" "name"] test-data-2)
-
- (testing "basic rename operations work correctly"
- (driver/rename-tables! driver db-id
- {qualified-table-1 qualified-temp-1
- qualified-table-2 qualified-temp-2})
-
- (is (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema}))
- (is (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))
- (is (not (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema})))
- (is (not (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))
-
- (is (= test-data-1 (table-rows qualified-temp-1)))
- (is (= test-data-2 (table-rows qualified-temp-2)))
-
- (driver/rename-tables! driver db-id
- {qualified-temp-1 qualified-table-1
- qualified-temp-2 qualified-table-2}))
-
- (testing "atomicity: all renames fail if any rename fails"
- (let [conflict-table (str test-table-2 "_conflict")
- qualified-conflict (qualified-table-name schema conflict-table)]
- (driver/create-table! driver db-id qualified-conflict {"id" "INTEGER"} {})
-
- (try
- (is (thrown? Exception
- (driver/rename-tables! driver db-id
- {qualified-table-1 qualified-temp-1
- qualified-table-2 qualified-conflict})))
-
- (testing "original tables should still exist after failed atomic rename"
- (is (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema}))
- (is (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))
-
- (testing "temp tables should not exist after failed atomic rename"
- (is (not (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema})))
- (is (not (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))))
-
- (testing "original data should be intact after failed atomic rename"
- (is (= test-data-1 (table-rows qualified-table-1)))
- (is (= test-data-2 (table-rows qualified-table-2))))
-
- (finally
- (driver/drop-table! driver db-id qualified-conflict)))))
-
- (finally
- (driver/drop-table! driver db-id qualified-table-1)
- (driver/drop-table! driver db-id qualified-table-2)))))))))
+ (mt/test-drivers (mt/normal-drivers-with-feature :atomic-renames)
+ (testing "rename-tables should rename multiple tables atomically"
+ (let [db-id (mt/id)
+ driver driver/*driver*
+ schema (sql.tx/session-schema driver)
+ test-table-1 (mt/random-name)
+ test-table-2 (mt/random-name)
+ qualified-table-1 (qualified-table-name schema test-table-1)
+ qualified-table-2 (qualified-table-name schema test-table-2)
+ temp-table-1 (str test-table-1 "_temp")
+ temp-table-2 (str test-table-2 "_temp")
+ qualified-temp-1 (qualified-table-name schema temp-table-1)
+ qualified-temp-2 (qualified-table-name schema temp-table-2)
+ test-data-1 [[1 "Alice"] [2 "Bob"]]
+ test-data-2 [[1 "Product A"] [2 "Product B"]]]
+
+ (driver/create-table! driver db-id qualified-table-1
+ {"id" "INTEGER", "name" "VARCHAR(255)"} {})
+ (driver/create-table! driver db-id qualified-table-2
+ {"id" "INTEGER", "name" "VARCHAR(255)"} {})
+
+ (try
+ (driver/insert-into! driver db-id qualified-table-1 ["id" "name"] test-data-1)
+ (driver/insert-into! driver db-id qualified-table-2 ["id" "name"] test-data-2)
+
+ (testing "basic rename operations work correctly"
+ (driver/rename-tables! driver db-id
+ {qualified-table-1 qualified-temp-1
+ qualified-table-2 qualified-temp-2})
+
+ (is (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema}))
+ (is (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))
+ (is (not (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema})))
+ (is (not (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))
+
+ (is (= test-data-1 (table-rows qualified-temp-1)))
+ (is (= test-data-2 (table-rows qualified-temp-2)))
+
+ (driver/rename-tables! driver db-id
+ {qualified-temp-1 qualified-table-1
+ qualified-temp-2 qualified-table-2}))
+
+ (testing "atomicity: all renames fail if any rename fails"
+ (let [conflict-table (str test-table-2 "_conflict")
+ qualified-conflict (qualified-table-name schema conflict-table)]
+ (driver/create-table! driver db-id qualified-conflict {"id" "INTEGER"} {})
+
+ (try
+ (is (thrown? Exception
+ (driver/rename-tables! driver db-id
+ {qualified-table-1 qualified-temp-1
+ qualified-table-2 qualified-conflict})))
+
+ (testing "original tables should still exist after failed atomic rename"
+ (is (driver/table-exists? driver (mt/db) {:name test-table-1 :schema schema}))
+ (is (driver/table-exists? driver (mt/db) {:name test-table-2 :schema schema})))
+
+ (testing "temp tables should not exist after failed atomic rename"
+ (is (not (driver/table-exists? driver (mt/db) {:name temp-table-1 :schema schema})))
+ (is (not (driver/table-exists? driver (mt/db) {:name temp-table-2 :schema schema}))))
+
+ (testing "original data should be intact after failed atomic rename"
+ (is (= test-data-1 (table-rows qualified-table-1)))
+ (is (= test-data-2 (table-rows qualified-table-2))))
+
+ (finally
+ (driver/drop-table! driver db-id qualified-conflict)))))
+
+ (finally
+ (driver/drop-table! driver db-id qualified-table-1)
+ (driver/drop-table! driver db-id qualified-table-2)))))))
+
+(deftest rename-table-test
+ (mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc
+ :+features [:rename]})
+ (testing "rename-table! should rename a single table correctly"
+ (let [db-id (mt/id)
+ driver driver/*driver*
+ schema (sql.tx/session-schema driver)
+ test-table (mt/random-name)
+ renamed-table (str test-table "_renamed")
+ qualified-table (qualified-table-name schema test-table)
+ qualified-renamed (qualified-table-name schema renamed-table)]
+ (driver/create-table! driver db-id qualified-table
+ {"id" "INTEGER", "name" "VARCHAR(255)"} {})
+ (try
+ (testing "single table rename works correctly"
+ (driver/rename-table! driver db-id qualified-table qualified-renamed)
+ (is (driver/table-exists? driver (mt/db) {:name renamed-table :schema schema})
+ "Renamed table should exist")
+ (is (not (driver/table-exists? driver (mt/db) {:name test-table :schema schema}))
+ "Original table should not exist"))
+
+ (finally
+ (when (driver/table-exists? driver (mt/db) {:name renamed-table :schema schema})
+ (driver/drop-table! driver db-id qualified-renamed))
+ (when (driver/table-exists? driver (mt/db) {:name test-table :schema schema})
+ (driver/drop-table! driver db-id qualified-table))))))))
diff --git a/test/metabase/driver_test.clj b/test/metabase/driver_test.clj
index d979a278506..fde4af96904 100644
--- a/test/metabase/driver_test.clj
+++ b/test/metabase/driver_test.clj
@@ -393,3 +393,14 @@
(qp/process-query success-query)
(is (= 3.0 (mt/metric-value system :metabase-query-processor/query {:driver driver/*driver* :status "success"})))
(is (= 2.0 (mt/metric-value system :metabase-query-processor/query {:driver driver/*driver* :status "failure"}))))))))
+
+(deftest python-transform-drivers-multimethods-support
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+ (let [driver driver/*driver*]
+ (is (get-method driver/create-table! driver))
+ (is (get-method driver/table-name-length-limit driver))
+ (is (get-method driver/drop-table! driver))
+ (is (let [should-be-supported-by-all #{:type/Number :type/Text :type/Date :type/DateTime :type/DateTimeWithTZ :type/Boolean}]
+ (and (get-method driver/type->database-type driver)
+ (every? #(driver/type->database-type driver %) should-be-supported-by-all))))
+ (is (get-method driver/insert-from-source! [driver :jsonl-file])))))