Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix bigquery/snowflake transforms tests (#69566)

Dan Stoned98776403499
diff --git a/.github/workflows/transforms-python-stress-test.yml b/.github/workflows/transforms-python-stress-test.yml
new file mode 100644
index 00000000000..38b0402711a
--- /dev/null
+++ b/.github/workflows/transforms-python-stress-test.yml
@@ -0,0 +1,495 @@
+name: Transforms Python Stress Test
+
+on:
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: 'Branch name or commit SHA to checkout'
+ type: string
+ required: false
+ default: ''
+ driver:
+ description: 'Which driver(s) to test'
+ required: true
+ default: 'all'
+ type: choice
+ options:
+ - all
+ - bigquery
+ - clickhouse
+ - mongo
+ - mysql
+ - postgres
+ - redshift
+ - snowflake
+ - sqlserver
+
+jobs:
+
+ be-tests-bigquery-transforms:
+ # inputs.driver is empty on push triggers, so treat '' as 'all'
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'bigquery' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CI: 'true'
+ DRIVERS: bigquery-cloud-sdk
+ MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}
+ MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}
+ MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}
+ MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}
+ MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
+ name: BigQuery Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Test BigQuery transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-bigquery-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: BigQuery Transforms
+
+ be-tests-clickhouse-transforms:
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'clickhouse' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CI: 'true'
+ DRIVERS: clickhouse
+ MB_CLICKHOUSE_TEST_HOST: localhost
+ MB_CLICKHOUSE_TEST_PORT: 8123
+ MB_CLICKHOUSE_TEST_OLD_PORT: 8124
+ MB_CLICKHOUSE_TEST_NGINX_PORT: 8127
+ name: ClickHouse Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Add ClickHouse TLS instance to /etc/hosts
+ run: |
+ sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts
+ - name: Start ClickHouse in Docker
+ uses: hoverkraft-tech/[email protected]
+ with:
+ compose-file: "modules/drivers/clickhouse/docker-compose.yml"
+ down-flags: "--volumes"
+ services: |
+ clickhouse
+ clickhouse_tls
+ clickhouse_older_version
+ clickhouse_cluster_node1
+ clickhouse_cluster_node2
+ nginx
+ - name: Test ClickHouse transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-clickhouse-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: ClickHouse Transforms
+
+ be-tests-mongo-transforms:
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'mongo' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CI: 'true'
+ DRIVERS: mongo
+ MB_MONGO_TEST_USER: metabase
+ MB_MONGO_TEST_PASSWORD: metasample123
+ services:
+ mongodb:
+ image: mongo:latest
+ ports:
+ - "27017:27017"
+ env:
+ MONGO_INITDB_ROOT_USERNAME: metabase
+ MONGO_INITDB_ROOT_PASSWORD: metasample123
+ name: MongoDB Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Test MongoDB transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-mongo-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: MongoDB Transforms
+
+ be-tests-mysql-transforms:
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'mysql' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ services:
+ mysql:
+ image: mysql:latest
+ ports:
+ - "3306:3306"
+ env:
+ MYSQL_ALLOW_EMPTY_PASSWORD: true
+ MYSQL_DATABASE: circle_test
+ env:
+ CI: 'true'
+ DRIVERS: mysql
+ MB_DB_TYPE: mysql
+ MB_DB_HOST: localhost
+ MB_DB_PORT: 3306
+ MB_DB_DBNAME: circle_test
+ MB_DB_USER: root
+ MB_MYSQL_TEST_USER: root
+ name: MySQL Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Test MySQL transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-mysql-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: MySQL Transforms
+
+ be-tests-postgres-transforms:
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'postgres' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CI: 'true'
+ DRIVERS: postgres
+ MB_DB_TYPE: postgres
+ MB_DB_PORT: 5432
+ MB_DB_HOST: localhost
+ MB_DB_DBNAME: mb_test
+ MB_DB_USER: mb_test
+ MB_POSTGRESQL_TEST_USER: mb_test
+ services:
+ postgres:
+ image: postgres:latest
+ ports:
+ - "5432:5432"
+ env:
+ POSTGRES_USER: mb_test
+ POSTGRES_DB: mb_test
+ POSTGRES_HOST_AUTH_METHOD: trust
+ name: Postgres Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Test Postgres transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-postgres-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: Postgres Transforms
+
+ be-tests-redshift-transforms:
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'redshift' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CI: 'true'
+ DRIVERS: redshift
+ MB_REDSHIFT_TEST_USER: metabase_ci
+ MB_REDSHIFT_TEST_DB: testdb
+ MB_REDSHIFT_TEST_DB_ROUTING: dev
+ MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }}
+ MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}
+ name: Redshift Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Test Redshift transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-redshift-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: Redshift Transforms
+
+ be-tests-snowflake-transforms:
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'snowflake' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CI: 'true'
+ DRIVERS: snowflake
+ MB_SNOWFLAKE_TEST_USER: METABASE CI
+ MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}
+ MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}
+ MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}
+ MB_SNOWFLAKE_TEST_PK_USER: METABASE PK
+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2 }}
+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY }}
+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2 }}
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DEFAULT_USER: RSA_ROLE_TEST_DEFAULT_USER
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB
+ name: Snowflake Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Test Snowflake transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-snowflake-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: Snowflake Transforms
+
+ be-tests-sqlserver-transforms:
+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'sqlserver' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CI: 'true'
+ DRIVERS: sqlserver
+ MB_SQLSERVER_TEST_HOST: localhost
+ MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd'
+ MB_SQLSERVER_TEST_USER: SA
+ services:
+ sqlserver:
+ image: mcr.microsoft.com/mssql/server:2022-latest
+ ports:
+ - "1433:1433"
+ env:
+ ACCEPT_EULA: Y
+ SA_PASSWORD: 'P@ssw0rd'
+ MSSQL_MEMORY_LIMIT_MB: 1024
+ name: SQL Server Transforms
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref || github.ref }}
+ - name: Setup python-runner
+ uses: ./.github/actions/setup-python-runner
+ with:
+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ - name: Test SQL Server transforms
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-sqlserver-transforms'
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ - name: Cleanup python-runner
+ if: always()
+ uses: ./.github/actions/cleanup-python-runner
+ with:
+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+ logs-artifact-suffix: SQL Server Transforms
+
+ transforms-stress-test-result:
+ needs:
+ - be-tests-bigquery-transforms
+ - be-tests-clickhouse-transforms
+ - be-tests-mongo-transforms
+ - be-tests-mysql-transforms
+ - be-tests-postgres-transforms
+ - be-tests-redshift-transforms
+ - be-tests-snowflake-transforms
+ - be-tests-sqlserver-transforms
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ name: transforms-stress-test-result
+ if: always() && !cancelled()
+ steps:
+ - name: Check job status
+ uses: actions/github-script@v7
+ env:
+ needs: ${{ toJson(needs) }}
+ with:
+ script: |
+ const needs = JSON.parse(process.env.needs);
+
+ const jobs = Object.entries(needs).map(
+ ([jobName, jobValues]) => ({
+ name: jobName,
+ result: jobValues.result
+ }));
+
+ if (jobs.every(job => (job.result === 'skipped' || job.result === 'success'))) {
+ console.log("All transforms tests have passed (or been skipped).");
+ process.exit(0);
+ }
+
+ console.log("Some transforms tests have failed:");
+ jobs.forEach((job) => {
+ if (job.result !== 'success' && job.result !== 'skipped') {
+ console.log(` ${job.name} - ${job.result}`);
+ }
+ });
+
+ process.exit(1);
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index 75c1d64e84d..9789ddcd39f 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -82,12 +82,14 @@
[schema checkpoint-config]
(let [{:keys [field-name template-tag-type]} checkpoint-config
timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
- query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"
+ query (format "SELECT *, %s AS %s FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"
timestamp-sql
+ (sql.u/quote-name driver/*driver* :field "load_timestamp")
(if schema
(sql.u/quote-name driver/*driver* :table schema "transforms_products")
"transforms_products")
- field-name field-name)]
+ (sql.u/quote-name driver/*driver* :field field-name)
+ (sql.u/quote-name driver/*driver* :field field-name))]
{:database (mt/id)
:type :native
:native {:query query
@@ -101,8 +103,9 @@
"Create a native query without template tags for testing automatic checkpoint insertion. "
[schema]
(let [timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
- query (format "SELECT *, %s AS load_timestamp FROM %s"
+ query (format "SELECT *, %s AS %s FROM %s"
timestamp-sql
+ (sql.u/quote-name driver/*driver* :field "load_timestamp")
(if schema
(sql.u/quote-name driver/*driver* :table schema "transforms_products")
"transforms_products"))]
@@ -131,7 +134,7 @@
transform-type can be :native, :native-auto-wrap, :mbql, or :python
checkpoint-config should be from checkpoint-configs"
- [transform-name target-table-name transform-type checkpoint-config]
+ [transform-name target-table transform-type checkpoint-config]
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
{:keys [field-name lib-column-key]} checkpoint-config]
{:name transform-name
@@ -155,33 +158,28 @@
:body incremental-python-body
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter-unique-key lib-column-key}})
- :target {:type "table-incremental"
- :schema schema
- :name target-table-name
- :database (mt/id)
- :target-incremental-strategy {:type "append"}}}))
+ :target (merge target-table {:type "table-incremental"
+ :target-incremental-strategy {:type "append"}})}))
(defn- get-table-row-count
"Get the row count of a table by name."
- [table-name]
- (let [table (t2/select-one :model/Table :name table-name)
- mp (mt/metadata-provider)
+ [{table-name :name}]
+ (let [table (t2/select-one :model/Table :name table-name)
+ mp (mt/metadata-provider)
table-metadata (lib.metadata/table mp (:id table))
- count-query (lib/aggregate (lib/query mp table-metadata) (lib/count))
- result (qp/process-query count-query)]
+ count-query (lib/aggregate (lib/query mp table-metadata) (lib/count))
+ result (qp/process-query count-query)]
(some-> result :data :rows first first bigint)))
(defn- get-distinct-timestamp-count
"Get the count of distinct load_timestamp values in a table."
- [table-name]
- (let [table (t2/select-one :model/Table :name table-name)
- native-query {:database (mt/id)
- :type :native
- :native {:query (format "SELECT COUNT(DISTINCT load_timestamp) FROM %s"
- (sql.u/quote-name driver/*driver* :table
- (:schema table)
- table-name))}}
- result (qp/process-query native-query)]
+ [{table-name :name, :keys [schema database]}]
+ (let [native-query {:database database
+ :type :native
+ :native {:query (format "SELECT COUNT(DISTINCT %s) FROM %s"
+ (sql.u/quote-name driver/*driver* :field "load_timestamp")
+ (sql.u/quote-name driver/*driver* :table schema table-name))}}
+ result (qp/process-query native-query)]
(some-> result :data :rows first first bigint)))
(defn get-checkpoint-value [transform]
@@ -192,7 +190,7 @@
"Compare two checkpoint values with type-appropriate logic. "
[checkpoint-type expected actual]
(case checkpoint-type
- :integer (= expected actual)
+ :integer (= (bigint expected) (bigint actual))
:float (and (number? actual)
(< (Math/abs (- expected actual)) 0.01))
:temporal (and (string? actual)
@@ -203,14 +201,13 @@
[products]
(let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))
spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
- schema-prefix (if schema (str schema ".") "")
values-list (str/join ", "
(map (fn [{:keys [name category price created-at]}]
(format "('%s', '%s', %s, '%s')" name category price created-at))
products))
- insert-sql (format "INSERT INTO %s%s (name, category, price, created_at) VALUES %s"
- schema-prefix
- source-table-name
+ insert-sql (format "INSERT INTO %s (%s) VALUES %s"
+ (sql.u/quote-name driver/*driver* :table schema source-table-name)
+ (str/join "," (map #(sql.u/quote-name driver/*driver* :field %) ["name" "category" "price" "created_at"]))
values-list)]
(driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))
@@ -218,10 +215,9 @@
[products]
(let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))
spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
- schema-prefix (if schema (str schema ".") "")
- delete-sql (format "DELETE FROM %s%s WHERE name IN (%s)"
- schema-prefix
- source-table-name
+ delete-sql (format "DELETE FROM %s WHERE %s IN (%s)"
+ (sql.u/quote-name driver/*driver* :table schema source-table-name)
+ (sql.u/quote-name driver/*driver* :field "name")
(str/join ", " (map (constantly "?") products)))]
(driver/execute-raw-queries! driver/*driver* spec [[delete-sql (mapv :name products)]])))
@@ -238,6 +234,12 @@
(defn- test-drivers []
(disj (mt/normal-drivers-with-feature :transforms/table) :redshift :clickhouse :sqlserver))
+(defn- target-table-gen [prefix]
+ {:type "table"
+ :name prefix
+ :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ :database (mt/id)})
+
(deftest create-incremental-transform-test
(testing "Creating an incremental transform with checkpoint strategy"
(doseq [checkpoint-type [:integer :float]
@@ -246,7 +248,7 @@
(mt/test-drivers (test-drivers)
(mt/with-premium-features #{:transforms}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "incremental_test"]
+ (with-transform-cleanup! [target-table (target-table-gen "incremental_test")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
{:keys [field-name]} checkpoint-config
transform-payload (make-incremental-transform-payload "Test Incremental Transform" target-table :native checkpoint-config)]
@@ -281,14 +283,14 @@
(mt/test-drivers (test-drivers)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "incremental_twice"]
+ (with-transform-cleanup! [target-table (target-table-gen "incremental_twice")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
{:keys [expected-initial-checkpoint expected-second-checkpoint]} checkpoint-config
transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)]
(mt/with-temp [:model/Transform transform transform-payload]
(testing "First run processes all data"
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (transforms.tu/wait-for-table target-table 10000)
+ (transforms.tu/wait-for-table (:name target-table) 10000)
(let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)]
(is (= 10 row-count) "First run should process the first 10 products")
@@ -315,17 +317,17 @@
transform-type [:native :mbql :python]
:when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]
(testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))
- (mt/test-drivers (test-drivers)
+ (mt/test-drivers (disj (test-drivers) :bigquery-cloud-sdk) ; will follow up with a fix via GDGT-1777
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "switch_incr_to_non_incr"]
+ (with-transform-cleanup! [target-table (target-table-gen "switch_incr_to_non_incr")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
{:keys [expected-initial-checkpoint]} checkpoint-config
initial-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)]
(mt/with-temp [:model/Transform transform initial-payload]
(testing "Initial incremental run processes first batch"
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (transforms.tu/wait-for-table target-table 10000)
+ (transforms.tu/wait-for-table (:name target-table) 10000)
(let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)
checkpoint (get-checkpoint-value transform)]
@@ -374,7 +376,7 @@
(mt/test-drivers (test-drivers)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "switch_non_incr_to_incr"]
+ (with-transform-cleanup! [target-table (target-table-gen "switch_non_incr_to_incr")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
{:keys [expected-second-checkpoint]} checkpoint-config
incremental-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)
@@ -385,7 +387,7 @@
(mt/with-temp [:model/Transform transform initial-payload]
(testing "Initial non-incremental run creates table"
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (transforms.tu/wait-for-table target-table 10000)
+ (transforms.tu/wait-for-table (:name target-table) 10000)
(let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)]
(is (= 10 row-count) "Initial run should process 10 products")
@@ -411,14 +413,16 @@
(is (= 2 distinct-timestamps) "Should have 2 distinct timestamp")
(is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))
- (when-not (= driver/*driver* :clickhouse)
+ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment
+ (not= driver/*driver* :clickhouse)
+ ;; this *should* work see #68965 for context, will plan follow-up task
+ (not= driver/*driver* :snowflake))
(testing "Add new data and run incrementally"
(with-insert-test-products!
[{:name "After Switch Product"
:category "Gadget"
:price 379.99
:created-at "2024-01-20T10:00:00"}]
-
(let [transform (t2/select-one :model/Transform (:id transform))]
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
(let [row-count (get-table-row-count target-table)
@@ -435,7 +439,7 @@
(mt/test-drivers (test-drivers)
(mt/with-premium-features #{:transforms}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "native_no_template"]
+ (with-transform-cleanup! [target-table (target-table-gen "native_no_template")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
{:keys [expected-second-checkpoint field-name]} checkpoint-config
schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
@@ -444,16 +448,14 @@
:query (make-incremental-source-query-without-template-tag schema)
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter field-name}}
- :target {:type "table-incremental"
- :schema schema
- :name target-table
- :database (mt/id)
- :target-incremental-strategy {:type "append"}}}]
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy {:type "append"}})}]
(mt/with-temp [:model/Transform transform transform-payload]
(testing "First run processes all existing data"
(transforms.execute/execute! transform {:run-method :manual})
- (transforms.tu/wait-for-table target-table 10000)
- (let [row-count (get-table-row-count target-table)
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (let [row-count (get-table-row-count target-table)
checkpoint (get-checkpoint-value transform)]
(is (= 16 row-count) "First run should process all 16 products")
(is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)
@@ -464,7 +466,10 @@
(let [row-count (get-table-row-count target-table)]
(is (= 16 row-count) "Should still have 16 rows, no new data")))
- (when-not (= driver/*driver* :clickhouse)
+ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment
+ (not= driver/*driver* :clickhouse)
+ ;; this *should* work see #68965 for context, will plan follow-up task
+ (not= driver/*driver* :snowflake))
(testing "After inserting new data, incremental run appends only new rows"
(with-insert-test-products!
[{:name "New Product 1"
@@ -484,7 +489,7 @@
(deftest unsupported-checkpoint-column-type-test
(testing "Transform fails at runtime with unsupported checkpoint column type"
- (mt/test-drivers (test-drivers)
+ (mt/test-drivers #{:h2 :postgres}
(mt/with-premium-features #{:transforms}
(mt/dataset transforms-dataset/transforms-test
(with-transform-cleanup! [target-table "unsupported_type_test"]
@@ -511,7 +516,7 @@
(mt/test-drivers [:postgres]
(mt/with-premium-features #{:transforms}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "native_temporal_cast"]
+ (with-transform-cleanup! [target-table (target-table-gen "native_temporal_cast")]
(let [checkpoint-config (get checkpoint-configs :temporal)
{:keys [expected-initial-checkpoint expected-second-checkpoint field-name]} checkpoint-config
schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
@@ -535,18 +540,16 @@
:required false}}}}
:source-incremental-strategy {:type "checkpoint"
:checkpoint-filter field-name}}
- :target {:type "table-incremental"
- :schema schema
- :name target-table
- :database (mt/id)
- :target-incremental-strategy {:type "append"}}}]
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy {:type "append"}})}]
(mt/with-temp [:model/Transform transform transform-payload]
(testing "First run processes first batch"
(transforms.execute/execute! transform {:run-method :manual})
- (transforms.tu/wait-for-table target-table 10000)
- (let [row-count (get-table-row-count target-table)
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)
- checkpoint (get-checkpoint-value transform)]
+ checkpoint (get-checkpoint-value transform)]
(is (= 10 row-count) "First run should process the first 10 products")
(is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
(is (compare-checkpoint-values :temporal expected-initial-checkpoint checkpoint)
@@ -554,7 +557,7 @@
(testing "Second run processes remaining data"
(transforms.execute/execute! transform {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
+ (let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)
checkpoint (get-checkpoint-value transform)]
(is (= 16 row-count) "Second run should add remaining 6 rows")
@@ -685,14 +688,13 @@
(mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "incremental_index"]
+ (with-transform-cleanup! [target-table (target-table-gen "incremental_index")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
- transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)
- schema (:schema (:target transform-payload))]
+ transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)]
(mt/with-temp [:model/Transform transform transform-payload]
(testing "First run creates index on checkpoint column"
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:schema schema, :name target-table})
+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)
field-name (:field-name checkpoint-config)]
(testing "Index was created"
(is (= 1 (count indexes)))
@@ -701,7 +703,7 @@
(is (= 10 (get-table-row-count target-table))))
(testing "Second run succeeds with existing index"
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:schema schema, :name target-table})
+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)
field-name (:field-name checkpoint-config)]
(testing "Index still exists"
(is (= 1 (count indexes)))
@@ -718,14 +720,14 @@
(mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "index_cleanup_non_incr"]
+ (with-transform-cleanup! [target-table (target-table-gen "index_cleanup_non_incr")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
{:keys [field-name]} checkpoint-config
incremental-payload (make-incremental-transform-payload "Index Cleanup Transform" target-table transform-type checkpoint-config)]
(mt/with-temp [:model/Transform transform incremental-payload]
(testing "First incremental run creates index"
(execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:name target-table})]
+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]
(is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes)))))
(testing "Switch to non-incremental via API"
(let [non-incremental-payload (-> incremental-payload
@@ -736,7 +738,7 @@
(testing "Non-incremental run removes automatic indexes indexes"
(let [transform (t2/select-one :model/Transform (:id transform))]
(execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:name target-table})
+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)
mb-indexes (filter #(str/starts-with? (:index-name %) "mb_transform_idx_") indexes)]
(testing "Automatic indexes are dropped"
(is (empty? mb-indexes))))))))))))))))
@@ -746,7 +748,7 @@
(mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))
(mt/with-premium-features #{:transforms}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table "icchange"]
+ (with-transform-cleanup! [target-table (target-table-gen "icchange")]
(let [integer-config (get checkpoint-configs :integer)
float-config (get checkpoint-configs :float)
initial-field (:field-name integer-config)
@@ -756,7 +758,7 @@
(mt/with-temp [:model/Transform transform initial-payload]
(testing "First run with integer checkpoint creates index on id column"
(execute-transform-with-ordering! transform transform-type initial-field {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:name target-table})]
+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]
(is (= 1 (count indexes)))
(is (=? {:value initial-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))
(testing "Switch checkpoint column via API"
@@ -767,8 +769,7 @@
(testing "Run with new checkpoint column updates indexes"
(let [transform (t2/select-one :model/Transform (:id transform))]
(execute-transform-with-ordering! transform transform-type new-field {:run-method :manual})
- (let [table (t2/select-one :model/Table :name target-table)
- indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:schema (:schema table) :name target-table})]
+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]
(testing "Old index removed, new checkpoint column is now indexed"
(is (= 1 (count indexes)))
(is (=? {:value new-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
index b700f8b94fb..293d281f987 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
@@ -106,7 +106,7 @@
(defn- test-run [& {:keys [program user features source-tables extra-opts]
:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]
user :crowberto
- source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}
+ source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}
features #{:transforms :transforms-python}}}]
(let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
(mt/with-premium-features features
@@ -154,12 +154,12 @@
(is (=? {:status 200} (test-run :user :lucky))))))))
(deftest test-run-input-limit-test
- (testing "maximum"
- (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))
- (testing "minimum"
- (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))
- (testing "truncates sources"
- (mt/dataset transforms-dataset/transforms-test
+ (mt/dataset transforms-dataset/transforms-test
+ (testing "maximum"
+ (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))
+ (testing "minimum"
+ (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))
+ (testing "truncates sources"
(let [program ["def transform(customers):" " return customers"]
source-tables {"customers" (mt/id :transforms_customers)}
response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})]
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 34fafad5141..248f264b20a 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -911,10 +911,10 @@
;;; +----------------------------------------------------------------------------------------------------------------+
(defn- get-table-str [table]
- (let [table-str (if (namespace table)
- (format "%s.%s" (namespace table) (name table))
- (name table))]
- (sql.u/quote-name :bigquery-cloud-sdk :table table-str)))
+ (let [qn #(sql.u/quote-name :bigquery-cloud-sdk :table %)]
+ (if (namespace table)
+ (format "%s.%s" (qn (namespace table)) (qn (name table)))
+ (qn (name table)))))
(defmethod driver/compile-transform :bigquery-cloud-sdk
[_driver {:keys [query output-table]}]
diff --git a/test/metabase/transforms/test_util.clj b/test/metabase/transforms/test_util.clj
index 79eca9a0a01..720d249348c 100644
--- a/test/metabase/transforms/test_util.clj
+++ b/test/metabase/transforms/test_util.clj
@@ -156,7 +156,9 @@
[]
(or (when (get-method driver.sql/default-schema driver/*driver*)
(driver.sql/default-schema driver/*driver*))
- "public"))
+ (if (= :bigquery-cloud-sdk driver/*driver*)
+ (t2/select-one-fn :schema :model/Table :db_id (mt/id))
+ "public")))
(defmulti delete-schema!
"Deletes a schema."