The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix bigquery/snowflake transforms tests (#69566)
Dan Stoned98776403499
diff --git a/.github/workflows/transforms-python-stress-test.yml b/.github/workflows/transforms-python-stress-test.ymlnew file mode 100644index 00000000000..38b0402711a--- /dev/null+++ b/.github/workflows/transforms-python-stress-test.yml@@ -0,0 +1,495 @@+name: Transforms Python Stress Test++on:+ workflow_dispatch:+ inputs:+ ref:+ description: 'Branch name or commit SHA to checkout'+ type: string+ required: false+ default: ''+ driver:+ description: 'Which driver(s) to test'+ required: true+ default: 'all'+ type: choice+ options:+ - all+ - bigquery+ - clickhouse+ - mongo+ - mysql+ - postgres+ - redshift+ - snowflake+ - sqlserver++jobs:++ be-tests-bigquery-transforms:+ # inputs.driver is empty on push triggers, so treat '' as 'all'+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'bigquery' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ CI: 'true'+ DRIVERS: bigquery-cloud-sdk+ MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}+ MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}+ MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}+ MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}+ MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}+ name: BigQuery Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Test BigQuery transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-bigquery-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: BigQuery Transforms++ be-tests-clickhouse-transforms:+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'clickhouse' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ CI: 'true'+ DRIVERS: clickhouse+ MB_CLICKHOUSE_TEST_HOST: localhost+ MB_CLICKHOUSE_TEST_PORT: 8123+ MB_CLICKHOUSE_TEST_OLD_PORT: 8124+ MB_CLICKHOUSE_TEST_NGINX_PORT: 8127+ name: ClickHouse Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Add ClickHouse TLS instance to /etc/hosts+ run: |+ sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts+ - name: Start ClickHouse in Docker+ uses: hoverkraft-tech/[email protected]+ with:+ compose-file: "modules/drivers/clickhouse/docker-compose.yml"+ down-flags: "--volumes"+ services: |+ clickhouse+ clickhouse_tls+ clickhouse_older_version+ clickhouse_cluster_node1+ clickhouse_cluster_node2+ nginx+ - name: Test ClickHouse transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-clickhouse-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: ClickHouse Transforms++ be-tests-mongo-transforms:+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'mongo' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ CI: 'true'+ DRIVERS: mongo+ MB_MONGO_TEST_USER: metabase+ MB_MONGO_TEST_PASSWORD: metasample123+ services:+ mongodb:+ image: mongo:latest+ ports:+ - "27017:27017"+ env:+ MONGO_INITDB_ROOT_USERNAME: metabase+ MONGO_INITDB_ROOT_PASSWORD: metasample123+ name: MongoDB Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Test MongoDB transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-mongo-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: MongoDB Transforms++ be-tests-mysql-transforms:+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'mysql' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ services:+ mysql:+ image: mysql:latest+ ports:+ - "3306:3306"+ env:+ MYSQL_ALLOW_EMPTY_PASSWORD: true+ MYSQL_DATABASE: circle_test+ env:+ CI: 'true'+ DRIVERS: mysql+ MB_DB_TYPE: mysql+ MB_DB_HOST: localhost+ MB_DB_PORT: 3306+ MB_DB_DBNAME: circle_test+ MB_DB_USER: root+ MB_MYSQL_TEST_USER: root+ name: MySQL Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Test MySQL transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-mysql-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: MySQL Transforms++ be-tests-postgres-transforms:+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'postgres' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ CI: 'true'+ DRIVERS: postgres+ MB_DB_TYPE: postgres+ MB_DB_PORT: 5432+ MB_DB_HOST: localhost+ MB_DB_DBNAME: mb_test+ MB_DB_USER: mb_test+ MB_POSTGRESQL_TEST_USER: mb_test+ services:+ postgres:+ image: postgres:latest+ ports:+ - "5432:5432"+ env:+ POSTGRES_USER: mb_test+ POSTGRES_DB: mb_test+ POSTGRES_HOST_AUTH_METHOD: trust+ name: Postgres Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Test Postgres transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-postgres-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: Postgres Transforms++ be-tests-redshift-transforms:+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'redshift' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ CI: 'true'+ DRIVERS: redshift+ MB_REDSHIFT_TEST_USER: metabase_ci+ MB_REDSHIFT_TEST_DB: testdb+ MB_REDSHIFT_TEST_DB_ROUTING: dev+ MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }}+ MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}+ name: Redshift Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Test Redshift transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-redshift-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: Redshift Transforms++ be-tests-snowflake-transforms:+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'snowflake' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ CI: 'true'+ DRIVERS: snowflake+ MB_SNOWFLAKE_TEST_USER: METABASE CI+ MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}+ MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}+ MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}+ MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}+ MB_SNOWFLAKE_TEST_PK_USER: METABASE PK+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY }}+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2 }}+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY }}+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2 }}+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DEFAULT_USER: RSA_ROLE_TEST_DEFAULT_USER+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB+ name: Snowflake Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Test Snowflake transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-snowflake-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: Snowflake Transforms++ be-tests-sqlserver-transforms:+ if: ${{ inputs.driver == 'all' || inputs.driver == '' || inputs.driver == 'sqlserver' }}+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ CI: 'true'+ DRIVERS: sqlserver+ MB_SQLSERVER_TEST_HOST: localhost+ MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd'+ MB_SQLSERVER_TEST_USER: SA+ services:+ sqlserver:+ image: mcr.microsoft.com/mssql/server:2022-latest+ ports:+ - "1433:1433"+ env:+ ACCEPT_EULA: Y+ SA_PASSWORD: 'P@ssw0rd'+ MSSQL_MEMORY_LIMIT_MB: 1024+ name: SQL Server Transforms+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref || github.ref }}+ - name: Setup python-runner+ uses: ./.github/actions/setup-python-runner+ with:+ metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ - name: Test SQL Server transforms+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-sqlserver-transforms'+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}+ aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ - name: Cleanup python-runner+ if: always()+ uses: ./.github/actions/cleanup-python-runner+ with:+ upload-logs: ${{ steps.test-driver.outcome == 'failure' }}+ logs-artifact-suffix: SQL Server Transforms++ transforms-stress-test-result:+ needs:+ - be-tests-bigquery-transforms+ - be-tests-clickhouse-transforms+ - be-tests-mongo-transforms+ - be-tests-mysql-transforms+ - be-tests-postgres-transforms+ - be-tests-redshift-transforms+ - be-tests-snowflake-transforms+ - be-tests-sqlserver-transforms+ runs-on: ubuntu-latest+ timeout-minutes: 5+ name: transforms-stress-test-result+ if: always() && !cancelled()+ steps:+ - name: Check job status+ uses: actions/github-script@v7+ env:+ needs: ${{ toJson(needs) }}+ with:+ script: |+ const needs = JSON.parse(process.env.needs);++ const jobs = Object.entries(needs).map(+ ([jobName, jobValues]) => ({+ name: jobName,+ result: jobValues.result+ }));++ if (jobs.every(job => (job.result === 'skipped' || job.result === 'success'))) {+ console.log("All transforms tests have passed (or been skipped).");+ process.exit(0);+ }++ console.log("Some transforms tests have failed:");+ jobs.forEach((job) => {+ if (job.result !== 'success' && job.result !== 'skipped') {+ console.log(` ${job.name} - ${job.result}`);+ }+ });++ process.exit(1);diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex 75c1d64e84d..9789ddcd39f 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -82,12 +82,14 @@[schema checkpoint-config](let [{:keys [field-name template-tag-type]} checkpoint-configtimestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))- query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"+ query (format "SELECT *, %s AS %s FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"timestamp-sql+ (sql.u/quote-name driver/*driver* :field "load_timestamp")(if schema(sql.u/quote-name driver/*driver* :table schema "transforms_products")"transforms_products")- field-name field-name)]+ (sql.u/quote-name driver/*driver* :field field-name)+ (sql.u/quote-name driver/*driver* :field field-name))]{:database (mt/id):type :native:native {:query query@@ -101,8 +103,9 @@"Create a native query without template tags for testing automatic checkpoint insertion. "[schema](let [timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))- query (format "SELECT *, %s AS load_timestamp FROM %s"+ query (format "SELECT *, %s AS %s FROM %s"timestamp-sql+ (sql.u/quote-name driver/*driver* :field "load_timestamp")(if schema(sql.u/quote-name driver/*driver* :table schema "transforms_products")"transforms_products"))]@@ -131,7 +134,7 @@transform-type can be :native, :native-auto-wrap, :mbql, or :pythoncheckpoint-config should be from checkpoint-configs"- [transform-name target-table-name transform-type checkpoint-config]+ [transform-name target-table transform-type checkpoint-config](let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)){:keys [field-name lib-column-key]} checkpoint-config]{:name transform-name@@ -155,33 +158,28 @@:body incremental-python-body:source-incremental-strategy {:type "checkpoint":checkpoint-filter-unique-key lib-column-key}})- :target {:type "table-incremental"- :schema schema- :name target-table-name- :database (mt/id)- :target-incremental-strategy {:type "append"}}}))+ :target (merge target-table {:type "table-incremental"+ :target-incremental-strategy {:type "append"}})}))(defn- get-table-row-count"Get the row count of a table by name."- [table-name]- (let [table (t2/select-one :model/Table :name table-name)- mp (mt/metadata-provider)+ [{table-name :name}]+ (let [table (t2/select-one :model/Table :name table-name)+ mp (mt/metadata-provider)table-metadata (lib.metadata/table mp (:id table))- count-query (lib/aggregate (lib/query mp table-metadata) (lib/count))- result (qp/process-query count-query)]+ count-query (lib/aggregate (lib/query mp table-metadata) (lib/count))+ result (qp/process-query count-query)](some-> result :data :rows first first bigint)))(defn- get-distinct-timestamp-count"Get the count of distinct load_timestamp values in a table."- [table-name]- (let [table (t2/select-one :model/Table :name table-name)- native-query {:database (mt/id)- :type :native- :native {:query (format "SELECT COUNT(DISTINCT load_timestamp) FROM %s"- (sql.u/quote-name driver/*driver* :table- (:schema table)- table-name))}}- result (qp/process-query native-query)]+ [{table-name :name, :keys [schema database]}]+ (let [native-query {:database database+ :type :native+ :native {:query (format "SELECT COUNT(DISTINCT %s) FROM %s"+ (sql.u/quote-name driver/*driver* :field "load_timestamp")+ (sql.u/quote-name driver/*driver* :table schema table-name))}}+ result (qp/process-query native-query)](some-> result :data :rows first first bigint)))(defn get-checkpoint-value [transform]@@ -192,7 +190,7 @@"Compare two checkpoint values with type-appropriate logic. "[checkpoint-type expected actual](case checkpoint-type- :integer (= expected actual)+ :integer (= (bigint expected) (bigint actual)):float (and (number? actual)(< (Math/abs (- expected actual)) 0.01)):temporal (and (string? actual)@@ -203,14 +201,13 @@[products](let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))- schema-prefix (if schema (str schema ".") "")values-list (str/join ", "(map (fn [{:keys [name category price created-at]}](format "('%s', '%s', %s, '%s')" name category price created-at))products))- insert-sql (format "INSERT INTO %s%s (name, category, price, created_at) VALUES %s"- schema-prefix- source-table-name+ insert-sql (format "INSERT INTO %s (%s) VALUES %s"+ (sql.u/quote-name driver/*driver* :table schema source-table-name)+ (str/join "," (map #(sql.u/quote-name driver/*driver* :field %) ["name" "category" "price" "created_at"]))values-list)](driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))@@ -218,10 +215,9 @@[products](let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))- schema-prefix (if schema (str schema ".") "")- delete-sql (format "DELETE FROM %s%s WHERE name IN (%s)"- schema-prefix- source-table-name+ delete-sql (format "DELETE FROM %s WHERE %s IN (%s)"+ (sql.u/quote-name driver/*driver* :table schema source-table-name)+ (sql.u/quote-name driver/*driver* :field "name")(str/join ", " (map (constantly "?") products)))](driver/execute-raw-queries! driver/*driver* spec [[delete-sql (mapv :name products)]])))@@ -238,6 +234,12 @@(defn- test-drivers [](disj (mt/normal-drivers-with-feature :transforms/table) :redshift :clickhouse :sqlserver))+(defn- target-table-gen [prefix]+ {:type "table"+ :name prefix+ :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ :database (mt/id)})+(deftest create-incremental-transform-test(testing "Creating an incremental transform with checkpoint strategy"(doseq [checkpoint-type [:integer :float]@@ -246,7 +248,7 @@(mt/test-drivers (test-drivers)(mt/with-premium-features #{:transforms}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "incremental_test"]+ (with-transform-cleanup! [target-table (target-table-gen "incremental_test")](let [checkpoint-config (get checkpoint-configs checkpoint-type){:keys [field-name]} checkpoint-configtransform-payload (make-incremental-transform-payload "Test Incremental Transform" target-table :native checkpoint-config)]@@ -281,14 +283,14 @@(mt/test-drivers (test-drivers)(mt/with-premium-features #{:transforms :transforms-python}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "incremental_twice"]+ (with-transform-cleanup! [target-table (target-table-gen "incremental_twice")](let [checkpoint-config (get checkpoint-configs checkpoint-type){:keys [expected-initial-checkpoint expected-second-checkpoint]} checkpoint-configtransform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)](mt/with-temp [:model/Transform transform transform-payload](testing "First run processes all data"(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (transforms.tu/wait-for-table target-table 10000)+ (transforms.tu/wait-for-table (:name target-table) 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)](is (= 10 row-count) "First run should process the first 10 products")@@ -315,17 +317,17 @@transform-type [:native :mbql :python]:when (valid-checkpoint-transform-combo? checkpoint-type transform-type)](testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))- (mt/test-drivers (test-drivers)+ (mt/test-drivers (disj (test-drivers) :bigquery-cloud-sdk) ; will follow up with a fix via GDGT-1777(mt/with-premium-features #{:transforms :transforms-python}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "switch_incr_to_non_incr"]+ (with-transform-cleanup! [target-table (target-table-gen "switch_incr_to_non_incr")](let [checkpoint-config (get checkpoint-configs checkpoint-type){:keys [expected-initial-checkpoint]} checkpoint-configinitial-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)](mt/with-temp [:model/Transform transform initial-payload](testing "Initial incremental run processes first batch"(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (transforms.tu/wait-for-table target-table 10000)+ (transforms.tu/wait-for-table (:name target-table) 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)checkpoint (get-checkpoint-value transform)]@@ -374,7 +376,7 @@(mt/test-drivers (test-drivers)(mt/with-premium-features #{:transforms :transforms-python}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "switch_non_incr_to_incr"]+ (with-transform-cleanup! [target-table (target-table-gen "switch_non_incr_to_incr")](let [checkpoint-config (get checkpoint-configs checkpoint-type){:keys [expected-second-checkpoint]} checkpoint-configincremental-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)@@ -385,7 +387,7 @@(mt/with-temp [:model/Transform transform initial-payload](testing "Initial non-incremental run creates table"(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (transforms.tu/wait-for-table target-table 10000)+ (transforms.tu/wait-for-table (:name target-table) 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)](is (= 10 row-count) "Initial run should process 10 products")@@ -411,14 +413,16 @@(is (= 2 distinct-timestamps) "Should have 2 distinct timestamp")(is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))- (when-not (= driver/*driver* :clickhouse)+ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment+ (not= driver/*driver* :clickhouse)+ ;; this *should* work see #68965 for context, will plan follow-up task+ (not= driver/*driver* :snowflake))(testing "Add new data and run incrementally"(with-insert-test-products![{:name "After Switch Product":category "Gadget":price 379.99:created-at "2024-01-20T10:00:00"}]-(let [transform (t2/select-one :model/Transform (:id transform))](execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})(let [row-count (get-table-row-count target-table)@@ -435,7 +439,7 @@(mt/test-drivers (test-drivers)(mt/with-premium-features #{:transforms}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "native_no_template"]+ (with-transform-cleanup! [target-table (target-table-gen "native_no_template")](let [checkpoint-config (get checkpoint-configs checkpoint-type){:keys [expected-second-checkpoint field-name]} checkpoint-configschema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))@@ -444,16 +448,14 @@:query (make-incremental-source-query-without-template-tag schema):source-incremental-strategy {:type "checkpoint":checkpoint-filter field-name}}- :target {:type "table-incremental"- :schema schema- :name target-table- :database (mt/id)- :target-incremental-strategy {:type "append"}}}]+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy {:type "append"}})}](mt/with-temp [:model/Transform transform transform-payload](testing "First run processes all existing data"(transforms.execute/execute! transform {:run-method :manual})- (transforms.tu/wait-for-table target-table 10000)- (let [row-count (get-table-row-count target-table)+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (let [row-count (get-table-row-count target-table)checkpoint (get-checkpoint-value transform)](is (= 16 row-count) "First run should process all 16 products")(is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)@@ -464,7 +466,10 @@(let [row-count (get-table-row-count target-table)](is (= 16 row-count) "Should still have 16 rows, no new data")))- (when-not (= driver/*driver* :clickhouse)+ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment+ (not= driver/*driver* :clickhouse)+ ;; this *should* work see #68965 for context, will plan follow-up task+ (not= driver/*driver* :snowflake))(testing "After inserting new data, incremental run appends only new rows"(with-insert-test-products![{:name "New Product 1"@@ -484,7 +489,7 @@(deftest unsupported-checkpoint-column-type-test(testing "Transform fails at runtime with unsupported checkpoint column type"- (mt/test-drivers (test-drivers)+ (mt/test-drivers #{:h2 :postgres}(mt/with-premium-features #{:transforms}(mt/dataset transforms-dataset/transforms-test(with-transform-cleanup! [target-table "unsupported_type_test"]@@ -511,7 +516,7 @@(mt/test-drivers [:postgres](mt/with-premium-features #{:transforms}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "native_temporal_cast"]+ (with-transform-cleanup! [target-table (target-table-gen "native_temporal_cast")](let [checkpoint-config (get checkpoint-configs :temporal){:keys [expected-initial-checkpoint expected-second-checkpoint field-name]} checkpoint-configschema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))@@ -535,18 +540,16 @@:required false}}}}:source-incremental-strategy {:type "checkpoint":checkpoint-filter field-name}}- :target {:type "table-incremental"- :schema schema- :name target-table- :database (mt/id)- :target-incremental-strategy {:type "append"}}}]+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy {:type "append"}})}](mt/with-temp [:model/Transform transform transform-payload](testing "First run processes first batch"(transforms.execute/execute! transform {:run-method :manual})- (transforms.tu/wait-for-table target-table 10000)- (let [row-count (get-table-row-count target-table)+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value transform)]+ checkpoint (get-checkpoint-value transform)](is (= 10 row-count) "First run should process the first 10 products")(is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")(is (compare-checkpoint-values :temporal expected-initial-checkpoint checkpoint)@@ -554,7 +557,7 @@(testing "Second run processes remaining data"(transforms.execute/execute! transform {:run-method :manual})- (let [row-count (get-table-row-count target-table)+ (let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)checkpoint (get-checkpoint-value transform)](is (= 16 row-count) "Second run should add remaining 6 rows")@@ -685,14 +688,13 @@(mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))(mt/with-premium-features #{:transforms :transforms-python}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "incremental_index"]+ (with-transform-cleanup! [target-table (target-table-gen "incremental_index")](let [checkpoint-config (get checkpoint-configs checkpoint-type)- transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)- schema (:schema (:target transform-payload))]+ transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)](mt/with-temp [:model/Transform transform transform-payload](testing "First run creates index on checkpoint column"(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:schema schema, :name target-table})+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)field-name (:field-name checkpoint-config)](testing "Index was created"(is (= 1 (count indexes)))@@ -701,7 +703,7 @@(is (= 10 (get-table-row-count target-table))))(testing "Second run succeeds with existing index"(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:schema schema, :name target-table})+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)field-name (:field-name checkpoint-config)](testing "Index still exists"(is (= 1 (count indexes)))@@ -718,14 +720,14 @@(mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))(mt/with-premium-features #{:transforms :transforms-python}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "index_cleanup_non_incr"]+ (with-transform-cleanup! [target-table (target-table-gen "index_cleanup_non_incr")](let [checkpoint-config (get checkpoint-configs checkpoint-type){:keys [field-name]} checkpoint-configincremental-payload (make-incremental-transform-payload "Index Cleanup Transform" target-table transform-type checkpoint-config)](mt/with-temp [:model/Transform transform incremental-payload](testing "First incremental run creates index"(execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:name target-table})]+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)](is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes)))))(testing "Switch to non-incremental via API"(let [non-incremental-payload (-> incremental-payload@@ -736,7 +738,7 @@(testing "Non-incremental run removes automatic indexes indexes"(let [transform (t2/select-one :model/Transform (:id transform))](execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:name target-table})+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)mb-indexes (filter #(str/starts-with? (:index-name %) "mb_transform_idx_") indexes)](testing "Automatic indexes are dropped"(is (empty? mb-indexes))))))))))))))))@@ -746,7 +748,7 @@(mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))(mt/with-premium-features #{:transforms}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "icchange"]+ (with-transform-cleanup! [target-table (target-table-gen "icchange")](let [integer-config (get checkpoint-configs :integer)float-config (get checkpoint-configs :float)initial-field (:field-name integer-config)@@ -756,7 +758,7 @@(mt/with-temp [:model/Transform transform initial-payload](testing "First run with integer checkpoint creates index on id column"(execute-transform-with-ordering! transform transform-type initial-field {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:name target-table})]+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)](is (= 1 (count indexes)))(is (=? {:value initial-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))(testing "Switch checkpoint column via API"@@ -767,8 +769,7 @@(testing "Run with new checkpoint column updates indexes"(let [transform (t2/select-one :model/Transform (:id transform))](execute-transform-with-ordering! transform transform-type new-field {:run-method :manual})- (let [table (t2/select-one :model/Table :name target-table)- indexes (driver/describe-table-indexes driver/*driver* (mt/id) {:schema (:schema table) :name target-table})]+ (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)](testing "Old index removed, new checkpoint column is now indexed"(is (= 1 (count indexes)))(is (=? {:value new-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.cljindex b700f8b94fb..293d281f987 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj@@ -106,7 +106,7 @@(defn- test-run [& {:keys [program user features source-tables extra-opts]:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]user :crowberto- source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}+ source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}features #{:transforms :transforms-python}}}](let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)](mt/with-premium-features features@@ -154,12 +154,12 @@(is (=? {:status 200} (test-run :user :lucky))))))))(deftest test-run-input-limit-test- (testing "maximum"- (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))- (testing "minimum"- (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))- (testing "truncates sources"- (mt/dataset transforms-dataset/transforms-test+ (mt/dataset transforms-dataset/transforms-test+ (testing "maximum"+ (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))+ (testing "minimum"+ (is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))+ (testing "truncates sources"(let [program ["def transform(customers):" " return customers"]source-tables {"customers" (mt/id :transforms_customers)}response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})]diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 34fafad5141..248f264b20a 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -911,10 +911,10 @@;;; +----------------------------------------------------------------------------------------------------------------+(defn- get-table-str [table]- (let [table-str (if (namespace table)- (format "%s.%s" (namespace table) (name table))- (name table))]- (sql.u/quote-name :bigquery-cloud-sdk :table table-str)))+ (let [qn #(sql.u/quote-name :bigquery-cloud-sdk :table %)]+ (if (namespace table)+ (format "%s.%s" (qn (namespace table)) (qn (name table)))+ (qn (name table)))))(defmethod driver/compile-transform :bigquery-cloud-sdk[_driver {:keys [query output-table]}]diff --git a/test/metabase/transforms/test_util.clj b/test/metabase/transforms/test_util.cljindex 79eca9a0a01..720d249348c 100644--- a/test/metabase/transforms/test_util.clj+++ b/test/metabase/transforms/test_util.clj@@ -156,7 +156,9 @@[](or (when (get-method driver.sql/default-schema driver/*driver*)(driver.sql/default-schema driver/*driver*))- "public"))+ (if (= :bigquery-cloud-sdk driver/*driver*)+ (t2/select-one-fn :schema :model/Table :db_id (mt/id))+ "public")))(defmulti delete-schema!"Deletes a schema."