Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Split Snowflake driver tests across 2 runners x 2 JVMs (#81813)

Riley Thompsondd4f301ae28f
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index f786203b6f9..0a257eee640 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -976,7 +976,7 @@ jobs:
needs: determine-driver-skips
if: ${{ needs.determine-driver-skips.outputs.snowflake-should-run == 'true' }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 70
+ timeout-minutes: 40
permissions:
# Required to mint a GitHub Actions OIDC token for the Snowflake WIF live test.
id-token: write
@@ -984,19 +984,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- job:
- - name: Driver Tests
- test-args: >-
- :only-tags [:mb/driver-tests]
- :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]'
- :fail-fast? true
- - name: Upload Tests
- test-args: >-
- :only-tags [:mb/upload-tests]
- - name: Transforms Python Tests
- test-args: >-
- :only-tags [:mb/transforms-python-test]
- needs-python-runner: true
+ runner-index: [0, 1]
env:
CI: 'true'
DRIVERS: snowflake
@@ -1026,11 +1014,89 @@ jobs:
MB_SNOWFLAKE_TEST_WIF_USER: ${{ github.event_name == 'pull_request' && 'METABASE WIF' || (github.ref == 'refs/heads/master' && 'METABASE WIF MAIN' || 'skip') }}
MB_SNOWFLAKE_TEST_WIF_DB: SNOWFLAKE_SAMPLE_DATA
MB_SNOWFLAKE_TEST_WIF_ROLE: PUBLIC
- name: Snowflake ${{ matrix.job.name }}
+ name: Snowflake Driver Tests ${{ matrix.runner-index }}
+ steps:
+ - uses: actions/checkout@v7
+ - name: Test Snowflake driver
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-snowflake-ee'
+ test-args: >-
+ :only-tags [:mb/driver-tests]
+ :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]'
+ :fail-fast? true
+ runner-index: ${{ matrix.runner-index }}
+ runner-count: ${{ strategy.job-total }}
+ jvms-per-runner: '2'
+ heap-per-jvm: '4g'
+ artifact-suffix: runner-${{ matrix.runner-index }}
+
+ be-tests-snowflake-upload-ee:
+ needs: determine-driver-skips
+ if: ${{ needs.determine-driver-skips.outputs.snowflake-should-run == 'true' }}
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 40
+ env:
+ CI: 'true'
+ DRIVERS: snowflake
+ MB_SNOWFLAKE_TEST_USER: METABASE CI
+ MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}
+ MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}
+ MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}
+ MB_SNOWFLAKE_TEST_PK_USER: METABASE PK
+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2 }}
+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY }}
+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2 }}
+
+ # RSA Role testing:
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DEFAULT_USER: RSA_ROLE_TEST_DEFAULT_USER
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB
+ name: Snowflake Upload Tests
+ steps:
+ - uses: actions/checkout@v7
+ - name: Test Snowflake driver
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ # This is the same as the driver-tests job above to preserve quarantine history in ci-conductor.
+ junit-name: 'be-tests-snowflake-ee'
+ test-args: >-
+ :only-tags [:mb/upload-tests]
+ artifact-suffix: upload
+
+ be-tests-snowflake-transforms-python-ee:
+ needs: determine-driver-skips
+ if: ${{ needs.determine-driver-skips.outputs.snowflake-should-run == 'true' }}
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 30
+ env:
+ CI: 'true'
+ DRIVERS: snowflake
+ MB_SNOWFLAKE_TEST_USER: METABASE CI
+ MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }}
+ MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }}
+ MB_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }}
+ MB_SNOWFLAKE_TEST_PK_USER: METABASE PK
+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY }}
+ MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY_2 }}
+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY }}
+ MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PUBLIC_KEY_2 }}
+
+ # RSA Role testing:
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DEFAULT_USER: RSA_ROLE_TEST_DEFAULT_USER
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE
+ MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB
+ name: Snowflake Transforms Python Tests
steps:
- uses: actions/checkout@v7
- name: Setup python-runner
- if: ${{ matrix.job.needs-python-runner == true }}
uses: ./.github/actions/setup-python-runner
with:
metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
@@ -1038,15 +1104,19 @@ jobs:
id: test-driver
uses: ./.github/actions/test-driver
with:
+ # This is the same as the driver-tests job above to preserve quarantine history in ci-conductor.
junit-name: 'be-tests-snowflake-ee'
- test-args: ${{ matrix.job.test-args }}
- artifact-suffix: ${{ matrix.job.name }}
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ artifact-suffix: transforms-python
- name: Cleanup python-runner
- if: ${{ always() && matrix.job.needs-python-runner == true }}
+ if: ${{ always() }}
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ matrix.job.name }}
+ # Driver-qualified: cleanup-python-runner names its artifact from this suffix alone, so it
+ # must not collide with the BigQuery transforms-python job's in the same run.
+ logs-artifact-suffix: snowflake-transforms-python
be-tests-sparksql-ee:
needs: determine-driver-skips
@@ -1266,6 +1336,8 @@ jobs:
- be-tests-presto-jdbc-ee
- be-tests-redshift-ee
- be-tests-snowflake-ee
+ - be-tests-snowflake-transforms-python-ee
+ - be-tests-snowflake-upload-ee
- be-tests-sparksql-ee
- be-tests-sqlite-ee
- be-tests-sqlserver