The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Skip cloud driver tests when possible + consolidate test skipping logic (#67568)
bryanf290bbbbe0b9
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.ymlindex e3fe8f4c41c..29ca9b049e1 100644--- a/.github/actions/test-driver/action.yml+++ b/.github/actions/test-driver/action.yml@@ -13,54 +13,21 @@ inputs:required: falsedescription: Suffix to append to the test logs artifact name to avoid conflicts in matrix jobs-outputs:- quarantine-skipped:- description: Whether the driver test was skipped due to quarantine- value: ${{ steps.check-driver.outputs.skip }}-runs:using: "composite"steps:- - name: Check if driver is ignored- id: check-driver- shell: bash- run: |- # If DRIVERS env var is not set, proceed with tests- if [ -z "$DRIVERS" ]; then- echo "No DRIVERS env var set. Proceeding with tests."- echo "skip=false" >> $GITHUB_OUTPUT- exit 0- fi-- # Fetch CI config and check if driver should be skipped- curl -sSf -o ci-test-config.json https://raw.githubusercontent.com/metabase/ci-test-config/refs/heads/master/ci-test-config.json || true-- # Check if the driver is in the ignored list (gracefully handle missing/invalid config)- IS_IGNORED=$(jq --arg driver "$DRIVERS" '.ignored.drivers // [] | contains([$driver])' ci-test-config.json 2>/dev/null || echo "false")-- if [ "$IS_IGNORED" = "true" ]; then- echo "⚠️ Driver '$DRIVERS' is in the ignored list. Skipping tests."- echo "View the ignored list here: https://github.com/metabase/ci-test-config/blob/master/ci-test-config.json"- echo "skip=true" >> $GITHUB_OUTPUT- else- echo "Driver '$DRIVERS' is not ignored. Proceeding with tests."- echo "skip=false" >> $GITHUB_OUTPUT- fi-- name: Prepare back-end environment- if: steps.check-driver.outputs.skip != 'true'uses: ./.github/actions/prepare-backend- name: Prepare front-end environment- if: ${{ steps.check-driver.outputs.skip != 'true' && inputs.build-static-viz == 'true' }}+ if: ${{ inputs.build-static-viz == 'true' }}uses: ./.github/actions/prepare-frontend- name: Build static viz frontend- if: ${{ steps.check-driver.outputs.skip != 'true' && inputs.build-static-viz == 'true' }}+ if: ${{ inputs.build-static-viz == 'true' }}run: yarn build-static-vizshell: bashenv:MB_EDITION: ee- name: Test database driver- if: steps.check-driver.outputs.skip != 'true'run: clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test ${{ inputs.test-args }}shell: bashid: run-test@@ -73,7 +40,7 @@ runs:# As per the documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-context# test.outcome gives us access to the result before `continue-on-error` overrides it and sets it to 'success'.# We want test reports even if the test step gets cancelled (e.g. times out), hence we match anything that is not a 'success'.- if: ${{ steps.run-test.outcome != 'success' && steps.check-driver.outputs.skip != 'true' }}+ if: ${{ steps.run-test.outcome != 'success' }}with:path: "target/junit/**/*_test.xml"name: JUnit Test Report ${{ inputs.junit-name }}@@ -82,7 +49,7 @@ runs:list-tests: failedfail-on-error: false- name: Upload Logs on Failure- if: always() && steps.run-test.outcome != 'success' && steps.check-driver.outputs.skip != 'true'+ if: always() && steps.run-test.outcome != 'success'uses: actions/upload-artifact@v4with:name: ${{ format('test-logs-{0}{1}', github.job, (inputs.logs-artifact-suffix == '' || inputs.logs-artifact-suffix == null) && '' || format('-{0}', inputs.logs-artifact-suffix)) }}diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 47dd8a34319..89f448597ff 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -12,12 +12,52 @@ concurrency:cancel-in-progress: truejobs:- # This job will determine if the tier 3 driver tests should be skipped.- # It will always run first.+ # This job determines which driver tests should run based on PR context.+ # All decision logic is consolidated in mage driver-decisions.determine-driver-skips:runs-on: ubuntu-latestoutputs:- skip: ${{ steps.check-branch-and-files-changed.outputs.skip }}+ # All driver run decisions (from mage driver-decisions)+ h2-should-run: ${{ steps.driver-decisions.outputs.h2-should-run }}+ athena-should-run: ${{ steps.driver-decisions.outputs.athena-should-run }}+ bigquery-should-run: ${{ steps.driver-decisions.outputs.bigquery-should-run }}+ clickhouse-should-run: ${{ steps.driver-decisions.outputs.clickhouse-should-run }}+ databricks-should-run: ${{ steps.driver-decisions.outputs.databricks-should-run }}+ druid-should-run: ${{ steps.driver-decisions.outputs.druid-should-run }}+ druid-jdbc-should-run: ${{ steps.driver-decisions.outputs.druid-jdbc-should-run }}+ mongo-should-run: ${{ steps.driver-decisions.outputs.mongo-should-run }}+ mongo-ssl-should-run: ${{ steps.driver-decisions.outputs.mongo-ssl-should-run }}+ mongo-sharded-cluster-should-run: ${{ steps.driver-decisions.outputs.mongo-sharded-cluster-should-run }}+ mysql-mariadb-should-run: ${{ steps.driver-decisions.outputs.mysql-mariadb-should-run }}+ oracle-should-run: ${{ steps.driver-decisions.outputs.oracle-should-run }}+ postgres-should-run: ${{ steps.driver-decisions.outputs.postgres-should-run }}+ presto-jdbc-should-run: ${{ steps.driver-decisions.outputs.presto-jdbc-should-run }}+ redshift-should-run: ${{ steps.driver-decisions.outputs.redshift-should-run }}+ snowflake-should-run: ${{ steps.driver-decisions.outputs.snowflake-should-run }}+ sparksql-should-run: ${{ steps.driver-decisions.outputs.sparksql-should-run }}+ sqlite-should-run: ${{ steps.driver-decisions.outputs.sqlite-should-run }}+ sqlserver-should-run: ${{ steps.driver-decisions.outputs.sqlserver-should-run }}+ vertica-should-run: ${{ steps.driver-decisions.outputs.vertica-should-run }}+ # Per-driver quarantine conflict flags (set when driver is quarantined but has file changes)+ athena-quarantine-conflict: ${{ steps.driver-decisions.outputs.athena-quarantine-conflict }}+ bigquery-quarantine-conflict: ${{ steps.driver-decisions.outputs.bigquery-quarantine-conflict }}+ clickhouse-quarantine-conflict: ${{ steps.driver-decisions.outputs.clickhouse-quarantine-conflict }}+ databricks-quarantine-conflict: ${{ steps.driver-decisions.outputs.databricks-quarantine-conflict }}+ druid-quarantine-conflict: ${{ steps.driver-decisions.outputs.druid-quarantine-conflict }}+ druid-jdbc-quarantine-conflict: ${{ steps.driver-decisions.outputs.druid-jdbc-quarantine-conflict }}+ mongo-quarantine-conflict: ${{ steps.driver-decisions.outputs.mongo-quarantine-conflict }}+ mongo-ssl-quarantine-conflict: ${{ steps.driver-decisions.outputs.mongo-ssl-quarantine-conflict }}+ mongo-sharded-cluster-quarantine-conflict: ${{ steps.driver-decisions.outputs.mongo-sharded-cluster-quarantine-conflict }}+ mysql-mariadb-quarantine-conflict: ${{ steps.driver-decisions.outputs.mysql-mariadb-quarantine-conflict }}+ oracle-quarantine-conflict: ${{ steps.driver-decisions.outputs.oracle-quarantine-conflict }}+ postgres-quarantine-conflict: ${{ steps.driver-decisions.outputs.postgres-quarantine-conflict }}+ presto-jdbc-quarantine-conflict: ${{ steps.driver-decisions.outputs.presto-jdbc-quarantine-conflict }}+ redshift-quarantine-conflict: ${{ steps.driver-decisions.outputs.redshift-quarantine-conflict }}+ snowflake-quarantine-conflict: ${{ steps.driver-decisions.outputs.snowflake-quarantine-conflict }}+ sparksql-quarantine-conflict: ${{ steps.driver-decisions.outputs.sparksql-quarantine-conflict }}+ sqlite-quarantine-conflict: ${{ steps.driver-decisions.outputs.sqlite-quarantine-conflict }}+ sqlserver-quarantine-conflict: ${{ steps.driver-decisions.outputs.sqlserver-quarantine-conflict }}+ vertica-quarantine-conflict: ${{ steps.driver-decisions.outputs.vertica-quarantine-conflict }}steps:- name: Checkout repositoryuses: actions/checkout@v4@@ -26,29 +66,26 @@ jobs:run: |# Fetch the base branch to ensure its history is available for diffinggit fetch --depth=1 origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}- # Only run this if it's a pull request and the base ref exists- - name: Check if driver tests should run- id: check-branch-and-files-changed+ - name: Determine which drivers should run+ id: driver-decisionsrun: |- IS_MASTER_OR_RELEASE="${{ github.ref_name == 'master' || startsWith(github.ref_name, 'release-') }}"- echo "Is master or release: ${IS_MASTER_OR_RELEASE}"-- if [[ "${IS_MASTER_OR_RELEASE}" == "true" ]]; then- echo "skip=false" >> $GITHUB_OUTPUT- echo "Merging into master or a release branch. All driver tests will run."- else- echo "Checking for changes in relevant modules..."- if ./bin/mage can-skip-driver-tests "${{ github.event.pull_request.base.ref }}"; then- echo 'skip=true' >> $GITHUB_OUTPUT- else- echo 'skip=false' >> $GITHUB_OUTPUT- fi- fi+ # All decision logic is in mage - it analyzes module dependencies,+ # branch context, labels, and file changes to decide what runs.+ MAGE_ARGS=(+ --git-ref="${{ github.event.pull_request.base.ref || 'master' }}"+ --is-master-or-release="${{ github.ref_name == 'master' || startsWith(github.ref_name, 'release-') }}"+ --pr-labels="${{ join(github.event.pull_request.labels.*.name, ',') }}"+ --skip="${{ inputs.skip }}"+ )+ # First call: show human-readable output for debugging+ ./bin/mage -driver-decisions "${MAGE_ARGS[@]}"+ # Second call: output only key=value lines for GITHUB_OUTPUT+ ./bin/mage -driver-decisions "${MAGE_ARGS[@]}" --github-output-only >> $GITHUB_OUTPUTshell: bashbe-tests-athena-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.athena-should-run == 'true' || needs.determine-driver-skips.outputs.athena-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -65,6 +102,16 @@ jobs:MB_ATHENA_TEST_WITHOUT_GET_TABLE_METADATA_SECRET_KEY: ${{ secrets.MB_ATHENA_TEST_WITHOUT_GET_TABLE_METADATA_SECRET_KEY }}name: Athenasteps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.athena-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Athena driver is quarantined but you changed files in modules/drivers/athena/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Athena tests, add the label: break-quarantine-athena"+ echo ""+ echo "If you merge without testing, you risk breaking the Athena driver!"+ exit 1- uses: actions/checkout@v4- name: Test Athena driverid: test-driver@@ -76,7 +123,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -88,7 +135,7 @@ jobs:be-tests-bigquery-cloud-sdk-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' || needs.determine-driver-skips.outputs.bigquery-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -116,6 +163,16 @@ jobs:MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}name: BigQuery ${{ matrix.job.name }}steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.bigquery-quarantine-conflict == 'true' }}+ run: |+ echo "::error::BigQuery driver is quarantined but you changed files in modules/drivers/bigquery-cloud-sdk/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run BigQuery tests, add the label: break-quarantine-bigquery"+ echo ""+ echo "If you merge without testing, you risk breaking the BigQuery driver!"+ exit 1- uses: actions/checkout@v4- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -131,7 +188,7 @@ jobs:logs-artifact-suffix: ${{ matrix.job.name }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -149,7 +206,7 @@ jobs:be-tests-clickhouse-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.clickhouse-should-run == 'true' || needs.determine-driver-skips.outputs.clickhouse-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -173,6 +230,16 @@ jobs:MB_CLICKHOUSE_TEST_NGINX_PORT: 8127name: Clickhouse ${{ matrix.job.name }}steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.clickhouse-quarantine-conflict == 'true' }}+ run: |+ echo "::error::ClickHouse driver is quarantined but you changed files in modules/drivers/clickhouse/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run ClickHouse tests, add the label: break-quarantine-clickhouse"+ echo ""+ echo "If you merge without testing, you risk breaking the ClickHouse driver!"+ exit 1- uses: actions/checkout@v4- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -203,7 +270,7 @@ jobs:logs-artifact-suffix: ${{ matrix.job.name }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -221,7 +288,7 @@ jobs:be-tests-druid-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.druid-should-run == 'true' || needs.determine-driver-skips.outputs.druid-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -236,6 +303,16 @@ jobs:CLUSTER_SIZE: nano-quickstartname: Druid (Legacy)steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.druid-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Druid driver is quarantined but you changed files in modules/drivers/druid/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Druid tests, add the label: break-quarantine-druid"+ echo ""+ echo "If you merge without testing, you risk breaking the Druid driver!"+ exit 1- uses: actions/checkout@v4- name: Test Druid driverid: test-driver@@ -247,7 +324,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -259,7 +336,7 @@ jobs:be-tests-druid-jdbc-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.druid-jdbc-should-run == 'true' || needs.determine-driver-skips.outputs.druid-jdbc-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -274,6 +351,16 @@ jobs:CLUSTER_SIZE: nano-quickstartname: Druid (JDBC)steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.druid-jdbc-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Druid JDBC driver is quarantined but you changed files in modules/drivers/druid-jdbc/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Druid JDBC tests, add the label: break-quarantine-druid-jdbc"+ echo ""+ echo "If you merge without testing, you risk breaking the Druid JDBC driver!"+ exit 1- uses: actions/checkout@v4- name: Test Druid JDBC driverid: test-driver@@ -285,7 +372,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -297,7 +384,7 @@ jobs:be-tests-databricks-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.databricks-should-run == 'true' || needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -312,6 +399,16 @@ jobs:MB_DATABRICKS_TEST_OAUTH_SECRET: ${{ secrets.MB_DATABRICKS_JDBC_TEST_OAUTH_SECRET }}name: Databrickssteps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Databricks driver is quarantined but you changed files in modules/drivers/databricks/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Databricks tests, add the label: break-quarantine-databricks"+ echo ""+ echo "If you merge without testing, you risk breaking the Databricks driver!"+ exit 1- uses: actions/checkout@v4- name: Test Databricks driverid: test-driver@@ -323,7 +420,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -335,7 +432,7 @@ jobs:be-tests-databricks-multi-catalog-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.databricks-should-run == 'true' || needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -351,6 +448,16 @@ jobs:MB_DATABRICKS_TEST_OAUTH_SECRET: ${{ secrets.MB_DATABRICKS_JDBC_TEST_OAUTH_SECRET }}name: Databricks (Multi-Catalog)steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Databricks driver is quarantined but you changed files in modules/drivers/databricks/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Databricks tests, add the label: break-quarantine-databricks"+ echo ""+ echo "If you merge without testing, you risk breaking the Databricks driver!"+ exit 1- uses: actions/checkout@v4- name: Test Databricks driver with multi-catalogid: test-driver@@ -362,7 +469,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -373,7 +480,8 @@ jobs:trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}be-tests-h2:- if: ${{ !inputs.skip }}+ needs: determine-driver-skips+ if: ${{ needs.determine-driver-skips.outputs.h2-should-run == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -391,7 +499,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -403,7 +511,7 @@ jobs:be-tests-mysql-mariadb:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.mysql-mariadb-should-run == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -521,7 +629,7 @@ jobs::exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -539,7 +647,7 @@ jobs:be-tests-mongo:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.mongo-should-run == 'true' || needs.determine-driver-skips.outputs.mongo-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -576,6 +684,16 @@ jobs:MONGO_INITDB_ROOT_PASSWORD: metasample123name: ${{ matrix.version.name }} ${{ matrix.job.name }}steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.mongo-quarantine-conflict == 'true' }}+ run: |+ echo "::error::MongoDB driver is quarantined but you changed files in modules/drivers/mongo/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run MongoDB tests, add the label: break-quarantine-mongo"+ echo ""+ echo "If you merge without testing, you risk breaking the MongoDB driver!"+ exit 1- uses: actions/checkout@v4- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -591,7 +709,7 @@ jobs:logs-artifact-suffix: ${{ matrix.job.name }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -609,7 +727,7 @@ jobs:be-tests-mongo-ssl:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.mongo-ssl-should-run == 'true' || needs.determine-driver-skips.outputs.mongo-ssl-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -627,6 +745,16 @@ jobs:MB_TEST_MONGO_REQUIRES_SSL: truename: "${{ matrix.version.name }} (SSL)"steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.mongo-ssl-quarantine-conflict == 'true' }}+ run: |+ echo "::error::MongoDB SSL driver is quarantined but you changed files in modules/drivers/mongo/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run MongoDB SSL tests, add the label: break-quarantine-mongo-ssl"+ echo ""+ echo "If you merge without testing, you risk breaking the MongoDB SSL driver!"+ exit 1- uses: actions/checkout@v4- name: Spin up Mongo docker containerrun: >-@@ -663,7 +791,7 @@ jobs::only-tags [:mb/driver-tests]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -675,7 +803,7 @@ jobs:be-tests-mongo-sharded-cluster-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.mongo-sharded-cluster-should-run == 'true' || needs.determine-driver-skips.outputs.mongo-sharded-cluster-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -688,6 +816,16 @@ jobs:- "27017:17017"name: MongoDB Sharded Clustersteps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.mongo-sharded-cluster-quarantine-conflict == 'true' }}+ run: |+ echo "::error::MongoDB Sharded Cluster driver is quarantined but you changed files in modules/drivers/mongo/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run MongoDB Sharded Cluster tests, add the label: break-quarantine-mongo-sharded-cluster"+ echo ""+ echo "If you merge without testing, you risk breaking the MongoDB Sharded Cluster driver!"+ exit 1- uses: actions/checkout@v4- name: Test MongoDB driver (Sharded cluster)id: test-driver@@ -698,7 +836,7 @@ jobs::only metabase.driver.mongo.sharded-cluster-test- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -710,7 +848,7 @@ jobs:be-tests-oracle:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.oracle-should-run == 'true' || needs.determine-driver-skips.outputs.oracle-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -761,6 +899,16 @@ jobs:- "1521:${{ matrix.version.port }}"name: ${{ matrix.version.name }}steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.oracle-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Oracle driver is quarantined but you changed files in modules/drivers/oracle/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Oracle tests, add the label: break-quarantine-oracle"+ echo ""+ echo "If you merge without testing, you risk breaking the Oracle driver!"+ exit 1- uses: actions/checkout@v4- name: Test ${{ matrix.version.name }}id: test-driver@@ -773,7 +921,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -785,7 +933,7 @@ jobs:be-tests-postgres:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.postgres-should-run == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -876,7 +1024,7 @@ jobs::exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -894,7 +1042,7 @@ jobs:be-tests-presto-jdbc-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.presto-jdbc-should-run == 'true' || needs.determine-driver-skips.outputs.presto-jdbc-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -917,6 +1065,16 @@ jobs:env:JAVA_TOOL_OPTIONS: "-Xmx2g"steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.presto-jdbc-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Presto JDBC driver is quarantined but you changed files in modules/drivers/presto-jdbc/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Presto JDBC tests, add the label: break-quarantine-presto-jdbc"+ echo ""+ echo "If you merge without testing, you risk breaking the Presto JDBC driver!"+ exit 1- uses: actions/checkout@v4- uses: ./.github/actions/await-portwith:@@ -953,7 +1111,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -965,7 +1123,7 @@ jobs:be-tests-redshift-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.redshift-should-run == 'true' || needs.determine-driver-skips.outputs.redshift-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -1010,6 +1168,16 @@ jobs:MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}name: ${{ matrix.job.name }}steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.redshift-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Redshift driver is quarantined but you changed files in modules/drivers/redshift/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Redshift tests, add the label: break-quarantine-redshift"+ echo ""+ echo "If you merge without testing, you risk breaking the Redshift driver!"+ exit 1- uses: actions/checkout@v4- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -1027,7 +1195,7 @@ jobs:logs-artifact-suffix: ${{ matrix.job.name }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: ${{ !matrix.job.skip && !cancelled() && steps.test-driver.outputs.quarantine-skipped != 'true' }}+ if: ${{ !matrix.job.skip && !cancelled() }}with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -1045,7 +1213,7 @@ jobs:be-tests-snowflake-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.snowflake-should-run == 'true' || needs.determine-driver-skips.outputs.snowflake-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60strategy:@@ -1083,6 +1251,16 @@ jobs:MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DBname: Snowflake ${{ matrix.job.name }}steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.snowflake-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Snowflake driver is quarantined but you changed files in modules/drivers/snowflake/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Snowflake tests, add the label: break-quarantine-snowflake"+ echo ""+ echo "If you merge without testing, you risk breaking the Snowflake driver!"+ exit 1- uses: actions/checkout@v4- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -1098,7 +1276,7 @@ jobs:logs-artifact-suffix: ${{ matrix.job.name }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -1116,7 +1294,7 @@ jobs:be-tests-sparksql-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.sparksql-should-run == 'true' || needs.determine-driver-skips.outputs.sparksql-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -1129,6 +1307,16 @@ jobs:- "10000:10000"name: Spark SQLsteps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.sparksql-quarantine-conflict == 'true' }}+ run: |+ echo "::error::SparkSQL driver is quarantined but you changed files in modules/drivers/sparksql/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run SparkSQL tests, add the label: break-quarantine-sparksql"+ echo ""+ echo "If you merge without testing, you risk breaking the SparkSQL driver!"+ exit 1- uses: actions/checkout@v4- name: Test Spark driverid: test-driver@@ -1140,7 +1328,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -1152,7 +1340,7 @@ jobs:be-tests-sqlite-ee:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.sqlite-should-run == 'true' || needs.determine-driver-skips.outputs.sqlite-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -1160,6 +1348,16 @@ jobs:DRIVERS: sqlitename: SQLitesteps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.sqlite-quarantine-conflict == 'true' }}+ run: |+ echo "::error::SQLite driver is quarantined but you changed files in modules/drivers/sqlite/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run SQLite tests, add the label: break-quarantine-sqlite"+ echo ""+ echo "If you merge without testing, you risk breaking the SQLite driver!"+ exit 1- uses: actions/checkout@v4- name: Test SQLite driverid: test-driver@@ -1171,7 +1369,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -1183,7 +1381,7 @@ jobs:be-tests-sqlserver:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.sqlserver-should-run == 'true' || needs.determine-driver-skips.outputs.sqlserver-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 30strategy:@@ -1222,6 +1420,16 @@ jobs:MSSQL_MEMORY_LIMIT_MB: 1024name: ${{ matrix.version.name }} ${{ matrix.job.name }}steps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.sqlserver-quarantine-conflict == 'true' }}+ run: |+ echo "::error::SQL Server driver is quarantined but you changed files in modules/drivers/sqlserver/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run SQL Server tests, add the label: break-quarantine-sqlserver"+ echo ""+ echo "If you merge without testing, you risk breaking the SQL Server driver!"+ exit 1- uses: actions/checkout@v4- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -1237,7 +1445,7 @@ jobs:logs-artifact-suffix: ${{ matrix.version.name }}- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}@@ -1294,7 +1502,7 @@ jobs:login-to-ecr:needs: determine-driver-skips- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.vertica-should-run == 'true' || needs.determine-driver-skips.outputs.vertica-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 15permissions:@@ -1324,7 +1532,7 @@ jobs:be-tests-vertica-ee:needs: [determine-driver-skips, login-to-ecr]- if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ if: ${{ needs.determine-driver-skips.outputs.vertica-should-run == 'true' || needs.determine-driver-skips.outputs.vertica-quarantine-conflict == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 60env:@@ -1340,6 +1548,16 @@ jobs:password: ${{ needs.login-to-ecr.outputs.docker_password }}name: Verticasteps:+ - name: Fail on quarantine conflict+ if: ${{ needs.determine-driver-skips.outputs.vertica-quarantine-conflict == 'true' }}+ run: |+ echo "::error::Vertica driver is quarantined but you changed files in modules/drivers/vertica/"+ echo ""+ echo "Your changes will NOT be tested because the driver is currently quarantined."+ echo "To run Vertica tests, add the label: break-quarantine-vertica"+ echo ""+ echo "If you merge without testing, you risk breaking the Vertica driver!"+ exit 1- uses: actions/checkout@v4- name: Make plugins directoryrun: mkdir plugins@@ -1353,7 +1571,7 @@ jobs::exclude-tags [:mb/transforms-python-test]- name: Upload Test Resultsuses: ./.github/actions/upload-test-results- if: always() && steps.test-driver.outputs.quarantine-skipped != 'true'+ if: always()with:input-path: ./target/junit/output-name: ${{ github.job }}diff --git a/bb.edn b/bb.ednindex 309ea33824c..e3411162029 100644--- a/bb.edn+++ b/bb.edn@@ -1,15 +1,15 @@{:min-bb-version "1.12.212";; we put path as bin, and everything is in the ./mage subdirectory,;; so the namespaces are mage.cli, mage.format, etc.- :paths ["mage/src" "mage/test" "bin/lint-migrations-file/src"]- :deps {dev.weavejester/cljfmt {:mvn/version "0.15.3"} ;; run cljfmt from bb- io.github.paintparty/bling {:mvn/version "0.8.8"} ;; printing bells and whistles- medley/medley {:mvn/version "1.3.0"} ;; utilities- metosin/malli {:mvn/version "0.17.0"} ;; data validation- mvxcvi/puget {:mvn/version "1.3.4"} ;; pretty printing- org.babashka/json {:mvn/version "0.1.6"} ;; json parsing- table/table {:mvn/version "0.5.0"} ;; table printing- }+ :paths ["mage/src" "mage/test" "bin/lint-migrations-file/src"]+ :deps {dev.weavejester/cljfmt {:mvn/version "0.15.3"} ;; run cljfmt from bb+ io.github.paintparty/bling {:mvn/version "0.8.8"} ;; printing bells and whistles+ medley/medley {:mvn/version "1.3.0"} ;; utilities+ metosin/malli {:mvn/version "0.17.0"} ;; data validation+ mvxcvi/puget {:mvn/version "1.3.4"} ;; pretty printing+ org.babashka/json {:mvn/version "0.1.6"} ;; json parsing+ table/table {:mvn/version "0.5.0"} ;; table printing+ }:tasks{:requires [[mage.cli :as cli][mage.color :as c]]@@ -21,91 +21,91 @@(cons 'do body))))alias- {:doc "Installs 'mage' alias and autocomplete"- :examples [["./bin/mage alias" "Prints instructions to install the 'mage' alias and autocomplete"]- ["./bin/mage alias install fish" "Installs mage alias for fish shell"]- ["./bin/mage alias print zsh" "Prints shell script to put into .zshrc for zsh users"]]- :requires [[mage.alias :as alias]]+ {:doc "Installs 'mage' alias and autocomplete"+ :examples [["./bin/mage alias" "Prints instructions to install the 'mage' alias and autocomplete"]+ ["./bin/mage alias install fish" "Installs mage alias for fish shell"]+ ["./bin/mage alias print zsh" "Prints shell script to put into .zshrc for zsh users"]]+ :requires [[mage.alias :as alias]]:arg-schema [:or[:tuple [:enum {:name "shell-type"} "bash" "zsh" "fish"]][:tuple [:enum {:name "mode"} "install" "print"]- [:enum {:name "shell-type"} "bash" "zsh" "fish"]]]- :task (task! (alias/instructions parsed))}+ [:enum {:name "shell-type"} "bash" "zsh" "fish"]]]+ :task (task! (alias/instructions parsed))}cljfmt-staged- {:doc "Runs cljfmt on staged files"+ {:doc "Runs cljfmt on staged files":requires [[mage.format :as format]]:examples [["./bin/mage cljfmt-staged" "Format staged files"]["./bin/mage cljfmt-staged -c" "Check staged files"]]- :options [["-c" "--force-check" "Check staged files"]]- :task (task! (format/staged parsed))}+ :options [["-c" "--force-check" "Check staged files"]]+ :task (task! (format/staged parsed))}cljfmt-files- {:doc "Runs cljfmt on the given files/directories"- :requires [[mage.format :as format]]- :examples [["./bin/mage cljfmt-files src/metabase/events.clj" "Format events.clj"]- ["./bin/mage cljfmt-files src" "Format all files in src"]- ["./bin/mage cljfmt-files -c src" "Check all files in src"]]- :options [["-c" "--force-check" "Check staged files"]]+ {:doc "Runs cljfmt on the given files/directories"+ :requires [[mage.format :as format]]+ :examples [["./bin/mage cljfmt-files src/metabase/events.clj" "Format events.clj"]+ ["./bin/mage cljfmt-files src" "Format all files in src"]+ ["./bin/mage cljfmt-files -c src" "Check all files in src"]]+ :options [["-c" "--force-check" "Check staged files"]]:arg-schema [:sequential [:string {:name "file-or-directory":description "Files or directories to format."}]]- :task (task! (format/files parsed))}+ :task (task! (format/files parsed))}cljfmt-all- {:doc "Runs cljfmt on all (clojure) files"+ {:doc "Runs cljfmt on all (clojure) files":requires [[mage.format :as format]]:examples [["./bin/mage cljfmt-all" "Format all files"]]- :options [["-c" "--force-check" "Check staged files"]]- :task (task! (format/all parsed))}+ :options [["-c" "--force-check" "Check staged files"]]+ :task (task! (format/all parsed))}cljfmt-updated- {:doc "Runs cljfmt on all (Clojure) files relative to a git ref (default HEAD)"- :requires [[mage.format :as format]]- :examples [["./bin/mage cljfmt-updated" "Format updated files relative to HEAD"]- ["./bin/mage cljfmt-updated master" "Format updated files relative to master"]]- :options [["-c" "--force-check" "Check staged files"]]+ {:doc "Runs cljfmt on all (Clojure) files relative to a git ref (default HEAD)"+ :requires [[mage.format :as format]]+ :examples [["./bin/mage cljfmt-updated" "Format updated files relative to HEAD"]+ ["./bin/mage cljfmt-updated master" "Format updated files relative to master"]]+ :options [["-c" "--force-check" "Check staged files"]]:arg-schema [:or [:tuple] [:tuple [:string {:name "git-ref" :description "Git ref to compare against"}]]]- :task (task! (format/updated parsed))}+ :task (task! (format/updated parsed))}kondo- {:doc "Runs Kondo against a file, directory, or everything we usually lint"+ {:doc "Runs Kondo against a file, directory, or everything we usually lint":examples [["./bin/mage kondo" "run Kondo against everything we usually lint"]["./bin/mage kondo src/metabase/db.clj" "run Kondo against a file"]["./bin/mage kondo src/metabase/db.clj src/metabase/config.clj" "run Kondo against 2 files"]["./bin/mage kondo src/metabase/api/macros" "run Kondo against a directory"]]:requires [[mage.kondo :as kondo]]- :task (task! (kondo/kondo (:arguments parsed)))}+ :task (task! (kondo/kondo (:arguments parsed)))}kondo-updated- {:doc "Runs kondo against files changed compared to a git ref"+ {:doc "Runs kondo against files changed compared to a git ref":examples [["./bin/mage kondo-updated" "run Kondo on files with changes relative to HEAD"]["./bin/mage kondo-updated master" "run Kondo on files with changes relative to master"]]:requires [[mage.kondo :as kondo]]- :task (task! (kondo/kondo-updated (:arguments parsed)))}+ :task (task! (kondo/kondo-updated (:arguments parsed)))}check- {:doc "Checks whether we can compile all source files (and finds circular dependencies)"+ {:doc "Checks whether we can compile all source files (and finds circular dependencies)":examples [["./bin/mage check"]]:requires [[mage.check]]- :task (task! (mage.check/check (:arguments parsed)))}+ :task (task! (mage.check/check (:arguments parsed)))}codespell- {:doc "Check for misspellings in the source code."+ {:doc "Check for misspellings in the source code.":examples [["./bin/mage codespell"]]:requires [[mage.codespell]]- :task (task! (mage.codespell/codespell (:arguments parsed)))}+ :task (task! (mage.codespell/codespell (:arguments parsed)))}codespell-interactive- {:doc "Check for misspellings in the source code, and interactively prompt to fix them."+ {:doc "Check for misspellings in the source code, and interactively prompt to fix them.":examples [["./bin/mage codespell-interactive"]]:requires [[mage.codespell]]- :task (task! (mage.codespell/codespell-interactive (:arguments parsed)))}+ :task (task! (mage.codespell/codespell-interactive (:arguments parsed)))}start-maildev- {:doc "Start Maildev"+ {:doc "Start Maildev":examples [["./bin/mage start-maildev -h" "print help for start-maildev"]]:requires [[mage.start-maildev :as start-maildev]]- :task (task! (start-maildev/start-maildev!))}+ :task (task! (start-maildev/start-maildev!))}start-python-runner{:doc "Start python-runner container with internal Moto S3 server for Python transforms"@@ -114,127 +114,127 @@:task (task! (start-python-runner/start-python-runner!))}start-db- {:doc "Start a db on a default port in docker"- :examples [["./bin/mage start-db postgres latest" "start the latest postgres db we support"]- ["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]]- :requires [[mage.start-db :as start-db]]+ {:doc "Start a db on a default port in docker"+ :examples [["./bin/mage start-db postgres latest" "start the latest postgres db we support"]+ ["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]]+ :requires [[mage.start-db :as start-db]]:arg-schema [:tuple[:enum {:name "db"} :postgres :mysql :mariadb :mongo][:enum {:name "relative-version"} :oldest :latest]]- :db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}- :mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}- :mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}- :mongo {:ports {:oldest 27017 :latest 27018} :eol-url "https://endoflife.date/api/mongodb.json"}}- :usage-fn start-db/usage- :task (task!- (let [[db version] (:arguments parsed)]- (start-db/start-db (:db-info (current-task)) db version)))}+ :db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}+ :mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}+ :mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}+ :mongo {:ports {:oldest 27017 :latest 27018} :eol-url "https://endoflife.date/api/mongodb.json"}}+ :usage-fn start-db/usage+ :task (task!+ (let [[db version] (:arguments parsed)]+ (start-db/start-db (:db-info (current-task)) db version)))}nrepl- {:doc "Starts the babashka nrepl: helpful for mage development"+ {:doc "Starts the babashka nrepl: helpful for mage development":requires [[babashka.nrepl.server :as nrepl.server][mage.color :as c]]:examples [["./bin/mage nrepl" "Starts the nrepl server"]["./bin/mage nrepl --port 50506" "Starts the nrepl server"]["./bin/mage nrepl -p 50506" "Starts the nrepl server"]]- :options [["-p" "--port PORT" "optionally provide a port to run the nrepl server on" :default 1667 :parse-fn Integer/parseInt]]- :task (task!- (let [{:keys [port]} (:options parsed)]- (spit ".nrepl-port" port)- (nrepl.server/start-server! {:port port})- (deref (promise))))}+ :options [["-p" "--port PORT" "optionally provide a port to run the nrepl server on" :default 1667 :parse-fn Integer/parseInt]]+ :task (task!+ (let [{:keys [port]} (:options parsed)]+ (spit ".nrepl-port" port)+ (nrepl.server/start-server! {:port port})+ (deref (promise))))}lint-migrations- {:doc "Lint migrations files"- :requires [[lint-migrations-file]- [babashka.process :as p]]- :examples [["./bin/mage lint-migrations" "Lint all migrations and print report"]]- :task (do (cli/parse! (current-task))- (p/shell "bin/pre-lint-migrations-file.sh")- (lint-migrations-file/-main))}+ {:doc "Lint migrations files"+ :requires [[lint-migrations-file]+ [babashka.process :as p]]+ :examples [["./bin/mage lint-migrations" "Lint all migrations and print report"]]+ :task (do (cli/parse! (current-task))+ (p/shell "bin/pre-lint-migrations-file.sh")+ (lint-migrations-file/-main))}jar-download- {:doc "Download (and optionally run) jar for a metabase version or branch"- :examples [["./bin/mage jar-download 50" "Download the latest enterprise version of release 50 to ~/path/to/my/metabase/jars"]- ["./bin/mage jar-download 1.45.2 -r" "Download and run metabase_1.45.2.jar"]- ["./bin/mage jar-download 1.45.2 -d ~/path/to/my/jars" "Download metabase_1.45.2.jar to ~/path/to/my/jars, deleting if it exists"]- ["./bin/mage jar-download master --run --port 3001" "Download and run the latest master branch jar on port 3001"]- ["JARS=~/my-stuff ./bin/mage jar-download 1.45.2" "Download metabase_1.45.2.jar to ~/my-stuff"]- ["./bin/mage jar-download v1.56.0-beta -r" "Download and run metabase_1.56.0-beta.jar"]]- :requires [[mage.jar-download :as jar-download]]- :options [["-r" "--run" "run the new jar after downloading it"]- ["-e" "--env-file FILE" "you can optionally provide a lein-env file to specify the environment to use when running the jar."]- ["-d" "--delete" "delete the old jar if found, by default does not re-download it"]- ["-p" "--port PORT" "you can optionally provide a port to run the jar on, if not, we use a sane port."]]+ {:doc "Download (and optionally run) jar for a metabase version or branch"+ :examples [["./bin/mage jar-download 50" "Download the latest enterprise version of release 50 to ~/path/to/my/metabase/jars"]+ ["./bin/mage jar-download 1.45.2 -r" "Download and run metabase_1.45.2.jar"]+ ["./bin/mage jar-download 1.45.2 -d ~/path/to/my/jars" "Download metabase_1.45.2.jar to ~/path/to/my/jars, deleting if it exists"]+ ["./bin/mage jar-download master --run --port 3001" "Download and run the latest master branch jar on port 3001"]+ ["JARS=~/my-stuff ./bin/mage jar-download 1.45.2" "Download metabase_1.45.2.jar to ~/my-stuff"]+ ["./bin/mage jar-download v1.56.0-beta -r" "Download and run metabase_1.56.0-beta.jar"]]+ :requires [[mage.jar-download :as jar-download]]+ :options [["-r" "--run" "run the new jar after downloading it"]+ ["-e" "--env-file FILE" "you can optionally provide a lein-env file to specify the environment to use when running the jar."]+ ["-d" "--delete" "delete the old jar if found, by default does not re-download it"]+ ["-p" "--port PORT" "you can optionally provide a port to run the jar on, if not, we use a sane port."]]:arg-schema [:or[:tuple [:string {:name "version"}]][:tuple [:string {:name "version"}] [:string {:name "path"}]]]- :usage-fn (fn [_]- (-> "Version can have the following forms:"- (str "\n -" (c/green "50") " for the latest enterprise version of release 50")- (str "\n -" (c/green "1.45.2") " for a specific version")- (str "\n -" (c/green "master") " for the latest jar created for master")- (str "\n -" (c/green "any-branch-with-a-pr") " for the latest uberjar created for that branch")- (str "\n -" (c/green "1.56.0-beta") " for a specific beta version")- (str "\n\nnote: If you get an error about 'Invalid or corrupt jarfile', run this command again with --delete.")))- :task (task! (jar-download/jar-download parsed))}+ :usage-fn (fn [_]+ (-> "Version can have the following forms:"+ (str "\n -" (c/green "50") " for the latest enterprise version of release 50")+ (str "\n -" (c/green "1.45.2") " for a specific version")+ (str "\n -" (c/green "master") " for the latest jar created for master")+ (str "\n -" (c/green "any-branch-with-a-pr") " for the latest uberjar created for that branch")+ (str "\n -" (c/green "1.56.0-beta") " for a specific beta version")+ (str "\n\nnote: If you get an error about 'Invalid or corrupt jarfile', run this command again with --delete.")))+ :task (task! (jar-download/jar-download parsed))}setup-stats-repl{:doc "Connect to the stats repl":examples [["./bin/mage connect-stats-repl" "Connects to the stats repl"]]:requires [[mage.stats-repl :as stats-repl]]- :task (stats-repl/connect (cli/parse! (current-task)))}+ :task (stats-repl/connect (cli/parse! (current-task)))}notify- {:doc "Send notification for a comment (that already exists)"+ {:doc "Send notification for a comment (that already exists)":requires [[mage.be-dev :refer [nrepl-eval]]]- :options [["-c" "--comment-id ID" "Notify about this comment id"]- ["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]+ :options [["-c" "--comment-id ID" "Notify about this comment id"]+ ["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]:examples [["./bin/mage notify -c 1234" "Notify about comment with id 1234"]["./bin/mage notify --comment-id 5678 --port 59498" "Notify about comment with id 5678 on nrepl port 59498"]]- :task (let [opts (:options (cli/parse! (current-task)))+ :task (let [opts (:options (cli/parse! (current-task)));; note: no quoting -> expect it to be int- code (format "(notify-comment-id! %d)" (parse-long (:comment-id opts)))]- (if (:port opts)- (nrepl-eval "metabase.comments.core" code (:port opts))- (nrepl-eval "metabase.comments.core" code)))}+ code (format "(notify-comment-id! %d)" (parse-long (:comment-id opts)))]+ (if (:port opts)+ (nrepl-eval "metabase.comments.core" code (:port opts))+ (nrepl-eval "metabase.comments.core" code)))}run-tests- {:doc "Quickly run tests locally. Pass dirs/files as arguments to run them."- :examples [["./bin/mage run-tests" "launch interactive test picker"]- ["./bin/mage run-tests test/metabase/db" "run tests in test/metabase/db"]- ["./bin/mage run-tests test/metabase/logger test/metabase/api_keys" "run tests in test/metabase/logger and src/metabase/api_keys"]- ["./bin/mage run-tests test/metabase/config_test.clj" "run all tests in the metabase.config-test namespace"]]+ {:doc "Quickly run tests locally. Pass dirs/files as arguments to run them."+ :examples [["./bin/mage run-tests" "launch interactive test picker"]+ ["./bin/mage run-tests test/metabase/db" "run tests in test/metabase/db"]+ ["./bin/mage run-tests test/metabase/logger test/metabase/api_keys" "run tests in test/metabase/logger and src/metabase/api_keys"]+ ["./bin/mage run-tests test/metabase/config_test.clj" "run all tests in the metabase.config-test namespace"]];; TODO: add option to select files as well as dirs from fzf with styles- :options [["-s" "--selecting STYLE" "Style of test selection" :default "all"]+ :options [["-s" "--selecting STYLE" "Style of test selection" :default "all"];; all, file, dir- ["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]+ ["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]:arg-schema [:sequential{:details "If none are given, you will be prompted to select directories recursively from test/ and enterprise/backend/test/"}[:string {:name "file-or-dir" :description "Files or directories (recursive) to test"}]]- :requires [[mage.quick-test-runner :as qtr]]- :task (task! (qtr/go parsed))}+ :requires [[mage.quick-test-runner :as qtr]]+ :task (task! (qtr/go parsed))}affected-modules- {:doc "Which backend modules have changes?"+ {:doc "Which backend modules have changes?":examples [["./bin/mage affected-modules master" "prints changes compared to `master`"]["./bin/mage affected-modules"]]:arg-schema [:or [:tuple] [:tuple [:string {:name "git-ref"}]]]:requires [[mage.modules]]- :task (task! (mage.modules/cli-print-affected-modules (:arguments parsed)))}+ :task (task! (mage.modules/cli-print-affected-modules (:arguments parsed)))}can-skip-driver-tests- {:doc "Can we skip driver tests for this branch?"+ {:doc "Can we skip driver tests for this branch?":examples [["./bin/mage can-skip-driver-tests master" "Returns zero (success) if we can skip driver tests based on changes related to `master`"]["./bin/mage can-skip-driver-tests" "Returns zero (success) if we can skip driver tests based on changes? (defaults to master)"]]:requires [[mage.modules]]- :task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}+ :task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}ls- {:doc "List all mage public and private tasks"+ {:doc "List all mage public and private tasks":examples [["./bin/mage ls" "List all tasks"]]:requires [[mage.util :as u]]- :task (task! (u/print-tasks parsed))}+ :task (task! (u/print-tasks parsed))}analytics:setup{:doc "Set up usage analytics development mode"@@ -273,12 +273,45 @@:examples [["./bin/mage doctor" "Run all health checks"]]:requires [[mage.doctor :as doctor]]:task (task! (doctor/doctor!))}-+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Private Tasks:;; These tasks are more for internal tooling, e.g. for use from git hooks etc.;; - hidden from `./bin/mage` listing and `bb tasks`, these all - start with a `-`+ -driver-decisions+ {:doc "Determine which driver tests should run based on PR context. Outputs decisions for all drivers."+ :examples [["./bin/mage -driver-decisions --git-ref=master --is-master-or-release=false --pr-labels= --skip=false" "Determine driver decisions for a PR"]+ ["./bin/mage -driver-decisions --pr-labels=ci:all-cloud-drivers" "Run all cloud drivers via label"]+ ["./bin/mage -driver-decisions --github-output-only" "Output only GITHUB_OUTPUT format (for CI)"]]+ :usage-fn+ (fn [_]+ (->>+ ["# How we decide to run a driver's tests. For each driver, we check these conditions in order (first match wins):"+ " |----------|-------------------------------------|--------|-------------------------------------------|"+ " | Priority | Condition | Result | Reason |"+ " |----------|-------------------------------------|--------|-------------------------------------------|"+ " | 1 | Global skip (no backend changes) | SKIP | workflow skip (no backend changes) |"+ " | 2 | Driver is :h2 | RUN | H2 always runs |"+ " | 3 | Driver is quarantined | SKIP | driver is quarantined |"+ " | 3(a) | ...and has break-quarantine-X label | RUN | anti-quarantine label present |"+ " | 4 | On master or release branch | RUN | master/release branch |"+ " | 5 | Cloud driver + ci:all-cloud-drivers | RUN | ci:all-cloud-drivers label |"+ " | 6 | Cloud driver + its files changed | RUN | driver files changed |"+ " | 7 | Cloud driver, no relevant changes | SKIP | no relevant changes for cloud driver |"+ " | 8 | Driver deps affected by changes | RUN | driver module affected by shared code |"+ " | 9 | Self-hosted driver, not affected | SKIP | driver module not affected |"+ " |----------|-------------------------------------|--------|-------------------------------------------|"+ ""] (str/join "\n")))+ :options [[nil "--git-ref REF" "The git ref to compute changed modules against"]+ [nil "--is-master-or-release MASTER_OR_RELEASE"]+ [nil "--pr-labels PR_LABELS" "Comma delimited labels on the pr"]+ [nil "--skip SKIP"]+ [nil "--github-output-only" "Only output GITHUB_OUTPUT format (no human-readable output)"]]+ :requires [[clojure.string :as str]+ [mage.modules]]+ :task (task! (mage.modules/-main parsed))}+-install-merge-drivers{:doc "Installs custom git merge drivers for YAML migration files.":requires [[mage.util :as u]@@ -319,38 +352,38 @@[:string {:name "base" :description "ancestor's version (base)"}][:string {:name "ours" :description "current version (ours)"}][:string {:name "theirs" :description "other branch's version (theirs)"}]- [:int {:name "conflict-marker-size" :description "conflict marker size" :min 0}]+ [:int {:name "conflict-marker-size" :description "conflict marker size" :min 0}][:string {:name "file-path" :description "file path"}]]:requires [[mage.merge-yaml-migrations]]:task (task! (mage.merge-yaml-migrations/-main parsed))}-+-play- {:doc "Play sounds from mage.sound"- :examples [["./bin/mage -play success" "play success sound"] ["./bin/mage -play s" "play success sound"]- ["./bin/mage -play error" "play error sound"] ["./bin/mage -play e" "play error sound"]- ["./bin/mage -play ping" "play ping sound"] ["./bin/mage -play p" "play ping sound"]- ["./bin/mage -play warning" "play warning sound"] ["./bin/mage -play w" "play warning sound"]]- :requires [[mage.sound :as sound]]+ {:doc "Play sounds from mage.sound"+ :examples [["./bin/mage -play success" "play success sound"] ["./bin/mage -play s" "play success sound"]+ ["./bin/mage -play error" "play error sound"] ["./bin/mage -play e" "play error sound"]+ ["./bin/mage -play ping" "play ping sound"] ["./bin/mage -play p" "play ping sound"]+ ["./bin/mage -play warning" "play warning sound"] ["./bin/mage -play w" "play warning sound"]]+ :requires [[mage.sound :as sound]]:arg-schema [:tuple [:enum "s" "success" "e" "error" "p" "ping" "w" "warning"]]- :task (task! (case (first (:arguments parsed))- ("s" "success") (sound/success)- ("e" "error") (sound/error)- ("p" "ping") (sound/ping)- ("w" "warning") (sound/warning)))}-+ :task (task! (case (first (:arguments parsed))+ ("s" "success") (sound/success)+ ("e" "error") (sound/error)+ ("p" "ping") (sound/ping)+ ("w" "warning") (sound/warning)))}+-token-scan- {:doc "Security token scanner for detecting leaked API keys and secrets"- :examples [["./bin/mage -token-scan deps.edn bb.edn" "Scan specific files"]- ["./bin/mage -token-scan -v deps.edn" "Scan with verbose output"]- ["./bin/mage -token-scan -a" "Scan all files in the project"]]- :requires [[mage.token-scan :as token-scan]]- :options [["-a" "--all-files" "Scan all files in the project"]- ["-v" "--verbose" "Verbose output"]]+ {:doc "Security token scanner for detecting leaked API keys and secrets"+ :examples [["./bin/mage -token-scan deps.edn bb.edn" "Scan specific files"]+ ["./bin/mage -token-scan -v deps.edn" "Scan with verbose output"]+ ["./bin/mage -token-scan -a" "Scan all files in the project"]]+ :requires [[mage.token-scan :as token-scan]]+ :options [["-a" "--all-files" "Scan all files in the project"]+ ["-v" "--verbose" "Verbose output"]]:arg-schema [:sequential [:string {:description "Files to scan"}]]- :task (task! (token-scan/run-scan parsed))}+ :task (task! (token-scan/run-scan parsed))}-repl- {:doc "Eval a string of clojure code in the backend dev server"+ {:doc "Eval a string of clojure code in the backend dev server":requires [[babashka.process :as process][mage.color :as c][mage.be-dev :refer [nrepl-eval]]]@@ -362,16 +395,16 @@"Evaluate a keyword inside a namespace"]]:options [["-n" "--namespace NAMESPACE" "Namespace to use for the task"]["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]- :task (let [parsed (cli/parse! (current-task))- code (first (:arguments parsed))- port (or (:port (:options parsed))- (slurp ".nrepl-port")- (throw (ex-info (c/red "No port specified, and no .nrepl-port file found.") {:babashka/exit 1})))- nns (or (:namespace (:options parsed)) "user")]- (nrepl-eval nns code port))}+ :task (let [parsed (cli/parse! (current-task))+ code (first (:arguments parsed))+ port (or (:port (:options parsed))+ (slurp ".nrepl-port")+ (throw (ex-info (c/red "No port specified, and no .nrepl-port file found.") {:babashka/exit 1})))+ nns (or (:namespace (:options parsed)) "user")]+ (nrepl-eval nns code port))}-check-readable- {:doc "Check that code is readable parens a file"+ {:doc "Check that code is readable parens a file":requires [[mage.readability-check :as readability]]:arg-schema [:or[:tuple [:string {:name "file"@@ -385,60 +418,59 @@"Check file for readability"]["./bin/mage -check-readable dev/src/dev.clj 300""Check top level form containing line 300 for readability"]]- :task (let [{:keys [arguments]} (cli/parse! (current-task))]- (apply readability/check arguments))}+ :task (let [{:keys [arguments]} (cli/parse! (current-task))]+ (apply readability/check arguments))};; Uncomment this if you want to play with the example calculator task:- #_#_- -example-calculator- {;; `:doc` is a short docstring for the task, will be listed in `./bin/mage -h` and other help info.+ #_#_-example-calculator+ {;; `:doc` is a short docstring for the task, will be listed in `./bin/mage -h` and other help info.;; Keep these short, they need to fit on one line in the output from `./bin/mage -h`.- :doc "The example calculator's task docstring"+ :doc "The example calculator's task docstring";; examples get printed out when you pass -h or --help.- :examples [["./bin/mage -example-calculator 1 + 99" "evaluates to 100"]+ :examples [["./bin/mage -example-calculator 1 + 99" "evaluates to 100"]["./bin/mage -example-calculator 100 - 99" "evaluates to 1"]];; The task is the actual code that runs when you run the task.- :task (task!- (println "Parsed input:") (u/pp parsed)- (let [{:keys [arguments data]} parsed- [a op b] arguments]- (println a (name op) b "=" (c/blue (({:+ + :- -} op) a b)))))+ :task (task!+ (println "Parsed input:") (u/pp parsed)+ (let [{:keys [arguments data]} parsed+ [a op b] arguments]+ (println a (name op) b "=" (c/blue (({:+ + :- -} op) a b)))));; (optional) `:require` lazily libraries for just your task:- :requires [[mage.color :as c]+ :requires [[mage.color :as c][mage.util :as u]];; (optional) `:options` are passed to [[clojure.tools.cli/parse-opts]].;; See: https://clojure.github.io/tools.cli/index.html#clojure.tools.cli/parse-opts- :options [["-n" "--negate" "Negate the result of the calculation"]- ["-a" "--also-add N" "Add N to the result of the calculation"]]+ :options [["-n" "--negate" "Negate the result of the calculation"]+ ["-a" "--also-add N" "Add N to the result of the calculation"]];; (optional) `:arg-schema` is a malli schema for the arguments passed to the task, after the options- :arg-schema [:tuple [:int {:name "number-a"}] [:enum {:name "operation"} :+ :-] [:int {:name "number-b"}]]+ :arg-schema [:tuple [:int {:name "number-a"}] [:enum {:name "operation"} :+ :-] [:int {:name "number-b"}]];; (optional) `:usage-fn` is a function called with the current-task map.;; Returns a string containing extra-detailed usage information.- :usage-fn (fn [{:keys [doc] :as _task}]+ :usage-fn (fn [{:keys [doc] :as _task}](str "Optional extra usage information, for explicit callouts.\n""Reversed docstring: '" (c/green (apply str (reverse doc))) "'"));; (optional) Any other keys are completely allowed. Put things to lookup or that you want to be easy to change;; here. These will be returned from `(current-task)`. See: start-db for an example of using `:ports` to define a;; top-level map of ports. If something should be easily accessible in the task, put it here.- :data {:a [:b :c] :b [:d]}}+ :data {:a [:b :c] :b [:d]}}- -test {:doc "run all mage tests"+ -test {:doc "run all mage tests":requires [[clojure.string :as str][clojure.test][mage.core-test] ;; loads the actual tests[mage.util :as u]]:arg-schema [:sequential [:string {:name "test-matching-pattern"}]]- :task (let [{:keys [arguments]} (task! parsed)- pattern (if (seq arguments)- (str "^mage\\..*(" (str/join "|" arguments) ").*")- "^mage\\..*")- _ (u/debug ["test regex matching pattern" pattern])- results (clojure.test/run-all-tests (re-pattern pattern))]- (if (zero? (+ (:fail results) (:error results)))- (println (c/green "All mage tests passed!"))- (do- (println (c/red "Some mage tests failed."))- (u/exit 1))))}+ :task (let [{:keys [arguments]} (task! parsed)+ pattern (if (seq arguments)+ (str "^mage\\..*(" (str/join "|" arguments) ").*")+ "^mage\\..*")+ _ (u/debug ["test regex matching pattern" pattern])+ results (clojure.test/run-all-tests (re-pattern pattern))]+ (if (zero? (+ (:fail results) (:error results)))+ (println (c/green "All mage tests passed!"))+ (do+ (println (c/red "Some mage tests failed."))+ (u/exit 1))))};;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Personal Tasks:@@ -447,14 +479,14 @@;; `./bin/mage` listing.-escherize-pr-summary- {:doc "Print a day's PR summary for the metabase/metabase repo"+ {:doc "Print a day's PR summary for the metabase/metabase repo":requires [[mage.escherize.pr-summary :as eps]]:examples [["./bin/mage -escherize-pr-summary | pbcopy" "Copy your daily"]]- :task (task! (eps/summarize-prs parsed))}+ :task (task! (eps/summarize-prs parsed))}-escherize-fix-whitespace{:doc "Removes all lines like '^<space>+$' from files in glob pattern.":requires [[mage.escherize.fix-whitespace :as fix-whitespace]]:examples [["mage -escherize-fix-whitespace '**/*.clj{,c,s}'" "Removes all lines like ' ' from all clj files"]]:arg-schema [:sequential [:string {:name "glob-pattern" :description "Glob pattern to match files"}]]- :task (task! (fix-whitespace/run-clean parsed))}}}+ :task (task! (fix-whitespace/run-clean parsed))}}}diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex b678b5c4f39..32e11d0d620 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -1,5 +1,7 @@(ns mage.modules(:require+ ^:clj-kondo/ignore+ [cheshire.core :as json][clojure.edn :as edn][clojure.set :as set][clojure.string :as str]@@ -8,6 +10,8 @@(set! *warn-on-reflection* true)+(def ^:dynamic ^:private *github-output-only?* false)+;;; TODO (Cam 2025-11-07) changes to test files should only cause us to run tests for that module as well, not;;; everything that depends on that module directly or indirectly in `src`(defn- file->module [filename]@@ -23,7 +27,7 @@updated-files))(defn- updated-modules [git-ref]- (let [git-ref (or git-ref "master")+ (let [git-ref (or git-ref "master")updated-files (u/updated-files git-ref)](updated-files->updated-modules updated-files)))@@ -31,7 +35,7 @@[module](case (namespace module)"enterprise" (str "enterprise/backend/test/metabase_enterprise/" (str/replace (name module) #"-" "_"))- nil (str "test/metabase/" (str/replace (name module) #"-" "_"))))+ nil (str "test/metabase/" (str/replace (name module) #"-" "_"))))(defn- dependencies"Read out the Kondo config for the modules linter; return a map of module => set of modules it directly depends on."@@ -65,8 +69,8 @@(indirect-dependents deps module (sorted-set)))([deps module acc](let [module-deps (direct-dependents deps module)- new-deps (set/difference module-deps acc)- acc (into acc new-deps)]+ new-deps (set/difference module-deps acc)+ acc (into acc new-deps)](reduce(fn [acc new-dep](indirect-dependents deps new-dep acc))@@ -90,11 +94,7 @@(comment(unaffected-modules (dependencies) '#{enterprise/billing}))-(defn- skip-driver-tests? [deps modules]- (let [unaffected (unaffected-modules deps modules)]- (contains? unaffected 'driver)))--(defn- print-updated-and-unaffected-modules [deps updated]+(defn- print-updated-and-unaffected-modules [deps updated driver-deps-affected?](let [unaffected (unaffected-modules deps updated)](println "These modules have changed:" (pr-str updated))(println)@@ -105,16 +105,17 @@(println "(By unaffected, this means these modules do not have a direct or indirect dependency on the modules that have been changed.)")(println)(println)- (println (if (skip-driver-tests? deps updated)- (c/green "Driver tests " (c/bold "CAN be skipped") "")- (c/red "Driver tests " (c/bold "MUST be run") ".")))))+ (println (if driver-deps-affected?+ (c/red "Driver tests " (c/bold "MUST be run") ".")+ (c/green "Driver tests " (c/bold "CAN be skipped") "")))))(defn cli-print-affected-modules[[git-ref, :as _command-line-args]]- (let [deps (dependencies)- updated (updated-modules git-ref)- affected (affected-modules deps updated)]- (print-updated-and-unaffected-modules deps updated)+ (let [deps (dependencies)+ updated (updated-modules git-ref)+ affected (affected-modules deps updated)+ driver-deps-affected? (not (contains? (unaffected-modules deps updated) 'driver))]+ (print-updated-and-unaffected-modules deps updated driver-deps-affected?)(println)(println)(println "You can run tests for these modules and all downstream modules as follows:")@@ -131,10 +132,10 @@(some (fn [filename](when (or (str/includes? filename "deps.edn")(str/includes? filename "modules/drivers/"))- (printf "Running driver tests because %s was changed\n" (pr-str filename))- (flush)+ (when-not *github-output-only?*+ (println (str "Running driver tests because " (pr-str filename) " was changed")))filename))- (u/updated-clojure-files (or git-ref "master"))))+ (u/updated-files (or git-ref "master"))))(defn- remove-non-driver-test-namespaces [files](into []@@ -144,11 +145,19 @@(not (some #(str/includes? filename %)["query_processor""driver"])))- (printf "Ignorning changes in test namespace %s\n" (pr-str filename))- (flush)+ (when-not *github-output-only?*+ (println (str "Ignorning changes in test namespace " (pr-str filename))))filename)))files))+(defn- driver-deps-affected?+ "Returns true if the driver module is affected by the changed modules."+ ([modules]+ (driver-deps-affected? (dependencies) modules))+ ([deps modules]+ (let [unaffected (unaffected-modules deps modules)]+ (not (contains? unaffected 'driver)))))+(defn cli-can-skip-driver-tests"Exits with zero status code if we can skip driver tests, nonzero if we cannot.@@ -156,14 +165,274 @@./bin/mage can-skip-driver-tests [git-ref]"[[git-ref, :as _arguments]]- (let [deps (dependencies)- git-ref (or git-ref "master")+ (let [deps (dependencies)+ git-ref (or git-ref "master")updated-files (remove-non-driver-test-namespaces (u/updated-files git-ref))- updated (updated-files->updated-modules updated-files)- skip-tests? (skip-driver-tests? deps updated)]+ updated (updated-files->updated-modules updated-files)+ drivers-affected? (driver-deps-affected? deps updated)];; Not strictly necessary, but people looking at CI will appreciate having this extra info.- (print-updated-and-unaffected-modules deps updated)+ (print-updated-and-unaffected-modules deps updated drivers-affected?)(u/exit (cond- (not skip-tests?) 1(changes-important-file-for-drivers? git-ref) 1- :else 0))))+ drivers-affected? 1+ :else 0))))++;;;; =============================================================================+;;;; Driver test decisions - consolidated logic for which drivers to run+;;;; =============================================================================++(def cloud-drivers+ "Drivers that run on cloud infrastructure and require secrets. These are more expensive to run,+ since they need round trip times, so we skip them on PRs unless specifically needed."+ #{:athena :bigquery :databricks :redshift :snowflake})++(def ^:private all-drivers+ "All driver test jobs in drivers.yml, in order."+ [:h2+ :athena+ :bigquery+ :clickhouse+ :databricks+ :druid+ :druid-jdbc+ :mongo+ :mongo-ssl+ :mongo-sharded-cluster+ :mysql-mariadb+ :oracle+ :postgres+ :presto-jdbc+ :redshift+ :snowflake+ :sparksql+ :sqlite+ :sqlserver+ :vertica])++(def ^:private driver-directory->drivers+ "Maps driver directory names to the driver keyword(s) they correspond to.+ Most directories map to a single driver, but some (like mongo) map to multiple test jobs."+ {"athena" [:athena]+ "bigquery-cloud-sdk" [:bigquery]+ "clickhouse" [:clickhouse]+ "databricks" [:databricks]+ "druid" [:druid]+ "druid-jdbc" [:druid-jdbc]+ "mongo" [:mongo :mongo-ssl :mongo-sharded-cluster]+ "oracle" [:oracle]+ "presto-jdbc" [:presto-jdbc]+ "redshift" [:redshift]+ "snowflake" [:snowflake]+ "sparksql" [:sparksql]+ "sqlite" [:sqlite]+ "sqlserver" [:sqlserver]+ ;; starburst tests are currently disabled in drivers.yml+ ;; "starburst" [:starburst]+ "vertica" [:vertica]})++(defn- drivers-with-file-changes+ "Returns a set of driver keywords that have file changes in modules/drivers/<driver>/."+ [git-ref]+ (let [updated-files (u/updated-files (or git-ref "master"))]+ (into #{}+ (mapcat (fn [filename]+ (when-let [[_ dir-name] (re-matches #"modules/drivers/([^/]+)/.*" filename)]+ (get driver-directory->drivers dir-name))))+ updated-files)))++;;; driver quarantine++(def ^:private ci-test-config-url+ "https://raw.githubusercontent.com/metabase/ci-test-config/refs/heads/master/ci-test-config.json")++(defn- read-ci-test-config []+ (json/parse-string (slurp ci-test-config-url) keyword))++(defn- quarantined-drivers []+ (-> (read-ci-test-config)+ (get-in [:ignored :drivers] [])+ (->> (map keyword))+ (set)))++(defn- parse-bool+ "Parse a string boolean from CLI args. Returns true for 'true', false otherwise."+ [s]+ (= (str/lower-case (str s)) "true"))++(defn- parse-labels+ "Parse comma-separated labels string into a set of label strings."+ [labels-str]+ (if (str/blank? labels-str)+ #{}+ (into #{} (map str/trim) (str/split labels-str #","))))++(defn break-quarantine-label [driver]+ (str "break-quarantine-" (name driver)))++(defn- driver-decision+ "Determine if a driver should run and why.++ Returns a map with :should-run (boolean) and :reason (string).++ For the decision priority order, see: mage -driver-decisions -h++ ## What counts as 'driver deps affected'?++ The driver module is considered affected when:+ - Files in modules/drivers/* are changed (triggers all drivers)+ - deps.edn is changed (triggers all drivers)+ - Clojure modules that the 'driver' module depends on are changed"+ [driver+ {:keys [is-master-or-release pr-labels skip particular-driver-changed? verbose?]}+ driver-deps-affected?+ quarantined-drivers]+ (cond+ ;; Priority 1: Global skip (no backend changes)+ skip+ {:should-run false+ :reason "workflow skip (no backend changes)"}++ ;; Priority 2: H2 always runs when backend tests run+ (= driver :h2)+ {:should-run true+ :reason "H2 always runs"}++ ;; Priority 3: Quarantined drivers (respected even on master/release)+ (contains? quarantined-drivers driver)+ (do+ (when verbose?+ (println "Driver" (name driver) "is quarantined; checking for '" (break-quarantine-label driver) "' label...."))+ (if (contains? pr-labels (break-quarantine-label driver))+ {:should-run true+ :reason "driver is quarantined, but anti-quarantine label present; running anyway"}+ {:should-run false+ :reason "driver is quarantined"}))++ ;; Priority 4: Master/release branch - all (non-quarantined) drivers run+ is-master-or-release+ {:should-run true+ :reason "master/release branch"}++ ;; Priority 5: Cloud driver + ci:all-cloud-drivers label+ (and (contains? cloud-drivers driver)+ (contains? pr-labels "ci:all-cloud-drivers"))+ {:should-run true+ :reason "ci:all-cloud-drivers label"}++ ;; Priority 6: Cloud driver + its files changed+ (and (contains? cloud-drivers driver)+ (contains? particular-driver-changed? driver))+ {:should-run true+ :reason (str "driver files changed (modules/drivers/" (name driver) "/**)")}++ ;; Priority 7: Cloud driver, no relevant changes+ (contains? cloud-drivers driver)+ {:should-run false+ :reason "no relevant changes for cloud driver"}++ ;; Priority 8: Driver deps affected by shared code changes+ driver-deps-affected?+ {:should-run true+ :reason "driver module affected by shared code changes"}++ ;; Priority 9: Self-hosted driver, not affected+ :else+ {:should-run false+ :reason "driver module not affected"}))++(defn- cli-driver-decisions+ "Determine which driver tests should run based on PR context.++ Outputs decisions in GITHUB_OUTPUT format (key=value lines) plus human-readable logs.+ Use --github-output-only to output only the key=value lines for CI.++ Usage:+ ./bin/mage -driver-decisions \\+ --git-ref=master \\+ --is-master-or-release=false \\+ --pr-labels=ci:all-cloud-drivers,other-label \\+ --skip=false"+ [{:keys [options] :as _parsed}]+ (let [github-output-only? (some? (:github-output-only options))+ git-ref (get options :git-ref "master")+ ;; Detect file changes for ALL drivers via git diff+ particular-driver-changed? (drivers-with-file-changes git-ref)+ ctx {:git-ref git-ref+ :is-master-or-release (parse-bool (:is-master-or-release options))+ :pr-labels (parse-labels (:pr-labels options))+ :skip (parse-bool (:skip options))+ :particular-driver-changed? particular-driver-changed?+ :verbose? (not github-output-only?)}+ quarantined (quarantined-drivers)+ updated-files (remove-non-driver-test-namespaces+ (u/updated-files git-ref))+ updated (updated-files->updated-modules updated-files)+ driver-affected? (driver-deps-affected? updated)+ important-file-changed? (changes-important-file-for-drivers? git-ref)+ ;; For module dependency check, combine both conditions+ effective-driver-affected? (or driver-affected? important-file-changed?)+ decisions (mapv (fn [driver]+ (assoc (driver-decision driver ctx effective-driver-affected? quarantined)+ :driver driver))+ all-drivers)+ ;; Check for quarantined drivers with file changes but no break-quarantine label+ quarantined-with-changes (into #{}+ (filter (fn [driver]+ (and (contains? quarantined driver)+ (contains? particular-driver-changed? driver)+ (not (contains? (:pr-labels ctx)+ (break-quarantine-label driver))))))+ all-drivers)]++ (if github-output-only?+ ;; In github-output-only mode, print just the key=value lines (no colors)+ (do+ (doseq [{:keys [driver should-run]} decisions]+ (println (str (name driver) "-should-run=" should-run)))+ (doseq [driver quarantined-with-changes]+ (println (str (name driver) "-quarantine-conflict=true"))))++ (do+ ;; Print module analysis summary+ (println "")+ (println "=== Module Analysis ===")+ (println "Changed modules:" (pr-str updated))+ (println "Driver module affected:" driver-affected?)+ (println "Important file changed:" (boolean important-file-changed?))+ (println "Drivers with file changes:" (pr-str particular-driver-changed?))+ (println "")++ ;; Print human-readable decision summary+ (println "=== Driver Decisions ===")+ (doseq [{:keys [driver should-run reason]} decisions]+ (println (format "%-25s %s - %s"+ (name driver)+ (if should-run (c/green "RUN ") (c/yellow "SKIP"))+ reason)))+ (println "")++ ;; Print GITHUB_OUTPUT preview with colors+ (let [{drivers-to-run true drivers-to-skip false} (group-by :should-run decisions)]+ (println (c/green (str "\n=== Drivers to Run (" (count drivers-to-run) ") ===")))+ (doseq [{:keys [driver]} drivers-to-run]+ (println (str (name driver) "-should-run=true")))+ (println (c/yellow (str "\n=== Drivers to Skip (" (count drivers-to-skip) ") ===")))+ (doseq [{:keys [driver]} drivers-to-skip]+ (println (str (name driver) "-should-run=false"))))++ ;; Output quarantine conflict warnings with colors+ (when (seq quarantined-with-changes)+ (println "")+ (println (c/red "⚠️ WARNING: Quarantined driver(s) have file changes but tests will NOT run!"))+ (println (c/red "=== Quarantine Conflicts ==="))+ (doseq [driver quarantined-with-changes]+ (println (c/red (str " • " (name driver) " - add label '" (break-quarantine-label driver) "' to run tests")))+ (println (str (name driver) "-quarantine-conflict=true"))))))++ (u/exit 0)))++(defn -main+ "See [[cli-driver-decisions]]."+ [{:keys [options] :as parsed}]+ (binding [*github-output-only?* (:github-output-only options)]+ (cli-driver-decisions parsed)))diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex 969dd233415..43413a55262 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -8,6 +8,7 @@[mage.alias-test][mage.doctor-test][mage.merge-yaml-migrations-test :as merge-yaml-migrations-test]+ [mage.modules-test][mage.token-scan-test][mage.util :as u][mage.util-test]))@@ -16,6 +17,7 @@;; Load test namespaces to ensure code coveragemage.doctor-test/keep-memage.util-test/keep-me+ mage.modules-test/keep-memerge-yaml-migrations-test/keep-metoken-scan-test/keep-me)diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.cljnew file mode 100644index 00000000000..f964fac7c0c--- /dev/null+++ b/mage/test/mage/modules_test.clj@@ -0,0 +1,191 @@+(ns mage.modules-test+ "Tests for driver decision logic.+ Run `mage -driver-decisions -h` to see the priority order."+ (:require+ [clojure.test :refer [deftest is testing]]+ [mage.modules]))++;; Access private functions for testing via @#'ns/var (works in babashka)+(defn driver-decision [& args]+ (apply @(resolve 'mage.modules/driver-decision) args))++(def cloud-drivers @(resolve 'mage.modules/cloud-drivers))++;; Referenced by core_test.clj to ensure namespace is loaded+(def keep-me :loaded)++(defn- make-ctx+ "Create a context map with sensible defaults, overridable by opts."+ [opts]+ (merge {:is-master-or-release false+ :pr-labels #{}+ :skip false+ :particular-driver-changed? #{}+ :verbose? false}+ opts))++;;; =============================================================================+;;; Priority 3: Quarantine (respected even on master/release)+;;; =============================================================================++(deftest quarantined-driver-skips+ (testing "Quarantined driver is skipped"+ (let [result (driver-decision :postgres+ (make-ctx {})+ false ; driver-module-affected?+ #{:postgres})] ; quarantined+ (is (false? (:should-run result)))+ (is (= "driver is quarantined" (:reason result))))))++(deftest quarantined-driver-with-break-label-runs+ (testing "Quarantined driver runs when break-quarantine label is present"+ (let [result (driver-decision :postgres+ (make-ctx {:pr-labels #{"break-quarantine-postgres"}})+ false+ #{:postgres})]+ (is (true? (:should-run result)))+ (is (re-find #"anti-quarantine label" (:reason result))))))++(deftest quarantine-respected-on-master+ (testing "Quarantined driver is skipped even on master/release"+ (let [result (driver-decision :postgres+ (make-ctx {:is-master-or-release true})+ true ; driver-deps-affected?+ #{:postgres})] ; quarantined+ (is (false? (:should-run result)))+ (is (= "driver is quarantined" (:reason result))))))++;;; =============================================================================+;;; Priority 1: Global skip+;;; =============================================================================++(deftest global-skip-skips-all-drivers+ (testing "Global skip (no backend changes) skips all drivers"+ (doseq [driver [:postgres :mysql :mongo :athena :bigquery]]+ (let [result (driver-decision driver+ (make-ctx {:skip true})+ true ; even if affected+ #{})]+ (is (false? (:should-run result))+ (str driver " should be skipped"))+ (is (= "workflow skip (no backend changes)" (:reason result)))))))++(deftest global-skip-beats-quarantine+ (testing "Global skip takes priority over quarantine"+ (let [result (driver-decision :postgres+ (make-ctx {:skip true})+ false+ #{:postgres})]+ (is (false? (:should-run result)))+ (is (= "workflow skip (no backend changes)" (:reason result))))))++;;; =============================================================================+;;; Priority 2: H2 always runs+;;; =============================================================================++(deftest h2-always-runs+ (testing "H2 always runs when not globally skipped"+ (let [result (driver-decision :h2+ (make-ctx {:is-master-or-release false})+ false ; driver module not affected+ #{})]+ (is (true? (:should-run result)))+ (is (= "H2 always runs" (:reason result))))))++(deftest h2-skipped-on-global-skip+ (testing "H2 is skipped when global skip is true"+ (let [result (driver-decision :h2+ (make-ctx {:skip true})+ false+ #{})]+ (is (false? (:should-run result)))+ (is (= "workflow skip (no backend changes)" (:reason result))))))++;;; =============================================================================+;;; Priority 4: Master/release branch+;;; =============================================================================++(deftest master-branch-runs-all-drivers+ (testing "All drivers run on master/release branch"+ (doseq [driver [:postgres :mysql :mongo :athena :bigquery :snowflake]]+ (let [result (driver-decision driver+ (make-ctx {:is-master-or-release true})+ false ; even if not affected+ #{})]+ (is (true? (:should-run result))+ (str driver " should run on master"))+ (is (= "master/release branch" (:reason result)))))))++;;; =============================================================================+;;; Priority 8: Driver deps affected (self-hosted only)+;;; =============================================================================++(deftest driver-deps-affected-runs-self-hosted-drivers+ (testing "Self-hosted drivers run when driver module is affected"+ (doseq [driver [:postgres :mysql :mongo :oracle :sqlserver]]+ (let [result (driver-decision driver+ (make-ctx {})+ true ; driver-deps-affected+ #{})]+ (is (true? (:should-run result))+ (str driver " should run when driver module affected"))+ (is (= "driver module affected by shared code changes" (:reason result)))))))++;;; =============================================================================+;;; Priority 5-7: Cloud driver special rules+;;; =============================================================================++(deftest cloud-driver-with-label-runs+ (testing "Cloud driver runs with ci:all-cloud-drivers label"+ (doseq [driver [:athena :bigquery :databricks :redshift :snowflake]]+ (let [result (driver-decision driver+ (make-ctx {:pr-labels #{"ci:all-cloud-drivers"}})+ false ; not affected+ #{})]+ (is (true? (:should-run result))+ (str driver " should run with label"))+ (is (= "ci:all-cloud-drivers label" (:reason result)))))))++(deftest cloud-driver-with-file-changes-runs+ (testing "Cloud driver runs when its files changed"+ (let [result (driver-decision :athena+ (make-ctx {:particular-driver-changed? #{:athena}})+ false+ #{})]+ (is (true? (:should-run result)))+ (is (re-find #"driver files changed" (:reason result))))))++(deftest cloud-driver-without-changes-skips+ (testing "Cloud driver skips when no relevant changes"+ (doseq [driver [:athena :bigquery :databricks :redshift :snowflake]]+ (let [result (driver-decision driver+ (make-ctx {})+ false ; not affected+ #{})]+ (is (false? (:should-run result))+ (str driver " should skip without changes"))+ (is (= "no relevant changes for cloud driver" (:reason result)))))))++;;; =============================================================================+;;; Priority 9: Self-hosted drivers+;;; =============================================================================++(deftest self-hosted-driver-not-affected-skips+ (testing "Self-hosted driver skips when driver module not affected"+ (doseq [driver [:postgres :mysql :mongo :oracle :sqlserver]]+ (let [result (driver-decision driver+ (make-ctx {})+ false ; not affected+ #{})]+ (is (false? (:should-run result))+ (str driver " should skip when not affected"))+ (is (= "driver module not affected" (:reason result)))))))++;;; =============================================================================+;;; Integration: Verify cloud vs self-hosted classification+;;; =============================================================================++(deftest cloud-drivers-are-correct+ (testing "Cloud drivers set matches expected"+ (is (= #{:athena :bigquery :databricks :redshift :snowflake}+ cloud-drivers))))