The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Run a configured-skip driver when its own files change (#79793)
Nemanja Glumac5147989defea
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex e6e216ef04f..20b2af2f570 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -45,26 +45,6 @@ jobs:sqlite-should-run: ${{ steps.driver-decisions.outputs.sqlite-should-run }}sqlserver-should-run: ${{ steps.driver-decisions.outputs.sqlserver-should-run }}vertica-should-run: ${{ steps.driver-decisions.outputs.vertica-should-run }}- # Per-driver quarantine conflict flags (set when driver is quarantined but has file changes)- athena-quarantine-conflict: ${{ steps.driver-decisions.outputs.athena-quarantine-conflict }}- bigquery-quarantine-conflict: ${{ steps.driver-decisions.outputs.bigquery-quarantine-conflict }}- clickhouse-quarantine-conflict: ${{ steps.driver-decisions.outputs.clickhouse-quarantine-conflict }}- databricks-quarantine-conflict: ${{ steps.driver-decisions.outputs.databricks-quarantine-conflict }}- druid-quarantine-conflict: ${{ steps.driver-decisions.outputs.druid-quarantine-conflict }}- druid-jdbc-quarantine-conflict: ${{ steps.driver-decisions.outputs.druid-jdbc-quarantine-conflict }}- mongo-quarantine-conflict: ${{ steps.driver-decisions.outputs.mongo-quarantine-conflict }}- mongo-ssl-quarantine-conflict: ${{ steps.driver-decisions.outputs.mongo-ssl-quarantine-conflict }}- mongo-sharded-cluster-quarantine-conflict: ${{ steps.driver-decisions.outputs.mongo-sharded-cluster-quarantine-conflict }}- mysql-mariadb-quarantine-conflict: ${{ steps.driver-decisions.outputs.mysql-mariadb-quarantine-conflict }}- oracle-quarantine-conflict: ${{ steps.driver-decisions.outputs.oracle-quarantine-conflict }}- postgres-quarantine-conflict: ${{ steps.driver-decisions.outputs.postgres-quarantine-conflict }}- presto-jdbc-quarantine-conflict: ${{ steps.driver-decisions.outputs.presto-jdbc-quarantine-conflict }}- redshift-quarantine-conflict: ${{ steps.driver-decisions.outputs.redshift-quarantine-conflict }}- snowflake-quarantine-conflict: ${{ steps.driver-decisions.outputs.snowflake-quarantine-conflict }}- sparksql-quarantine-conflict: ${{ steps.driver-decisions.outputs.sparksql-quarantine-conflict }}- sqlite-quarantine-conflict: ${{ steps.driver-decisions.outputs.sqlite-quarantine-conflict }}- sqlserver-quarantine-conflict: ${{ steps.driver-decisions.outputs.sqlserver-quarantine-conflict }}- vertica-quarantine-conflict: ${{ steps.driver-decisions.outputs.vertica-quarantine-conflict }}steps:- name: Checkout repositoryuses: actions/checkout@v6@@ -101,7 +81,7 @@ jobs:be-tests-athena-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.athena-should-run == 'true' || needs.determine-driver-skips.outputs.athena-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.athena-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40env:@@ -118,16 +98,6 @@ jobs:MB_ATHENA_TEST_WITHOUT_GET_TABLE_METADATA_SECRET_KEY: ${{ secrets.MB_ATHENA_TEST_WITHOUT_GET_TABLE_METADATA_SECRET_KEY }}name: Athenasteps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.athena-quarantine-conflict == 'true' }}- run: |- echo "::error::Athena driver is quarantined but you changed files in modules/drivers/athena/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Athena tests, add the label: ci:run-athena"- echo ""- echo "If you merge without testing, you risk breaking the Athena driver!"- exit 1- uses: actions/checkout@v6- name: Test Athena driverid: test-driver@@ -141,7 +111,7 @@ jobs:be-tests-bigquery-cloud-sdk-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' || needs.determine-driver-skips.outputs.bigquery-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 70strategy:@@ -178,16 +148,6 @@ jobs:MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}name: BigQuery ${{ matrix.job.name }}steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.bigquery-quarantine-conflict == 'true' }}- run: |- echo "::error::BigQuery driver is quarantined but you changed files in modules/drivers/bigquery-cloud-sdk/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run BigQuery tests, add the label: ci:run-bigquery"- echo ""- echo "If you merge without testing, you risk breaking the BigQuery driver!"- exit 1- uses: actions/checkout@v6- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -211,7 +171,7 @@ jobs:be-tests-clickhouse-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.clickhouse-should-run == 'true' || needs.determine-driver-skips.outputs.clickhouse-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.clickhouse-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40strategy:@@ -235,16 +195,6 @@ jobs:MB_CLICKHOUSE_TEST_NGINX_PORT: 8127name: Clickhouse ${{ matrix.job.name }}steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.clickhouse-quarantine-conflict == 'true' }}- run: |- echo "::error::ClickHouse driver is quarantined but you changed files in modules/drivers/clickhouse/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run ClickHouse tests, add the label: ci:run-clickhouse"- echo ""- echo "If you merge without testing, you risk breaking the ClickHouse driver!"- exit 1- uses: actions/checkout@v6- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -284,7 +234,7 @@ jobs:be-tests-druid-jdbc-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.druid-jdbc-should-run == 'true' || needs.determine-driver-skips.outputs.druid-jdbc-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.druid-jdbc-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40env:@@ -299,16 +249,6 @@ jobs:CLUSTER_SIZE: nano-quickstartname: Druid (JDBC)steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.druid-jdbc-quarantine-conflict == 'true' }}- run: |- echo "::error::Druid JDBC driver is quarantined but you changed files in modules/drivers/druid-jdbc/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Druid JDBC tests, add the label: ci:run-druid-jdbc"- echo ""- echo "If you merge without testing, you risk breaking the Druid JDBC driver!"- exit 1- uses: actions/checkout@v6- name: Test Druid JDBC driverid: test-driver@@ -322,7 +262,7 @@ jobs:be-tests-databricks-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.databricks-should-run == 'true' || needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.databricks-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 50strategy:@@ -362,16 +302,6 @@ jobs:MB_DATABRICKS_TEST_OAUTH_SECRET: ${{ secrets.MB_DATABRICKS_JDBC_TEST_OAUTH_SECRET }}name: Databrickssteps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}- run: |- echo "::error::Databricks driver is quarantined but you changed files in modules/drivers/databricks/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Databricks tests, add the label: ci:run-databricks"- echo ""- echo "If you merge without testing, you risk breaking the Databricks driver!"- exit 1- uses: actions/checkout@v6- name: Test Databricks driverid: test-driver@@ -384,7 +314,7 @@ jobs:be-tests-databricks-multi-catalog-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.databricks-should-run == 'true' || needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.databricks-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 50strategy:@@ -425,16 +355,6 @@ jobs:MB_DATABRICKS_TEST_OAUTH_SECRET: ${{ secrets.MB_DATABRICKS_JDBC_TEST_OAUTH_SECRET }}name: Databricks (Multi-Catalog)steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.databricks-quarantine-conflict == 'true' }}- run: |- echo "::error::Databricks driver is quarantined but you changed files in modules/drivers/databricks/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Databricks tests, add the label: ci:run-databricks"- echo ""- echo "If you merge without testing, you risk breaking the Databricks driver!"- exit 1- uses: actions/checkout@v6- name: Test Databricks driver with multi-catalogid: test-driver@@ -625,7 +545,7 @@ jobs:be-tests-mongo:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.mongo-should-run == 'true' || needs.determine-driver-skips.outputs.mongo-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.mongo-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40strategy:@@ -662,16 +582,6 @@ jobs:MONGO_INITDB_ROOT_PASSWORD: metasample123name: ${{ matrix.version.name }} ${{ matrix.job.name }}steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.mongo-quarantine-conflict == 'true' }}- run: |- echo "::error::MongoDB driver is quarantined but you changed files in modules/drivers/mongo/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run MongoDB tests, add the label: ci:run-mongo"- echo ""- echo "If you merge without testing, you risk breaking the MongoDB driver!"- exit 1- uses: actions/checkout@v6- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -695,7 +605,7 @@ jobs:be-tests-mongo-ssl:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.mongo-ssl-should-run == 'true' || needs.determine-driver-skips.outputs.mongo-ssl-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.mongo-ssl-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40strategy:@@ -713,16 +623,6 @@ jobs:MB_TEST_MONGO_REQUIRES_SSL: truename: "${{ matrix.version.name }} (SSL)"steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.mongo-ssl-quarantine-conflict == 'true' }}- run: |- echo "::error::MongoDB SSL driver is quarantined but you changed files in modules/drivers/mongo/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run MongoDB SSL tests, add the label: ci:run-mongo-ssl"- echo ""- echo "If you merge without testing, you risk breaking the MongoDB SSL driver!"- exit 1- uses: actions/checkout@v6- name: Spin up Mongo docker containerrun: >-@@ -761,7 +661,7 @@ jobs:be-tests-mongo-sharded-cluster-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.mongo-sharded-cluster-should-run == 'true' || needs.determine-driver-skips.outputs.mongo-sharded-cluster-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.mongo-sharded-cluster-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40env:@@ -774,16 +674,6 @@ jobs:- "27017:17017"name: MongoDB Sharded Clustersteps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.mongo-sharded-cluster-quarantine-conflict == 'true' }}- run: |- echo "::error::MongoDB Sharded Cluster driver is quarantined but you changed files in modules/drivers/mongo/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run MongoDB Sharded Cluster tests, add the label: ci:run-mongo-sharded-cluster"- echo ""- echo "If you merge without testing, you risk breaking the MongoDB Sharded Cluster driver!"- exit 1- uses: actions/checkout@v6- name: Test MongoDB driver (Sharded cluster)id: test-driver@@ -796,7 +686,7 @@ jobs:be-tests-oracle:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.oracle-should-run == 'true' || needs.determine-driver-skips.outputs.oracle-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.oracle-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40strategy:@@ -847,16 +737,6 @@ jobs:- "1521:${{ matrix.version.port }}"name: ${{ matrix.version.name }}steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.oracle-quarantine-conflict == 'true' }}- run: |- echo "::error::Oracle driver is quarantined but you changed files in modules/drivers/oracle/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Oracle tests, add the label: ci:run-oracle"- echo ""- echo "If you merge without testing, you risk breaking the Oracle driver!"- exit 1- uses: actions/checkout@v6- name: Test ${{ matrix.version.name }}id: test-driver@@ -971,7 +851,7 @@ jobs:be-tests-presto-jdbc-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.presto-jdbc-should-run == 'true' || needs.determine-driver-skips.outputs.presto-jdbc-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.presto-jdbc-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40env:@@ -994,16 +874,6 @@ jobs:env:JAVA_TOOL_OPTIONS: "-Xmx2g"steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.presto-jdbc-quarantine-conflict == 'true' }}- run: |- echo "::error::Presto JDBC driver is quarantined but you changed files in modules/drivers/presto-jdbc/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Presto JDBC tests, add the label: ci:run-presto-jdbc"- echo ""- echo "If you merge without testing, you risk breaking the Presto JDBC driver!"- exit 1- uses: actions/checkout@v6- uses: ./.github/actions/await-portwith:@@ -1042,7 +912,7 @@ jobs:be-tests-redshift-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.redshift-should-run == 'true' || needs.determine-driver-skips.outputs.redshift-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.redshift-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 50strategy:@@ -1087,16 +957,6 @@ jobs:MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }}name: ${{ matrix.job.name }}steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.redshift-quarantine-conflict == 'true' }}- run: |- echo "::error::Redshift driver is quarantined but you changed files in modules/drivers/redshift/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Redshift tests, add the label: ci:run-redshift"- echo ""- echo "If you merge without testing, you risk breaking the Redshift driver!"- exit 1- uses: actions/checkout@v6- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -1122,7 +982,7 @@ jobs:be-tests-snowflake-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.snowflake-should-run == 'true' || needs.determine-driver-skips.outputs.snowflake-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.snowflake-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 70strategy:@@ -1162,16 +1022,6 @@ jobs:MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DBname: Snowflake ${{ matrix.job.name }}steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.snowflake-quarantine-conflict == 'true' }}- run: |- echo "::error::Snowflake driver is quarantined but you changed files in modules/drivers/snowflake/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Snowflake tests, add the label: ci:run-snowflake"- echo ""- echo "If you merge without testing, you risk breaking the Snowflake driver!"- exit 1- uses: actions/checkout@v6- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -1195,7 +1045,7 @@ jobs:be-tests-sparksql-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.sparksql-should-run == 'true' || needs.determine-driver-skips.outputs.sparksql-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.sparksql-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40env:@@ -1208,16 +1058,6 @@ jobs:- "10000:10000"name: Spark SQLsteps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.sparksql-quarantine-conflict == 'true' }}- run: |- echo "::error::SparkSQL driver is quarantined but you changed files in modules/drivers/sparksql/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run SparkSQL tests, add the label: ci:run-sparksql"- echo ""- echo "If you merge without testing, you risk breaking the SparkSQL driver!"- exit 1- uses: actions/checkout@v6- name: Test Spark driverid: test-driver@@ -1231,7 +1071,7 @@ jobs:be-tests-sqlite-ee:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.sqlite-should-run == 'true' || needs.determine-driver-skips.outputs.sqlite-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.sqlite-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40env:@@ -1239,16 +1079,6 @@ jobs:DRIVERS: sqlitename: SQLitesteps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.sqlite-quarantine-conflict == 'true' }}- run: |- echo "::error::SQLite driver is quarantined but you changed files that affect it"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run SQLite tests, add the label: ci:run-sqlite"- echo ""- echo "If you merge without testing, you risk breaking the SQLite driver!"- exit 1- uses: actions/checkout@v6- name: Test SQLite driverid: test-driver@@ -1262,7 +1092,7 @@ jobs:be-tests-sqlserver:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.sqlserver-should-run == 'true' || needs.determine-driver-skips.outputs.sqlserver-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.sqlserver-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 30strategy:@@ -1301,16 +1131,6 @@ jobs:MSSQL_MEMORY_LIMIT_MB: 1024name: ${{ matrix.version.name }} ${{ matrix.job.name }}steps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.sqlserver-quarantine-conflict == 'true' }}- run: |- echo "::error::SQL Server driver is quarantined but you changed files in modules/drivers/sqlserver/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run SQL Server tests, add the label: ci:run-sqlserver"- echo ""- echo "If you merge without testing, you risk breaking the SQL Server driver!"- exit 1- uses: actions/checkout@v6- name: Setup python-runnerif: ${{ matrix.job.needs-python-runner == true }}@@ -1362,7 +1182,7 @@ jobs:login-to-ecr:needs: determine-driver-skips- if: ${{ needs.determine-driver-skips.outputs.vertica-should-run == 'true' || needs.determine-driver-skips.outputs.vertica-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.vertica-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 15permissions:@@ -1394,7 +1214,7 @@ jobs:be-tests-vertica-ee:needs: [determine-driver-skips, login-to-ecr]- if: ${{ needs.determine-driver-skips.outputs.vertica-should-run == 'true' || needs.determine-driver-skips.outputs.vertica-quarantine-conflict == 'true' }}+ if: ${{ needs.determine-driver-skips.outputs.vertica-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 40env:@@ -1410,16 +1230,6 @@ jobs:password: ${{ needs.login-to-ecr.outputs.docker_password }}name: Verticasteps:- - name: Fail on quarantine conflict- if: ${{ needs.determine-driver-skips.outputs.vertica-quarantine-conflict == 'true' }}- run: |- echo "::error::Vertica driver is quarantined but you changed files in modules/drivers/vertica/"- echo ""- echo "Your changes will NOT be tested because the driver is currently quarantined."- echo "To run Vertica tests, add the label: ci:run-vertica"- echo ""- echo "If you merge without testing, you risk breaking the Vertica driver!"- exit 1- uses: actions/checkout@v6- name: Make plugins directoryrun: mkdir pluginsdiff --git a/bb.edn b/bb.ednindex e0e9ef323af..fc0ca4e6bc7 100644--- a/bb.edn+++ b/bb.edn@@ -409,13 +409,10 @@["1" "Global skip (no backend changes)" "SKIP" "workflow skip (no backend changes)"]["2" "Driver is :h2 or :postgres" "RUN" "H2/Postgres always run"]["3" "ci:run-all-drivers or ci:run-DRIVER" "RUN" "ci:run-all-drivers or ci:run-DRIVER label"]- ["4" "Configured status = skip (PR branches)" "SKIP" "driver is quarantined"]- ["5" "On master or release branch" "RUN" "master/release branch (quarantine ignored)"]- ["6" "Cloud driver + ci:run-all-cloud-drivers" "RUN" "ci:run-all-cloud-drivers label"]- ["7"- "Cloud driver + its files changed"- "RUN"- (str "any of " (pr-str driver-triggers) " affected by cloud-driver module changes")]+ ["4" "Driver's own modules/drivers/* files changed" "RUN" "driver files changed (skip list ignored)"]+ ["5" "Configured status = skip (PR branches)" "SKIP" "driver is skipped by CI config"]+ ["6" "On master or release branch" "RUN" "master/release branch (skip list ignored)"]+ ["7" "Cloud driver + ci:run-all-cloud-drivers" "RUN" "ci:run-all-cloud-drivers label"]["8" "Cloud driver + query-processor updated" "RUN" "query-processor module updated"]["9" "Cloud driver + driver deps affected" "RUN" "driver module affected by shared code changes"]["10" "Cloud driver, no relevant changes" "SKIP" "no relevant changes for cloud driver"]@@ -426,16 +423,18 @@["12" "Self-hosted driver, not affected" "SKIP" "driver module not affected"]]))"\nThis table decides only WHETHER a driver runs. Whether its failures GATE is a separate\n""question, answered at the end of the job by ci-conductor's quarantine rules.\n"- "\nNote: the remote quarantine list (status = skip) is ignored entirely on master and\n"- "release-* branches -- every driver runs and gates there (Priority 4 is bypassed).\n"+ "\nNote: the remote skip list (status = skip) is ignored entirely on master and\n"+ "release-* branches -- every driver runs and gates there (Priority 5 is bypassed). It is\n"+ "also ignored for any driver whose own modules/drivers/* files the run changes.\n""\n\n## Labels of Interest\n\n""These PR labels influence driver test decisions:\n\n"- " ci:run-all-drivers Force-run ALL drivers (overrides quarantine)\n"- " ci:run-DRIVER Force-run a specific driver, e.g. ci:run-mysql-mariadb (overrides quarantine)\n"+ " ci:run-all-drivers Force-run ALL drivers (overrides the skip list)\n"+ " ci:run-DRIVER Force-run a specific driver, e.g. ci:run-mysql-mariadb (overrides the skip list)\n"" ci:run-all-cloud-drivers Run all cloud drivers (athena, bigquery, databricks, redshift, snowflake)\n""\n## Driver Status (from the CI driver config)\n\n"- " skip Listed with status \"skip\" in ci-test-config.json: not run at all (quarantined;\n"- " ci:run-DRIVER overrides). Every other driver runs and gates as usual.\n"+ " skip Listed with status \"skip\" in ci-test-config.json: not run at all (changing the\n"+ " driver's own files or ci:run-DRIVER overrides). Every other driver runs and\n"+ " gates as usual.\n""\nTo let a driver run but not gate, add a suite-quarantine rule in CI Conductor instead.\n""\n## All Drivers\n\n"(str/join ", " (map name (sort mage.modules/all-drivers)))diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex c2ed37d5887..19f1a093b3f 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -339,7 +339,7 @@(get driver-directory->drivers dir-name))))updated-files)))-;;; driver quarantine / status+;;; driver status(def ^:private ci-test-config-url"https://raw.githubusercontent.com/metabase/ci-test-config/refs/heads/master/ci-test-config.json")@@ -358,8 +358,9 @@(defn- skip-drivers"Set of driver keywords listed with status `\"skip\"` in the top-level `drivers` array ofci-test-config.json. Each entry identifies a driver by its short `name` (e.g. \"snowflake\");- a few names fan out to several jobs. `skip` means: do not run the driver at all (subject to- the ci:run-<driver> label which overrides the `skip` keyword and runs the driver anyways).+ a few names fan out to several jobs. `skip` means: do not run the driver at all, unless the run+ changes files under the driver's own `modules/drivers/*` directory or carries the+ ci:run-<driver> label -- either of those overrides the `skip` keyword and runs the driver.Drivers absent from the config run and gate as usual.@@ -382,15 +383,14 @@(.getMessage e) "); treating all drivers as required."))))#{})))-(defn- effective-quarantined-drivers- "Set of quarantined drivers to actually enforce for this run.+(defn- effective-skipped-drivers+ "Set of skipped drivers to actually enforce for this run.- The quarantine list is fetched on the fly from a remote file (ci-test-config.json, see+ The skip list is fetched on the fly from a remote file (ci-test-config.json, see[[skip-drivers]]) and can change at any time. We intentionally IGNORE it on `master` and- `release-*` branches: there, every driver must run and gate, so a stray remote quarantine entry- can't silently disable a driver's tests (nor raise a quarantine-conflict, which would fail a push- demanding a PR-only label). On PR/feature branches the quarantine is honored, subject to the- ci:run-<driver> label."+ `release-*` branches: there, every driver must run and gate, so a stray remote skip entry+ can't silently disable a driver's tests. On PR/feature branches the skip list is honored,+ subject to changes to the driver's own files and the ci:run-<driver> label."[skipped is-master-or-release](if is-master-or-release#{}@@ -429,7 +429,7 @@[driver{:keys [is-master-or-release pr-labels skip particular-driver-changed?]}driver-deps-affected?- quarantined-drivers+ skipped-driversupdated](cond;; Priority 1: Global skip (no backend changes)@@ -450,30 +450,30 @@"ci:run-all-drivers label"(str (run-driver-label driver) " label"))}- ;; Priority 4: Quarantined drivers are skipped; Priority 3 is the way to force one to run.- ;; On master/release this set is empty (see [[effective-quarantined-drivers]]), so quarantine is- ;; ignored there and we fall through to Priority 5.- (contains? quarantined-drivers driver)+ ;; Priority 4: The driver's own source changed - run it even when the CI config skips it,+ ;; since the change is exactly what needs testing.+ (contains? particular-driver-changed? driver)+ {:should-run true+ :reason "driver files changed"}++ ;; Priority 5: Drivers the CI config marks `skip`; Priorities 3 and 4 are the ways to force+ ;; one to run. On master/release this set is empty (see [[effective-skipped-drivers]]), so the+ ;; skip list is ignored there and we fall through to Priority 6.+ (contains? skipped-drivers driver){:should-run false- :reason "driver is quarantined"}+ :reason "driver is skipped by CI config"}- ;; Priority 5: Master/release branch - all (non-quarantined) drivers run+ ;; Priority 6: Master/release branch - every driver the CI config does not skip runsis-master-or-release{:should-run true:reason "master/release branch"}- ;; Priority 6: Cloud driver + ci:run-all-cloud-drivers label+ ;; Priority 7: Cloud driver + ci:run-all-cloud-drivers label(and (contains? cloud-drivers driver)(contains? pr-labels "ci:run-all-cloud-drivers")){:should-run true:reason "ci:run-all-cloud-drivers label"}- ;; Priority 7: Cloud driver + its files changed- (and (contains? cloud-drivers driver)- (contains? particular-driver-changed? driver))- {:should-run true- :reason (str "driver files changed (modules/drivers/" (name driver) "/**)")}-;; Priority 8: Cloud driver + module triggering cloud dbs updated → run it(and (contains? cloud-drivers driver)(seq (set/intersection updated modules-triggering-cloud-drivers)))@@ -524,10 +524,9 @@:pr-labels (parse-labels (:pr-labels options)):skip (parse-bool (:skip options)):particular-driver-changed? particular-driver-changed?}- ;; On master/release we drop the remote quarantine list entirely (see- ;; [[effective-quarantined-drivers]]) so every driver runs and gates, and no- ;; quarantine-conflict is raised.- quarantined (effective-quarantined-drivers (skip-drivers) is-master-or-release)+ ;; On master/release we drop the remote skip list entirely (see+ ;; [[effective-skipped-drivers]]) so every driver runs and gates.+ skipped (effective-skipped-drivers (skip-drivers) is-master-or-release)updated-files (u/updated-files git-ref)updated (updated-files->updated-modules updated-files)driver-affected? (driver-deps-affected? updated)@@ -535,24 +534,13 @@;; For module dependency check, combine both conditionseffective-driver-affected? (or driver-affected? important-file-changed?)decisions (mapv (fn [driver]- (assoc (driver-decision driver ctx effective-driver-affected? quarantined updated)+ (assoc (driver-decision driver ctx effective-driver-affected? skipped updated):driver driver))- all-drivers)- ;; Check for quarantined drivers with file changes but no ci:run-<driver> label- quarantined-with-changes (into #{}- (filter (fn [driver]- (and (contains? quarantined driver)- (contains? particular-driver-changed? driver)- (not (contains? (:pr-labels ctx)- (run-driver-label driver))))))- all-drivers)]+ all-drivers)](if github-output-only?;; In github-output-only mode, print just the key=value lines (no colors)- (do- (doseq [{:keys [driver should-run]} decisions]- (println (str (name driver) "-should-run=" should-run)))- (doseq [driver quarantined-with-changes]- (println (str (name driver) "-quarantine-conflict=true"))))+ (doseq [{:keys [driver should-run]} decisions]+ (println (str (name driver) "-should-run=" should-run)))(do;; Print module analysis summary(println "")@@ -577,15 +565,7 @@(println (str (name driver) "-should-run=true")))(println (c/yellow (str "\n=== Drivers to Skip (" (count drivers-to-skip) ") ===")))(doseq [{:keys [driver]} drivers-to-skip]- (println (str (name driver) "-should-run=false"))))- ;; Output quarantine conflict warnings with colors- (when (seq quarantined-with-changes)- (println "")- (println (c/red "⚠️ WARNING: Quarantined driver(s) have file changes but tests will NOT run!"))- (println (c/red "=== Quarantine Conflicts ==="))- (doseq [driver quarantined-with-changes]- (println (c/red (str " • " (name driver) " - add label '" (run-driver-label driver) "' to run tests")))- (println (str (name driver) "-quarantine-conflict=true"))))))+ (println (str (name driver) "-should-run=false"))))))(u/exit 0)))(defn -maindiff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.cljindex ff206dbd66f..24dc96b4a4a 100644--- a/mage/test/mage/modules_test.clj+++ b/mage/test/mage/modules_test.clj@@ -18,52 +18,71 @@opts));;; =============================================================================-;;; Priority 4: Quarantine (respected on PR branches, ignored on master/release)+;;; Priority 4: Driver's own files changed (beats the CI config skip list);;; =============================================================================-(deftest quarantined-driver-skips- (testing "Quarantined driver is skipped"- (let [result (mage.modules/driver-decision :mysql- (make-ctx {})- false ; driver-module-affected?- #{:mysql} ; quarantined- #{})] ; updated- (is (false? (:should-run result)))- (is (= "driver is quarantined" (:reason result))))))+(deftest particular-driver-changes-beat-configured-skip+ (testing "a skipped driver still runs when its own files changed -- that's what needs testing"+ (doseq [driver [:mysql :mongo :snowflake :databricks]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {:particular-driver-changed? #{driver}})+ false ; driver-deps-affected?+ #{driver} ; skipped+ #{})] ; updated+ (is (true? (:should-run result))+ (str driver " should run despite the CI config skip"))+ (is (= "driver files changed" (:reason result)))))))++;;; =============================================================================+;;; Priority 5: CI config skip list (respected on PR branches, ignored on master/release)+;;; =============================================================================-(deftest effective-quarantine-ignored-on-master-and-release- (testing "the remote quarantine list is dropped on master/release, but honored on PR branches"+(deftest skipped-driver-does-not-run+ (testing "Driver the CI config marks skip does not run"+ (doseq [[what changed] {"nothing changed" #{}+ "another driver's files changed" #{:mysql}}]+ (testing what+ (let [result (mage.modules/driver-decision :snowflake+ (make-ctx {:particular-driver-changed? changed})+ false ; driver-deps-affected?+ #{:snowflake :mysql} ; skipped+ #{})] ; updated+ (is (false? (:should-run result)))+ (is (= "driver is skipped by CI config" (:reason result))))))))++(deftest effective-skip-list-ignored-on-master-and-release+ (testing "the remote skip list is dropped on master/release, but honored on PR branches"(let [skipped #{:databricks}]- (is (= #{} (#'mage.modules/effective-quarantined-drivers skipped true))- "master/release: nothing is quarantined")- (is (= #{:databricks} (#'mage.modules/effective-quarantined-drivers skipped false))- "PR/feature branch: skipped drivers remain quarantined"))))--(deftest quarantined-driver-runs-on-master- (testing "a driver quarantined in the remote config still runs (and gates) on master/release"- ;; Production clears the quarantine set on master/release via effective-quarantined-drivers,- ;; so the decision falls through Priority 4 to Priority 5.- (let [quarantined (#'mage.modules/effective-quarantined-drivers #{:mysql} true)+ (is (= #{} (#'mage.modules/effective-skipped-drivers skipped true))+ "master/release: nothing is skipped")+ (is (= #{:databricks} (#'mage.modules/effective-skipped-drivers skipped false))+ "PR/feature branch: skipped drivers stay skipped"))))++(deftest skipped-driver-runs-on-master+ (testing "a driver marked skip in the remote config still runs (and gates) on master/release"+ ;; Production clears the skip set on master/release via effective-skipped-drivers,+ ;; so the decision falls through Priority 5 to Priority 6.+ (let [skipped (#'mage.modules/effective-skipped-drivers #{:mysql} true)result (mage.modules/driver-decision :mysql(make-ctx {:is-master-or-release true})true ; driver-deps-affected?- quarantined+ skipped#{})] ; updated(is (true? (:should-run result)))(is (= "master/release branch" (:reason result))))))-(deftest quarantine-config-names-are-translated+(deftest skip-config-names-are-translated(testing "Config names from ci-test-config.json are translated to internal driver keywords via driver-directory->drivers"(testing "directory names are translated to internal keywords";; bigquery-cloud-sdk -> :bigquery (via driver-directory->drivers mapping).- ;; Use a PR/feature branch here, since quarantine is ignored on master/release.+ ;; Use a PR/feature branch here, since the skip list is ignored on master/release.(let [result (mage.modules/driver-decision :bigquery(make-ctx {:is-master-or-release false})false#{:bigquery} ; as if translated from "bigquery-cloud-sdk"#{})](is (false? (:should-run result)))- (is (= "driver is quarantined" (:reason result)))))+ (is (= "driver is skipped by CI config" (:reason result)))))(testing "the driver-directory->drivers mapping contains expected translations";; Verify the mapping exists and has expected entries(is (= [:bigquery] (get @#'mage.modules/driver-directory->drivers "bigquery-cloud-sdk"))@@ -81,14 +100,14 @@(let [result (mage.modules/driver-decision driver(make-ctx {:skip true})true ; even if affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (false? (:should-run result))(str driver " should be skipped"))(is (= "workflow skip (no backend changes)" (:reason result)))))))-(deftest global-skip-beats-quarantine- (testing "Global skip takes priority over quarantine"+(deftest workflow-skip-beats-ci-config-skip+ (testing "Workflow skip takes priority over the CI config skip list"(let [result (mage.modules/driver-decision :mysql(make-ctx {:skip true})false@@ -107,7 +126,7 @@(let [result (mage.modules/driver-decision driver(make-ctx {:is-master-or-release false})false ; driver module not affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (true? (:should-run result))(str driver " should always run"))@@ -119,7 +138,7 @@(let [result (mage.modules/driver-decision driver(make-ctx {:skip true})false- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (false? (:should-run result))(str driver " should be skipped on global skip"))@@ -135,7 +154,7 @@(let [result (mage.modules/driver-decision driver(make-ctx {:pr-labels #{"ci:run-all-drivers"}})false ; not affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (true? (:should-run result))(str driver " should run with ci:run-all-drivers"))@@ -146,7 +165,7 @@(let [result (mage.modules/driver-decision :mysql(make-ctx {:pr-labels #{"ci:run-mysql"}})false- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (true? (:should-run result)))(is (= "ci:run-mysql label" (:reason result))))))@@ -156,42 +175,42 @@(let [result (mage.modules/driver-decision :mongo(make-ctx {:pr-labels #{"ci:run-mysql"}})false- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (false? (:should-run result))))))-(deftest ci-run-all-drivers-overrides-quarantine- (testing "ci:run-all-drivers overrides quarantine"+(deftest ci-run-all-drivers-overrides-skip-list+ (testing "ci:run-all-drivers overrides the skip list"(let [result (mage.modules/driver-decision :mysql(make-ctx {:pr-labels #{"ci:run-all-drivers"}})false- #{:mysql} ; quarantined+ #{:mysql} ; skipped#{})] ; updated(is (true? (:should-run result)))(is (= "ci:run-all-drivers label" (:reason result))))))-(deftest ci-run-specific-driver-overrides-quarantine- (testing "ci:run-<driver> overrides quarantine"+(deftest ci-run-specific-driver-overrides-skip-list+ (testing "ci:run-<driver> overrides the skip list"(let [result (mage.modules/driver-decision :mysql(make-ctx {:pr-labels #{"ci:run-mysql"}})false- #{:mysql} ; quarantined+ #{:mysql} ; skipped#{})] ; updated(is (true? (:should-run result)))(is (= "ci:run-mysql label" (:reason result))))));;; =============================================================================-;;; Priority 5: Master/release branch+;;; Priority 6: Master/release branch;;; =============================================================================(deftest master-branch-runs-all-drivers(testing "All drivers run on master/release branch"- ;; H2/Postgres hit priority 2 first, others hit priority 5+ ;; H2/Postgres hit priority 2 first, others hit priority 6(doseq [driver [:mysql :mongo :athena :bigquery :snowflake]](let [result (mage.modules/driver-decision driver(make-ctx {:is-master-or-release true})false ; even if not affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (true? (:should-run result))(str driver " should run on master"))@@ -208,14 +227,14 @@(let [result (mage.modules/driver-decision driver(make-ctx {})true ; driver-deps-affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (true? (:should-run result))(str driver " should run when driver module affected"))(is (= "driver module affected by shared code changes" (:reason result)))))));;; =============================================================================-;;; Priority 6-10: Cloud driver special rules+;;; Priority 7-10: Cloud driver special rules;;; =============================================================================(deftest cloud-driver-with-label-runs@@ -224,22 +243,12 @@(let [result (mage.modules/driver-decision driver(make-ctx {:pr-labels #{"ci:run-all-cloud-drivers"}})false ; not affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (true? (:should-run result))(str driver " should run with label"))(is (= "ci:run-all-cloud-drivers label" (:reason result)))))))-(deftest cloud-driver-with-file-changes-runs- (testing "Cloud driver runs when its files changed"- (let [result (mage.modules/driver-decision :athena- (make-ctx {:particular-driver-changed? #{:athena}})- false- #{} ; quarantined- #{})] ; updated- (is (true? (:should-run result)))- (is (re-find #"driver files changed" (:reason result))))))-(deftest modules-can-trigger-cloud-drivers(doseq [module '#{query-processor transformsenterprise/transforms enterprise/transforms-python}@@ -248,7 +257,7 @@(let [result (mage.modules/driver-decision driver(make-ctx {})false ; not affected- #{} ; quarantined+ #{} ; skipped#{module})] ; updated(is (true? (:should-run result))(str driver " should run when query-processor updated"))@@ -261,7 +270,7 @@(let [result (mage.modules/driver-decision driver(make-ctx {})true ; driver-deps-affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (true? (:should-run result))(str driver " should run when driver deps affected"))@@ -273,7 +282,7 @@(let [result (mage.modules/driver-decision driver(make-ctx {})false ; not affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (false? (:should-run result))(str driver " should skip without changes"))@@ -290,7 +299,7 @@(let [result (mage.modules/driver-decision driver(make-ctx {})false ; not affected- #{} ; quarantined+ #{} ; skipped#{})] ; updated(is (false? (:should-run result))(str driver " should skip when not affected"))@@ -394,11 +403,11 @@{:name "snowflake" :status "required"}]})(deftest skip-drivers-selects-only-skip-status- (testing "only drivers listed with status \"skip\" are quarantined"+ (testing "only drivers listed with status \"skip\" are skipped"(with-redefs [mage.modules/read-ci-test-config (constantly example-config)](is (= #{:databricks :mongo :mongo-ssl :mongo-sharded-cluster}(#'mage.modules/skip-drivers)))))- (testing "drivers absent from the config are not quarantined"+ (testing "drivers absent from the config are not skipped"(with-redefs [mage.modules/read-ci-test-config (constantly example-config)](is (not (contains? (#'mage.modules/skip-drivers) :mysql))))))