Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve App DB Test Sharding (#82538)

Ryan Laurie1a32584bb85c
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index f2885377d11..5430e42f781 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -84,6 +84,7 @@
:metabase/validate-defendpoint-route-uses-kebab-case {:level :warning}
:metabase/validate-defendpoint-query-params-use-kebab-case {:level :warning}
:metabase/validate-logging {:level :warning}
+ :metabase/validate-mb-app-db-migrations-test {:level :warning}
:metabase/validate-mb-driver-tests {:level :warning}
:metabase/validate-mb-once {:level :warning}
@@ -617,7 +618,7 @@
metabase-enterprise.transforms-python.drivers-test/with-test-table hooks.common/with-two-bindings
metabase.api.common.internal-test/with-jetty-server hooks.common/with-one-binding
metabase.api.macros/defendpoint hooks.metabase.api.macros/defendpoint
- metabase.app-db.schema-migrations-test.impl/test-migrations hooks.metabase.app-db.schema-migrations-test.impl/test-migrations
+ metabase.app-db.schema-migrations-test.impl/test-migrations hooks.metabase.app-db.schema-migrations-test.impl/test-migrations!
metabase.channel.render.js.pool/with-static-viz-context hooks.common/with-one-top-level-binding
metabase.channel.template.handlebars-helper/defhelper hooks.metabase.channel.template.handlebars-helper/defhelper
metabase.channel.template.handlebars-test/with-temp-template! hooks.common/with-vec-first-binding
diff --git a/.clj-kondo/src/hooks/metabase/app_db/schema_migrations_test/impl.clj b/.clj-kondo/src/hooks/metabase/app_db/schema_migrations_test/impl.clj
index c31d5169dc3..9b53bb0266b 100644
--- a/.clj-kondo/src/hooks/metabase/app_db/schema_migrations_test/impl.clj
+++ b/.clj-kondo/src/hooks/metabase/app_db/schema_migrations_test/impl.clj
@@ -1,8 +1,22 @@
(ns hooks.metabase.app-db.schema-migrations-test.impl
- (:require [clj-kondo.hooks-api :as hooks]))
+ (:require
+ [clj-kondo.hooks-api :as hooks]))
-(defn test-migrations
- [{{[_ migration-range {[binding] :children} & body] :children} :node}]
+(defn- validate-mb-app-db-migrations-test!
+ "CI runs the namespaces tagged `:mb/app-db-migrations-test` on a shard of their own, because a count-based split
+ can't balance them (see `.github/workflows/app-db.yml`). An untagged one still runs, just on a shard sized for
+ everything else."
+ [{:keys [ns node]}]
+ (when-not (:mb/app-db-migrations-test (meta ns))
+ (hooks/reg-finding!
+ (assoc (meta node)
+ :message (str "Namespaces that run migration tests (test-migrations) should be marked"
+ " ^:mb/app-db-migrations-test [:metabase/validate-mb-app-db-migrations-test]")
+ :type :metabase/validate-mb-app-db-migrations-test))))
+
+(defn test-migrations!
+ [{{[_ migration-range {[binding] :children} & body] :children} :node, :as x}]
+ (validate-mb-app-db-migrations-test! x)
(let [node* (hooks/list-node
(list*
(hooks/token-node `let)
diff --git a/.clj-kondo/test/hooks/metabase/app_db/schema_migrations_test/impl_test.clj b/.clj-kondo/test/hooks/metabase/app_db/schema_migrations_test/impl_test.clj
new file mode 100644
index 00000000000..d70dcac940b
--- /dev/null
+++ b/.clj-kondo/test/hooks/metabase/app_db/schema_migrations_test/impl_test.clj
@@ -0,0 +1,25 @@
+(ns hooks.metabase.app-db.schema-migrations-test.impl-test
+ (:require
+ [clj-kondo.hooks-api :as hooks]
+ [clj-kondo.impl.utils]
+ [clojure.test :refer :all]
+ [hooks.metabase.app-db.schema-migrations-test.impl]))
+
+(defn- finding-types!
+ [ns-symb]
+ (binding [clj-kondo.impl.utils/*ctx* {:config {:linters {:metabase/validate-mb-app-db-migrations-test {:level :warning}}}
+ :ignores (atom nil)
+ :findings (atom [])
+ :namespaces (atom {})}]
+ (hooks.metabase.app-db.schema-migrations-test.impl/test-migrations!
+ {:ns ns-symb
+ :node (hooks/parse-string (pr-str '(impl/test-migrations ["v64.2026-09-01T00:00:00"] [migrate!] (migrate!))))})
+ (mapv :type @(:findings clj-kondo.impl.utils/*ctx*))))
+
+(deftest ^:synchronized require-app-db-migrations-test-tag-test
+ (testing "a namespace that runs migration tests must be tagged, so CI shards it with the other migration tests"
+ (is (= [:metabase/validate-mb-app-db-migrations-test]
+ (finding-types! 'metabase.some-migrations-test))))
+ (testing "a tagged namespace is fine"
+ (is (= []
+ (finding-types! (with-meta 'metabase.some-migrations-test {:mb/app-db-migrations-test true}))))))
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.yml
index 7cb787324cb..349d0ee58a3 100644
--- a/.github/workflows/app-db.yml
+++ b/.github/workflows/app-db.yml
@@ -64,8 +64,9 @@ jobs:
mariadb: ${{ steps.versions.outputs.mariadb }}
mysql: ${{ steps.versions.outputs.mysql }}
postgres: ${{ steps.versions.outputs.postgres }}
+ shards: ${{ steps.versions.outputs.shards }}
steps:
- - name: Select versions to test
+ - name: Select versions and shards to test
id: versions
uses: actions/github-script@v9
env:
@@ -133,6 +134,32 @@ jobs:
core.setOutput(db, JSON.stringify(selected));
}
+ const commonExcludes = ":mb/driver-tests :mb/transforms-python-test :mb/old-migrations-test";
+
+ const shards = [
+ {
+ name: 'Enterprise Tests',
+ 'build-static-viz': false,
+ 'test-args': `:only '"enterprise/backend/test"' :exclude-tags '[${commonExcludes}]'`,
+ },
+ {
+ name: 'EE App DB Tests (Part 1)', // migrations shard
+ 'build-static-viz': true,
+ 'test-args': `:only '"test"' :only-tags '[:mb/app-db-migrations-test]' :exclude-tags '[${commonExcludes}]'`,
+ },
+ {
+ name: 'EE App DB Tests (Part 2)',
+ 'build-static-viz': true,
+ 'test-args': `:only '"test"' :partition/total 2 :partition/index 0 :exclude-tags '[${commonExcludes} :mb/app-db-migrations-test]'`,
+ },
+ {
+ name: 'EE App DB Tests (Part 3)',
+ 'build-static-viz': true,
+ 'test-args': `:only '"test"' :partition/total 2 :partition/index 1 :exclude-tags '[${commonExcludes} :mb/app-db-migrations-test]'`,
+ },
+ ];
+ core.setOutput('shards', JSON.stringify(shards));
+
be-tests-mariadb:
needs: app-db-matrix
if: ${{ !inputs.skip && (inputs.app-db == '' || inputs.app-db == 'mariadb') }}
@@ -142,26 +169,7 @@ jobs:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.app-db-matrix.outputs.mariadb) }}
- job:
- - name: Enterprise Tests
- build-static-viz: false
- test-args: >-
- :only '"enterprise/backend/test"'
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- - name: EE App DB Tests (Part 1)
- build-static-viz: true
- test-args: >-
- :only '"test"'
- :partition/total 2
- :partition/index 0
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- - name: EE App DB Tests (Part 2)
- build-static-viz: true
- test-args: >-
- :only '"test"'
- :partition/total 2
- :partition/index 1
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
+ job: ${{ fromJSON(needs.app-db-matrix.outputs.shards) }}
services:
mysql:
@@ -175,10 +183,6 @@ jobs:
MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: true
MARIADB_DATABASE: mb_test
env:
- # __ADDITIONAL_EXCLUDED_TAG__: ${{ !inputs.force-run && ':mb/old-migrations-test' || '' }}
- # GDGT-3126: re-enable and properly fix old migrations tests
- __ADDITIONAL_EXCLUDED_TAG__: ':mb/old-migrations-test'
- # actual serious env vars below
CI: 'true'
DRIVERS: mysql
MB_DB_TYPE: mysql
@@ -205,7 +209,8 @@ jobs:
# for historic reasons (I don't want to mess around with required jobs) the job name should be something like
# "be-tests-mariadb-10-6-ee (0)"
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
- steps:
+ # Shared by every app DB job below via `*app-db-test-steps`.
+ steps: &app-db-test-steps
- uses: actions/checkout@v6
- name: Test ${{ matrix.version.name }}
id: test-driver
@@ -214,9 +219,7 @@ jobs:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
artifact-suffix: ${{ matrix.job.name }}
- test-args: >-
- ${{ matrix.job.test-args }}
- :exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
+ test-args: ${{ matrix.job.test-args }}
be-tests-mysql:
needs: app-db-matrix
@@ -227,26 +230,7 @@ jobs:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.app-db-matrix.outputs.mysql) }}
- job:
- - name: Enterprise Tests
- build-static-viz: false
- test-args: >-
- :only '"enterprise/backend/test"'
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- - name: EE App DB Tests (Part 1)
- build-static-viz: true
- test-args: >-
- :only '"test"'
- :partition/total 2
- :partition/index 0
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- - name: EE App DB Tests (Part 2)
- build-static-viz: true
- test-args: >-
- :only '"test"'
- :partition/total 2
- :partition/index 1
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
+ job: ${{ fromJSON(needs.app-db-matrix.outputs.shards) }}
services:
mysql:
@@ -260,10 +244,6 @@ jobs:
MYSQL_ALLOW_EMPTY_PASSWORD: true
MYSQL_DATABASE: mb_test
env:
- # __ADDITIONAL_EXCLUDED_TAG__: ${{ !inputs.force-run && ':mb/old-migrations-test' || '' }}
- # GDGT-3126: re-enable and properly fix old migrations tests
- __ADDITIONAL_EXCLUDED_TAG__: ':mb/old-migrations-test'
- # actual serious env vars below
CI: 'true'
DRIVERS: mysql
MB_DB_TYPE: mysql
@@ -298,18 +278,7 @@ jobs:
# for historic reasons (I don't want to mess around with required jobs) the job name should be something like
# "be-tests-mariadb-10-6-ee (0)"
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
- steps:
- - uses: actions/checkout@v6
- - name: Test ${{ matrix.version.name }}
- id: test-driver
- uses: ./.github/actions/test-driver
- with:
- build-static-viz: ${{ matrix.job.build-static-viz }}
- junit-name: ${{ matrix.version.junit-name }}
- artifact-suffix: ${{ matrix.job.name }}
- test-args: >-
- ${{ matrix.job.test-args }}
- :exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
+ steps: *app-db-test-steps
be-tests-postgres:
needs: app-db-matrix
@@ -320,32 +289,9 @@ jobs:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.app-db-matrix.outputs.postgres) }}
- job:
- - name: Enterprise Tests
- build-static-viz: false
- test-args: >-
- :only '"enterprise/backend/test"'
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- - name: EE App DB Tests (Part 1)
- build-static-viz: true
- test-args: >-
- :only '"test"'
- :partition/total 2
- :partition/index 0
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
- - name: EE App DB Tests (Part 2)
- build-static-viz: true
- test-args: >-
- :only '"test"'
- :partition/total 2
- :partition/index 1
- exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
+ job: ${{ fromJSON(needs.app-db-matrix.outputs.shards) }}
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
env:
- # __ADDITIONAL_EXCLUDED_TAG__: ${{ !inputs.force-run && ':mb/old-migrations-test' || '' }}
- # GDGT-3126: re-enable and properly fix old migrations tests
- __ADDITIONAL_EXCLUDED_TAG__: ':mb/old-migrations-test'
- # actual serious env vars below
CI: 'true'
DRIVERS: postgres
MB_DB_TYPE: postgres
@@ -374,18 +320,7 @@ jobs:
POSTGRES_USER: mb_test
POSTGRES_DB: mb_test
POSTGRES_HOST_AUTH_METHOD: trust
- steps:
- - uses: actions/checkout@v6
- - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
- id: test-driver
- uses: ./.github/actions/test-driver
- with:
- build-static-viz: ${{ matrix.job.build-static-viz }}
- junit-name: ${{ matrix.version.junit-name }}
- artifact-suffix: ${{ matrix.job.name }}
- test-args: >-
- ${{ matrix.job.test-args }}
- :exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
+ steps: *app-db-test-steps
app-db-tests-result:
needs:
diff --git a/test/metabase/app_db/custom_migrations/llm_providers_test.clj b/test/metabase/app_db/custom_migrations/llm_providers_test.clj
index 888b09370f6..09e92aacae0 100644
--- a/test/metabase/app_db/custom_migrations/llm_providers_test.clj
+++ b/test/metabase/app_db/custom_migrations/llm_providers_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.app-db.custom-migrations.llm-providers-test
+(ns ^:mb/app-db-migrations-test metabase.app-db.custom-migrations.llm-providers-test
"Tests for the v64.7qmx3p migration that moves the per-provider LLM credential settings onto the `llm-providers`
connection list. The migration itself is [[metabase.app-db.custom-migrations.llm-providers]], reached through the
`MigrateLlmProviderSettings` custom change."
diff --git a/test/metabase/app_db/custom_migrations/metrics_v2_test.clj b/test/metabase/app_db/custom_migrations/metrics_v2_test.clj
index 5566ffc0f85..59360a3389d 100644
--- a/test/metabase/app_db/custom_migrations/metrics_v2_test.clj
+++ b/test/metabase/app_db/custom_migrations/metrics_v2_test.clj
@@ -1,4 +1,4 @@
-(ns ^:mb/old-migrations-test metabase.app-db.custom-migrations.metrics-v2-test
+(ns ^:mb/old-migrations-test ^:mb/app-db-migrations-test metabase.app-db.custom-migrations.metrics-v2-test
"These are 'old' tests now since this migration happened in 51."
(:require
[clojure.test :refer [deftest is testing use-fixtures]]
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index 2e72908d3a9..2b47fecfc4a 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.app-db.custom-migrations-test
+(ns ^:mb/app-db-migrations-test metabase.app-db.custom-migrations-test
"Tests to make sure the custom migrations work as expected.
As of #52254, any tests marked `^:mb/old-migrations-test` are only run on pushes to `master` or `release-`
diff --git a/test/metabase/app_db/force_migration_test.clj b/test/metabase/app_db/force_migration_test.clj
index a23fdc5480a..7ab18d77222 100644
--- a/test/metabase/app_db/force_migration_test.clj
+++ b/test/metabase/app_db/force_migration_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.app-db.force-migration-test
+(ns ^:mb/app-db-migrations-test metabase.app-db.force-migration-test
(:require
[clojure.test :refer :all]
[metabase.app-db.core :as mdb]
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index f5ace6c1f42..185faaece2c 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.app-db.schema-migrations-test
+(ns ^:mb/app-db-migrations-test metabase.app-db.schema-migrations-test
"Tests for the schema migrations defined in the Liquibase YAML files. The basic idea is:
1. Create a temporary H2/Postgres/MySQL/MariaDB database
diff --git a/test/metabase/app_db/schema_migrations_test/impl.clj b/test/metabase/app_db/schema_migrations_test/impl.clj
index b1a59cb9ce1..8cd1c30f2f6 100644
--- a/test/metabase/app_db/schema_migrations_test/impl.clj
+++ b/test/metabase/app_db/schema_migrations_test/impl.clj
@@ -1,4 +1,4 @@
-(ns metabase.app-db.schema-migrations-test.impl
+(ns ^:mb/app-db-migrations-test metabase.app-db.schema-migrations-test.impl
"Tests for the schema migrations defined in the Liquibase YAML files. The basic idea is:
1. Create a temporary H2/Postgres/MySQL/MariaDB database
diff --git a/test/metabase/app_db/setup_test.clj b/test/metabase/app_db/setup_test.clj
index a57a475fb7c..980876fb312 100644
--- a/test/metabase/app_db/setup_test.clj
+++ b/test/metabase/app_db/setup_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.app-db.setup-test
+(ns ^:mb/app-db-migrations-test metabase.app-db.setup-test
(:require
[clojure.java.jdbc :as jdbc]
[clojure.test :refer :all]
diff --git a/test/metabase/permissions/models/data_permissions_test.clj b/test/metabase/permissions/models/data_permissions_test.clj
index 987d3cc9b74..a2e347c0da8 100644
--- a/test/metabase/permissions/models/data_permissions_test.clj
+++ b/test/metabase/permissions/models/data_permissions_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.permissions.models.data-permissions-test
+(ns ^:mb/app-db-migrations-test metabase.permissions.models.data-permissions-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]