Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Run project checks in a separate job (#81318)

Chris Truter3c5d1111d77a
diff --git a/.claude/agents/modules-backend-expert.md b/.claude/agents/modules-backend-expert.md
index e6f29f6281a..4389f068653 100644
--- a/.claude/agents/modules-backend-expert.md
+++ b/.claude/agents/modules-backend-expert.md
@@ -81,7 +81,7 @@ Everything else in the module is **internal** and not allowed to be referenced f
`dev/src/dev/modularization_help.clj` — `potemkin-ns!` helper that generates `.core` re-export boilerplate from a target namespace.
-`test/metabase/core/modules_test.clj` — runs on config changes; validates structure, team assignments, API and model boundaries.
+`dev/test/metabase/core/modules_test.clj` — runs on config changes; validates structure, team assignments, API and model boundaries.
### API Design Inside a Module
@@ -119,7 +119,7 @@ When the linter complains, the answer is **always refactoring**, never suppressi
- `dev/src/dev/module_score.clj` — quality scoring
- `dev/src/dev/modularization_help.clj` — Potemkin re-export helper
- `dev/src/dev/model_boundary_config.clj` + `dev/src/dev/model_tracking.clj` — model-level enforcement
-- `test/metabase/core/modules_test.clj` — config validation tests
+- `dev/test/metabase/core/modules_test.clj` — config validation tests
- `src/metabase/core/init.clj` — OSS module init aggregation
- `enterprise/backend/src/metabase_enterprise/core/init.clj` — EE module init aggregation
- `src/metabase/models/resolution.clj` — `:model/X` keyword → namespace map
@@ -198,6 +198,6 @@ Use the `clojure-eval` skill (preferred) or `clj-nrepl-eval` to drive the dev to
(dev.module-score/scores (dev.module-score/deps) (dev.module-score/config))
```
-For tests of the config itself, use `./bin/test-agent :only '[metabase.core.modules-test]'`. After editing Clojure files, run `clj-paren-repair`. After editing `config.edn`, re-run the modules-test.
+For tests of the config itself, use `./bin/mage project-tests modules`. After editing Clojure files, run `clj-paren-repair`. After editing `config.edn`, re-run the modules-test.
**Update your agent memory** as you discover idioms: which modules use `:friends` and why, which models are most-imported (suggesting they belong in their own module), which settings are accessed cross-module via `(setting/get …)` instead of `:uses` deps, and which refactors the team is actively running.
diff --git a/.claude/skills/clojure-write/SKILL.md b/.claude/skills/clojure-write/SKILL.md
index 0760a2c7c25..28badaf5f17 100644
--- a/.claude/skills/clojure-write/SKILL.md
+++ b/.claude/skills/clojure-write/SKILL.md
@@ -123,5 +123,5 @@ breakage keeps happening, *that* is the signal a comment was warranted.
- Spaces on a line with nothing after it is not allowed
- After changing module boundaries (adding/removing/renaming a `src` namespace, a cross-module `require`
or `:model/X` reference, or a new module), run `./bin/mage fix-modules-config` to regenerate
- `.clj-kondo/config/modules/config.edn` and keep `metabase.core.modules-test` green. No-op when nothing
- drifted; see "Module Boundaries" in the project `CLAUDE.md`.
+ `.clj-kondo/config/modules/config.edn`, then run `./bin/mage project-tests modules`. The fixer is a no-op when
+ nothing drifted; see "Module Boundaries" in the project `CLAUDE.md`.
diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
index bd0b75c6f99..f0b70fc97af 100644
--- a/.github/actionlint.yaml
+++ b/.github/actionlint.yaml
@@ -8,6 +8,9 @@ paths:
# GitHub added parallel steps (`parallel`, `background`, `wait`, `wait-all`,
# `cancel`) in June 2026, and actionlint 1.7.12 does not know the schema yet:
# https://github.blog/changelog/2026-06-25-actions-steps-can-now-be-run-in-parallel/
+ .github/workflows/backend.yml:
+ ignore:
+ - 'step must run script with "run" section or run action with "uses" section'
.github/workflows/lint.yml:
ignore:
- 'step must run script with "run" section or run action with "uses" section'
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 87740280feb..9ad7a4328c0 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -117,6 +117,7 @@ backend_sources: &backend_sources
backend_specs: &backend_specs
- *shared_specs
- "test/**"
+ - "dev/test/**"
- "enterprise/backend/test/**"
- "test_modules/**"
- "modules/drivers/*/test/**"
@@ -129,6 +130,45 @@ backend_all: &backend_all
- *backend_sources
- *backend_specs
+# Project checks have narrower triggers than the backend matrix.
+# Changes to their shared CI entry points exercise every suite; otherwise each
+# suite follows only the files that can affect its result.
+project_tests_infra: &project_tests_infra
+ - ".github/actions/prepare-backend/**"
+ - ".github/file-paths.yaml"
+ - ".github/workflows/backend.yml"
+ - ".github/workflows/run-tests.yml"
+ - "bb.edn"
+ - "bin/mage"
+ - "bin/mage.bb"
+ - "mage/src/mage/color.clj"
+ - "mage/src/mage/project_tests.clj"
+ - "mage/src/mage/shell.clj"
+ - "mage/src/mage/util.clj"
+
+project_backend_checks:
+ - *project_tests_infra
+ - "{src,test,enterprise,dev,bin,mage}/**/*.{clj,cljc,cljs}"
+ - "modules/**/*.clj"
+ - "modules/**/*.cljc"
+ - "modules/**/*.cljs"
+ - "**deps.edn"
+ - ".clj-kondo/config.edn"
+ - ".clj-kondo/config/modules/config.edn"
+ - ".github/team.json"
+ - "dev/resources/dev/modules_config_test/**"
+
+# Only the ratchet check reads the budgets, so a diff that touches nothing else runs that suite alone.
+project_ratchet_checks:
+ - ".clj-kondo/ratchets.edn"
+
+project_migration_checks:
+ - *project_tests_infra
+ - "bin/lint-migrations-file/**"
+ - "resources/migrations/**/*.{yaml,yml}"
+ - "src/metabase/app_db/custom_migrations.clj"
+ - "src/metabase/app_db/custom_migrations/**"
+
sources: &sources
- *shared_sources
- *frontend_sources
diff --git a/.github/scripts/cache-keys.sh b/.github/scripts/cache-keys.sh
index d0aea2bd03e..7d3abb9d560 100755
--- a/.github/scripts/cache-keys.sh
+++ b/.github/scripts/cache-keys.sh
@@ -50,7 +50,11 @@ digest() {
}
shopt -s nullglob
-DEPS_FILES=(deps.edn modules/drivers/deps.edn modules/drivers/*/deps.edn)
+DEPS_FILES=(deps.edn
+ bin/lint-migrations-file/deps.edn
+ bin/load-namespaces/deps.edn
+ modules/drivers/deps.edn
+ modules/drivers/*/deps.edn)
shopt -u nullglob
DEPS_HASH="$(digest "${DEPS_FILES[@]}")"
diff --git a/.github/scripts/check-preresolve-aliases.sh b/.github/scripts/check-preresolve-aliases.sh
index b1982661c26..4828d656947 100755
--- a/.github/scripts/check-preresolve-aliases.sh
+++ b/.github/scripts/check-preresolve-aliases.sh
@@ -1,6 +1,5 @@
#!/usr/bin/env bash
-# Fails when a `clojure` invocation under .github/ uses an alias combination that cache-warm does not
-# pre-resolve.
+# Fails when a Clojure classpath used by CI is not pre-resolved by cache-warm.
#
# tools.deps selects dependency versions globally per classpath, so two alias combinations can resolve
# the same transitive library to different versions. A combination the warm job never resolved is
@@ -30,10 +29,18 @@ EOF
# :deps - tools.deps built-in used by write-poms.sh; contributes no dependencies of its own.
# :outdated - scheduled dependency-bump workflow only; a cold resolve there costs nothing anyone waits on.
-# Alias combinations the warm job pre-resolves.
-declared="$(sed -n '/combos=(/,/^ *)/p' "$WARM" | grep -oE '":[^"]*"' | tr -d '"' | sort -u)"
+# Dependency roots and alias combinations the warm job pre-resolves.
+declared="$({
+ sed -n '/^[[:space:]]*combos=(/,/^ *)/p' "$WARM" \
+ | grep -oE '":[^"]*"' \
+ | tr -d '"' \
+ | sed 's#^#.|#'
+ sed -n '/^[[:space:]]*nested_combos=(/,/^ *)/p' "$WARM" \
+ | grep -oE '"[^"]+\|:[^"]+"' \
+ | tr -d '"'
+} | sort -u)"
if [ -z "$declared" ]; then
- echo "::error::check-preresolve-aliases.sh: found no combos=( ... ) list in $WARM" >&2
+ echo "::error::check-preresolve-aliases.sh: found no pre-resolved classpaths in $WARM" >&2
exit 1
fi
@@ -43,27 +50,60 @@ unknown="$(grep -rhoE '\$\{\{[^}]*\}\}' --include='*' \
--exclude='*.png' --exclude='*.jpg' .github/ 2>/dev/null \
| sort -u | grep -v 'edition' || true)"
-scan() { # scan <edition>: alias strings used by clojure invocations, with expressions substituted
+scan() { # scan <edition>: dependency root and aliases used by .github Clojure invocations
grep -rhE 'clojure[[:space:]]+(-P[[:space:]]+)?-[XMATP]?:' \
--include='*' --exclude='*.png' --exclude='*.jpg' .github/ 2>/dev/null \
| sed "s/\\\${{[^}]*edition[^}]*}}/$1/g" \
- | grep -oE 'clojure[[:space:]]+(-P[[:space:]]+)?-[XMATP]?:[A-Za-z0-9:_.-]+' \
- | grep -oE ':[A-Za-z0-9:_.-]+' \
- | sed 's/:$//'
+ | grep -oE 'clojure[[:space:]]+(-P[[:space:]]+)?-[XMATP]?:[A-Za-z0-9:_./-]+' \
+ | grep -oE ':[A-Za-z0-9:_./-]+' \
+ | sed -e 's/:$//' -e 's#^#.|#'
}
-found="$( { scan ee; scan oss; } | sort -u )"
+scan_project_tests() { # scan_project_tests <function> <dependency-root>
+ # Mage shell commands are vectors split across lines, so flatten each function body before
+ # extracting its alias.
+ sed -n "/(defn- $1 /,/^(defn/p" mage/src/mage/project_tests.clj \
+ | tr '\n' ' ' \
+ | grep -oE '"clojure"[[:space:]]+("-P"[[:space:]]+)?"-[XMATP]?:[A-Za-z0-9:_./-]+' \
+ | grep -oE ':[A-Za-z0-9:_./-]+' \
+ | sed -e 's/:$//' -e "s#^#$2|#"
+}
+
+# Check each scan separately. One scan still finding commands would otherwise conceal that another has
+# stopped matching, which silently drops everything it used to cover.
+require_scan() { # require_scan <what> <result>
+ [ -n "$2" ] && return
+ echo "::error::check-preresolve-aliases.sh: found no Clojure invocation in $1; the scan pattern no longer matches" >&2
+ exit 1
+}
+workflow_checks=$( { scan ee; scan oss; } || true)
+clojure_checks=$(scan_project_tests run-clojure-checks! . || true)
+migration_checks=$(scan_project_tests run-migration-checks! bin/lint-migrations-file || true)
+require_scan ".github/" "$workflow_checks"
+require_scan "run-clojure-checks! (mage/src/mage/project_tests.clj)" "$clojure_checks"
+require_scan "run-migration-checks! (mage/src/mage/project_tests.clj)" "$migration_checks"
+
+scanned="$(printf '%s\n' "$workflow_checks" "$clojure_checks" "$migration_checks" | sort -u)"
+
+found="$( {
+ printf '%s\n' "$scanned"
+ # build-scripts.yml runs this classpath from its own dependency root.
+ printf '%s\n' 'bin/load-namespaces|:test'
+} | sort -u )"
-covered="$(printf '%s\n%s\n' "$declared" "$WAIVED" | sort -u)"
+covered="$( {
+ printf '%s\n' "$declared"
+ printf '%s\n' "$WAIVED" | sed 's#^#.|#'
+} | sort -u)"
missing="$(comm -23 <(printf '%s\n' "$found") <(printf '%s\n' "$covered") || true)"
if [ -n "$missing" ]; then
- echo "::error::Alias combinations used under .github/ that cache-warm does not pre-resolve:" >&2
+ echo "::error::Clojure classpaths used by CI that cache-warm does not pre-resolve:" >&2
printf '%s\n' "$missing" | sed 's/^/ /' >&2
echo >&2
- echo "Add each to the combos list in $WARM, or waive it in this script with a reason." >&2
+ echo "Add each to the matching root or nested combo list in $WARM, or waive it here with a reason." >&2
echo "Leaving it uncovered means those jobs re-fetch their version deltas from Maven on every run." >&2
exit 1
fi
-echo "All $(printf '%s\n' "$found" | grep -c .) clojure alias combinations under .github/ are pre-resolved."
+echo "All $(printf '%s\n' "$found" | grep -c .) Clojure classpaths used by CI are pre-resolved."
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.yml
index c8693a374d7..8e5ce6c6482 100644
--- a/.github/workflows/app-db.yml
+++ b/.github/workflows/app-db.yml
@@ -332,7 +332,6 @@ jobs:
build-static-viz: true
test-args: >-
:only '"test"'
- :exclude-tags [:mb/driver-tests]
:partition/total 2
:partition/index 0
exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index f9ad2b54541..9b0590e33a1 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -10,6 +10,18 @@ on:
description: Run the full suite regardless of the diff (master / release branches).
type: boolean
default: false
+ run-project-backend-checks:
+ description: Run project-level backend checks.
+ type: boolean
+ default: false
+ run-project-migration-checks:
+ description: Run project-level migration checks.
+ type: boolean
+ default: false
+ run-project-ratchet-checks:
+ description: Run the ratchet check on its own, when the backend checks (which include it) do not run.
+ type: boolean
+ default: false
# Run the backend test suite on demand, against the ref you dispatch on, optionally under a
# profiler. Everything else about the jobs -- matrix legs, tag filters, quarantine gating -- is
@@ -46,6 +58,94 @@ env:
ONLY_TESTS: ${{ inputs.tests }}
jobs:
+ project-checks:
+ name: 'Project tests: Suites'
+ if: >-
+ ${{ inputs.force-run ||
+ inputs.run-project-backend-checks ||
+ inputs.run-project-migration-checks ||
+ inputs.run-project-ratchet-checks }}
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v6
+ - name: Prepare back-end environment
+ uses: ./.github/actions/prepare-backend
+ # Bootstrap the shared Mage runtime before concurrent commands try to install it themselves.
+ - name: Prepare Mage
+ run: ./bin/mage
+ - parallel:
+ - name: Run backend checks
+ if: ${{ inputs.force-run || inputs.run-project-backend-checks }}
+ run: ./bin/mage project-tests backend
+ - name: Run migration checks
+ if: ${{ inputs.force-run || inputs.run-project-migration-checks }}
+ run: ./bin/mage project-tests migrations
+ # The backend checks already run this suite, so it only runs when they do not.
+ - name: Run ratchet checks
+ if: ${{ inputs.run-project-ratchet-checks && !inputs.force-run && !inputs.run-project-backend-checks }}
+ run: ./bin/mage project-tests ratchets
+
+ project-tests:
+ name: Project tests
+ needs: project-checks
+ if: ${{ always() && !cancelled() }}
+ runs-on: ${{ vars.SLIM_RUNNER_KEY }}
+ timeout-minutes: 5
+ steps:
+ - name: Check Project tests status
+ uses: actions/github-script@v9
+ env:
+ needs: ${{ toJson(needs) }}
+ with:
+ script: | # js
+ const needs = JSON.parse(process.env.needs);
+ const jobs = Object.entries(needs).map(
+ ([jobName, jobValues]) => ({
+ name: jobName,
+ result: jobValues.result
+ }));
+
+ if (jobs.every(job => job.result === 'skipped' || job.result === 'success')) {
+ console.log('');
+ console.log(' |>>> |>>> |>>>');
+ console.log(' | | |');
+ console.log(' _ _|_ _ _ _|_ _ _ _|_ _');
+ console.log(' | |_| |_| |______| |_| |_| |______| |_| |_| |');
+ console.log(' | | | | | |');
+ console.log(' | [] [] | | [] [] | | [] [] |');
+ console.log(' | |______| |______| |');
+ console.log(' | ___ |');
+ console.log(' |___________________| |___________________|');
+ console.log(' |___|');
+ console.log('');
+ console.log('Everything is up to code and structurally sound.');
+ process.exit(0);
+ }
+
+ console.log('');
+ console.log(' .-===========-.____');
+ console.log(' /_______________\\ \\____');
+ console.log(' | .-----------. | \\___');
+ console.log(' | | x x | | \\__');
+ console.log(' | | ~ | | _\\_');
+ console.log(" | `-----------' | /___\\");
+ console.log(' | o o |');
+ console.log(" `-----------------'");
+ console.log(' /=============\\');
+ console.log(' / O O O O O O O \\');
+ console.log(' \\_______________/');
+ console.log('');
+ console.log('One or more project checks are off the blueprint. Hard hats required.');
+
+ jobs.forEach((job) => {
+ if (job.result !== 'success' && job.result !== 'skipped') {
+ console.log(`${job.name} - ${job.result}`);
+ }
+ });
+
+ process.exit(1);
+
be-linter-clj-kondo:
if: ${{ !inputs.skip }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
@@ -80,7 +180,7 @@ jobs:
- uses: actions/checkout@v6
- name: Prepare back-end environment
uses: ./.github/actions/prepare-backend
- - run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood/test
+ - run: clojure -X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:eastwood/test
name: Run Eastwood linter on our backend tests
be-linter-eastwood-test-changed:
@@ -108,13 +208,14 @@ jobs:
# last synced don't widen our scope. Falls back to BASE_SHA if older than fetched depth.
git fetch --no-tags --depth=200 origin "$BASE_SHA"
DIFF_BASE=$(git merge-base "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "$BASE_SHA")
- # Test roots we lint: core, EE, and each driver module's test tree.
+ # Test roots we lint: core, EE, dev/test, and each driver module's test tree.
# :eastwood/test derives matching :source-paths from the classpath, so any
# namespace we name here is reachable. Include renames (R) so a test that
# was just renamed still gets linted under its new name.
FILES=$(git diff --name-only --diff-filter=AMR "$DIFF_BASE" "$HEAD_SHA" \
-- 'test/**/*.clj' 'test/**/*.cljc' \
'enterprise/backend/test/**/*.clj' 'enterprise/backend/test/**/*.cljc' \
+ 'dev/test/**/*.clj' 'dev/test/**/*.cljc' \
'modules/drivers/**/test/**/*.clj' 'modules/drivers/**/test/**/*.cljc')
if [ -z "$FILES" ]; then
@@ -130,7 +231,14 @@ jobs:
exit 1
fi
- NS=$(sed -E 's#^(test/|enterprise/backend/test/|modules/drivers/[^/]+/test/)##; s#\.cljc?$##; s#/#.#g; s#_#-#g' <<<"$FILES" \
+ # Convert each test path to a Clojure namespace: strip its known test root and
+ # extension, then translate path separators and underscores. Keep these roots
+ # aligned with FILES above and the Eastwood classpath aliases below.
+ NS=$(sed -E \
+ -e 's#^(test/|enterprise/backend/test/|dev/test/|modules/drivers/[^/]+/test/)##' \
+ -e 's#\.cljc?$##' \
+ -e 's#/#.#g' \
+ -e 's#_#-#g' <<<"$FILES" \
| sort -u \
| paste -sd' ' -)
echo "Changed test namespaces: $NS"
@@ -139,7 +247,7 @@ jobs:
if: steps.changed.outputs.namespaces != ''
env:
NAMESPACES: ${{ steps.changed.outputs.namespaces }}
- run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood/test :namespaces "[$NAMESPACES]"
+ run: clojure -X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:eastwood/test :namespaces "[$NAMESPACES]"
be-tests:
if: ${{ !inputs.skip }}
@@ -368,6 +476,7 @@ jobs:
be-tests-result:
needs:
+ - project-tests
- be-linter-eastwood
- be-linter-eastwood-test
- be-linter-eastwood-test-changed
diff --git a/.github/workflows/build-scripts.yml b/.github/workflows/build-scripts.yml
index 2f87185af88..2b13c13d26b 100644
--- a/.github/workflows/build-scripts.yml
+++ b/.github/workflows/build-scripts.yml
@@ -79,6 +79,7 @@ jobs:
test-build-scripts:
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 40
steps:
- uses: actions/checkout@v6
- name: Prepare back-end environment
@@ -88,10 +89,6 @@ jobs:
- name: Run build and release script tests
run: clojure -X:dev:drivers:build:build-dev:build-test:ci
timeout-minutes: 15
- - name: Run Liquibase migrations linter tests
- run: clojure -M:test
- working-directory: bin/lint-migrations-file
- timeout-minutes: 15
- name: Run load-namespaces tests
run: clojure -M:test
working-directory: bin/load-namespaces
diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml
index f9d99c42a55..bc8d3f2abbf 100644
--- a/.github/workflows/cache-warm.yml
+++ b/.github/workflows/cache-warm.yml
@@ -118,7 +118,9 @@ jobs:
combos=(
":build:dev:ee:ee-dev:drivers:drivers-dev:cljs" # build + uberjar (combined classpath)
":cljs" # shadow-cljs (build:cljs); resolves :cljs alone -> covers asm:9.2
- ":dev:ee:ee-dev:drivers:drivers-dev:test:eastwood" # backend.yml (eastwood + eastwood/test)
+ ":dev:ee:ee-dev:drivers:drivers-dev:test:eastwood" # backend.yml (eastwood)
+ ":dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:eastwood/test" # backend.yml (eastwood tests)
+ ":dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" # run-tests.yml (project tests)
":dev:ci:ee:ee-dev:drivers:drivers-dev:test" # test-driver.yml (ee) -> covers data.json:2.4.0
":dev:ci:oss:oss-dev:drivers:drivers-dev:test" # test-driver.yml (oss)
":dev:ci:test:ee:ee-dev" # backend.yml matrix (ee)
@@ -140,6 +142,20 @@ jobs:
echo "::endgroup::"
done
+ # These commands use nested tools.deps projects; resolving the same alias from the
+ # repository root does not warm their dependencies.
+ nested_combos=(
+ "bin/lint-migrations-file|:test" # project migration checks
+ "bin/load-namespaces|:test" # build-scripts.yml
+ )
+ for spec in "${nested_combos[@]}"; do
+ project=${spec%%|*}
+ combo=${spec#*|}
+ echo "::group::${project}: clojure -P -M${combo}"
+ (cd "$project" && clojure -P -M"${combo}")
+ echo "::endgroup::"
+ done
+
echo "::group::./bin/mage"
./bin/mage
echo "::endgroup::"
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b508e7888a7..531899a8e0d 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -65,6 +65,9 @@ jobs:
embedding_sdk_host_sample_apps: ${{ steps.changes.outputs.embedding_sdk_host_sample_apps }}
embedding_sdk_ci: ${{ steps.changes.outputs.embedding_sdk_ci }}
custom_viz_all: ${{ steps.changes.outputs.custom_viz_all }}
+ project_backend_checks: ${{ steps.changes.outputs.project_backend_checks }}
+ project_migration_checks: ${{ steps.changes.outputs.project_migration_checks }}
+ project_ratchet_checks: ${{ steps.changes.outputs.project_ratchet_checks }}
no_source_code_changes: >-
${{ steps.changes.outputs.documentation == 'true'
&& steps.changes.outputs.backend_all == 'false'
@@ -215,6 +218,9 @@ jobs:
needs.files-changed.outputs.backend_all != 'true' &&
needs.static-viz-files-changed.outputs.static_viz != 'true' }}
force-run: ${{ needs.should-run.outputs.verdict == 'force-run' }}
+ run-project-backend-checks: ${{ needs.files-changed.outputs.project_backend_checks == 'true' }}
+ run-project-migration-checks: ${{ needs.files-changed.outputs.project_migration_checks == 'true' }}
+ run-project-ratchet-checks: ${{ needs.files-changed.outputs.project_ratchet_checks == 'true' }}
semantic-search-tests:
if: |
diff --git a/CLAUDE.md b/CLAUDE.md
index 561425517d8..75280e89f2b 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -77,6 +77,12 @@ It piggybacks on a running dev nREPL (~5s) and auto-spawns a JVM if none is runn
the four generated keys; structural changes it can't safely make (a new module needs a human `:team`, or
modules need reordering) are printed as `WARNING:` lines for you to resolve by hand.
+Run the repository-level module, ratchet, and migration checks with:
+
+```bash
+./bin/mage project-tests
+```
+
## Kondo Ignore Ratchets
`.clj-kondo/ratchets.edn` records, per linter, how many inline `:clj-kondo/ignore` forms the backend source
diff --git a/bb.edn b/bb.edn
index dc53048899f..9fdf426a0eb 100644
--- a/bb.edn
+++ b/bb.edn
@@ -276,6 +276,25 @@
:requires [[mage.modules]]
:task (task! (mage.modules/cli-fix-config (cli/parse! (current-task))))}
+ project-tests
+ {:doc "Run project-level backend and migration checks."
+ :examples [["./bin/mage project-tests" "Run backend and migration checks"]
+ ["./bin/mage project-tests backend" "Run backend checks"]
+ ["./bin/mage project-tests migrations" "Run migration checks"]
+ ["./bin/mage project-tests modules" "Run module checks"]
+ ["./bin/mage project-tests ratchets" "Run ratchet checks"]]
+ :arg-schema [:or
+ [:tuple]
+ [:tuple [:enum {:name "suite"}
+ "backend"
+ "migrations"
+ "modules"
+ "ratchets"]]]
+ :requires [[mage.project-tests :as project-tests]]
+ :task (task! (if-let [suite (first (:arguments parsed))]
+ (project-tests/run! suite)
+ (project-tests/run!)))}
+
fix-kondo-ratchets
{:doc "Lower the per-linter ignore budgets in .clj-kondo/ratchets.edn to match the source tree."
:examples [["./bin/mage fix-kondo-ratchets" "Ratchet budgets down (never raises one; over-budget linters are only warned about)"]
diff --git a/bin/test-agent b/bin/test-agent
index ef89788cded..e2db8aa06c9 100755
--- a/bin/test-agent
+++ b/bin/test-agent
@@ -2,6 +2,8 @@
# Agent-friendly test runner for Metabase backend (Clojure) tests.
#
# Differences from a normal `clj -X:dev:test` run:
+# - Puts dev/test on the test path (:dev/test), so the project-level checks under
+# dev/test/metabase/core are runnable with :only.
# - Uses HAWK_MODE=cli/ci so eftest.report.pretty is used instead of the
# progress-bar reporter. Failures print inline as they happen.
# - Strips ANSI escape codes from output so results are plain text.
@@ -43,9 +45,9 @@ for arg in "$@"; do
done
if $INCLUDE_OSS; then
- ALIASES=":dev:ci:test:oss:oss-dev"
+ ALIASES=":dev:dev/test:ci:test:oss:oss-dev"
else
- ALIASES=":dev:ee:ee-dev:test"
+ ALIASES=":dev:dev/test:ee:ee-dev:test"
fi
if $INCLUDE_DRIVERS; then
diff --git a/deps.edn b/deps.edn
index 258f24e21f1..839bbc3718b 100644
--- a/deps.edn
+++ b/deps.edn
@@ -362,6 +362,10 @@
;; The actual port is read from the running Jetty server via server.instance/server-port.
"-Dmb.jetty.port=0"]}
+ ;; Includes the small generator tests that validate development-only repository configuration.
+ :dev/test
+ {:extra-paths ["dev/test"]}
+
;; run the dev server with
;; clojure -M:run
;; clojure -M:run:drivers (include all drivers)
@@ -562,6 +566,7 @@
"test"
"enterprise/backend/src"
"enterprise/backend/test"
+ "dev/test/metabase"
"bin/build/src"
"bin/build/test"
"bin/lint-migrations-file/src"
@@ -698,7 +703,11 @@
;; Syntax error compiling fn* in Eastwood's analyzer
metabase.sql-tools.macaw.references-test
;; Method too large for Eastwood's analyzer
- metabase.sql-tools.sqlglot.references-test]}}
+ metabase.sql-tools.sqlglot.references-test
+ ;; dev/test namespaces that no longer load; nothing runs them
+ dev.deps-graph-test
+ dev.migrate-test
+ dev.model-tracking-test]}}
;; clojure -T:whitespace-linter
:whitespace-linter
diff --git a/dev/notes/model-boundary-enforcement.md b/dev/notes/model-boundary-enforcement.md
index 6edfb9e9995..e4d72f2b937 100644
--- a/dev/notes/model-boundary-enforcement.md
+++ b/dev/notes/model-boundary-enforcement.md
@@ -145,7 +145,7 @@ Use sparingly. Prefer explicit sets where feasible.
### Verifying changes
```bash
-./bin/test-agent :only '[metabase.core.modules-test]'
+./bin/mage project-tests modules
```
Or at the REPL:
diff --git a/dev/test/dev/deps_graph_test.clj b/dev/test/dev/deps_graph_test.clj
index 9f10f51d23a..5eca962e2eb 100644
--- a/dev/test/dev/deps_graph_test.clj
+++ b/dev/test/dev/deps_graph_test.clj
@@ -1,4 +1,4 @@
-(ns test.dev.deps-graph-test
+(ns dev.deps-graph-test
(:require
[clojure.test :refer :all]
[dev.deps-graph]
@@ -14,7 +14,7 @@
"(require '[malli.generator :as mg])" #{malli.generator}
"(require '[malli.generator])" #{malli.generator}
"(-> 'malli.generator requiring-resolve)" #{malli.generator}))
- ;; TODO: reader conditionals don't work properly
+;; TODO: reader conditionals don't work properly
; "(#?(:clj requiring-resolve :cljs resolve) 'malli.generator)" #{malli.generator}))
(deftest ^:parallel find-requires-test
diff --git a/dev/test/dev/n_plus_one_detection_test.clj b/dev/test/dev/n_plus_one_detection_test.clj
index 187df95d771..845724d6528 100644
--- a/dev/test/dev/n_plus_one_detection_test.clj
+++ b/dev/test/dev/n_plus_one_detection_test.clj
@@ -1,4 +1,4 @@
-(ns test.dev.n-plus-one-detection-test
+(ns dev.n-plus-one-detection-test
(:require [clojure.test :refer [deftest testing is]]
[dev]
[methodical.core :as methodical]
diff --git a/test/metabase/core/kondo_ratchet_test.clj b/dev/test/metabase/core/kondo_ratchet_test.clj
similarity index 100%
rename from test/metabase/core/kondo_ratchet_test.clj
rename to dev/test/metabase/core/kondo_ratchet_test.clj
diff --git a/test/metabase/core/modules_test.clj b/dev/test/metabase/core/modules_test.clj
similarity index 100%
rename from test/metabase/core/modules_test.clj
rename to dev/test/metabase/core/modules_test.clj
diff --git a/mage/src/mage/project_tests.clj b/mage/src/mage/project_tests.clj
new file mode 100644
index 00000000000..7825e7077ed
--- /dev/null
+++ b/mage/src/mage/project_tests.clj
@@ -0,0 +1,76 @@
+(ns mage.project-tests
+ (:refer-clojure :exclude [run!])
+ (:require
+ [clojure.string :as str]
+ [mage.shell :as shell]
+ [mage.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private module-check-namespaces
+ '[dev.modules-config-test
+ metabase.core.modules-test])
+
+(def ^:private ratchet-check-namespaces
+ '[metabase.core.kondo-ratchet-test])
+
+(def ^:private backend-check-namespaces
+ (vec (concat module-check-namespaces ratchet-check-namespaces)))
+
+(def ^:private default-suites
+ "Suites the bare `project-tests` command runs, in order."
+ ["migrations" "backend"])
+
+;; `sh` is a [[mage.shell/sh*]]-compatible function so unit tests can inspect commands without running them.
+;; .github/scripts/check-preresolve-aliases.sh reads the alias strings out of these two functions.
+
+(defn- run-migration-checks! [sh]
+ (sh {:dir (str u/project-root-directory "/bin/lint-migrations-file")}
+ "clojure" "-M:test"))
+
+(defn- run-clojure-checks! [sh namespaces]
+ (sh "clojure"
+ "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci"
+ ":only"
+ (pr-str namespaces)))
+
+(def ^:private suite-labels
+ {"backend" "backend checks"
+ "migrations" "migration checks"
+ "modules" "module checks"
+ "ratchets" "ratchet checks"})
+
+(defn- run-suite!
+ "Run one suite and return its exit code.
+ A command that cannot be started, or that `sh` times out, counts as a failure rather than aborting the run."
+ [sh suite]
+ (println "Running" (suite-labels suite))
+ (try
+ (:exit (case suite
+ "backend" (run-clojure-checks! sh backend-check-namespaces)
+ "migrations" (run-migration-checks! sh)
+ "modules" (run-clojure-checks! sh module-check-namespaces)
+ "ratchets" (run-clojure-checks! sh ratchet-check-namespaces)))
+ (catch Exception e
+ (println "Could not run" (suite-labels suite) "--" (ex-message e))
+ 1)))
+
+(defn run-suites!
+ "Run every suite in `suites` with `sh`, a [[mage.shell/sh*]]-compatible function, and return the names of
+ the suites that failed.
+ A failing suite does not stop the later ones.
+ `sh` streams each command's output as it runs; nothing is printed again afterwards."
+ [sh suites]
+ (let [failed (into [] (remove #(zero? (run-suite! sh %))) suites)]
+ (when (seq failed)
+ (println "Failed:" (str/join ", " (map suite-labels failed))))
+ failed))
+
+(defn run!
+ "Run the project-level checks: every default suite, or just `suite`.
+ Exits nonzero when any suite fails."
+ ([]
+ (run! nil))
+ ([suite]
+ (when (seq (run-suites! shell/sh* (if suite [suite] default-suites)))
+ (u/exit 1))))
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index b79aa8fc82f..ba3774e6eb8 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -15,6 +15,7 @@
[mage.kondo-ratchet-test]
[mage.merge-yaml-migrations-test :as merge-yaml-migrations-test]
[mage.modules-test]
+ [mage.project-tests-test]
[mage.shell-test]
[mage.token-scan-test]
[mage.util :as u]
@@ -27,6 +28,7 @@
mage.kondo-ratchet-test/keep-me
mage.util-test/keep-me
mage.modules-test/keep-me
+ mage.project-tests-test/keep-me
mage.shell-test/keep-me
merge-yaml-migrations-test/keep-me
token-scan-test/keep-me)
diff --git a/mage/test/mage/project_tests_test.clj b/mage/test/mage/project_tests_test.clj
new file mode 100644
index 00000000000..baac7d6bde4
--- /dev/null
+++ b/mage/test/mage/project_tests_test.clj
@@ -0,0 +1,86 @@
+(ns mage.project-tests-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [mage.project-tests :as project-tests]))
+
+;; Referenced by core_test.clj to ensure namespace is loaded
+(def keep-me :loaded)
+
+(set! *warn-on-reflection* true)
+
+(defn- fake-sh
+ "A [[mage.shell/sh*]] stand-in.
+ Records each command in `calls`, prints one line for it, and exits with the code in `exits` at that
+ call's position, or 0 past the end. An exception in `exits` is thrown instead, like a command that
+ never started."
+ [calls exits]
+ (fn [& args]
+ (let [args (if (map? (first args)) (rest args) args)
+ exit (get exits (count @calls) 0)]
+ (swap! calls conj (vec args))
+ (println "ran:" (str/join " " args))
+ (when (instance? Exception exit)
+ (throw exit))
+ {:exit exit, :out [], :err []})))
+
+(defn- run-suites!
+ "Run `suites` through [[project-tests/run-suites!]] with the given exit codes.
+ Returns the failed suites, the recorded commands, and everything printed."
+ [exits suites]
+ (let [calls (atom [])
+ out (java.io.StringWriter.)
+ failed (binding [*out* out]
+ (project-tests/run-suites! (fake-sh calls exits) suites))]
+ {:failed failed, :calls @calls, :out (str out)}))
+
+(defn- occurrences [s substring]
+ (count (re-seq (re-pattern (java.util.regex.Pattern/quote substring)) s)))
+
+(deftest first-suite-failure-still-runs-second-suite-test
+ (let [{:keys [failed calls out]} (run-suites! [1 0] ["migrations" "backend"])]
+ (testing "both suites run, in order"
+ (is (= [["clojure" "-M:test"]
+ ["clojure" "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" ":only"
+ "[dev.modules-config-test metabase.core.modules-test metabase.core.kondo-ratchet-test]"]]
+ calls)))
+ (testing "the failed suite is reported"
+ (is (= ["migrations"] failed))
+ (is (str/includes? out "Failed: migration checks")))
+ (testing "each command's output appears exactly once"
+ (is (= 1 (occurrences out "ran: clojure -M:test")))
+ (is (= 1 (occurrences out "ran: clojure -X:dev:dev/test"))))))
+
+(deftest first-suite-exception-still-runs-second-suite-test
+ (let [{:keys [failed calls out]} (run-suites! [(ex-info "clojure: command not found" {})]
+ ["migrations" "backend"])]
+ (testing "the second suite still runs"
+ (is (= 2 (count calls))))
+ (testing "the suite that could not run is reported once, as a failure"
+ (is (= ["migrations"] failed))
+ (is (= 1 (occurrences out "Could not run migration checks -- clojure: command not found")))
+ (is (str/includes? out "Failed: migration checks")))))
+
+(deftest second-suite-failure-test
+ (let [{:keys [failed calls]} (run-suites! [0 1] ["migrations" "backend"])]
+ (is (= 2 (count calls)))
+ (is (= ["backend"] failed))))
+
+(deftest both-suites-fail-test
+ (let [{:keys [failed out]} (run-suites! [1 1] ["migrations" "backend"])]
+ (is (= ["migrations" "backend"] failed))
+ (is (str/includes? out "Failed: migration checks, backend checks"))))
+
+(deftest all-suites-pass-test
+ (let [{:keys [failed out]} (run-suites! [] ["migrations" "backend"])]
+ (is (= [] failed))
+ (is (not (str/includes? out "Failed:")))))
+
+(deftest targeted-suites-test
+ (testing "modules and ratchets run only their own namespaces"
+ (is (= [["clojure" "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" ":only"
+ "[dev.modules-config-test metabase.core.modules-test]"]]
+ (:calls (run-suites! [] ["modules"]))))
+ (is (= [["clojure" "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" ":only"
+ "[metabase.core.kondo-ratchet-test]"]]
+ (:calls (run-suites! [] ["ratchets"]))))))