Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use content-based cache keys for m2, bun, cypress, eslint. (#80219)

Ben Grabow9add75e67fa0
diff --git a/.github/actions/get-cache-keys/action.yml b/.github/actions/get-cache-keys/action.yml
deleted file mode 100644
index 9f3644ec1c3..00000000000
--- a/.github/actions/get-cache-keys/action.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-name: Get cache keys
-description: |
- Get deterministic cache keys for Maven and Node.js caches based on the centrally controlled cache prefix,
- and the suffix based on the runner OS and the current week of the year.
-
-outputs:
- m2-cache-key:
- description: "The cache key for the Maven cache."
- value: ${{ steps.keys-factory.outputs.m2-cache-key }}
- m2-restore-key:
- description: "The fallback restore key prefix for the Maven cache."
- value: ${{ steps.keys-factory.outputs.m2-restore-key }}
- node-cache-key:
- description: "The cache key for the Node.js cache."
- value: ${{ steps.keys-factory.outputs.node-cache-key }}
- node-restore-key:
- description: "The fallback restore key prefix for the Node.js cache."
- value: ${{ steps.keys-factory.outputs.node-restore-key }}
- eslint-cache-key:
- description: "The cache key for Eslint cache"
- value: ${{ steps.keys-factory.outputs.eslint-cache-key }}
- cypress-cache-key:
- description: "The cache key for Cypress cache"
- value: ${{ steps.keys-factory.outputs.cypress-cache-key }}
-
-runs:
- using: "composite"
- steps:
- - name: Generate and output cache keys
- id: keys-factory
- run: |
- M2_CACHE_PREFIX='M2'
- NODE_CACHE_PREFIX='node-modules'
- ESLINT_CACHE_PREFIX='eslint'
- CYPRESS_CACHE_PREFIX='cypress'
- WEEK_OF_YEAR=$(/bin/date "+%G-%V")
- CACHE_SUFFIX="${{ runner.os }}-$WEEK_OF_YEAR"
-
- echo "m2-cache-key=$M2_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"
- echo "m2-restore-key=$M2_CACHE_PREFIX-" >> "$GITHUB_OUTPUT"
- echo "node-cache-key=$NODE_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"
- echo "node-restore-key=$NODE_CACHE_PREFIX-" >> "$GITHUB_OUTPUT"
- echo "eslint-cache-key=$ESLINT_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"
- echo "cypress-cache-key=$CYPRESS_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"
- shell: bash
diff --git a/.github/actions/prepare-backend/action.yml b/.github/actions/prepare-backend/action.yml
index e83704741bc..423ef245fb8 100644
--- a/.github/actions/prepare-backend/action.yml
+++ b/.github/actions/prepare-backend/action.yml
@@ -32,10 +32,6 @@ runs:
exit 1
fi
- - name: Get M2 cache key
- uses: ./.github/actions/get-cache-keys
- id: get-cache-keys
-
- name: Install uv (for Python package installation)
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
@@ -43,14 +39,14 @@ runs:
version: "0.10.2" # keep in sync with mise.toml
github-token: ${{ github.token }}
+ - name: Cache spec
+ id: cache-spec
+ shell: bash
+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"
+
- name: Restore M2 cache
uses: actions/cache/restore@v5
with:
- path: |
- ~/.m2
- ~/.gitlibs
- ~/.deps.clj
- bin/bb
- key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}
- restore-keys: |
- ${{ steps.get-cache-keys.outputs.m2-restore-key }}
+ path: ${{ steps.cache-spec.outputs.m2-path }}
+ key: ${{ steps.cache-spec.outputs.m2-key }}
+ restore-keys: ${{ steps.cache-spec.outputs.m2-restore-key }}
diff --git a/.github/actions/prepare-cypress/action.yml b/.github/actions/prepare-cypress/action.yml
index 71957a4b590..1f69a5216a1 100644
--- a/.github/actions/prepare-cypress/action.yml
+++ b/.github/actions/prepare-cypress/action.yml
@@ -25,15 +25,17 @@ runs:
chrome-version: 144
id: setup-chrome
- - name: Get cache keys
- uses: ./.github/actions/get-cache-keys
- id: get-cache-keys
+ - name: Cache spec
+ id: cache-spec
+ shell: bash
+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"
- - name: Get Cypress cache
+ - name: Restore Cypress binary
uses: actions/cache/restore@v5
with:
- path: ~/.cache/Cypress
- key: ${{ steps.get-cache-keys.outputs.cypress-cache-key }}
+ path: ${{ steps.cache-spec.outputs.cypress-path }}
+ key: ${{ steps.cache-spec.outputs.cypress-key }}
+ restore-keys: ${{ steps.cache-spec.outputs.cypress-restore-key }}
- name: Ensure that Cypress executable is ready
run: |
diff --git a/.github/actions/prepare-frontend/action.yml b/.github/actions/prepare-frontend/action.yml
index c6156adaf14..20c08e587a5 100644
--- a/.github/actions/prepare-frontend/action.yml
+++ b/.github/actions/prepare-frontend/action.yml
@@ -16,50 +16,38 @@ runs:
with:
node-version: ${{ inputs.node-version }}
- - name: Get cache keys
- uses: ./.github/actions/get-cache-keys
- id: get-cache-keys
-
- - name: Restore M2 cache
- uses: actions/cache/restore@v5
- with:
- path: |
- ~/.m2
- ~/.gitlibs
- key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}
- restore-keys: |
- ${{ steps.get-cache-keys.outputs.m2-restore-key }}
+ - name: Cache spec
+ id: cache-spec
+ shell: bash
+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"
- - name: Get bun cache directory
- id: bun-cache
- run: |
- BUN_CACHE_DIR=$(bun pm cache)
- echo "Bun cache directory: $BUN_CACHE_DIR"
- echo "dir=$BUN_CACHE_DIR" >> "$GITHUB_OUTPUT"
+ # The spec names the store path rather than discovering it, so that a job can restore the cache
+ # before bun exists. Exported job-wide rather than per-step because every later bun invocation has
+ # to resolve against the same store. Confirm bun agreed: if the variable stopped taking effect, bun
+ # would fill its default directory and the cache would silently save an empty tree.
+ - name: Point bun at the cached store
shell: bash
+ env:
+ STORE: ${{ steps.cache-spec.outputs.bun-store-path }}
+ run: |
+ echo "BUN_INSTALL_CACHE_DIR=$STORE" >> "$GITHUB_ENV"
+ ACTUAL=$(BUN_INSTALL_CACHE_DIR="$STORE" bun pm cache)
+ if [ "$ACTUAL" != "$STORE" ]; then
+ echo "::error::bun reports its store at $ACTUAL, not $STORE; BUN_INSTALL_CACHE_DIR is not taking effect"
+ exit 1
+ fi
- - name: Restore node_modules cache
+ - name: Restore bun store
uses: actions/cache/restore@v5
- id: node-modules-cache
with:
- path: |
- ${{ steps.bun-cache.outputs.dir }}
- node_modules
- key: ${{ steps.get-cache-keys.outputs.node-cache-key }}
- restore-keys: |
- ${{ steps.get-cache-keys.outputs.node-restore-key }}
+ path: ${{ steps.cache-spec.outputs.bun-store-path }}
+ key: ${{ steps.cache-spec.outputs.bun-store-key }}
+ restore-keys: ${{ steps.cache-spec.outputs.bun-store-restore-key }}
# postinstall lifecycle hook can be dangerous in CI so we only run it locally (see package.json)
# If a job wants to patch packages, it does so explicitly in the CI. This setting is opt-out.
- # For example, we never want to cache node_modules with patches already applied so cache-generator.yml skips that step.
- - name: Clean install
- # Remove the restored node_modules before installing. The os-week cache can
- # carry a stale tree (for example a different React major), and
- # `--frozen-lockfile` alone does not prune it. Rebuilding from the cached bun
- # store gives an exact, lockfile-correct node_modules with no extra download.
- run: |
- rm -rf node_modules
- bun install --frozen-lockfile
+ - name: Install front-end dependencies
+ run: bun install --frozen-lockfile
shell: bash
- name: Patch frontend dependencies
if: inputs.apply-patches == 'true'
diff --git a/.github/actions/update-cache/action.yml b/.github/actions/update-cache/action.yml
deleted file mode 100644
index 465a69bc069..00000000000
--- a/.github/actions/update-cache/action.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-name: Update cache
-description: |
- Update a GitHub Actions cache based on a provided path and key.
- If a cache entry already exists for the given key, it is first removed.
- This ensures immutability of cache entries, and prevents runtime errors.
-
-inputs:
- path:
- description: "The path(s) to cache, separated by newlines."
- required: true
- key:
- description: "The cache key to use."
- required: true
-
-runs:
- using: "composite"
- steps:
- - name: Check if the cache entry exists
- id: cache-lookup
- uses: actions/cache/restore@v5
- with:
- path: ${{ inputs.path }}
- key: ${{ inputs.key }}
- lookup-only: true # Important bit: the action will not fetch anything, just check for the cache existence.
- - name: Delete the cache entry if it exists
- if: ${{ steps.cache-lookup.outputs.cache-hit == 'true' }}
- run: |
- echo "Cache entry found for key '${{ inputs.key }}'. Deleting it to ensure immutability."
- gh api \
- --method DELETE \
- -H "Accept: application/vnd.github+json" \
- -H "X-GitHub-Api-Version: 2022-11-28" \
- /repos/${{ github.repository }}/actions/caches?key=${{ inputs.key }}
- env:
- GH_TOKEN: ${{ github.token }}
- shell: bash
- - name: Save the new cache entry
- uses: actions/cache/save@v5
- with:
- path: ${{ inputs.path }}
- key: ${{ inputs.key }}
diff --git a/.github/scripts/cache-keys.sh b/.github/scripts/cache-keys.sh
new file mode 100755
index 00000000000..5141f430b5f
--- /dev/null
+++ b/.github/scripts/cache-keys.sh
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+# Single definition of every CI dependency cache: what it holds, what identifies it, what it falls back to.
+#
+# A GitHub cache entry is identified by its key, a hash of the declared path list, and the ref that saved
+# it. A step declaring a different path list than the one a cache was saved with computes a different
+# identity and can never match it, whatever its key says. So paths and keys are emitted together here and
+# every cache step takes both from this script.
+#
+# Emits `name=value` lines in $GITHUB_OUTPUT format:
+#
+# .github/scripts/cache-keys.sh >> "$GITHUB_OUTPUT"
+#
+# Run it with no redirect to see the whole scheme, or pass one name to print a single value:
+#
+# .github/scripts/cache-keys.sh
+# .github/scripts/cache-keys.sh m2-key
+#
+# Hashes are computed here rather than with GitHub's hashFiles() so that a local run and a CI run of the
+# same commit produce the same keys, and so that an empty match can fail loudly - a hashFiles() call that
+# matches nothing silently yields "", collapsing the key onto its own restore prefix and pointing every
+# dependency set at one entry.
+
+set -euo pipefail
+
+cd "$(git rev-parse --show-toplevel)"
+
+OS="${RUNNER_OS:-$(uname -s)}"
+[ "$OS" = "Darwin" ] && OS="macOS"
+
+sha256() {
+ if command -v sha256sum >/dev/null 2>&1; then
+ sha256sum | cut -d' ' -f1
+ else
+ shasum -a 256 | cut -d' ' -f1
+ fi
+}
+
+# Digest of a set of files: hash each one, then hash the concatenation, so the result depends on every
+# file's contents and on which files exist.
+digest() {
+ local f
+ if [ "$#" -eq 0 ]; then
+ echo "::error::cache-keys.sh: no files matched; refusing to emit a key that would collide across dependency sets" >&2
+ exit 1
+ fi
+ for f in "$@"; do
+ [ -f "$f" ] || { echo "::error::cache-keys.sh: $f does not exist" >&2; exit 1; }
+ sha256 < "$f"
+ done | sha256
+}
+
+shopt -s nullglob
+DEPS_FILES=(deps.edn modules/drivers/deps.edn modules/drivers/*/deps.edn)
+shopt -u nullglob
+
+DEPS_HASH="$(digest "${DEPS_FILES[@]}")"
+LOCK_HASH="$(digest bun.lock)"
+
+# The Cypress binary depends on the Cypress version alone, so keying it on the whole lockfile would
+# throw away a 200MB download every time any frontend dependency moved.
+#
+# Reading a version out of a lockfile with a regex is only safe while exactly one line matches. Insist
+# on that rather than taking the first of several: a second match means the format moved under us, and
+# picking one at random would key the binary under a version it is not.
+CYPRESS_VERSIONS="$(sed -n 's/.*"cypress": \["cypress@\([^"]*\)".*/\1/p' bun.lock)"
+CYPRESS_MATCHES="$(printf '%s\n' "$CYPRESS_VERSIONS" | grep -c . || true)"
+if [ "$CYPRESS_MATCHES" != "1" ]; then
+ echo "::error::cache-keys.sh: expected exactly one resolved cypress version in bun.lock, found $CYPRESS_MATCHES" >&2
+ exit 1
+fi
+CYPRESS_VERSION="$CYPRESS_VERSIONS"
+
+# An incremental lint cache is only ever an accelerator, so it keys on the commit and always falls back
+# to the newest entry on the prefix. The exact key never pre-exists, which keeps the entry current.
+ESLINT_SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"
+
+emit_multiline() {
+ local name="$1"; shift
+ printf '%s<<CACHE_KEYS_EOF\n' "$name"
+ printf '%s\n' "$@"
+ printf 'CACHE_KEYS_EOF\n'
+}
+
+# Paths under the home directory are emitted absolute rather than as `~/...`. actions/cache would expand
+# a tilde, but a shell assigning one of these to a variable would not, and would go on to create a
+# directory named `~`. Paths inside the checkout stay relative; actions/cache resolves those against the
+# workspace.
+spec() {
+ emit_multiline m2-path "$HOME/.m2" "$HOME/.gitlibs" "$HOME/.deps.clj" 'bin/bb'
+ echo "m2-key=M2-$OS-$DEPS_HASH"
+ echo "m2-restore-key=M2-$OS-"
+
+ # Pinned rather than discovered via `bun pm cache` so that the path is known before bun is installed;
+ # prepare-frontend exports BUN_INSTALL_CACHE_DIR to match and fails if bun disagrees.
+ echo "bun-store-path=$HOME/.bun/install/cache"
+ echo "bun-store-key=bun-store-$OS-$LOCK_HASH"
+ echo "bun-store-restore-key=bun-store-$OS-"
+
+ echo "cypress-path=$HOME/.cache/Cypress"
+ echo "cypress-key=cypress-$OS-$CYPRESS_VERSION"
+ echo "cypress-restore-key=cypress-$OS-"
+
+ echo "eslint-path=.eslintcache"
+ echo "eslint-key=eslint-$OS-$ESLINT_SHA"
+ echo "eslint-restore-key=eslint-$OS-"
+}
+
+# Print one output's value, including the multiline form, and fail on a name that does not exist. A
+# caller that silently received "" would go on to build a key with a hole in it.
+query() {
+ local want="$1" line delim found=0 in_block=0
+ while IFS= read -r line; do
+ if [ "$in_block" = 1 ]; then
+ if [ "$line" = "$delim" ]; then
+ in_block=0
+ else
+ printf '%s\n' "$line"
+ fi
+ continue
+ fi
+ case "$line" in
+ "$want="*)
+ printf '%s\n' "${line#*=}"
+ found=1
+ ;;
+ "$want<<"*)
+ delim="${line#*<<}"
+ in_block=1
+ found=1
+ ;;
+ esac
+ done <<EOF
+$(spec)
+EOF
+ if [ "$found" = 0 ]; then
+ echo "::error::cache-keys.sh: no output named '$want'" >&2
+ exit 1
+ fi
+}
+
+if [ "$#" -eq 0 ]; then
+ spec
+else
+ query "$1"
+fi
diff --git a/.github/scripts/cache-keys.unit.spec.ts b/.github/scripts/cache-keys.unit.spec.ts
new file mode 100644
index 00000000000..98b45392332
--- /dev/null
+++ b/.github/scripts/cache-keys.unit.spec.ts
@@ -0,0 +1,80 @@
+import { execFileSync } from "child_process";
+import { join } from "path";
+
+const SCRIPT = join(__dirname, "cache-keys.sh");
+
+const CACHES = ["m2", "bun-store", "cypress", "eslint"];
+
+// stderr is inherited by default, so the deliberate-failure case below would print a literal
+// `::error::` line - which GitHub Actions reads as an annotation directive, making a passing run
+// annotate itself with an error.
+const run = (args: string[] = [], env: NodeJS.ProcessEnv = {}, quiet = false) =>
+ execFileSync(SCRIPT, args, {
+ encoding: "utf8",
+ stdio: quiet ? ["ignore", "pipe", "ignore"] : "pipe",
+ env: { ...process.env, RUNNER_OS: "Linux", GITHUB_SHA: "abc123", ...env },
+ });
+
+const get = (name: string, env: NodeJS.ProcessEnv = {}) =>
+ run([name], env).trimEnd();
+
+describe("cache-keys.sh", () => {
+ it("is deterministic for a given tree", () => {
+ expect(run()).toEqual(run());
+ });
+
+ it("emits every cache's path, key and restore key", () => {
+ for (const cache of CACHES) {
+ expect(get(`${cache}-path`)).not.toEqual("");
+ expect(get(`${cache}-key`)).not.toEqual("");
+ expect(get(`${cache}-restore-key`)).not.toEqual("");
+ }
+ });
+
+ // Silently returning "" for a name that does not exist is how a caller ends up building a key with a
+ // hole in it.
+ it("fails on an unknown output name", () => {
+ expect(() => run(["no-such-output"], {}, true)).toThrow();
+ });
+
+ // A key equal to its own restore prefix points every dependency set at one entry, and nothing reports
+ // it. This is the failure the script exists to prevent.
+ it("never emits a key equal to its own restore prefix", () => {
+ for (const cache of CACHES) {
+ const key = get(`${cache}-key`);
+ const prefix = get(`${cache}-restore-key`);
+ expect(key.startsWith(prefix)).toBe(true);
+ expect(key.length).toBeGreaterThan(prefix.length);
+ }
+ });
+
+ it("scopes every key to the runner OS", () => {
+ for (const cache of CACHES) {
+ expect(get(`${cache}-key`, { RUNNER_OS: "Linux" })).not.toEqual(
+ get(`${cache}-key`, { RUNNER_OS: "macOS" }),
+ );
+ }
+ });
+
+ // Home paths must be absolute: actions/cache expands a tilde but a shell assigning one to a variable
+ // does not, and would create a directory literally named `~`.
+ it("emits absolute paths for home directory locations", () => {
+ expect(get("bun-store-path").startsWith("/")).toBe(true);
+ expect(get("cypress-path").startsWith("/")).toBe(true);
+ for (const line of get("m2-path").split("\n")) {
+ expect(line.startsWith("~")).toBe(false);
+ }
+ });
+
+ // The Cypress binary depends on the Cypress version alone, so an unrelated dependency bump must not
+ // invalidate a 200MB download.
+ it("keys the Cypress binary on the resolved version, not the lockfile", () => {
+ expect(get("cypress-key")).toMatch(/^cypress-Linux-\d+\.\d+\.\d+/);
+ });
+
+ it("keys the ESLint cache on the commit", () => {
+ expect(get("eslint-key", { GITHUB_SHA: "deadbeef" })).toEqual(
+ "eslint-Linux-deadbeef",
+ );
+ });
+});
diff --git a/.github/scripts/check-preresolve-aliases.sh b/.github/scripts/check-preresolve-aliases.sh
new file mode 100755
index 00000000000..b1982661c26
--- /dev/null
+++ b/.github/scripts/check-preresolve-aliases.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Fails when a `clojure` invocation under .github/ uses an alias combination that cache-warm does not
+# pre-resolve.
+#
+# tools.deps selects dependency versions globally per classpath, so two alias combinations can resolve
+# the same transitive library to different versions. A combination the warm job never resolved is
+# therefore not merely absent from the cache - the cache holds *a* version of those libraries, just not
+# the one this classpath wants, and the job fetches its delta straight from Maven Central on every run.
+# That is the intermittent-403 failure of DEV-2094, and it is invisible: the build still passes.
+#
+# Nothing about a missing combination shows up at runtime, so this check is the only thing standing
+# between a newly added alias combination and a permanently half-cold cache.
+#
+# .github/scripts/check-preresolve-aliases.sh
+
+set -euo pipefail
+
+cd "$(git rev-parse --show-toplevel)"
+
+WARM=.github/workflows/cache-warm.yml
+
+# Invocations deliberately left out of the warm job. Each needs a reason, because the default answer is
+# to add the combination to the list instead.
+WAIVED="$(
+ cat <<'EOF'
+:deps
+:outdated
+EOF
+)"
+# :deps - tools.deps built-in used by write-poms.sh; contributes no dependencies of its own.
+# :outdated - scheduled dependency-bump workflow only; a cold resolve there costs nothing anyone waits on.
+
+# Alias combinations the warm job pre-resolves.
+declared="$(sed -n '/combos=(/,/^ *)/p' "$WARM" | grep -oE '":[^"]*"' | tr -d '"' | sort -u)"
+if [ -z "$declared" ]; then
+ echo "::error::check-preresolve-aliases.sh: found no combos=( ... ) list in $WARM" >&2
+ exit 1
+fi
+
+# Expressions we know how to expand. An unrecognised one has to stop the check rather than be skipped:
+# silently ignoring it is how a whole matrix of combinations would go unchecked.
+unknown="$(grep -rhoE '\$\{\{[^}]*\}\}' --include='*' \
+ --exclude='*.png' --exclude='*.jpg' .github/ 2>/dev/null \
+ | sort -u | grep -v 'edition' || true)"
+
+scan() { # scan <edition>: alias strings used by clojure invocations, with expressions substituted
+ grep -rhE 'clojure[[:space:]]+(-P[[:space:]]+)?-[XMATP]?:' \
+ --include='*' --exclude='*.png' --exclude='*.jpg' .github/ 2>/dev/null \
+ | sed "s/\\\${{[^}]*edition[^}]*}}/$1/g" \
+ | grep -oE 'clojure[[:space:]]+(-P[[:space:]]+)?-[XMATP]?:[A-Za-z0-9:_.-]+' \
+ | grep -oE ':[A-Za-z0-9:_.-]+' \
+ | sed 's/:$//'
+}
+
+found="$( { scan ee; scan oss; } | sort -u )"
+
+covered="$(printf '%s\n%s\n' "$declared" "$WAIVED" | sort -u)"
+missing="$(comm -23 <(printf '%s\n' "$found") <(printf '%s\n' "$covered") || true)"
+
+if [ -n "$missing" ]; then
+ echo "::error::Alias combinations used under .github/ that cache-warm does not pre-resolve:" >&2
+ printf '%s\n' "$missing" | sed 's/^/ /' >&2
+ echo >&2
+ echo "Add each to the combos list in $WARM, or waive it in this script with a reason." >&2
+ echo "Leaving it uncovered means those jobs re-fetch their version deltas from Maven on every run." >&2
+ exit 1
+fi
+
+echo "All $(printf '%s\n' "$found" | grep -c .) clojure alias combinations under .github/ are pre-resolved."
diff --git a/.github/workflows/cache-generator.yml b/.github/workflows/cache-generator.yml
deleted file mode 100644
index 0fe81577217..00000000000
--- a/.github/workflows/cache-generator.yml
+++ /dev/null
@@ -1,185 +0,0 @@
-# This workflow is the single source of truth for generating and updating various caches used by our CI pipeline.
-# It sets the fresh cache each Monday but provides a manual trigger to update it on-demand.
-#
-# All other jobs only restore from this cache; *they never update it*!
-# They should always use `actions/cache/restore@vX` because `actions/cache@vX` would attempt to save the cache.
-#
-# Our agreed-upon strategy is to use a deterministic cache key based on the week of the year.
-# Jobs can utilize the prior week's cache if the current week's cache is not found.
-#
-#
-# PRINCIPLES:
-# - The cache is never wrong!
-# - Caches are immutable once created.
-# - Caches are for speeding up builds, not for correctness.
-# - It is faster to download a large cache than to re-resolve missing dependencies.
-#
-#
-# DESIGN:
-#
-# 1. Prepare
-# - A job first prepares the local environment (e.g., installs JDK, Clojure, Node.js, etc.)
-# - This fetches the dependencies from the cache (if it exists) and warms up the local cache for this workflow.
-# 2. Resolve
-# - The next step pre-resolves all dependencies, ensuring that the local cache is fully populated and up-to-date.
-# 3. Update
-# - The final step saves the local cache back to GitHub Actions cache with a deterministic key.
-# - If a cache entry already exists for that key, it is deleted first to ensure immutability.
-#
-name: Cache Generator
-
-on:
- schedule:
- - cron: "0 3 * * 1" # Mondays at 03:00 UTC
- workflow_dispatch: # manually update the cache
-
-concurrency:
- group: cache-generator
- cancel-in-progress: true
-
-jobs:
- get-cache-keys:
- if: ${{ github.event_name != 'schedule' || github.repository == 'metabase/metabase' }}
- runs-on: ${{ vars.SLIM_RUNNER_KEY }}
- timeout-minutes: 5
- steps:
- - uses: actions/checkout@v6
- - name: Get cache keys
- uses: ./.github/actions/get-cache-keys
- id: get-cache-keys
- outputs:
- m2-cache-key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}
- node-cache-key: ${{ steps.get-cache-keys.outputs.node-cache-key }}
- eslint-cache-key: ${{ steps.get-cache-keys.outputs.eslint-cache-key }}
- cypress-cache-key: ${{ steps.get-cache-keys.outputs.cypress-cache-key }}
-
- generate-caches:
- # Prevent a workflow_dispatch triggered from any branch besides master. This would overwrite the default
- # cache keys but the cache would be innacessible to other branches since Github prevents cross-branch
- # cache hits (except for the default branch), breaking the cache entirely.
- if: ${{ github.ref_name == 'master' }}
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 30
- needs: get-cache-keys
- env:
- M2_CACHE_KEY: ${{ needs.get-cache-keys.outputs.m2-cache-key }}
- NODE_CACHE_KEY: ${{ needs.get-cache-keys.outputs.node-cache-key }}
- CYPRESS_CACHE_KEY: ${{ needs.get-cache-keys.outputs.cypress-cache-key }}
- ESLINT_CACHE_KEY: ${{ needs.get-cache-keys.outputs.eslint-cache-key }}
- steps:
- - uses: actions/checkout@v6
-
- # Backend preparation and cache
- - name: Prepare backend
- uses: ./.github/actions/prepare-backend
-
- # Pre-resolve every Clojure dependency classpath CI actually uses, so the
- # saved cache is a SUPERSET of what each job resolves at runtime.
- #
- # Why a single `-P` is not enough: tools.deps selects dependency *versions*
- # globally per classpath, so different alias combinations can resolve the
- # same transitive lib to different versions (e.g. `org.clojure/data.json`
- # 2.4.0 vs 2.5.x, `org.ow2.asm/asm` 9.2 vs newer). Resolving only one combo
- # leaves the others to fetch their version-deltas straight from Maven
- # Central / Clojars on every run -> intermittent 403s. See DEV-2094.
- #
- # Keep this list in sync with the `clojure` invocations across .github/.
- # Each entry notes its consumer. We resolve the FULL alias strings (not a
- # hand-reduced set) on purpose: if a currently dep-less alias later gains a
- # dependency, this still covers it instead of silently drifting back into a
- # cache gap.
- - name: Pre-resolve Clojure dependencies
- shell: bash
- run: |
- set -euo pipefail
- combos=(
- ":build:dev:ee:ee-dev:drivers:drivers-dev:cljs" # build + uberjar (combined classpath)
- ":cljs" # shadow-cljs (build:cljs); resolves :cljs alone -> covers asm:9.2
- ":dev:ee:ee-dev:drivers:drivers-dev:test:eastwood" # backend.yml (eastwood + eastwood/test)
- ":dev:ci:ee:ee-dev:drivers:drivers-dev:test" # test-driver.yml (ee) -> covers data.json:2.4.0
- ":dev:ci:oss:oss-dev:drivers:drivers-dev:test" # test-driver.yml (oss)
- ":dev:ci:test:ee:ee-dev" # backend.yml matrix (ee)
- ":dev:ci:test:oss:oss-dev" # backend.yml matrix (oss)
- ":dev:ci:ee:ee-dev:test:cloverage" # backend-cloverage.yml
- ":dev:drivers:drivers-dev:test" # reset-cloud-test-dataset.yml
- ":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml
- ":ee:drivers:load-namespaces" # backend.yml (load-namespaces)
- ":ee:doc" # docs-generate-base.yml
- ":kondo" # resolve-backport-conflicts.yml (replace-deps)
- ":run:ee" # cross-branch-migration-test
- ":dev:ee:migrate" # cross-branch-migration-test
- ":whitespace-linter" # whitespace linter
- )
- for combo in "${combos[@]}"; do
- echo "::group::clojure -P -A${combo}"
- clojure -P -A"${combo}"
- echo "::endgroup::"
- done
-
- echo "::group::./bin/mage"
- ./bin/mage
- echo "::endgroup::"
-
- - name: Update M2 cache
- uses: ./.github/actions/update-cache
- with:
- path: |
- ~/.m2
- ~/.gitlibs
- ~/.deps.clj
- bin/bb
- key: ${{ env.M2_CACHE_KEY }}
-
- # Frontend preparation and caches
- # `prepare-frontend` action (1) prepares Node.js and (2) fetches the node_modules cache if it exists.
- # It then (3) runs `bun install --frozen-lockfile` which ensures the local cache is fully populated.
- # IMPORTANT: Caching node_modules after patches are already applied will break any PR that attempts to either:
- # - upgrade a patched library
- # - change a patch
- # Hence why we explicitly exclude patches from the cache (controlled by the `apply-patches` action input).
- - name: Install front-end dependencies
- uses: ./.github/actions/prepare-frontend
- with:
- apply-patches: "false"
-
- - name: Get bun cache directory
- id: bun-cache
- run: |
- BUN_CACHE_DIR=$(bun pm cache)
- echo "Bun cache directory: $BUN_CACHE_DIR"
- echo "dir=$BUN_CACHE_DIR" >> "$GITHUB_OUTPUT"
-
- - name: Ensure that Cypress executable is ready
- run: |
- bun run cypress install
- bun run cypress cache path
- bun run cypress cache list
- bun run cypress verify
- shell: bash
-
- - name: Update node_modules cache
- uses: ./.github/actions/update-cache
- with:
- path: |
- ${{ steps.bun-cache.outputs.dir }}
- node_modules
- key: ${{ env.NODE_CACHE_KEY }}
-
- - name: Update Cypress cache
- uses: ./.github/actions/update-cache
- with:
- path: ~/.cache/Cypress
- key: ${{ env.CYPRESS_CACHE_KEY }}
-
- # ESLint cache (requires both backend and frontend)
- - name: Compile CLJS
- run: bun run build-pure:cljs
-
- - name: Run ESLint to populate cache
- run: bun run lint-eslint-pure
-
- - name: Update eslint cache
- uses: ./.github/actions/update-cache
- with:
- path: ${{ github.workspace}}/.eslintcache
- key: ${{ env.ESLINT_CACHE_KEY }}
diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml
new file mode 100644
index 00000000000..ee6e932dc45
--- /dev/null
+++ b/.github/workflows/cache-warm.yml
@@ -0,0 +1,172 @@
+# Writes the three large dependency caches. Every other job restores them and none of them writes.
+#
+# A single writer rather than the usual let-every-job-save arrangement, because a pull request here fans
+# out to roughly 200 jobs, ~75 of which install Cypress. Letting each of those save would have them all
+# build a 200MB archive and race for one reservation on every lockfile change, and all pull the Cypress
+# binary at once on a version bump.
+#
+# The ESLint cache is the exception and saves from the lint job itself: it is under a megabyte and only a
+# couple of jobs write it, so there is no herd to avoid.
+#
+# A cache is readable from the ref that saved it, from that ref's pull requests, and - for every ref -
+# from the default branch. Warming on push to each merge target therefore serves the pull requests aimed
+# at it, and master's copy backs up every ref that has none of its own.
+#
+# Keys are content-derived (see .github/scripts/cache-keys.sh), so a push whose dependencies already have
+# an entry does nothing but three lookups. There is no cron: nothing here is keyed by date.
+#
+#
+# PRINCIPLES:
+# - The cache is never wrong!
+# - Caches are immutable once created.
+# - Caches are for speeding up builds, not for correctness.
+# - It is faster to download a large cache than to re-resolve missing dependencies.
+
+name: Cache Warm
+
+on:
+ push:
+ branches:
+ - master
+ - "release-x.*.x"
+ workflow_dispatch:
+
+concurrency:
+ group: cache-warm-${{ github.ref }}
+ # Pushes to master land a median of 27 minutes apart and a full resolve takes longer, so cancelling in
+ # progress would abort most warms before they save and a ref could stay cold indefinitely. Queueing
+ # costs one wait instead: the queued run finds every key present and exits in about a minute.
+ cancel-in-progress: false
+
+jobs:
+ warm:
+ # Forks read the upstream default branch's caches and cannot usefully write their own, so warming
+ # there would burn a runner for nothing.
+ if: ${{ github.repository == 'metabase/metabase' }}
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Cache spec
+ id: cache-spec
+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"
+
+ # `lookup-only` with no restore-keys asks whether this exact dependency set is already cached.
+ # Paths come from the spec, the same ones the save below declares - a lookup that named different
+ # paths would compute a different cache identity and report a miss every time.
+ - name: Look up M2 cache
+ id: m2-lookup
+ uses: actions/cache/restore@v5
+ with:
+ path: ${{ steps.cache-spec.outputs.m2-path }}
+ key: ${{ steps.cache-spec.outputs.m2-key }}
+ lookup-only: true
+ - name: Look up bun store
+ id: bun-lookup
+ uses: actions/cache/restore@v5
+ with:
+ path: ${{ steps.cache-spec.outputs.bun-store-path }}
+ key: ${{ steps.cache-spec.outputs.bun-store-key }}
+ lookup-only: true
+ - name: Look up Cypress binary
+ id: cypress-lookup
+ uses: actions/cache/restore@v5
+ with:
+ path: ${{ steps.cache-spec.outputs.cypress-path }}
+ key: ${{ steps.cache-spec.outputs.cypress-key }}
+ lookup-only: true
+
+ - name: Prepare backend
+ if: steps.m2-lookup.outputs.cache-hit != 'true'
+ uses: ./.github/actions/prepare-backend
+
+ # Pre-resolve every Clojure dependency classpath CI actually uses, so the saved cache is a SUPERSET
+ # of what each job resolves at runtime.
+ #
+ # Why a single `-P` is not enough: tools.deps selects dependency *versions* globally per classpath,
+ # so different alias combinations can resolve the same transitive lib to different versions (e.g.
+ # `org.clojure/data.json` 2.4.0 vs 2.5.x, `org.ow2.asm/asm` 9.2 vs newer). Resolving only one combo
+ # leaves the others to fetch their version-deltas straight from Maven Central / Clojars on every
+ # run -> intermittent 403s. See DEV-2094.
+ #
+ # Keep this list in sync with the `clojure` invocations across .github/. Each entry notes its
+ # consumer. We resolve the FULL alias strings (not a hand-reduced set) on purpose: if a currently
+ # dep-less alias later gains a dependency, this still covers it instead of silently drifting back
+ # into a cache gap.
+ - name: Pre-resolve Clojure dependencies
+ if: steps.m2-lookup.outputs.cache-hit != 'true'
+ shell: bash
+ run: |
+ set -euo pipefail
+ combos=(
+ ":build:dev:ee:ee-dev:drivers:drivers-dev:cljs" # build + uberjar (combined classpath)
+ ":cljs" # shadow-cljs (build:cljs); resolves :cljs alone -> covers asm:9.2
+ ":dev:ee:ee-dev:drivers:drivers-dev:test:eastwood" # backend.yml (eastwood + eastwood/test)
+ ":dev:ci:ee:ee-dev:drivers:drivers-dev:test" # test-driver.yml (ee) -> covers data.json:2.4.0
+ ":dev:ci:oss:oss-dev:drivers:drivers-dev:test" # test-driver.yml (oss)
+ ":dev:ci:test:ee:ee-dev" # backend.yml matrix (ee)
+ ":dev:ci:test:oss:oss-dev" # backend.yml matrix (oss)
+ ":dev:ci:ee:ee-dev:test:cloverage" # backend-cloverage.yml
+ ":dev:drivers:drivers-dev:test" # reset-cloud-test-dataset.yml
+ ":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml
+ ":test" # build-scripts.yml (bin/build unit tests)
+ ":ee:drivers:load-namespaces" # backend.yml (load-namespaces)
+ ":ee:doc" # docs-generate-base.yml
+ ":kondo" # resolve-backport-conflicts.yml (replace-deps)
+ ":run:ee" # cross-branch-migration-test
+ ":dev:ee:migrate" # cross-branch-migration-test
+ ":whitespace-linter" # whitespace linter
+ )
+ for combo in "${combos[@]}"; do
+ echo "::group::clojure -P -A${combo}"
+ clojure -P -A"${combo}"
+ echo "::endgroup::"
+ done
+
+ echo "::group::./bin/mage"
+ ./bin/mage
+ echo "::endgroup::"
+
+ - name: Save M2 cache
+ if: steps.m2-lookup.outputs.cache-hit != 'true'
+ uses: actions/cache/save@v5
+ with:
+ path: ${{ steps.cache-spec.outputs.m2-path }}
+ key: ${{ steps.cache-spec.outputs.m2-key }}
+
+ # `prepare-frontend` installs bun, points it at the store path, and runs a frozen-lockfile install,
+ # which is what fills the store. Patches are deliberately not applied: a store cached with patches
+ # already in place breaks any pull request that changes or upgrades a patched dependency.
+ #
+ # This also runs when only the Cypress cache is missing, because `bun run cypress install` below
+ # needs the node_modules this step creates. Narrowing the condition to the bun store alone would
+ # leave that step with no Cypress binary to run.
+ - name: Install front-end dependencies
+ if: steps.bun-lookup.outputs.cache-hit != 'true' || steps.cypress-lookup.outputs.cache-hit != 'true'
+ uses: ./.github/actions/prepare-frontend
+ with:
+ apply-patches: "false"
+
+ - name: Save bun store
+ if: steps.bun-lookup.outputs.cache-hit != 'true'
+ uses: actions/cache/save@v5
+ with:
+ path: ${{ steps.cache-spec.outputs.bun-store-path }}
+ key: ${{ steps.cache-spec.outputs.bun-store-key }}
+
+ - name: Ensure that Cypress executable is ready
+ if: steps.cypress-lookup.outputs.cache-hit != 'true'
+ shell: bash
+ run: |
+ bun run cypress install
+ bun run cypress cache path
+ bun run cypress cache list
+ bun run cypress verify
+
+ - name: Save Cypress binary
+ if: steps.cypress-lookup.outputs.cache-hit != 'true'
+ uses: actions/cache/save@v5
+ with:
+ path: ${{ steps.cache-spec.outputs.cypress-path }}
+ key: ${{ steps.cache-spec.outputs.cypress-key }}
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index 8be01ad8616..552b43b6c9a 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -50,14 +50,17 @@ jobs:
- name: Prepare back-end environment
uses: ./.github/actions/prepare-backend
- - name: Get Eslint cache key
- uses: ./.github/actions/get-cache-keys
- id: get-cache-keys
- - name: Restore ESLint cache
- uses: actions/cache/restore@v5
+ - name: Cache spec
+ id: cache-spec
+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"
+ # Small enough, and written by few enough jobs, that this one reads and writes its own cache
+ # rather than waiting on cache-warm.
+ - name: ESLint cache
+ uses: actions/cache@v5
with:
- path: ${{ github.workspace}}/.eslintcache
- key: ${{ steps.get-cache-keys.outputs.eslint-cache-key }}
+ path: ${{ steps.cache-spec.outputs.eslint-path }}
+ key: ${{ steps.cache-spec.outputs.eslint-key }}
+ restore-keys: ${{ steps.cache-spec.outputs.eslint-restore-key }}
- name: Compile CLJS
run: bun run build-pure:cljs
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index f10c857b52c..98b28981422 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -64,6 +64,12 @@ jobs:
fi
- parallel:
+ # Deliberately ungated. This guards a coupling that drifts silently, so running it only when a
+ # path filter says .github changed would reintroduce the failure it exists to catch - and the
+ # filters here cover .github/workflows only, not the actions and scripts it also reads.
+ - name: Check cache-warm pre-resolves every alias combination CI uses
+ run: .github/scripts/check-preresolve-aliases.sh
+
# Lint whitespace, always runs
- name: Run Whitespace Linter
run: clojure -T:whitespace-linter lint :whitespace-linter/progress-bar false