The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Use content-based cache keys for m2, bun, cypress, eslint. (#80219)
Ben Grabow9add75e67fa0
diff --git a/.github/actions/get-cache-keys/action.yml b/.github/actions/get-cache-keys/action.ymldeleted file mode 100644index 9f3644ec1c3..00000000000--- a/.github/actions/get-cache-keys/action.yml+++ /dev/null@@ -1,45 +0,0 @@-name: Get cache keys-description: |- Get deterministic cache keys for Maven and Node.js caches based on the centrally controlled cache prefix,- and the suffix based on the runner OS and the current week of the year.--outputs:- m2-cache-key:- description: "The cache key for the Maven cache."- value: ${{ steps.keys-factory.outputs.m2-cache-key }}- m2-restore-key:- description: "The fallback restore key prefix for the Maven cache."- value: ${{ steps.keys-factory.outputs.m2-restore-key }}- node-cache-key:- description: "The cache key for the Node.js cache."- value: ${{ steps.keys-factory.outputs.node-cache-key }}- node-restore-key:- description: "The fallback restore key prefix for the Node.js cache."- value: ${{ steps.keys-factory.outputs.node-restore-key }}- eslint-cache-key:- description: "The cache key for Eslint cache"- value: ${{ steps.keys-factory.outputs.eslint-cache-key }}- cypress-cache-key:- description: "The cache key for Cypress cache"- value: ${{ steps.keys-factory.outputs.cypress-cache-key }}--runs:- using: "composite"- steps:- - name: Generate and output cache keys- id: keys-factory- run: |- M2_CACHE_PREFIX='M2'- NODE_CACHE_PREFIX='node-modules'- ESLINT_CACHE_PREFIX='eslint'- CYPRESS_CACHE_PREFIX='cypress'- WEEK_OF_YEAR=$(/bin/date "+%G-%V")- CACHE_SUFFIX="${{ runner.os }}-$WEEK_OF_YEAR"-- echo "m2-cache-key=$M2_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"- echo "m2-restore-key=$M2_CACHE_PREFIX-" >> "$GITHUB_OUTPUT"- echo "node-cache-key=$NODE_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"- echo "node-restore-key=$NODE_CACHE_PREFIX-" >> "$GITHUB_OUTPUT"- echo "eslint-cache-key=$ESLINT_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"- echo "cypress-cache-key=$CYPRESS_CACHE_PREFIX-$CACHE_SUFFIX" >> "$GITHUB_OUTPUT"- shell: bashdiff --git a/.github/actions/prepare-backend/action.yml b/.github/actions/prepare-backend/action.ymlindex e83704741bc..423ef245fb8 100644--- a/.github/actions/prepare-backend/action.yml+++ b/.github/actions/prepare-backend/action.yml@@ -32,10 +32,6 @@ runs:exit 1fi- - name: Get M2 cache key- uses: ./.github/actions/get-cache-keys- id: get-cache-keys-- name: Install uv (for Python package installation)uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0with:@@ -43,14 +39,14 @@ runs:version: "0.10.2" # keep in sync with mise.tomlgithub-token: ${{ github.token }}+ - name: Cache spec+ id: cache-spec+ shell: bash+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"+- name: Restore M2 cacheuses: actions/cache/restore@v5with:- path: |- ~/.m2- ~/.gitlibs- ~/.deps.clj- bin/bb- key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}- restore-keys: |- ${{ steps.get-cache-keys.outputs.m2-restore-key }}+ path: ${{ steps.cache-spec.outputs.m2-path }}+ key: ${{ steps.cache-spec.outputs.m2-key }}+ restore-keys: ${{ steps.cache-spec.outputs.m2-restore-key }}diff --git a/.github/actions/prepare-cypress/action.yml b/.github/actions/prepare-cypress/action.ymlindex 71957a4b590..1f69a5216a1 100644--- a/.github/actions/prepare-cypress/action.yml+++ b/.github/actions/prepare-cypress/action.yml@@ -25,15 +25,17 @@ runs:chrome-version: 144id: setup-chrome- - name: Get cache keys- uses: ./.github/actions/get-cache-keys- id: get-cache-keys+ - name: Cache spec+ id: cache-spec+ shell: bash+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"- - name: Get Cypress cache+ - name: Restore Cypress binaryuses: actions/cache/restore@v5with:- path: ~/.cache/Cypress- key: ${{ steps.get-cache-keys.outputs.cypress-cache-key }}+ path: ${{ steps.cache-spec.outputs.cypress-path }}+ key: ${{ steps.cache-spec.outputs.cypress-key }}+ restore-keys: ${{ steps.cache-spec.outputs.cypress-restore-key }}- name: Ensure that Cypress executable is readyrun: |diff --git a/.github/actions/prepare-frontend/action.yml b/.github/actions/prepare-frontend/action.ymlindex c6156adaf14..20c08e587a5 100644--- a/.github/actions/prepare-frontend/action.yml+++ b/.github/actions/prepare-frontend/action.yml@@ -16,50 +16,38 @@ runs:with:node-version: ${{ inputs.node-version }}- - name: Get cache keys- uses: ./.github/actions/get-cache-keys- id: get-cache-keys-- - name: Restore M2 cache- uses: actions/cache/restore@v5- with:- path: |- ~/.m2- ~/.gitlibs- key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}- restore-keys: |- ${{ steps.get-cache-keys.outputs.m2-restore-key }}+ - name: Cache spec+ id: cache-spec+ shell: bash+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"- - name: Get bun cache directory- id: bun-cache- run: |- BUN_CACHE_DIR=$(bun pm cache)- echo "Bun cache directory: $BUN_CACHE_DIR"- echo "dir=$BUN_CACHE_DIR" >> "$GITHUB_OUTPUT"+ # The spec names the store path rather than discovering it, so that a job can restore the cache+ # before bun exists. Exported job-wide rather than per-step because every later bun invocation has+ # to resolve against the same store. Confirm bun agreed: if the variable stopped taking effect, bun+ # would fill its default directory and the cache would silently save an empty tree.+ - name: Point bun at the cached storeshell: bash+ env:+ STORE: ${{ steps.cache-spec.outputs.bun-store-path }}+ run: |+ echo "BUN_INSTALL_CACHE_DIR=$STORE" >> "$GITHUB_ENV"+ ACTUAL=$(BUN_INSTALL_CACHE_DIR="$STORE" bun pm cache)+ if [ "$ACTUAL" != "$STORE" ]; then+ echo "::error::bun reports its store at $ACTUAL, not $STORE; BUN_INSTALL_CACHE_DIR is not taking effect"+ exit 1+ fi- - name: Restore node_modules cache+ - name: Restore bun storeuses: actions/cache/restore@v5- id: node-modules-cachewith:- path: |- ${{ steps.bun-cache.outputs.dir }}- node_modules- key: ${{ steps.get-cache-keys.outputs.node-cache-key }}- restore-keys: |- ${{ steps.get-cache-keys.outputs.node-restore-key }}+ path: ${{ steps.cache-spec.outputs.bun-store-path }}+ key: ${{ steps.cache-spec.outputs.bun-store-key }}+ restore-keys: ${{ steps.cache-spec.outputs.bun-store-restore-key }}# postinstall lifecycle hook can be dangerous in CI so we only run it locally (see package.json)# If a job wants to patch packages, it does so explicitly in the CI. This setting is opt-out.- # For example, we never want to cache node_modules with patches already applied so cache-generator.yml skips that step.- - name: Clean install- # Remove the restored node_modules before installing. The os-week cache can- # carry a stale tree (for example a different React major), and- # `--frozen-lockfile` alone does not prune it. Rebuilding from the cached bun- # store gives an exact, lockfile-correct node_modules with no extra download.- run: |- rm -rf node_modules- bun install --frozen-lockfile+ - name: Install front-end dependencies+ run: bun install --frozen-lockfileshell: bash- name: Patch frontend dependenciesif: inputs.apply-patches == 'true'diff --git a/.github/actions/update-cache/action.yml b/.github/actions/update-cache/action.ymldeleted file mode 100644index 465a69bc069..00000000000--- a/.github/actions/update-cache/action.yml+++ /dev/null@@ -1,41 +0,0 @@-name: Update cache-description: |- Update a GitHub Actions cache based on a provided path and key.- If a cache entry already exists for the given key, it is first removed.- This ensures immutability of cache entries, and prevents runtime errors.--inputs:- path:- description: "The path(s) to cache, separated by newlines."- required: true- key:- description: "The cache key to use."- required: true--runs:- using: "composite"- steps:- - name: Check if the cache entry exists- id: cache-lookup- uses: actions/cache/restore@v5- with:- path: ${{ inputs.path }}- key: ${{ inputs.key }}- lookup-only: true # Important bit: the action will not fetch anything, just check for the cache existence.- - name: Delete the cache entry if it exists- if: ${{ steps.cache-lookup.outputs.cache-hit == 'true' }}- run: |- echo "Cache entry found for key '${{ inputs.key }}'. Deleting it to ensure immutability."- gh api \- --method DELETE \- -H "Accept: application/vnd.github+json" \- -H "X-GitHub-Api-Version: 2022-11-28" \- /repos/${{ github.repository }}/actions/caches?key=${{ inputs.key }}- env:- GH_TOKEN: ${{ github.token }}- shell: bash- - name: Save the new cache entry- uses: actions/cache/save@v5- with:- path: ${{ inputs.path }}- key: ${{ inputs.key }}diff --git a/.github/scripts/cache-keys.sh b/.github/scripts/cache-keys.shnew file mode 100755index 00000000000..5141f430b5f--- /dev/null+++ b/.github/scripts/cache-keys.sh@@ -0,0 +1,145 @@+#!/usr/bin/env bash+# Single definition of every CI dependency cache: what it holds, what identifies it, what it falls back to.+#+# A GitHub cache entry is identified by its key, a hash of the declared path list, and the ref that saved+# it. A step declaring a different path list than the one a cache was saved with computes a different+# identity and can never match it, whatever its key says. So paths and keys are emitted together here and+# every cache step takes both from this script.+#+# Emits `name=value` lines in $GITHUB_OUTPUT format:+#+# .github/scripts/cache-keys.sh >> "$GITHUB_OUTPUT"+#+# Run it with no redirect to see the whole scheme, or pass one name to print a single value:+#+# .github/scripts/cache-keys.sh+# .github/scripts/cache-keys.sh m2-key+#+# Hashes are computed here rather than with GitHub's hashFiles() so that a local run and a CI run of the+# same commit produce the same keys, and so that an empty match can fail loudly - a hashFiles() call that+# matches nothing silently yields "", collapsing the key onto its own restore prefix and pointing every+# dependency set at one entry.++set -euo pipefail++cd "$(git rev-parse --show-toplevel)"++OS="${RUNNER_OS:-$(uname -s)}"+[ "$OS" = "Darwin" ] && OS="macOS"++sha256() {+ if command -v sha256sum >/dev/null 2>&1; then+ sha256sum | cut -d' ' -f1+ else+ shasum -a 256 | cut -d' ' -f1+ fi+}++# Digest of a set of files: hash each one, then hash the concatenation, so the result depends on every+# file's contents and on which files exist.+digest() {+ local f+ if [ "$#" -eq 0 ]; then+ echo "::error::cache-keys.sh: no files matched; refusing to emit a key that would collide across dependency sets" >&2+ exit 1+ fi+ for f in "$@"; do+ [ -f "$f" ] || { echo "::error::cache-keys.sh: $f does not exist" >&2; exit 1; }+ sha256 < "$f"+ done | sha256+}++shopt -s nullglob+DEPS_FILES=(deps.edn modules/drivers/deps.edn modules/drivers/*/deps.edn)+shopt -u nullglob++DEPS_HASH="$(digest "${DEPS_FILES[@]}")"+LOCK_HASH="$(digest bun.lock)"++# The Cypress binary depends on the Cypress version alone, so keying it on the whole lockfile would+# throw away a 200MB download every time any frontend dependency moved.+#+# Reading a version out of a lockfile with a regex is only safe while exactly one line matches. Insist+# on that rather than taking the first of several: a second match means the format moved under us, and+# picking one at random would key the binary under a version it is not.+CYPRESS_VERSIONS="$(sed -n 's/.*"cypress": \["cypress@\([^"]*\)".*/\1/p' bun.lock)"+CYPRESS_MATCHES="$(printf '%s\n' "$CYPRESS_VERSIONS" | grep -c . || true)"+if [ "$CYPRESS_MATCHES" != "1" ]; then+ echo "::error::cache-keys.sh: expected exactly one resolved cypress version in bun.lock, found $CYPRESS_MATCHES" >&2+ exit 1+fi+CYPRESS_VERSION="$CYPRESS_VERSIONS"++# An incremental lint cache is only ever an accelerator, so it keys on the commit and always falls back+# to the newest entry on the prefix. The exact key never pre-exists, which keeps the entry current.+ESLINT_SHA="${GITHUB_SHA:-$(git rev-parse HEAD)}"++emit_multiline() {+ local name="$1"; shift+ printf '%s<<CACHE_KEYS_EOF\n' "$name"+ printf '%s\n' "$@"+ printf 'CACHE_KEYS_EOF\n'+}++# Paths under the home directory are emitted absolute rather than as `~/...`. actions/cache would expand+# a tilde, but a shell assigning one of these to a variable would not, and would go on to create a+# directory named `~`. Paths inside the checkout stay relative; actions/cache resolves those against the+# workspace.+spec() {+ emit_multiline m2-path "$HOME/.m2" "$HOME/.gitlibs" "$HOME/.deps.clj" 'bin/bb'+ echo "m2-key=M2-$OS-$DEPS_HASH"+ echo "m2-restore-key=M2-$OS-"++ # Pinned rather than discovered via `bun pm cache` so that the path is known before bun is installed;+ # prepare-frontend exports BUN_INSTALL_CACHE_DIR to match and fails if bun disagrees.+ echo "bun-store-path=$HOME/.bun/install/cache"+ echo "bun-store-key=bun-store-$OS-$LOCK_HASH"+ echo "bun-store-restore-key=bun-store-$OS-"++ echo "cypress-path=$HOME/.cache/Cypress"+ echo "cypress-key=cypress-$OS-$CYPRESS_VERSION"+ echo "cypress-restore-key=cypress-$OS-"++ echo "eslint-path=.eslintcache"+ echo "eslint-key=eslint-$OS-$ESLINT_SHA"+ echo "eslint-restore-key=eslint-$OS-"+}++# Print one output's value, including the multiline form, and fail on a name that does not exist. A+# caller that silently received "" would go on to build a key with a hole in it.+query() {+ local want="$1" line delim found=0 in_block=0+ while IFS= read -r line; do+ if [ "$in_block" = 1 ]; then+ if [ "$line" = "$delim" ]; then+ in_block=0+ else+ printf '%s\n' "$line"+ fi+ continue+ fi+ case "$line" in+ "$want="*)+ printf '%s\n' "${line#*=}"+ found=1+ ;;+ "$want<<"*)+ delim="${line#*<<}"+ in_block=1+ found=1+ ;;+ esac+ done <<EOF+$(spec)+EOF+ if [ "$found" = 0 ]; then+ echo "::error::cache-keys.sh: no output named '$want'" >&2+ exit 1+ fi+}++if [ "$#" -eq 0 ]; then+ spec+else+ query "$1"+fidiff --git a/.github/scripts/cache-keys.unit.spec.ts b/.github/scripts/cache-keys.unit.spec.tsnew file mode 100644index 00000000000..98b45392332--- /dev/null+++ b/.github/scripts/cache-keys.unit.spec.ts@@ -0,0 +1,80 @@+import { execFileSync } from "child_process";+import { join } from "path";++const SCRIPT = join(__dirname, "cache-keys.sh");++const CACHES = ["m2", "bun-store", "cypress", "eslint"];++// stderr is inherited by default, so the deliberate-failure case below would print a literal+// `::error::` line - which GitHub Actions reads as an annotation directive, making a passing run+// annotate itself with an error.+const run = (args: string[] = [], env: NodeJS.ProcessEnv = {}, quiet = false) =>+ execFileSync(SCRIPT, args, {+ encoding: "utf8",+ stdio: quiet ? ["ignore", "pipe", "ignore"] : "pipe",+ env: { ...process.env, RUNNER_OS: "Linux", GITHUB_SHA: "abc123", ...env },+ });++const get = (name: string, env: NodeJS.ProcessEnv = {}) =>+ run([name], env).trimEnd();++describe("cache-keys.sh", () => {+ it("is deterministic for a given tree", () => {+ expect(run()).toEqual(run());+ });++ it("emits every cache's path, key and restore key", () => {+ for (const cache of CACHES) {+ expect(get(`${cache}-path`)).not.toEqual("");+ expect(get(`${cache}-key`)).not.toEqual("");+ expect(get(`${cache}-restore-key`)).not.toEqual("");+ }+ });++ // Silently returning "" for a name that does not exist is how a caller ends up building a key with a+ // hole in it.+ it("fails on an unknown output name", () => {+ expect(() => run(["no-such-output"], {}, true)).toThrow();+ });++ // A key equal to its own restore prefix points every dependency set at one entry, and nothing reports+ // it. This is the failure the script exists to prevent.+ it("never emits a key equal to its own restore prefix", () => {+ for (const cache of CACHES) {+ const key = get(`${cache}-key`);+ const prefix = get(`${cache}-restore-key`);+ expect(key.startsWith(prefix)).toBe(true);+ expect(key.length).toBeGreaterThan(prefix.length);+ }+ });++ it("scopes every key to the runner OS", () => {+ for (const cache of CACHES) {+ expect(get(`${cache}-key`, { RUNNER_OS: "Linux" })).not.toEqual(+ get(`${cache}-key`, { RUNNER_OS: "macOS" }),+ );+ }+ });++ // Home paths must be absolute: actions/cache expands a tilde but a shell assigning one to a variable+ // does not, and would create a directory literally named `~`.+ it("emits absolute paths for home directory locations", () => {+ expect(get("bun-store-path").startsWith("/")).toBe(true);+ expect(get("cypress-path").startsWith("/")).toBe(true);+ for (const line of get("m2-path").split("\n")) {+ expect(line.startsWith("~")).toBe(false);+ }+ });++ // The Cypress binary depends on the Cypress version alone, so an unrelated dependency bump must not+ // invalidate a 200MB download.+ it("keys the Cypress binary on the resolved version, not the lockfile", () => {+ expect(get("cypress-key")).toMatch(/^cypress-Linux-\d+\.\d+\.\d+/);+ });++ it("keys the ESLint cache on the commit", () => {+ expect(get("eslint-key", { GITHUB_SHA: "deadbeef" })).toEqual(+ "eslint-Linux-deadbeef",+ );+ });+});diff --git a/.github/scripts/check-preresolve-aliases.sh b/.github/scripts/check-preresolve-aliases.shnew file mode 100755index 00000000000..b1982661c26--- /dev/null+++ b/.github/scripts/check-preresolve-aliases.sh@@ -0,0 +1,69 @@+#!/usr/bin/env bash+# Fails when a `clojure` invocation under .github/ uses an alias combination that cache-warm does not+# pre-resolve.+#+# tools.deps selects dependency versions globally per classpath, so two alias combinations can resolve+# the same transitive library to different versions. A combination the warm job never resolved is+# therefore not merely absent from the cache - the cache holds *a* version of those libraries, just not+# the one this classpath wants, and the job fetches its delta straight from Maven Central on every run.+# That is the intermittent-403 failure of DEV-2094, and it is invisible: the build still passes.+#+# Nothing about a missing combination shows up at runtime, so this check is the only thing standing+# between a newly added alias combination and a permanently half-cold cache.+#+# .github/scripts/check-preresolve-aliases.sh++set -euo pipefail++cd "$(git rev-parse --show-toplevel)"++WARM=.github/workflows/cache-warm.yml++# Invocations deliberately left out of the warm job. Each needs a reason, because the default answer is+# to add the combination to the list instead.+WAIVED="$(+ cat <<'EOF'+:deps+:outdated+EOF+)"+# :deps - tools.deps built-in used by write-poms.sh; contributes no dependencies of its own.+# :outdated - scheduled dependency-bump workflow only; a cold resolve there costs nothing anyone waits on.++# Alias combinations the warm job pre-resolves.+declared="$(sed -n '/combos=(/,/^ *)/p' "$WARM" | grep -oE '":[^"]*"' | tr -d '"' | sort -u)"+if [ -z "$declared" ]; then+ echo "::error::check-preresolve-aliases.sh: found no combos=( ... ) list in $WARM" >&2+ exit 1+fi++# Expressions we know how to expand. An unrecognised one has to stop the check rather than be skipped:+# silently ignoring it is how a whole matrix of combinations would go unchecked.+unknown="$(grep -rhoE '\$\{\{[^}]*\}\}' --include='*' \+ --exclude='*.png' --exclude='*.jpg' .github/ 2>/dev/null \+ | sort -u | grep -v 'edition' || true)"++scan() { # scan <edition>: alias strings used by clojure invocations, with expressions substituted+ grep -rhE 'clojure[[:space:]]+(-P[[:space:]]+)?-[XMATP]?:' \+ --include='*' --exclude='*.png' --exclude='*.jpg' .github/ 2>/dev/null \+ | sed "s/\\\${{[^}]*edition[^}]*}}/$1/g" \+ | grep -oE 'clojure[[:space:]]+(-P[[:space:]]+)?-[XMATP]?:[A-Za-z0-9:_.-]+' \+ | grep -oE ':[A-Za-z0-9:_.-]+' \+ | sed 's/:$//'+}++found="$( { scan ee; scan oss; } | sort -u )"++covered="$(printf '%s\n%s\n' "$declared" "$WAIVED" | sort -u)"+missing="$(comm -23 <(printf '%s\n' "$found") <(printf '%s\n' "$covered") || true)"++if [ -n "$missing" ]; then+ echo "::error::Alias combinations used under .github/ that cache-warm does not pre-resolve:" >&2+ printf '%s\n' "$missing" | sed 's/^/ /' >&2+ echo >&2+ echo "Add each to the combos list in $WARM, or waive it in this script with a reason." >&2+ echo "Leaving it uncovered means those jobs re-fetch their version deltas from Maven on every run." >&2+ exit 1+fi++echo "All $(printf '%s\n' "$found" | grep -c .) clojure alias combinations under .github/ are pre-resolved."diff --git a/.github/workflows/cache-generator.yml b/.github/workflows/cache-generator.ymldeleted file mode 100644index 0fe81577217..00000000000--- a/.github/workflows/cache-generator.yml+++ /dev/null@@ -1,185 +0,0 @@-# This workflow is the single source of truth for generating and updating various caches used by our CI pipeline.-# It sets the fresh cache each Monday but provides a manual trigger to update it on-demand.-#-# All other jobs only restore from this cache; *they never update it*!-# They should always use `actions/cache/restore@vX` because `actions/cache@vX` would attempt to save the cache.-#-# Our agreed-upon strategy is to use a deterministic cache key based on the week of the year.-# Jobs can utilize the prior week's cache if the current week's cache is not found.-#-#-# PRINCIPLES:-# - The cache is never wrong!-# - Caches are immutable once created.-# - Caches are for speeding up builds, not for correctness.-# - It is faster to download a large cache than to re-resolve missing dependencies.-#-#-# DESIGN:-#-# 1. Prepare-# - A job first prepares the local environment (e.g., installs JDK, Clojure, Node.js, etc.)-# - This fetches the dependencies from the cache (if it exists) and warms up the local cache for this workflow.-# 2. Resolve-# - The next step pre-resolves all dependencies, ensuring that the local cache is fully populated and up-to-date.-# 3. Update-# - The final step saves the local cache back to GitHub Actions cache with a deterministic key.-# - If a cache entry already exists for that key, it is deleted first to ensure immutability.-#-name: Cache Generator--on:- schedule:- - cron: "0 3 * * 1" # Mondays at 03:00 UTC- workflow_dispatch: # manually update the cache--concurrency:- group: cache-generator- cancel-in-progress: true--jobs:- get-cache-keys:- if: ${{ github.event_name != 'schedule' || github.repository == 'metabase/metabase' }}- runs-on: ${{ vars.SLIM_RUNNER_KEY }}- timeout-minutes: 5- steps:- - uses: actions/checkout@v6- - name: Get cache keys- uses: ./.github/actions/get-cache-keys- id: get-cache-keys- outputs:- m2-cache-key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}- node-cache-key: ${{ steps.get-cache-keys.outputs.node-cache-key }}- eslint-cache-key: ${{ steps.get-cache-keys.outputs.eslint-cache-key }}- cypress-cache-key: ${{ steps.get-cache-keys.outputs.cypress-cache-key }}-- generate-caches:- # Prevent a workflow_dispatch triggered from any branch besides master. This would overwrite the default- # cache keys but the cache would be innacessible to other branches since Github prevents cross-branch- # cache hits (except for the default branch), breaking the cache entirely.- if: ${{ github.ref_name == 'master' }}- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}- timeout-minutes: 30- needs: get-cache-keys- env:- M2_CACHE_KEY: ${{ needs.get-cache-keys.outputs.m2-cache-key }}- NODE_CACHE_KEY: ${{ needs.get-cache-keys.outputs.node-cache-key }}- CYPRESS_CACHE_KEY: ${{ needs.get-cache-keys.outputs.cypress-cache-key }}- ESLINT_CACHE_KEY: ${{ needs.get-cache-keys.outputs.eslint-cache-key }}- steps:- - uses: actions/checkout@v6-- # Backend preparation and cache- - name: Prepare backend- uses: ./.github/actions/prepare-backend-- # Pre-resolve every Clojure dependency classpath CI actually uses, so the- # saved cache is a SUPERSET of what each job resolves at runtime.- #- # Why a single `-P` is not enough: tools.deps selects dependency *versions*- # globally per classpath, so different alias combinations can resolve the- # same transitive lib to different versions (e.g. `org.clojure/data.json`- # 2.4.0 vs 2.5.x, `org.ow2.asm/asm` 9.2 vs newer). Resolving only one combo- # leaves the others to fetch their version-deltas straight from Maven- # Central / Clojars on every run -> intermittent 403s. See DEV-2094.- #- # Keep this list in sync with the `clojure` invocations across .github/.- # Each entry notes its consumer. We resolve the FULL alias strings (not a- # hand-reduced set) on purpose: if a currently dep-less alias later gains a- # dependency, this still covers it instead of silently drifting back into a- # cache gap.- - name: Pre-resolve Clojure dependencies- shell: bash- run: |- set -euo pipefail- combos=(- ":build:dev:ee:ee-dev:drivers:drivers-dev:cljs" # build + uberjar (combined classpath)- ":cljs" # shadow-cljs (build:cljs); resolves :cljs alone -> covers asm:9.2- ":dev:ee:ee-dev:drivers:drivers-dev:test:eastwood" # backend.yml (eastwood + eastwood/test)- ":dev:ci:ee:ee-dev:drivers:drivers-dev:test" # test-driver.yml (ee) -> covers data.json:2.4.0- ":dev:ci:oss:oss-dev:drivers:drivers-dev:test" # test-driver.yml (oss)- ":dev:ci:test:ee:ee-dev" # backend.yml matrix (ee)- ":dev:ci:test:oss:oss-dev" # backend.yml matrix (oss)- ":dev:ci:ee:ee-dev:test:cloverage" # backend-cloverage.yml- ":dev:drivers:drivers-dev:test" # reset-cloud-test-dataset.yml- ":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml- ":ee:drivers:load-namespaces" # backend.yml (load-namespaces)- ":ee:doc" # docs-generate-base.yml- ":kondo" # resolve-backport-conflicts.yml (replace-deps)- ":run:ee" # cross-branch-migration-test- ":dev:ee:migrate" # cross-branch-migration-test- ":whitespace-linter" # whitespace linter- )- for combo in "${combos[@]}"; do- echo "::group::clojure -P -A${combo}"- clojure -P -A"${combo}"- echo "::endgroup::"- done-- echo "::group::./bin/mage"- ./bin/mage- echo "::endgroup::"-- - name: Update M2 cache- uses: ./.github/actions/update-cache- with:- path: |- ~/.m2- ~/.gitlibs- ~/.deps.clj- bin/bb- key: ${{ env.M2_CACHE_KEY }}-- # Frontend preparation and caches- # `prepare-frontend` action (1) prepares Node.js and (2) fetches the node_modules cache if it exists.- # It then (3) runs `bun install --frozen-lockfile` which ensures the local cache is fully populated.- # IMPORTANT: Caching node_modules after patches are already applied will break any PR that attempts to either:- # - upgrade a patched library- # - change a patch- # Hence why we explicitly exclude patches from the cache (controlled by the `apply-patches` action input).- - name: Install front-end dependencies- uses: ./.github/actions/prepare-frontend- with:- apply-patches: "false"-- - name: Get bun cache directory- id: bun-cache- run: |- BUN_CACHE_DIR=$(bun pm cache)- echo "Bun cache directory: $BUN_CACHE_DIR"- echo "dir=$BUN_CACHE_DIR" >> "$GITHUB_OUTPUT"-- - name: Ensure that Cypress executable is ready- run: |- bun run cypress install- bun run cypress cache path- bun run cypress cache list- bun run cypress verify- shell: bash-- - name: Update node_modules cache- uses: ./.github/actions/update-cache- with:- path: |- ${{ steps.bun-cache.outputs.dir }}- node_modules- key: ${{ env.NODE_CACHE_KEY }}-- - name: Update Cypress cache- uses: ./.github/actions/update-cache- with:- path: ~/.cache/Cypress- key: ${{ env.CYPRESS_CACHE_KEY }}-- # ESLint cache (requires both backend and frontend)- - name: Compile CLJS- run: bun run build-pure:cljs-- - name: Run ESLint to populate cache- run: bun run lint-eslint-pure-- - name: Update eslint cache- uses: ./.github/actions/update-cache- with:- path: ${{ github.workspace}}/.eslintcache- key: ${{ env.ESLINT_CACHE_KEY }}diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.ymlnew file mode 100644index 00000000000..ee6e932dc45--- /dev/null+++ b/.github/workflows/cache-warm.yml@@ -0,0 +1,172 @@+# Writes the three large dependency caches. Every other job restores them and none of them writes.+#+# A single writer rather than the usual let-every-job-save arrangement, because a pull request here fans+# out to roughly 200 jobs, ~75 of which install Cypress. Letting each of those save would have them all+# build a 200MB archive and race for one reservation on every lockfile change, and all pull the Cypress+# binary at once on a version bump.+#+# The ESLint cache is the exception and saves from the lint job itself: it is under a megabyte and only a+# couple of jobs write it, so there is no herd to avoid.+#+# A cache is readable from the ref that saved it, from that ref's pull requests, and - for every ref -+# from the default branch. Warming on push to each merge target therefore serves the pull requests aimed+# at it, and master's copy backs up every ref that has none of its own.+#+# Keys are content-derived (see .github/scripts/cache-keys.sh), so a push whose dependencies already have+# an entry does nothing but three lookups. There is no cron: nothing here is keyed by date.+#+#+# PRINCIPLES:+# - The cache is never wrong!+# - Caches are immutable once created.+# - Caches are for speeding up builds, not for correctness.+# - It is faster to download a large cache than to re-resolve missing dependencies.++name: Cache Warm++on:+ push:+ branches:+ - master+ - "release-x.*.x"+ workflow_dispatch:++concurrency:+ group: cache-warm-${{ github.ref }}+ # Pushes to master land a median of 27 minutes apart and a full resolve takes longer, so cancelling in+ # progress would abort most warms before they save and a ref could stay cold indefinitely. Queueing+ # costs one wait instead: the queued run finds every key present and exits in about a minute.+ cancel-in-progress: false++jobs:+ warm:+ # Forks read the upstream default branch's caches and cannot usefully write their own, so warming+ # there would burn a runner for nothing.+ if: ${{ github.repository == 'metabase/metabase' }}+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 30+ steps:+ - uses: actions/checkout@v6++ - name: Cache spec+ id: cache-spec+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"++ # `lookup-only` with no restore-keys asks whether this exact dependency set is already cached.+ # Paths come from the spec, the same ones the save below declares - a lookup that named different+ # paths would compute a different cache identity and report a miss every time.+ - name: Look up M2 cache+ id: m2-lookup+ uses: actions/cache/restore@v5+ with:+ path: ${{ steps.cache-spec.outputs.m2-path }}+ key: ${{ steps.cache-spec.outputs.m2-key }}+ lookup-only: true+ - name: Look up bun store+ id: bun-lookup+ uses: actions/cache/restore@v5+ with:+ path: ${{ steps.cache-spec.outputs.bun-store-path }}+ key: ${{ steps.cache-spec.outputs.bun-store-key }}+ lookup-only: true+ - name: Look up Cypress binary+ id: cypress-lookup+ uses: actions/cache/restore@v5+ with:+ path: ${{ steps.cache-spec.outputs.cypress-path }}+ key: ${{ steps.cache-spec.outputs.cypress-key }}+ lookup-only: true++ - name: Prepare backend+ if: steps.m2-lookup.outputs.cache-hit != 'true'+ uses: ./.github/actions/prepare-backend++ # Pre-resolve every Clojure dependency classpath CI actually uses, so the saved cache is a SUPERSET+ # of what each job resolves at runtime.+ #+ # Why a single `-P` is not enough: tools.deps selects dependency *versions* globally per classpath,+ # so different alias combinations can resolve the same transitive lib to different versions (e.g.+ # `org.clojure/data.json` 2.4.0 vs 2.5.x, `org.ow2.asm/asm` 9.2 vs newer). Resolving only one combo+ # leaves the others to fetch their version-deltas straight from Maven Central / Clojars on every+ # run -> intermittent 403s. See DEV-2094.+ #+ # Keep this list in sync with the `clojure` invocations across .github/. Each entry notes its+ # consumer. We resolve the FULL alias strings (not a hand-reduced set) on purpose: if a currently+ # dep-less alias later gains a dependency, this still covers it instead of silently drifting back+ # into a cache gap.+ - name: Pre-resolve Clojure dependencies+ if: steps.m2-lookup.outputs.cache-hit != 'true'+ shell: bash+ run: |+ set -euo pipefail+ combos=(+ ":build:dev:ee:ee-dev:drivers:drivers-dev:cljs" # build + uberjar (combined classpath)+ ":cljs" # shadow-cljs (build:cljs); resolves :cljs alone -> covers asm:9.2+ ":dev:ee:ee-dev:drivers:drivers-dev:test:eastwood" # backend.yml (eastwood + eastwood/test)+ ":dev:ci:ee:ee-dev:drivers:drivers-dev:test" # test-driver.yml (ee) -> covers data.json:2.4.0+ ":dev:ci:oss:oss-dev:drivers:drivers-dev:test" # test-driver.yml (oss)+ ":dev:ci:test:ee:ee-dev" # backend.yml matrix (ee)+ ":dev:ci:test:oss:oss-dev" # backend.yml matrix (oss)+ ":dev:ci:ee:ee-dev:test:cloverage" # backend-cloverage.yml+ ":dev:drivers:drivers-dev:test" # reset-cloud-test-dataset.yml+ ":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml+ ":test" # build-scripts.yml (bin/build unit tests)+ ":ee:drivers:load-namespaces" # backend.yml (load-namespaces)+ ":ee:doc" # docs-generate-base.yml+ ":kondo" # resolve-backport-conflicts.yml (replace-deps)+ ":run:ee" # cross-branch-migration-test+ ":dev:ee:migrate" # cross-branch-migration-test+ ":whitespace-linter" # whitespace linter+ )+ for combo in "${combos[@]}"; do+ echo "::group::clojure -P -A${combo}"+ clojure -P -A"${combo}"+ echo "::endgroup::"+ done++ echo "::group::./bin/mage"+ ./bin/mage+ echo "::endgroup::"++ - name: Save M2 cache+ if: steps.m2-lookup.outputs.cache-hit != 'true'+ uses: actions/cache/save@v5+ with:+ path: ${{ steps.cache-spec.outputs.m2-path }}+ key: ${{ steps.cache-spec.outputs.m2-key }}++ # `prepare-frontend` installs bun, points it at the store path, and runs a frozen-lockfile install,+ # which is what fills the store. Patches are deliberately not applied: a store cached with patches+ # already in place breaks any pull request that changes or upgrades a patched dependency.+ #+ # This also runs when only the Cypress cache is missing, because `bun run cypress install` below+ # needs the node_modules this step creates. Narrowing the condition to the bun store alone would+ # leave that step with no Cypress binary to run.+ - name: Install front-end dependencies+ if: steps.bun-lookup.outputs.cache-hit != 'true' || steps.cypress-lookup.outputs.cache-hit != 'true'+ uses: ./.github/actions/prepare-frontend+ with:+ apply-patches: "false"++ - name: Save bun store+ if: steps.bun-lookup.outputs.cache-hit != 'true'+ uses: actions/cache/save@v5+ with:+ path: ${{ steps.cache-spec.outputs.bun-store-path }}+ key: ${{ steps.cache-spec.outputs.bun-store-key }}++ - name: Ensure that Cypress executable is ready+ if: steps.cypress-lookup.outputs.cache-hit != 'true'+ shell: bash+ run: |+ bun run cypress install+ bun run cypress cache path+ bun run cypress cache list+ bun run cypress verify++ - name: Save Cypress binary+ if: steps.cypress-lookup.outputs.cache-hit != 'true'+ uses: actions/cache/save@v5+ with:+ path: ${{ steps.cache-spec.outputs.cypress-path }}+ key: ${{ steps.cache-spec.outputs.cypress-key }}diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.ymlindex 8be01ad8616..552b43b6c9a 100644--- a/.github/workflows/frontend.yml+++ b/.github/workflows/frontend.yml@@ -50,14 +50,17 @@ jobs:- name: Prepare back-end environmentuses: ./.github/actions/prepare-backend- - name: Get Eslint cache key- uses: ./.github/actions/get-cache-keys- id: get-cache-keys- - name: Restore ESLint cache- uses: actions/cache/restore@v5+ - name: Cache spec+ id: cache-spec+ run: .github/scripts/cache-keys.sh | tee -a "$GITHUB_OUTPUT"+ # Small enough, and written by few enough jobs, that this one reads and writes its own cache+ # rather than waiting on cache-warm.+ - name: ESLint cache+ uses: actions/cache@v5with:- path: ${{ github.workspace}}/.eslintcache- key: ${{ steps.get-cache-keys.outputs.eslint-cache-key }}+ path: ${{ steps.cache-spec.outputs.eslint-path }}+ key: ${{ steps.cache-spec.outputs.eslint-key }}+ restore-keys: ${{ steps.cache-spec.outputs.eslint-restore-key }}- name: Compile CLJSrun: bun run build-pure:cljsdiff --git a/.github/workflows/lint.yml b/.github/workflows/lint.ymlindex f10c857b52c..98b28981422 100644--- a/.github/workflows/lint.yml+++ b/.github/workflows/lint.yml@@ -64,6 +64,12 @@ jobs:fi- parallel:+ # Deliberately ungated. This guards a coupling that drifts silently, so running it only when a+ # path filter says .github changed would reintroduce the failure it exists to catch - and the+ # filters here cover .github/workflows only, not the actions and scripts it also reads.+ - name: Check cache-warm pre-resolves every alias combination CI uses+ run: .github/scripts/check-preresolve-aliases.sh+# Lint whitespace, always runs- name: Run Whitespace Linterrun: clojure -T:whitespace-linter lint :whitespace-linter/progress-bar false