Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Introduce Metabase CI caching strategy (#64988)

Nemanja Glumac8a542dd80aca
diff --git a/.github/actions/get-cache-keys/action.yml b/.github/actions/get-cache-keys/action.yml
new file mode 100644
index 00000000000..a2cf8716fcd
--- /dev/null
+++ b/.github/actions/get-cache-keys/action.yml
@@ -0,0 +1,27 @@
+name: Get cache keys
+description: |
+ Get deterministic cache keys for Maven and Node.js caches based on the centrally controlled cache prefix,
+ and the suffix based on the runner OS and the current week of the year.
+
+outputs:
+ m2-cache-key:
+ description: "The cache key for the Maven cache."
+ value: ${{ steps.keys-factory.outputs.m2-cache-key }}
+ node-cache-key:
+ description: "The cache key for the Node.js cache."
+ value: ${{ steps.keys-factory.outputs.node-cache-key }}
+
+runs:
+ using: "composite"
+ steps:
+ - name: Generate and output cache keys
+ id: keys-factory
+ run: |
+ M2_CACHE_PREFIX='M2'
+ NODE_CACHE_PREFIX='node-modules'
+ WEEK_OF_YEAR=$(/bin/date "+%G-%V")
+ CACHE_SUFFIX="${{ runner.os }}-$WEEK_OF_YEAR"
+
+ echo "m2-cache-key=$M2_CACHE_PREFIX-$CACHE_SUFFIX" >> $GITHUB_OUTPUT
+ echo "node-cache-key=$NODE_CACHE_PREFIX-$CACHE_SUFFIX" >> $GITHUB_OUTPUT
+ shell: bash
diff --git a/.github/actions/update-cache/action.yml b/.github/actions/update-cache/action.yml
new file mode 100644
index 00000000000..de2e3e48d7a
--- /dev/null
+++ b/.github/actions/update-cache/action.yml
@@ -0,0 +1,41 @@
+name: Update cache
+description: |
+ Update a GitHub Actions cache based on a provided path and key.
+ If a cache entry already exists for the given key, it is first removed.
+ This ensures immutability of cache entries, and prevents runtime errors.
+
+inputs:
+ path:
+ description: "The path(s) to cache, separated by newlines."
+ required: true
+ key:
+ description: "The cache key to use."
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: Check if the cache entry exists
+ id: cache-lookup
+ uses: actions/cache/restore@v4
+ with:
+ path: ${{ inputs.path }}
+ key: ${{ inputs.key }}
+ lookup-only: true # Important bit: the action will not fetch anything, just check for the cache existence.
+ - name: Delete the cache entry if it exists
+ if: ${{ steps.cache-lookup.outputs.cache-hit == 'true' }}
+ run: |
+ echo "Cache entry found for key '${{ inputs.key }}'. Deleting it to ensure immutability."
+ gh api \
+ --method DELETE \
+ -H "Accept: application/vnd.github+json" \
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ /repos/${{ github.repository }}/actions/caches?key=${{ inputs.key }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ shell: bash
+ - name: Save the new cache entry
+ uses: actions/cache/save@v4
+ with:
+ path: ${{ inputs.path }}
+ key: ${{ inputs.key }}
diff --git a/.github/workflows/cache-generator.yml b/.github/workflows/cache-generator.yml
new file mode 100644
index 00000000000..2f09e7aa0b9
--- /dev/null
+++ b/.github/workflows/cache-generator.yml
@@ -0,0 +1,93 @@
+# This workflow is the single source of truth for generating and updating various caches used by our CI pipeline.
+# It sets the fresh cache each Monday but provides a manual trigger to update it on-demand.
+#
+# All other jobs only restore from this cache; *they never update it*!
+# They should always use `actions/cache/restore@v4` because `actions/cache@v4` would attempt to save the cache.
+#
+# Our agreed-upon strategy is to use a deterministic cache key based on the week of the year.
+# Jobs can utilize the prior week's cache if the current week's cache is not found.
+#
+#
+# PRINCIPLES:
+# - The cache is never wrong!
+# - Caches are immutable once created.
+# - Caches are for speeding up builds, not for correctness.
+# - It is faster to download a large cache than to re-resolve missing dependencies.
+#
+#
+# DESIGN:
+#
+# 1. Prepare
+# - A job first prepares the local environment (e.g., installs JDK, Clojure, Node.js, etc.)
+# - This fetches the dependencies from the cache (if it exists) and warms up the local cache for this workflow.
+# 2. Resolve
+# - The next step pre-resolves all dependencies, ensuring that the local cache is fully populated and up-to-date.
+# 3. Update
+# - The final step saves the local cache back to GitHub Actions cache with a deterministic key.
+# - If a cache entry already exists for that key, it is deleted first to ensure immutability.
+#
+name: Cache Generator
+
+on:
+ schedule:
+ - cron: '0 3 * * 1' # Mondays at 03:00 UTC
+ workflow_dispatch: # manually update the cache
+
+concurrency:
+ group: cache-generator
+ cancel-in-progress: true
+
+jobs:
+ get-cache-keys:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v4
+ - name: Get cache keys
+ uses: ./.github/actions/get-cache-keys
+ id: get-cache-keys
+ outputs:
+ m2-cache-key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}
+ node-cache-key: ${{ steps.get-cache-keys.outputs.node-cache-key }}
+
+ generate-m2-cache:
+ runs-on: ubuntu-22.04
+ timeout-minutes: 15
+ needs: get-cache-keys
+ env:
+ M2_CACHE_KEY: ${{ needs.get-cache-keys.outputs.m2-cache-key }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Prepare backend
+ uses: ./.github/actions/prepare-backend
+
+ # Add or remove aliases to affect the cache size and its contents.
+ # Adjust to taste, then redeploy this workflow.
+ - name: Pre-resolve dependencies
+ run: clojure -A:build:dev:ee:ee-dev:drivers:drivers-dev:cljs -P
+
+ - name: Update M2 cache
+ uses: ./.github/actions/update-cache
+ with:
+ path: |
+ ~/.m2
+ ~/.gitlibs
+ key: ${{ env.M2_CACHE_KEY }}
+
+ generate-node-modules-cache:
+ runs-on: ubuntu-22.04
+ timeout-minutes: 15
+ needs: get-cache-keys
+ env:
+ NODE_CACHE_KEY: ${{ needs.get-cache-keys.outputs.node-cache-key }}
+ steps:
+ - uses: actions/checkout@v4
+ # `prepare-frontend` action (1) prepares Node.js and (2) fetches the node_modules cache if it exists.
+ # It then (3) runs `yarn install --frozen-lockfile --prefer-offline` which ensures the local cache is fully populated.
+ - name: Install front-end dependencies
+ uses: ./.github/actions/prepare-frontend
+
+ - name: Update node_modules cache
+ uses: ./.github/actions/update-cache
+ with:
+ path: node_modules
+ key: ${{ env.NODE_CACHE_KEY }}