The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Introduce Metabase CI caching strategy (#64988)
Nemanja Glumac8a542dd80aca
diff --git a/.github/actions/get-cache-keys/action.yml b/.github/actions/get-cache-keys/action.ymlnew file mode 100644index 00000000000..a2cf8716fcd--- /dev/null+++ b/.github/actions/get-cache-keys/action.yml@@ -0,0 +1,27 @@+name: Get cache keys+description: |+ Get deterministic cache keys for Maven and Node.js caches based on the centrally controlled cache prefix,+ and the suffix based on the runner OS and the current week of the year.++outputs:+ m2-cache-key:+ description: "The cache key for the Maven cache."+ value: ${{ steps.keys-factory.outputs.m2-cache-key }}+ node-cache-key:+ description: "The cache key for the Node.js cache."+ value: ${{ steps.keys-factory.outputs.node-cache-key }}++runs:+ using: "composite"+ steps:+ - name: Generate and output cache keys+ id: keys-factory+ run: |+ M2_CACHE_PREFIX='M2'+ NODE_CACHE_PREFIX='node-modules'+ WEEK_OF_YEAR=$(/bin/date "+%G-%V")+ CACHE_SUFFIX="${{ runner.os }}-$WEEK_OF_YEAR"++ echo "m2-cache-key=$M2_CACHE_PREFIX-$CACHE_SUFFIX" >> $GITHUB_OUTPUT+ echo "node-cache-key=$NODE_CACHE_PREFIX-$CACHE_SUFFIX" >> $GITHUB_OUTPUT+ shell: bashdiff --git a/.github/actions/update-cache/action.yml b/.github/actions/update-cache/action.ymlnew file mode 100644index 00000000000..de2e3e48d7a--- /dev/null+++ b/.github/actions/update-cache/action.yml@@ -0,0 +1,41 @@+name: Update cache+description: |+ Update a GitHub Actions cache based on a provided path and key.+ If a cache entry already exists for the given key, it is first removed.+ This ensures immutability of cache entries, and prevents runtime errors.++inputs:+ path:+ description: "The path(s) to cache, separated by newlines."+ required: true+ key:+ description: "The cache key to use."+ required: true++runs:+ using: "composite"+ steps:+ - name: Check if the cache entry exists+ id: cache-lookup+ uses: actions/cache/restore@v4+ with:+ path: ${{ inputs.path }}+ key: ${{ inputs.key }}+ lookup-only: true # Important bit: the action will not fetch anything, just check for the cache existence.+ - name: Delete the cache entry if it exists+ if: ${{ steps.cache-lookup.outputs.cache-hit == 'true' }}+ run: |+ echo "Cache entry found for key '${{ inputs.key }}'. Deleting it to ensure immutability."+ gh api \+ --method DELETE \+ -H "Accept: application/vnd.github+json" \+ -H "X-GitHub-Api-Version: 2022-11-28" \+ /repos/${{ github.repository }}/actions/caches?key=${{ inputs.key }}+ env:+ GH_TOKEN: ${{ github.token }}+ shell: bash+ - name: Save the new cache entry+ uses: actions/cache/save@v4+ with:+ path: ${{ inputs.path }}+ key: ${{ inputs.key }}diff --git a/.github/workflows/cache-generator.yml b/.github/workflows/cache-generator.ymlnew file mode 100644index 00000000000..2f09e7aa0b9--- /dev/null+++ b/.github/workflows/cache-generator.yml@@ -0,0 +1,93 @@+# This workflow is the single source of truth for generating and updating various caches used by our CI pipeline.+# It sets the fresh cache each Monday but provides a manual trigger to update it on-demand.+#+# All other jobs only restore from this cache; *they never update it*!+# They should always use `actions/cache/restore@v4` because `actions/cache@v4` would attempt to save the cache.+#+# Our agreed-upon strategy is to use a deterministic cache key based on the week of the year.+# Jobs can utilize the prior week's cache if the current week's cache is not found.+#+#+# PRINCIPLES:+# - The cache is never wrong!+# - Caches are immutable once created.+# - Caches are for speeding up builds, not for correctness.+# - It is faster to download a large cache than to re-resolve missing dependencies.+#+#+# DESIGN:+#+# 1. Prepare+# - A job first prepares the local environment (e.g., installs JDK, Clojure, Node.js, etc.)+# - This fetches the dependencies from the cache (if it exists) and warms up the local cache for this workflow.+# 2. Resolve+# - The next step pre-resolves all dependencies, ensuring that the local cache is fully populated and up-to-date.+# 3. Update+# - The final step saves the local cache back to GitHub Actions cache with a deterministic key.+# - If a cache entry already exists for that key, it is deleted first to ensure immutability.+#+name: Cache Generator++on:+ schedule:+ - cron: '0 3 * * 1' # Mondays at 03:00 UTC+ workflow_dispatch: # manually update the cache++concurrency:+ group: cache-generator+ cancel-in-progress: true++jobs:+ get-cache-keys:+ runs-on: ubuntu-22.04+ steps:+ - uses: actions/checkout@v4+ - name: Get cache keys+ uses: ./.github/actions/get-cache-keys+ id: get-cache-keys+ outputs:+ m2-cache-key: ${{ steps.get-cache-keys.outputs.m2-cache-key }}+ node-cache-key: ${{ steps.get-cache-keys.outputs.node-cache-key }}++ generate-m2-cache:+ runs-on: ubuntu-22.04+ timeout-minutes: 15+ needs: get-cache-keys+ env:+ M2_CACHE_KEY: ${{ needs.get-cache-keys.outputs.m2-cache-key }}+ steps:+ - uses: actions/checkout@v4+ - name: Prepare backend+ uses: ./.github/actions/prepare-backend++ # Add or remove aliases to affect the cache size and its contents.+ # Adjust to taste, then redeploy this workflow.+ - name: Pre-resolve dependencies+ run: clojure -A:build:dev:ee:ee-dev:drivers:drivers-dev:cljs -P++ - name: Update M2 cache+ uses: ./.github/actions/update-cache+ with:+ path: |+ ~/.m2+ ~/.gitlibs+ key: ${{ env.M2_CACHE_KEY }}++ generate-node-modules-cache:+ runs-on: ubuntu-22.04+ timeout-minutes: 15+ needs: get-cache-keys+ env:+ NODE_CACHE_KEY: ${{ needs.get-cache-keys.outputs.node-cache-key }}+ steps:+ - uses: actions/checkout@v4+ # `prepare-frontend` action (1) prepares Node.js and (2) fetches the node_modules cache if it exists.+ # It then (3) runs `yarn install --frozen-lockfile --prefer-offline` which ensures the local cache is fully populated.+ - name: Install front-end dependencies+ uses: ./.github/actions/prepare-frontend++ - name: Update node_modules cache+ uses: ./.github/actions/update-cache+ with:+ path: node_modules+ key: ${{ env.NODE_CACHE_KEY }}