Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

build: cache Go build and module dirs to speed up CI

Nick Craig-Wood8b981d8a6957
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b1792fa32..9dacf0ebf 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -97,16 +97,58 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
+ - name: Get runner parameters
+ id: get-runner-parameters
+ run: |
+ echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
+ echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
+
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Go
+ id: setup-go
uses: actions/setup-go@v6
with:
go-version: ${{ matrix.go }}
check-latest: true
+ # Caching is handled explicitly below. setup-go's built-in cache
+ # keys only on go.sum, so all matrix jobs collide on one key and
+ # only the first to finish saves - which loses the cross-compile
+ # build cache.
+ cache: false
+
+ - name: Get Go paths
+ id: go-paths
+ run: |
+ echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+ echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
+
+ # The module cache holds downloaded source and is identical across
+ # jobs (it depends only on go.sum, not on OS, arch or Go version), so
+ # it is shared via a single key to avoid storing the same ~600 MiB in
+ # every job's cache and overflowing the repo cache limit.
+ - name: Module cache
+ uses: actions/cache@v5
+ with:
+ path: ${{ steps.go-paths.outputs.mod-cache }}
+ key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
+ restore-keys: |
+ go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
+ go-mod-
+
+ # The build cache holds compiled artifacts which are specific to the
+ # OS, arch and Go version, so it is kept per job.
+ - name: Build cache
+ uses: actions/cache@v5
+ with:
+ path: ${{ steps.go-paths.outputs.build-cache }}
+ key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
+ restore-keys: |
+ go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
+ go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
- name: Set environment variables
run: |
@@ -205,6 +247,15 @@ jobs:
# Deploy binaries if enabled in config && not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
+ - name: Trim module cache before saving
+ if: always()
+ run: |
+ # The downloaded module .zip archives are not needed to build from
+ # the already-extracted module cache, so drop them before the cache
+ # is saved - this roughly halves the module cache size.
+ modcache=$(go env GOMODCACHE | tr '\\' '/')
+ find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
+
lint:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -231,11 +282,19 @@ jobs:
check-latest: true
cache: false
+ - name: Module cache
+ uses: actions/cache@v5
+ with:
+ path: ~/go/pkg/mod
+ key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
+ restore-keys: |
+ go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
+ go-mod-
+
- name: Cache
uses: actions/cache@v5
with:
path: |
- ~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
@@ -302,6 +361,15 @@ jobs:
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
if: github.event_name == 'pull_request'
+ - name: Trim module cache before saving
+ if: always()
+ run: |
+ # The downloaded module .zip archives are not needed to build from
+ # the already-extracted module cache, so drop them before the cache
+ # is saved - this roughly halves the module cache size.
+ modcache=$(go env GOMODCACHE | tr '\\' '/')
+ find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
+
android:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -309,6 +377,12 @@ jobs:
runs-on: ubuntu-latest
steps:
+ - name: Get runner parameters
+ id: get-runner-parameters
+ run: |
+ echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
+ echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
+
- name: Checkout
uses: actions/checkout@v6
with:
@@ -316,9 +390,37 @@ jobs:
# Upgrade together with NDK version
- name: Set up Go
+ id: setup-go
uses: actions/setup-go@v6
with:
go-version: '~1.26.3'
+ # Caching is handled explicitly below to share the module cache
+ # with the other jobs - see the build job for the rationale.
+ cache: false
+
+ - name: Get Go paths
+ id: go-paths
+ run: |
+ echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+ echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
+
+ - name: Module cache
+ uses: actions/cache@v5
+ with:
+ path: ${{ steps.go-paths.outputs.mod-cache }}
+ key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
+ restore-keys: |
+ go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
+ go-mod-
+
+ - name: Build cache
+ uses: actions/cache@v5
+ with:
+ path: ${{ steps.go-paths.outputs.build-cache }}
+ key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
+ restore-keys: |
+ go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
+ go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
- name: Set global environment variables
run: |
@@ -394,3 +496,12 @@ jobs:
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
# Upload artifacts if not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
+
+ - name: Trim module cache before saving
+ if: always()
+ run: |
+ # The downloaded module .zip archives are not needed to build from
+ # the already-extracted module cache, so drop them before the cache
+ # is saved - this roughly halves the module cache size.
+ modcache=$(go env GOMODCACHE | tr '\\' '/')
+ find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true