Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf(ci): restore the dev shell in about 20 s on an empty cache volume (#9858)

teoa2bf842ab72e
diff --git a/.github/actions/setup-nix-dev-shell/action.yml b/.github/actions/setup-nix-dev-shell/action.yml
index 5bbd3ed4f5..be51e659cf 100644
--- a/.github/actions/setup-nix-dev-shell/action.yml
+++ b/.github/actions/setup-nix-dev-shell/action.yml
@@ -27,3 +27,7 @@ runs:
echo "BASH_ENV=$dev_env" >> "$GITHUB_ENV"
source "$dev_env"
printf '%s\n' "$PATH" | tr ':' '\n' >> "$GITHUB_PATH"
+ if [ "${NIX_SEED_SAVE:-}" = true ]; then
+ echo "NIX_SEED_SAVE=false" >> "$GITHUB_ENV"
+ .github/actions/setup-nix/nix_seed.sh save || echo "::warning::Could not save the Nix seed"
+ fi
diff --git a/.github/actions/setup-nix/action.yml b/.github/actions/setup-nix/action.yml
index 419ca489e9..3c261a0a12 100644
--- a/.github/actions/setup-nix/action.yml
+++ b/.github/actions/setup-nix/action.yml
@@ -30,6 +30,13 @@ inputs:
description: AWS secret access key for reading the private cache bucket.
required: false
default: ''
+ seed-cache:
+ description: >-
+ Namespace cache name holding a copy of /nix with the dev shell, restored
+ when the volume is empty. Its writers are PR jobs, so only PR check jobs
+ pass this, never jobs that deploy or hold deploy secrets. Empty disables.
+ required: false
+ default: ''
runs:
using: composite
@@ -41,6 +48,8 @@ runs:
NIX_CACHE_PUBLIC_KEY: ${{ inputs.nix-cache-public-key }}
NIX_CACHE_AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
NIX_CACHE_AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+ NIX_SEED_CACHE: ${{ inputs.seed-cache }}
+ NIX_SEED_KEY: ${{ hashFiles('flake.lock', 'flake.nix', 'nix/**', 'rust-toolchain.toml') }}
run: |
set -euo pipefail
@@ -120,6 +129,10 @@ runs:
fi
}
+ if [[ ! -x "$nix_bin" && -n "$NIX_SEED_CACHE" ]]; then
+ "$GITHUB_ACTION_PATH/nix_seed.sh" restore || echo "No Nix seed restored."
+ fi
+
if [[ -x "$nix_bin" ]]; then
# Store + profiles persist on the cache volume, but config, build
# users, and the daemon live outside /nix and are gone on a fresh
diff --git a/.github/actions/setup-nix/nix_seed.sh b/.github/actions/setup-nix/nix_seed.sh
new file mode 100755
index 0000000000..108210d289
--- /dev/null
+++ b/.github/actions/setup-nix/nix_seed.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Keep a copy of /nix with the dev shell in Namespace's HTTP cache. A job on an empty
+# volume restores it instead of installing Nix and downloading the shell, which takes
+# about a minute longer and depends on cache.nixos.org. It is split into shards because
+# one stream from the cache varies between 70 and 190 MB/s. About one transfer in three
+# stalls partway, so a restore that runs over restore_limit falls back to the install.
+
+action="$1"
+shards=8
+restore_limit=40
+
+credentials="$(mktemp "$RUNNER_TEMP/nix-seed.XXXXXX")"
+work="$(mktemp -d "$RUNNER_TEMP/nix-seed-shards.XXXXXX")"
+trap 'rm -rf "$credentials" "$work"' EXIT
+
+# Forks have no remote cache credentials.
+nsc cache sccache setup --cache_name "$NIX_SEED_CACHE" > "$credentials"
+endpoint="$(sed -n 's/^SCCACHE_WEBDAV_ENDPOINT=//p' "$credentials")"
+token="$(sed -n 's/^SCCACHE_WEBDAV_TOKEN=//p' "$credentials")"
+echo "::add-mask::$token"
+
+base="$endpoint/nix-seed/$NIX_SEED_KEY"
+
+# The token goes through a config descriptor so it stays out of curl's arguments.
+cache_request() {
+ curl --fail --silent --show-error --speed-limit 1048576 --speed-time 20 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$token") "$@"
+}
+
+wait_all() {
+ local failed=0 pid
+ for pid in "$@"; do
+ wait "$pid" || failed=1
+ done
+
+ return "$failed"
+}
+
+restore() {
+ local pids=() shard
+ for shard in $(seq 0 $((shards - 1))); do
+ (cache_request --max-time "$restore_limit" "$base/$shard" | zstd -d -T2 | sudo tar -xf - -C /) &
+ pids+=("$!")
+ done
+
+ wait_all "${pids[@]}"
+}
+
+# Largest entries first, dealt in turn, so the shards come out about the same size.
+# .links only holds hard links to files that are in the store anyway.
+split_store() {
+ sudo find /nix /nix/store -mindepth 1 -maxdepth 1 ! -path /nix/store ! -path /nix/store/.links -print0 \
+ | sudo xargs -0 du -s \
+ | sort -rn \
+ | cut -f2- \
+ | sed 's#^/##' \
+ | awk -v shards="$shards" -v work="$work" '{ print > (work "/" (NR % shards)) }'
+}
+
+save() {
+ local pids=() shard
+ split_store
+
+ for shard in $(seq 0 $((shards - 1))); do
+ (
+ sudo tar -C / -cf - -T "$work/$shard" | zstd -T2 -3 -q -o "$work/$shard.tar.zst"
+ cache_request --upload-file "$work/$shard.tar.zst" "$base/$shard"
+ ) &
+ pids+=("$!")
+ done
+
+ wait_all "${pids[@]}"
+}
+
+case "$action" in
+ restore)
+ # No seed yet: the job installs from scratch and saves one once its dev shell is ready.
+ if ! cache_request --head --output /dev/null "$base/0"; then
+ {
+ echo "NIX_SEED_SAVE=true"
+ echo "NIX_SEED_CACHE=$NIX_SEED_CACHE"
+ echo "NIX_SEED_KEY=$NIX_SEED_KEY"
+ } >> "$GITHUB_ENV"
+ exit 1
+ fi
+
+ if restore; then
+ echo "Restored /nix from the seed"
+ exit 0
+ fi
+
+ # An interrupted restore leaves a store that looks installed.
+ sudo find /nix -mindepth 1 -delete
+ exit 1
+ ;;
+ save)
+ # Nothing writes to the store between the dev shell and this step.
+ save
+ echo "Saved /nix to the seed"
+ ;;
+esac
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml
index 62e69daf8b..ac393e98f6 100644
--- a/.github/workflows/code_check_cloud_storage.yml
+++ b/.github/workflows/code_check_cloud_storage.yml
@@ -347,6 +347,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
timeout-minutes: 15
@@ -565,6 +567,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
timeout-minutes: 15
@@ -750,6 +754,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
timeout-minutes: 15
diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.yml
index 28d0b8542d..e9cd5c8c66 100644
--- a/.github/workflows/web-app-check-main.yml
+++ b/.github/workflows/web-app-check-main.yml
@@ -398,6 +398,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup Prereqs
uses: './.github/actions/setup-reqs-web'
timeout-minutes: 15
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
index aa56a9f74f..42b75ff97c 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
@@ -105,7 +105,7 @@ fn check() -> Job {
.map(with_check_env)
.add_step(steps::checkout(false, false))
.add_step(steps::mount_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES))
.add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
.add_step(steps::code_mode_assets())
@@ -142,7 +142,7 @@ fn doppler_config() -> Job {
// git-remote-https at Nix's glibc, which aborts `git fetch`.
.add_step(compute_changed_files())
.add_step(steps::mount_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES))
.add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
.add_step(steps::code_mode_assets())
@@ -198,7 +198,7 @@ fn test() -> Job {
.add_service("redis", redis_service())
.add_step(steps::checkout(false, false))
.add_step(steps::mount_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES))
.add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
.add_step(steps::start_sccache_server())
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
index 95655ca699..39e379a231 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
@@ -192,6 +192,13 @@ pub fn setup_nix_with_cache() -> Step<Use> {
.add_with(("aws-secret-access-key", vars::AWS_SECRET_ACCESS_KEY))
}
+/// [`setup_nix`] restoring `/nix` from Namespace's HTTP cache when the volume is
+/// empty. PR jobs write that copy, so this is only for PR check jobs: never a job
+/// that deploys or holds deploy secrets.
+pub fn setup_nix_with_seed() -> Step<Use> {
+ setup_nix().add_with(("seed-cache", vars::CI_SCCACHE_NAME))
+}
+
/// Push built store paths to the private S3 binary cache, signing on upload —
/// the `cachix watch-store` role. `targets` lands in the step's `TARGETS` env
/// (space-separated flake refs or store paths; GH expressions fine) and is
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
index 3cfcff4c17..12b630c25f 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
@@ -167,7 +167,7 @@ fn generated_code() -> Job {
.add_env(("RUSTFLAGS", "-C link-arg=-fuse-ld=mold"))
.add_step(checkout("Checkout Repo", false))
.add_step(steps::mount_web_cache_volume(true))
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(
steps::setup_reqs_web("Setup Prereqs", false)
.timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES),