Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

ci(code-mode): stop rebuilding assets on most Rust jobs by sharing them through a remote cache (#9743)

teo0bf9f97445c2
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml
index dd0f188a39..2116aee545 100644
--- a/.github/workflows/code_check_cloud_storage.yml
+++ b/.github/workflows/code_check_cloud_storage.yml
@@ -349,8 +349,46 @@ jobs:
fi
- name: Build code-mode assets
run: |
- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"
+ set -euo pipefail
+
+ # Build the code-mode assets once per SDK state, not once per job. Cache volumes
+ # are forks of past jobs, so most jobs miss the store path another job just built.
+ # The key is the store path, which covers every input of the build.
+ assets="$(nix eval --raw .#code-mode-assets.outPath)"
echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"
+
+ if nix-store --check-validity "$assets" 2>/dev/null; then
+ echo "Code-mode assets already in the store"
+ exit 0
+ fi
+
+ # Forks have no remote cache credentials.
+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ nix build --no-link --print-build-logs .#code-mode-assets
+ exit 0
+ fi
+
+ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"
+
+ # The token goes through a config descriptor so it stays out of curl's arguments.
+ cache_request() {
+ curl --fail --silent --show-error --max-time 30 --retry 2 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"
+ }
+
+ if cache_request "$cache_url" | gunzip | nix-store --import; then
+ echo "Code-mode assets restored from the remote cache"
+ exit 0
+ fi
+
+ nix build --no-link --print-build-logs .#code-mode-assets
+
+ # Losing the upload costs the next job a rebuild, not a failure.
+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
+ echo "Code-mode assets saved to the remote cache"
+ else
+ echo "::warning::Could not save code-mode assets to the remote cache"
+ fi
- name: fmt
run: cargo fmt --check
- name: clippy
@@ -523,8 +561,46 @@ jobs:
fi
- name: Build code-mode assets
run: |
- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"
+ set -euo pipefail
+
+ # Build the code-mode assets once per SDK state, not once per job. Cache volumes
+ # are forks of past jobs, so most jobs miss the store path another job just built.
+ # The key is the store path, which covers every input of the build.
+ assets="$(nix eval --raw .#code-mode-assets.outPath)"
echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"
+
+ if nix-store --check-validity "$assets" 2>/dev/null; then
+ echo "Code-mode assets already in the store"
+ exit 0
+ fi
+
+ # Forks have no remote cache credentials.
+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ nix build --no-link --print-build-logs .#code-mode-assets
+ exit 0
+ fi
+
+ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"
+
+ # The token goes through a config descriptor so it stays out of curl's arguments.
+ cache_request() {
+ curl --fail --silent --show-error --max-time 30 --retry 2 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"
+ }
+
+ if cache_request "$cache_url" | gunzip | nix-store --import; then
+ echo "Code-mode assets restored from the remote cache"
+ exit 0
+ fi
+
+ nix build --no-link --print-build-logs .#code-mode-assets
+
+ # Losing the upload costs the next job a rebuild, not a failure.
+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
+ echo "Code-mode assets saved to the remote cache"
+ else
+ echo "::warning::Could not save code-mode assets to the remote cache"
+ fi
- id: doppler-bins
name: compute affected Doppler config bins
run: |
@@ -664,8 +740,46 @@ jobs:
run: sccache --start-server
- name: Build code-mode assets
run: |
- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"
+ set -euo pipefail
+
+ # Build the code-mode assets once per SDK state, not once per job. Cache volumes
+ # are forks of past jobs, so most jobs miss the store path another job just built.
+ # The key is the store path, which covers every input of the build.
+ assets="$(nix eval --raw .#code-mode-assets.outPath)"
echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"
+
+ if nix-store --check-validity "$assets" 2>/dev/null; then
+ echo "Code-mode assets already in the store"
+ exit 0
+ fi
+
+ # Forks have no remote cache credentials.
+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ nix build --no-link --print-build-logs .#code-mode-assets
+ exit 0
+ fi
+
+ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"
+
+ # The token goes through a config descriptor so it stays out of curl's arguments.
+ cache_request() {
+ curl --fail --silent --show-error --max-time 30 --retry 2 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"
+ }
+
+ if cache_request "$cache_url" | gunzip | nix-store --import; then
+ echo "Code-mode assets restored from the remote cache"
+ exit 0
+ fi
+
+ nix build --no-link --print-build-logs .#code-mode-assets
+
+ # Losing the upload costs the next job a rebuild, not a failure.
+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
+ echo "Code-mode assets saved to the remote cache"
+ else
+ echo "::warning::Could not save code-mode assets to the remote cache"
+ fi
- name: configure postgres for concurrent tests
run: |
postgres_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 == "pgvector/pgvector:pg14" { print $1; exit }')"
diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.yml
index 97627bacd7..10615d957a 100644
--- a/.github/workflows/web-app-check-main.yml
+++ b/.github/workflows/web-app-check-main.yml
@@ -387,8 +387,46 @@ jobs:
fi
- name: Build code-mode assets
run: |
- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"
+ set -euo pipefail
+
+ # Build the code-mode assets once per SDK state, not once per job. Cache volumes
+ # are forks of past jobs, so most jobs miss the store path another job just built.
+ # The key is the store path, which covers every input of the build.
+ assets="$(nix eval --raw .#code-mode-assets.outPath)"
echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"
+
+ if nix-store --check-validity "$assets" 2>/dev/null; then
+ echo "Code-mode assets already in the store"
+ exit 0
+ fi
+
+ # Forks have no remote cache credentials.
+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ nix build --no-link --print-build-logs .#code-mode-assets
+ exit 0
+ fi
+
+ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"
+
+ # The token goes through a config descriptor so it stays out of curl's arguments.
+ cache_request() {
+ curl --fail --silent --show-error --max-time 30 --retry 2 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"
+ }
+
+ if cache_request "$cache_url" | gunzip | nix-store --import; then
+ echo "Code-mode assets restored from the remote cache"
+ exit 0
+ fi
+
+ nix build --no-link --print-build-logs .#code-mode-assets
+
+ # Losing the upload costs the next job a rebuild, not a failure.
+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
+ echo "Code-mode assets saved to the remote cache"
+ else
+ echo "::warning::Could not save code-mode assets to the remote cache"
+ fi
- name: Build schema binaries
run: |
set -euo pipefail
diff --git a/crates/code_mode_assets/AGENTS.md b/crates/code_mode_assets/AGENTS.md
index a1e26326c4..dfd8df067b 100644
--- a/crates/code_mode_assets/AGENTS.md
+++ b/crates/code_mode_assets/AGENTS.md
@@ -11,7 +11,9 @@ built from `packages/sdk` when this crate compiles and are never committed.
`build.rs` takes the assets from exactly one place:
- `CODE_MODE_ASSETS_DIR` set (Nix builds and Rust CI): that directory, built by
- the `code-mode-assets` flake package from the Nix `node_modules`.
+ the `code-mode-assets` flake package from the Nix `node_modules`. Rust CI
+ restores it from Namespace's remote cache, keyed by its store path, and builds
+ it only on a miss (`steps::code_mode_assets`).
- Unset (local cargo): it runs `packages/code-mode-sdk/scripts/generate.ts` with
`bun` into `OUT_DIR`, rerunning when SDK, generator or lockfile sources change.
diff --git a/docs/development/infrastructure.md b/docs/development/infrastructure.md
index 073dfde270..adf4960937 100644
--- a/docs/development/infrastructure.md
+++ b/docs/development/infrastructure.md
@@ -125,7 +125,8 @@ an entire checkout. Include any new `include_str!` or `include_bytes!` assets in
its source filter. Assets generated from JavaScript are built in their own
derivation instead: the `code-mode-assets` package bundles the SDK from the Nix
`node_modules`, and every crane build of `crates/code_mode_assets` reads it
-through `CODE_MODE_ASSETS_DIR`; Rust CI exports the same variable before cargo.
+through `CODE_MODE_ASSETS_DIR`; Rust CI exports the same variable before cargo,
+restoring the output from Namespace's remote cache when another job already built it.
Run it locally with `nix build --no-link .#code-mode-assets`.
## Other resources
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh
new file mode 100644
index 0000000000..0ac472f885
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh
@@ -0,0 +1,40 @@
+set -euo pipefail
+
+# Build the code-mode assets once per SDK state, not once per job. Cache volumes
+# are forks of past jobs, so most jobs miss the store path another job just built.
+# The key is the store path, which covers every input of the build.
+assets="$(nix eval --raw .#code-mode-assets.outPath)"
+echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"
+
+if nix-store --check-validity "$assets" 2>/dev/null; then
+ echo "Code-mode assets already in the store"
+ exit 0
+fi
+
+# Forks have no remote cache credentials.
+if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ nix build --no-link --print-build-logs .#code-mode-assets
+ exit 0
+fi
+
+cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"
+
+# The token goes through a config descriptor so it stays out of curl's arguments.
+cache_request() {
+ curl --fail --silent --show-error --max-time 30 --retry 2 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"
+}
+
+if cache_request "$cache_url" | gunzip | nix-store --import; then
+ echo "Code-mode assets restored from the remote cache"
+ exit 0
+fi
+
+nix build --no-link --print-build-logs .#code-mode-assets
+
+# Losing the upload costs the next job a rebuild, not a failure.
+if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
+ echo "Code-mode assets saved to the remote cache"
+else
+ echo "::warning::Could not save code-mode assets to the remote cache"
+fi
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
index e114d1964a..58db6c66fb 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
@@ -430,15 +430,13 @@ pub fn setup_reqs_web(name: &str, playwright: bool) -> Step<Use> {
.when(playwright, |step| step.add_with(("playwright", "true")))
}
-/// Build the code-mode assets (`crates/code_mode_assets`) in Nix and point
-/// later cargo steps at them, so Rust jobs need no bun install. Any job that
-/// compiles that crate or a dependent (DSS, the agent harness) needs this.
+/// Build the code-mode assets (`crates/code_mode_assets`) in Nix, or restore them
+/// from Namespace's remote cache, and point later cargo steps at them, so Rust jobs
+/// need no bun install. Any job that compiles that crate or a dependent (DSS, the
+/// agent harness) needs this. Goes after [`configure_namespace_sccache`], whose
+/// credentials it reuses.
pub fn code_mode_assets() -> Step<Run> {
- // A separate assignment, so a failed build fails the step under `bash -e`.
- Step::new("Build code-mode assets").run(indoc::indoc! {r#"
- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"
- echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"
- "#})
+ Step::new("Build code-mode assets").run(include_str!("scripts/code_mode_assets.sh"))
}
/// Start the server before parallel compiler invocations can race to launch it.