Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf(ci): cut about 45 s from code-mode asset builds when the SDK changes (#9871)

teod5872ce30065
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml
index ac393e98f6..ff45578c3c 100644
--- a/.github/workflows/code_check_cloud_storage.yml
+++ b/.github/workflows/code_check_cloud_storage.yml
@@ -419,8 +419,29 @@ jobs:
exit 0
fi
+ # Installing node_modules is most of a build and changes far less often than the SDK,
+ # so it has its own entry.
+ modules="$(nix eval --raw .#macro-js-node-modules.outPath)"
+ modules_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$modules").zst"
+ save_modules=false
+
+ if ! nix-store --check-validity "$modules" 2>/dev/null; then
+ # No retries: a retry would restart the stream into the same pipe, and past the limit
+ # a rebuild is faster.
+ if cache_request --retry 0 --max-time 60 "$modules_url" | zstd -d | nix-store --import; then
+ echo "node_modules restored from the remote cache"
+ else
+ save_modules=true
+ fi
+ fi
+
nix build --no-link --print-build-logs .#code-mode-assets
+ if [ "$save_modules" = true ] \
+ && ! nix-store --export "$modules" | zstd -T0 -3 | cache_request --max-time 120 --upload-file - "$modules_url"; then
+ echo "::warning::Could not save node_modules to the remote cache"
+ fi
+
# Losing the upload costs the next job a rebuild, not a failure.
if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
echo "Code-mode assets saved to the remote cache"
@@ -639,8 +660,29 @@ jobs:
exit 0
fi
+ # Installing node_modules is most of a build and changes far less often than the SDK,
+ # so it has its own entry.
+ modules="$(nix eval --raw .#macro-js-node-modules.outPath)"
+ modules_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$modules").zst"
+ save_modules=false
+
+ if ! nix-store --check-validity "$modules" 2>/dev/null; then
+ # No retries: a retry would restart the stream into the same pipe, and past the limit
+ # a rebuild is faster.
+ if cache_request --retry 0 --max-time 60 "$modules_url" | zstd -d | nix-store --import; then
+ echo "node_modules restored from the remote cache"
+ else
+ save_modules=true
+ fi
+ fi
+
nix build --no-link --print-build-logs .#code-mode-assets
+ if [ "$save_modules" = true ] \
+ && ! nix-store --export "$modules" | zstd -T0 -3 | cache_request --max-time 120 --upload-file - "$modules_url"; then
+ echo "::warning::Could not save node_modules to the remote cache"
+ fi
+
# Losing the upload costs the next job a rebuild, not a failure.
if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
echo "Code-mode assets saved to the remote cache"
@@ -828,8 +870,29 @@ jobs:
exit 0
fi
+ # Installing node_modules is most of a build and changes far less often than the SDK,
+ # so it has its own entry.
+ modules="$(nix eval --raw .#macro-js-node-modules.outPath)"
+ modules_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$modules").zst"
+ save_modules=false
+
+ if ! nix-store --check-validity "$modules" 2>/dev/null; then
+ # No retries: a retry would restart the stream into the same pipe, and past the limit
+ # a rebuild is faster.
+ if cache_request --retry 0 --max-time 60 "$modules_url" | zstd -d | nix-store --import; then
+ echo "node_modules restored from the remote cache"
+ else
+ save_modules=true
+ fi
+ fi
+
nix build --no-link --print-build-logs .#code-mode-assets
+ if [ "$save_modules" = true ] \
+ && ! nix-store --export "$modules" | zstd -T0 -3 | cache_request --max-time 120 --upload-file - "$modules_url"; then
+ echo "::warning::Could not save node_modules to the remote cache"
+ fi
+
# Losing the upload costs the next job a rebuild, not a failure.
if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
echo "Code-mode assets saved to the remote cache"
diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.yml
index e9cd5c8c66..4d4a499054 100644
--- a/.github/workflows/web-app-check-main.yml
+++ b/.github/workflows/web-app-check-main.yml
@@ -470,8 +470,29 @@ jobs:
exit 0
fi
+ # Installing node_modules is most of a build and changes far less often than the SDK,
+ # so it has its own entry.
+ modules="$(nix eval --raw .#macro-js-node-modules.outPath)"
+ modules_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$modules").zst"
+ save_modules=false
+
+ if ! nix-store --check-validity "$modules" 2>/dev/null; then
+ # No retries: a retry would restart the stream into the same pipe, and past the limit
+ # a rebuild is faster.
+ if cache_request --retry 0 --max-time 60 "$modules_url" | zstd -d | nix-store --import; then
+ echo "node_modules restored from the remote cache"
+ else
+ save_modules=true
+ fi
+ fi
+
nix build --no-link --print-build-logs .#code-mode-assets
+ if [ "$save_modules" = true ] \
+ && ! nix-store --export "$modules" | zstd -T0 -3 | cache_request --max-time 120 --upload-file - "$modules_url"; then
+ echo "::warning::Could not save node_modules to the remote cache"
+ fi
+
# Losing the upload costs the next job a rebuild, not a failure.
if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
echo "Code-mode assets saved to the remote cache"
diff --git a/crates/code_mode_assets/AGENTS.md b/crates/code_mode_assets/AGENTS.md
index dfd8df067b..3efd3df50b 100644
--- a/crates/code_mode_assets/AGENTS.md
+++ b/crates/code_mode_assets/AGENTS.md
@@ -13,7 +13,8 @@ built from `packages/sdk` when this crate compiles and are never committed.
- `CODE_MODE_ASSETS_DIR` set (Nix builds and Rust CI): that directory, built by
the `code-mode-assets` flake package from the Nix `node_modules`. Rust CI
restores it from Namespace's remote cache, keyed by its store path, and builds
- it only on a miss (`steps::code_mode_assets`).
+ it only on a miss (`steps::code_mode_assets`). A build
+ restores node_modules the same way, under its own store path.
- Unset (local cargo): it runs `packages/code-mode-sdk/scripts/generate.ts` with
`bun` into `OUT_DIR`, rerunning when SDK, generator or lockfile sources change.
diff --git a/docs/development/infrastructure.md b/docs/development/infrastructure.md
index adf4960937..709bc70b17 100644
--- a/docs/development/infrastructure.md
+++ b/docs/development/infrastructure.md
@@ -126,7 +126,7 @@ its source filter. Assets generated from JavaScript are built in their own
derivation instead: the `code-mode-assets` package bundles the SDK from the Nix
`node_modules`, and every crane build of `crates/code_mode_assets` reads it
through `CODE_MODE_ASSETS_DIR`; Rust CI exports the same variable before cargo,
-restoring the output from Namespace's remote cache when another job already built it.
+restoring the output from Namespace's remote cache when another job already built it. On a miss, node_modules comes from the same cache, keyed by its own store path.
Run it locally with `nix build --no-link .#code-mode-assets`.
## Other resources
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix
index b2335914f4..aa96eff640 100644
--- a/nix/cloud-storage.nix
+++ b/nix/cloud-storage.nix
@@ -1203,6 +1203,7 @@
;
default = cargoArtifacts;
code-mode-assets = codeModeAssets;
+ macro-js-node-modules = jsNodeModules;
}
// dopplerConfigBinPackages
// deployServiceBinaryPackages
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh
index 0ac472f885..f4fc502460 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh
@@ -30,8 +30,29 @@ if cache_request "$cache_url" | gunzip | nix-store --import; then
exit 0
fi
+# Installing node_modules is most of a build and changes far less often than the SDK,
+# so it has its own entry.
+modules="$(nix eval --raw .#macro-js-node-modules.outPath)"
+modules_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$modules").zst"
+save_modules=false
+
+if ! nix-store --check-validity "$modules" 2>/dev/null; then
+ # No retries: a retry would restart the stream into the same pipe, and past the limit
+ # a rebuild is faster.
+ if cache_request --retry 0 --max-time 60 "$modules_url" | zstd -d | nix-store --import; then
+ echo "node_modules restored from the remote cache"
+ else
+ save_modules=true
+ fi
+fi
+
nix build --no-link --print-build-logs .#code-mode-assets
+if [ "$save_modules" = true ] \
+ && ! nix-store --export "$modules" | zstd -T0 -3 | cache_request --max-time 120 --upload-file - "$modules_url"; then
+ echo "::warning::Could not save node_modules to the remote cache"
+fi
+
# Losing the upload costs the next job a rebuild, not a failure.
if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
echo "Code-mode assets saved to the remote cache"