Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
ci(code-mode): stop rebuilding assets on most Rust jobs by sharing them through a remote cache (#9743)
teo0bf9f97445c2
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.ymlindex dd0f188a39..2116aee545 100644--- a/.github/workflows/code_check_cloud_storage.yml+++ b/.github/workflows/code_check_cloud_storage.yml@@ -349,8 +349,46 @@ jobs:fi- name: Build code-mode assetsrun: |- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"+ set -euo pipefail++ # Build the code-mode assets once per SDK state, not once per job. Cache volumes+ # are forks of past jobs, so most jobs miss the store path another job just built.+ # The key is the store path, which covers every input of the build.+ assets="$(nix eval --raw .#code-mode-assets.outPath)"echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"++ if nix-store --check-validity "$assets" 2>/dev/null; then+ echo "Code-mode assets already in the store"+ exit 0+ fi++ # Forks have no remote cache credentials.+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then+ nix build --no-link --print-build-logs .#code-mode-assets+ exit 0+ fi++ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"++ # The token goes through a config descriptor so it stays out of curl's arguments.+ cache_request() {+ curl --fail --silent --show-error --max-time 30 --retry 2 \+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"+ }++ if cache_request "$cache_url" | gunzip | nix-store --import; then+ echo "Code-mode assets restored from the remote cache"+ exit 0+ fi++ nix build --no-link --print-build-logs .#code-mode-assets++ # Losing the upload costs the next job a rebuild, not a failure.+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then+ echo "Code-mode assets saved to the remote cache"+ else+ echo "::warning::Could not save code-mode assets to the remote cache"+ fi- name: fmtrun: cargo fmt --check- name: clippy@@ -523,8 +561,46 @@ jobs:fi- name: Build code-mode assetsrun: |- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"+ set -euo pipefail++ # Build the code-mode assets once per SDK state, not once per job. Cache volumes+ # are forks of past jobs, so most jobs miss the store path another job just built.+ # The key is the store path, which covers every input of the build.+ assets="$(nix eval --raw .#code-mode-assets.outPath)"echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"++ if nix-store --check-validity "$assets" 2>/dev/null; then+ echo "Code-mode assets already in the store"+ exit 0+ fi++ # Forks have no remote cache credentials.+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then+ nix build --no-link --print-build-logs .#code-mode-assets+ exit 0+ fi++ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"++ # The token goes through a config descriptor so it stays out of curl's arguments.+ cache_request() {+ curl --fail --silent --show-error --max-time 30 --retry 2 \+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"+ }++ if cache_request "$cache_url" | gunzip | nix-store --import; then+ echo "Code-mode assets restored from the remote cache"+ exit 0+ fi++ nix build --no-link --print-build-logs .#code-mode-assets++ # Losing the upload costs the next job a rebuild, not a failure.+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then+ echo "Code-mode assets saved to the remote cache"+ else+ echo "::warning::Could not save code-mode assets to the remote cache"+ fi- id: doppler-binsname: compute affected Doppler config binsrun: |@@ -664,8 +740,46 @@ jobs:run: sccache --start-server- name: Build code-mode assetsrun: |- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"+ set -euo pipefail++ # Build the code-mode assets once per SDK state, not once per job. Cache volumes+ # are forks of past jobs, so most jobs miss the store path another job just built.+ # The key is the store path, which covers every input of the build.+ assets="$(nix eval --raw .#code-mode-assets.outPath)"echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"++ if nix-store --check-validity "$assets" 2>/dev/null; then+ echo "Code-mode assets already in the store"+ exit 0+ fi++ # Forks have no remote cache credentials.+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then+ nix build --no-link --print-build-logs .#code-mode-assets+ exit 0+ fi++ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"++ # The token goes through a config descriptor so it stays out of curl's arguments.+ cache_request() {+ curl --fail --silent --show-error --max-time 30 --retry 2 \+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"+ }++ if cache_request "$cache_url" | gunzip | nix-store --import; then+ echo "Code-mode assets restored from the remote cache"+ exit 0+ fi++ nix build --no-link --print-build-logs .#code-mode-assets++ # Losing the upload costs the next job a rebuild, not a failure.+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then+ echo "Code-mode assets saved to the remote cache"+ else+ echo "::warning::Could not save code-mode assets to the remote cache"+ fi- name: configure postgres for concurrent testsrun: |postgres_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 == "pgvector/pgvector:pg14" { print $1; exit }')"diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.ymlindex 97627bacd7..10615d957a 100644--- a/.github/workflows/web-app-check-main.yml+++ b/.github/workflows/web-app-check-main.yml@@ -387,8 +387,46 @@ jobs:fi- name: Build code-mode assetsrun: |- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"+ set -euo pipefail++ # Build the code-mode assets once per SDK state, not once per job. Cache volumes+ # are forks of past jobs, so most jobs miss the store path another job just built.+ # The key is the store path, which covers every input of the build.+ assets="$(nix eval --raw .#code-mode-assets.outPath)"echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"++ if nix-store --check-validity "$assets" 2>/dev/null; then+ echo "Code-mode assets already in the store"+ exit 0+ fi++ # Forks have no remote cache credentials.+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then+ nix build --no-link --print-build-logs .#code-mode-assets+ exit 0+ fi++ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"++ # The token goes through a config descriptor so it stays out of curl's arguments.+ cache_request() {+ curl --fail --silent --show-error --max-time 30 --retry 2 \+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"+ }++ if cache_request "$cache_url" | gunzip | nix-store --import; then+ echo "Code-mode assets restored from the remote cache"+ exit 0+ fi++ nix build --no-link --print-build-logs .#code-mode-assets++ # Losing the upload costs the next job a rebuild, not a failure.+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then+ echo "Code-mode assets saved to the remote cache"+ else+ echo "::warning::Could not save code-mode assets to the remote cache"+ fi- name: Build schema binariesrun: |set -euo pipefaildiff --git a/crates/code_mode_assets/AGENTS.md b/crates/code_mode_assets/AGENTS.mdindex a1e26326c4..dfd8df067b 100644--- a/crates/code_mode_assets/AGENTS.md+++ b/crates/code_mode_assets/AGENTS.md@@ -11,7 +11,9 @@ built from `packages/sdk` when this crate compiles and are never committed.`build.rs` takes the assets from exactly one place:- `CODE_MODE_ASSETS_DIR` set (Nix builds and Rust CI): that directory, built by- the `code-mode-assets` flake package from the Nix `node_modules`.+ the `code-mode-assets` flake package from the Nix `node_modules`. Rust CI+ restores it from Namespace's remote cache, keyed by its store path, and builds+ it only on a miss (`steps::code_mode_assets`).- Unset (local cargo): it runs `packages/code-mode-sdk/scripts/generate.ts` with`bun` into `OUT_DIR`, rerunning when SDK, generator or lockfile sources change.diff --git a/docs/development/infrastructure.md b/docs/development/infrastructure.mdindex 073dfde270..adf4960937 100644--- a/docs/development/infrastructure.md+++ b/docs/development/infrastructure.md@@ -125,7 +125,8 @@ an entire checkout. Include any new `include_str!` or `include_bytes!` assets inits source filter. Assets generated from JavaScript are built in their ownderivation instead: the `code-mode-assets` package bundles the SDK from the Nix`node_modules`, and every crane build of `crates/code_mode_assets` reads it-through `CODE_MODE_ASSETS_DIR`; Rust CI exports the same variable before cargo.+through `CODE_MODE_ASSETS_DIR`; Rust CI exports the same variable before cargo,+restoring the output from Namespace's remote cache when another job already built it.Run it locally with `nix build --no-link .#code-mode-assets`.## Other resourcesdiff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.shnew file mode 100644index 0000000000..0ac472f885--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/code_mode_assets.sh@@ -0,0 +1,40 @@+set -euo pipefail++# Build the code-mode assets once per SDK state, not once per job. Cache volumes+# are forks of past jobs, so most jobs miss the store path another job just built.+# The key is the store path, which covers every input of the build.+assets="$(nix eval --raw .#code-mode-assets.outPath)"+echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"++if nix-store --check-validity "$assets" 2>/dev/null; then+ echo "Code-mode assets already in the store"+ exit 0+fi++# Forks have no remote cache credentials.+if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then+ nix build --no-link --print-build-logs .#code-mode-assets+ exit 0+fi++cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"++# The token goes through a config descriptor so it stays out of curl's arguments.+cache_request() {+ curl --fail --silent --show-error --max-time 30 --retry 2 \+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"+}++if cache_request "$cache_url" | gunzip | nix-store --import; then+ echo "Code-mode assets restored from the remote cache"+ exit 0+fi++nix build --no-link --print-build-logs .#code-mode-assets++# Losing the upload costs the next job a rebuild, not a failure.+if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then+ echo "Code-mode assets saved to the remote cache"+else+ echo "::warning::Could not save code-mode assets to the remote cache"+fidiff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rsindex e114d1964a..58db6c66fb 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs@@ -430,15 +430,13 @@ pub fn setup_reqs_web(name: &str, playwright: bool) -> Step<Use> {.when(playwright, |step| step.add_with(("playwright", "true")))}-/// Build the code-mode assets (`crates/code_mode_assets`) in Nix and point-/// later cargo steps at them, so Rust jobs need no bun install. Any job that-/// compiles that crate or a dependent (DSS, the agent harness) needs this.+/// Build the code-mode assets (`crates/code_mode_assets`) in Nix, or restore them+/// from Namespace's remote cache, and point later cargo steps at them, so Rust jobs+/// need no bun install. Any job that compiles that crate or a dependent (DSS, the+/// agent harness) needs this. Goes after [`configure_namespace_sccache`], whose+/// credentials it reuses.pub fn code_mode_assets() -> Step<Run> {- // A separate assignment, so a failed build fails the step under `bash -e`.- Step::new("Build code-mode assets").run(indoc::indoc! {r#"- assets="$(nix build --no-link --print-out-paths --print-build-logs .#code-mode-assets)"- echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"- "#})+ Step::new("Build code-mode assets").run(include_str!("scripts/code_mode_assets.sh"))}/// Start the server before parallel compiler invocations can race to launch it.