Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
ci(path-check): cut 22 s from every PR by reusing the registry and filter build (#9742)
teofde88f223517
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.ymlindex 86bf02c7c8..e9187d2b8c 100644--- a/.github/workflows/code_check_cloud_storage.yml+++ b/.github/workflows/code_check_cloud_storage.yml@@ -19,7 +19,7 @@ env:- mainjobs:path-check:- runs-on: namespace-profile-linux-small;overrides.cache-tag=nextest-filter+ runs-on: namespace-profile-linux-small;overrides.cache-tag=sccache-cioutputs:should_run: ${{ steps.filter.outputs.should_run }}doppler_candidates: ${{ steps.filter.outputs.doppler_candidates }}@@ -123,12 +123,43 @@ jobs:printf '%s\n' "$compare_rev" > /tmp/base-revisiongit diff --name-only --no-renames "$compare_rev" "$GITHUB_SHA" > /tmp/changed-filesshell: bash- - name: Mount nextest-filter binary cache+ - name: Mount nextest-filter and cargo registry cacheif: steps.filter.outputs.should_run == 'true' && ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository)uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0with:- path: /home/runner/.cache/macro-nextest-filter+ path: |-+ /home/runner/.cache/macro-nextest-filter+ /home/runner/.cargo/registry+ /home/runner/.cargo/gitcontinue-on-error: true+ - name: Configure Namespace remote sccache+ if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository+ run: |-+ set -euo pipefail+ env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"+ trap 'rm -f "$env_file"' EXIT+ if nsc cache sccache setup --cache_name sccache-ci > "$env_file"; then+ # Register credential values as masked BEFORE exporting: GitHub only+ # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,+ # ~24h Namespace workspace token (registry + cache write) — printed+ # verbatim in every subsequent step's env dump. Masking is selective by+ # key name: masking non-secrets like the endpoint URL or key prefix+ # would redact those strings everywhere in the logs.+ while IFS= read -r line; do+ k="${line%%=*}"+ v="${line#*=}"+ [ -n "$v" ] && [ "$k" != "$line" ] || continue+ case "$k" in+ *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;+ esac+ done < "$env_file"+ cat "$env_file" >> "$GITHUB_ENV"+ # Force the next compiler invocation to start a server with the new remote+ # backend even if a setup hook happened to launch one already.+ sccache --stop-server >/dev/null 2>&1 || true+ else+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"+ fi- id: nextest-filtername: compute nextest package filterif: steps.filter.outputs.should_run == 'true'@@ -150,40 +181,59 @@ jobs:# full-suite list and turned every dev-shell tweak into a full-suite rebuild.# The filter runs as a release binary cached per source tree: a debug build- # spends ~20 s in guppy on this runner, and compiling it took ~40 s more. The- # key covers every input of the build, so a hit is exactly what `cargo build`- # would produce. FILTER_CACHE_DIR is a Namespace cache volume for same-repo- # PRs and a plain scratch directory for forks.+ # spends ~20 s in guppy on this runner, and compiling it took ~25 s more. The+ # key covers every input of the build. It lives in Namespace's remote cache, which+ # every runner sees, because a cache volume holds it on only some jobs. Forks have+ # no credentials for that cache and always build, in a plain scratch directory.filter_bin=""++ # The token goes through a config descriptor so it stays out of curl's arguments.+ filter_cache_request() {+ curl --fail --silent --show-error --max-time 30 --retry 2 \+ --config <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"+ }++ # With no arguments a working filter exits non-zero and prints its usage. A corrupt+ # binary in the cache would otherwise fail path-check for every PR on that key.+ filter_bin_runs() {+ chmod +x "$filter_bin"+ case "$("$filter_bin" 2>&1 || true)" in+ *usage:*) return 0 ;;+ *) return 1 ;;+ esac+ }+ensure_filter_bin() {[ -n "$filter_bin" ] && return 0- local key bin_dir staging+ local key cache_url=""key="$(git rev-parse \HEAD:tooling/xtask/crates/xtask_nextest_filter \HEAD:tooling/xtask/crates/xtask_graph \HEAD:tooling/xtask/crates/xtask_paths \HEAD:Cargo.lock \+ HEAD:Cargo.toml \HEAD:rust-toolchain.toml | sha256sum | cut -c1-16)"- bin_dir="$FILTER_CACHE_DIR/bin/$key"- filter_bin="$bin_dir/xtask_nextest_filter"- if [ -x "$filter_bin" ]; then- echo "Using cached nextest-filter build $key"- touch "$bin_dir"- return 0+ filter_bin="$RUNNER_TEMP/xtask_nextest_filter-$key"++ if [ -n "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then+ cache_url="$SCCACHE_WEBDAV_ENDPOINT/nextest-filter/$key"+ if filter_cache_request --output "$filter_bin" "$cache_url" && filter_bin_runs; then+ echo "Using cached nextest-filter build $key"+ return 0+ fifiecho "Building nextest-filter $key"CARGO_TARGET_DIR="$FILTER_CACHE_DIR/target" \cargo build --release --locked --quiet -p xtask_nextest_filter- staging="$(mktemp -d "$FILTER_CACHE_DIR/bin/.staging.XXXXXX")"- cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$staging/"- # Another job may have published the same key meanwhile; either copy works.- mv -T "$staging" "$bin_dir" 2>/dev/null || rm -rf "$staging"- # Drop builds no PR has used for a week (every Cargo.lock change adds one).- find "$FILTER_CACHE_DIR/bin" -mindepth 1 -maxdepth 1 -type d -mtime +7 \- -exec rm -rf {} + || true+ cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$filter_bin"++ # Losing the upload costs the next job a rebuild, not a failure.+ if [ -n "$cache_url" ]; then+ filter_cache_request --upload-file "$filter_bin" "$cache_url" \+ || echo "::warning::Could not save the nextest filter to the remote cache"+ fi}- mkdir -p "$FILTER_CACHE_DIR/bin"emit() {echo "rust_packages=$1" >> "$GITHUB_OUTPUT"diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rsindex e0373dc4d1..6ed1d3200c 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs@@ -55,11 +55,12 @@ const FILTER_CACHE_DIR: RuntimePath<'static> =xtask_paths::runtime_path!("/home/runner/.cache/macro-nextest-filter");/// Decide whether the rest of the workflow runs and compute the nextest-/// filter. Deliberately light: no Nix, no sccache, and a cached release-/// binary for the filter.+/// filter. Deliberately light: no Nix, and a cached release binary for the filter.+/// The sccache step supplies only that cache's credentials; nothing compiles through+/// it here.fn path_check() -> Job {Job::default()- .runs_on(runners::Runner::Small.with_cache_tag(vars::NEXTEST_FILTER_CACHE_TAG))+ .runs_on(runners::Runner::Small.with_cache_tag(vars::CI_CACHE_TAG)).add_output("should_run", "${{ steps.filter.outputs.should_run }}").add_output("doppler_candidates",@@ -87,11 +88,11 @@ fn path_check() -> Job {.if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")),).add_step(- steps::mount_path_cache_volume("Mount nextest-filter binary cache", FILTER_CACHE_DIR)- .if_condition(Expression::new(format!(- "steps.filter.outputs.should_run == 'true' && ({TRUSTED_CONTEXT})"- ))),+ steps::mount_nextest_filter_cache(FILTER_CACHE_DIR).if_condition(Expression::new(+ format!("steps.filter.outputs.should_run == 'true' && ({TRUSTED_CONTEXT})"),+ )),)+ .add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME)).add_step(compute_nextest_filter())}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rsindex 5517319f72..c7d770c4ef 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs@@ -54,7 +54,7 @@ fn filter_binary_cache_is_trusted_only() {.as_array().unwrap().iter()- .find(|step| step["name"] == "Mount nextest-filter binary cache")+ .find(|step| step["name"] == "Mount nextest-filter and cargo registry cache").expect("binary cache mount");let condition = mount["if"].as_str().unwrap();assert!(diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.shindex 7f269f76f6..9bb946351b 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh@@ -15,40 +15,59 @@ set -euo pipefail# full-suite list and turned every dev-shell tweak into a full-suite rebuild.# The filter runs as a release binary cached per source tree: a debug build-# spends ~20 s in guppy on this runner, and compiling it took ~40 s more. The-# key covers every input of the build, so a hit is exactly what `cargo build`-# would produce. FILTER_CACHE_DIR is a Namespace cache volume for same-repo-# PRs and a plain scratch directory for forks.+# spends ~20 s in guppy on this runner, and compiling it took ~25 s more. The+# key covers every input of the build. It lives in Namespace's remote cache, which+# every runner sees, because a cache volume holds it on only some jobs. Forks have+# no credentials for that cache and always build, in a plain scratch directory.filter_bin=""++# The token goes through a config descriptor so it stays out of curl's arguments.+filter_cache_request() {+ curl --fail --silent --show-error --max-time 30 --retry 2 \+ --config <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"+}++# With no arguments a working filter exits non-zero and prints its usage. A corrupt+# binary in the cache would otherwise fail path-check for every PR on that key.+filter_bin_runs() {+ chmod +x "$filter_bin"+ case "$("$filter_bin" 2>&1 || true)" in+ *usage:*) return 0 ;;+ *) return 1 ;;+ esac+}+ensure_filter_bin() {[ -n "$filter_bin" ] && return 0- local key bin_dir staging+ local key cache_url=""key="$(git rev-parse \HEAD:tooling/xtask/crates/xtask_nextest_filter \HEAD:tooling/xtask/crates/xtask_graph \HEAD:tooling/xtask/crates/xtask_paths \HEAD:Cargo.lock \+ HEAD:Cargo.toml \HEAD:rust-toolchain.toml | sha256sum | cut -c1-16)"- bin_dir="$FILTER_CACHE_DIR/bin/$key"- filter_bin="$bin_dir/xtask_nextest_filter"- if [ -x "$filter_bin" ]; then- echo "Using cached nextest-filter build $key"- touch "$bin_dir"- return 0+ filter_bin="$RUNNER_TEMP/xtask_nextest_filter-$key"++ if [ -n "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then+ cache_url="$SCCACHE_WEBDAV_ENDPOINT/nextest-filter/$key"+ if filter_cache_request --output "$filter_bin" "$cache_url" && filter_bin_runs; then+ echo "Using cached nextest-filter build $key"+ return 0+ fifiecho "Building nextest-filter $key"CARGO_TARGET_DIR="$FILTER_CACHE_DIR/target" \cargo build --release --locked --quiet -p xtask_nextest_filter- staging="$(mktemp -d "$FILTER_CACHE_DIR/bin/.staging.XXXXXX")"- cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$staging/"- # Another job may have published the same key meanwhile; either copy works.- mv -T "$staging" "$bin_dir" 2>/dev/null || rm -rf "$staging"- # Drop builds no PR has used for a week (every Cargo.lock change adds one).- find "$FILTER_CACHE_DIR/bin" -mindepth 1 -maxdepth 1 -type d -mtime +7 \- -exec rm -rf {} + || true+ cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$filter_bin"++ # Losing the upload costs the next job a rebuild, not a failure.+ if [ -n "$cache_url" ]; then+ filter_cache_request --upload-file "$filter_bin" "$cache_url" \+ || echo "::warning::Could not save the nextest filter to the remote cache"+ fi}-mkdir -p "$FILTER_CACHE_DIR/bin"emit() {echo "rust_packages=$1" >> "$GITHUB_OUTPUT"diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rsindex aaef0ebd9f..e114d1964a 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs@@ -257,11 +257,18 @@ pub fn mount_cache_volume() -> Step<Use> {.add_with(("path", xtask_paths::runtime_path!("/nix").as_str()))}-/// Mount only `path` from the profile's cache volume, without a framework-/// mode. Unlike [`mount_cache_volume`] (`cache: rust` plus `/nix`, ~11 s on-/// `linux-rust-ci`), a plain path mounts in under a second.-pub fn mount_path_cache_volume(name: &str, path: RuntimePath<'_>) -> Step<Use> {- nscloud_cache_action(name).add_with(("path", path.as_str()))+/// Mount a scratch directory for building the nextest filter, plus cargo's registry+/// and git checkouts. `cargo metadata` fetches every crate in the lockfile, which took+/// most of the filter's time when `path-check` had no registry to start from.+pub fn mount_nextest_filter_cache(filter_cache_dir: RuntimePath<'_>) -> Step<Use> {+ let paths = format!(+ "{}\n{}\n{}",+ filter_cache_dir.as_str(),+ xtask_paths::runtime_path!("/home/runner/.cargo/registry").as_str(),+ xtask_paths::runtime_path!("/home/runner/.cargo/git").as_str(),+ );++ nscloud_cache_action("Mount nextest-filter and cargo registry cache").add_with(("path", paths))}/// The pinned `nscloud-cache-action`, shared by every mount helper below.diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rsindex 2db17e7d76..767792b3b3 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs@@ -82,12 +82,6 @@ pub const NIX_CHECKS_CACHE_TAG: &str = "nix-checks";/// a different workload and runner profile.pub const WEB_SCCACHE_NAME: &str = "web-ci";-/// Namespace cache tag for `path-check`, which keeps its release-/// `xtask_nextest_filter` binary on the volume. Without a fixed tag the small-/// profile's volume is scoped per branch and PR runs never persisted, so every-/// run rebuilt the filter.-pub const NEXTEST_FILTER_CACHE_TAG: &str = "nextest-filter";-/// Namespace cache tag for the sync-service worker deploy. Its own pool: this/// job compiles for `wasm32-unknown-unknown`, so nothing in the host-target/// volumes ([`CI_CACHE_TAG`]) would serve it anyway.