Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

ci(test): build the service binaries on pull requests (#9861)

teo76e3ec5bd923
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml
index 02ed4d38d2..e221e57721 100644
--- a/.github/workflows/code_check_cloud_storage.yml
+++ b/.github/workflows/code_check_cloud_storage.yml
@@ -1049,6 +1049,159 @@ jobs:
- name: Teardown Nix
if: always()
uses: './.github/actions/teardown-nix'
+ build-service-bins:
+ needs:
+ - path-check
+ if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.skip_tests != 'true' && (needs.path-check.outputs.rust_packages == 'all' || contains(needs.path-check.outputs.rust_packages, 'document_storage_service') || contains(needs.path-check.outputs.rust_packages, 'email_service') || contains(needs.path-check.outputs.rust_packages, 'authentication_service'))
+ runs-on: namespace-profile-linux-large-build;overrides.cache-tag=sccache-ci
+ env:
+ RUST_PACKAGES: ${{ needs.path-check.outputs.rust_packages }}
+ CARGO_PROFILE_DEV_DEBUG: '0'
+ CARGO_PROFILE_TEST_DEBUG: '0'
+ RUSTFLAGS: '-Dwarnings -C link-arg=-fuse-ld=mold'
+ SQLX_OFFLINE: 'true'
+ timeout-minutes: 45
+ steps:
+ - name: Checkout
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ clean: 'false'
+ persist-credentials: 'false'
+ - name: Mount Namespace cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: |-
+ /nix
+ ./target
+ /home/runner/.cargo/registry
+ /home/runner/.cargo/git
+ /home/runner/.cargo/.global-cache
+ continue-on-error: true
+ - name: Setup Nix
+ uses: './.github/actions/setup-nix'
+ - name: Setup Nix dev shell
+ uses: './.github/actions/setup-nix-dev-shell'
+ timeout-minutes: 15
+ - name: Configure Namespace remote sccache
+ if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
+ run: |-
+ set -euo pipefail
+ env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"
+ trap 'rm -f "$env_file"' EXIT
+ if nsc cache sccache setup --cache_name sccache-ci > "$env_file"; then
+ # Register credential values as masked BEFORE exporting: GitHub only
+ # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,
+ # ~24h Namespace workspace token (registry + cache write) — printed
+ # verbatim in every subsequent step's env dump. Masking is selective by
+ # key name: masking non-secrets like the endpoint URL or key prefix
+ # would redact those strings everywhere in the logs.
+ while IFS= read -r line; do
+ k="${line%%=*}"
+ v="${line#*=}"
+ [ -n "$v" ] && [ "$k" != "$line" ] || continue
+ case "$k" in
+ *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;
+ esac
+ done < "$env_file"
+ cat "$env_file" >> "$GITHUB_ENV"
+ # The server checks the WebDAV backend before it reports ready, and the
+ # client's default 10s wait can run out. This limit has no env var.
+ sccache_conf="$RUNNER_TEMP/sccache.toml"
+ echo 'server_startup_timeout_ms = 60000' > "$sccache_conf"
+ echo "SCCACHE_CONF=$sccache_conf" >> "$GITHUB_ENV"
+ # Force the next compiler invocation to start a server with the new remote
+ # backend even if a setup hook happened to launch one already.
+ sccache --stop-server >/dev/null 2>&1 || true
+ else
+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
+ fi
+ - name: Build code-mode assets
+ run: |
+ set -euo pipefail
+
+ # Build the code-mode assets once per SDK state, not once per job. Cache volumes
+ # are forks of past jobs, so most jobs miss the store path another job just built.
+ # The key is the store path, which covers every input of the build.
+ assets="$(nix eval --raw .#code-mode-assets.outPath)"
+ echo "CODE_MODE_ASSETS_DIR=$assets" >> "$GITHUB_ENV"
+
+ if nix-store --check-validity "$assets" 2>/dev/null; then
+ echo "Code-mode assets already in the store"
+ exit 0
+ fi
+
+ # Forks have no remote cache credentials.
+ if [ -z "${SCCACHE_WEBDAV_ENDPOINT:-}" ]; then
+ nix build --no-link --print-build-logs .#code-mode-assets
+ exit 0
+ fi
+
+ cache_url="$SCCACHE_WEBDAV_ENDPOINT/code-mode-assets/$(basename "$assets").closure.gz"
+
+ # The token goes through a config descriptor so it stays out of curl's arguments.
+ cache_request() {
+ curl --fail --silent --show-error --max-time 30 --retry 2 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"
+ }
+
+ if cache_request "$cache_url" | gunzip | nix-store --import; then
+ echo "Code-mode assets restored from the remote cache"
+ exit 0
+ fi
+
+ nix build --no-link --print-build-logs .#code-mode-assets
+
+ # Losing the upload costs the next job a rebuild, not a failure.
+ if nix-store --export "$assets" | gzip | cache_request --upload-file - "$cache_url"; then
+ echo "Code-mode assets saved to the remote cache"
+ else
+ echo "::warning::Could not save code-mode assets to the remote cache"
+ fi
+ - name: build service binaries
+ run: |
+ set -euo pipefail
+
+ : "${RUST_PACKAGES:?RUST_PACKAGES is required}"
+
+ # The test job never compiles these services' `run` bodies: each is an `impl
+ # Future` polled only by the bin's `main`, and a bin's test target doesn't call
+ # `main`. Clippy stops before monomorphization. So a failure only a real build
+ # finds, such as a recursion limit overflow while instantiating, shows up here.
+ selected=()
+ for service in document_storage_service email_service authentication_service; do
+ if [ "$RUST_PACKAGES" = "all" ] || [[ " $RUST_PACKAGES " == *" $service "* ]]; then
+ selected+=("$service")
+ fi
+ done
+
+ if [ "${#selected[@]}" -eq 0 ]; then
+ echo "No service binaries were affected"
+ exit 0
+ fi
+
+ # The same packages and features as run_tests.sh. Building each service's
+ # `service_binary_builds` test target, which nothing runs, makes cargo build the
+ # real binaries under the features `cargo test` resolves, so every other unit is
+ # one the test job builds too and both jobs share sccache entries. `cargo build`
+ # would resolve without dev-dependency features and rebuild most of the workspace.
+ if [ "$RUST_PACKAGES" = "all" ]; then
+ package_args=(--workspace --exclude sync_service)
+ else
+ package_args=()
+ for package in $RUST_PACKAGES; do
+ [ "$package" = "sync_service" ] || package_args+=(-p "$package")
+ done
+ fi
+
+ cargo test --no-run --locked --all-features \
+ --config 'profile.dev.package."*".debug=false' "${package_args[@]}" \
+ --test service_binary_builds
+ - name: show sccache stats
+ if: always()
+ run: sccache --show-stats || true
+ - name: Teardown Nix
+ if: always()
+ uses: './.github/actions/teardown-nix'
graphql-schema-compat:
needs:
- path-check
@@ -1075,6 +1228,7 @@ jobs:
- check
- doppler-config
- test
+ - build-service-bins
- graphql-schema-compat
if: always()
name: Cloud Storage Status Check
@@ -1087,6 +1241,7 @@ jobs:
echo "check: ${{ needs.check.result }}"
echo "doppler-config: ${{ needs.doppler-config.result }}"
echo "test: ${{ needs.test.result }}"
+ echo "build-service-bins: ${{ needs.build-service-bins.result }}"
echo "graphql-schema-compat: ${{ needs.graphql-schema-compat.result }}"
# Only success and skipped are OK. A job that hits its timeout reports
@@ -1096,6 +1251,7 @@ jobs:
"${{ needs.check.result }}" \
"${{ needs.doppler-config.result }}" \
"${{ needs.test.result }}" \
+ "${{ needs.build-service-bins.result }}" \
"${{ needs.graphql-schema-compat.result }}"; do
if [[ "$result" != "success" && "$result" != "skipped" ]]; then
echo "❌ One or more jobs failed or were cancelled"
diff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.toml
index 1c0d94373d..1bf4e390db 100644
--- a/services/authentication_service/Cargo.toml
+++ b/services/authentication_service/Cargo.toml
@@ -21,6 +21,14 @@ path = "src/openapi.rs"
name = "authentication_service_doppler_config"
path = "src/doppler_config.rs"
+
+# Built only by the `build-service-bins` CI job, which needs the real binaries;
+# `test = false` keeps it out of the test job.
+[[test]]
+name = "service_binary_builds"
+path = "tests/service_binary_builds.rs"
+test = false
+
[features]
default = ["rate_limit"]
local = []
diff --git a/services/authentication_service/tests/service_binary_builds.rs b/services/authentication_service/tests/service_binary_builds.rs
new file mode 100644
index 0000000000..d46b18eb18
--- /dev/null
+++ b/services/authentication_service/tests/service_binary_builds.rs
@@ -0,0 +1,8 @@
+//! Built only by the `build-service-bins` CI job, never run: building it makes
+//! cargo build this package's real binaries under the same features as `cargo
+//! test`, so that job compiles nothing the test job doesn't already compile.
+
+#[test]
+fn service_binary_builds() {
+ assert!(std::path::Path::new(env!("CARGO_BIN_EXE_authentication_service")).exists());
+}
diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml
index a9efddf15f..13fb64c1f4 100644
--- a/services/document_storage_service/Cargo.toml
+++ b/services/document_storage_service/Cargo.toml
@@ -7,6 +7,7 @@ version = "0.1.0"
# The bins hold no tests and the library no doctests. Neither target builds under
# `cargo test`, so the test job compiles the library once, as its test binary.
+# The `build-service-bins` job builds the real binary.
[lib]
doctest = false
@@ -26,6 +27,14 @@ path = "src/doppler_config.rs"
test = false
+
+# Built only by the `build-service-bins` CI job, which needs the real binaries;
+# `test = false` keeps it out of the test job.
+[[test]]
+name = "service_binary_builds"
+path = "tests/service_binary_builds.rs"
+test = false
+
[features]
connection_gateway = []
default = ["connection_gateway", "delete_document_worker", "location_check"]
diff --git a/services/document_storage_service/tests/service_binary_builds.rs b/services/document_storage_service/tests/service_binary_builds.rs
new file mode 100644
index 0000000000..0400878a98
--- /dev/null
+++ b/services/document_storage_service/tests/service_binary_builds.rs
@@ -0,0 +1,8 @@
+//! Built only by the `build-service-bins` CI job, never run: building it makes
+//! cargo build this package's real binaries under the same features as `cargo
+//! test`, so that job compiles nothing the test job doesn't already compile.
+
+#[test]
+fn service_binary_builds() {
+ assert!(std::path::Path::new(env!("CARGO_BIN_EXE_document_storage_service")).exists());
+}
diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.toml
index f304887175..48079ebce4 100644
--- a/services/email_service/Cargo.toml
+++ b/services/email_service/Cargo.toml
@@ -61,6 +61,14 @@ path = "src/openapi.rs"
name = "email_service_doppler_config"
path = "src/doppler_config.rs"
+
+# Built only by the `build-service-bins` CI job, which needs the real binaries;
+# `test = false` keeps it out of the test job.
+[[test]]
+name = "service_binary_builds"
+path = "tests/service_binary_builds.rs"
+test = false
+
[features]
attachment_upload = []
connection_gateway = []
diff --git a/services/email_service/tests/service_binary_builds.rs b/services/email_service/tests/service_binary_builds.rs
new file mode 100644
index 0000000000..88b48d708d
--- /dev/null
+++ b/services/email_service/tests/service_binary_builds.rs
@@ -0,0 +1,8 @@
+//! Built only by the `build-service-bins` CI job, never run: building it makes
+//! cargo build this package's real binaries under the same features as `cargo
+//! test`, so that job compiles nothing the test job doesn't already compile.
+
+#[test]
+fn service_binary_builds() {
+ assert!(std::path::Path::new(env!("CARGO_BIN_EXE_email_service")).exists());
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
index 5bdc679ea0..99c7a6bd4d 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
@@ -7,7 +7,7 @@
//! AWS credentials anywhere).
//!
//! Critical path: `path-check` (no Rust compile on a warm binary cache) →
-//! `test`. `check` (fmt + clippy), `doppler-config`, and
+//! `test`. `check` (fmt + clippy), `build-service-bins`, `doppler-config`, and
//! `graphql-schema-compat` run alongside `test`.
use gh_workflow::{
@@ -41,6 +41,7 @@ pub fn code_check_cloud_storage() -> Workflow {
.add_job("check", check())
.add_job("doppler-config", doppler_config())
.add_job("test", test())
+ .add_job("build-service-bins", build_service_bins())
.add_job("graphql-schema-compat", graphql_schema_compat())
.add_job("status-check", status_check())
}
@@ -186,11 +187,7 @@ fn test() -> Job {
"${{ needs.path-check.outputs.rust_packages }}",
))
.add_env(("NEXTEST_TEST_THREADS", vars::NEXTEST_TEST_THREADS))
- // Debug info is most of what test builds write and link. Panic locations
- // don't depend on it; only backtrace frames lose their file and line.
- .add_env(("CARGO_PROFILE_DEV_DEBUG", "0"))
- .add_env(("CARGO_PROFILE_TEST_DEBUG", "0"))
- .add_env(("RUSTFLAGS", "-Dwarnings -C link-arg=-fuse-ld=mold"))
+ .map(with_test_compile_env)
// The default 10-minute idle timeout can stop the server during long
// link or test phases, which resets the stats reported at the end.
.add_env(("SCCACHE_IDLE_TIMEOUT", "0"))
@@ -212,6 +209,51 @@ fn test() -> Job {
.add_step(steps::teardown_nix())
}
+/// The env the jobs that compile for `test` share, so their sccache keys stay
+/// identical.
+fn with_test_compile_env(job: Job) -> Job {
+ // Debug info is most of what test builds write and link. Panic locations
+ // don't depend on it; only backtrace frames lose their file and line.
+ job.add_env(("CARGO_PROFILE_DEV_DEBUG", "0"))
+ .add_env(("CARGO_PROFILE_TEST_DEBUG", "0"))
+ .add_env(("RUSTFLAGS", "-Dwarnings -C link-arg=-fuse-ld=mold"))
+}
+
+/// Build the real binaries of the services whose `run` bodies only a real build
+/// compiles. The script picks them from the selected packages; this gate only
+/// avoids a dev shell when none can match.
+fn build_service_bins() -> Job {
+ steps::gated_job()
+ .timeout_minutes(45u32)
+ .cond(Expression::new(BUILD_SERVICE_BINS_CONDITION))
+ .runs_on(runners::Runner::Large.with_cache_tag(vars::CI_CACHE_TAG))
+ .add_env((
+ "RUST_PACKAGES",
+ "${{ needs.path-check.outputs.rust_packages }}",
+ ))
+ .map(with_test_compile_env)
+ .add_env(("SQLX_OFFLINE", "true"))
+ .add_step(steps::checkout(false, false))
+ .add_step(steps::mount_cache_volume())
+ .add_step(steps::setup_nix())
+ .add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES))
+ .add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
+ .add_step(steps::code_mode_assets())
+ .add_step(
+ Step::new("build service binaries").run(include_str!("scripts/build_service_bins.sh")),
+ )
+ .add_step(steps::show_sccache_stats())
+ .add_step(steps::teardown_nix())
+}
+
+const BUILD_SERVICE_BINS_CONDITION: &str = "needs.path-check.outputs.should_run == 'true' \
+ && github.event.pull_request.draft == false \
+ && needs.path-check.outputs.skip_tests != 'true' \
+ && (needs.path-check.outputs.rust_packages == 'all' \
+ || contains(needs.path-check.outputs.rust_packages, 'document_storage_service') \
+ || contains(needs.path-check.outputs.rust_packages, 'email_service') \
+ || contains(needs.path-check.outputs.rust_packages, 'authentication_service'))";
+
/// Install the same pinned runtime shipped in the agent harness container.
fn setup_execution_runtime() -> Step<gh_workflow::Use> {
Step::new("Set up code execution runtime")
@@ -251,6 +293,7 @@ fn status_check() -> Job {
"check".to_string(),
"doppler-config".to_string(),
"test".to_string(),
+ "build-service-bins".to_string(),
"graphql-schema-compat".to_string(),
])
.add_step(check_job_results())
@@ -439,6 +482,7 @@ fn check_job_results() -> Step<Run> {
echo "check: ${{ needs.check.result }}"
echo "doppler-config: ${{ needs.doppler-config.result }}"
echo "test: ${{ needs.test.result }}"
+ echo "build-service-bins: ${{ needs.build-service-bins.result }}"
echo "graphql-schema-compat: ${{ needs.graphql-schema-compat.result }}"
# Only success and skipped are OK. A job that hits its timeout reports
@@ -448,6 +492,7 @@ fn check_job_results() -> Step<Run> {
"${{ needs.check.result }}" \
"${{ needs.doppler-config.result }}" \
"${{ needs.test.result }}" \
+ "${{ needs.build-service-bins.result }}" \
"${{ needs.graphql-schema-compat.result }}"; do
if [[ "$result" != "success" && "$result" != "skipped" ]]; then
echo "❌ One or more jobs failed or were cancelled"
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
index 8825dbf80c..a9a9617426 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
@@ -74,6 +74,7 @@ fn status_check_covers_every_job() {
"check",
"doppler-config",
"test",
+ "build-service-bins",
"graphql-schema-compat",
] {
let check = format!(r#""${{{{ needs.{needed}.result }}}}""#);
@@ -352,3 +353,22 @@ fn test_postgres_keeps_its_data_in_shared_memory_across_a_restart() {
);
assert!(!options.contains("--tmpfs"), "{options}");
}
+
+/// The gate may run the job with nothing to build, but it must never hide a
+/// service the script builds, or a real-build failure in it passes the PR.
+#[test]
+fn service_bins_gate_covers_every_service_the_script_builds() {
+ let script = include_str!("../scripts/build_service_bins.sh");
+ let services = script
+ .split("for service in ")
+ .nth(1)
+ .and_then(|rest| rest.split("; do").next())
+ .expect("service list");
+
+ for service in services.split_whitespace() {
+ assert!(
+ BUILD_SERVICE_BINS_CONDITION.contains(&format!("'{service}'")),
+ "{service} is built by the script but missing from the job's gate"
+ );
+ }
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/build_service_bins.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/build_service_bins.sh
new file mode 100644
index 0000000000..3c4ea2bd04
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/build_service_bins.sh
@@ -0,0 +1,37 @@
+set -euo pipefail
+
+: "${RUST_PACKAGES:?RUST_PACKAGES is required}"
+
+# The test job never compiles these services' `run` bodies: each is an `impl
+# Future` polled only by the bin's `main`, and a bin's test target doesn't call
+# `main`. Clippy stops before monomorphization. So a failure only a real build
+# finds, such as a recursion limit overflow while instantiating, shows up here.
+selected=()
+for service in document_storage_service email_service authentication_service; do
+ if [ "$RUST_PACKAGES" = "all" ] || [[ " $RUST_PACKAGES " == *" $service "* ]]; then
+ selected+=("$service")
+ fi
+done
+
+if [ "${#selected[@]}" -eq 0 ]; then
+ echo "No service binaries were affected"
+ exit 0
+fi
+
+# The same packages and features as run_tests.sh. Building each service's
+# `service_binary_builds` test target, which nothing runs, makes cargo build the
+# real binaries under the features `cargo test` resolves, so every other unit is
+# one the test job builds too and both jobs share sccache entries. `cargo build`
+# would resolve without dev-dependency features and rebuild most of the workspace.
+if [ "$RUST_PACKAGES" = "all" ]; then
+ package_args=(--workspace --exclude sync_service)
+else
+ package_args=()
+ for package in $RUST_PACKAGES; do
+ [ "$package" = "sync_service" ] || package_args+=(-p "$package")
+ done
+fi
+
+cargo test --no-run --locked --all-features \
+ --config 'profile.dev.package."*".debug=false' "${package_args[@]}" \
+ --test service_binary_builds