Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
ci(rust): stop stalled PR jobs and fail the required check on a timeout (#9773)
teo6e6777597442
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.ymlindex c4c0258b0a..e7c9a09002 100644--- a/.github/workflows/code_check_cloud_storage.yml+++ b/.github/workflows/code_check_cloud_storage.yml@@ -26,6 +26,7 @@ jobs:graphql_schema: ${{ steps.filter.outputs.graphql_schema }}rust_packages: ${{ steps.nextest-filter.outputs.rust_packages }}skip_tests: ${{ steps.nextest-filter.outputs.skip_tests }}+ timeout-minutes: 15steps:- name: Checkoutuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd@@ -303,6 +304,7 @@ jobs:RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold'RUSTDOCFLAGS: '-Dwarnings'SQLX_OFFLINE: 'true'+ timeout-minutes: 45steps:- name: Checkoutuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd@@ -323,6 +325,7 @@ jobs:uses: './.github/actions/setup-nix'- name: Setup Nix dev shelluses: './.github/actions/setup-nix-dev-shell'+ timeout-minutes: 15- name: Configure Namespace remote sccacheif: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repositoryrun: |-@@ -478,6 +481,7 @@ jobs:RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold'RUSTDOCFLAGS: '-Dwarnings'SQLX_OFFLINE: 'true'+ timeout-minutes: 45steps:- name: Checkoutuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd@@ -539,6 +543,7 @@ jobs:uses: './.github/actions/setup-nix'- name: Setup Nix dev shelluses: './.github/actions/setup-nix-dev-shell'+ timeout-minutes: 15- name: Configure Namespace remote sccacheif: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repositoryrun: |-@@ -687,6 +692,7 @@ jobs:CARGO_PROFILE_TEST_DEBUG: '0'RUSTFLAGS: '-Dwarnings -C link-arg=-fuse-ld=mold'SCCACHE_IDLE_TIMEOUT: '0'+ timeout-minutes: 90services:postgres:image: pgvector/pgvector:pg14@@ -722,6 +728,7 @@ jobs:uses: './.github/actions/setup-nix'- name: Setup Nix dev shelluses: './.github/actions/setup-nix-dev-shell'+ timeout-minutes: 15- name: Configure Namespace remote sccacheif: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repositoryrun: |-@@ -952,6 +959,7 @@ jobs:- path-checkif: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.graphql_schema == 'true'runs-on: namespace-profile-linux-small+ timeout-minutes: 15steps:- name: Checkoutuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd@@ -976,6 +984,7 @@ jobs:if: always()name: Cloud Storage Status Checkruns-on: namespace-profile-linux-small+ timeout-minutes: 5steps:- name: Check job resultsrun: |@@ -985,15 +994,19 @@ jobs:echo "test: ${{ needs.test.result }}"echo "graphql-schema-compat: ${{ needs.graphql-schema-compat.result }}"- # Fail if any job failed (skipped and success are both OK)- if [[ "${{ needs.path-check.result }}" == "failure" ]] || \- [[ "${{ needs.check.result }}" == "failure" ]] || \- [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \- [[ "${{ needs.test.result }}" == "failure" ]] || \- [[ "${{ needs.graphql-schema-compat.result }}" == "failure" ]]; then- echo "❌ One or more jobs failed"- exit 1- fi+ # Only success and skipped are OK. A job that hits its timeout reports+ # cancelled, and passing on that would merge code whose tests never ran.+ for result in \+ "${{ needs.path-check.result }}" \+ "${{ needs.check.result }}" \+ "${{ needs.doppler-config.result }}" \+ "${{ needs.test.result }}" \+ "${{ needs.graphql-schema-compat.result }}"; do+ if [[ "$result" != "success" && "$result" != "skipped" ]]; then+ echo "❌ One or more jobs failed or were cancelled"+ exit 1+ fi+ doneecho "✅ All jobs passed or were skipped"concurrency:diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.ymlindex 10615d957a..f9ec4fa6e7 100644--- a/.github/workflows/web-app-check-main.yml+++ b/.github/workflows/web-app-check-main.yml@@ -336,6 +336,7 @@ jobs:env:CARGO_PROFILE_DEV_DEBUG: '0'RUSTFLAGS: '-C link-arg=-fuse-ld=mold'+ timeout-minutes: 45steps:- name: Checkout Repouses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10@@ -352,6 +353,7 @@ jobs:uses: './.github/actions/setup-nix'- name: Setup Prereqsuses: './.github/actions/setup-reqs-web'+ timeout-minutes: 15- name: Configure Namespace remote sccacheif: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repositoryrun: |-@@ -1060,11 +1062,13 @@ jobs:echo "ios-native-check: ${{ needs.ios-native-check.result }}"# Fail if any job failed (skipped and success are both OK). The iOS- # gate is the merge gate, so a cancelled gate is not OK either.+ # gate is the merge gate, so a cancelled gate is not OK either. The same+ # goes for generated code: a job that hits its timeout reports cancelled.if [[ "${{ needs.android-native-check.result }}" != "success" ]] || \[[ "${{ needs.path-check.result }}" == "failure" ]] || \[[ "${{ needs.typescript.result }}" == "failure" ]] || \[[ "${{ needs.generated-code.result }}" == "failure" ]] || \+ [[ "${{ needs.generated-code.result }}" == "cancelled" ]] || \[[ "${{ needs.biome-check.result }}" == "failure" ]] || \[[ "${{ needs.test.result }}" == "failure" ]] || \[[ "${{ needs.cycles.result }}" == "failure" ]] || \diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rsindex 00ef805a92..aa56a9f74f 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs@@ -60,6 +60,7 @@ const FILTER_CACHE_DIR: RuntimePath<'static> =/// it here.fn path_check() -> Job {Job::default()+ .timeout_minutes(15u32).runs_on(runners::Runner::Small.with_cache_tag(vars::CI_CACHE_TAG)).add_output("should_run", "${{ steps.filter.outputs.should_run }}").add_output(@@ -99,12 +100,13 @@ fn path_check() -> Job {/// fmt + clippy.fn check() -> Job {steps::gated_job()+ .timeout_minutes(45u32).runs_on(runners::Runner::Large.with_cache_tag(vars::CI_CACHE_TAG)).map(with_check_env).add_step(steps::checkout(false, false)).add_step(steps::mount_cache_volume()).add_step(steps::setup_nix())- .add_step(steps::setup_dev_shell())+ .add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES)).add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME)).add_step(steps::code_mode_assets()).add_step(cargo_fmt())@@ -129,6 +131,7 @@ fn with_check_env(job: Job) -> Job {/// path filter gates it; the xtask then picks the exact binaries.fn doppler_config() -> Job {steps::gated_job()+ .timeout_minutes(45u32).cond(Expression::new("needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.doppler_candidates == 'true'",))@@ -140,7 +143,7 @@ fn doppler_config() -> Job {.add_step(compute_changed_files()).add_step(steps::mount_cache_volume()).add_step(steps::setup_nix())- .add_step(steps::setup_dev_shell())+ .add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES)).add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME)).add_step(steps::code_mode_assets()).add_step(compute_doppler_bins())@@ -154,6 +157,7 @@ fn doppler_config() -> Job {/// merge commit, so its first parent is the base branch tip.fn graphql_schema_compat() -> Job {steps::gated_job()+ .timeout_minutes(15u32).cond(Expression::new("needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.graphql_schema == 'true'",))@@ -172,6 +176,7 @@ fn graphql_schema_compat() -> Job {/// when `skip_tests` is set (`.sqlx`-only diffs still clippy the snapshot).fn test() -> Job {steps::gated_job()+ .timeout_minutes(90u32).cond(Expression::new("needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.rust_packages != 'none' && needs.path-check.outputs.skip_tests != 'true'",))@@ -194,7 +199,7 @@ fn test() -> Job {.add_step(steps::checkout(false, false)).add_step(steps::mount_cache_volume()).add_step(steps::setup_nix())- .add_step(steps::setup_dev_shell())+ .add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES)).add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME)).add_step(steps::start_sccache_server()).add_step(steps::code_mode_assets())@@ -235,6 +240,7 @@ fn test_execution_runtime() -> Step<Run> {/// stable — branch protection references it.fn status_check() -> Job {Job::default()+ .timeout_minutes(5u32).name("Cloud Storage Status Check").runs_on(runners::Runner::Small.to_string()).cond(Expression::new("always()"))@@ -433,15 +439,19 @@ fn check_job_results() -> Step<Run> {echo "test: ${{ needs.test.result }}"echo "graphql-schema-compat: ${{ needs.graphql-schema-compat.result }}"- # Fail if any job failed (skipped and success are both OK)- if [[ "${{ needs.path-check.result }}" == "failure" ]] || \- [[ "${{ needs.check.result }}" == "failure" ]] || \- [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \- [[ "${{ needs.test.result }}" == "failure" ]] || \- [[ "${{ needs.graphql-schema-compat.result }}" == "failure" ]]; then- echo "❌ One or more jobs failed"- exit 1- fi+ # Only success and skipped are OK. A job that hits its timeout reports+ # cancelled, and passing on that would merge code whose tests never ran.+ for result in \+ "${{ needs.path-check.result }}" \+ "${{ needs.check.result }}" \+ "${{ needs.doppler-config.result }}" \+ "${{ needs.test.result }}" \+ "${{ needs.graphql-schema-compat.result }}"; do+ if [[ "$result" != "success" && "$result" != "skipped" ]]; then+ echo "❌ One or more jobs failed or were cancelled"+ exit 1+ fi+ doneecho "✅ All jobs passed or were skipped""#})diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rsindex 0193e8d16a..8825dbf80c 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs@@ -76,7 +76,7 @@ fn status_check_covers_every_job() {"test","graphql-schema-compat",] {- let check = format!(r#"[[ "${{{{ needs.{needed}.result }}}}" == "failure" ]]"#);+ let check = format!(r#""${{{{ needs.{needed}.result }}}}""#);assert!(script.contains(&check), "{check}: {script}");assert!(status["needs"]@@ -291,6 +291,45 @@ fn schema_changes_run_the_compatibility_check() {assert!(job.contains("graphql-schema-compat --base HEAD^1"), "{job}");}+/// A stalled step must not hold a runner for the default six hours, and the dev+/// shell, where stalls have happened, must fail well before the job limit.+#[test]+fn rust_jobs_have_timeouts() {+ for job in [+ path_check(),+ check(),+ doppler_config(),+ graphql_schema_compat(),+ test(),+ status_check(),+ ] {+ let job = serde_json::to_value(job).unwrap();+ assert!(job["timeout-minutes"].is_u64(), "{job}");+ }+ for job in [check(), doppler_config(), test()] {+ let job = serde_json::to_value(job).unwrap();+ let shell = job["steps"]+ .as_array()+ .unwrap()+ .iter()+ .find(|step| step["name"] == "Setup Nix dev shell")+ .expect("dev shell step");+ assert_eq!(shell["timeout-minutes"], vars::DEV_SHELL_TIMEOUT_MINUTES);+ }+}++/// A job that hits its timeout reports `cancelled`, so the required check can't+/// treat only `failure` as a problem.+#[test]+fn status_check_rejects_cancelled_jobs() {+ let status = serde_json::to_value(status_check()).unwrap();+ let script = status["steps"][0]["run"].as_str().unwrap();+ assert!(+ script.contains(r#"[[ "$result" != "success" && "$result" != "skipped" ]]"#),+ "{script}"+ );+}+/// Plain `cargo test` here rebuilt hundreds of crates under different flags to run/// about four seconds of tests.#[test]diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rsindex b43a015d5b..5177257957 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs@@ -47,6 +47,11 @@ pub const NIX_CACHE_URL: &str = "${{ vars.NIX_CACHE_URL }}";/// paths verify.pub const NIX_CACHE_PUBLIC_KEY: &str = "${{ vars.NIX_CACHE_PUBLIC_KEY }}";+/// Limit on entering the Nix dev shell. Healthy runs take under 9 minutes even+/// from an empty cache volume, while a stalled store download has run for over+/// an hour with no other limit to stop it.+pub const DEV_SHELL_TIMEOUT_MINUTES: u32 = 15;+/// Nextest thread count for the test job, one per vCPU of [`Runner::Large`].////// [`Runner::Large`]: crate::workflows::runners::Runner::Largediff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rsindex 31a74a6bf6..9422c80f5e 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs@@ -144,6 +144,7 @@ fn generated_code() -> Job {"needs.path-check.outputs.codegen_changed == 'true'",)).name("Generated Code Check")+ .timeout_minutes(45u32).runs_on(mid_web_runner())// The generators run once and are thrown away, so debug info is pure// cost: it made the 15 binaries 12.7 GB instead of 466 MB. Job-level,@@ -158,7 +159,10 @@ fn generated_code() -> Job {.add_step(checkout("Checkout Repo", false)).add_step(steps::mount_web_cache_volume(true)).add_step(steps::setup_nix())- .add_step(steps::setup_reqs_web("Setup Prereqs", false))+ .add_step(+ steps::setup_reqs_web("Setup Prereqs", false)+ .timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES),+ ).add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME)).add_step(steps::code_mode_assets()).add_step(build_schema_binaries())@@ -542,11 +546,13 @@ fn check_job_results() -> Step<Run> {echo "ios-native-check: ${{ needs.ios-native-check.result }}"# Fail if any job failed (skipped and success are both OK). The iOS- # gate is the merge gate, so a cancelled gate is not OK either.+ # gate is the merge gate, so a cancelled gate is not OK either. The same+ # goes for generated code: a job that hits its timeout reports cancelled.if [[ "${{ needs.android-native-check.result }}" != "success" ]] || \[[ "${{ needs.path-check.result }}" == "failure" ]] || \[[ "${{ needs.typescript.result }}" == "failure" ]] || \[[ "${{ needs.generated-code.result }}" == "failure" ]] || \+ [[ "${{ needs.generated-code.result }}" == "cancelled" ]] || \[[ "${{ needs.biome-check.result }}" == "failure" ]] || \[[ "${{ needs.test.result }}" == "failure" ]] || \[[ "${{ needs.cycles.result }}" == "failure" ]] || \diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rsindex 35bb984a99..ff544270e4 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs@@ -658,3 +658,11 @@ fn ios_gate_status_and_comment_scenarios() {String::from_utf8_lossy(&output.stderr));}++/// A job that hits its timeout reports `cancelled`, so the required check must+/// not pass on a generated code job that never finished.+#[test]+fn status_check_rejects_a_cancelled_generated_code_job() {+ let yaml = web_app_check_main().to_string().expect("workflow yaml");+ assert!(yaml.contains(r#"[[ "${{ needs.generated-code.result }}" == "cancelled" ]]"#));+}