Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

ci(rust): stop stalled PR jobs and fail the required check on a timeout (#9773)

teo6e6777597442
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml
index c4c0258b0a..e7c9a09002 100644
--- a/.github/workflows/code_check_cloud_storage.yml
+++ b/.github/workflows/code_check_cloud_storage.yml
@@ -26,6 +26,7 @@ jobs:
graphql_schema: ${{ steps.filter.outputs.graphql_schema }}
rust_packages: ${{ steps.nextest-filter.outputs.rust_packages }}
skip_tests: ${{ steps.nextest-filter.outputs.skip_tests }}
+ timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -303,6 +304,7 @@ jobs:
RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold'
RUSTDOCFLAGS: '-Dwarnings'
SQLX_OFFLINE: 'true'
+ timeout-minutes: 45
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -323,6 +325,7 @@ jobs:
uses: './.github/actions/setup-nix'
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
+ timeout-minutes: 15
- name: Configure Namespace remote sccache
if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
run: |-
@@ -478,6 +481,7 @@ jobs:
RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold'
RUSTDOCFLAGS: '-Dwarnings'
SQLX_OFFLINE: 'true'
+ timeout-minutes: 45
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -539,6 +543,7 @@ jobs:
uses: './.github/actions/setup-nix'
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
+ timeout-minutes: 15
- name: Configure Namespace remote sccache
if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
run: |-
@@ -687,6 +692,7 @@ jobs:
CARGO_PROFILE_TEST_DEBUG: '0'
RUSTFLAGS: '-Dwarnings -C link-arg=-fuse-ld=mold'
SCCACHE_IDLE_TIMEOUT: '0'
+ timeout-minutes: 90
services:
postgres:
image: pgvector/pgvector:pg14
@@ -722,6 +728,7 @@ jobs:
uses: './.github/actions/setup-nix'
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
+ timeout-minutes: 15
- name: Configure Namespace remote sccache
if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
run: |-
@@ -952,6 +959,7 @@ jobs:
- path-check
if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.graphql_schema == 'true'
runs-on: namespace-profile-linux-small
+ timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -976,6 +984,7 @@ jobs:
if: always()
name: Cloud Storage Status Check
runs-on: namespace-profile-linux-small
+ timeout-minutes: 5
steps:
- name: Check job results
run: |
@@ -985,15 +994,19 @@ jobs:
echo "test: ${{ needs.test.result }}"
echo "graphql-schema-compat: ${{ needs.graphql-schema-compat.result }}"
- # Fail if any job failed (skipped and success are both OK)
- if [[ "${{ needs.path-check.result }}" == "failure" ]] || \
- [[ "${{ needs.check.result }}" == "failure" ]] || \
- [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \
- [[ "${{ needs.test.result }}" == "failure" ]] || \
- [[ "${{ needs.graphql-schema-compat.result }}" == "failure" ]]; then
- echo "❌ One or more jobs failed"
- exit 1
- fi
+ # Only success and skipped are OK. A job that hits its timeout reports
+ # cancelled, and passing on that would merge code whose tests never ran.
+ for result in \
+ "${{ needs.path-check.result }}" \
+ "${{ needs.check.result }}" \
+ "${{ needs.doppler-config.result }}" \
+ "${{ needs.test.result }}" \
+ "${{ needs.graphql-schema-compat.result }}"; do
+ if [[ "$result" != "success" && "$result" != "skipped" ]]; then
+ echo "❌ One or more jobs failed or were cancelled"
+ exit 1
+ fi
+ done
echo "✅ All jobs passed or were skipped"
concurrency:
diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.yml
index 10615d957a..f9ec4fa6e7 100644
--- a/.github/workflows/web-app-check-main.yml
+++ b/.github/workflows/web-app-check-main.yml
@@ -336,6 +336,7 @@ jobs:
env:
CARGO_PROFILE_DEV_DEBUG: '0'
RUSTFLAGS: '-C link-arg=-fuse-ld=mold'
+ timeout-minutes: 45
steps:
- name: Checkout Repo
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
@@ -352,6 +353,7 @@ jobs:
uses: './.github/actions/setup-nix'
- name: Setup Prereqs
uses: './.github/actions/setup-reqs-web'
+ timeout-minutes: 15
- name: Configure Namespace remote sccache
if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
run: |-
@@ -1060,11 +1062,13 @@ jobs:
echo "ios-native-check: ${{ needs.ios-native-check.result }}"
# Fail if any job failed (skipped and success are both OK). The iOS
- # gate is the merge gate, so a cancelled gate is not OK either.
+ # gate is the merge gate, so a cancelled gate is not OK either. The same
+ # goes for generated code: a job that hits its timeout reports cancelled.
if [[ "${{ needs.android-native-check.result }}" != "success" ]] || \
[[ "${{ needs.path-check.result }}" == "failure" ]] || \
[[ "${{ needs.typescript.result }}" == "failure" ]] || \
[[ "${{ needs.generated-code.result }}" == "failure" ]] || \
+ [[ "${{ needs.generated-code.result }}" == "cancelled" ]] || \
[[ "${{ needs.biome-check.result }}" == "failure" ]] || \
[[ "${{ needs.test.result }}" == "failure" ]] || \
[[ "${{ needs.cycles.result }}" == "failure" ]] || \
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
index 00ef805a92..aa56a9f74f 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
@@ -60,6 +60,7 @@ const FILTER_CACHE_DIR: RuntimePath<'static> =
/// it here.
fn path_check() -> Job {
Job::default()
+ .timeout_minutes(15u32)
.runs_on(runners::Runner::Small.with_cache_tag(vars::CI_CACHE_TAG))
.add_output("should_run", "${{ steps.filter.outputs.should_run }}")
.add_output(
@@ -99,12 +100,13 @@ fn path_check() -> Job {
/// fmt + clippy.
fn check() -> Job {
steps::gated_job()
+ .timeout_minutes(45u32)
.runs_on(runners::Runner::Large.with_cache_tag(vars::CI_CACHE_TAG))
.map(with_check_env)
.add_step(steps::checkout(false, false))
.add_step(steps::mount_cache_volume())
.add_step(steps::setup_nix())
- .add_step(steps::setup_dev_shell())
+ .add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES))
.add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
.add_step(steps::code_mode_assets())
.add_step(cargo_fmt())
@@ -129,6 +131,7 @@ fn with_check_env(job: Job) -> Job {
/// path filter gates it; the xtask then picks the exact binaries.
fn doppler_config() -> Job {
steps::gated_job()
+ .timeout_minutes(45u32)
.cond(Expression::new(
"needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.doppler_candidates == 'true'",
))
@@ -140,7 +143,7 @@ fn doppler_config() -> Job {
.add_step(compute_changed_files())
.add_step(steps::mount_cache_volume())
.add_step(steps::setup_nix())
- .add_step(steps::setup_dev_shell())
+ .add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES))
.add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
.add_step(steps::code_mode_assets())
.add_step(compute_doppler_bins())
@@ -154,6 +157,7 @@ fn doppler_config() -> Job {
/// merge commit, so its first parent is the base branch tip.
fn graphql_schema_compat() -> Job {
steps::gated_job()
+ .timeout_minutes(15u32)
.cond(Expression::new(
"needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.graphql_schema == 'true'",
))
@@ -172,6 +176,7 @@ fn graphql_schema_compat() -> Job {
/// when `skip_tests` is set (`.sqlx`-only diffs still clippy the snapshot).
fn test() -> Job {
steps::gated_job()
+ .timeout_minutes(90u32)
.cond(Expression::new(
"needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.rust_packages != 'none' && needs.path-check.outputs.skip_tests != 'true'",
))
@@ -194,7 +199,7 @@ fn test() -> Job {
.add_step(steps::checkout(false, false))
.add_step(steps::mount_cache_volume())
.add_step(steps::setup_nix())
- .add_step(steps::setup_dev_shell())
+ .add_step(steps::setup_dev_shell().timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES))
.add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
.add_step(steps::start_sccache_server())
.add_step(steps::code_mode_assets())
@@ -235,6 +240,7 @@ fn test_execution_runtime() -> Step<Run> {
/// stable — branch protection references it.
fn status_check() -> Job {
Job::default()
+ .timeout_minutes(5u32)
.name("Cloud Storage Status Check")
.runs_on(runners::Runner::Small.to_string())
.cond(Expression::new("always()"))
@@ -433,15 +439,19 @@ fn check_job_results() -> Step<Run> {
echo "test: ${{ needs.test.result }}"
echo "graphql-schema-compat: ${{ needs.graphql-schema-compat.result }}"
- # Fail if any job failed (skipped and success are both OK)
- if [[ "${{ needs.path-check.result }}" == "failure" ]] || \
- [[ "${{ needs.check.result }}" == "failure" ]] || \
- [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \
- [[ "${{ needs.test.result }}" == "failure" ]] || \
- [[ "${{ needs.graphql-schema-compat.result }}" == "failure" ]]; then
- echo "❌ One or more jobs failed"
- exit 1
- fi
+ # Only success and skipped are OK. A job that hits its timeout reports
+ # cancelled, and passing on that would merge code whose tests never ran.
+ for result in \
+ "${{ needs.path-check.result }}" \
+ "${{ needs.check.result }}" \
+ "${{ needs.doppler-config.result }}" \
+ "${{ needs.test.result }}" \
+ "${{ needs.graphql-schema-compat.result }}"; do
+ if [[ "$result" != "success" && "$result" != "skipped" ]]; then
+ echo "❌ One or more jobs failed or were cancelled"
+ exit 1
+ fi
+ done
echo "✅ All jobs passed or were skipped"
"#})
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
index 0193e8d16a..8825dbf80c 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
@@ -76,7 +76,7 @@ fn status_check_covers_every_job() {
"test",
"graphql-schema-compat",
] {
- let check = format!(r#"[[ "${{{{ needs.{needed}.result }}}}" == "failure" ]]"#);
+ let check = format!(r#""${{{{ needs.{needed}.result }}}}""#);
assert!(script.contains(&check), "{check}: {script}");
assert!(
status["needs"]
@@ -291,6 +291,45 @@ fn schema_changes_run_the_compatibility_check() {
assert!(job.contains("graphql-schema-compat --base HEAD^1"), "{job}");
}
+/// A stalled step must not hold a runner for the default six hours, and the dev
+/// shell, where stalls have happened, must fail well before the job limit.
+#[test]
+fn rust_jobs_have_timeouts() {
+ for job in [
+ path_check(),
+ check(),
+ doppler_config(),
+ graphql_schema_compat(),
+ test(),
+ status_check(),
+ ] {
+ let job = serde_json::to_value(job).unwrap();
+ assert!(job["timeout-minutes"].is_u64(), "{job}");
+ }
+ for job in [check(), doppler_config(), test()] {
+ let job = serde_json::to_value(job).unwrap();
+ let shell = job["steps"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|step| step["name"] == "Setup Nix dev shell")
+ .expect("dev shell step");
+ assert_eq!(shell["timeout-minutes"], vars::DEV_SHELL_TIMEOUT_MINUTES);
+ }
+}
+
+/// A job that hits its timeout reports `cancelled`, so the required check can't
+/// treat only `failure` as a problem.
+#[test]
+fn status_check_rejects_cancelled_jobs() {
+ let status = serde_json::to_value(status_check()).unwrap();
+ let script = status["steps"][0]["run"].as_str().unwrap();
+ assert!(
+ script.contains(r#"[[ "$result" != "success" && "$result" != "skipped" ]]"#),
+ "{script}"
+ );
+}
+
/// Plain `cargo test` here rebuilt hundreds of crates under different flags to run
/// about four seconds of tests.
#[test]
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
index b43a015d5b..5177257957 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
@@ -47,6 +47,11 @@ pub const NIX_CACHE_URL: &str = "${{ vars.NIX_CACHE_URL }}";
/// paths verify.
pub const NIX_CACHE_PUBLIC_KEY: &str = "${{ vars.NIX_CACHE_PUBLIC_KEY }}";
+/// Limit on entering the Nix dev shell. Healthy runs take under 9 minutes even
+/// from an empty cache volume, while a stalled store download has run for over
+/// an hour with no other limit to stop it.
+pub const DEV_SHELL_TIMEOUT_MINUTES: u32 = 15;
+
/// Nextest thread count for the test job, one per vCPU of [`Runner::Large`].
///
/// [`Runner::Large`]: crate::workflows::runners::Runner::Large
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
index 31a74a6bf6..9422c80f5e 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
@@ -144,6 +144,7 @@ fn generated_code() -> Job {
"needs.path-check.outputs.codegen_changed == 'true'",
))
.name("Generated Code Check")
+ .timeout_minutes(45u32)
.runs_on(mid_web_runner())
// The generators run once and are thrown away, so debug info is pure
// cost: it made the 15 binaries 12.7 GB instead of 466 MB. Job-level,
@@ -158,7 +159,10 @@ fn generated_code() -> Job {
.add_step(checkout("Checkout Repo", false))
.add_step(steps::mount_web_cache_volume(true))
.add_step(steps::setup_nix())
- .add_step(steps::setup_reqs_web("Setup Prereqs", false))
+ .add_step(
+ steps::setup_reqs_web("Setup Prereqs", false)
+ .timeout_minutes(vars::DEV_SHELL_TIMEOUT_MINUTES),
+ )
.add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
.add_step(steps::code_mode_assets())
.add_step(build_schema_binaries())
@@ -542,11 +546,13 @@ fn check_job_results() -> Step<Run> {
echo "ios-native-check: ${{ needs.ios-native-check.result }}"
# Fail if any job failed (skipped and success are both OK). The iOS
- # gate is the merge gate, so a cancelled gate is not OK either.
+ # gate is the merge gate, so a cancelled gate is not OK either. The same
+ # goes for generated code: a job that hits its timeout reports cancelled.
if [[ "${{ needs.android-native-check.result }}" != "success" ]] || \
[[ "${{ needs.path-check.result }}" == "failure" ]] || \
[[ "${{ needs.typescript.result }}" == "failure" ]] || \
[[ "${{ needs.generated-code.result }}" == "failure" ]] || \
+ [[ "${{ needs.generated-code.result }}" == "cancelled" ]] || \
[[ "${{ needs.biome-check.result }}" == "failure" ]] || \
[[ "${{ needs.test.result }}" == "failure" ]] || \
[[ "${{ needs.cycles.result }}" == "failure" ]] || \
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
index 35bb984a99..ff544270e4 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
@@ -658,3 +658,11 @@ fn ios_gate_status_and_comment_scenarios() {
String::from_utf8_lossy(&output.stderr)
);
}
+
+/// A job that hits its timeout reports `cancelled`, so the required check must
+/// not pass on a generated code job that never finished.
+#[test]
+fn status_check_rejects_a_cancelled_generated_code_job() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ assert!(yaml.contains(r#"[[ "${{ needs.generated-code.result }}" == "cancelled" ]]"#));
+}