Skip to content

The open-source app everyone uses to manage agents at work

[codex] Parallelize release verify workflow (#9168)

Dottacec0fc249a24
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index af04adf61d..abfe65bd10 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -58,6 +58,9 @@ jobs:
- name: Test general-server shard partition
run: node --test ./scripts/__tests__/run-vitest-stable-shard.test.mjs
+ - name: Test release verify workflow wiring
+ run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs
+
- name: Test standalone package build concurrency
run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs
diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml
new file mode 100644
index 0000000000..08bac6e4c8
--- /dev/null
+++ b/.github/workflows/release-verify.yml
@@ -0,0 +1,175 @@
+name: Release Verify
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ description: Commit SHA, branch, or tag to verify
+ required: true
+ type: string
+
+jobs:
+ typecheck:
+ name: Typecheck
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@v6
+ with:
+ version: 9.15.4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 24
+ cache: pnpm
+
+ - name: Validate release package manifest
+ run: node ./scripts/release-package-map.mjs check
+
+ - name: Install dependencies
+ run: pnpm install --no-frozen-lockfile
+
+ - name: Typecheck
+ run: pnpm -r typecheck
+
+ general_tests:
+ name: General tests (${{ matrix.group_label }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - group: general-server
+ group_label: server (1/3)
+ shard_index: 0
+ shard_count: 3
+ - group: general-server
+ group_label: server (2/3)
+ shard_index: 1
+ shard_count: 3
+ - group: general-server
+ group_label: server (3/3)
+ shard_index: 2
+ shard_count: 3
+ - group: general-workspaces-a
+ group_label: workspaces-a
+ - group: general-workspaces-b
+ group_label: workspaces-b
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@v6
+ with:
+ version: 9.15.4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 24
+ cache: pnpm
+
+ - name: Install dependencies
+ run: pnpm install --no-frozen-lockfile
+
+ - name: Run grouped general test suites
+ run: |
+ if [ -n "${{ matrix.shard_count }}" ]; then
+ pnpm test:run:general -- --group '${{ matrix.group }}' \
+ --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }}
+ else
+ pnpm test:run:general -- --group '${{ matrix.group }}'
+ fi
+
+ serialized_tests:
+ name: Serialized tests (${{ matrix.shard_label }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - shard_index: 0
+ shard_count: 4
+ shard_label: 1/4
+ - shard_index: 1
+ shard_count: 4
+ shard_label: 2/4
+ - shard_index: 2
+ shard_count: 4
+ shard_label: 3/4
+ - shard_index: 3
+ shard_count: 4
+ shard_label: 4/4
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@v6
+ with:
+ version: 9.15.4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 24
+ cache: pnpm
+
+ - name: Install dependencies
+ run: pnpm install --no-frozen-lockfile
+
+ - name: Run serialized server test shard
+ run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }}
+
+ build:
+ name: Build
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@v6
+ with:
+ version: 9.15.4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 24
+ cache: pnpm
+
+ - name: Install dependencies
+ run: pnpm install --no-frozen-lockfile
+
+ - name: Build
+ run: pnpm build
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f97fd15136..17233901f2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -28,42 +28,9 @@ concurrency:
jobs:
verify_canary:
if: github.event_name == 'push'
- runs-on: ubuntu-latest
- timeout-minutes: 30
- permissions:
- contents: read
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v7
- with:
- fetch-depth: 0
-
- - name: Setup pnpm
- uses: pnpm/action-setup@v6
- with:
- version: 9.15.4
-
- - name: Setup Node.js
- uses: actions/setup-node@v6
- with:
- node-version: 24
- cache: pnpm
-
- - name: Validate release package manifest
- run: node ./scripts/release-package-map.mjs check
-
- - name: Install dependencies
- run: pnpm install --no-frozen-lockfile
-
- - name: Typecheck
- run: pnpm -r typecheck
-
- - name: Run tests
- run: pnpm test:run
-
- - name: Build
- run: pnpm build
+ uses: ./.github/workflows/release-verify.yml
+ with:
+ ref: ${{ github.sha }}
publish_canary:
if: github.event_name == 'push'
@@ -122,43 +89,9 @@ jobs:
verify_stable:
if: github.event_name == 'workflow_dispatch'
- runs-on: ubuntu-latest
- timeout-minutes: 30
- permissions:
- contents: read
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v7
- with:
- fetch-depth: 0
- ref: ${{ inputs.source_ref }}
-
- - name: Setup pnpm
- uses: pnpm/action-setup@v6
- with:
- version: 9.15.4
-
- - name: Setup Node.js
- uses: actions/setup-node@v6
- with:
- node-version: 24
- cache: pnpm
-
- - name: Validate release package manifest
- run: node ./scripts/release-package-map.mjs check
-
- - name: Install dependencies
- run: pnpm install --no-frozen-lockfile
-
- - name: Typecheck
- run: pnpm -r typecheck
-
- - name: Run tests
- run: pnpm test:run
-
- - name: Build
- run: pnpm build
+ uses: ./.github/workflows/release-verify.yml
+ with:
+ ref: ${{ inputs.source_ref }}
preview_stable:
if: github.event_name == 'workflow_dispatch' && inputs.dry_run
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
new file mode 100644
index 0000000000..3bd0a616a8
--- /dev/null
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+function readWorkflow(name) {
+ return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8");
+}
+
+test("release workflow delegates stable and canary verification to the reusable workflow", () => {
+ const releaseWorkflow = readWorkflow("release.yml");
+
+ assert.match(
+ releaseWorkflow,
+ /verify_canary:\n\s+if: github\.event_name == 'push'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/,
+ );
+ assert.match(
+ releaseWorkflow,
+ /verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ inputs\.source_ref \}\}/,
+ );
+ assert.doesNotMatch(releaseWorkflow, /verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/);
+});
+
+test("release verify workflow covers the same split test surface as stable PR verification", () => {
+ const verifyWorkflow = readWorkflow("release-verify.yml");
+
+ assert.match(verifyWorkflow, /workflow_call:/);
+ assert.match(verifyWorkflow, /node \.\/scripts\/release-package-map\.mjs check/);
+ assert.match(verifyWorkflow, /pnpm -r typecheck/);
+ assert.match(verifyWorkflow, /pnpm build/);
+
+ for (const group of ["general-server", "general-workspaces-a", "general-workspaces-b"]) {
+ assert.match(verifyWorkflow, new RegExp(`group: ${group}`));
+ }
+
+ for (const shardIndex of [0, 1, 2]) {
+ assert.match(
+ verifyWorkflow,
+ new RegExp(`group: general-server[\\s\\S]*?shard_index: ${shardIndex}[\\s\\S]*?shard_count: 3`),
+ );
+ }
+
+ for (const shardIndex of [0, 1, 2, 3]) {
+ assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 4`));
+ }
+
+ assert.match(verifyWorkflow, /pnpm test:run:general -- --group/);
+ assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/);
+});
diff --git a/server/src/__tests__/heartbeat-worktree-suppression.test.ts b/server/src/__tests__/heartbeat-worktree-suppression.test.ts
index 3a3144dcef..b740512df9 100644
--- a/server/src/__tests__/heartbeat-worktree-suppression.test.ts
+++ b/server/src/__tests__/heartbeat-worktree-suppression.test.ts
@@ -133,29 +133,35 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {
return { companyId, agentId, issueId };
}
- async function waitForTerminalRun(runId: string) {
- for (let attempt = 0; attempt < 20; attempt += 1) {
+ async function waitForCompletedRun(runId: string, agentId: string) {
+ let latestStatus: string | null = null;
+ let latestLastRunId: string | null = null;
+
+ for (let attempt = 0; attempt < 100; attempt += 1) {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null);
- if (run && run.status !== "queued" && run.status !== "running") return run.status;
- await new Promise((resolve) => setTimeout(resolve, 25));
- }
- return null;
- }
-
- async function waitForRuntimeStateLastRun(agentId: string, runId: string) {
- for (let attempt = 0; attempt < 100; attempt += 1) {
const state = await db
.select({ lastRunId: agentRuntimeState.lastRunId })
.from(agentRuntimeState)
.where(eq(agentRuntimeState.agentId, agentId))
.then((rows) => rows[0] ?? null);
- if (state?.lastRunId === runId) return;
+
+ latestStatus = run?.status ?? null;
+ latestLastRunId = state?.lastRunId ?? null;
+
+ if (run && run.status !== "queued" && run.status !== "running" && state?.lastRunId === runId) {
+ return run.status;
+ }
+
await new Promise((resolve) => setTimeout(resolve, 50));
}
+
+ throw new Error(
+ `Timed out waiting for heartbeat run ${runId} to finish; latest status=${latestStatus ?? "missing"}, runtime lastRunId=${latestLastRunId ?? "missing"}`,
+ );
}
it("suppresses new assignment wakes in worktree instances without creating heartbeat runs", async () => {
@@ -239,6 +245,11 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {
it("still creates live-plane assignment runs when suppression is not active", async () => {
const { agentId, issueId } = await insertAgentAndIssue();
+ await db
+ .update(issues)
+ .set({ status: "in_review", updatedAt: new Date() })
+ .where(eq(issues.id, issueId));
+
const heartbeat = heartbeatService(db, { runtimeEnv: {} });
const run = await heartbeat.wakeup(agentId, {
@@ -246,27 +257,22 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {
triggerDetail: "system",
reason: "issue_assigned",
payload: { issueId },
- contextSnapshot: { issueId, wakeReason: "issue_assigned" },
+ contextSnapshot: { issueId, wakeReason: "issue_assigned", skipIssueComment: true },
requestedByActorType: "system",
requestedByActorId: "issue_assignment",
});
expect(run).not.toBeNull();
- const terminalStatus = await waitForTerminalRun(run!.id);
- expect(["succeeded", null]).toContain(terminalStatus);
+ const terminalStatus = await waitForCompletedRun(run!.id, agentId);
+ await heartbeat.waitForRunExecutionDrain(run!.id);
+ expect(terminalStatus).toBe("succeeded");
const runCount = await db
.select({ count: sql<number>`count(*)::int` })
.from(heartbeatRuns)
.then((rows) => rows[0]?.count ?? 0);
expect(runCount).toBe(1);
-
- await db
- .update(issues)
- .set({ status: "done", updatedAt: new Date() })
- .where(eq(issues.id, issueId));
- await waitForRuntimeStateLastRun(agentId, run!.id);
- });
+ }, 10_000);
it("recognizes explicit restore-in-progress suppression", () => {
expect(resolveHeartbeatSchedulingSuppression({
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index a97a4dadbc..620fb16d06 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -2640,6 +2640,8 @@ export function shouldResetTaskSessionForWake(
function shouldRequireIssueCommentForWake(
contextSnapshot: Record<string, unknown> | null | undefined,
) {
+ if (contextSnapshot?.skipIssueComment === true) return false;
+
const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
return (
wakeReason === "issue_assigned" ||
@@ -15043,6 +15045,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
}
return {
+ waitForRunExecutionDrain: async (
+ runId: string,
+ options: { timeoutMs?: number; intervalMs?: number } = {},
+ ) => {
+ const timeoutMs = options.timeoutMs ?? 5_000;
+ const intervalMs = options.intervalMs ?? 25;
+ const deadline = Date.now() + timeoutMs;
+
+ while (liveRunExecutions.has(runId)) {
+ if (Date.now() >= deadline) {
+ throw new Error(`Timed out waiting for heartbeat run ${runId} execution to drain`);
+ }
+ await new Promise((resolve) => setTimeout(resolve, intervalMs));
+ }
+ },
list: async (
companyId: string,
agentId?: string,