The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
[codex] Parallelize release verify workflow (#9168)
Dottacec0fc249a24
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.ymlindex af04adf61d..abfe65bd10 100644--- a/.github/workflows/pr.yml+++ b/.github/workflows/pr.yml@@ -58,6 +58,9 @@ jobs:- name: Test general-server shard partitionrun: node --test ./scripts/__tests__/run-vitest-stable-shard.test.mjs+ - name: Test release verify workflow wiring+ run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs+- name: Test standalone package build concurrencyrun: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjsdiff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.ymlnew file mode 100644index 0000000000..08bac6e4c8--- /dev/null+++ b/.github/workflows/release-verify.yml@@ -0,0 +1,175 @@+name: Release Verify++on:+ workflow_call:+ inputs:+ ref:+ description: Commit SHA, branch, or tag to verify+ required: true+ type: string++jobs:+ typecheck:+ name: Typecheck+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: read++ steps:+ - name: Checkout repository+ uses: actions/checkout@v7+ with:+ ref: ${{ inputs.ref }}++ - name: Setup pnpm+ uses: pnpm/action-setup@v6+ with:+ version: 9.15.4++ - name: Setup Node.js+ uses: actions/setup-node@v6+ with:+ node-version: 24+ cache: pnpm++ - name: Validate release package manifest+ run: node ./scripts/release-package-map.mjs check++ - name: Install dependencies+ run: pnpm install --no-frozen-lockfile++ - name: Typecheck+ run: pnpm -r typecheck++ general_tests:+ name: General tests (${{ matrix.group_label }})+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: read+ strategy:+ fail-fast: false+ matrix:+ include:+ - group: general-server+ group_label: server (1/3)+ shard_index: 0+ shard_count: 3+ - group: general-server+ group_label: server (2/3)+ shard_index: 1+ shard_count: 3+ - group: general-server+ group_label: server (3/3)+ shard_index: 2+ shard_count: 3+ - group: general-workspaces-a+ group_label: workspaces-a+ - group: general-workspaces-b+ group_label: workspaces-b++ steps:+ - name: Checkout repository+ uses: actions/checkout@v7+ with:+ ref: ${{ inputs.ref }}++ - name: Setup pnpm+ uses: pnpm/action-setup@v6+ with:+ version: 9.15.4++ - name: Setup Node.js+ uses: actions/setup-node@v6+ with:+ node-version: 24+ cache: pnpm++ - name: Install dependencies+ run: pnpm install --no-frozen-lockfile++ - name: Run grouped general test suites+ run: |+ if [ -n "${{ matrix.shard_count }}" ]; then+ pnpm test:run:general -- --group '${{ matrix.group }}' \+ --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }}+ else+ pnpm test:run:general -- --group '${{ matrix.group }}'+ fi++ serialized_tests:+ name: Serialized tests (${{ matrix.shard_label }})+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: read+ strategy:+ fail-fast: false+ matrix:+ include:+ - shard_index: 0+ shard_count: 4+ shard_label: 1/4+ - shard_index: 1+ shard_count: 4+ shard_label: 2/4+ - shard_index: 2+ shard_count: 4+ shard_label: 3/4+ - shard_index: 3+ shard_count: 4+ shard_label: 4/4++ steps:+ - name: Checkout repository+ uses: actions/checkout@v7+ with:+ ref: ${{ inputs.ref }}++ - name: Setup pnpm+ uses: pnpm/action-setup@v6+ with:+ version: 9.15.4++ - name: Setup Node.js+ uses: actions/setup-node@v6+ with:+ node-version: 24+ cache: pnpm++ - name: Install dependencies+ run: pnpm install --no-frozen-lockfile++ - name: Run serialized server test shard+ run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }}++ build:+ name: Build+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: read++ steps:+ - name: Checkout repository+ uses: actions/checkout@v7+ with:+ ref: ${{ inputs.ref }}++ - name: Setup pnpm+ uses: pnpm/action-setup@v6+ with:+ version: 9.15.4++ - name: Setup Node.js+ uses: actions/setup-node@v6+ with:+ node-version: 24+ cache: pnpm++ - name: Install dependencies+ run: pnpm install --no-frozen-lockfile++ - name: Build+ run: pnpm builddiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex f97fd15136..17233901f2 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -28,42 +28,9 @@ concurrency:jobs:verify_canary:if: github.event_name == 'push'- runs-on: ubuntu-latest- timeout-minutes: 30- permissions:- contents: read-- steps:- - name: Checkout repository- uses: actions/checkout@v7- with:- fetch-depth: 0-- - name: Setup pnpm- uses: pnpm/action-setup@v6- with:- version: 9.15.4-- - name: Setup Node.js- uses: actions/setup-node@v6- with:- node-version: 24- cache: pnpm-- - name: Validate release package manifest- run: node ./scripts/release-package-map.mjs check-- - name: Install dependencies- run: pnpm install --no-frozen-lockfile-- - name: Typecheck- run: pnpm -r typecheck-- - name: Run tests- run: pnpm test:run-- - name: Build- run: pnpm build+ uses: ./.github/workflows/release-verify.yml+ with:+ ref: ${{ github.sha }}publish_canary:if: github.event_name == 'push'@@ -122,43 +89,9 @@ jobs:verify_stable:if: github.event_name == 'workflow_dispatch'- runs-on: ubuntu-latest- timeout-minutes: 30- permissions:- contents: read-- steps:- - name: Checkout repository- uses: actions/checkout@v7- with:- fetch-depth: 0- ref: ${{ inputs.source_ref }}-- - name: Setup pnpm- uses: pnpm/action-setup@v6- with:- version: 9.15.4-- - name: Setup Node.js- uses: actions/setup-node@v6- with:- node-version: 24- cache: pnpm-- - name: Validate release package manifest- run: node ./scripts/release-package-map.mjs check-- - name: Install dependencies- run: pnpm install --no-frozen-lockfile-- - name: Typecheck- run: pnpm -r typecheck-- - name: Run tests- run: pnpm test:run-- - name: Build- run: pnpm build+ uses: ./.github/workflows/release-verify.yml+ with:+ ref: ${{ inputs.source_ref }}preview_stable:if: github.event_name == 'workflow_dispatch' && inputs.dry_rundiff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjsnew file mode 100644index 0000000000..3bd0a616a8--- /dev/null+++ b/scripts/__tests__/release-verify-workflow.test.mjs@@ -0,0 +1,52 @@+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import path from "node:path";+import { fileURLToPath } from "node:url";+import test from "node:test";++const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");++function readWorkflow(name) {+ return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8");+}++test("release workflow delegates stable and canary verification to the reusable workflow", () => {+ const releaseWorkflow = readWorkflow("release.yml");++ assert.match(+ releaseWorkflow,+ /verify_canary:\n\s+if: github\.event_name == 'push'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/,+ );+ assert.match(+ releaseWorkflow,+ /verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ inputs\.source_ref \}\}/,+ );+ assert.doesNotMatch(releaseWorkflow, /verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/);+});++test("release verify workflow covers the same split test surface as stable PR verification", () => {+ const verifyWorkflow = readWorkflow("release-verify.yml");++ assert.match(verifyWorkflow, /workflow_call:/);+ assert.match(verifyWorkflow, /node \.\/scripts\/release-package-map\.mjs check/);+ assert.match(verifyWorkflow, /pnpm -r typecheck/);+ assert.match(verifyWorkflow, /pnpm build/);++ for (const group of ["general-server", "general-workspaces-a", "general-workspaces-b"]) {+ assert.match(verifyWorkflow, new RegExp(`group: ${group}`));+ }++ for (const shardIndex of [0, 1, 2]) {+ assert.match(+ verifyWorkflow,+ new RegExp(`group: general-server[\\s\\S]*?shard_index: ${shardIndex}[\\s\\S]*?shard_count: 3`),+ );+ }++ for (const shardIndex of [0, 1, 2, 3]) {+ assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 4`));+ }++ assert.match(verifyWorkflow, /pnpm test:run:general -- --group/);+ assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/);+});diff --git a/server/src/__tests__/heartbeat-worktree-suppression.test.ts b/server/src/__tests__/heartbeat-worktree-suppression.test.tsindex 3a3144dcef..b740512df9 100644--- a/server/src/__tests__/heartbeat-worktree-suppression.test.ts+++ b/server/src/__tests__/heartbeat-worktree-suppression.test.ts@@ -133,29 +133,35 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {return { companyId, agentId, issueId };}- async function waitForTerminalRun(runId: string) {- for (let attempt = 0; attempt < 20; attempt += 1) {+ async function waitForCompletedRun(runId: string, agentId: string) {+ let latestStatus: string | null = null;+ let latestLastRunId: string | null = null;++ for (let attempt = 0; attempt < 100; attempt += 1) {const run = await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then((rows) => rows[0] ?? null);- if (run && run.status !== "queued" && run.status !== "running") return run.status;- await new Promise((resolve) => setTimeout(resolve, 25));- }- return null;- }-- async function waitForRuntimeStateLastRun(agentId: string, runId: string) {- for (let attempt = 0; attempt < 100; attempt += 1) {const state = await db.select({ lastRunId: agentRuntimeState.lastRunId }).from(agentRuntimeState).where(eq(agentRuntimeState.agentId, agentId)).then((rows) => rows[0] ?? null);- if (state?.lastRunId === runId) return;++ latestStatus = run?.status ?? null;+ latestLastRunId = state?.lastRunId ?? null;++ if (run && run.status !== "queued" && run.status !== "running" && state?.lastRunId === runId) {+ return run.status;+ }+await new Promise((resolve) => setTimeout(resolve, 50));}++ throw new Error(+ `Timed out waiting for heartbeat run ${runId} to finish; latest status=${latestStatus ?? "missing"}, runtime lastRunId=${latestLastRunId ?? "missing"}`,+ );}it("suppresses new assignment wakes in worktree instances without creating heartbeat runs", async () => {@@ -239,6 +245,11 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {it("still creates live-plane assignment runs when suppression is not active", async () => {const { agentId, issueId } = await insertAgentAndIssue();+ await db+ .update(issues)+ .set({ status: "in_review", updatedAt: new Date() })+ .where(eq(issues.id, issueId));+const heartbeat = heartbeatService(db, { runtimeEnv: {} });const run = await heartbeat.wakeup(agentId, {@@ -246,27 +257,22 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => {triggerDetail: "system",reason: "issue_assigned",payload: { issueId },- contextSnapshot: { issueId, wakeReason: "issue_assigned" },+ contextSnapshot: { issueId, wakeReason: "issue_assigned", skipIssueComment: true },requestedByActorType: "system",requestedByActorId: "issue_assignment",});expect(run).not.toBeNull();- const terminalStatus = await waitForTerminalRun(run!.id);- expect(["succeeded", null]).toContain(terminalStatus);+ const terminalStatus = await waitForCompletedRun(run!.id, agentId);+ await heartbeat.waitForRunExecutionDrain(run!.id);+ expect(terminalStatus).toBe("succeeded");const runCount = await db.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns).then((rows) => rows[0]?.count ?? 0);expect(runCount).toBe(1);-- await db- .update(issues)- .set({ status: "done", updatedAt: new Date() })- .where(eq(issues.id, issueId));- await waitForRuntimeStateLastRun(agentId, run!.id);- });+ }, 10_000);it("recognizes explicit restore-in-progress suppression", () => {expect(resolveHeartbeatSchedulingSuppression({diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex a97a4dadbc..620fb16d06 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -2640,6 +2640,8 @@ export function shouldResetTaskSessionForWake(function shouldRequireIssueCommentForWake(contextSnapshot: Record<string, unknown> | null | undefined,) {+ if (contextSnapshot?.skipIssueComment === true) return false;+const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);return (wakeReason === "issue_assigned" ||@@ -15043,6 +15045,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})}return {+ waitForRunExecutionDrain: async (+ runId: string,+ options: { timeoutMs?: number; intervalMs?: number } = {},+ ) => {+ const timeoutMs = options.timeoutMs ?? 5_000;+ const intervalMs = options.intervalMs ?? 25;+ const deadline = Date.now() + timeoutMs;++ while (liveRunExecutions.has(runId)) {+ if (Date.now() >= deadline) {+ throw new Error(`Timed out waiting for heartbeat run ${runId} execution to drain`);+ }+ await new Promise((resolve) => setTimeout(resolve, intervalMs));+ }+ },list: async (companyId: string,agentId?: string,