Skip to content

The open-source app everyone uses to manage agents at work

fix(e2e): wait on the server's own deferral signal in the signoff wakeup helper (#15345)

Nicky Leachceabc3bc8806
diff --git a/tests/e2e/signoff-policy.spec.ts b/tests/e2e/signoff-policy.spec.ts
index 71868b9a6a..471589cf3c 100644
--- a/tests/e2e/signoff-policy.spec.ts
+++ b/tests/e2e/signoff-policy.spec.ts
@@ -57,14 +57,91 @@ async function createAgentRequest(token: string): Promise<APIRequestContext> {
});
}
-/** Invoke a heartbeat run for an agent, returning the run ID. */
+// The wait below follows the server's own deferral signal. It does not guess
+// a delay. WAKEUP_WAIT_TOTAL_MS is a per-call ceiling, not a fixed wait: one
+// test can call invokeHeartbeat several times under the suite's 60_000 ms
+// per-test timeout (see tests/e2e/playwright.config.ts), so each call caps
+// itself to the test's own remaining time, minus a reserve left for the
+// assertions that still have to run after the call returns. A stuck wait
+// then throws a named error, not a generic Playwright timeout.
+// The server's rewake cooldown (ISSUE_REWAKE_BASE_COOLDOWN_MS, 120_000 ms in
+// server/src/services/issue-rewake-throttle.ts) can outlast one test's life.
+// When that happens, this helper still fails fast and states the reason.
+const WAKEUP_WAIT_TOTAL_MS = 45_000;
+const WAKEUP_WAIT_RESERVE_MS = 10_000;
+const WAKEUP_POLL_INTERVAL_MS = 100;
+const MAX_CONSECUTIVE_RUN_READ_FAILURES = 5;
+
+// First-seen start time for the current test, keyed by Playwright's own
+// TestInfo object. Playwright does not expose a test start timestamp, so
+// this helper records one the first time a test calls invokeHeartbeat.
+const testStartTimesMs = new WeakMap<ReturnType<typeof test.info>, number>();
+
+/**
+ * Return how long one invokeHeartbeat call may wait this time: the smaller
+ * of WAKEUP_WAIT_TOTAL_MS and the current test's remaining time, after
+ * WAKEUP_WAIT_RESERVE_MS is set aside for whatever the test still does once
+ * this call returns. Returns 0 when the test has no time left to spare, so
+ * the caller fails fast instead of starting a wait it cannot finish.
+ */
+function wakeupWaitBudgetMs(): number {
+ const info = test.info();
+ const testTimeoutMs = info.timeout;
+ if (testTimeoutMs <= 0) return WAKEUP_WAIT_TOTAL_MS; // no test timeout set
+
+ let startedAtMs = testStartTimesMs.get(info);
+ if (startedAtMs === undefined) {
+ startedAtMs = Date.now();
+ testStartTimesMs.set(info, startedAtMs);
+ }
+
+ const elapsedMs = Date.now() - startedAtMs;
+ const remainingMs = testTimeoutMs - elapsedMs - WAKEUP_WAIT_RESERVE_MS;
+ return Math.max(0, Math.min(WAKEUP_WAIT_TOTAL_MS, remainingMs));
+}
+
+/**
+ * Fetch a candidate run and confirm it belongs to this agent and this issue.
+ * Return null when the run fails to load, or when it names a different
+ * agent or a different issue. A caller must never treat an unverified lock
+ * field as the answer.
+ */
+async function fetchVerifiedRun(
+ board: APIRequestContext,
+ candidateRunId: string | null | undefined,
+ agentId: string,
+ issueId: string,
+): Promise<{ id: string; status: string } | null> {
+ if (!candidateRunId) return null;
+ const runRes = await board.get(`${BASE_URL}/api/heartbeat-runs/${candidateRunId}`);
+ if (!runRes.ok()) return null;
+ const candidateRun = await runRes.json();
+ const context = candidateRun.contextSnapshot ?? {};
+ if (candidateRun.agentId !== agentId) return null;
+ if (context.issueId !== issueId && context.taskId !== issueId) return null;
+ return { id: candidateRunId, status: candidateRun.status };
+}
+
+/**
+ * Invoke a heartbeat run for an agent, returning the run ID.
+ *
+ * Posts the wakeup exactly once. A second on-demand wake for the same
+ * (agent, issue) pair while the first has not yet produced visible progress
+ * reads as a redundant re-poll to the server's rewake throttle, so this
+ * never re-posts — it only reads state after the single post.
+ *
+ * Every returned run ID passes `fetchVerifiedRun` first. The issue's
+ * run-lock fields name a candidate, not a verified answer, because the lock
+ * holder and the issue assignee can differ during a stage transition.
+ */
async function invokeHeartbeat(
board: APIRequestContext,
agentId: string,
issueId: string,
): Promise<string> {
- const res = await board.post(`${BASE_URL}/api/agents/${agentId}/heartbeat/invoke`, {
+ const res = await board.post(`${BASE_URL}/api/agents/${agentId}/wakeup`, {
data: {
+ source: "on_demand",
reason: "issue_assigned",
payload: { issueId, taskId: issueId, taskKey: issueId },
},
@@ -73,49 +150,112 @@ async function invokeHeartbeat(
const run = await res.json();
if (typeof run.id === "string" && run.id.length > 0) return run.id;
- // A stage transition can already be replacing the previous executor's run
- // with the participant's queued run. If the legacy invoke is skipped and
- // that run has already released the issue lock, recover it from the agent's
- // recent run receipts.
- const deadline = Date.now() + 3_000;
- do {
+ const waitBudgetMs = wakeupWaitBudgetMs();
+ const deadline = Date.now() + waitBudgetMs;
+ const lastSeen = {
+ reason: typeof run.reason === "string" ? run.reason : null,
+ checkoutRunId: null as string | null,
+ executionRunId: typeof run.executionRunId === "string" ? run.executionRunId : null,
+ blockingRunStatus: null as string | null,
+ blockingRunReadFailed: false,
+ };
+
+ // A stage transition can invoke the next participant before the prior
+ // stage's run releases the issue's execution lock. The server names the
+ // run that holds the lock for two skip reasons: `issue_execution_deferred`
+ // (an active execution run) and `execution_reconciliation_required` (an
+ // execution blocker). Both name the blocking run in the same two fields,
+ // so this code handles them the same way.
+ // A third reason, `wakeup_skipped`, can also carry an `executionRunId`,
+ // but only when that run already finished. A finished run is never this
+ // agent's turn. So for `wakeup_skipped`, this code ignores `executionRunId`
+ // and falls through to the verified wait below.
+ if (
+ (lastSeen.reason === "issue_execution_deferred" || lastSeen.reason === "execution_reconciliation_required") &&
+ typeof run.executionRunId === "string"
+ ) {
+ if (run.executionAgentId === agentId) {
+ // A run for this same agent on this same issue already holds the lock
+ // (for example one started when the issue was assigned). Verify it
+ // before this code treats it as the agent's current turn.
+ const verified = await fetchVerifiedRun(board, run.executionRunId, agentId, issueId);
+ if (verified) return verified.id;
+ } else {
+ // A different agent holds the lock (the prior stage's participant, or
+ // the blocker's owner). Wait for that run to finish. It promotes this
+ // wake into a real run as part of releasing the lock, so this code
+ // waits for that instead of guessing how long the release takes.
+ let consecutiveRunReadFailures = 0;
+ while (Date.now() < deadline) {
+ const runRes = await board.get(`${BASE_URL}/api/heartbeat-runs/${run.executionRunId}`);
+ if (runRes.ok()) {
+ consecutiveRunReadFailures = 0;
+ const blockingRun = await runRes.json();
+ lastSeen.blockingRunStatus = typeof blockingRun.status === "string" ? blockingRun.status : null;
+ if (lastSeen.blockingRunStatus !== "queued" && lastSeen.blockingRunStatus !== "running") break;
+ } else {
+ // A 404 or a 500 response must not spin this loop forever. Count
+ // repeated read failures and stop once they dominate the wait.
+ // The failure then reads as "the run read kept failing", not as a
+ // generic timeout.
+ consecutiveRunReadFailures += 1;
+ if (consecutiveRunReadFailures >= MAX_CONSECUTIVE_RUN_READ_FAILURES) {
+ lastSeen.blockingRunReadFailed = true;
+ break;
+ }
+ }
+ await new Promise((resolve) => setTimeout(resolve, WAKEUP_POLL_INTERVAL_MS));
+ }
+ }
+ }
+
+ // Read the issue's own run-lock fields, and this agent's recent runs,
+ // until this agent's run for this issue appears and passes verification.
+ // Negative-authorization tests invoke a non-participant on purpose, so an
+ // assignee mismatch is a genuine rejection to return untouched, not a
+ // signal to keep waiting for a run that must never exist for that agent.
+ while (Date.now() < deadline) {
const issueRunLock = await getIssueRunLockState(board, issueId);
+ lastSeen.checkoutRunId = issueRunLock.checkoutRunId;
+ lastSeen.executionRunId = issueRunLock.executionRunId;
if (issueRunLock.assigneeAgentId !== agentId) {
- // Negative authorization cases intentionally invoke a non-participant.
- // Preserve the server rejection instead of waiting for a run that must
- // never be assigned to that agent.
return issueRunLock.executionRunId ?? issueRunLock.checkoutRunId ?? "";
}
- const candidates = new Set<string>([
- run.executionRunId,
- issueRunLock.executionRunId,
- issueRunLock.checkoutRunId,
- ].filter((candidate): candidate is string => Boolean(candidate)));
+
+ const verifiedCheckout = await fetchVerifiedRun(board, issueRunLock.checkoutRunId, agentId, issueId);
+ if (verifiedCheckout) return verifiedCheckout.id;
+ const verifiedExecution = await fetchVerifiedRun(board, issueRunLock.executionRunId, agentId, issueId);
+ if (verifiedExecution) return verifiedExecution.id;
+
const recentRunsRes = await board.get(
`${BASE_URL}/api/companies/${issueRunLock.companyId}/heartbeat-runs?agentId=${agentId}&limit=20`,
);
if (recentRunsRes.ok()) {
const recentRuns = await recentRunsRes.json();
for (const recentRun of Array.isArray(recentRuns) ? recentRuns : []) {
- if (typeof recentRun.id === "string") candidates.add(recentRun.id);
+ const context = recentRun.contextSnapshot ?? {};
+ if (
+ typeof recentRun.id === "string" &&
+ recentRun.agentId === agentId &&
+ (context.issueId === issueId || context.taskId === issueId)
+ ) {
+ return recentRun.id;
+ }
}
}
- for (const candidate of candidates) {
- const runRes = await board.get(`${BASE_URL}/api/heartbeat-runs/${candidate}`);
- if (!runRes.ok()) continue;
- const candidateRun = await runRes.json();
- const context = candidateRun.contextSnapshot ?? {};
- if (
- candidateRun.agentId === agentId &&
- (context.issueId === issueId || context.taskId === issueId)
- ) {
- return candidate;
- }
- }
- await new Promise((resolve) => setTimeout(resolve, 50));
- } while (Date.now() < deadline);
+ await new Promise((resolve) => setTimeout(resolve, WAKEUP_POLL_INTERVAL_MS));
+ }
- throw new Error(`No issue-bound heartbeat run became available for agent ${agentId}`);
+ throw new Error(
+ `No verified heartbeat run became available for agent ${agentId} on issue ${issueId} ` +
+ `within ${waitBudgetMs}ms` +
+ `${waitBudgetMs <= 0 ? " (the test's remaining time budget was already spent before this wait could start)" : ""}. ` +
+ `Last skip reason: ${lastSeen.reason ?? "none"}. ` +
+ `Last checkoutRunId: ${lastSeen.checkoutRunId ?? "none"}. ` +
+ `Last executionRunId: ${lastSeen.executionRunId ?? "none"}. ` +
+ `Blocking run status: ${lastSeen.blockingRunStatus ?? "none"}` +
+ `${lastSeen.blockingRunReadFailed ? " (the blocking run read failed repeatedly)" : ""}.`,
+ );
}
async function getIssueRunLockState(board: APIRequestContext, issueId: string): Promise<IssueRunLockState> {