Skip to content

The open-source app everyone uses to manage agents at work

refactor(server): move the release half of the deferred wake state machine into a wake-queue module (#13132)

Nicky Leach6dd48cad439e
diff --git a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts
index 9bc0e48011..a054564853 100644
--- a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts
+++ b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts
@@ -619,152 +619,6 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
}
}, 120_000);
- it("cancels an empty deferred comment wake instead of promoting deleted input", async () => {
- const gateway = await createControlledGatewayServer();
- const companyId = randomUUID();
- const agentId = randomUUID();
- const issueId = randomUUID();
- const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
- const heartbeat = heartbeatService(db);
-
- try {
- await db.insert(companies).values({
- id: companyId,
- name: "Paperclip",
- issuePrefix,
- requireBoardApprovalForNewAgents: false,
- defaultResponsibleUserId: "responsible-user",
- });
- await db.insert(agents).values({
- id: agentId,
- companyId,
- name: "Gateway Agent",
- role: "engineer",
- status: "idle",
- adapterType: "openclaw_gateway",
- adapterConfig: {
- url: gateway.url,
- headers: { "x-openclaw-token": "gateway-token" },
- payloadTemplate: { message: "wake now" },
- waitTimeoutMs: 2_000,
- },
- runtimeConfig: {},
- permissions: {},
- });
- await db.insert(issues).values({
- id: issueId,
- companyId,
- title: "Discard deferred follow-up",
- status: "todo",
- priority: "medium",
- responsibleUserId: "responsible-user",
- assigneeAgentId: agentId,
- issueNumber: 1,
- identifier: `${issuePrefix}-1`,
- });
- const firstComment = await db
- .insert(issueComments)
- .values({
- companyId,
- issueId,
- authorUserId: "user-1",
- body: "First comment",
- })
- .returning()
- .then((rows) => rows[0]);
- const firstRun = await heartbeat.wakeup(agentId, {
- source: "automation",
- triggerDetail: "system",
- reason: "issue_commented",
- payload: { issueId, commentId: firstComment.id },
- contextSnapshot: {
- issueId,
- taskId: issueId,
- commentId: firstComment.id,
- wakeReason: "issue_commented",
- },
- requestedByActorType: "user",
- requestedByActorId: "user-1",
- });
- expect(firstRun).not.toBeNull();
- await waitFor(async () => {
- const current = await db
- .select({ status: heartbeatRuns.status })
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.id, firstRun!.id))
- .then((rows) => rows[0] ?? null);
- return current?.status === "running";
- });
-
- const discardedComment = await db
- .insert(issueComments)
- .values({
- companyId,
- issueId,
- authorUserId: "user-1",
- body: "Delete this before the current turn finishes",
- })
- .returning()
- .then((rows) => rows[0]);
- expect(await heartbeat.wakeup(agentId, {
- source: "automation",
- triggerDetail: "system",
- reason: "issue_commented",
- payload: { issueId, commentId: discardedComment.id },
- contextSnapshot: {
- issueId,
- taskId: issueId,
- commentId: discardedComment.id,
- wakeReason: "issue_commented",
- },
- requestedByActorType: "user",
- requestedByActorId: "user-1",
- })).toBeNull();
- await waitFor(async () => db
- .select({ id: agentWakeupRequests.id })
- .from(agentWakeupRequests)
- .where(and(
- eq(agentWakeupRequests.companyId, companyId),
- eq(agentWakeupRequests.agentId, agentId),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- ))
- .then((rows) => Boolean(rows[0])));
- const deferredWake = await db
- .select()
- .from(agentWakeupRequests)
- .where(and(
- eq(agentWakeupRequests.companyId, companyId),
- eq(agentWakeupRequests.agentId, agentId),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- ))
- .then((rows) => rows[0]);
- if (!deferredWake) throw new Error("Expected a deferred comment wake");
- await db.delete(issueComments).where(eq(issueComments.id, discardedComment.id));
-
- gateway.releaseFirstWait();
- await waitFor(async () => {
- const wake = await db
- .select({ status: agentWakeupRequests.status })
- .from(agentWakeupRequests)
- .where(eq(agentWakeupRequests.id, deferredWake.id))
- .then((rows) => rows[0] ?? null);
- return wake?.status === "cancelled";
- }, 90_000);
- await heartbeat.drainActiveRunExecutions();
-
- expect(gateway.getAgentPayloads()).toHaveLength(1);
- const runs = await db
- .select({ id: heartbeatRuns.id })
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.agentId, agentId));
- expect(runs.map((run) => run.id)).toEqual([firstRun!.id]);
- } finally {
- gateway.releaseFirstWait();
- await heartbeat.drainActiveRunExecutions();
- await gateway.close();
- }
- }, 120_000);
-
it("retains deferred comments for reconciliation after cancelling an unknown provider outcome", async () => {
const gateway = await createControlledGatewayServer();
const companyId = randomUUID();
@@ -1312,7 +1166,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
}
}, 120_000);
- it("cancels a deferred wake containing only a comment authored by the closing run", async () => {
+ it("still reopens a finished issue when a deferred batch mixes self-authored and human comments", async () => {
const gateway = await createControlledGatewayServer();
const companyId = randomUUID();
const agentId = randomUUID();
@@ -1353,7 +1207,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
await db.insert(issues).values({
id: issueId,
companyId,
- title: "Self-comment must not reopen",
+ title: "Human follow-up must survive mixed deferred batches",
status: "todo",
priority: "medium",
responsibleUserId: "responsible-user",
@@ -1385,12 +1239,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
.then((rows) => rows[0] ?? null);
return run?.status === "running";
});
- await waitFor(() => gateway.getAgentPayloads().length === 1);
- // Local-CLI agents post comments under user auth, but stamp the heartbeat
- // run id on each comment via createdByRunId. Simulate that here: a "user"
- // comment that was actually authored by the run that is about to close
- // the issue. Without the Path A guard this would trigger a reopen.
const selfComment = await db
.insert(issueComments)
.values({
@@ -1398,12 +1247,12 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
issueId,
authorUserId: "local-cli-user",
createdByRunId: firstRun?.id ?? null,
- body: "Closing comment from the same run",
+ body: "Closing note from the same run",
})
.returning()
.then((rows) => rows[0]);
- const deferredRun = await heartbeat.wakeup(agentId, {
+ const firstDeferredRun = await heartbeat.wakeup(agentId, {
source: "automation",
triggerDetail: "system",
reason: "issue_commented",
@@ -1419,7 +1268,36 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
requestedByActorId: "local-cli-user",
});
- expect(deferredRun).toBeNull();
+ expect(firstDeferredRun).toBeNull();
+
+ const humanComment = await db
+ .insert(issueComments)
+ .values({
+ companyId,
+ issueId,
+ authorUserId: "user-1",
+ body: "Real follow-up from a human after the run closes",
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ const secondDeferredRun = await heartbeat.wakeup(agentId, {
+ source: "automation",
+ triggerDetail: "system",
+ reason: "issue_commented",
+ payload: { issueId, commentId: humanComment.id },
+ contextSnapshot: {
+ issueId,
+ taskId: issueId,
+ commentId: humanComment.id,
+ wakeCommentId: humanComment.id,
+ wakeReason: "issue_commented",
+ },
+ requestedByActorType: "user",
+ requestedByActorId: "user-1",
+ });
+
+ expect(secondDeferredRun).toBeNull();
await waitFor(async () => {
const deferred = await db
@@ -1454,36 +1332,21 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
gateway.releaseFirstWait();
+ await waitFor(() => gateway.getAgentPayloads().length >= 2, 90_000);
await waitFor(async () => {
- const run = await db
+ const runs = await db
.select()
.from(heartbeatRuns)
- .where(eq(heartbeatRuns.id, firstRun!.id))
- .then((rows) => rows[0] ?? null);
- const deferred = await db
- .select()
- .from(agentWakeupRequests)
- .where(
- and(
- eq(agentWakeupRequests.companyId, companyId),
- eq(agentWakeupRequests.agentId, agentId),
- ),
- )
- .then((rows) => rows.find((request) => request.status === "cancelled") ?? null);
+ .where(eq(heartbeatRuns.agentId, agentId))
+ .orderBy(asc(heartbeatRuns.createdAt));
+ const [initialRun, promotedRun] = runs;
return (
- run?.status === "succeeded" &&
- deferred?.error ===
- "Deferred wake contained only comments authored by the finishing run"
+ initialRun?.id === firstRun?.id &&
+ initialRun.status === "succeeded" &&
+ promotedRun?.status === "succeeded"
);
}, 90_000);
- expect(gateway.getAgentPayloads()).toHaveLength(1);
- const runs = await db
- .select()
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.agentId, agentId));
- expect(runs).toHaveLength(1);
-
const issueAfterPromotion = await db
.select({
status: issues.status,
@@ -1494,21 +1357,37 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
.then((rows) => rows[0] ?? null);
expect(issueAfterPromotion).toMatchObject({
- status: "done",
+ status: "in_progress",
+ completedAt: null,
});
- expect(issueAfterPromotion?.completedAt).not.toBeNull();
+
+ const secondPayload = gateway.getAgentPayloads()[1] ?? {};
+ expect(secondPayload.paperclip).toBeUndefined();
+ const secondWake = parseWakePayloadFromMessage(secondPayload.message);
+ expect(secondWake).toMatchObject({
+ reason: "issue_commented",
+ commentIds: [humanComment.id],
+ latestCommentId: humanComment.id,
+ issue: {
+ id: issueId,
+ identifier: `${issuePrefix}-1`,
+ title: "Human follow-up must survive mixed deferred batches",
+ status: "in_progress",
+ priority: "medium",
+ },
+ });
+ expect(String(secondPayload.message ?? "")).toContain("Real follow-up from a human after the run closes");
} finally {
gateway.releaseFirstWait();
await gateway.close();
}
}, 120_000);
- it("promotes an interaction continuation after removing a coalesced self-authored comment", async () => {
+ it("cancels a deferred comment wake when its only queued comment is deleted before promotion", async () => {
const gateway = await createControlledGatewayServer();
const companyId = randomUUID();
const agentId = randomUUID();
const issueId = randomUUID();
- const interactionId = randomUUID();
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
const heartbeat = heartbeatService(db);
@@ -1524,7 +1403,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
await db.insert(agents).values({
id: agentId,
companyId,
- name: "Local CLI Agent",
+ name: "Gateway Agent",
role: "engineer",
status: "idle",
adapterType: "openclaw_gateway",
@@ -1545,7 +1424,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
await db.insert(issues).values({
id: issueId,
companyId,
- title: "Interaction continuation survives self-comment filtering",
+ title: "Deleted follow-up must not reopen",
status: "todo",
priority: "medium",
responsibleUserId: "responsible-user",
@@ -1577,99 +1456,122 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
.then((rows) => rows[0] ?? null);
return run?.status === "running";
});
+ await waitFor(() => gateway.getAgentPayloads().length === 1);
- const selfComment = await db
+ const queuedComment = await db
.insert(issueComments)
.values({
companyId,
issueId,
- authorUserId: "local-cli-user",
- createdByRunId: firstRun!.id,
- body: "Completion note from the source run",
+ authorUserId: "user-1",
+ body: "Please look at this once you finish",
})
.returning()
.then((rows) => rows[0]);
- expect(await heartbeat.wakeup(agentId, {
- source: "automation",
- triggerDetail: "system",
- reason: "issue_commented",
- payload: { issueId, commentId: selfComment.id },
- contextSnapshot: {
- issueId,
- taskId: issueId,
- commentId: selfComment.id,
- wakeCommentId: selfComment.id,
- wakeReason: "issue_commented",
- },
- requestedByActorType: "user",
- requestedByActorId: "local-cli-user",
- })).toBeNull();
-
- expect(await heartbeat.wakeup(agentId, {
+ const deferredRun = await heartbeat.wakeup(agentId, {
source: "automation",
triggerDetail: "system",
reason: "issue_commented",
- payload: {
- issueId,
- interactionId,
- interactionKind: "request_confirmation",
- interactionStatus: "accepted",
- mutation: "interaction",
- },
+ payload: { issueId, commentId: queuedComment.id },
contextSnapshot: {
issueId,
taskId: issueId,
- interactionId,
- interactionKind: "request_confirmation",
- interactionStatus: "accepted",
+ commentId: queuedComment.id,
+ wakeCommentId: queuedComment.id,
wakeReason: "issue_commented",
- source: "issue.interaction.respond",
},
requestedByActorType: "user",
requestedByActorId: "user-1",
- })).toBeNull();
+ });
- gateway.releaseFirstWait();
+ expect(deferredRun).toBeNull();
await waitFor(async () => {
- const runs = await db
+ const deferred = await db
.select()
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.agentId, agentId))
- .orderBy(asc(heartbeatRuns.createdAt));
- return (
- runs.length === 2 &&
- runs[0]?.status === "succeeded" &&
- runs[1]?.status === "succeeded"
- );
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.agentId, agentId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+ return Boolean(deferred);
+ });
+
+ const deferredWake = await db
+ .select({ id: agentWakeupRequests.id })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.agentId, agentId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+ const deferredWakeId = deferredWake!.id;
+
+ // The author retracts the comment before the first run finishes, so the
+ // real comment-liveness query must find zero live comments left in the
+ // queued batch.
+ await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, queuedComment.id));
+
+ // Running records admission. Wait for provider acceptance before
+ // simulating completion by that provider, or startup correctly rejects
+ // the already-closed task before this scenario reaches its follow-up.
+ await waitFor(() => gateway.getAgentPayloads().length >= 1);
+ await db
+ .update(issues)
+ .set({
+ status: "done",
+ completedAt: new Date(),
+ executionRunId: null,
+ executionAgentNameKey: null,
+ executionLockedAt: null,
+ updatedAt: new Date(),
+ })
+ .where(eq(issues.id, issueId));
+
+ gateway.releaseFirstWait();
+
+ await waitFor(async () => {
+ const [wake] = await db
+ .select({ status: agentWakeupRequests.status })
+ .from(agentWakeupRequests)
+ .where(eq(agentWakeupRequests.id, deferredWakeId));
+ return wake?.status === "cancelled";
}, 90_000);
- const promotedRun = await db
- .select()
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.agentId, agentId))
- .orderBy(asc(heartbeatRuns.createdAt))
- .then((runs) => runs[1] ?? null);
- expect(promotedRun?.contextSnapshot).toMatchObject({
- interactionId,
- interactionKind: "request_confirmation",
- interactionStatus: "accepted",
- });
- expect(promotedRun?.contextSnapshot).not.toMatchObject({
- wakeCommentIds: expect.anything(),
- });
- expect(promotedRun?.contextSnapshot).not.toMatchObject({
- commentId: selfComment.id,
+ const [cancelledWake] = await db
+ .select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
+ .from(agentWakeupRequests)
+ .where(eq(agentWakeupRequests.id, deferredWakeId));
+ expect(cancelledWake).toMatchObject({
+ status: "cancelled",
+ error: "Queued messages were discarded before promotion",
});
- expect(gateway.getAgentPayloads()).toHaveLength(2);
+
+ // No live comment remained, so the queue must not promote a second run
+ // and the issue must stay in the state the first run's completion left it in.
+ expect(gateway.getAgentPayloads()).toHaveLength(1);
+ const issueAfterCompletion = await db
+ .select({ status: issues.status, completedAt: issues.completedAt })
+ .from(issues)
+ .where(eq(issues.id, issueId))
+ .then((rows) => rows[0] ?? null);
+ expect(issueAfterCompletion).toMatchObject({ status: "done" });
+ expect(issueAfterCompletion?.completedAt).not.toBeNull();
} finally {
gateway.releaseFirstWait();
await gateway.close();
}
}, 120_000);
- it("still reopens a finished issue when a deferred batch mixes self-authored and human comments", async () => {
+ it("cancels a deferred comment wake when its only queued comment was authored by the finishing run", async () => {
const gateway = await createControlledGatewayServer();
const companyId = randomUUID();
const agentId = randomUUID();
@@ -1710,7 +1612,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
await db.insert(issues).values({
id: issueId,
companyId,
- title: "Human follow-up must survive mixed deferred batches",
+ title: "Self-authored note must not reopen",
status: "todo",
priority: "medium",
responsibleUserId: "responsible-user",
@@ -1755,7 +1657,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
.returning()
.then((rows) => rows[0]);
- const firstDeferredRun = await heartbeat.wakeup(agentId, {
+ const deferredRun = await heartbeat.wakeup(agentId, {
source: "automation",
triggerDetail: "system",
reason: "issue_commented",
@@ -1771,36 +1673,7 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
requestedByActorId: "local-cli-user",
});
- expect(firstDeferredRun).toBeNull();
-
- const humanComment = await db
- .insert(issueComments)
- .values({
- companyId,
- issueId,
- authorUserId: "user-1",
- body: "Real follow-up from a human after the run closes",
- })
- .returning()
- .then((rows) => rows[0]);
-
- const secondDeferredRun = await heartbeat.wakeup(agentId, {
- source: "automation",
- triggerDetail: "system",
- reason: "issue_commented",
- payload: { issueId, commentId: humanComment.id },
- contextSnapshot: {
- issueId,
- taskId: issueId,
- commentId: humanComment.id,
- wakeCommentId: humanComment.id,
- wakeReason: "issue_commented",
- },
- requestedByActorType: "user",
- requestedByActorId: "user-1",
- });
-
- expect(secondDeferredRun).toBeNull();
+ expect(deferredRun).toBeNull();
await waitFor(async () => {
const deferred = await db
@@ -1817,6 +1690,18 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
return Boolean(deferred);
});
+ const queuedWake = await db
+ .select({ id: agentWakeupRequests.id })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.agentId, agentId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+
// Running records admission. Wait for provider acceptance before
// simulating completion by that provider, or startup correctly rejects
// the already-closed task before this scenario reaches its follow-up.
@@ -1835,51 +1720,34 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
gateway.releaseFirstWait();
- await waitFor(() => gateway.getAgentPayloads().length >= 2, 90_000);
await waitFor(async () => {
- const runs = await db
- .select()
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.agentId, agentId))
- .orderBy(asc(heartbeatRuns.createdAt));
- const [initialRun, promotedRun] = runs;
- return (
- initialRun?.id === firstRun?.id &&
- initialRun.status === "succeeded" &&
- promotedRun?.status === "succeeded"
- );
+ const [wake] = await db
+ .select({ status: agentWakeupRequests.status })
+ .from(agentWakeupRequests)
+ .where(eq(agentWakeupRequests.id, queuedWake!.id));
+ return wake?.status === "cancelled";
}, 90_000);
- const issueAfterPromotion = await db
- .select({
- status: issues.status,
- completedAt: issues.completedAt,
- })
+ const [cancelledWake] = await db
+ .select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
+ .from(agentWakeupRequests)
+ .where(eq(agentWakeupRequests.id, queuedWake!.id));
+ expect(cancelledWake).toMatchObject({
+ status: "cancelled",
+ error: "Deferred wake contained only comments authored by the finishing run",
+ });
+
+ // The only queued comment came from the run that just finished, so the
+ // queue must not promote a second run for the same agent to re-read its
+ // own note, and the issue must stay in the state the run's completion left it in.
+ expect(gateway.getAgentPayloads()).toHaveLength(1);
+ const issueAfterCompletion = await db
+ .select({ status: issues.status, completedAt: issues.completedAt })
.from(issues)
.where(eq(issues.id, issueId))
.then((rows) => rows[0] ?? null);
-
- expect(issueAfterPromotion).toMatchObject({
- status: "in_progress",
- completedAt: null,
- });
-
- const secondPayload = gateway.getAgentPayloads()[1] ?? {};
- expect(secondPayload.paperclip).toBeUndefined();
- const secondWake = parseWakePayloadFromMessage(secondPayload.message);
- expect(secondWake).toMatchObject({
- reason: "issue_commented",
- commentIds: [humanComment.id],
- latestCommentId: humanComment.id,
- issue: {
- id: issueId,
- identifier: `${issuePrefix}-1`,
- title: "Human follow-up must survive mixed deferred batches",
- status: "in_progress",
- priority: "medium",
- },
- });
- expect(String(secondPayload.message ?? "")).toContain("Real follow-up from a human after the run closes");
+ expect(issueAfterCompletion).toMatchObject({ status: "done" });
+ expect(issueAfterCompletion?.completedAt).not.toBeNull();
} finally {
gateway.releaseFirstWait();
await gateway.close();
diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts
new file mode 100644
index 0000000000..f49ec7cdb4
--- /dev/null
+++ b/server/src/modules/wake-queue/adapters/postgres.test.ts
@@ -0,0 +1,439 @@
+import { randomUUID } from "node:crypto";
+import { eq } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
+import {
+ agentWakeupRequests,
+ agents,
+ companies,
+ createDb,
+ heartbeatRuns,
+ issueComments,
+ issues,
+} from "@paperclipai/db";
+import {
+ getEmbeddedPostgresTestSupport,
+ startEmbeddedPostgresTestDatabase,
+} from "../../../__tests__/helpers/embedded-postgres.js";
+import { createPostgresWakeQueueAdapter } from "./postgres.js";
+import type { WakeQueuePostgresAdapterDeps } from "./postgres.js";
+
+// Proves the atomicity and company-scope properties the security review
+// requires: every mutation names `companyId` in its own SQL `WHERE` clause,
+// a foreign-company row is invisible to a read, and a deferred-status
+// compare-and-set that affects no row leaves no other trace. The decision
+// branching itself is proven against plain facts in `domain/policy.test.ts`.
+const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+
+if (!embeddedPostgresSupport.supported) {
+ console.warn(
+ `Skipping embedded Postgres wake-queue adapter tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
+ );
+}
+
+describeEmbeddedPostgres("wake-queue postgres adapter", () => {
+ let db!: ReturnType<typeof createDb>;
+ let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
+
+ const stubDeps: WakeQueuePostgresAdapterDeps = {
+ resolveResponsibleUserId: async () => "responsible-user",
+ getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }),
+ resolveSessionBeforeForWakeup: async () => null,
+ };
+
+ beforeAll(async () => {
+ tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
+ db = createDb(tempDb.connectionString);
+ }, 20_000);
+
+ afterEach(async () => {
+ await db.delete(issueComments);
+ // `heartbeat_runs.wakeup_request_id` references `agent_wakeup_requests.id`,
+ // so the run row must go first.
+ await db.delete(heartbeatRuns);
+ await db.delete(agentWakeupRequests);
+ await db.delete(issues);
+ await db.delete(agents);
+ await db.delete(companies);
+ });
+
+ afterAll(async () => {
+ await tempDb?.cleanup();
+ });
+
+ async function seedCompany(): Promise<string> {
+ const companyId = randomUUID();
+ await db.insert(companies).values({
+ id: companyId,
+ name: "Paperclip",
+ issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
+ requireBoardApprovalForNewAgents: false,
+ defaultResponsibleUserId: "responsible-user",
+ });
+ return companyId;
+ }
+
+ async function seedAgent(input: { companyId: string; name?: string }): Promise<string> {
+ const agentId = randomUUID();
+ await db.insert(agents).values({
+ id: agentId,
+ companyId: input.companyId,
+ name: input.name ?? "CodexCoder",
+ role: "engineer",
+ status: "active",
+ adapterType: "codex_local",
+ adapterConfig: {},
+ runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
+ permissions: {},
+ });
+ return agentId;
+ }
+
+ async function seedIssue(input: {
+ companyId: string;
+ issueId?: string;
+ status?: string;
+ assigneeAgentId?: string | null;
+ executionRunId?: string | null;
+ checkoutRunId?: string | null;
+ }): Promise<string> {
+ const issueId = input.issueId ?? randomUUID();
+ await db.insert(issues).values({
+ id: issueId,
+ companyId: input.companyId,
+ title: "Wake-queue adapter fixture issue",
+ status: input.status ?? "in_progress",
+ priority: "medium",
+ assigneeAgentId: input.assigneeAgentId ?? null,
+ executionRunId: input.executionRunId ?? null,
+ checkoutRunId: input.checkoutRunId ?? null,
+ });
+ return issueId;
+ }
+
+ async function seedRun(input: {
+ companyId: string;
+ agentId: string;
+ status?: string;
+ contextSnapshot?: Record<string, unknown>;
+ errorCode?: string | null;
+ runtimeMode?: string;
+ }): Promise<string> {
+ const runId = randomUUID();
+ await db.insert(heartbeatRuns).values({
+ id: runId,
+ companyId: input.companyId,
+ agentId: input.agentId,
+ invocationSource: "on_demand",
+ status: input.status ?? "failed",
+ contextSnapshot: input.contextSnapshot ?? {},
+ errorCode: input.errorCode ?? null,
+ });
+ return runId;
+ }
+
+ async function seedDeferredWake(input: {
+ companyId: string;
+ agentId: string;
+ issueId: string;
+ requestedByActorType?: string;
+ requestedByActorId?: string | null;
+ payload?: Record<string, unknown>;
+ }): Promise<string> {
+ const id = randomUUID();
+ await db.insert(agentWakeupRequests).values({
+ id,
+ companyId: input.companyId,
+ agentId: input.agentId,
+ source: "automation",
+ reason: "issue_commented",
+ status: "deferred_issue_execution",
+ requestedByActorType: input.requestedByActorType ?? "user",
+ requestedByActorId: input.requestedByActorId ?? null,
+ payload: { issueId: input.issueId, ...(input.payload ?? {}) },
+ });
+ return id;
+ }
+
+ // Review test (a): a foreign-company agent id produces the current failed
+ // wake status and the current error text, and creates no run.
+ it("fails a deferred wake whose agent belongs to a different company, without creating a run", async () => {
+ const companyId = await seedCompany();
+ const otherCompanyId = await seedCompany();
+ const foreignAgentId = await seedAgent({ companyId: otherCompanyId });
+ const finishingAgentId = await seedAgent({ companyId });
+ const issueId = await seedIssue({ companyId, assigneeAgentId: finishingAgentId, status: "in_progress" });
+ // A finishing run status other than the legacy-reconciliation set (failed,
+ // timed_out, interrupted, cancelled) reaches the module's own drain logic.
+ const runId = await seedRun({ companyId, agentId: finishingAgentId, contextSnapshot: { issueId }, status: "succeeded" });
+ await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
+ const wakeId = await seedDeferredWake({ companyId, agentId: foreignAgentId, issueId });
+
+ const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
+ const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
+ const candidate = await ports.writer.claimNextDeferredWake({ companyId, issueId: locked.primaryIssue.id });
+ expect(candidate?.id).toBe(wakeId);
+ const agent = await ports.reader.findInvokableAgent({ companyId, agentId: foreignAgentId });
+ expect(agent).toBeNull();
+ const failed = await ports.writer.failDeferredWake({ companyId, wakeId: candidate!.id, now: new Date() });
+ expect(failed).toBe(true);
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ });
+ expect(result.outcome.kind).toBe("released");
+
+ const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
+ expect(wakeRow?.status).toBe("failed");
+ expect(wakeRow?.error).toBe("Deferred wake could not be promoted: agent is not invokable");
+ expect(wakeRow?.runId).toBeNull();
+ const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
+ expect(runs).toHaveLength(1);
+ expect(runs[0]!.id).toBe(runId);
+ });
+
+ // Review test (b): each release adapter mutation with a foreign company
+ // affects no row.
+ it("scopes every release mutation to its own company and affects no row across a company boundary", async () => {
+ const companyId = await seedCompany();
+ const otherCompanyId = await seedCompany();
+ const agentId = await seedAgent({ companyId });
+ const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
+ await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
+ const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
+ const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
+
+ const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
+ await adapter.withIssueExecutionLock(
+ { companyId, runId, now: new Date() },
+ async (_locked, ports) => {
+ const cancelledUnderWrongCompany = await ports.writer.cancelDeferredWake({
+ companyId: otherCompanyId,
+ wakeId,
+ reason: "cross-company cancel attempt",
+ now: new Date(),
+ });
+ expect(cancelledUnderWrongCompany).toBe(false);
+
+ const failedUnderWrongCompany = await ports.writer.failDeferredWake({
+ companyId: otherCompanyId,
+ wakeId,
+ now: new Date(),
+ });
+ expect(failedUnderWrongCompany).toBe(false);
+
+ const normalizedUnderWrongCompany = await ports.writer.normalizeDeferredWakeCommentIds({
+ companyId: otherCompanyId,
+ wakeId,
+ payload: { issueId },
+ liveCommentIds: ["c1"],
+ now: new Date(),
+ });
+ expect(normalizedUnderWrongCompany).toBeNull();
+
+ const reopenedUnderWrongCompany = await ports.writer.reopenIssue({
+ companyId: otherCompanyId,
+ issueId,
+ runId,
+ });
+ expect(reopenedUnderWrongCompany).toBeNull();
+
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ },
+ );
+
+ const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
+ expect(wakeRow?.status).toBe("deferred_issue_execution");
+ expect(wakeRow?.error).toBeNull();
+ const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
+ expect(issueRow?.status).toBe("blocked");
+ expect(issueRow?.executionState).toEqual({ phase: "running" });
+ });
+
+ // Review defect: the reopen path must carry the company into every read,
+ // lock, and write. A check before the write is not a boundary, because
+ // `issues.company_id` can change between that check and the write.
+ it("refuses to reopen an issue for a company that does not own it, and leaves the issue untouched", async () => {
+ const companyId = await seedCompany();
+ const otherCompanyId = await seedCompany();
+ const agentId = await seedAgent({ companyId });
+ const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
+ await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
+ const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
+
+ const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
+ const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
+ reopened: null,
+ };
+ await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
+ captured.reopened = await ports.writer.reopenIssue({ companyId: otherCompanyId, issueId, runId });
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ });
+ expect(captured.reopened).toBeNull();
+
+ const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
+ expect(issueRow?.status).toBe("blocked");
+ expect(issueRow?.executionState).toEqual({ phase: "running" });
+ });
+
+ it("reopens an issue for the company that owns it, and clears the execution state", async () => {
+ const companyId = await seedCompany();
+ const agentId = await seedAgent({ companyId });
+ const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
+ await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
+ const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
+
+ const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
+ const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
+ reopened: null,
+ };
+ await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
+ captured.reopened = await ports.writer.reopenIssue({ companyId, issueId, runId });
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ });
+ expect(captured.reopened?.status).toBe("todo");
+ expect(captured.reopened?.executionState).toBeNull();
+
+ const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
+ expect(issueRow?.status).toBe("todo");
+ expect(issueRow?.executionState).toBeNull();
+ });
+
+ // Review test (c): a deferred-status compare-and-set that affects no row
+ // claims nothing, and no other write in the promotion path ever runs.
+ it("fails the promotion claim when the deferred-status compare-and-set loses the race, before any other write", async () => {
+ const companyId = await seedCompany();
+ const agentId = await seedAgent({ companyId });
+ const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
+ const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
+ // A concurrent finalization already claimed this wake before the promotion claim runs.
+ await db.update(agentWakeupRequests).set({ status: "cancelled" }).where(eq(agentWakeupRequests.id, wakeId));
+
+ const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
+ const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
+ const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
+ const claimed = await ports.writer.claimDeferredWakeForPromotion({ companyId, wakeId, now: new Date() });
+ expect(claimed).toBe(false);
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ });
+ expect(result.outcome.kind).toBe("released");
+
+ const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
+ expect(runs).toHaveLength(1);
+ expect(runs[0]!.id).toBe(runId);
+ const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
+ expect(issueRow?.executionRunId).toBeNull();
+ const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
+ expect(wakeRow?.status).toBe("cancelled");
+ });
+
+ // `finalizePromotedWake`'s own writes guard against clobbering state a
+ // concurrent write already changed: it never takes the issue's execution
+ // lock away from a run that already holds it, and it never overwrites a
+ // `runId` a wake row already carries. Both guards only matter as defense
+ // in depth today (the release drain calls this at most once per
+ // transaction), so this drives the port directly to prove the SQL itself,
+ // independent of that call pattern.
+ it("guards finalizePromotedWake's own writes against clobbering an already-set execution lock or runId", async () => {
+ const companyId = await seedCompany();
+ const agentId = await seedAgent({ companyId });
+ const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
+ const wakeIdA = await seedDeferredWake({ companyId, agentId, issueId });
+ const wakeIdB = await seedDeferredWake({ companyId, agentId, issueId });
+ const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
+ const deferredAgent = { id: agentId, companyId, name: "CodexCoder", invokable: true };
+
+ const finalizedRunIds: string[] = [];
+ const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
+ await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
+ const finalize = async (wakeId: string) => {
+ const promoted = await ports.writer.finalizePromotedWake({
+ companyId,
+ wakeId,
+ deferredAgent,
+ issue: locked.primaryIssue,
+ finishingRun: locked.run,
+ contextSnapshot: { issueId },
+ reason: "issue_execution_promoted",
+ source: "automation",
+ triggerDetail: null,
+ payload: {},
+ responsibleUserId: "responsible-user",
+ sessionBefore: null,
+ now: new Date(),
+ });
+ finalizedRunIds.push(promoted.id);
+ };
+ // The issue's execution lock is free; this call takes it.
+ await finalize(wakeIdA);
+ // The issue's execution lock is already held by the first call's run,
+ // so this call's issue-lock write must no-op even though a run is
+ // still created.
+ await finalize(wakeIdB);
+ // Repeating the same wake must not overwrite its now-set `runId`.
+ await finalize(wakeIdA);
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ });
+ const [runA, runB, runC] = finalizedRunIds;
+
+ const wakeRowA = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdA)))[0];
+ const wakeRowB = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdB)))[0];
+ expect(wakeRowA?.runId).toBe(runA);
+ expect(wakeRowB?.runId).toBe(runB);
+ const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
+ expect(issueRow?.executionRunId).toBe(runA);
+ expect(issueRow?.executionRunId).not.toBe(runB);
+ expect(issueRow?.executionRunId).not.toBe(runC);
+ const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
+ // All three finalize calls each still insert their own run row.
+ expect(runs.map((run) => run.id).sort()).toEqual([runId, runA, runB, runC].sort());
+ });
+
+ it("locks the context issue and every sibling issue in id order, and two concurrent releases do not deadlock", async () => {
+ const companyId = await seedCompany();
+ const agentId = await seedAgent({ companyId });
+ const issueA = await seedIssue({ companyId, assigneeAgentId: agentId });
+ const issueB = await seedIssue({ companyId, assigneeAgentId: agentId });
+ const runA = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueA } });
+ const runB = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueB } });
+ await db.update(issues).set({ executionRunId: runA, checkoutRunId: runB }).where(eq(issues.id, issueA));
+ await db.update(issues).set({ executionRunId: runB, checkoutRunId: runA }).where(eq(issues.id, issueB));
+
+ const adapterA = createPostgresWakeQueueAdapter(db, stubDeps);
+ const adapterB = createPostgresWakeQueueAdapter(db, stubDeps);
+ const releaseA = adapterA.withIssueExecutionLock({ companyId, runId: runA, now: new Date() }, async (_locked, _ports) => {
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ });
+ const releaseB = adapterB.withIssueExecutionLock({ companyId, runId: runB, now: new Date() }, async (_locked, _ports) => {
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ return { outcome: { kind: "released" as const }, postCommitEffects: [] };
+ });
+
+ await expect(Promise.all([releaseA, releaseB])).resolves.toBeDefined();
+
+ const rows = await db.select().from(issues).where(eq(issues.companyId, companyId));
+ for (const row of rows) {
+ expect(row.executionRunId).toBeNull();
+ expect(row.checkoutRunId).toBeNull();
+ }
+ });
+
+ it("clears both lock columns on every sibling and keeps a transferred executionRunId", async () => {
+ const companyId = await seedCompany();
+ const agentId = await seedAgent({ companyId });
+ const finishingRunId = await seedRun({ companyId, agentId, contextSnapshot: {} });
+ const retryRunId = await seedRun({ companyId, agentId, contextSnapshot: {}, status: "queued" });
+ const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, executionRunId: retryRunId, checkoutRunId: finishingRunId });
+
+ const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
+ const result = await adapter.withIssueExecutionLock({ companyId, runId: finishingRunId, now: new Date() }, async () => ({
+ outcome: { kind: "released" as const },
+ postCommitEffects: [],
+ }));
+ expect(result.outcome.kind).toBe("released");
+
+ const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
+ // executionRunId already pointed at the retry, not the finishing run, so it must survive.
+ expect(issueRow?.executionRunId).toBe(retryRunId);
+ expect(issueRow?.checkoutRunId).toBeNull();
+ });
+});
diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts
new file mode 100644
index 0000000000..f3fd525818
--- /dev/null
+++ b/server/src/modules/wake-queue/adapters/postgres.ts
@@ -0,0 +1,817 @@
+import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import {
+ agentWakeupRequests,
+ agents,
+ heartbeatRuns,
+ issueComments,
+ issueRecoveryActions,
+ issueRelations,
+ issues,
+ nativeRunFinalizations,
+} from "@paperclipai/db";
+import { legacyExecutionNeedsReconciliation } from "../../../services/legacy-execution-recovery.js";
+import { evaluateAgentInvokability } from "../../../services/agent-invokability.js";
+import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake } from "../../../services/issue-tree-control.js";
+import { isAutomaticRecoverySuppressedByPauseHold } from "../../../services/recovery/pause-hold-guard.js";
+import { issueService } from "../../../services/issues.js";
+import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js";
+import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js";
+import { withRecoveryContext } from "../../../services/recovery/status-only-context.js";
+import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js";
+import {
+ buildConfigurationIncompleteRecoveryNoticeSeed,
+ buildExecutionReviewParticipantRecoveryNoticeSeed,
+ buildImmediateExecutionPathRecoveryNoticeSeed,
+ buildWorkspaceValidationRecoveryNoticeSeed,
+} from "../../../services/recovery/stranded-notice.js";
+import {
+ queuedCommentIdsFromWakePayload,
+ withQueuedCommentIdsInWakePayload,
+} from "../../../services/issue-queued-comment-queue.js";
+import { extractWakeCommentIds } from "../../run-dispatch/index.js";
+import { hasInteractionContinuationWakeContext } from "../domain/context.js";
+import type {
+ DeferredWakeCandidate,
+ InvokableAgentSnapshot,
+ IssueLockWriter,
+ IssueSnapshot,
+ LockedIssueExecution,
+ ReleaseTransactionResult,
+ RunSnapshot,
+ WakeQueueReader,
+ WakeQueueWriter,
+} from "../application/ports.js";
+import type { RunSummary } from "../application/types.js";
+import { WakeQueueApplicationError } from "../application/types.js";
+
+const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
+const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
+const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
+const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
+const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
+const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete";
+const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery";
+const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = "execution_review_participant_recovery";
+const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery";
+const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
+
+type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
+type IssueRow = typeof issues.$inferSelect;
+
+function parseObject(value: unknown): Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value)
+ ? (value as Record<string, unknown>)
+ : {};
+}
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+function normalizeAgentNameKey(value: string | null | undefined): string | null {
+ if (typeof value !== "string") return null;
+ const normalized = value.trim().toLowerCase();
+ return normalized.length > 0 ? normalized : null;
+}
+
+function isWorkspaceValidationFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
+ return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
+}
+
+function isConfigurationIncompleteFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
+ return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found";
+}
+
+function toRunSnapshot(row: HeartbeatRunRow): RunSnapshot {
+ const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete);
+ return {
+ id: row.id,
+ companyId: row.companyId,
+ agentId: row.agentId,
+ status: row.status,
+ runtimeMode: row.runtimeMode,
+ errorCode: row.errorCode,
+ responsibleUserId: row.responsibleUserId,
+ contextSnapshot: parseObject(row.contextSnapshot),
+ configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null,
+ };
+}
+
+function toIssueSnapshot(row: IssueRow): IssueSnapshot {
+ return {
+ id: row.id,
+ companyId: row.companyId,
+ identifier: row.identifier ?? "",
+ status: row.status,
+ assigneeAgentId: row.assigneeAgentId,
+ assigneeUserId: row.assigneeUserId,
+ hiddenAt: row.hiddenAt,
+ originKind: row.originKind,
+ monitorNextCheckAt: row.monitorNextCheckAt,
+ executionState: (row.executionState as Record<string, unknown> | null) ?? null,
+ responsibleUserId: row.responsibleUserId,
+ parentId: row.parentId,
+ originId: row.originId,
+ originRunId: row.originRunId,
+ };
+}
+
+function toRunSummary(row: HeartbeatRunRow): RunSummary {
+ return {
+ id: row.id,
+ companyId: row.companyId,
+ agentId: row.agentId,
+ invocationSource: row.invocationSource,
+ triggerDetail: row.triggerDetail,
+ wakeupRequestId: row.wakeupRequestId,
+ };
+}
+
+function toDeferredWakeCandidate(row: typeof agentWakeupRequests.$inferSelect): DeferredWakeCandidate {
+ const payload = parseObject(row.payload);
+ const queuedCommentIds = queuedCommentIdsFromWakePayload(payload);
+ const deferredContextSeed = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
+ const deferredCommentIds = extractWakeCommentIds(deferredContextSeed);
+ const wakeReason = readNonEmptyString(deferredContextSeed.wakeReason);
+ const queuedReason = wakeReason ?? readNonEmptyString(row.reason);
+ const queuedWakeIsCommentOnly =
+ !queuedReason ||
+ queuedReason === "issue_commented" ||
+ queuedReason === "issue_reopened_via_comment" ||
+ queuedReason === "issue_comment_mentioned";
+ const preservesIndependentContinuation =
+ hasInteractionContinuationWakeContext(deferredContextSeed) ||
+ deferredContextSeed.resumeIntent === true ||
+ !queuedWakeIsCommentOnly;
+
+ return {
+ id: row.id,
+ companyId: row.companyId,
+ agentId: row.agentId,
+ reason: row.reason,
+ source: row.source,
+ triggerDetail: row.triggerDetail,
+ requestedByActorType: row.requestedByActorType,
+ requestedByActorId: row.requestedByActorId,
+ payload,
+ queuedCommentIds,
+ preservesIndependentContinuation,
+ deferredContextSeed,
+ deferredCommentIds,
+ wakeReason,
+ };
+}
+
+export type WakeQueuePostgresAdapterDeps = {
+ resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"];
+ getRoutineEnv: WakeQueueReader["getRoutineEnv"];
+ resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"];
+};
+
+function buildReader(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueReader {
+ return {
+ async findInvokableAgent({ companyId, agentId }): Promise<InvokableAgentSnapshot | null> {
+ const agent = await tx
+ .select()
+ .from(agents)
+ .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)))
+ .then((rows) => rows[0] ?? null);
+ if (!agent) return null;
+ const companyAgents = await tx
+ .select({ id: agents.id, companyId: agents.companyId, name: agents.name, reportsTo: agents.reportsTo, status: agents.status })
+ .from(agents)
+ .where(eq(agents.companyId, companyId));
+ const invokability = evaluateAgentInvokability(agent, companyAgents);
+ return { id: agent.id, companyId: agent.companyId, name: agent.name, invokable: invokability.invokable };
+ },
+ resolveResponsibleUserId: deps.resolveResponsibleUserId,
+ getRoutineEnv: deps.getRoutineEnv,
+ resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup,
+ };
+}
+
+function buildWriter(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueWriter {
+ const treeControlSvc = issueTreeControlService(tx);
+ const issuesSvc = issueService(tx);
+
+ return {
+ async claimNextDeferredWake({ companyId, issueId }) {
+ const row = await tx
+ .select()
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
+ sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
+ ),
+ )
+ .orderBy(asc(agentWakeupRequests.requestedAt))
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ return row ? toDeferredWakeCandidate(row) : null;
+ },
+
+ async getQueuedCommentLiveness({ companyId, issueId, wakeAgentId, finishingRunId, finishingRunAgentId, queuedCommentIds }) {
+ const rows = await tx
+ .select({ id: issueComments.id, deletedAt: issueComments.deletedAt, createdByRunId: issueComments.createdByRunId })
+ .from(issueComments)
+ .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, queuedCommentIds)));
+ const targetsFinishingRunAgent = wakeAgentId === finishingRunAgentId;
+ const liveNonSelfCommentIds = queuedCommentIds.filter((commentId) => {
+ const row = rows.find((candidate) => candidate.id === commentId);
+ return Boolean(row && !row.deletedAt && (!targetsFinishingRunAgent || row.createdByRunId !== finishingRunId));
+ });
+ const containedSelfAuthoredComment = rows.some(
+ (row) => targetsFinishingRunAgent && !row.deletedAt && row.createdByRunId === finishingRunId,
+ );
+ return { liveNonSelfCommentIds, containedSelfAuthoredComment };
+ },
+
+ async cancelDeferredWake({ companyId, wakeId, reason, now }) {
+ const rows = await tx
+ .update(agentWakeupRequests)
+ .set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now })
+ .where(
+ and(
+ eq(agentWakeupRequests.id, wakeId),
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
+ ),
+ )
+ .returning({ id: agentWakeupRequests.id });
+ return rows.length > 0;
+ },
+
+ async normalizeDeferredWakeCommentIds({ companyId, wakeId, payload, liveCommentIds, now }) {
+ const rows = await tx
+ .update(agentWakeupRequests)
+ .set({ payload: withQueuedCommentIdsInWakePayload(payload, liveCommentIds), updatedAt: now })
+ .where(
+ and(
+ eq(agentWakeupRequests.id, wakeId),
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
+ ),
+ )
+ .returning();
+ const row = rows[0];
+ return row ? toDeferredWakeCandidate(row) : null;
+ },
+
+ async failDeferredWake({ companyId, wakeId, now }) {
+ const rows = await tx
+ .update(agentWakeupRequests)
+ .set({
+ status: "failed",
+ finishedAt: now,
+ error: "Deferred wake could not be promoted: agent is not invokable",
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(agentWakeupRequests.id, wakeId),
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
+ ),
+ )
+ .returning({ id: agentWakeupRequests.id });
+ return rows.length > 0;
+ },
+
+ async getPauseHoldFacts({ companyId, issueId, wakeAgentId, deferredContextSeed, requestedByActorType, requestedByActorId }) {
+ const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId);
+ if (!activePauseHold) {
+ return {
+ activePauseHold: false,
+ treeHoldInteractionWake: false,
+ holdId: null,
+ rootIssueId: null,
+ mode: null,
+ reason: null,
+ releasePolicy: null,
+ };
+ }
+ const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(tx, {
+ companyId,
+ issueId,
+ agentId: wakeAgentId,
+ contextSnapshot: deferredContextSeed,
+ requestedByActorType,
+ requestedByActorId,
+ });
+ return {
+ activePauseHold: true,
+ treeHoldInteractionWake,
+ holdId: activePauseHold.holdId,
+ rootIssueId: activePauseHold.rootIssueId,
+ mode: activePauseHold.mode,
+ reason: activePauseHold.reason,
+ releasePolicy: activePauseHold.releasePolicy,
+ };
+ },
+
+ async getCommentSelfAuthorship({ companyId, issueId, finishingRunId, commentIds }) {
+ const rows = await tx
+ .select({ createdByRunId: issueComments.createdByRunId })
+ .from(issueComments)
+ .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, commentIds)));
+ return { allSelfAuthored: rows.length > 0 && rows.every((row) => row.createdByRunId === finishingRunId) };
+ },
+
+ async reopenIssue({ companyId, issueId }) {
+ const updated = await issuesSvc.updateForCompany(issueId, companyId, { status: "todo", executionState: null }, tx);
+ return updated ? toIssueSnapshot(updated as unknown as IssueRow) : null;
+ },
+
+ async claimDeferredWakeForPromotion({ companyId, wakeId, now }) {
+ const claimed = await tx
+ .update(agentWakeupRequests)
+ .set({
+ status: "queued",
+ reason: "issue_execution_promoted",
+ claimedAt: null,
+ finishedAt: null,
+ error: null,
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(agentWakeupRequests.id, wakeId),
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
+ ),
+ )
+ .returning({ id: agentWakeupRequests.id });
+ return claimed.length > 0;
+ },
+
+ async finalizePromotedWake(input) {
+ const newRun = await tx
+ .insert(heartbeatRuns)
+ .values({
+ companyId: input.deferredAgent.companyId,
+ agentId: input.deferredAgent.id,
+ invocationSource: input.source,
+ triggerDetail: input.triggerDetail,
+ status: "queued",
+ wakeupRequestId: input.wakeId,
+ contextSnapshot: input.contextSnapshot,
+ responsibleUserId: input.responsibleUserId,
+ sessionIdBefore: input.sessionBefore,
+ continuationAttempt: readContinuationAttempt(input.contextSnapshot.livenessContinuationAttempt),
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ // `claimDeferredWakeForPromotion` already moved this row off
+ // `deferred_issue_execution` inside this same transaction, so no
+ // concurrent claimer can still match that guard; this extra `runId is
+ // null` guard only protects against writing the link twice.
+ await tx
+ .update(agentWakeupRequests)
+ .set({ runId: newRun.id, updatedAt: input.now })
+ .where(
+ and(
+ eq(agentWakeupRequests.id, input.wakeId),
+ eq(agentWakeupRequests.companyId, input.companyId),
+ isNull(agentWakeupRequests.runId),
+ ),
+ );
+
+ // Promoted mention wakes are issue-scoped, not issue ownership
+ // transfers. The lock-clearing step earlier in this transaction
+ // already set `executionRunId` to null for this issue, so the `is
+ // null` guard only protects against taking the lock twice.
+ await tx
+ .update(issues)
+ .set({
+ executionRunId: newRun.id,
+ executionAgentNameKey: normalizeAgentNameKey(input.deferredAgent.name),
+ executionLockedAt: input.now,
+ updatedAt: input.now,
+ })
+ .where(
+ and(
+ eq(issues.id, input.issue.id),
+ eq(issues.companyId, input.companyId),
+ eq(issues.assigneeAgentId, input.deferredAgent.id),
+ isNull(issues.executionRunId),
+ ),
+ );
+
+ return toRunSummary(newRun);
+ },
+
+ async hasExistingExecutionPath({ companyId, issueId, excludeRunId, agentId }) {
+ const row = await tx
+ .select({ id: heartbeatRuns.id })
+ .from(heartbeatRuns)
+ .where(
+ and(
+ eq(heartbeatRuns.companyId, companyId),
+ inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]),
+ sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
+ sql`${heartbeatRuns.id} <> ${excludeRunId}`,
+ agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`,
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ return row !== null;
+ },
+
+ async hasExplicitBlockerPath({ companyId, issueId }) {
+ const row = await tx
+ .select({ issueId: issueRelations.issueId })
+ .from(issueRelations)
+ .innerJoin(issues, eq(issueRelations.issueId, issues.id))
+ .where(
+ and(
+ eq(issueRelations.companyId, companyId),
+ eq(issueRelations.relatedIssueId, issueId),
+ eq(issueRelations.type, "blocks"),
+ eq(issues.companyId, companyId),
+ notInArray(issues.status, ["done", "cancelled"]),
+ isNull(issues.hiddenAt),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ return row !== null;
+ },
+
+ async isAutomaticRecoverySuppressedByPauseHold({ companyId, issueId }) {
+ return isAutomaticRecoverySuppressedByPauseHold(tx, companyId, issueId, treeControlSvc);
+ },
+
+ async buildBlockedRecoveryNotice({ noticeKind, issueStatus, finishingRun }) {
+ if (noticeKind === "workspace_validation") {
+ return { notice: buildWorkspaceValidationRecoveryNoticeSeed(), recoveryCause: WORKSPACE_VALIDATION_RECOVERY_CAUSE };
+ }
+ if (noticeKind === "configuration_incomplete") {
+ return {
+ notice: buildConfigurationIncompleteRecoveryNoticeSeed(finishingRun.configurationIncompletePayload),
+ recoveryCause: CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE,
+ };
+ }
+ if (noticeKind === "execution_review_participant") {
+ return {
+ notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
+ recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE,
+ };
+ }
+ return { notice: buildImmediateExecutionPathRecoveryNoticeSeed({ status: issueStatus }), recoveryCause: null };
+ },
+
+ async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
+ const executionState = parseIssueExecutionState(issue.executionState);
+ const wakeupRequest = await tx
+ .insert(agentWakeupRequests)
+ .values({
+ companyId,
+ agentId: recoveryAgent.id,
+ source: "automation",
+ triggerDetail: "system",
+ reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
+ payload: withRecoveryContext(
+ {
+ issueId: issue.id,
+ retryOfRunId: finishingRun.id,
+ retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
+ currentStageId: executionState?.currentStageId ?? null,
+ currentStageType: executionState?.currentStageType ?? null,
+ },
+ "normal_model",
+ ),
+ status: "queued",
+ requestedByActorType: "system",
+ requestedByActorId: null,
+ updatedAt: now,
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ const queuedRun = await tx
+ .insert(heartbeatRuns)
+ .values({
+ companyId,
+ agentId: recoveryAgent.id,
+ invocationSource: "automation",
+ triggerDetail: "system",
+ status: "queued",
+ wakeupRequestId: wakeupRequest.id,
+ contextSnapshot: withRecoveryContext(
+ {
+ issueId: issue.id,
+ taskId: issue.id,
+ wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
+ retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
+ source: "issue.execution_review_recovery",
+ retryOfRunId: finishingRun.id,
+ currentStageId: executionState?.currentStageId ?? null,
+ currentStageType: executionState?.currentStageType ?? null,
+ reviewRecoveryInstruction:
+ "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",
+ },
+ "normal_model",
+ ),
+ sessionIdBefore: sessionBefore,
+ retryOfRunId: finishingRun.id,
+ updatedAt: now,
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ await tx
+ .update(agentWakeupRequests)
+ .set({ runId: queuedRun.id, updatedAt: now })
+ .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
+
+ await tx
+ .update(issues)
+ .set({
+ executionRunId: queuedRun.id,
+ executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
+ executionLockedAt: now,
+ updatedAt: now,
+ })
+ .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
+
+ return toRunSummary(queuedRun);
+ },
+
+ async queueImmediateRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
+ const retryReason = issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed";
+ const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed";
+ const recoverySource = issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery";
+ const recoveryContextSnapshot = withRecoveryContext(
+ {
+ issueId: issue.id,
+ taskId: issue.id,
+ wakeReason: recoveryReason,
+ retryReason,
+ source: recoverySource,
+ retryOfRunId: finishingRun.id,
+ },
+ "normal_model",
+ );
+
+ const routineEnvContext = await deps.getRoutineEnv({ companyId, issue });
+ const responsibleUserId = await deps.resolveResponsibleUserId({
+ companyId,
+ contextSnapshot: recoveryContextSnapshot,
+ issue,
+ routineEnvContext,
+ requestedByActorType: "system",
+ requestedByActorId: null,
+ source: "automation",
+ triggerDetail: "system",
+ existingRunResponsibleUserId: finishingRun.responsibleUserId,
+ });
+ if (!responsibleUserId) {
+ throw new WakeQueueApplicationError(
+ "responsible_user_unresolved",
+ "Unable to resolve responsible user for recovery heartbeat run",
+ {
+ runId: finishingRun.id,
+ agentId: recoveryAgent.id,
+ companyId,
+ issueId: issue.id,
+ wakeReason: recoveryReason,
+ },
+ );
+ }
+
+ const wakeupRequest = await tx
+ .insert(agentWakeupRequests)
+ .values({
+ companyId,
+ agentId: recoveryAgent.id,
+ source: "automation",
+ triggerDetail: "system",
+ reason: recoveryReason,
+ payload: withRecoveryContext({ issueId: issue.id, retryOfRunId: finishingRun.id }, "normal_model"),
+ status: "queued",
+ requestedByActorType: "system",
+ requestedByActorId: null,
+ updatedAt: now,
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ const queuedRun = await tx
+ .insert(heartbeatRuns)
+ .values({
+ companyId,
+ agentId: recoveryAgent.id,
+ invocationSource: "automation",
+ triggerDetail: "system",
+ status: "queued",
+ wakeupRequestId: wakeupRequest.id,
+ contextSnapshot: recoveryContextSnapshot,
+ responsibleUserId,
+ sessionIdBefore: sessionBefore,
+ retryOfRunId: finishingRun.id,
+ updatedAt: now,
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ await tx
+ .update(agentWakeupRequests)
+ .set({ runId: queuedRun.id, updatedAt: now })
+ .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
+
+ await tx
+ .update(issues)
+ .set({
+ executionRunId: queuedRun.id,
+ executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
+ executionLockedAt: now,
+ updatedAt: now,
+ })
+ .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
+
+ return toRunSummary(queuedRun);
+ },
+ };
+}
+
+async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: HeartbeatRunRow, issue: IssueRow, now: Date): Promise<boolean> {
+ const applies =
+ run.runtimeMode === "native" &&
+ ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) &&
+ issue.assigneeAgentId === run.agentId &&
+ !["done", "cancelled"].includes(issue.status);
+ if (!applies) return false;
+
+ const existing = await tx
+ .select({ id: issueRecoveryActions.id })
+ .from(issueRecoveryActions)
+ .where(
+ and(
+ eq(issueRecoveryActions.companyId, issue.companyId),
+ eq(issueRecoveryActions.sourceIssueId, issue.id),
+ or(
+ inArray(issueRecoveryActions.status, ["active", "escalated"]),
+ sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`,
+ ),
+ ),
+ )
+ .limit(1);
+ if (!existing.length) {
+ await tx
+ .update(nativeRunFinalizations)
+ .set({
+ phase: "terminal_failure",
+ leaseOwner: null,
+ leaseExpiresAt: null,
+ nextAttemptAt: null,
+ recoveryState: "blocked",
+ failureCode: "native_continuation_requires_reconciliation",
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(nativeRunFinalizations.companyId, issue.companyId),
+ eq(nativeRunFinalizations.runId, run.id),
+ isNull(nativeRunFinalizations.resultId),
+ ),
+ );
+ await issueRecoveryActionService(tx).upsertSourceScoped({
+ companyId: issue.companyId,
+ sourceIssueId: issue.id,
+ kind: "active_run_watchdog",
+ ownerType: "board",
+ returnOwnerAgentId: run.agentId,
+ cause: "native_continuation_requires_reconciliation",
+ fingerprint: `native-continuation:${run.id}`,
+ evidence: { runId: run.id, originalFailureCode: run.errorCode },
+ nextAction:
+ "Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.",
+ maxAttempts: 3,
+ wakePolicy: null,
+ supersedeOnIdentityChange: true,
+ });
+ }
+ return true;
+}
+
+export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter {
+ return {
+ async withIssueExecutionLock(input, fn): Promise<ReleaseTransactionResult & { run: RunSnapshot }> {
+ return db.transaction(async (rawTx) => {
+ const tx = rawTx as unknown as Db;
+ const run = await tx
+ .select()
+ .from(heartbeatRuns)
+ .where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)))
+ .then((rows) => rows[0] ?? null);
+ if (!run) {
+ throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`);
+ }
+ const runSnapshot = toRunSnapshot(run);
+ const contextIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
+
+ // Lock the context issue (if any) and every issue that still references this
+ // run, in id order, so two concurrent finalizations can never deadlock on
+ // each other's row-lock acquisition order.
+ await tx.execute(
+ contextIssueId
+ ? sql`
+ select id from issues
+ where company_id = ${input.companyId}
+ and (
+ id = ${contextIssueId}
+ or execution_run_id = ${run.id}
+ or checkout_run_id = ${run.id}
+ )
+ order by id
+ for update
+ `
+ : sql`
+ select id from issues
+ where company_id = ${input.companyId}
+ and (execution_run_id = ${run.id} or checkout_run_id = ${run.id})
+ order by id
+ for update
+ `,
+ );
+
+ const candidateIssues = await tx
+ .select()
+ .from(issues)
+ .where(
+ and(
+ eq(issues.companyId, input.companyId),
+ contextIssueId
+ ? or(eq(issues.id, contextIssueId), eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id))
+ : or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)),
+ ),
+ )
+ .orderBy(asc(issues.id));
+
+ // Two separate updates: a retry can move `executionRunId` to a new run
+ // while `checkoutRunId` still points at this one finishing.
+ await tx
+ .update(issues)
+ .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now })
+ .where(and(eq(issues.companyId, input.companyId), eq(issues.executionRunId, run.id)));
+ await tx
+ .update(issues)
+ .set({ checkoutRunId: null, updatedAt: input.now })
+ .where(and(eq(issues.companyId, input.companyId), eq(issues.checkoutRunId, run.id)));
+
+ const issueRow =
+ (contextIssueId ? candidateIssues.find((candidate) => candidate.id === contextIssueId) : candidateIssues[0]) ?? null;
+
+ if (!issueRow || (issueRow.executionRunId && issueRow.executionRunId !== run.id)) {
+ return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
+ }
+
+ if (
+ (isWorkspaceValidationFailedRun(run) || isConfigurationIncompleteFailedRun(run)) &&
+ (issueRow.status === "todo" || issueRow.status === "in_progress") &&
+ !issueRow.assigneeUserId &&
+ issueRow.assigneeAgentId === run.agentId
+ ) {
+ const configurationIncomplete = isConfigurationIncompleteFailedRun(run);
+ const notice = configurationIncomplete
+ ? buildConfigurationIncompleteRecoveryNoticeSeed(runSnapshot.configurationIncompletePayload)
+ : buildWorkspaceValidationRecoveryNoticeSeed();
+ return {
+ outcome: {
+ kind: "blocked",
+ issue: toIssueSnapshot(issueRow),
+ previousStatus: issueRow.status as "todo" | "in_progress",
+ notice,
+ recoveryCause: configurationIncomplete ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE : WORKSPACE_VALIDATION_RECOVERY_CAUSE,
+ },
+ postCommitEffects: [],
+ run: runSnapshot,
+ };
+ }
+
+ if (legacyExecutionNeedsReconciliation(run)) {
+ return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
+ }
+
+ // An operator stop never promotes old queued work by itself. The next
+ // explicit wake adopts those messages atomically when it queues a run.
+ if (run.status === "cancelled" && parseObject(run.resultJson?.executionCancellation).state === "acknowledged") {
+ return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
+ }
+
+ if (await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) {
+ return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
+ }
+
+ const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot };
+ const result = await fn(locked, { reader: buildReader(tx, deps), writer: buildWriter(tx, deps) });
+ return { ...result, run: runSnapshot };
+ });
+ },
+ };
+}
diff --git a/server/src/modules/wake-queue/application/ports.ts b/server/src/modules/wake-queue/application/ports.ts
new file mode 100644
index 0000000000..bce32babfd
--- /dev/null
+++ b/server/src/modules/wake-queue/application/ports.ts
@@ -0,0 +1,244 @@
+import type { ReleaseRecoveryBlockedNoticeKind } from "../domain/policy.js";
+import type {
+ InvokableAgentSnapshot,
+ IssueSnapshot,
+ PostCommitEffect,
+ ReleaseOutcome,
+ RunSnapshot,
+ RunSummary,
+} from "./types.js";
+
+export type { InvokableAgentSnapshot, IssueSnapshot, RunSnapshot, RunSummary };
+
+/** The primary issue a locked release resolves to, plus the finishing run the lock step already loaded. */
+export type LockedIssueExecution = {
+ primaryIssue: IssueSnapshot;
+ run: RunSnapshot;
+};
+
+export type ReleaseTransactionResult = {
+ outcome: ReleaseOutcome;
+ postCommitEffects: PostCommitEffect[];
+};
+
+/** Read-only lookups the release use case needs, each scoped to a company. */
+export interface WakeQueueReader {
+ findInvokableAgent(input: { companyId: string; agentId: string }): Promise<InvokableAgentSnapshot | null>;
+ /**
+ * Takes the transaction-scoped issue snapshot, not an issue id, so this
+ * port never re-reads the issue on a separate connection while the
+ * module's own transaction is open.
+ */
+ resolveResponsibleUserId(input: {
+ companyId: string;
+ contextSnapshot: Record<string, unknown>;
+ issue: IssueSnapshot;
+ /** From a prior `getRoutineEnv` call against the same issue; pass `{ routineId: null, env: null, responsibleUserId: null }` when the issue is not a routine execution. */
+ routineEnvContext: { routineId: string | null; env: unknown; responsibleUserId: string | null };
+ requestedByActorType: "user" | "agent" | "system" | null;
+ requestedByActorId: string | null;
+ source: string;
+ triggerDetail: string | null;
+ existingRunResponsibleUserId: string | null;
+ }): Promise<string | null>;
+ /**
+ * Takes the transaction-scoped issue snapshot, not an issue id, so this
+ * port never re-reads the issue on a separate connection while the
+ * module's own transaction is open.
+ */
+ getRoutineEnv(input: {
+ companyId: string;
+ issue: IssueSnapshot;
+ }): Promise<{ routineId: string | null; env: unknown; responsibleUserId: string | null }>;
+ resolveSessionBeforeForWakeup(input: {
+ companyId: string;
+ agentId: string;
+ taskKey: string | null;
+ }): Promise<string | null>;
+}
+
+export type DeferredWakeCandidate = {
+ id: string;
+ companyId: string;
+ agentId: string;
+ reason: string | null;
+ source: string | null;
+ triggerDetail: string | null;
+ requestedByActorType: string | null;
+ requestedByActorId: string | null;
+ payload: Record<string, unknown>;
+ /** The queued comment ids the wake's queued-comment context carries, already extracted from the payload. */
+ queuedCommentIds: string[];
+ /** True when the wake carries an independent reason to continue even with no live queued comments. */
+ preservesIndependentContinuation: boolean;
+ /** `payload._paperclipWakeContext`, already parsed to a plain object. */
+ deferredContextSeed: Record<string, unknown>;
+ /** The comment ids the wake's context snapshot carries (a separate set from queuedCommentIds), used for the reopen check. */
+ deferredCommentIds: string[];
+ wakeReason: string | null;
+};
+
+export type PromoteDeferredWakeInput = {
+ companyId: string;
+ wakeId: string;
+ deferredAgent: InvokableAgentSnapshot;
+ issue: IssueSnapshot;
+ finishingRun: RunSnapshot;
+ contextSnapshot: Record<string, unknown>;
+ reason: string;
+ source: string;
+ triggerDetail: string | null;
+ payload: Record<string, unknown>;
+ responsibleUserId: string;
+ sessionBefore: string | null;
+ now: Date;
+};
+
+/** The transaction-scoped write operations that drain and resolve the deferred-wake queue. */
+export interface WakeQueueWriter {
+ claimNextDeferredWake(input: { companyId: string; issueId: string }): Promise<DeferredWakeCandidate | null>;
+ getQueuedCommentLiveness(input: {
+ companyId: string;
+ issueId: string;
+ wakeAgentId: string;
+ finishingRunId: string;
+ finishingRunAgentId: string;
+ queuedCommentIds: string[];
+ }): Promise<{ liveNonSelfCommentIds: string[]; containedSelfAuthoredComment: boolean }>;
+ /** Cancels the wake with `status = 'deferred_issue_execution'` as an atomic compare-and-set guard. */
+ cancelDeferredWake(input: {
+ companyId: string;
+ wakeId: string;
+ reason: string;
+ now: Date;
+ }): Promise<boolean>;
+ normalizeDeferredWakeCommentIds(input: {
+ companyId: string;
+ wakeId: string;
+ /** The wake's current payload, as already read by `claimNextDeferredWake`, used as the rewrite base. */
+ payload: Record<string, unknown>;
+ liveCommentIds: string[];
+ now: Date;
+ }): Promise<DeferredWakeCandidate | null>;
+ /** Sets `status = 'failed'` guarded by the current `deferred_issue_execution` status. */
+ failDeferredWake(input: { companyId: string; wakeId: string; now: Date }): Promise<boolean>;
+ getPauseHoldFacts(input: {
+ companyId: string;
+ issueId: string;
+ wakeAgentId: string;
+ deferredContextSeed: Record<string, unknown>;
+ requestedByActorType: string | null;
+ requestedByActorId: string | null;
+ }): Promise<{
+ activePauseHold: boolean;
+ treeHoldInteractionWake: boolean;
+ holdId: string | null;
+ rootIssueId: string | null;
+ mode: string | null;
+ reason: string | null;
+ releasePolicy: unknown;
+ }>;
+ getCommentSelfAuthorship(input: {
+ companyId: string;
+ issueId: string;
+ finishingRunId: string;
+ commentIds: string[];
+ }): Promise<{ allSelfAuthored: boolean }>;
+ reopenIssue(input: { companyId: string; issueId: string; runId: string }): Promise<IssueSnapshot | null>;
+ /**
+ * Atomically claims the wake for promotion, guarded on its current
+ * `deferred_issue_execution` status. Call this before any other write in
+ * the promotion path (including a reopen), so a lost race here can never
+ * leave another write committed underneath it. Returns `false` when a
+ * concurrent writer already changed the wake's status.
+ */
+ claimDeferredWakeForPromotion(input: { companyId: string; wakeId: string; now: Date }): Promise<boolean>;
+ /**
+ * Finalizes a wake that `claimDeferredWakeForPromotion` already claimed:
+ * inserts the queued run, links it back onto the wake row, and takes the
+ * issue's execution lock. Call only after that claim returns `true`.
+ */
+ finalizePromotedWake(input: PromoteDeferredWakeInput): Promise<RunSummary>;
+ /** An open run already on this issue (optionally scoped to one agent) that would race a new recovery run. */
+ hasExistingExecutionPath(input: {
+ companyId: string;
+ issueId: string;
+ excludeRunId: string;
+ agentId: string | null;
+ }): Promise<boolean>;
+ /** An open, non-hidden issue that still lists this issue as a `blocks` predecessor. */
+ hasExplicitBlockerPath(input: { companyId: string; issueId: string }): Promise<boolean>;
+ isAutomaticRecoverySuppressedByPauseHold(input: { companyId: string; issueId: string }): Promise<boolean>;
+ /** Builds the stranded-recovery notice content for a `blocked` outcome; pure formatting, kept behind the writer so `services/recovery/stranded-notice` stays out of the application layer. */
+ buildBlockedRecoveryNotice(input: {
+ noticeKind: ReleaseRecoveryBlockedNoticeKind;
+ issueStatus: "todo" | "in_progress";
+ finishingRun: RunSnapshot;
+ }): Promise<{ notice: Record<string, unknown>; recoveryCause: string | null }>;
+ queueReviewParticipantRecoveryRun(input: {
+ companyId: string;
+ issue: IssueSnapshot;
+ finishingRun: RunSnapshot;
+ recoveryAgent: InvokableAgentSnapshot;
+ sessionBefore: string | null;
+ now: Date;
+ }): Promise<RunSummary>;
+ /**
+ * Builds the recovery context snapshot, resolves the responsible user
+ * from it, and queues the run. Throws `WakeQueueApplicationError` with
+ * code `responsible_user_unresolved` when no responsible user resolves,
+ * without queuing anything.
+ */
+ queueImmediateRecoveryRun(input: {
+ companyId: string;
+ issue: IssueSnapshot;
+ finishingRun: RunSnapshot;
+ recoveryAgent: InvokableAgentSnapshot;
+ sessionBefore: string | null;
+ now: Date;
+ }): Promise<RunSummary>;
+}
+
+/**
+ * Owns the module's own transaction: loads the finishing run, locks the
+ * context issue and every sibling issue in id order, clears the two
+ * release-lock columns, and picks the primary issue. When the primary
+ * issue is missing, already reclaimed, or resolved by an early exit
+ * (workspace-validation block, legacy reconciliation, a native-runtime
+ * terminal failure), the adapter returns that outcome directly without
+ * calling `fn`. Otherwise it calls `fn` with the locked issue and run, and
+ * with `reader`/`writer` ports bound to the same transaction, so every
+ * call `fn` makes through them participates in the one transaction this
+ * method owns.
+ */
+export interface IssueLockWriter {
+ withIssueExecutionLock(
+ input: { companyId: string; runId: string; now: Date },
+ fn: (
+ locked: LockedIssueExecution,
+ ports: { reader: WakeQueueReader; writer: WakeQueueWriter },
+ ) => Promise<ReleaseTransactionResult>,
+ ): Promise<ReleaseTransactionResult & { run: RunSnapshot }>;
+}
+
+export type StrandedAssignedIssueEscalationInput = {
+ issue: IssueSnapshot;
+ previousStatus: "todo" | "in_progress" | "in_review";
+ latestRun: RunSnapshot;
+ notice: Record<string, unknown>;
+ recoveryCause: string | null;
+};
+
+export type StrandedRecoveryInPlaceEscalationInput = {
+ issue: IssueSnapshot;
+ previousStatus: "todo" | "in_progress" | "in_review";
+ latestRun: RunSnapshot;
+};
+
+/** Wraps `services/recovery`'s stranded-issue escalation, called only after the release transaction commits. */
+export interface RecoveryEscalationPort {
+ escalateStrandedAssignedIssue(input: StrandedAssignedIssueEscalationInput): Promise<void>;
+ escalateStrandedRecoveryIssueInPlace(input: StrandedRecoveryInPlaceEscalationInput): Promise<void>;
+}
+
+export type { PostCommitEffect, ReleaseOutcome };
diff --git a/server/src/modules/wake-queue/application/types.ts b/server/src/modules/wake-queue/application/types.ts
new file mode 100644
index 0000000000..cb126d644d
--- /dev/null
+++ b/server/src/modules/wake-queue/application/types.ts
@@ -0,0 +1,99 @@
+export type RunSummary = {
+ id: string;
+ companyId: string;
+ agentId: string;
+ invocationSource: string;
+ triggerDetail: string | null;
+ wakeupRequestId: string | null;
+};
+
+export type RunSnapshot = {
+ id: string;
+ companyId: string;
+ agentId: string;
+ status: string;
+ runtimeMode: string | null;
+ errorCode: string | null;
+ responsibleUserId: string | null;
+ /** The run's own context snapshot, kept as plain JSON so ports carry no drizzle types. */
+ contextSnapshot: Record<string, unknown>;
+ /** `resultJson.configurationIncomplete`, already parsed; non-null only on a configuration-incomplete failed run. */
+ configurationIncompletePayload: Record<string, unknown> | null;
+};
+
+export type IssueSnapshot = {
+ id: string;
+ companyId: string;
+ identifier: string;
+ status: string;
+ assigneeAgentId: string | null;
+ assigneeUserId: string | null;
+ hiddenAt: Date | null;
+ originKind: string | null;
+ monitorNextCheckAt: Date | null;
+ executionState: Record<string, unknown> | null;
+ /** Carried so the routine-env and responsible-user reader ports can use this
+ * transaction-scoped snapshot instead of reading the issue again. */
+ responsibleUserId: string | null;
+ parentId: string | null;
+ originId: string | null;
+ originRunId: string | null;
+};
+
+export type InvokableAgentSnapshot = {
+ id: string;
+ companyId: string;
+ name: string | null;
+ invokable: boolean;
+};
+
+/** A new heartbeat run reached the queued state and should be published and dispatched. */
+export type RunQueuedEffect = {
+ kind: "run_queued";
+ run: RunSummary;
+};
+
+/** An issue reopened from done/cancelled because a live deferred comment wake promoted on it. */
+export type IssueReopenedEffect = {
+ kind: "issue_reopened";
+ companyId: string;
+ agentId: string;
+ runId: string;
+ issueId: string;
+ identifier: string;
+ reopenedFrom: string;
+};
+
+/** Explicit post-commit work a caller applies only after the release transaction commits. */
+export type PostCommitEffect = RunQueuedEffect | IssueReopenedEffect;
+
+export type ReleaseOutcome =
+ | { kind: "released" }
+ | { kind: "promoted"; run: RunSummary }
+ | { kind: "queued_review_participant_recovery"; run: RunSummary }
+ | { kind: "queued_recovery"; run: RunSummary }
+ | {
+ kind: "blocked";
+ issue: IssueSnapshot;
+ previousStatus: "todo" | "in_progress" | "in_review";
+ notice: Record<string, unknown>;
+ recoveryCause: string | null;
+ }
+ | {
+ kind: "blocked_recovery_in_place";
+ issue: IssueSnapshot;
+ previousStatus: "todo" | "in_progress" | "in_review";
+ };
+
+export type WakeQueueApplicationErrorCode = "responsible_user_unresolved";
+
+export class WakeQueueApplicationError extends Error {
+ constructor(
+ readonly code: WakeQueueApplicationErrorCode,
+ message: string,
+ readonly details: Record<string, unknown> = {},
+ ) {
+ super(message);
+ this.name = "WakeQueueApplicationError";
+ }
+}
diff --git a/server/src/modules/wake-queue/application/use-cases.test.ts b/server/src/modules/wake-queue/application/use-cases.test.ts
new file mode 100644
index 0000000000..81f0719e00
--- /dev/null
+++ b/server/src/modules/wake-queue/application/use-cases.test.ts
@@ -0,0 +1,331 @@
+import { describe, expect, it, vi } from "vitest";
+import { createReleaseIssueExecution } from "./use-cases.js";
+import { WakeQueueApplicationError } from "./types.js";
+import type {
+ DeferredWakeCandidate,
+ InvokableAgentSnapshot,
+ IssueLockWriter,
+ IssueSnapshot,
+ PromoteDeferredWakeInput,
+ RecoveryEscalationPort,
+ RunSnapshot,
+ RunSummary,
+ WakeQueueReader,
+ WakeQueueWriter,
+} from "./ports.js";
+
+const RUN: RunSnapshot = {
+ id: "run-1",
+ companyId: "company-1",
+ agentId: "finishing-agent",
+ status: "failed",
+ runtimeMode: "process",
+ errorCode: null,
+ responsibleUserId: "user-1",
+ contextSnapshot: {},
+ configurationIncompletePayload: null,
+};
+
+const ISSUE: IssueSnapshot = {
+ id: "issue-1",
+ companyId: "company-1",
+ identifier: "PAP-1",
+ status: "in_progress",
+ assigneeAgentId: "finishing-agent",
+ assigneeUserId: null,
+ hiddenAt: null,
+ originKind: null,
+ monitorNextCheckAt: null,
+ executionState: null,
+ responsibleUserId: null,
+ parentId: null,
+ originId: null,
+ originRunId: null,
+};
+
+const AGENT: InvokableAgentSnapshot = {
+ id: "deferred-agent",
+ companyId: "company-1",
+ name: "Deferred Agent",
+ invokable: true,
+};
+
+function wakeCandidate(overrides: Partial<DeferredWakeCandidate> = {}): DeferredWakeCandidate {
+ return {
+ id: overrides.id ?? "wake-1",
+ companyId: "company-1",
+ agentId: "deferred-agent",
+ reason: "issue_commented",
+ source: "automation",
+ triggerDetail: null,
+ requestedByActorType: "user",
+ requestedByActorId: "actor-1",
+ payload: {},
+ queuedCommentIds: [],
+ preservesIndependentContinuation: false,
+ deferredContextSeed: {},
+ deferredCommentIds: [],
+ wakeReason: "issue_commented",
+ ...overrides,
+ };
+}
+
+function runSummary(id: string): RunSummary {
+ return {
+ id,
+ companyId: "company-1",
+ agentId: "deferred-agent",
+ invocationSource: "automation",
+ triggerDetail: null,
+ wakeupRequestId: `wakeup-${id}`,
+ };
+}
+
+function createFakeReader(overrides: Partial<WakeQueueReader> = {}): WakeQueueReader {
+ return {
+ findInvokableAgent: vi.fn(async () => AGENT),
+ resolveResponsibleUserId: vi.fn(async () => "user-1"),
+ getRoutineEnv: vi.fn(async () => ({ routineId: null, env: null, responsibleUserId: null })),
+ resolveSessionBeforeForWakeup: vi.fn(async () => null),
+ ...overrides,
+ };
+}
+
+function createFakeWriter(overrides: Partial<WakeQueueWriter> = {}): WakeQueueWriter {
+ return {
+ claimNextDeferredWake: vi.fn(async () => null),
+ getQueuedCommentLiveness: vi.fn(async () => ({ liveNonSelfCommentIds: [], containedSelfAuthoredComment: false })),
+ cancelDeferredWake: vi.fn(async () => true),
+ normalizeDeferredWakeCommentIds: vi.fn(async (input) => wakeCandidate({ id: input.wakeId, queuedCommentIds: input.liveCommentIds })),
+ failDeferredWake: vi.fn(async () => true),
+ getPauseHoldFacts: vi.fn(async () => ({
+ activePauseHold: false,
+ treeHoldInteractionWake: false,
+ holdId: null,
+ rootIssueId: null,
+ mode: null,
+ reason: null,
+ releasePolicy: null,
+ })),
+ getCommentSelfAuthorship: vi.fn(async () => ({ allSelfAuthored: false })),
+ reopenIssue: vi.fn(async () => null),
+ claimDeferredWakeForPromotion: vi.fn(async () => true),
+ finalizePromotedWake: vi.fn(async (input) => runSummary(input.wakeId)),
+ hasExistingExecutionPath: vi.fn(async () => false),
+ hasExplicitBlockerPath: vi.fn(async () => false),
+ isAutomaticRecoverySuppressedByPauseHold: vi.fn(async () => false),
+ buildBlockedRecoveryNotice: vi.fn(async () => ({ notice: {}, recoveryCause: null })),
+ queueReviewParticipantRecoveryRun: vi.fn(async () => runSummary("review-recovery")),
+ queueImmediateRecoveryRun: vi.fn(async () => runSummary("immediate-recovery")),
+ ...overrides,
+ };
+}
+
+function createFakeIssueLock(reader: WakeQueueReader, writer: WakeQueueWriter): IssueLockWriter {
+ return {
+ withIssueExecutionLock: vi.fn(async (_input, fn) => {
+ const result = await fn({ primaryIssue: ISSUE, run: RUN }, { reader, writer });
+ return { ...result, run: RUN };
+ }),
+ };
+}
+
+function createFakeRecovery(): RecoveryEscalationPort {
+ return {
+ escalateStrandedAssignedIssue: vi.fn(async () => {}),
+ escalateStrandedRecoveryIssueInPlace: vi.fn(async () => {}),
+ };
+}
+
+describe("releaseIssueExecution", () => {
+ it("processes the deferred wakes in requestedAt order", async () => {
+ const claimOrder: string[] = [];
+ const queue = [wakeCandidate({ id: "wake-earliest" }), wakeCandidate({ id: "wake-latest" })];
+ const writer = createFakeWriter({
+ claimNextDeferredWake: vi.fn(async () => {
+ const next = queue.shift() ?? null;
+ if (next) claimOrder.push(next.id);
+ return next;
+ }),
+ });
+ // Every wake fails invokability so the loop keeps draining without promoting.
+ const reader = createFakeReader({ findInvokableAgent: vi.fn(async () => null) });
+ const issueLock = createFakeIssueLock(reader, writer);
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
+
+ await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
+
+ expect(claimOrder).toEqual(["wake-earliest", "wake-latest"]);
+ expect(writer.failDeferredWake).toHaveBeenCalledTimes(2);
+ });
+
+ it("stops the loop after the first promotion", async () => {
+ const claimNextDeferredWake = vi.fn(async () => wakeCandidate({ id: "wake-promotes" }));
+ const writer = createFakeWriter({ claimNextDeferredWake });
+ const reader = createFakeReader();
+ const issueLock = createFakeIssueLock(reader, writer);
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
+
+ const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
+
+ expect(result.outcome.kind).toBe("promoted");
+ expect(claimNextDeferredWake).toHaveBeenCalledTimes(1);
+ });
+
+ it("continues the loop after a cancel outcome, a fail outcome, and a normalize outcome, then promotes", async () => {
+ const queue = [
+ // cancel_empty: queued comments, none live, no independent continuation.
+ wakeCandidate({ id: "wake-cancel", queuedCommentIds: ["c1"] }),
+ // fail_not_invokable: agent lookup misses for this one wake only.
+ wakeCandidate({ id: "wake-fail", agentId: "uninvokable-agent" }),
+ // normalize: queued comments differ from the live set, then promotes.
+ wakeCandidate({ id: "wake-normalize", queuedCommentIds: ["c1", "c2"] }),
+ ];
+ const claimNextDeferredWake = vi.fn(async () => queue.shift() ?? null);
+ const findInvokableAgent = vi.fn(async (input: { agentId: string }) =>
+ input.agentId === "deferred-agent" ? AGENT : null,
+ );
+ const getQueuedCommentLiveness = vi.fn(async (input: { queuedCommentIds: string[] }) =>
+ input.queuedCommentIds.length === 1
+ ? { liveNonSelfCommentIds: [], containedSelfAuthoredComment: false }
+ : { liveNonSelfCommentIds: ["c2"], containedSelfAuthoredComment: false },
+ );
+ const writer = createFakeWriter({ claimNextDeferredWake, getQueuedCommentLiveness });
+ const reader = createFakeReader({ findInvokableAgent });
+ const issueLock = createFakeIssueLock(reader, writer);
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
+
+ const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
+
+ expect(writer.cancelDeferredWake).toHaveBeenCalledTimes(1);
+ expect(writer.failDeferredWake).toHaveBeenCalledTimes(1);
+ expect(writer.normalizeDeferredWakeCommentIds).toHaveBeenCalledTimes(1);
+ expect(claimNextDeferredWake).toHaveBeenCalledTimes(3);
+ expect(result.outcome.kind).toBe("promoted");
+ });
+
+ it("returns the post-commit effects as data without running them", async () => {
+ const writer = createFakeWriter({ claimNextDeferredWake: vi.fn(async () => wakeCandidate()) });
+ const reader = createFakeReader();
+ const issueLock = createFakeIssueLock(reader, writer);
+ const recovery = createFakeRecovery();
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery });
+
+ const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
+
+ expect(result.postCommitEffects).toEqual([{ kind: "run_queued", run: runSummary("wake-1") }]);
+ expect(recovery.escalateStrandedAssignedIssue).not.toHaveBeenCalled();
+ expect(recovery.escalateStrandedRecoveryIssueInPlace).not.toHaveBeenCalled();
+ });
+
+ it("carries the deferred wake's raw issue, interaction, execution-stage, and accepted-plan context onto the promoted run, and clears only the rendered text projections", async () => {
+ const finalizePromotedWake = vi.fn(async (input: PromoteDeferredWakeInput) => runSummary(input.wakeId));
+ const writer = createFakeWriter({
+ claimNextDeferredWake: vi.fn(async () =>
+ wakeCandidate({
+ deferredContextSeed: {
+ issueId: ISSUE.id,
+ wakeCommentIds: ["comment-1"],
+ // A queue-time render from a prior coalesced run. Promotion must
+ // not persist this alongside the current (unrelated) comment id.
+ paperclipTaskMarkdown: "queue-time markdown",
+ paperclipTaskMarkdownCompact: "queue-time compact markdown",
+ paperclipWake: { commentId: "comment-1" },
+ executionStage: { stage: "review" },
+ planReviewInteraction: { acceptedTargetRevision: { revisionId: "revision-1" } },
+ acceptedPlanWakeRouting: { targetAgentId: "agent-1" },
+ },
+ }),
+ ),
+ finalizePromotedWake,
+ });
+ const reader = createFakeReader();
+ const issueLock = createFakeIssueLock(reader, writer);
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
+
+ const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
+
+ expect(result.outcome.kind).toBe("promoted");
+ expect(finalizePromotedWake).toHaveBeenCalledTimes(1);
+ const promotedContextSnapshot = finalizePromotedWake.mock.calls[0]![0].contextSnapshot;
+ // The rendered text is cleared; `executeRun` rebuilds it, with proper
+ // trust-based redaction, from the current issue and comment rows before
+ // the run dispatches.
+ expect(promotedContextSnapshot.paperclipTaskMarkdown).toBeUndefined();
+ expect(promotedContextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
+ expect(promotedContextSnapshot.paperclipWake).toBeUndefined();
+ // The raw fields that render depends on are not dropped.
+ expect(promotedContextSnapshot.issueId).toBe(ISSUE.id);
+ expect(promotedContextSnapshot.executionStage).toEqual({ stage: "review" });
+ expect(promotedContextSnapshot.planReviewInteraction).toEqual({
+ acceptedTargetRevision: { revisionId: "revision-1" },
+ });
+ expect(promotedContextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" });
+ });
+
+ it("never reopens the issue when the promotion claim loses the race, and moves on to the next wake", async () => {
+ const doneIssue: IssueSnapshot = { ...ISSUE, status: "done" };
+ const queue = [
+ // Carries a comment that would reopen the done issue, but the
+ // promotion claim below loses the race before that reopen can run.
+ wakeCandidate({
+ id: "wake-lost-race",
+ deferredCommentIds: ["c1"],
+ requestedByActorType: "user",
+ }),
+ wakeCandidate({ id: "wake-promotes" }),
+ ];
+ const claimNextDeferredWake = vi.fn(async () => queue.shift() ?? null);
+ const claimDeferredWakeForPromotion = vi.fn(async ({ wakeId }: { wakeId: string }) => wakeId !== "wake-lost-race");
+ const reopenIssue = vi.fn(async () => null);
+ const writer = createFakeWriter({ claimNextDeferredWake, claimDeferredWakeForPromotion, reopenIssue });
+ const reader = createFakeReader();
+ const issueLock: IssueLockWriter = {
+ withIssueExecutionLock: vi.fn(async (_input, fn) => {
+ const result = await fn({ primaryIssue: doneIssue, run: RUN }, { reader, writer });
+ return { ...result, run: RUN };
+ }),
+ };
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
+
+ const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
+
+ expect(reopenIssue).not.toHaveBeenCalled();
+ expect(claimDeferredWakeForPromotion).toHaveBeenCalledTimes(2);
+ expect(result.outcome.kind).toBe("promoted");
+ expect(result.postCommitEffects).toEqual([{ kind: "run_queued", run: runSummary("wake-promotes") }]);
+ });
+
+ it("throws WakeQueueApplicationError with code responsible_user_unresolved when the responsible user cannot resolve", async () => {
+ const writer = createFakeWriter({ claimNextDeferredWake: vi.fn(async () => wakeCandidate()) });
+ const reader = createFakeReader({ resolveResponsibleUserId: vi.fn(async () => null) });
+ const issueLock = createFakeIssueLock(reader, writer);
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
+
+ await expect(
+ releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }),
+ ).rejects.toMatchObject({
+ constructor: WakeQueueApplicationError,
+ code: "responsible_user_unresolved",
+ });
+ });
+
+ it("escalates through the recovery port for a blocked outcome, after the transaction resolves", async () => {
+ const writer = createFakeWriter({
+ claimNextDeferredWake: vi.fn(async () => null),
+ hasExistingExecutionPath: vi.fn(async () => false),
+ isAutomaticRecoverySuppressedByPauseHold: vi.fn(async () => false),
+ buildBlockedRecoveryNotice: vi.fn(async () => ({ notice: { kind: "immediate_execution_path" }, recoveryCause: "immediate_execution_path" })),
+ });
+ // The recovery agent (the finishing run's own agent) is not invokable, which forces "blocked".
+ const reader = createFakeReader({ findInvokableAgent: vi.fn(async () => null) });
+ const issueLock = createFakeIssueLock(reader, writer);
+ const recovery = createFakeRecovery();
+ const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery });
+
+ const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
+
+ expect(result.outcome.kind).toBe("blocked");
+ expect(recovery.escalateStrandedAssignedIssue).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/server/src/modules/wake-queue/application/use-cases.ts b/server/src/modules/wake-queue/application/use-cases.ts
new file mode 100644
index 0000000000..6a07e32498
--- /dev/null
+++ b/server/src/modules/wake-queue/application/use-cases.ts
@@ -0,0 +1,503 @@
+import { enrichPromotedWakeContext } from "../domain/context.js";
+import { decideDeferredWake, decideReleaseRecovery } from "../domain/policy.js";
+import type {
+ IssueLockWriter,
+ IssueSnapshot,
+ LockedIssueExecution,
+ RecoveryEscalationPort,
+ ReleaseTransactionResult,
+ RunSnapshot,
+ WakeQueueReader,
+ WakeQueueWriter,
+} from "./ports.js";
+import type { PostCommitEffect, ReleaseOutcome } from "./types.js";
+import { WakeQueueApplicationError } from "./types.js";
+
+const ISSUE_DISPOSITION_REPAIR_RETRY_REASON = "issue_disposition_repair";
+const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery";
+const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS = new Set([
+ "execution_review_requested",
+ "execution_approval_requested",
+]);
+const HEARTBEAT_RUN_TERMINAL_STATUSES = new Set([
+ "succeeded",
+ "failed",
+ "timed_out",
+ "cancelled",
+]);
+const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = new Set([
+ "failed",
+ "timed_out",
+ "cancelled",
+]);
+const STRANDED_ISSUE_RECOVERY_ORIGIN_KIND = "stranded_issue_recovery";
+const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
+const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+function isWorkspaceValidationFailedRun(run: Pick<RunSnapshot, "errorCode">): boolean {
+ return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
+}
+
+function isConfigurationIncompleteFailedRun(run: Pick<RunSnapshot, "errorCode">): boolean {
+ return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found";
+}
+
+function isExecutionReviewParticipantRecoveryRun(run: Pick<RunSnapshot, "contextSnapshot">): boolean {
+ return readNonEmptyString(run.contextSnapshot.retryReason) === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON;
+}
+
+function isExecutionReviewParticipantRecoveryEligibleRun(run: Pick<RunSnapshot, "contextSnapshot">): boolean {
+ const wakeReason = readNonEmptyString(run.contextSnapshot.wakeReason);
+ return (wakeReason !== null && EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS.has(wakeReason))
+ || isExecutionReviewParticipantRecoveryRun(run);
+}
+
+function didAutomaticRecoveryFail(
+ run: Pick<RunSnapshot, "status" | "contextSnapshot">,
+ expectedRetryReason: "assignment_recovery" | "issue_continuation_needed",
+): boolean {
+ const latestRetryReason = readNonEmptyString(run.contextSnapshot.retryReason);
+ return latestRetryReason === expectedRetryReason && UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.has(run.status);
+}
+
+function currentAgentParticipant(issue: IssueSnapshot): { agentId: string } | null {
+ const executionState = issue.executionState;
+ if (!executionState || executionState.status !== "pending") return null;
+ const participant = executionState.currentParticipant as Record<string, unknown> | null | undefined;
+ if (!participant || participant.type !== "agent") return null;
+ const agentId = readNonEmptyString(participant.agentId);
+ return agentId ? { agentId } : null;
+}
+
+export type ReleaseIssueExecutionInput = {
+ companyId: string;
+ runId: string;
+ now: Date;
+ suppressImmediateRecovery?: boolean;
+};
+
+/**
+ * Drains the deferred-wake queue for the issue a run just released, in
+ * `requestedAt` order, promoting at most one wake. When the queue empties
+ * without a promotion, decides the release-recovery outcome. Every read and
+ * write happens through `ports`, already bound to the module's own
+ * transaction by the caller.
+ */
+async function runReleaseDrain(
+ locked: LockedIssueExecution,
+ ports: { reader: WakeQueueReader; writer: WakeQueueWriter },
+ input: ReleaseIssueExecutionInput,
+): Promise<ReleaseTransactionResult> {
+ const { run } = locked;
+ let issue = locked.primaryIssue;
+ const postCommitEffects: PostCommitEffect[] = [];
+
+ while (true) {
+ const candidate = await ports.writer.claimNextDeferredWake({ companyId: run.companyId, issueId: issue.id });
+ if (!candidate) break;
+
+ let liveness = { liveNonSelfCommentIds: candidate.queuedCommentIds, containedSelfAuthoredComment: false };
+ if (candidate.queuedCommentIds.length > 0) {
+ liveness = await ports.writer.getQueuedCommentLiveness({
+ companyId: run.companyId,
+ issueId: issue.id,
+ wakeAgentId: candidate.agentId,
+ finishingRunId: run.id,
+ finishingRunAgentId: run.agentId,
+ queuedCommentIds: candidate.queuedCommentIds,
+ });
+ }
+ const liveCommentIdsChanged =
+ liveness.liveNonSelfCommentIds.length !== candidate.queuedCommentIds.length ||
+ liveness.liveNonSelfCommentIds.some((id, index) => id !== candidate.queuedCommentIds[index]);
+
+ const deferredAgent = await ports.reader.findInvokableAgent({ companyId: run.companyId, agentId: candidate.agentId });
+ const pauseHold = await ports.writer.getPauseHoldFacts({
+ companyId: run.companyId,
+ issueId: issue.id,
+ wakeAgentId: candidate.agentId,
+ deferredContextSeed: candidate.deferredContextSeed,
+ requestedByActorType: candidate.requestedByActorType,
+ requestedByActorId: candidate.requestedByActorId,
+ });
+
+ let decision = decideDeferredWake({
+ queuedComment: {
+ hasQueuedCommentIds: candidate.queuedCommentIds.length > 0,
+ liveNonSelfCommentIdsLength: liveness.liveNonSelfCommentIds.length,
+ queuedCommentIdsLength: candidate.queuedCommentIds.length,
+ liveCommentIdsChanged,
+ containedSelfAuthoredComment: liveness.containedSelfAuthoredComment,
+ preservesIndependentContinuation: candidate.preservesIndependentContinuation,
+ },
+ agent: { agentFound: deferredAgent !== null, invokable: deferredAgent?.invokable ?? false },
+ pauseHold: { activePauseHold: pauseHold.activePauseHold, treeHoldInteractionWake: pauseHold.treeHoldInteractionWake },
+ });
+
+ if (decision.kind === "cancel_empty") {
+ await ports.writer.cancelDeferredWake({
+ companyId: run.companyId,
+ wakeId: candidate.id,
+ reason: decision.selfAuthored
+ ? "Deferred wake contained only comments authored by the finishing run"
+ : "Queued messages were discarded before promotion",
+ now: input.now,
+ });
+ continue;
+ }
+
+ let workingCandidate = candidate;
+ if (decision.kind === "normalize") {
+ const normalized = await ports.writer.normalizeDeferredWakeCommentIds({
+ companyId: run.companyId,
+ wakeId: candidate.id,
+ payload: candidate.payload,
+ liveCommentIds: liveness.liveNonSelfCommentIds,
+ now: input.now,
+ });
+ if (!normalized) continue;
+ workingCandidate = normalized;
+ // Re-decide with the same agent/pause-hold facts already fetched above; the
+ // comment-id set now matches, so only fail/cancel-pause-hold/promote can result.
+ decision = decideDeferredWake({
+ queuedComment: {
+ hasQueuedCommentIds: workingCandidate.queuedCommentIds.length > 0,
+ liveNonSelfCommentIdsLength: liveness.liveNonSelfCommentIds.length,
+ queuedCommentIdsLength: liveness.liveNonSelfCommentIds.length,
+ liveCommentIdsChanged: false,
+ containedSelfAuthoredComment: liveness.containedSelfAuthoredComment,
+ preservesIndependentContinuation: workingCandidate.preservesIndependentContinuation,
+ },
+ agent: { agentFound: deferredAgent !== null, invokable: deferredAgent?.invokable ?? false },
+ pauseHold: { activePauseHold: pauseHold.activePauseHold, treeHoldInteractionWake: pauseHold.treeHoldInteractionWake },
+ });
+ }
+
+ if (decision.kind === "fail_not_invokable") {
+ await ports.writer.failDeferredWake({ companyId: run.companyId, wakeId: workingCandidate.id, now: input.now });
+ continue;
+ }
+
+ if (decision.kind === "cancel_pause_hold") {
+ await ports.writer.cancelDeferredWake({
+ companyId: run.companyId,
+ wakeId: workingCandidate.id,
+ reason: "Deferred wake suppressed by active subtree pause hold",
+ now: input.now,
+ });
+ continue;
+ }
+
+ // decision.kind === "promote"
+ const invokableAgent = deferredAgent!;
+
+ // Claim the wake for promotion before any other write in this branch
+ // (design choice: claim first, then reopen). A reopen write, or its
+ // `issue_reopened` post-commit effect, must never survive a lost race on
+ // this compare-and-set. When the claim fails, a concurrent writer already
+ // changed the wake's status, so this candidate is gone; move on to the
+ // next one instead of ending the drain.
+ const claimedForPromotion = await ports.writer.claimDeferredWakeForPromotion({
+ companyId: run.companyId,
+ wakeId: workingCandidate.id,
+ now: input.now,
+ });
+ if (!claimedForPromotion) continue;
+
+ let currentIssue = issue;
+
+ if (workingCandidate.deferredCommentIds.length > 0 && (currentIssue.status === "done" || currentIssue.status === "cancelled")) {
+ const selfAuthorship = await ports.writer.getCommentSelfAuthorship({
+ companyId: run.companyId,
+ issueId: currentIssue.id,
+ finishingRunId: run.id,
+ commentIds: workingCandidate.deferredCommentIds,
+ });
+ const shouldReopen =
+ !selfAuthorship.allSelfAuthored &&
+ (workingCandidate.requestedByActorType === "user" || workingCandidate.wakeReason === "issue_reopened_via_comment");
+ if (shouldReopen) {
+ const reopened = await ports.writer.reopenIssue({ companyId: run.companyId, issueId: currentIssue.id, runId: run.id });
+ if (reopened) {
+ postCommitEffects.push({
+ kind: "issue_reopened",
+ companyId: reopened.companyId,
+ agentId: invokableAgent.id,
+ runId: run.id,
+ issueId: reopened.id,
+ identifier: reopened.identifier,
+ reopenedFrom: currentIssue.status,
+ });
+ currentIssue = reopened;
+ issue = reopened;
+ }
+ }
+ }
+
+ const promotedReason = workingCandidate.reason ?? "issue_execution_promoted";
+ const promotedSource = workingCandidate.source ?? "automation";
+ const promotedTriggerDetail = workingCandidate.triggerDetail ?? null;
+ const promotedPayload = { ...workingCandidate.payload };
+ delete promotedPayload["_paperclipWakeContext"];
+
+ const promotedContextSeed: Record<string, unknown> = { ...workingCandidate.deferredContextSeed };
+ if (pauseHold.activePauseHold) {
+ promotedContextSeed.treeHoldInteraction = true;
+ promotedContextSeed.activeTreeHold = {
+ holdId: pauseHold.holdId,
+ rootIssueId: pauseHold.rootIssueId,
+ mode: pauseHold.mode,
+ reason: pauseHold.reason,
+ releasePolicy: pauseHold.releasePolicy,
+ interaction: true,
+ };
+ }
+
+ const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey } = enrichPromotedWakeContext({
+ contextSnapshot: promotedContextSeed,
+ reason: promotedReason,
+ source: promotedSource,
+ triggerDetail: promotedTriggerDetail,
+ payload: promotedPayload,
+ });
+
+ const sessionBefore =
+ readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
+ (await ports.reader.resolveSessionBeforeForWakeup({
+ companyId: run.companyId,
+ agentId: invokableAgent.id,
+ taskKey: promotedTaskKey,
+ }));
+
+ const promotedRoutineEnvContext = await ports.reader.getRoutineEnv({
+ companyId: invokableAgent.companyId,
+ issue: currentIssue,
+ });
+ const responsibleUserId = await ports.reader.resolveResponsibleUserId({
+ companyId: invokableAgent.companyId,
+ contextSnapshot: promotedContextSnapshot,
+ issue: currentIssue,
+ routineEnvContext: promotedRoutineEnvContext,
+ requestedByActorType: workingCandidate.requestedByActorType as "user" | "agent" | "system" | null,
+ requestedByActorId: workingCandidate.requestedByActorId,
+ source: promotedSource,
+ triggerDetail: promotedTriggerDetail,
+ existingRunResponsibleUserId: run.responsibleUserId,
+ });
+ if (!responsibleUserId) {
+ throw new WakeQueueApplicationError(
+ "responsible_user_unresolved",
+ "Unable to resolve responsible user for promoted heartbeat run",
+ {
+ runId: run.id,
+ agentId: invokableAgent.id,
+ companyId: invokableAgent.companyId,
+ issueId: currentIssue.id,
+ wakeReason: readNonEmptyString(promotedContextSnapshot.wakeReason),
+ },
+ );
+ }
+
+ const promotedRun = await ports.writer.finalizePromotedWake({
+ companyId: run.companyId,
+ wakeId: workingCandidate.id,
+ deferredAgent: invokableAgent,
+ issue: currentIssue,
+ finishingRun: run,
+ contextSnapshot: promotedContextSnapshot,
+ reason: promotedReason,
+ source: promotedSource,
+ triggerDetail: promotedTriggerDetail,
+ payload: promotedPayload,
+ responsibleUserId,
+ sessionBefore,
+ now: input.now,
+ });
+
+ postCommitEffects.push({ kind: "run_queued", run: promotedRun });
+ return { outcome: { kind: "promoted", run: promotedRun }, postCommitEffects };
+ }
+
+ return runReleaseRecoveryTail(issue, run, ports.reader, ports.writer, input, postCommitEffects);
+}
+
+async function runReleaseRecoveryTail(
+ issue: IssueSnapshot,
+ run: RunSnapshot,
+ reader: WakeQueueReader,
+ writer: WakeQueueWriter,
+ input: ReleaseIssueExecutionInput,
+ postCommitEffects: PostCommitEffect[],
+): Promise<ReleaseTransactionResult> {
+ const suppressImmediateRecovery = input.suppressImmediateRecovery ?? false;
+ const isStrandedRecoveryOrigin = issue.originKind === STRANDED_ISSUE_RECOVERY_ORIGIN_KIND;
+ const recoveryAgent = await reader.findInvokableAgent({ companyId: issue.companyId, agentId: run.agentId });
+
+ const currentParticipant = currentAgentParticipant(issue);
+ const reviewParticipantApplies =
+ issue.status === "in_review" &&
+ !issue.assigneeUserId &&
+ currentParticipant !== null &&
+ currentParticipant.agentId === run.agentId &&
+ isExecutionReviewParticipantRecoveryEligibleRun(run) &&
+ HEARTBEAT_RUN_TERMINAL_STATUSES.has(run.status);
+
+ const immediateApplies =
+ (issue.status === "todo" || issue.status === "in_progress") &&
+ !issue.assigneeUserId &&
+ !issue.hiddenAt &&
+ issue.assigneeAgentId === run.agentId &&
+ (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled");
+
+ const suppressedByPauseHold = (reviewParticipantApplies || immediateApplies)
+ ? await writer.isAutomaticRecoverySuppressedByPauseHold({ companyId: issue.companyId, issueId: issue.id })
+ : false;
+
+ const hasExistingExecutionPath = reviewParticipantApplies
+ ? await writer.hasExistingExecutionPath({
+ companyId: issue.companyId,
+ issueId: issue.id,
+ excludeRunId: run.id,
+ agentId: currentParticipant?.agentId ?? null,
+ })
+ : immediateApplies
+ ? await writer.hasExistingExecutionPath({ companyId: issue.companyId, issueId: issue.id, excludeRunId: run.id, agentId: null })
+ : false;
+
+ const hasExplicitBlockerPath = immediateApplies && !reviewParticipantApplies
+ ? await writer.hasExplicitBlockerPath({ companyId: issue.companyId, issueId: issue.id })
+ : false;
+
+ const expectedRetryReason: "assignment_recovery" | "issue_continuation_needed" =
+ issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed";
+
+ const decision = decideReleaseRecovery({
+ suppressImmediateRecovery,
+ reviewParticipant: {
+ applies: reviewParticipantApplies,
+ hasExistingExecutionPath,
+ hasPersistedMonitor: Boolean(issue.monitorNextCheckAt),
+ suppressedByPauseHold,
+ isStrandedRecoveryOrigin,
+ recoveryAgentPresent: recoveryAgent !== null,
+ recoveryAgentInvokable: recoveryAgent?.invokable ?? false,
+ isExecutionReviewParticipantRecoveryRun: isExecutionReviewParticipantRecoveryRun(run),
+ },
+ immediate: {
+ applies: immediateApplies,
+ isDispositionRepairRetry: readNonEmptyString(run.contextSnapshot.retryReason) === ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
+ hasExistingExecutionPath,
+ hasPersistedMonitor: Boolean(issue.monitorNextCheckAt),
+ hasExplicitBlockerPath,
+ suppressedByPauseHold,
+ isStrandedRecoveryOrigin,
+ recoveryAgentPresent: recoveryAgent !== null,
+ recoveryAgentInvokable: recoveryAgent?.invokable ?? false,
+ isWorkspaceValidationFailedRun: isWorkspaceValidationFailedRun(run),
+ isConfigurationIncompleteFailedRun: isConfigurationIncompleteFailedRun(run),
+ automaticRecoveryAlreadyFailed: didAutomaticRecoveryFail(run, expectedRetryReason),
+ },
+ });
+
+ if (decision.kind === "released") {
+ return { outcome: { kind: "released" }, postCommitEffects };
+ }
+
+ if (decision.kind === "blocked_recovery_in_place") {
+ return {
+ outcome: { kind: "blocked_recovery_in_place", issue, previousStatus: statusForBlock(issue) },
+ postCommitEffects,
+ };
+ }
+
+ if (decision.kind === "blocked") {
+ const { notice, recoveryCause } = await writer.buildBlockedRecoveryNotice({
+ noticeKind: decision.notice,
+ issueStatus: issue.status === "todo" ? "todo" : "in_progress",
+ finishingRun: run,
+ });
+ return {
+ outcome: {
+ kind: "blocked",
+ issue,
+ previousStatus: statusForBlock(issue),
+ notice,
+ recoveryCause,
+ },
+ postCommitEffects,
+ };
+ }
+
+ const sessionBefore = await reader.resolveSessionBeforeForWakeup({
+ companyId: issue.companyId,
+ agentId: recoveryAgent!.id,
+ taskKey: readNonEmptyString(run.contextSnapshot.taskKey) ?? readNonEmptyString(run.contextSnapshot.issueId),
+ });
+
+ if (decision.kind === "queue_review_participant_recovery") {
+ const queuedRun = await writer.queueReviewParticipantRecoveryRun({
+ companyId: issue.companyId,
+ issue,
+ finishingRun: run,
+ recoveryAgent: recoveryAgent!,
+ sessionBefore,
+ now: input.now,
+ });
+ postCommitEffects.push({ kind: "run_queued", run: queuedRun });
+ return { outcome: { kind: "queued_review_participant_recovery", run: queuedRun }, postCommitEffects };
+ }
+
+ // decision.kind === "queue_recovery"; the adapter builds the recovery
+ // context snapshot and resolves the responsible user from it, throwing
+ // WakeQueueApplicationError when no responsible user resolves.
+ const queuedRun = await writer.queueImmediateRecoveryRun({
+ companyId: issue.companyId,
+ issue,
+ finishingRun: run,
+ recoveryAgent: recoveryAgent!,
+ sessionBefore,
+ now: input.now,
+ });
+ postCommitEffects.push({ kind: "run_queued", run: queuedRun });
+ return { outcome: { kind: "queued_recovery", run: queuedRun }, postCommitEffects };
+}
+
+function statusForBlock(issue: IssueSnapshot): "todo" | "in_progress" | "in_review" {
+ return issue.status === "todo" || issue.status === "in_review" ? issue.status : "in_progress";
+}
+
+export function createReleaseIssueExecution(deps: {
+ issueLock: IssueLockWriter;
+ recovery: RecoveryEscalationPort;
+}) {
+ return async function releaseIssueExecution(
+ input: ReleaseIssueExecutionInput,
+ ): Promise<{ outcome: ReleaseOutcome; postCommitEffects: PostCommitEffect[] }> {
+ const result = await deps.issueLock.withIssueExecutionLock(
+ { companyId: input.companyId, runId: input.runId, now: input.now },
+ (locked, ports) => runReleaseDrain(locked, ports, input),
+ );
+
+ if (result.outcome.kind === "blocked") {
+ await deps.recovery.escalateStrandedAssignedIssue({
+ issue: result.outcome.issue,
+ previousStatus: result.outcome.previousStatus,
+ latestRun: result.run,
+ notice: result.outcome.notice,
+ recoveryCause: result.outcome.recoveryCause,
+ });
+ } else if (result.outcome.kind === "blocked_recovery_in_place") {
+ await deps.recovery.escalateStrandedRecoveryIssueInPlace({
+ issue: result.outcome.issue,
+ previousStatus: result.outcome.previousStatus,
+ latestRun: result.run,
+ });
+ }
+
+ return { outcome: result.outcome, postCommitEffects: result.postCommitEffects };
+ };
+}
diff --git a/server/src/modules/wake-queue/domain/context.test.ts b/server/src/modules/wake-queue/domain/context.test.ts
new file mode 100644
index 0000000000..d4fe457f71
--- /dev/null
+++ b/server/src/modules/wake-queue/domain/context.test.ts
@@ -0,0 +1,124 @@
+import { describe, expect, it } from "vitest";
+import { enrichPromotedWakeContext } from "./context.js";
+
+describe("enrichPromotedWakeContext", () => {
+ it("clears stale derived comment projections when the canonical id list is empty, but keeps an independent interaction continuation", () => {
+ const contextSnapshot: Record<string, unknown> = {
+ // Normalization already dropped the canonical wake comment ids upstream
+ // of this function; no wakeCommentIds field remains on the snapshot.
+ paperclipWake: { commentId: "comment-2" },
+ paperclipWakeComment: { id: "comment-2", body: "stale body" },
+ paperclipTaskMarkdown: "stale markdown",
+ paperclipTaskMarkdownCompact: "stale compact markdown",
+ // Independent interaction continuation the promoted run must keep.
+ interactionId: "interaction-1",
+ interactionKind: "request_confirmation",
+ interactionStatus: "accepted",
+ };
+
+ const result = enrichPromotedWakeContext({
+ contextSnapshot,
+ reason: "issue_execution_promoted",
+ source: "automation",
+ triggerDetail: null,
+ payload: { mutation: "interaction" },
+ });
+
+ expect(result.contextSnapshot.wakeCommentIds).toBeUndefined();
+ expect(result.contextSnapshot.commentId).toBeUndefined();
+ expect(result.contextSnapshot.wakeCommentId).toBeUndefined();
+ expect(result.contextSnapshot.paperclipWake).toBeUndefined();
+ expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined();
+ expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined();
+ expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
+ // The independent interaction continuation survives the clear.
+ expect(result.contextSnapshot.interactionId).toBe("interaction-1");
+ expect(result.contextSnapshot.interactionStatus).toBe("accepted");
+ });
+
+ it("sets the canonical id and latest-id fields when the recomputed list has entries, and still clears the stale projections", () => {
+ const contextSnapshot: Record<string, unknown> = {
+ wakeCommentIds: ["comment-1"],
+ paperclipWake: { commentId: "comment-1" },
+ paperclipWakeComment: { id: "comment-1", body: "will be rebuilt" },
+ paperclipTaskMarkdown: "will be rebuilt",
+ paperclipTaskMarkdownCompact: "will be rebuilt",
+ };
+
+ const result = enrichPromotedWakeContext({
+ contextSnapshot,
+ reason: "issue_execution_promoted",
+ source: "automation",
+ triggerDetail: null,
+ payload: {},
+ });
+
+ expect(result.contextSnapshot.wakeCommentIds).toEqual(["comment-1"]);
+ expect(result.contextSnapshot.commentId).toBe("comment-1");
+ expect(result.contextSnapshot.wakeCommentId).toBe("comment-1");
+ // The derived projections are cleared so the run rebuilds them from the
+ // canonical ids instead of carrying forward a queue-time snapshot.
+ expect(result.contextSnapshot.paperclipWake).toBeUndefined();
+ expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined();
+ expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined();
+ expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
+ });
+
+ it("adds a new id from the payload to the canonical list and clears the stale projections", () => {
+ const contextSnapshot: Record<string, unknown> = {
+ wakeCommentIds: ["comment-1"],
+ paperclipWakeComment: { id: "comment-1", body: "stale body" },
+ };
+
+ const result = enrichPromotedWakeContext({
+ contextSnapshot,
+ reason: "issue_execution_promoted",
+ source: "automation",
+ triggerDetail: null,
+ payload: { commentId: "comment-2" },
+ });
+
+ expect(result.contextSnapshot.wakeCommentIds).toEqual(["comment-1", "comment-2"]);
+ expect(result.contextSnapshot.commentId).toBe("comment-2");
+ expect(result.contextSnapshot.wakeCommentId).toBe("comment-2");
+ expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined();
+ });
+
+ it("keeps the raw issue, execution-stage, and accepted-plan fields a dispatch rebuild needs, alongside the cleared text projections", () => {
+ const contextSnapshot: Record<string, unknown> = {
+ wakeCommentIds: ["comment-1"],
+ paperclipWake: { commentId: "comment-1" },
+ paperclipTaskMarkdown: "stale markdown",
+ paperclipTaskMarkdownCompact: "stale compact markdown",
+ // Raw context a dispatch-time rebuild reads directly off the promoted
+ // run; none of it is derived from the comment ids above, so none of it
+ // should be cleared alongside the stale text projections.
+ issueId: "issue-1",
+ executionStage: { stage: "review" },
+ planReviewInteraction: { acceptedTargetRevision: { revisionId: "revision-1" } },
+ acceptedPlanWakeRouting: { targetAgentId: "agent-1" },
+ workspaceRefreshReason: "accepted_plan_confirmation",
+ };
+
+ const result = enrichPromotedWakeContext({
+ contextSnapshot,
+ reason: "issue_execution_promoted",
+ source: "automation",
+ triggerDetail: null,
+ payload: {},
+ });
+
+ // The rendered text is cleared; dispatch rebuilds it from the raw fields.
+ expect(result.contextSnapshot.paperclipWake).toBeUndefined();
+ expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined();
+ expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
+ // The raw fields the rebuild needs are untouched.
+ expect(result.contextSnapshot.issueId).toBe("issue-1");
+ expect(result.contextSnapshot.executionStage).toEqual({ stage: "review" });
+ expect(result.contextSnapshot.planReviewInteraction).toEqual({
+ acceptedTargetRevision: { revisionId: "revision-1" },
+ });
+ expect(result.contextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" });
+ expect(result.contextSnapshot.workspaceRefreshReason).toBe("accepted_plan_confirmation");
+ });
+});
diff --git a/server/src/modules/wake-queue/domain/context.ts b/server/src/modules/wake-queue/domain/context.ts
new file mode 100644
index 0000000000..d9f31ef6e3
--- /dev/null
+++ b/server/src/modules/wake-queue/domain/context.ts
@@ -0,0 +1,165 @@
+// Pure helpers that build the context snapshot for a promoted deferred wake.
+// `extractWakeCommentIds`, `deriveCommentId`, and `WAKE_COMMENT_IDS_KEY` come
+// from the run-dispatch module so the two modules read one wake context
+// shape; this file holds only the small pieces that are specific to
+// building the release half's promoted-run snapshot.
+
+import { extractWakeCommentIds, WAKE_COMMENT_IDS_KEY } from "../../run-dispatch/index.js";
+
+const PAPERCLIP_WAKE_PAYLOAD_KEY = "paperclipWake";
+const PAPERCLIP_WAKE_COMMENT_KEY = "paperclipWakeComment";
+const PAPERCLIP_TASK_MARKDOWN_KEY = "paperclipTaskMarkdown";
+const PAPERCLIP_TASK_MARKDOWN_COMPACT_KEY = "paperclipTaskMarkdownCompact";
+
+const INTERACTION_CONTINUATION_CONTEXT_KEYS = [
+ "interactionId",
+ "interactionKind",
+ "interactionStatus",
+ "continuationPolicy",
+ "checkboxSelection",
+ "itemVerdicts",
+ "newlyResolvedItemIds",
+] as const;
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+function deriveTaskKey(
+ contextSnapshot: Record<string, unknown> | null | undefined,
+ payload: Record<string, unknown> | null | undefined,
+): string | null {
+ return (
+ readNonEmptyString(contextSnapshot?.taskKey) ??
+ readNonEmptyString(contextSnapshot?.taskId) ??
+ readNonEmptyString(contextSnapshot?.issueId) ??
+ readNonEmptyString(payload?.taskKey) ??
+ readNonEmptyString(payload?.taskId) ??
+ readNonEmptyString(payload?.issueId) ??
+ null
+ );
+}
+
+function mergeWakeCommentIds(...values: Array<unknown>): string[] {
+ const merged: string[] = [];
+ const append = (value: unknown) => {
+ const normalized = readNonEmptyString(value);
+ if (!normalized || merged.includes(normalized)) return;
+ merged.push(normalized);
+ };
+ for (const value of values) {
+ if (Array.isArray(value)) {
+ for (const entry of value) append(entry);
+ continue;
+ }
+ if (typeof value === "object" && value !== null) {
+ const candidate = value as Record<string, unknown>;
+ const batched = extractWakeCommentIds(candidate);
+ if (batched.length > 0) {
+ for (const entry of batched) append(entry);
+ continue;
+ }
+ append(candidate.wakeCommentId);
+ append(candidate.commentId);
+ continue;
+ }
+ append(value);
+ }
+ return merged;
+}
+
+export function hasInteractionContinuationWakeContext(contextSnapshot: Record<string, unknown>): boolean {
+ return INTERACTION_CONTINUATION_CONTEXT_KEYS.some((key) => readNonEmptyString(contextSnapshot[key]));
+}
+
+function isInteractionResolutionWakePayload(payload: Record<string, unknown> | null | undefined): boolean {
+ return readNonEmptyString(payload?.mutation) === "interaction";
+}
+
+function normalizeInteractionContinuationWakeContext(
+ contextSnapshot: Record<string, unknown>,
+ payload: Record<string, unknown> | null | undefined,
+): void {
+ if (isInteractionResolutionWakePayload(payload)) return;
+ for (const key of INTERACTION_CONTINUATION_CONTEXT_KEYS) {
+ delete contextSnapshot[key];
+ }
+}
+
+export type EnrichPromotedWakeContextInput = {
+ contextSnapshot: Record<string, unknown>;
+ reason: string | null;
+ source: string | null;
+ triggerDetail: string | null;
+ payload: Record<string, unknown> | null;
+};
+
+export type EnrichPromotedWakeContextResult = {
+ contextSnapshot: Record<string, unknown>;
+ taskKey: string | null;
+};
+
+/**
+ * Fills the promoted run's context snapshot with the fields the original
+ * wake enrichment always fills, without overwriting a field the deferred
+ * wake already carried. The deferred wake's own context snapshot was
+ * already enriched once when it was first queued, so most calls only
+ * normalize the interaction-continuation keys.
+ */
+export function enrichPromotedWakeContext(
+ input: EnrichPromotedWakeContextInput,
+): EnrichPromotedWakeContextResult {
+ const contextSnapshot = { ...input.contextSnapshot };
+ const { reason, source, triggerDetail, payload } = input;
+ const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]);
+ const taskKey = deriveTaskKey(contextSnapshot, payload);
+ const wakeCommentIds = mergeWakeCommentIds(contextSnapshot, commentIdFromPayload);
+
+ if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) {
+ contextSnapshot.wakeReason = reason;
+ }
+ if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) {
+ contextSnapshot.taskKey = taskKey;
+ }
+ if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) {
+ contextSnapshot.commentId = commentIdFromPayload;
+ }
+ // The wake payload, resolved comment, and task-markdown snapshots below are
+ // rendered text built from the canonical comment ids and the issue state at
+ // queue time. This function recomputes the canonical ids on every call
+ // (`wakeCommentIds`), so it must not let a rendered snapshot from a stale
+ // id list carry forward: clear all four, then restore only the canonical
+ // id and latest-id fields, and only when the recomputed list actually has
+ // entries.
+ //
+ // This function does not render a replacement in its place. The
+ // promoted run still carries every raw field the render depends on
+ // (`issueId`, `wakeReason`, interaction and accepted-plan fields, and the
+ // canonical comment ids set below): `heartbeatService`'s `executeRun`
+ // rebuilds the rendered text from those raw fields and the current issue
+ // and comment rows, with the trust-based redaction that rendering needs,
+ // before it persists the run and dispatches it to the agent. Rendering a
+ // second, simplified copy here would either skip that redaction (an unsafe
+ // shortcut for quarantined comment content) or duplicate it outside the
+ // one place it is proven correct, so this function leaves the render to
+ // dispatch and only guarantees the raw inputs survive.
+ delete contextSnapshot[PAPERCLIP_WAKE_PAYLOAD_KEY];
+ delete contextSnapshot[PAPERCLIP_WAKE_COMMENT_KEY];
+ delete contextSnapshot[PAPERCLIP_TASK_MARKDOWN_KEY];
+ delete contextSnapshot[PAPERCLIP_TASK_MARKDOWN_COMPACT_KEY];
+ if (wakeCommentIds.length > 0) {
+ const latestCommentId = wakeCommentIds[wakeCommentIds.length - 1];
+ contextSnapshot[WAKE_COMMENT_IDS_KEY] = wakeCommentIds;
+ contextSnapshot.commentId = latestCommentId;
+ contextSnapshot.wakeCommentId = latestCommentId;
+ }
+ if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) {
+ contextSnapshot.wakeSource = source;
+ }
+ if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
+ contextSnapshot.wakeTriggerDetail = triggerDetail;
+ }
+ normalizeInteractionContinuationWakeContext(contextSnapshot, payload);
+
+ return { contextSnapshot, taskKey };
+}
diff --git a/server/src/modules/wake-queue/domain/policy.test.ts b/server/src/modules/wake-queue/domain/policy.test.ts
new file mode 100644
index 0000000000..61eb44d0e6
--- /dev/null
+++ b/server/src/modules/wake-queue/domain/policy.test.ts
@@ -0,0 +1,353 @@
+import { describe, expect, it } from "vitest";
+import {
+ decideDeferredWake,
+ decideReleaseRecovery,
+ type DeferredWakeFacts,
+ type ReleaseRecoveryFacts,
+} from "./policy.js";
+
+const baseDeferredWakeFacts: DeferredWakeFacts = {
+ queuedComment: {
+ hasQueuedCommentIds: false,
+ liveNonSelfCommentIdsLength: 0,
+ queuedCommentIdsLength: 0,
+ liveCommentIdsChanged: false,
+ containedSelfAuthoredComment: false,
+ preservesIndependentContinuation: false,
+ },
+ agent: { agentFound: true, invokable: true },
+ pauseHold: { activePauseHold: false, treeHoldInteractionWake: false },
+};
+
+describe("decideDeferredWake", () => {
+ const cases: Array<{
+ name: string;
+ facts: DeferredWakeFacts;
+ expected: ReturnType<typeof decideDeferredWake>;
+ }> = [
+ {
+ name: "cancel_empty: all queued comments discarded and no independent continuation",
+ facts: {
+ ...baseDeferredWakeFacts,
+ queuedComment: {
+ hasQueuedCommentIds: true,
+ liveNonSelfCommentIdsLength: 0,
+ queuedCommentIdsLength: 2,
+ liveCommentIdsChanged: true,
+ containedSelfAuthoredComment: false,
+ preservesIndependentContinuation: false,
+ },
+ },
+ expected: { kind: "cancel_empty", selfAuthored: false },
+ },
+ {
+ name: "cancel_empty: self-authored comments discarded, error text reflects self-authorship",
+ facts: {
+ ...baseDeferredWakeFacts,
+ queuedComment: {
+ hasQueuedCommentIds: true,
+ liveNonSelfCommentIdsLength: 0,
+ queuedCommentIdsLength: 1,
+ liveCommentIdsChanged: true,
+ containedSelfAuthoredComment: true,
+ preservesIndependentContinuation: false,
+ },
+ },
+ expected: { kind: "cancel_empty", selfAuthored: true },
+ },
+ {
+ name: "normalize: no live comments, but an independent continuation reason still rewrites the queued id list",
+ facts: {
+ ...baseDeferredWakeFacts,
+ queuedComment: {
+ hasQueuedCommentIds: true,
+ liveNonSelfCommentIdsLength: 0,
+ queuedCommentIdsLength: 1,
+ liveCommentIdsChanged: true,
+ containedSelfAuthoredComment: false,
+ preservesIndependentContinuation: true,
+ },
+ },
+ expected: { kind: "normalize" },
+ },
+ {
+ name: "promote: an independent continuation reason keeps the wake alive with the live id set already matching",
+ facts: {
+ ...baseDeferredWakeFacts,
+ queuedComment: {
+ hasQueuedCommentIds: true,
+ liveNonSelfCommentIdsLength: 0,
+ queuedCommentIdsLength: 0,
+ liveCommentIdsChanged: false,
+ containedSelfAuthoredComment: false,
+ preservesIndependentContinuation: true,
+ },
+ },
+ expected: { kind: "promote" },
+ },
+ {
+ name: "normalize: the live comment id set differs from the queued set",
+ facts: {
+ ...baseDeferredWakeFacts,
+ queuedComment: {
+ hasQueuedCommentIds: true,
+ liveNonSelfCommentIdsLength: 1,
+ queuedCommentIdsLength: 2,
+ liveCommentIdsChanged: true,
+ containedSelfAuthoredComment: false,
+ preservesIndependentContinuation: false,
+ },
+ },
+ expected: { kind: "normalize" },
+ },
+ {
+ name: "fail_not_invokable: the agent lookup returns not-found",
+ facts: {
+ ...baseDeferredWakeFacts,
+ agent: { agentFound: false, invokable: false },
+ },
+ expected: { kind: "fail_not_invokable" },
+ },
+ {
+ name: "fail_not_invokable: the agent is found but not invokable",
+ facts: {
+ ...baseDeferredWakeFacts,
+ agent: { agentFound: true, invokable: false },
+ },
+ expected: { kind: "fail_not_invokable" },
+ },
+ {
+ name: "cancel_pause_hold: an active pause hold with no verified tree-hold interaction",
+ facts: {
+ ...baseDeferredWakeFacts,
+ pauseHold: { activePauseHold: true, treeHoldInteractionWake: false },
+ },
+ expected: { kind: "cancel_pause_hold" },
+ },
+ {
+ name: "promote: an active pause hold but a verified tree-hold interaction wake survives it",
+ facts: {
+ ...baseDeferredWakeFacts,
+ pauseHold: { activePauseHold: true, treeHoldInteractionWake: true },
+ },
+ expected: { kind: "promote" },
+ },
+ {
+ name: "promote: no queued comments, an invokable agent, and no pause hold",
+ facts: baseDeferredWakeFacts,
+ expected: { kind: "promote" },
+ },
+ ];
+
+ for (const testCase of cases) {
+ it(testCase.name, () => {
+ expect(decideDeferredWake(testCase.facts)).toEqual(testCase.expected);
+ });
+ }
+});
+
+const baseReleaseRecoveryFacts: ReleaseRecoveryFacts = {
+ suppressImmediateRecovery: false,
+ reviewParticipant: {
+ applies: false,
+ hasExistingExecutionPath: false,
+ hasPersistedMonitor: false,
+ suppressedByPauseHold: false,
+ isStrandedRecoveryOrigin: false,
+ recoveryAgentPresent: true,
+ recoveryAgentInvokable: true,
+ isExecutionReviewParticipantRecoveryRun: false,
+ },
+ immediate: {
+ applies: false,
+ isDispositionRepairRetry: false,
+ hasExistingExecutionPath: false,
+ hasPersistedMonitor: false,
+ hasExplicitBlockerPath: false,
+ suppressedByPauseHold: false,
+ isStrandedRecoveryOrigin: false,
+ recoveryAgentPresent: true,
+ recoveryAgentInvokable: true,
+ isWorkspaceValidationFailedRun: false,
+ isConfigurationIncompleteFailedRun: false,
+ automaticRecoveryAlreadyFailed: false,
+ },
+};
+
+describe("decideReleaseRecovery", () => {
+ const cases: Array<{
+ name: string;
+ facts: ReleaseRecoveryFacts;
+ expected: ReturnType<typeof decideReleaseRecovery>;
+ }> = [
+ {
+ name: "released: neither the review-participant nor the immediate-recovery branch applies",
+ facts: baseReleaseRecoveryFacts,
+ expected: { kind: "released" },
+ },
+ {
+ name: "released: immediate recovery applies but the caller asked to suppress it",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ suppressImmediateRecovery: true,
+ immediate: { ...baseReleaseRecoveryFacts.immediate, applies: true },
+ },
+ expected: { kind: "released" },
+ },
+ {
+ name: "released: immediate recovery applies but an existing execution path already covers it",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ hasExistingExecutionPath: true,
+ },
+ },
+ expected: { kind: "released" },
+ },
+ {
+ name: "released: immediate recovery applies but the finishing run carried the disposition-repair retry reason",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ isDispositionRepairRetry: true,
+ },
+ },
+ expected: { kind: "released" },
+ },
+ {
+ name: "released: immediate recovery applies but an explicit blocker path exists",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ hasExplicitBlockerPath: true,
+ },
+ },
+ expected: { kind: "released" },
+ },
+ {
+ name: "blocked_recovery_in_place: immediate recovery applies on a stranded-issue-recovery origin",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ isStrandedRecoveryOrigin: true,
+ },
+ },
+ expected: { kind: "blocked_recovery_in_place" },
+ },
+ {
+ name: "blocked: immediate recovery applies but the recovery agent is not invokable",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ recoveryAgentInvokable: false,
+ },
+ },
+ expected: { kind: "blocked", notice: "immediate_execution_path" },
+ },
+ {
+ name: "blocked: immediate recovery applies and the run failed workspace validation",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ isWorkspaceValidationFailedRun: true,
+ },
+ },
+ expected: { kind: "blocked", notice: "workspace_validation" },
+ },
+ {
+ name: "blocked: immediate recovery applies and the run failed on incomplete configuration",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ isConfigurationIncompleteFailedRun: true,
+ },
+ },
+ expected: { kind: "blocked", notice: "configuration_incomplete" },
+ },
+ {
+ name: "queue_recovery: immediate recovery applies and no suppression or block condition fires",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ immediate: { ...baseReleaseRecoveryFacts.immediate, applies: true },
+ },
+ expected: { kind: "queue_recovery" },
+ },
+ {
+ name: "released: review-participant recovery applies but a persisted monitor already covers it",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ reviewParticipant: {
+ ...baseReleaseRecoveryFacts.reviewParticipant,
+ applies: true,
+ hasPersistedMonitor: true,
+ },
+ },
+ expected: { kind: "released" },
+ },
+ {
+ name: "blocked_recovery_in_place: review-participant recovery applies on a stranded-issue-recovery origin",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ reviewParticipant: {
+ ...baseReleaseRecoveryFacts.reviewParticipant,
+ applies: true,
+ isStrandedRecoveryOrigin: true,
+ },
+ },
+ expected: { kind: "blocked_recovery_in_place" },
+ },
+ {
+ name: "blocked: review-participant recovery applies but the finishing run was itself that recovery retry",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ reviewParticipant: {
+ ...baseReleaseRecoveryFacts.reviewParticipant,
+ applies: true,
+ isExecutionReviewParticipantRecoveryRun: true,
+ },
+ },
+ expected: { kind: "blocked", notice: "execution_review_participant" },
+ },
+ {
+ name: "queue_review_participant_recovery: review-participant recovery applies and no suppression or block condition fires",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ reviewParticipant: { ...baseReleaseRecoveryFacts.reviewParticipant, applies: true },
+ },
+ expected: { kind: "queue_review_participant_recovery" },
+ },
+ {
+ name: "review-participant recovery is evaluated before immediate recovery when both apply",
+ facts: {
+ ...baseReleaseRecoveryFacts,
+ reviewParticipant: { ...baseReleaseRecoveryFacts.reviewParticipant, applies: true },
+ immediate: {
+ ...baseReleaseRecoveryFacts.immediate,
+ applies: true,
+ isStrandedRecoveryOrigin: true,
+ },
+ },
+ expected: { kind: "queue_review_participant_recovery" },
+ },
+ ];
+
+ for (const testCase of cases) {
+ it(testCase.name, () => {
+ expect(decideReleaseRecovery(testCase.facts)).toEqual(testCase.expected);
+ });
+ }
+});
diff --git a/server/src/modules/wake-queue/domain/policy.ts b/server/src/modules/wake-queue/domain/policy.ts
new file mode 100644
index 0000000000..7fd959a2d8
--- /dev/null
+++ b/server/src/modules/wake-queue/domain/policy.ts
@@ -0,0 +1,200 @@
+// Pure decision rules for the release half of the deferred issue-execution
+// wake state machine:
+// - the per-wake decision (decideDeferredWake), applied to the earliest
+// deferred wake queued against the issue a run just released
+// - the release-recovery decision (decideReleaseRecovery), applied once
+// the deferred-wake queue is empty and no wake was promoted
+// The caller reads the database and packs the result into a facts object.
+// This file only branches on that facts object; it never queries a
+// database, reads the clock, or reads the wake context payload directly.
+
+export type DeferredWakeQueuedCommentFacts = {
+ /** True when the wake carries one or more queued comment ids to check. */
+ hasQueuedCommentIds: boolean;
+ /** Count of queued comment ids that are still live and not self-authored by the finishing run. */
+ liveNonSelfCommentIdsLength: number;
+ /** Count of queued comment ids the wake originally carried. */
+ queuedCommentIdsLength: number;
+ /** True when the live, non-self comment id list differs from the queued list. */
+ liveCommentIdsChanged: boolean;
+ /** True when every discarded comment id was authored by the finishing run. */
+ containedSelfAuthoredComment: boolean;
+ /** True when the wake carries an independent reason to continue even with no live comments. */
+ preservesIndependentContinuation: boolean;
+};
+
+export type DeferredWakeAgentFacts = {
+ /** True when the wake's agent exists in the issue's own company. */
+ agentFound: boolean;
+ /** True when the agent is invokable (status, org chain). Meaningless when agentFound is false. */
+ invokable: boolean;
+};
+
+export type DeferredWakePauseHoldFacts = {
+ /** True when an active subtree pause hold covers the issue. */
+ activePauseHold: boolean;
+ /** True when the wake is a verified issue-tree-control interaction wake that survives a pause hold. */
+ treeHoldInteractionWake: boolean;
+};
+
+export type DeferredWakeFacts = {
+ queuedComment: DeferredWakeQueuedCommentFacts;
+ agent: DeferredWakeAgentFacts;
+ pauseHold: DeferredWakePauseHoldFacts;
+};
+
+export type DeferredWakeDecision =
+ | { kind: "cancel_empty"; selfAuthored: boolean }
+ | { kind: "normalize" }
+ | { kind: "fail_not_invokable" }
+ | { kind: "cancel_pause_hold" }
+ | { kind: "promote" };
+
+/**
+ * Decides what to do with the earliest deferred wake queued against the
+ * issue a run just released. The caller applies a "normalize" decision (a
+ * queued-comment-id rewrite) and calls this function again with facts that
+ * reflect the rewrite, so a single wake can normalize and then also fail,
+ * cancel, or promote in the same drain step — matching the order the
+ * original state machine always evaluated them in.
+ */
+export function decideDeferredWake(facts: DeferredWakeFacts): DeferredWakeDecision {
+ const { queuedComment, agent, pauseHold } = facts;
+
+ if (
+ queuedComment.hasQueuedCommentIds &&
+ queuedComment.liveNonSelfCommentIdsLength === 0 &&
+ !queuedComment.preservesIndependentContinuation
+ ) {
+ return { kind: "cancel_empty", selfAuthored: queuedComment.containedSelfAuthoredComment };
+ }
+
+ if (queuedComment.hasQueuedCommentIds && queuedComment.liveCommentIdsChanged) {
+ return { kind: "normalize" };
+ }
+
+ if (!agent.agentFound || !agent.invokable) {
+ return { kind: "fail_not_invokable" };
+ }
+
+ if (pauseHold.activePauseHold && !pauseHold.treeHoldInteractionWake) {
+ return { kind: "cancel_pause_hold" };
+ }
+
+ return { kind: "promote" };
+}
+
+export type ReleaseRecoveryReviewParticipantFacts = {
+ /** True when the issue is in_review, unassigned to a user, and waiting on the finishing run as the current agent participant. */
+ applies: boolean;
+ hasExistingExecutionPath: boolean;
+ hasPersistedMonitor: boolean;
+ suppressedByPauseHold: boolean;
+ isStrandedRecoveryOrigin: boolean;
+ recoveryAgentPresent: boolean;
+ recoveryAgentInvokable: boolean;
+ /** True when the finishing run was itself a review-participant-recovery retry. */
+ isExecutionReviewParticipantRecoveryRun: boolean;
+};
+
+export type ReleaseRecoveryImmediateFacts = {
+ /** True when the issue is todo/in_progress, unassigned to a user, not hidden, still assigned to the finishing run's agent, and the run ended failed/timed_out/cancelled. */
+ applies: boolean;
+ /** True when the finishing run itself carried the disposition-repair retry reason. */
+ isDispositionRepairRetry: boolean;
+ hasExistingExecutionPath: boolean;
+ hasPersistedMonitor: boolean;
+ hasExplicitBlockerPath: boolean;
+ suppressedByPauseHold: boolean;
+ isStrandedRecoveryOrigin: boolean;
+ recoveryAgentPresent: boolean;
+ recoveryAgentInvokable: boolean;
+ isWorkspaceValidationFailedRun: boolean;
+ isConfigurationIncompleteFailedRun: boolean;
+ /** didAutomaticRecoveryFail(run, expectedRetryReason) for the issue's own status branch. */
+ automaticRecoveryAlreadyFailed: boolean;
+};
+
+export type ReleaseRecoveryFacts = {
+ /** options.suppressImmediateRecovery on the caller's release request. */
+ suppressImmediateRecovery: boolean;
+ reviewParticipant: ReleaseRecoveryReviewParticipantFacts;
+ immediate: ReleaseRecoveryImmediateFacts;
+};
+
+export type ReleaseRecoveryBlockedNoticeKind =
+ | "workspace_validation"
+ | "configuration_incomplete"
+ | "execution_review_participant"
+ | "immediate_execution_path";
+
+export type ReleaseRecoveryDecision =
+ | { kind: "released" }
+ | { kind: "blocked_recovery_in_place" }
+ | { kind: "blocked"; notice: ReleaseRecoveryBlockedNoticeKind }
+ | { kind: "queue_review_participant_recovery" }
+ | { kind: "queue_recovery" };
+
+/**
+ * Decides the release-recovery outcome once the deferred-wake queue is
+ * empty and no wake was promoted. The review-participant branch and the
+ * assignment-and-continuation branch are the two ways a released issue can
+ * still need automatic recovery; this function evaluates both as one
+ * decision, review-participant first, matching the original evaluation
+ * order.
+ */
+export function decideReleaseRecovery(facts: ReleaseRecoveryFacts): ReleaseRecoveryDecision {
+ const { reviewParticipant, immediate } = facts;
+
+ if (reviewParticipant.applies) {
+ if (
+ facts.suppressImmediateRecovery ||
+ reviewParticipant.hasExistingExecutionPath ||
+ reviewParticipant.hasPersistedMonitor ||
+ reviewParticipant.suppressedByPauseHold
+ ) {
+ return { kind: "released" };
+ }
+ if (reviewParticipant.isStrandedRecoveryOrigin) {
+ return { kind: "blocked_recovery_in_place" };
+ }
+ const shouldBlock =
+ !reviewParticipant.recoveryAgentInvokable ||
+ !reviewParticipant.recoveryAgentPresent ||
+ reviewParticipant.isExecutionReviewParticipantRecoveryRun;
+ if (shouldBlock) {
+ return { kind: "blocked", notice: "execution_review_participant" };
+ }
+ return { kind: "queue_review_participant_recovery" };
+ }
+
+ if (immediate.isDispositionRepairRetry) return { kind: "released" };
+ if (!immediate.applies) return { kind: "released" };
+ if (facts.suppressImmediateRecovery) return { kind: "released" };
+ if (
+ immediate.hasExistingExecutionPath ||
+ immediate.hasPersistedMonitor ||
+ immediate.hasExplicitBlockerPath
+ ) {
+ return { kind: "released" };
+ }
+ if (immediate.suppressedByPauseHold) return { kind: "released" };
+ if (immediate.isStrandedRecoveryOrigin) return { kind: "blocked_recovery_in_place" };
+
+ const shouldBlockImmediately =
+ !immediate.recoveryAgentInvokable ||
+ !immediate.recoveryAgentPresent ||
+ immediate.isWorkspaceValidationFailedRun ||
+ immediate.isConfigurationIncompleteFailedRun ||
+ immediate.automaticRecoveryAlreadyFailed;
+ if (shouldBlockImmediately) {
+ const notice: ReleaseRecoveryBlockedNoticeKind = immediate.isWorkspaceValidationFailedRun
+ ? "workspace_validation"
+ : immediate.isConfigurationIncompleteFailedRun
+ ? "configuration_incomplete"
+ : "immediate_execution_path";
+ return { kind: "blocked", notice };
+ }
+
+ return { kind: "queue_recovery" };
+}
diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts
new file mode 100644
index 0000000000..4c794e452e
--- /dev/null
+++ b/server/src/modules/wake-queue/index.ts
@@ -0,0 +1,50 @@
+import type { Db } from "@paperclipai/db";
+import { createPostgresWakeQueueAdapter } from "./adapters/postgres.js";
+import { createReleaseIssueExecution } from "./application/use-cases.js";
+import type {
+ IssueSnapshot,
+ RecoveryEscalationPort,
+ RunSnapshot,
+ WakeQueueReader,
+} from "./application/ports.js";
+
+export type {
+ PostCommitEffect,
+ ReleaseOutcome,
+ RunSummary,
+} from "./application/types.js";
+export { WakeQueueApplicationError } from "./application/types.js";
+export type { IssueSnapshot, RunSnapshot, RecoveryEscalationPort } from "./application/ports.js";
+export type { ReleaseIssueExecutionInput } from "./application/use-cases.js";
+
+export type WakeQueueDeps = {
+ /** Stays in `heartbeat.ts`; resolves the responsible user for a promoted or recovery run seed. */
+ resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"];
+ /** Stays in `heartbeat.ts`; reads the routine environment context for an execution issue. */
+ getRoutineEnv: WakeQueueReader["getRoutineEnv"];
+ /** Stays in `heartbeat.ts`; resolves the session-before display id for a wakeup. */
+ resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"];
+ /** `services/recovery`'s stranded-issue escalation, called only after the release transaction commits. */
+ recovery: RecoveryEscalationPort;
+};
+
+/**
+ * Composes the wake-queue module: the Postgres adapter (which owns the
+ * release transaction) and the release use case. `heartbeat.ts` holds the
+ * only caller: it builds one instance per process next to
+ * `createRunDispatch(db)` and delegates `releaseIssueExecutionAndPromote`'s
+ * body to `releaseIssueExecution`.
+ */
+export function createWakeQueue(db: Db, deps: WakeQueueDeps) {
+ const issueLock = createPostgresWakeQueueAdapter(db, {
+ resolveResponsibleUserId: deps.resolveResponsibleUserId,
+ getRoutineEnv: deps.getRoutineEnv,
+ resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup,
+ });
+
+ return {
+ releaseIssueExecution: createReleaseIssueExecution({ issueLock, recovery: deps.recovery }),
+ };
+}
+
+export type WakeQueue = ReturnType<typeof createWakeQueue>;
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 78bfa70dcb..37a38c592f 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -3,7 +3,6 @@ import { legacyExecutionNeedsReconciliation, terminalizeLegacyExecution } from "
import { adapterExecutionControls, createAdapterExecutionControl, waitForAdapterStop } from "./adapter-execution-control.js";
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";
-import { issueRecoveryActionService } from "./issue-recovery-actions.js";
export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";
import { rebindContinuationContract } from "./native-runtime/continuation-contract.js";
import { buildExecutionContinuation } from "./execution-continuation.js";
@@ -376,13 +375,9 @@ import {
SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY,
readContinuationAttempt,
} from "./recovery/index.js";
-import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js";
import {
SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,
- buildConfigurationIncompleteRecoveryNoticeSeed,
- buildExecutionReviewParticipantRecoveryNoticeSeed,
- buildImmediateExecutionPathRecoveryNoticeSeed,
- buildWorkspaceValidationRecoveryNoticeSeed,
+ type StrandedRecoveryNoticeSeed,
} from "./recovery/stranded-notice.js";
import { withRecoveryContext } from "./recovery/status-only-context.js";
import {
@@ -403,6 +398,11 @@ import {
allowsIssueInteractionWake,
isResolvedInteractionContinuationWakeContext,
} from "../modules/run-dispatch/index.js";
+import {
+ createWakeQueue,
+ WakeQueueApplicationError,
+ type PostCommitEffect as WakeQueuePostCommitEffect,
+} from "../modules/wake-queue/index.js";
import {
buildIssueReviewPathLostIdempotencyKey,
decideIssueReviewPathRecovery,
@@ -795,13 +795,6 @@ function buildSandboxProviderPluginNotReadyResultJson(
};
}
-function readConfigurationIncompletePayload(
- run: Pick<typeof heartbeatRuns.$inferSelect, "resultJson"> | null | undefined,
-): Record<string, unknown> | null {
- const payload = parseObject(parseObject(run?.resultJson).configurationIncomplete);
- return Object.keys(payload).length > 0 ? payload : null;
-}
-
function buildUnresolvedWorkspaceBaseRefResultJson(
run: typeof heartbeatRuns.$inferSelect,
error: UnresolvedWorkspaceBaseRefError,
@@ -4830,52 +4823,6 @@ export function summarizeHeartbeatRunListResultJson(input: {
return Object.keys(summary).length > 0 ? summary : null;
}
-function didAutomaticRecoveryFail(
- latestRun: Pick<
- typeof heartbeatRuns.$inferSelect,
- "status" | "contextSnapshot"
- > | null,
- expectedRetryReason:
- | "assignment_recovery"
- | "issue_continuation_needed"
- | typeof EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
-) {
- if (!latestRun) return false;
-
- const latestContext = parseObject(latestRun.contextSnapshot);
- const latestRetryReason = readNonEmptyString(latestContext.retryReason);
- return (
- latestRetryReason === expectedRetryReason &&
- UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
- latestRun.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
- )
- );
-}
-
-function isExecutionReviewParticipantRecoveryRun(
- run: Pick<typeof heartbeatRuns.$inferSelect, "contextSnapshot"> | null,
-) {
- if (!run) return false;
- const context = parseObject(run.contextSnapshot);
- return (
- readNonEmptyString(context.retryReason) ===
- EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON
- );
-}
-
-function isExecutionReviewParticipantRecoveryEligibleRun(
- run: Pick<typeof heartbeatRuns.$inferSelect, "contextSnapshot"> | null,
-) {
- if (!run) return false;
- const context = parseObject(run.contextSnapshot);
- const wakeReason = readNonEmptyString(context.wakeReason);
- return (
- wakeReason === "execution_review_requested" ||
- wakeReason === "execution_approval_requested" ||
- isExecutionReviewParticipantRecoveryRun(run)
- );
-}
-
function normalizeLedgerBillingType(value: unknown): BillingType {
const raw = readNonEmptyString(value);
switch (raw) {
@@ -8499,6 +8446,134 @@ export function heartbeatService(
}
}
+ const wakeQueue = createWakeQueue(db, {
+ resolveResponsibleUserId: async (input) => {
+ // `input.issue` is the wake-queue module's own transaction-scoped
+ // snapshot; using it here, instead of re-reading the issue through
+ // `getIssueExecutionContext`, keeps this read off a second connection
+ // while the module's transaction is open, and keeps it seeing the
+ // in-transaction issue status rather than a stale one.
+ return resolveResponsibleUserIdForRunSeed({
+ companyId: input.companyId,
+ contextSnapshot: input.contextSnapshot,
+ issueContext: input.issue,
+ // The wake-queue module's port type widens `env` to `unknown` so its
+ // application layer stays free of this file's routine env type; the
+ // value always comes from this file's own getRoutineEnvForExecutionIssue.
+ routineEnvContext: input.routineEnvContext as Awaited<
+ ReturnType<typeof getRoutineEnvForExecutionIssue>
+ >,
+ requestedByActorType: input.requestedByActorType,
+ requestedByActorId: input.requestedByActorId,
+ source: input.source as WakeupOptions["source"],
+ triggerDetail: input.triggerDetail as WakeupOptions["triggerDetail"],
+ existingRunResponsibleUserId: input.existingRunResponsibleUserId,
+ });
+ },
+ getRoutineEnv: async (input) => {
+ // Same reason as `resolveResponsibleUserId` above: use the passed-in
+ // transaction-scoped issue snapshot instead of reading the issue again.
+ return getRoutineEnvForExecutionIssue(input.companyId, input.issue);
+ },
+ resolveSessionBeforeForWakeup: async (input) => {
+ // Scoped to this port only, so a wake-queue agent id can never resolve
+ // a session against another company's agent row. The shared `getAgent`
+ // helper below has no company predicate, so this reads the agent
+ // directly with the company named in its own `WHERE` clause.
+ const agent = await db
+ .select()
+ .from(agents)
+ .where(and(eq(agents.id, input.agentId), eq(agents.companyId, input.companyId)))
+ .then((rows) => rows[0] ?? null);
+ if (!agent) return null;
+ return resolveSessionBeforeForWakeup(agent, input.taskKey);
+ },
+ recovery: {
+ // The wake-queue module's plain snapshots hold only the fields the
+ // release decision needs; escalation needs the full row, so this
+ // re-reads both by id after the release transaction has committed.
+ escalateStrandedAssignedIssue: async (input) => {
+ const [issueRow] = await db
+ .select()
+ .from(issues)
+ .where(and(eq(issues.id, input.issue.id), eq(issues.companyId, input.issue.companyId)));
+ const [runRow] = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(and(eq(heartbeatRuns.id, input.latestRun.id), eq(heartbeatRuns.companyId, input.latestRun.companyId)));
+ if (!issueRow || !runRow) return;
+ await recovery.escalateStrandedAssignedIssue({
+ issue: issueRow,
+ previousStatus: input.previousStatus,
+ latestRun: runRow,
+ notice: input.notice as StrandedRecoveryNoticeSeed,
+ recoveryCause: (input.recoveryCause ?? undefined) as
+ | typeof WORKSPACE_VALIDATION_RECOVERY_CAUSE
+ | typeof CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE
+ | typeof EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE
+ | undefined,
+ });
+ },
+ escalateStrandedRecoveryIssueInPlace: async (input) => {
+ const [issueRow] = await db
+ .select()
+ .from(issues)
+ .where(and(eq(issues.id, input.issue.id), eq(issues.companyId, input.issue.companyId)));
+ const [runRow] = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(and(eq(heartbeatRuns.id, input.latestRun.id), eq(heartbeatRuns.companyId, input.latestRun.companyId)));
+ if (!issueRow || !runRow) return;
+ await recovery.escalateStrandedRecoveryIssueInPlace({
+ issue: issueRow,
+ previousStatus: input.previousStatus,
+ latestRun: runRow,
+ });
+ },
+ },
+ });
+
+ // Applies the post-commit effects a wake-queue release returns, exactly as
+ // the original release function did before its writes moved into that
+ // module: publish + dispatch a promoted or recovery run, log a reopened
+ // issue's activity entry.
+ async function applyWakeQueuePostCommitEffects(effects: WakeQueuePostCommitEffect[]) {
+ for (const effect of effects) {
+ if (effect.kind === "run_queued") {
+ publishLiveEvent({
+ companyId: effect.run.companyId,
+ type: "heartbeat.run.queued",
+ payload: {
+ runId: effect.run.id,
+ agentId: effect.run.agentId,
+ invocationSource: effect.run.invocationSource,
+ triggerDetail: effect.run.triggerDetail,
+ wakeupRequestId: effect.run.wakeupRequestId,
+ },
+ });
+ await startNextQueuedRunForAgent(effect.run.agentId);
+ } else {
+ await logActivity(db, {
+ companyId: effect.companyId,
+ actorType: "system",
+ actorId: "heartbeat",
+ agentId: effect.agentId,
+ runId: effect.runId,
+ action: "issue.updated",
+ entityType: "issue",
+ entityId: effect.issueId,
+ details: {
+ status: "todo",
+ reopened: true,
+ reopenedFrom: effect.reopenedFrom,
+ source: "deferred_comment_wake",
+ identifier: effect.identifier,
+ },
+ });
+ }
+ }
+ }
+
function isPlanApprovalConfirmationPayload(payload: unknown) {
const target = parseObject(parseObject(payload).target);
return (
@@ -9164,7 +9239,7 @@ export function heartbeatService(
async function getRoutineEnvForExecutionIssue(
companyId: string,
- issueContext: Awaited<ReturnType<typeof getIssueExecutionContext>> | null,
+ issueContext: { originKind: string | null; originId: string | null; originRunId: string | null } | null,
) {
if (
!issueContext ||
@@ -9317,7 +9392,7 @@ export function heartbeatService(
async function resolveResponsibleUserIdForRunSeed(input: {
companyId: string;
contextSnapshot: Record<string, unknown>;
- issueContext: Awaited<ReturnType<typeof getIssueExecutionContext>> | null;
+ issueContext: { id: string; responsibleUserId: string | null; parentId: string | null } | null;
routineEnvContext: Awaited<
ReturnType<typeof getRoutineEnvForExecutionIssue>
>;
@@ -22602,1015 +22677,25 @@ export function heartbeatService(
run: typeof heartbeatRuns.$inferSelect,
options: { suppressImmediateRecovery?: boolean } = {},
) {
- const runContext = parseObject(run.contextSnapshot);
- const contextIssueId = readNonEmptyString(runContext.issueId);
- const taskKey = deriveTaskKeyWithHeartbeatFallback(runContext, null);
- const recoveryAgent = await getAgent(run.agentId);
- const recoveryAgentInvokable =
- recoveryAgent &&
- recoveryAgent.status !== "paused" &&
- recoveryAgent.status !== "terminated" &&
- recoveryAgent.status !== "pending_approval";
- const recoverySessionBefore = recoveryAgentInvokable
- ? await resolveSessionBeforeForWakeup(recoveryAgent, taskKey)
- : null;
- const recoveryAgentNameKey = normalizeAgentNameKey(recoveryAgent?.name);
-
- const promotionResult = await db.transaction(async (tx) => {
- // Lock the context issue (if any) AND every issue that still references this run.
- //
- // A single run can hold execution locks on multiple issues: the caller's context
- // issue (set via svc.checkout) plus any additional issues stamped by
- // enqueueWakeup's "legacy run" fallback when the run was the only queued/running
- // run matching their contextSnapshot.issueId. Historically this function only
- // resolved and cleared the lock on *one* issue (rows[0]), leaving the others
- // with an executionRunId pointing at a finalized run. Subsequent checkouts from
- // the assigned agent then failed with 409 and the issue stayed blocked forever.
- // `order by id` makes row-lock acquisition deterministic across concurrent
- // finalizations, which keeps deadlock risk independent of PostgreSQL's plan
- // choice when multiple issues match.
- await tx.execute(
- contextIssueId
- ? sql`
- select id from issues
- where company_id = ${run.companyId}
- and (
- id = ${contextIssueId}
- or execution_run_id = ${run.id}
- or checkout_run_id = ${run.id}
- )
- order by id
- for update
- `
- : sql`
- select id from issues
- where company_id = ${run.companyId}
- and (execution_run_id = ${run.id} or checkout_run_id = ${run.id})
- order by id
- for update
- `,
- );
-
- const candidateIssues = await tx
- .select()
- .from(issues)
- .where(
- and(
- eq(issues.companyId, run.companyId),
- contextIssueId
- ? or(
- eq(issues.id, contextIssueId),
- eq(issues.executionRunId, run.id),
- eq(issues.checkoutRunId, run.id),
- )
- : or(
- eq(issues.executionRunId, run.id),
- eq(issues.checkoutRunId, run.id),
- ),
- ),
- )
- .orderBy(asc(issues.id));
-
- // Clear orphaned execution-lock columns that still point at this finalizing
- // run, across every sibling issue in one statement so it scales with N
- // orphans without N round-trips. Rows are already held under FOR UPDATE from
- // the lock query above.
- //
- // The two columns are cleared in separate UPDATEs so we never clobber a
- // retry's executionRunId pointer: when a process-loss or codex-transient
- // retry is scheduled mid-finalization, it moves `executionRunId` from this
- // run to the retry run while leaving `checkoutRunId` pinned at this run.
- // Only the checkout column should be released in that case; the execution
- // column now belongs to the retry.
- const promotionUpdateTimestamp = new Date();
- await tx
- .update(issues)
- .set({
- executionRunId: null,
- executionAgentNameKey: null,
- executionLockedAt: null,
- updatedAt: promotionUpdateTimestamp,
- })
- .where(
- and(
- eq(issues.companyId, run.companyId),
- eq(issues.executionRunId, run.id),
- ),
- );
- // `checkoutRunId` clear is symmetric to #6008's per-issue self-heal,
- // extended to all siblings: covers paths where the issue's assignee or
- // status changed between checkout and termination, which
- // adoptStaleCheckoutRun's narrow WHERE clause cannot reach.
- await tx
- .update(issues)
- .set({
- checkoutRunId: null,
- updatedAt: promotionUpdateTimestamp,
- })
- .where(
- and(
- eq(issues.companyId, run.companyId),
- eq(issues.checkoutRunId, run.id),
- ),
- );
-
- // Deferred-wake promotion is bound to a single primary issue: the run's context
- // issue when present, otherwise the first candidate we found (preserves the
- // legacy rows[0] selection for runs that were not tied to a specific issue).
- let issue =
- (contextIssueId
- ? candidateIssues.find((candidate) => candidate.id === contextIssueId)
- : candidateIssues[0]) ?? null;
-
- if (!issue) return null;
- if (issue.executionRunId && issue.executionRunId !== run.id) return null;
-
- // Workspace-validation recovery: if the finalizing run failed workspace
- // validation, surface the primary issue for the blocked-recovery comment path.
- // Sibling lock cleanup is already done above; only the primary issue carries
- // the recovery surface because the comment is attached to a single issue.
- if (
- (isWorkspaceValidationFailedRun(run) ||
- isConfigurationIncompleteFailedRun(run)) &&
- (issue.status === "todo" || issue.status === "in_progress") &&
- !issue.assigneeUserId &&
- issue.assigneeAgentId === run.agentId
- ) {
- const configurationIncomplete = isConfigurationIncompleteFailedRun(run);
- return {
- kind: "blocked" as const,
- issue,
- previousStatus: issue.status,
- notice: configurationIncomplete
- ? buildConfigurationIncompleteRecoveryNoticeSeed(readConfigurationIncompletePayload(run))
- : buildWorkspaceValidationRecoveryNoticeSeed(),
- recoveryCause: configurationIncomplete
- ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE
- : WORKSPACE_VALIDATION_RECOVERY_CAUSE,
- };
- }
-
- if (legacyExecutionNeedsReconciliation(run)) return { kind: "released" as const };
- // An operator stop never promotes old queued work by itself. The next
- // explicit wake adopts those messages atomically when it queues a run.
- if (run.status === "cancelled" && parseObject(run.resultJson?.executionCancellation).state === "acknowledged") {
- return { kind: "released" as const };
- }
-
- // Native recovery owns its entire incident. The legacy stranded-work
- // fallback must not create a fresh run and reset a failed native budget,
- // including failures before the native executor acquired its lease.
- if (run.runtimeMode === "native" && ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status)
- && issue.assigneeAgentId === run.agentId && !["done", "cancelled"].includes(issue.status)) {
- const existing = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
- eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.sourceIssueId, issue.id),
- or(inArray(issueRecoveryActions.status, ["active", "escalated"]),
- sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`),
- )).limit(1);
- if (!existing.length) {
- await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null,
- nextAttemptAt: null, recoveryState: "blocked", failureCode: "native_continuation_requires_reconciliation",
- updatedAt: promotionUpdateTimestamp,
- }).where(and(eq(nativeRunFinalizations.runId, run.id), isNull(nativeRunFinalizations.resultId)));
- await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({
- companyId: issue.companyId, sourceIssueId: issue.id, kind: "active_run_watchdog", ownerType: "board",
- returnOwnerAgentId: run.agentId, cause: "native_continuation_requires_reconciliation",
- fingerprint: `native-continuation:${run.id}`, evidence: { runId: run.id, originalFailureCode: run.errorCode },
- nextAction: "Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.",
- maxAttempts: 3, wakePolicy: null, supersedeOnIdentityChange: true,
- });
- }
- return { kind: "released" as const };
- }
-
- while (true) {
- let deferred = await tx
- .select()
- .from(agentWakeupRequests)
- .where(
- and(
- eq(agentWakeupRequests.companyId, issue.companyId),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`,
- ),
- )
- .orderBy(asc(agentWakeupRequests.requestedAt))
- .limit(1)
- .then((rows) => rows[0] ?? null);
-
- if (!deferred) break;
-
- const queuedCommentIds = queuedCommentIdsFromWakePayload(
- deferred.payload,
- );
- if (queuedCommentIds.length > 0) {
- const queuedCommentRows = await tx
- .select({
- id: issueComments.id,
- deletedAt: issueComments.deletedAt,
- createdByRunId: issueComments.createdByRunId,
- })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, issue.companyId),
- eq(issueComments.issueId, issue.id),
- inArray(issueComments.id, queuedCommentIds),
- ),
- );
- const targetsFinishingRunAgent = deferred.agentId === run.agentId;
- const liveNonSelfCommentIds = queuedCommentIds.filter((commentId) => {
- const row = queuedCommentRows.find(
- (candidate) => candidate.id === commentId,
- );
- return Boolean(
- row &&
- !row.deletedAt &&
- (!targetsFinishingRunAgent || row.createdByRunId !== run.id),
- );
- });
- const now = new Date();
- const queuedContext = parseObject(
- parseObject(deferred.payload)[DEFERRED_WAKE_CONTEXT_KEY],
- );
- const queuedReason =
- readNonEmptyString(queuedContext.wakeReason) ??
- readNonEmptyString(deferred.reason);
- const queuedWakeIsCommentOnly =
- !queuedReason ||
- queuedReason === "issue_commented" ||
- queuedReason === "issue_reopened_via_comment" ||
- queuedReason === "issue_comment_mentioned";
- const preservesIndependentContinuation =
- hasInteractionContinuationWakeContext(queuedContext) ||
- queuedContext.resumeIntent === true ||
- !queuedWakeIsCommentOnly;
-
- if (
- liveNonSelfCommentIds.length === 0 &&
- !preservesIndependentContinuation
- ) {
- const containedSelfAuthoredComment = queuedCommentRows.some(
- (row) =>
- targetsFinishingRunAgent &&
- !row.deletedAt &&
- row.createdByRunId === run.id,
- );
- await tx
- .update(agentWakeupRequests)
- .set({
- status: "cancelled",
- finishedAt: now,
- error: containedSelfAuthoredComment
- ? "Deferred wake contained only comments authored by the finishing run"
- : "Queued messages were discarded before promotion",
- updatedAt: now,
- })
- .where(
- and(
- eq(agentWakeupRequests.id, deferred.id),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- ),
- );
- continue;
- }
-
- if (
- liveNonSelfCommentIds.length !== queuedCommentIds.length ||
- liveNonSelfCommentIds.some(
- (commentId, index) => commentId !== queuedCommentIds[index],
- )
- ) {
- const normalizedWake = await tx
- .update(agentWakeupRequests)
- .set({
- payload: withQueuedCommentIdsInWakePayload(
- deferred.payload,
- liveNonSelfCommentIds,
- ),
- updatedAt: now,
- })
- .where(
- and(
- eq(agentWakeupRequests.id, deferred.id),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- ),
- )
- .returning()
- .then((rows) => rows[0] ?? null);
- if (!normalizedWake) continue;
- deferred = normalizedWake;
- }
- }
-
- const deferredAgent = await tx
- .select()
- .from(agents)
- .where(eq(agents.id, deferred.agentId))
- .then((rows) => rows[0] ?? null);
-
- const companyAgents = deferredAgent
- ? await tx
- .select({
- id: agents.id,
- companyId: agents.companyId,
- name: agents.name,
- reportsTo: agents.reportsTo,
- status: agents.status,
- })
- .from(agents)
- .where(eq(agents.companyId, issue.companyId))
- : [];
- const deferredInvokability =
- deferredAgent?.companyId === issue.companyId
- ? evaluateAgentInvokability(deferredAgent, companyAgents)
- : evaluateAgentInvokability(null, companyAgents);
-
- if (
- !deferredAgent ||
- deferredAgent.companyId !== issue.companyId ||
- !deferredInvokability.invokable
- ) {
- await tx
- .update(agentWakeupRequests)
- .set({
- status: "failed",
- finishedAt: new Date(),
- error:
- "Deferred wake could not be promoted: agent is not invokable",
- updatedAt: new Date(),
- })
- .where(eq(agentWakeupRequests.id, deferred.id));
- continue;
- }
-
- const deferredPayload = parseObject(deferred.payload);
- const deferredContextSeed = parseObject(
- deferredPayload[DEFERRED_WAKE_CONTEXT_KEY],
- );
- const activePauseHold = await treeControlSvc.getActivePauseHoldGate(
- issue.companyId,
- issue.id,
- );
- const treeHoldInteractionWake =
- activePauseHold &&
- (await isVerifiedIssueTreeControlInteractionWake(tx, {
- companyId: issue.companyId,
- issueId: issue.id,
- agentId: deferred.agentId,
- contextSnapshot: deferredContextSeed,
- requestedByActorType: deferred.requestedByActorType,
- requestedByActorId: deferred.requestedByActorId,
- }));
- if (activePauseHold && !treeHoldInteractionWake) {
- await tx
- .update(agentWakeupRequests)
- .set({
- status: "cancelled",
- finishedAt: new Date(),
- error: "Deferred wake suppressed by active subtree pause hold",
- updatedAt: new Date(),
- })
- .where(eq(agentWakeupRequests.id, deferred.id));
- continue;
- }
-
- const promotedContextSeed: Record<string, unknown> = {
- ...deferredContextSeed,
- };
- if (activePauseHold) {
- promotedContextSeed.treeHoldInteraction = true;
- promotedContextSeed.activeTreeHold = {
- holdId: activePauseHold.holdId,
- rootIssueId: activePauseHold.rootIssueId,
- mode: activePauseHold.mode,
- reason: activePauseHold.reason,
- releasePolicy: activePauseHold.releasePolicy,
- interaction: true,
- };
- }
- const deferredCommentIds = extractWakeCommentIds(deferredContextSeed);
- const deferredWakeReason = readNonEmptyString(
- deferredContextSeed.wakeReason,
- );
- // Local-CLI agents post comments under user auth, so a self-comment from
- // the run that is now ending would otherwise look like a real human
- // comment and trigger a reopen on the very issue this run just closed.
- // Suppress reopen only when every referenced comment came from this run;
- // mixed batches must still reopen because they contain a real follow-up.
- let deferredCommentWakeIsSelfAuthored = false;
- if (deferredCommentIds.length > 0) {
- const deferredComments = await tx
- .select({ createdByRunId: issueComments.createdByRunId })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, issue.companyId),
- eq(issueComments.issueId, issue.id),
- inArray(issueComments.id, deferredCommentIds),
- ),
- )
- .then((rows) => rows);
- deferredCommentWakeIsSelfAuthored =
- deferredComments.length > 0 &&
- deferredComments.every(
- (comment) => comment.createdByRunId === run.id,
- );
- }
- // Only human/comment-reopen interactions should revive completed issues;
- // system follow-ups such as retry or cleanup wakes must not reopen closed work.
- const shouldReopenDeferredCommentWake =
- deferredCommentIds.length > 0 &&
- !deferredCommentWakeIsSelfAuthored &&
- (issue.status === "done" || issue.status === "cancelled") &&
- (deferred.requestedByActorType === "user" ||
- deferredWakeReason === "issue_reopened_via_comment");
- let reopenedActivity: LogActivityInput | null = null;
-
- if (shouldReopenDeferredCommentWake) {
- const reopenedFromStatus = issue.status;
- const reopenedIssue = await issuesSvc.update(
- issue.id,
- {
- status: "todo",
- executionState: null,
- },
- tx,
- );
- if (reopenedIssue) {
- issue = {
- ...issue,
- identifier: reopenedIssue.identifier,
- status: reopenedIssue.status,
- executionRunId: reopenedIssue.executionRunId,
- };
- if (!readNonEmptyString(promotedContextSeed.reopenedFrom)) {
- promotedContextSeed.reopenedFrom = reopenedFromStatus;
- }
- reopenedActivity = {
- companyId: issue.companyId,
- actorType: "system",
- actorId: "heartbeat",
- agentId: deferred.agentId,
- runId: run.id,
- action: "issue.updated",
- entityType: "issue",
- entityId: issue.id,
- details: {
- status: "todo",
- reopened: true,
- reopenedFrom: reopenedFromStatus,
- source: "deferred_comment_wake",
- identifier: issue.identifier,
- },
- };
- }
- }
-
- const promotedReason =
- readNonEmptyString(deferred.reason) ?? "issue_execution_promoted";
- const promotedSource =
- (readNonEmptyString(deferred.source) as WakeupOptions["source"]) ??
- "automation";
- const promotedTriggerDetail =
- (readNonEmptyString(
- deferred.triggerDetail,
- ) as WakeupOptions["triggerDetail"]) ?? null;
- const promotedPayload = deferredPayload;
- delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY];
-
- const {
- contextSnapshot: promotedContextSnapshot,
- taskKey: promotedTaskKey,
- } = enrichWakeContextSnapshot({
- contextSnapshot: promotedContextSeed,
- reason: promotedReason,
- source: promotedSource,
- triggerDetail: promotedTriggerDetail,
- payload: promotedPayload,
- });
-
- const sessionBefore =
- readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
- (await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey));
- const promotedContinuationAttempt = readContinuationAttempt(
- promotedContextSnapshot.livenessContinuationAttempt,
- );
- const promotedResponsibleUserId =
- await resolveResponsibleUserIdForRunSeed({
- companyId: deferredAgent.companyId,
- contextSnapshot: promotedContextSnapshot,
- issueContext: issue,
- routineEnvContext: await getRoutineEnvForExecutionIssue(
- deferredAgent.companyId,
- issue,
- ),
- requestedByActorType: deferred.requestedByActorType as
- "user" | "agent" | "system" | null,
- requestedByActorId: deferred.requestedByActorId,
- source: promotedSource,
- triggerDetail: promotedTriggerDetail,
- existingRunResponsibleUserId: run.responsibleUserId,
- });
- if (!promotedResponsibleUserId) {
- throw new HttpError(
- 422,
- "Unable to resolve responsible user for promoted heartbeat run",
- {
- code: "responsible_user_unresolved",
- runId: run.id,
- agentId: deferredAgent.id,
- companyId: deferredAgent.companyId,
- issueId: issue.id,
- wakeReason: readNonEmptyString(
- promotedContextSnapshot.wakeReason,
- ),
- },
- );
- }
- const now = new Date();
- const newRun = await tx
- .insert(heartbeatRuns)
- .values({
- companyId: deferredAgent.companyId,
- agentId: deferredAgent.id,
- invocationSource: promotedSource,
- triggerDetail: promotedTriggerDetail,
- status: "queued",
- wakeupRequestId: deferred.id,
- contextSnapshot: promotedContextSnapshot,
- responsibleUserId: promotedResponsibleUserId,
- sessionIdBefore: sessionBefore,
- continuationAttempt: promotedContinuationAttempt,
- })
- .returning()
- .then((rows) => rows[0]);
-
- await tx
- .update(agentWakeupRequests)
- .set({
- status: "queued",
- reason: "issue_execution_promoted",
- runId: newRun.id,
- claimedAt: null,
- finishedAt: null,
- error: null,
- updatedAt: now,
- })
- .where(eq(agentWakeupRequests.id, deferred.id));
-
- await tx
- .update(issues)
- .set({
- executionRunId: newRun.id,
- executionAgentNameKey: normalizeAgentNameKey(deferredAgent.name),
- executionLockedAt: now,
- updatedAt: now,
- })
- // Promoted mention wakes are issue-scoped, not issue ownership transfers.
- .where(
- and(
- eq(issues.id, issue.id),
- eq(issues.assigneeAgentId, deferredAgent.id),
- ),
- );
-
- return {
- kind: "promoted" as const,
- run: newRun,
- reopenedActivity,
- };
- }
-
- const findExistingExecutionPath = (agentId?: string | null) =>
- tx
- .select({ id: heartbeatRuns.id })
- .from(heartbeatRuns)
- .where(
- and(
- eq(heartbeatRuns.companyId, issue.companyId),
- inArray(heartbeatRuns.status, [
- ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,
- ]),
- sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`,
- sql`${heartbeatRuns.id} <> ${run.id}`,
- agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`,
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null);
-
- const issueHasPersistedMonitor = Boolean(issue.monitorNextCheckAt);
- const findExplicitBlockerPath = () =>
- tx
- .select({ id: issueRelations.issueId })
- .from(issueRelations)
- .innerJoin(issues, eq(issueRelations.issueId, issues.id))
- .where(
- and(
- eq(issueRelations.companyId, issue.companyId),
- eq(issueRelations.relatedIssueId, issue.id),
- eq(issueRelations.type, "blocks"),
- eq(issues.companyId, issue.companyId),
- notInArray(issues.status, ["done", "cancelled"]),
- isNull(issues.hiddenAt),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null);
- const executionState = parseIssueExecutionState(issue.executionState);
- const currentParticipant =
- executionState?.status === "pending"
- ? executionState.currentParticipant
- : null;
- const issueNeedsReviewParticipantRecovery =
- issue.status === "in_review" &&
- !issue.assigneeUserId &&
- currentParticipant?.type === "agent" &&
- currentParticipant.agentId === run.agentId &&
- isExecutionReviewParticipantRecoveryEligibleRun(run) &&
- HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
- run.status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
- );
-
- if (issueNeedsReviewParticipantRecovery) {
- const existingReviewParticipantExecutionPath =
- await findExistingExecutionPath(currentParticipant.agentId);
- if (
- options.suppressImmediateRecovery ||
- existingReviewParticipantExecutionPath ||
- issueHasPersistedMonitor ||
- (await isAutomaticRecoverySuppressedByPauseHold(
- db,
- issue.companyId,
- issue.id,
- treeControlSvc,
- ))
- ) {
- return { kind: "released" as const };
- }
-
- if (issue.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery) {
- return {
- kind: "blocked_recovery_in_place" as const,
- issue,
- previousStatus: issue.status,
- };
- }
-
- const shouldBlockReviewRecovery =
- !recoveryAgentInvokable ||
- !recoveryAgent ||
- isExecutionReviewParticipantRecoveryRun(run);
- if (shouldBlockReviewRecovery) {
- return {
- kind: "blocked" as const,
- issue,
- previousStatus: issue.status,
- notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
- recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE,
- };
- }
-
- const now = new Date();
- const wakeupRequest = await tx
- .insert(agentWakeupRequests)
- .values({
- companyId: issue.companyId,
- agentId: recoveryAgent.id,
- source: "automation",
- triggerDetail: "system",
- reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
- payload: withRecoveryContext(
- {
- issueId: issue.id,
- retryOfRunId: run.id,
- retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
- currentStageId: executionState?.currentStageId ?? null,
- currentStageType: executionState?.currentStageType ?? null,
- },
- "normal_model",
- ),
- status: "queued",
- requestedByActorType: "system",
- requestedByActorId: null,
- updatedAt: now,
- })
- .returning()
- .then((rows) => rows[0]);
-
- const queuedRun = await tx
- .insert(heartbeatRuns)
- .values({
- companyId: issue.companyId,
- agentId: recoveryAgent.id,
- invocationSource: "automation",
- triggerDetail: "system",
- status: "queued",
- wakeupRequestId: wakeupRequest.id,
- contextSnapshot: withRecoveryContext(
- {
- issueId: issue.id,
- taskId: issue.id,
- wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
- retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
- source: "issue.execution_review_recovery",
- retryOfRunId: run.id,
- currentStageId: executionState?.currentStageId ?? null,
- currentStageType: executionState?.currentStageType ?? null,
- reviewRecoveryInstruction:
- "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",
- },
- "normal_model",
- ),
- sessionIdBefore: recoverySessionBefore,
- retryOfRunId: run.id,
- updatedAt: now,
- })
- .returning()
- .then((rows) => rows[0]);
-
- await tx
- .update(agentWakeupRequests)
- .set({
- runId: queuedRun.id,
- updatedAt: now,
- })
- .where(eq(agentWakeupRequests.id, wakeupRequest.id));
-
- await tx
- .update(issues)
- .set({
- executionRunId: queuedRun.id,
- executionAgentNameKey: recoveryAgentNameKey,
- executionLockedAt: now,
- updatedAt: now,
- })
- .where(eq(issues.id, issue.id));
-
- return {
- kind: "queued_recovery" as const,
- run: queuedRun,
- };
- }
-
- const issueNeedsImmediateRecovery =
- (issue.status === "todo" || issue.status === "in_progress") &&
- !issue.assigneeUserId &&
- !issue.hiddenAt &&
- issue.assigneeAgentId === run.agentId &&
- (run.status === "failed" ||
- run.status === "timed_out" ||
- run.status === "cancelled");
-
- if (
- readNonEmptyString(parseObject(run.contextSnapshot).retryReason) ===
- ISSUE_DISPOSITION_REPAIR_RETRY_REASON
- ) {
- return { kind: "released" as const };
- }
-
- if (!issueNeedsImmediateRecovery) {
- return { kind: "released" as const };
- }
- if (options.suppressImmediateRecovery) {
- return { kind: "released" as const };
- }
-
- const existingExecutionPath = await findExistingExecutionPath();
- if (
- existingExecutionPath ||
- issueHasPersistedMonitor ||
- (await findExplicitBlockerPath())
- ) {
- return { kind: "released" as const };
- }
-
+ try {
+ const { postCommitEffects } = await wakeQueue.releaseIssueExecution({
+ companyId: run.companyId,
+ runId: run.id,
+ now: new Date(),
+ suppressImmediateRecovery: options.suppressImmediateRecovery,
+ });
+ await applyWakeQueuePostCommitEffects(postCommitEffects);
+ } catch (error) {
if (
- await isAutomaticRecoverySuppressedByPauseHold(
- db,
- issue.companyId,
- issue.id,
- treeControlSvc,
- )
+ error instanceof WakeQueueApplicationError &&
+ error.code === "responsible_user_unresolved"
) {
- return { kind: "released" as const };
- }
-
- if (issue.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery) {
- return {
- kind: "blocked_recovery_in_place" as const,
- issue,
- previousStatus: issue.status,
- };
- }
-
- const shouldBlockImmediately =
- !recoveryAgentInvokable ||
- !recoveryAgent ||
- isWorkspaceValidationFailedRun(run) ||
- isConfigurationIncompleteFailedRun(run) ||
- didAutomaticRecoveryFail(
- run,
- issue.status === "todo"
- ? "assignment_recovery"
- : "issue_continuation_needed",
- );
- if (shouldBlockImmediately) {
- const workspaceValidationFailure = isWorkspaceValidationFailedRun(run);
- const configurationIncompleteFailure =
- isConfigurationIncompleteFailedRun(run);
- const notice = workspaceValidationFailure
- ? buildWorkspaceValidationRecoveryNoticeSeed()
- : configurationIncompleteFailure
- ? buildConfigurationIncompleteRecoveryNoticeSeed(readConfigurationIncompletePayload(run))
- : buildImmediateExecutionPathRecoveryNoticeSeed({
- status: issue.status as "todo" | "in_progress",
- });
- return {
- kind: "blocked" as const,
- issue,
- previousStatus: issue.status,
- notice,
- recoveryCause: workspaceValidationFailure
- ? WORKSPACE_VALIDATION_RECOVERY_CAUSE
- : configurationIncompleteFailure
- ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE
- : undefined,
- };
+ // Every other `responsible_user_unresolved` HttpError in this file
+ // carries `code` inside its own `details`; match that shape here too.
+ throw new HttpError(422, error.message, { code: error.code, ...error.details });
}
-
- const retryReason =
- issue.status === "todo"
- ? "assignment_recovery"
- : "issue_continuation_needed";
- const recoveryReason =
- issue.status === "todo"
- ? "issue_assignment_recovery"
- : "issue_continuation_needed";
- const recoverySource =
- issue.status === "todo"
- ? "issue.assignment_recovery"
- : "issue.continuation_recovery";
- const now = new Date();
- const recoveryContextSnapshot = withRecoveryContext(
- {
- issueId: issue.id,
- taskId: issue.id,
- wakeReason: recoveryReason,
- retryReason,
- source: recoverySource,
- retryOfRunId: run.id,
- },
- "normal_model",
- );
- const responsibleUserId = await resolveResponsibleUserIdForRunSeed({
- companyId: issue.companyId,
- contextSnapshot: recoveryContextSnapshot,
- issueContext: issue,
- routineEnvContext: await getRoutineEnvForExecutionIssue(
- issue.companyId,
- issue,
- ),
- requestedByActorType: "system",
- requestedByActorId: null,
- source: "automation",
- triggerDetail: "system",
- existingRunResponsibleUserId: run.responsibleUserId,
- });
- if (!responsibleUserId) {
- throw new HttpError(
- 422,
- "Unable to resolve responsible user for recovery heartbeat run",
- {
- code: "responsible_user_unresolved",
- runId: run.id,
- agentId: recoveryAgent.id,
- companyId: issue.companyId,
- issueId: issue.id,
- wakeReason: recoveryReason,
- },
- );
- }
- const wakeupRequest = await tx
- .insert(agentWakeupRequests)
- .values({
- companyId: issue.companyId,
- agentId: recoveryAgent.id,
- source: "automation",
- triggerDetail: "system",
- reason: recoveryReason,
- payload: withRecoveryContext(
- {
- issueId: issue.id,
- retryOfRunId: run.id,
- },
- "normal_model",
- ),
- status: "queued",
- requestedByActorType: "system",
- requestedByActorId: null,
- updatedAt: now,
- })
- .returning()
- .then((rows) => rows[0]);
-
- const queuedRun = await tx
- .insert(heartbeatRuns)
- .values({
- companyId: issue.companyId,
- agentId: recoveryAgent.id,
- invocationSource: "automation",
- triggerDetail: "system",
- status: "queued",
- wakeupRequestId: wakeupRequest.id,
- contextSnapshot: recoveryContextSnapshot,
- responsibleUserId,
- sessionIdBefore: recoverySessionBefore,
- retryOfRunId: run.id,
- updatedAt: now,
- })
- .returning()
- .then((rows) => rows[0]);
-
- await tx
- .update(agentWakeupRequests)
- .set({
- runId: queuedRun.id,
- updatedAt: now,
- })
- .where(eq(agentWakeupRequests.id, wakeupRequest.id));
-
- await tx
- .update(issues)
- .set({
- executionRunId: queuedRun.id,
- executionAgentNameKey: recoveryAgentNameKey,
- executionLockedAt: now,
- updatedAt: now,
- })
- .where(eq(issues.id, issue.id));
-
- return {
- kind: "queued_recovery" as const,
- run: queuedRun,
- };
- });
-
- if (promotionResult?.kind === "blocked") {
- await recovery.escalateStrandedAssignedIssue({
- issue: promotionResult.issue,
- previousStatus: promotionResult.previousStatus as
- "todo" | "in_progress" | "in_review",
- latestRun: run,
- notice: promotionResult.notice,
- recoveryCause:
- promotionResult.recoveryCause === WORKSPACE_VALIDATION_RECOVERY_CAUSE
- ? WORKSPACE_VALIDATION_RECOVERY_CAUSE
- : promotionResult.recoveryCause ===
- CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE
- ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE
- : promotionResult.recoveryCause ===
- EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE
- ? EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE
- : undefined,
- });
- return;
- }
-
- if (promotionResult?.kind === "blocked_recovery_in_place") {
- await recovery.escalateStrandedRecoveryIssueInPlace({
- issue: promotionResult.issue,
- previousStatus: promotionResult.previousStatus as
- "todo" | "in_progress" | "in_review",
- latestRun: run,
- });
- return;
- }
-
- const promotedRun = promotionResult?.run ?? null;
- if (!promotedRun) return;
-
- if (
- promotionResult?.kind === "promoted" &&
- promotionResult.reopenedActivity
- ) {
- await logActivity(db, promotionResult.reopenedActivity);
+ throw error;
}
-
- publishLiveEvent({
- companyId: promotedRun.companyId,
- type: "heartbeat.run.queued",
- payload: {
- runId: promotedRun.id,
- agentId: promotedRun.agentId,
- invocationSource: promotedRun.invocationSource,
- triggerDetail: promotedRun.triggerDetail,
- wakeupRequestId: promotedRun.wakeupRequestId,
- },
- });
-
- await startNextQueuedRunForAgent(promotedRun.agentId);
}
async function enqueueWakeup(agentId: string, opts: WakeupOptions = {}) {
diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts
index 91684013af..709132b2bf 100644
--- a/server/src/services/issues.ts
+++ b/server/src/services/issues.ts
@@ -5638,7 +5638,7 @@ export function issueService(db: Db) {
return row;
}
- return {
+ const service = {
clearExecutionRunIfTerminal,
clearCheckoutRunIfTerminal,
addStopRelayCommentIfNeeded,
@@ -7783,6 +7783,7 @@ export function issueService(db: Db) {
blockedByIssueIds?: string[];
actorAgentId?: string | null;
actorUserId?: string | null;
+ companyGuard?: string;
},
dbOrTx: any = db,
postCommitActivityPublications?: ActivityPublication[],
@@ -7792,10 +7793,18 @@ export function issueService(db: Db) {
const activityPublications = postCommitActivityPublications ?? ownedActivityPublications;
const ownedPostCommitActions: IssuePostCommitAction[] = [];
const queuedPostCommitActions = postCommitActions ?? ownedPostCommitActions;
+ // A caller that supplies `companyGuard` gets the company added to
+ // every read, lock, and write predicate below. A check before this
+ // call is not a boundary: `issues.company_id` can change between
+ // that check and this write, so the predicate must carry the
+ // company itself.
+ const idPredicate = data.companyGuard !== undefined
+ ? and(eq(issues.id, id), eq(issues.companyId, data.companyGuard))
+ : eq(issues.id, id);
const existing = await dbOrTx
.select()
.from(issues)
- .where(eq(issues.id, id))
+ .where(idPredicate)
.then((rows: Array<typeof issues.$inferSelect>) => rows[0] ?? null);
if (!existing) return null;
@@ -7804,6 +7813,7 @@ export function issueService(db: Db) {
blockedByIssueIds,
actorAgentId,
actorUserId,
+ companyGuard,
...issueData
} = data;
const isolatedWorkspacesEnabled = (await instanceSettings.getExperimental()).enableIsolatedWorkspaces;
@@ -7955,7 +7965,7 @@ export function issueService(db: Db) {
const receiptExisting = await tx
.select()
.from(issues)
- .where(eq(issues.id, id))
+ .where(idPredicate)
.for("update")
.then((rows: Array<typeof issues.$inferSelect>) => rows[0] ?? null);
if (!receiptExisting) return null;
@@ -7994,7 +8004,7 @@ export function issueService(db: Db) {
const updated = await tx
.update(issues)
.set(patch)
- .where(eq(issues.id, id))
+ .where(idPredicate)
.returning()
.then((rows: Array<typeof issues.$inferSelect>) => rows[0] ?? null);
if (!updated) return null;
@@ -9535,4 +9545,38 @@ export function issueService(db: Db) {
}));
},
};
+
+ type IssueServiceApi = typeof service & {
+ updateForCompany: (
+ id: string,
+ companyId: string,
+ data: Parameters<typeof service.update>[1],
+ dbOrTx?: any,
+ postCommitActivityPublications?: ActivityPublication[],
+ postCommitActions?: IssuePostCommitAction[],
+ ) => ReturnType<typeof service.update>;
+ };
+ const serviceApi = service as IssueServiceApi;
+
+ // A company-scoped wrapper around `update`. It passes the company as a
+ // guard on every read, lock, and write predicate, so a caller with only
+ // a company id and an issue id cannot update an issue in another company.
+ serviceApi.updateForCompany = async (
+ id,
+ companyId,
+ data,
+ dbOrTx = db,
+ postCommitActivityPublications,
+ postCommitActions,
+ ) => {
+ return service.update(
+ id,
+ { ...data, companyGuard: companyId },
+ dbOrTx,
+ postCommitActivityPublications,
+ postCommitActions,
+ );
+ };
+
+ return serviceApi;
}