The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(runner): accept the indeterminate command result after a runner restart (#12646)
Zannis Kalampoukis0cc40037acfd
diff --git a/packages/paperclip-runner/src/contracts/durable-recovery.ts b/packages/paperclip-runner/src/contracts/durable-recovery.tsindex 5cd271b687..b04d41c1e4 100644--- a/packages/paperclip-runner/src/contracts/durable-recovery.ts+++ b/packages/paperclip-runner/src/contracts/durable-recovery.ts@@ -37,7 +37,23 @@ export interface DurableRecoveryProcessedCommand {commandId: string;controllerSeq: number;commandDigest: string;- status: "completed" | "failed" | "rejected";+ /**+ * The runner's command journal is persisted before a command's effect and+ * re-persisted after recovery, so a recovered trace carries the whole+ * lifecycle, not just the settled end of it:+ *+ * - `pending` — journaled, effect not yet confirmed. Written by+ * `DurableState::begin_command` and durable from that moment.+ * - `indeterminate` — the crash-recovery verdict.+ * `DurableState::reconcile_pending_commands` promotes every `pending`+ * entry on load and saves the state back, so the command is never+ * executed twice. Terminal.+ *+ * Both are values a consumer can read off `processedCommands`; a union that+ * omits them tells the compiler a state the runner routinely writes is+ * impossible.+ */+ status: "pending" | "completed" | "failed" | "rejected" | "indeterminate";logicalEffectCount: number;result: Record<string, unknown>;}@@ -79,7 +95,13 @@ export interface DurableRecoveryCoreCommand {type: string;issuedAt: string;payload: Record<string, unknown>;- status: "pending" | "completed" | "failed" | "rejected";+ /**+ * `indeterminate` is the runner's crash-recovery verdict: the command was+ * journaled but its effect was never confirmed, so the runner will not+ * execute it a second time. It is terminal, like the other non-pending+ * statuses.+ */+ status: "pending" | "completed" | "failed" | "rejected" | "indeterminate";result: Record<string, unknown> | null;}diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.tsindex af7aae9b87..90995500e1 100644--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts@@ -672,4 +672,98 @@ describe.sequential("DurablePrpControlPlane", () => {rmSync(root, { recursive: true, force: true });}});++ it("keeps a recovered runner attached when it reports an indeterminate command", async () => {+ const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-indeterminate-"));+ const controlPlane = new DurablePrpControlPlane({+ stateDirectory: root,+ identity,+ expectedRunnerVersion,+ expectedRunnerDigest,+ });+ try {+ await controlPlane.start();+ const journaled = controlPlane.queueCommand(+ "semantic_tool.result",+ { callId: "call-1" },+ "command-tool-1",+ );+ controlPlane.queueCommand(+ "turn.interrupt",+ { turnId: "turn-1" },+ "command-interrupt-1",+ );+ const client = await authenticate(+ controlPlane,+ controlPlane.issueBootstrapTicket(),+ );+ expect(client?.welcome.payload).toMatchObject({+ pendingCommands: [+ expect.objectContaining({ commandId: "command-tool-1" }),+ ],+ });++ // Exactly what runnerd replays after it is killed between journaling a+ // command and confirming its effect. Its durable contract promotes such a+ // command to `indeterminate` so that it is never executed twice.+ const indeterminateResult = {+ protocol: "paperclip.runner",+ version: 1,+ kind: "command_result",+ payload: {+ commandId: journaled.commandId,+ commandType: journaled.type,+ controllerSeq: journaled.controllerSeq,+ status: "indeterminate",+ result: {+ code: "execution_indeterminate",+ message:+ "runner recovered after journaling this command; it will not execute twice",+ },+ },+ };+ sendSecure(client!, indeterminateResult);++ // The authority has to accept that terminal status and hand out the next+ // command. Closing the connection instead strands the runner in a silent+ // reconnect loop that never re-reports its provider identity.+ await expect(receiveSecure(client!)).resolves.toMatchObject({+ kind: "command",+ payload: { commandId: "command-interrupt-1" },+ });+ expect(controlPlane.store.state.commands).toMatchObject([+ { commandId: "command-tool-1", status: "indeterminate" },+ { commandId: "command-interrupt-1", status: "pending" },+ ]);++ // The runner replays its journal on every reconnect, so the same+ // indeterminate result arrives again. It has to be absorbed as a+ // duplicate rather than treated as a conflicting result.+ sendSecure(client!, indeterminateResult);+ await expect(receiveSecure(client!)).resolves.toMatchObject({+ kind: "command",+ payload: { commandId: "command-interrupt-1" },+ });+ expect(controlPlane.store.state.duplicateCommandResults).toBe(1);++ client?.socket.destroy();+ await controlPlane.stop();++ // That result is now persisted, so a control plane restarted over the+ // same directory has to be able to read its own state back.+ const restarted = new DurablePrpControlPlane({+ stateDirectory: root,+ identity,+ expectedRunnerVersion,+ expectedRunnerDigest,+ });+ expect(restarted.store.state.commands).toMatchObject([+ { commandId: "command-tool-1", status: "indeterminate" },+ { commandId: "command-interrupt-1", status: "pending" },+ ]);+ } finally {+ await controlPlane.stop();+ rmSync(root, { recursive: true, force: true });+ }+ });});diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.tsindex e7cb078e5d..408ba27940 100644--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts@@ -380,9 +380,13 @@ function isStoredCoreState(commandTypes.has(command.type) &&typeof command.issuedAt === "string" &&isRecord(command.payload) &&- ["pending", "completed", "failed", "rejected"].includes(- String(command.status),- ) &&+ [+ "pending",+ "completed",+ "failed",+ "rejected",+ "indeterminate",+ ].includes(String(command.status)) &&(command.result === null || isRecord(command.result)),)) {@@ -1641,10 +1645,16 @@ export class DurablePrpControlPlane {return;}const status = result.status;+ // `indeterminate` is terminal too: a runner that crashed between journaling+ // a command and confirming its effect reports it on recovery and will not+ // execute it again. Rejecting it closes the connection, and since the+ // runner replays the same result on every reconnect, the session never+ // recovers.if (status !== "completed" &&status !== "failed" &&- status !== "rejected"+ status !== "rejected" &&+ status !== "indeterminate") {connection.close();return;diff --git a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts b/packages/paperclip-runner/src/control-plane/prp-transport-types.tsindex 6a9920573a..dda8167292 100644--- a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts+++ b/packages/paperclip-runner/src/control-plane/prp-transport-types.ts@@ -14,7 +14,13 @@ export interface DurableRecoveryCoreCommand {type: string;issuedAt: string;payload: Record<string, unknown>;- status: "pending" | "completed" | "failed" | "rejected";+ /**+ * `indeterminate` is the runner's crash-recovery verdict: the command was+ * journaled but its effect was never confirmed, so the runner will not+ * execute it a second time. It is terminal, like the other non-pending+ * statuses.+ */+ status: "pending" | "completed" | "failed" | "rejected" | "indeterminate";result: Record<string, unknown> | null;}