Skip to content

The open-source app everyone uses to manage agents at work

fix(runner): accept the indeterminate command result after a runner restart (#12646)

Zannis Kalampoukis0cc40037acfd
diff --git a/packages/paperclip-runner/src/contracts/durable-recovery.ts b/packages/paperclip-runner/src/contracts/durable-recovery.ts
index 5cd271b687..b04d41c1e4 100644
--- a/packages/paperclip-runner/src/contracts/durable-recovery.ts
+++ b/packages/paperclip-runner/src/contracts/durable-recovery.ts
@@ -37,7 +37,23 @@ export interface DurableRecoveryProcessedCommand {
commandId: string;
controllerSeq: number;
commandDigest: string;
- status: "completed" | "failed" | "rejected";
+ /**
+ * The runner's command journal is persisted before a command's effect and
+ * re-persisted after recovery, so a recovered trace carries the whole
+ * lifecycle, not just the settled end of it:
+ *
+ * - `pending` — journaled, effect not yet confirmed. Written by
+ * `DurableState::begin_command` and durable from that moment.
+ * - `indeterminate` — the crash-recovery verdict.
+ * `DurableState::reconcile_pending_commands` promotes every `pending`
+ * entry on load and saves the state back, so the command is never
+ * executed twice. Terminal.
+ *
+ * Both are values a consumer can read off `processedCommands`; a union that
+ * omits them tells the compiler a state the runner routinely writes is
+ * impossible.
+ */
+ status: "pending" | "completed" | "failed" | "rejected" | "indeterminate";
logicalEffectCount: number;
result: Record<string, unknown>;
}
@@ -79,7 +95,13 @@ export interface DurableRecoveryCoreCommand {
type: string;
issuedAt: string;
payload: Record<string, unknown>;
- status: "pending" | "completed" | "failed" | "rejected";
+ /**
+ * `indeterminate` is the runner's crash-recovery verdict: the command was
+ * journaled but its effect was never confirmed, so the runner will not
+ * execute it a second time. It is terminal, like the other non-pending
+ * statuses.
+ */
+ status: "pending" | "completed" | "failed" | "rejected" | "indeterminate";
result: Record<string, unknown> | null;
}
diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts
index af7aae9b87..90995500e1 100644
--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts
+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts
@@ -672,4 +672,98 @@ describe.sequential("DurablePrpControlPlane", () => {
rmSync(root, { recursive: true, force: true });
}
});
+
+ it("keeps a recovered runner attached when it reports an indeterminate command", async () => {
+ const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-indeterminate-"));
+ const controlPlane = new DurablePrpControlPlane({
+ stateDirectory: root,
+ identity,
+ expectedRunnerVersion,
+ expectedRunnerDigest,
+ });
+ try {
+ await controlPlane.start();
+ const journaled = controlPlane.queueCommand(
+ "semantic_tool.result",
+ { callId: "call-1" },
+ "command-tool-1",
+ );
+ controlPlane.queueCommand(
+ "turn.interrupt",
+ { turnId: "turn-1" },
+ "command-interrupt-1",
+ );
+ const client = await authenticate(
+ controlPlane,
+ controlPlane.issueBootstrapTicket(),
+ );
+ expect(client?.welcome.payload).toMatchObject({
+ pendingCommands: [
+ expect.objectContaining({ commandId: "command-tool-1" }),
+ ],
+ });
+
+ // Exactly what runnerd replays after it is killed between journaling a
+ // command and confirming its effect. Its durable contract promotes such a
+ // command to `indeterminate` so that it is never executed twice.
+ const indeterminateResult = {
+ protocol: "paperclip.runner",
+ version: 1,
+ kind: "command_result",
+ payload: {
+ commandId: journaled.commandId,
+ commandType: journaled.type,
+ controllerSeq: journaled.controllerSeq,
+ status: "indeterminate",
+ result: {
+ code: "execution_indeterminate",
+ message:
+ "runner recovered after journaling this command; it will not execute twice",
+ },
+ },
+ };
+ sendSecure(client!, indeterminateResult);
+
+ // The authority has to accept that terminal status and hand out the next
+ // command. Closing the connection instead strands the runner in a silent
+ // reconnect loop that never re-reports its provider identity.
+ await expect(receiveSecure(client!)).resolves.toMatchObject({
+ kind: "command",
+ payload: { commandId: "command-interrupt-1" },
+ });
+ expect(controlPlane.store.state.commands).toMatchObject([
+ { commandId: "command-tool-1", status: "indeterminate" },
+ { commandId: "command-interrupt-1", status: "pending" },
+ ]);
+
+ // The runner replays its journal on every reconnect, so the same
+ // indeterminate result arrives again. It has to be absorbed as a
+ // duplicate rather than treated as a conflicting result.
+ sendSecure(client!, indeterminateResult);
+ await expect(receiveSecure(client!)).resolves.toMatchObject({
+ kind: "command",
+ payload: { commandId: "command-interrupt-1" },
+ });
+ expect(controlPlane.store.state.duplicateCommandResults).toBe(1);
+
+ client?.socket.destroy();
+ await controlPlane.stop();
+
+ // That result is now persisted, so a control plane restarted over the
+ // same directory has to be able to read its own state back.
+ const restarted = new DurablePrpControlPlane({
+ stateDirectory: root,
+ identity,
+ expectedRunnerVersion,
+ expectedRunnerDigest,
+ });
+ expect(restarted.store.state.commands).toMatchObject([
+ { commandId: "command-tool-1", status: "indeterminate" },
+ { commandId: "command-interrupt-1", status: "pending" },
+ ]);
+ } finally {
+ await controlPlane.stop();
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
});
diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
index e7cb078e5d..408ba27940 100644
--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
@@ -380,9 +380,13 @@ function isStoredCoreState(
commandTypes.has(command.type) &&
typeof command.issuedAt === "string" &&
isRecord(command.payload) &&
- ["pending", "completed", "failed", "rejected"].includes(
- String(command.status),
- ) &&
+ [
+ "pending",
+ "completed",
+ "failed",
+ "rejected",
+ "indeterminate",
+ ].includes(String(command.status)) &&
(command.result === null || isRecord(command.result)),
)
) {
@@ -1641,10 +1645,16 @@ export class DurablePrpControlPlane {
return;
}
const status = result.status;
+ // `indeterminate` is terminal too: a runner that crashed between journaling
+ // a command and confirming its effect reports it on recovery and will not
+ // execute it again. Rejecting it closes the connection, and since the
+ // runner replays the same result on every reconnect, the session never
+ // recovers.
if (
status !== "completed" &&
status !== "failed" &&
- status !== "rejected"
+ status !== "rejected" &&
+ status !== "indeterminate"
) {
connection.close();
return;
diff --git a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts b/packages/paperclip-runner/src/control-plane/prp-transport-types.ts
index 6a9920573a..dda8167292 100644
--- a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts
+++ b/packages/paperclip-runner/src/control-plane/prp-transport-types.ts
@@ -14,7 +14,13 @@ export interface DurableRecoveryCoreCommand {
type: string;
issuedAt: string;
payload: Record<string, unknown>;
- status: "pending" | "completed" | "failed" | "rejected";
+ /**
+ * `indeterminate` is the runner's crash-recovery verdict: the command was
+ * journaled but its effect was never confirmed, so the runner will not
+ * execute it a second time. It is terminal, like the other non-pending
+ * statuses.
+ */
+ status: "pending" | "completed" | "failed" | "rejected" | "indeterminate";
result: Record<string, unknown> | null;
}