Skip to content

The open-source app everyone uses to manage agents at work

fix(codex): classify mid-turn harness crashes structurally as retriable infra (#10210)

Dotta665408c6d015
diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts
index b2723ca0bd..7284c4dc92 100644
--- a/packages/adapters/codex-local/src/server/execute.ts
+++ b/packages/adapters/codex-local/src/server/execute.ts
@@ -53,6 +53,7 @@ import {
parseCodexJsonl,
classifyCodexAuthRefreshFailure,
extractCodexRetryNotBefore,
+ isCodexHarnessCrash,
isCodexProviderQuotaError,
isCodexTransientUpstreamError,
isCodexUnknownSessionError,
@@ -1283,7 +1284,18 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
stderr: attempt.proc.stderr,
errorMessage: fallbackErrorMessage,
});
- const errorFamily = authRefreshFailure ?? (providerQuota ? "provider_quota" : transientUpstream ? "transient_upstream" : null);
+ const harnessCrash =
+ !authRefreshFailure &&
+ !providerQuota &&
+ !transientUpstream &&
+ isCodexHarnessCrash({
+ exitCode: attempt.proc.exitCode,
+ sawProtocolEvent: attempt.parsed.sawProtocolEvent,
+ sawProtocolTerminalEvent: attempt.parsed.sawProtocolTerminalEvent,
+ });
+ const errorFamily =
+ authRefreshFailure ??
+ (providerQuota ? "provider_quota" : transientUpstream || harnessCrash ? "transient_upstream" : null);
return {
exitCode: attempt.proc.exitCode,
@@ -1300,6 +1312,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
? "provider_quota"
: transientUpstream
? "codex_transient_upstream"
+ : harnessCrash
+ ? "codex_harness_crash"
: null,
errorFamily,
retryNotBefore: transientRetryNotBefore ? transientRetryNotBefore.toISOString() : null,
diff --git a/packages/adapters/codex-local/src/server/index.ts b/packages/adapters/codex-local/src/server/index.ts
index 96bb912145..04be1f1bf1 100644
--- a/packages/adapters/codex-local/src/server/index.ts
+++ b/packages/adapters/codex-local/src/server/index.ts
@@ -23,7 +23,7 @@ export {
} from "./codex-home.js";
export { listCodexSkills, syncCodexSkills } from "./skills.js";
export { testEnvironment } from "./test.js";
-export { parseCodexJsonl, isCodexProviderQuotaError, isCodexTransientUpstreamError, isCodexUnknownSessionError } from "./parse.js";
+export { parseCodexJsonl, isCodexHarnessCrash, isCodexProviderQuotaError, isCodexTransientUpstreamError, isCodexUnknownSessionError } from "./parse.js";
export {
getQuotaWindows,
readCodexAuthInfo,
diff --git a/packages/adapters/codex-local/src/server/parse.test.ts b/packages/adapters/codex-local/src/server/parse.test.ts
index 3d6d2a77e3..52ee27e51d 100644
--- a/packages/adapters/codex-local/src/server/parse.test.ts
+++ b/packages/adapters/codex-local/src/server/parse.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
classifyCodexAuthRefreshFailure,
extractCodexRetryNotBefore,
+ isCodexHarnessCrash,
isCodexProviderQuotaError,
isCodexTransientUpstreamError,
isCodexUnknownSessionError,
@@ -33,6 +34,8 @@ describe("parseCodexJsonl", () => {
},
usageBasis: "per_run",
errorMessage: "resume failed",
+ sawProtocolEvent: true,
+ sawProtocolTerminalEvent: true,
});
});
@@ -67,10 +70,80 @@ describe("parseCodexJsonl", () => {
},
usageBasis: "per_run",
errorMessage: null,
+ sawProtocolEvent: true,
+ sawProtocolTerminalEvent: true,
});
});
});
+describe("isCodexHarnessCrash", () => {
+ const crashedMidTurnStream = [
+ JSON.stringify({ type: "thread.started", thread_id: "thread_123" }),
+ JSON.stringify({
+ type: "item.completed",
+ item: { type: "agent_message", text: "Checking out the issue now." },
+ }),
+ JSON.stringify({ type: "item.started", item: { type: "command_execution" } }),
+ ].join("\n");
+
+ it("classifies a nonzero exit with no protocol-terminal event as a harness crash", () => {
+ const parsed = parseCodexJsonl(crashedMidTurnStream);
+ expect(parsed.sawProtocolEvent).toBe(true);
+ expect(parsed.sawProtocolTerminalEvent).toBe(false);
+ expect(isCodexHarnessCrash({ exitCode: 1, ...parsed })).toBe(true);
+ });
+
+ it("does not classify runs whose turn reached a protocol-terminal event", () => {
+ const failedInProtocol = parseCodexJsonl(
+ [
+ JSON.stringify({ type: "thread.started", thread_id: "thread_123" }),
+ JSON.stringify({ type: "turn.failed", error: { message: "the model rejected the request" } }),
+ ].join("\n"),
+ );
+ expect(isCodexHarnessCrash({ exitCode: 1, ...failedInProtocol })).toBe(false);
+
+ const completedThenFailedExit = parseCodexJsonl(
+ [
+ JSON.stringify({ type: "thread.started", thread_id: "thread_123" }),
+ JSON.stringify({
+ type: "turn.completed",
+ usage: { input_tokens: 10, cached_input_tokens: 2, output_tokens: 4 },
+ }),
+ ].join("\n"),
+ );
+ expect(isCodexHarnessCrash({ exitCode: 1, ...completedThenFailedExit })).toBe(false);
+ });
+
+ it("does not classify successful exits or streams that never spoke the protocol", () => {
+ expect(isCodexHarnessCrash({ exitCode: 0, ...parseCodexJsonl(crashedMidTurnStream) })).toBe(false);
+ expect(isCodexHarnessCrash({ exitCode: null, ...parseCodexJsonl(crashedMidTurnStream) })).toBe(false);
+
+ const neverStarted = parseCodexJsonl("error: unexpected argument '--bogus-flag'\n");
+ expect(neverStarted.sawProtocolEvent).toBe(false);
+ expect(isCodexHarnessCrash({ exitCode: 2, ...neverStarted })).toBe(false);
+ });
+
+ it("stays structural: agent output discussing network errors does not affect classification", () => {
+ const parsed = parseCodexJsonl(
+ [
+ JSON.stringify({ type: "thread.started", thread_id: "thread_123" }),
+ JSON.stringify({
+ type: "item.completed",
+ item: { type: "agent_message", text: "The deploy failed with connection reset by peer; investigating." },
+ }),
+ JSON.stringify({ type: "turn.failed", error: { message: "agent gave up" } }),
+ ].join("\n"),
+ );
+ expect(isCodexHarnessCrash({ exitCode: 1, ...parsed })).toBe(false);
+ expect(
+ isCodexTransientUpstreamError({
+ stdout: "connection reset by peer while running the deploy",
+ errorMessage: "agent gave up",
+ }),
+ ).toBe(false);
+ });
+});
+
describe("classifyCodexAuthRefreshFailure", () => {
it("classifies explicit refresh-token failure messages", () => {
expect(classifyCodexAuthRefreshFailure({ errorMessage: "provider error: refresh_token_reused" })).toBe(
diff --git a/packages/adapters/codex-local/src/server/parse.ts b/packages/adapters/codex-local/src/server/parse.ts
index a69931757f..7698977f40 100644
--- a/packages/adapters/codex-local/src/server/parse.ts
+++ b/packages/adapters/codex-local/src/server/parse.ts
@@ -31,6 +31,8 @@ export function parseCodexJsonl(stdout: string) {
let sessionId: string | null = null;
let finalMessage: string | null = null;
let errorMessage: string | null = null;
+ let sawProtocolEvent = false;
+ let sawProtocolTerminalEvent = false;
const usage = {
inputTokens: 0,
cachedInputTokens: 0,
@@ -45,6 +47,10 @@ export function parseCodexJsonl(stdout: string) {
if (!event) continue;
const type = asString(event.type, "");
+ if (type) sawProtocolEvent = true;
+ if (type === "error" || type === "turn.completed" || type === "turn.failed") {
+ sawProtocolTerminalEvent = true;
+ }
if (type === "thread.started") {
sessionId = asString(event.thread_id, sessionId ?? "") || sessionId;
continue;
@@ -86,9 +92,30 @@ export function parseCodexJsonl(stdout: string) {
usage,
usageBasis: "per_run" as const,
errorMessage,
+ sawProtocolEvent,
+ sawProtocolTerminalEvent,
};
}
+/**
+ * Structural crash detection: the codex CLI can only report an agent-level
+ * failure through the JSONL protocol (an `error` event, `turn.failed`, or a
+ * finished `turn.completed` followed by a nonzero exit). A nonzero exit after
+ * the protocol stream started but before any terminal event means the process
+ * died out from under the agent (MCP transport crash, worker panic, killed
+ * tool server) — retriable infrastructure, not agent behavior. This
+ * deliberately does not match error text: transport failure strings vary, and
+ * stdout/stderr can quote agent output that merely discusses network errors.
+ */
+export function isCodexHarnessCrash(input: {
+ exitCode: number | null;
+ sawProtocolEvent: boolean;
+ sawProtocolTerminalEvent: boolean;
+}): boolean {
+ if ((input.exitCode ?? 0) === 0) return false;
+ return input.sawProtocolEvent && !input.sawProtocolTerminalEvent;
+}
+
export function isCodexUnknownSessionError(stdout: string, stderr: string): boolean {
const haystack = `${stdout}\n${stderr}`
.split(/\r?\n/)
diff --git a/server/src/__tests__/codex-local-execute.test.ts b/server/src/__tests__/codex-local-execute.test.ts
index 96b7cee8ae..a907119970 100644
--- a/server/src/__tests__/codex-local-execute.test.ts
+++ b/server/src/__tests__/codex-local-execute.test.ts
@@ -698,6 +698,68 @@ describe("codex execute", () => {
}
});
+ it("classifies mid-turn harness crashes as retryable transient upstream errors", async () => {
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-harness-crash-"));
+ const workspace = path.join(root, "workspace");
+ const commandPath = path.join(root, "codex");
+ await fs.mkdir(workspace, { recursive: true });
+ // Faithful to the observed MCP transport crash: the protocol stream starts,
+ // then the process dies with only a harness tracing line on stderr — no
+ // protocol-terminal event (error / turn.failed / turn.completed).
+ const script = `#!/usr/bin/env node
+console.log(JSON.stringify({ type: "thread.started", thread_id: "thread-crash-1" }));
+console.log(JSON.stringify({ type: "item.completed", item: { type: "agent_message", text: "Starting the task." } }));
+console.error("2026-07-23T22:58:56.007042Z ERROR rmcp::transport::worker: worker quit with fatal: Transport channel closed, when UnexpectedContentType(Some(\\"text/plain\\"))");
+process.exit(1);
+`;
+ await fs.writeFile(commandPath, script, "utf8");
+ await fs.chmod(commandPath, 0o755);
+
+ const previousHome = process.env.HOME;
+ process.env.HOME = root;
+ await seedSharedCodexAuth(root);
+
+ try {
+ const result = await execute({
+ runId: "run-harness-crash",
+ agent: {
+ id: "agent-1",
+ companyId: "company-1",
+ name: "Codex Coder",
+ adapterType: "codex_local",
+ adapterConfig: { engine: "cli" },
+ },
+ runtime: {
+ sessionId: null,
+ sessionParams: null,
+ sessionDisplayId: null,
+ taskKey: null,
+ },
+ config: {
+ engine: "cli",
+ command: commandPath,
+ cwd: workspace,
+ promptTemplate: "Follow the paperclip heartbeat.",
+ },
+ context: {},
+ authToken: "run-jwt-token",
+ onLog: async () => {},
+ });
+
+ expect(result.exitCode).toBe(1);
+ expect(result.errorCode).toBe("codex_harness_crash");
+ expect(result.errorFamily).toBe("transient_upstream");
+ expect(result.errorMessage).toContain("Transport channel closed");
+ expect(result.sessionId).toBe("thread-crash-1");
+ expect(result.clearSession).toBe(false);
+ expect((result.resultJson as Record<string, unknown>).errorFamily).toBe("transient_upstream");
+ } finally {
+ if (previousHome === undefined) delete process.env.HOME;
+ else process.env.HOME = previousHome;
+ await fs.rm(root, { recursive: true, force: true });
+ }
+ });
+
it("persists retry-not-before metadata for codex provider quota failures", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-usage-limit-"));
const workspace = path.join(root, "workspace");
diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts
index a4f43a1609..9d8a1cbe0e 100644
--- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts
+++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts
@@ -2133,6 +2133,66 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
}
});
+ it("schedules a recovery continuation for codex harness crashes", async () => {
+ const companyId = randomUUID();
+ const agentId = randomUUID();
+ const runId = randomUUID();
+ const now = new Date("2026-07-24T12:00:00.000Z");
+
+ await seedRetryFixture({
+ runId,
+ companyId,
+ agentId,
+ now,
+ errorCode: "codex_harness_crash",
+ errorFamily: "transient_upstream",
+ });
+
+ const scheduled = await heartbeat.scheduleBoundedRetry(runId, {
+ now,
+ random: () => 0.5,
+ });
+
+ expect(scheduled.outcome).toBe("scheduled");
+ if (scheduled.outcome !== "scheduled") return;
+
+ expect(scheduled.run.scheduledRetryAttempt).toBe(1);
+ expect(scheduled.run.scheduledRetryReason).toBe("transient_failure");
+ const contextSnapshot = scheduled.run.contextSnapshot as Record<string, unknown>;
+ expect(contextSnapshot.codexTransientFallbackMode).toBe("same_session");
+ expect(contextSnapshot.retryOfRunId).toBe(runId);
+
+ await cleanupRetryFixture();
+ });
+
+ it("schedules a harness-crash recovery from the error code alone when the result json lost the error family", async () => {
+ const companyId = randomUUID();
+ const agentId = randomUUID();
+ const runId = randomUUID();
+ const now = new Date("2026-07-24T13:00:00.000Z");
+
+ await seedRetryFixture({
+ runId,
+ companyId,
+ agentId,
+ now,
+ errorCode: "codex_harness_crash",
+ errorFamily: null,
+ });
+
+ const scheduled = await heartbeat.scheduleBoundedRetry(runId, {
+ now,
+ random: () => 0.5,
+ });
+
+ expect(scheduled.outcome).toBe("scheduled");
+ if (scheduled.outcome !== "scheduled") return;
+ expect(scheduled.run.scheduledRetryReason).toBe("transient_failure");
+ expect((scheduled.run.contextSnapshot as Record<string, unknown>).codexTransientFallbackMode).toBe("same_session");
+
+ await cleanupRetryFixture();
+ });
+
it("honors codex retry-not-before timestamps when they exceed the default bounded backoff", async () => {
const companyId = randomUUID();
const agentId = randomUUID();
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 4dd31b4730..5a878a32c6 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -436,7 +436,11 @@ function readHeartbeatRunErrorFamily(
if (run.errorCode === "provider_quota") {
return "provider_quota";
}
- if (run.errorCode === "codex_transient_upstream" || run.errorCode === "claude_transient_upstream") {
+ if (
+ run.errorCode === "codex_transient_upstream" ||
+ run.errorCode === "claude_transient_upstream" ||
+ run.errorCode === "codex_harness_crash"
+ ) {
return "transient_upstream";
}
return null;
diff --git a/server/src/services/recovery/service.pause-durability.test.ts b/server/src/services/recovery/service.pause-durability.test.ts
index 5e63043f8e..cd873a16a1 100644
--- a/server/src/services/recovery/service.pause-durability.test.ts
+++ b/server/src/services/recovery/service.pause-durability.test.ts
@@ -26,6 +26,12 @@ describe("pause durability: continuation retry classification", () => {
expect(c.maxAttempts).toBeGreaterThan(0);
});
+ it("codex harness crashes retry as transient infra", () => {
+ const c = classifyContinuationFailure(run("codex_harness_crash"));
+ expect(c.kind).toBe("transient_infra");
+ expect(c.maxAttempts).toBeGreaterThan(0);
+ });
+
it("generic cancelled (non-pause cancellation) is NOT non-retryable", () => {
// non-pause cancellations (the internal invokability cancel and budget pause) keep errorCode "cancelled" -> default branch
expect(classifyContinuationFailure(run("cancelled")).kind).toBe("default");
diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts
index b370f7b711..ed37976de5 100644
--- a/server/src/services/recovery/service.ts
+++ b/server/src/services/recovery/service.ts
@@ -274,6 +274,7 @@ function isTerminalIssueRun(latestRun: LatestIssueRun) {
const TRANSIENT_INFRA_CONTINUATION_ERROR_CODES = new Set<string>([
"adapter_failed",
"codex_transient_upstream",
+ "codex_harness_crash",
"claude_transient_upstream",
"provider_quota",
"timeout",