Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tasks): retry codex turns that hit a flex capacity refusal (#119190)

posthog[bot]172974f3fe4b
diff --git a/packages/agent/packages/agent/src/adapters/error-classification.test.ts b/packages/agent/packages/agent/src/adapters/error-classification.test.ts
index 5aeafe437a8f..a51a9153d81b 100644
--- a/packages/agent/packages/agent/src/adapters/error-classification.test.ts
+++ b/packages/agent/packages/agent/src/adapters/error-classification.test.ts
@@ -46,6 +46,8 @@ describe("classifyAgentError", () => {
"upstream_provider_failure",
],
["unexpected status 403 Forbidden: needs a paid plan", "agent_error"],
+ ["Flex capacity unavailable.", "upstream_provider_failure"],
+ ["Internal error: Flex capacity unavailable.", "upstream_provider_failure"],
[
"[ede_diagnostic] result_type=user last_content_type=n/a stop_reason=null",
"turn_ended_without_response",
diff --git a/packages/agent/packages/agent/src/adapters/error-classification.ts b/packages/agent/packages/agent/src/adapters/error-classification.ts
index 7fcbaf6b8a69..01fa7b281b72 100644
--- a/packages/agent/packages/agent/src/adapters/error-classification.ts
+++ b/packages/agent/packages/agent/src/adapters/error-classification.ts
@@ -34,6 +34,9 @@ const UPSTREAM_PROVIDER_ERROR_STATUS_PATTERN = /API Error:\s*(?:429|5\d\d)\b/i;
// "unexpected status <code> <reason>: <body>" instead of the "API Error:" wording.
const CODEX_PROVIDER_ERROR_STATUS_PATTERN =
/unexpected status\s*(?:429|5\d\d)\b/i;
+// OpenAI refuses a flex-tier request when it has no flex capacity. The
+// refusal carries no HTTP status in the text, but it is transient.
+const UPSTREAM_CAPACITY_UNAVAILABLE_PATTERN = /\bcapacity unavailable\b/i;
const SANDBOX_TASK_SPEND_LIMIT_PATTERN =
/This agent run reached its spend limit/i;
const TURN_ENDED_WITHOUT_RESPONSE_PATTERN =
@@ -87,7 +90,8 @@ export function classifyAgentError(
}
if (
UPSTREAM_PROVIDER_ERROR_STATUS_PATTERN.test(text) ||
- CODEX_PROVIDER_ERROR_STATUS_PATTERN.test(text)
+ CODEX_PROVIDER_ERROR_STATUS_PATTERN.test(text) ||
+ UPSTREAM_CAPACITY_UNAVAILABLE_PATTERN.test(text)
) {
return "upstream_provider_failure";
}
diff --git a/packages/agent/packages/agent/src/server/agent-server.test.ts b/packages/agent/packages/agent/src/server/agent-server.test.ts
index 8f87666f1e42..3b353975e8f8 100644
--- a/packages/agent/packages/agent/src/server/agent-server.test.ts
+++ b/packages/agent/packages/agent/src/server/agent-server.test.ts
@@ -2445,7 +2445,7 @@ describe("AgentServer HTTP Mode", () => {
sessionId: "acp-1",
prompt: [{ type: "text", text: "do the task" }],
});
- await vi.advanceTimersByTimeAsync(10_000);
+ await vi.advanceTimersByTimeAsync(15_000);
await expect(resultPromise).resolves.toEqual({
stopReason: "end_turn",
@@ -2622,7 +2622,7 @@ describe("AgentServer HTTP Mode", () => {
}),
});
- await vi.advanceTimersByTimeAsync(10_000);
+ await vi.advanceTimersByTimeAsync(15_000);
await assertion;
expect(testServer.posthogAPI.updateTaskRun).not.toHaveBeenCalled();
} finally {
@@ -2771,7 +2771,7 @@ describe("AgentServer HTTP Mode", () => {
const assertion = expect(resultPromise).rejects.toThrow(
"API Error: terminated",
);
- await vi.advanceTimersByTimeAsync(10_000);
+ await vi.advanceTimersByTimeAsync(15_000);
await assertion;
expect(prompt).toHaveBeenCalledTimes(3);
diff --git a/packages/agent/packages/agent/src/server/agent-server.ts b/packages/agent/packages/agent/src/server/agent-server.ts
index c35dcf376853..523accfa9886 100644
--- a/packages/agent/packages/agent/src/server/agent-server.ts
+++ b/packages/agent/packages/agent/src/server/agent-server.ts
@@ -228,6 +228,13 @@ export const SSE_KEEPALIVE_INTERVAL_MS = 25_000;
// cut once, without letting a hard upstream outage loop forever.
const MAX_UPSTREAM_TURN_RETRIES = 2;
const UPSTREAM_TURN_RETRY_DELAY_MS = 5_000;
+
+// The delay doubles per attempt. Half of it is random, so many runs that hit
+// the same capacity gap do not retry at the same moment.
+export function upstreamTurnRetryDelayMs(attempt: number): number {
+ const ceiling = UPSTREAM_TURN_RETRY_DELAY_MS * 2 ** (attempt - 1);
+ return ceiling / 2 + Math.random() * (ceiling / 2);
+}
const PENDING_ARTIFACT_MAX_ATTEMPTS = 4;
const PENDING_ARTIFACT_RETRY_DELAY_MS = 500;
@@ -2965,7 +2972,7 @@ export class AgentServer {
},
);
await new Promise((resolve) =>
- setTimeout(resolve, UPSTREAM_TURN_RETRY_DELAY_MS),
+ setTimeout(resolve, upstreamTurnRetryDelayMs(retries)),
);
if (this.session !== originatingSession) {
throw new Error(
diff --git a/packages/agent/packages/agent/src/server/question-relay.test.ts b/packages/agent/packages/agent/src/server/question-relay.test.ts
index 5458460ece66..94ddb16ce334 100644
--- a/packages/agent/packages/agent/src/server/question-relay.test.ts
+++ b/packages/agent/packages/agent/src/server/question-relay.test.ts
@@ -1069,7 +1069,7 @@ describe("Question relay", () => {
vi.useFakeTimers();
try {
const sendPromise = startInitialTaskMessage(server, TEST_PAYLOAD);
- await vi.advanceTimersByTimeAsync(10_000);
+ await vi.advanceTimersByTimeAsync(15_000);
await sendPromise;
} finally {
vi.useRealTimers();