A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
retry provider errors before surfacing them (#347)
kt645f5147e617
diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.mdindex bf87c1b681..ee6a729b24 100644--- a/packages/agent/CHANGELOG.md+++ b/packages/agent/CHANGELOG.md@@ -2,6 +2,8 @@## [Unreleased]+- Fixed run-lifecycle failures carrying no diagnostics, so hosts can distinguish local failures from provider failures.+## [0.2.6] - 2026-07-06## [0.2.5] - 2026-07-06diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.tsindex bc75d86c54..141eca0e9c 100644--- a/packages/agent/src/agent.ts+++ b/packages/agent/src/agent.ts@@ -1,4 +1,5 @@import {+ createAssistantMessageDiagnostic,type ImageContent,type Message,type Model,@@ -502,6 +503,9 @@ export class Agent {usage: EMPTY_USAGE,stopReason: aborted ? "aborted" : "error",errorMessage: error instanceof Error ? error.message : String(error),+ diagnostics: aborted+ ? undefined+ : [createAssistantMessageDiagnostic("agent_lifecycle_failure", error, { source: "run_with_lifecycle" })],timestamp: Date.now(),} satisfies AgentMessage;this._state.messages.push(failureMessage);diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.mdindex 3c7a921a1c..a59b23f22e 100644--- a/packages/coding-agent/CHANGELOG.md+++ b/packages/coding-agent/CHANGELOG.md@@ -10,6 +10,7 @@- Fixed provider auth failures leaving stale credentials shown as connected in `/login` ([ENG-4491](https://linear.app/primeintellect/issue/ENG-4491/mark-provider-stale-after-repeated-401s)).- Fixed typing into the prompt after highlighting an inline subagent ([ENG-4494](https://linear.app/primeintellect/issue/ENG-4494/allow-typing-after-highlighting-a-subagent)).- Fixed session-targeted heartbeat jobs staying scheduled after sessions are killed or saved sessions are deleted ([#332](https://github.com/PrimeIntellect-ai/prime-agent/pull/332)).+- Fixed provider errors being surfaced instead of retried within the retry budget ([ENG-4503](https://linear.app/primeintellect/issue/ENG-4503/restarting-old-session-returns-empty-model-response)).## [0.2.6] - 2026-07-06diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.tsindex af46064b4d..37d03ecbf6 100644--- a/packages/coding-agent/src/core/agent-session.ts+++ b/packages/coding-agent/src/core/agent-session.ts@@ -1871,19 +1871,25 @@ export class AgentSession {// Check for retryable errors first (overloaded, rate limit, server errors)const concreteAuthFailure = this._isConcreteProviderAuthFailure(msg);- if (this._isRetryableError(msg) || concreteAuthFailure) {- if (concreteAuthFailure) {+ const retryConcreteAuthFailure =+ concreteAuthFailure && !this._isStructuredPermanentProviderRetryExhausted(msg);+ if (this._isRetryableError(msg) || retryConcreteAuthFailure) {+ if (retryConcreteAuthFailure) {this._captureRetryAuthFailureSource(msg);}const didRetry = await this._handleRetryableError(msg, {- markAuthStaleOnFailure: concreteAuthFailure,- authSourceTokens: concreteAuthFailure ? this._retryAuthFailureSources : undefined,+ markAuthStaleOnFailure: retryConcreteAuthFailure,+ authSourceTokens: retryConcreteAuthFailure ? this._retryAuthFailureSources : undefined,});if (didRetry) return; // Retry was initiated, don't proceed to compaction}- this._resolveRetry();const compactionWillRetry = await this._checkCompaction(msg);+ if (compactionWillRetry && this._retryAttempt > 0) {+ return;+ }+ this._finishActiveRetryWithFailure(msg);+ this._resolveRetry();if (!compactionWillRetry) {this._finishGoalForTerminalAssistantMessage(msg);this._scheduleAutoRefineAfterAgentEnd();@@ -3222,8 +3228,8 @@ export class AgentSession {* @returns Object with steering and followUp arrays*/clearQueue(): { steering: string[]; followUp: string[] } {- const steering = this._steeringMessages.map(queuedMessagePreview);- const followUp = this._followUpMessages.map(queuedMessagePreview);+ const steering = this._steeringMessages.map((message) => message.text);+ const followUp = this._followUpMessages.map((message) => message.text);this._rejectQueuedAgentMessageDeliveries(new Error("Queued agent message was cleared before delivery."));this._steeringMessages = [];this._followUpMessages = [];@@ -5626,34 +5632,64 @@ export class AgentSession {const contextWindow = this.model?.contextWindow ?? 0;if (isContextOverflow(message, contextWindow)) return false;- // Structured classification from the provider (stream-failure.ts) beats- // message-text matching. Safety/malformed/unknown kinds fall through to- // the regex, which distinguishes transient content_filter cases.- const failure = message.diagnostics?.find((d) => d.type === "provider_stream_failure");- const kind = failure?.details?.kind;- if (kind === "overloaded" || kind === "rate_limit" || kind === "server_error") return true;- if (kind === "refusal" || kind === "auth" || kind === "invalid_request") return false;-- const err = message.errorMessage;- // Match: overloaded_error, provider returned error, rate limit, 429, 500, 502, 503, 504, service unavailable, network/connection errors (including connection lost), WebSocket transport closes/errors, fetch failed, request ended without sending chunks, HTTP/2 closed before response, terminated, retry delay exceeded, transient content_filter finish_reason, provider policy-flag prose, and prose-form transient 5xx ("an error occurred while processing your request. you can retry")- return /overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|http2 request did not get a response|timed? out|timeout|terminated|retry delay|content.?filter|flagged .*(cybersecurity|usage polic|violat)|error occurred while processing|you can retry/i.test(- err,- );+ if (this._isFauxProviderQueueExhausted(message)) {+ return false;+ }++ if (this._isAgentLifecycleFailure(message)) {+ return false;+ }++ if (this._isStructuredPermanentProviderRetryExhausted(message)) {+ return false;+ }++ return true;}- private _getProviderStreamFailureAuthStatus(message: AssistantMessage): number | undefined {+ private _isFauxProviderQueueExhausted(message: AssistantMessage): boolean {+ return message.provider === "faux" && message.errorMessage === "No more faux responses queued";+ }++ private _isAgentLifecycleFailure(message: AssistantMessage): boolean {+ return message.diagnostics?.some((diagnostic) => diagnostic.type === "agent_lifecycle_failure") ?? false;+ }++ private _getProviderStreamFailureDetails(message: AssistantMessage): Record<string, unknown> | undefined {const failure = message.diagnostics?.find((diagnostic) => diagnostic.type === "provider_stream_failure");const details = failure?.details;if (!details || typeof details !== "object") {return undefined;}+ return details;+ }- const kind = "kind" in details ? details.kind : undefined;+ private _getProviderStreamFailureKind(message: AssistantMessage): string | undefined {+ const kind = this._getProviderStreamFailureDetails(message)?.kind;+ return typeof kind === "string" ? kind : undefined;+ }++ private _isStructuredPermanentProviderFailure(message: AssistantMessage): boolean {+ const kind = this._getProviderStreamFailureKind(message);+ return kind === "auth" || kind === "invalid_request" || kind === "refusal";+ }++ private _isStructuredPermanentProviderRetryExhausted(message: AssistantMessage): boolean {+ return this._retryAttempt > 0 && this._isStructuredPermanentProviderFailure(message);+ }++ private _getProviderStreamFailureAuthStatus(message: AssistantMessage): number | undefined {+ const details = this._getProviderStreamFailureDetails(message);+ if (!details) {+ return undefined;+ }++ const kind = details.kind;if (kind !== "auth") {return undefined;}- const status = "status" in details ? details.status : undefined;+ const status = details.status;if (typeof status === "number") {return status;}@@ -5735,6 +5771,21 @@ export class AgentSession {return false;}+ private _finishActiveRetryWithFailure(message: AssistantMessage): void {+ if (this._retryAttempt === 0) {+ return;+ }+ this._markProviderAuthStaleForRetryFailure(message);+ this._emit({+ type: "auto_retry_end",+ success: false,+ attempt: this._retryAttempt,+ finalError: message.errorMessage,+ });+ this._retryAttempt = 0;+ this._retryAuthFailureSources = [];+ }+/*** Handle retryable errors with exponential backoff.* @returns true if retry was initiated, false if max retries exceeded or disabled@@ -5832,6 +5883,17 @@ export class AgentSession {this._retryAbortController.abort();return;}+ if (this._retryAttempt > 0) {+ this._autoCompactionAbortController?.abort();+ this._cancelPostCompactionContinue();+ this._emit({+ type: "auto_retry_end",+ success: false,+ attempt: this._retryAttempt,+ finalError: "Retry cancelled",+ });+ this._retryAttempt = 0;+ }this._retryAuthFailureSources = [];this._resolveRetry();}diff --git a/packages/coding-agent/test/agent-session-retry.test.ts b/packages/coding-agent/test/agent-session-retry.test.tsindex 67717dc180..223c87adf1 100644--- a/packages/coding-agent/test/agent-session-retry.test.ts+++ b/packages/coding-agent/test/agent-session-retry.test.ts@@ -363,9 +363,11 @@ describe("AgentSession retry", () => {expect(callCount).toBe(4);});- // Transient provider stream artifacts the classifier previously missed (ENG-4390).+ // Provider errors are retried within the bounded retry budget (ENG-4390, ENG-4503).const transientProviderErrors: Array<[string, string]> = [["content_filter finish_reason", "Provider finish_reason: content_filter"],+ ["empty provider response", "Provider returned an empty response"],+ ["plain invalid API key", "invalid_api_key"],["cybersecurity policy flag", "Your request was flagged for cybersecurity risk and cannot be processed."],["usage policy flag", "flagged as potentially violating our usage policy"],["prose-form transient 5xx", "An error occurred while processing your request. You can retry your request."],diff --git a/packages/coding-agent/test/suite/agent-session-retry-events.test.ts b/packages/coding-agent/test/suite/agent-session-retry-events.test.tsindex a6d61b3d67..e14901b296 100644--- a/packages/coding-agent/test/suite/agent-session-retry-events.test.ts+++ b/packages/coding-agent/test/suite/agent-session-retry-events.test.ts@@ -1,5 +1,5 @@-import type { AgentTool } from "@earendil-works/pi-agent-core";-import { fauxAssistantMessage, fauxThinking, fauxToolCall } from "@earendil-works/pi-ai";+import type { AgentEvent, AgentTool } from "@earendil-works/pi-agent-core";+import { type AssistantMessage, fauxAssistantMessage, fauxThinking, fauxToolCall } from "@earendil-works/pi-ai";import { Type } from "typebox";import { afterEach, describe, expect, it } from "vitest";import { createHarness, type Harness } from "./harness.js";@@ -21,6 +21,34 @@ function normalizeEventOrder(events: Harness["events"]): string[] {return normalized;}+function structuredProviderFailure(kind: "auth" | "invalid_request" | "refusal"): AssistantMessage {+ return {+ ...fauxAssistantMessage("", {+ stopReason: "error",+ errorMessage: `provider ${kind} failure`,+ }),+ diagnostics: [+ {+ type: "provider_stream_failure",+ timestamp: Date.now(),+ details: { kind },+ },+ ],+ };+}++type SessionRetryCompactionInternals = {+ _retryAttempt: number;+ _retryPromise: Promise<void> | undefined;+ _retryResolve: (() => void) | undefined;+ _autoCompactionAbortController: AbortController | undefined;+ _postCompactionContinuationScheduled: boolean;+ _processAgentEvent: (event: AgentEvent) => Promise<void>;+ _checkCompaction: (message: AssistantMessage) => Promise<boolean>;+ _schedulePostCompactionContinue: () => void;+ _cancelPostCompactionContinue: () => void;+};+describe("AgentSession retry and event characterization", () => {const harnesses: Harness[] = [];@@ -157,15 +185,134 @@ describe("AgentSession retry and event characterization", () => {expect(harness.eventsOfType("auto_retry_start")).toEqual([]);});- it("does not retry non-retryable errors", async () => {+ it("does not retry faux provider queue exhaustion", async () => {+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });+ harnesses.push(harness);++ await harness.session.prompt("test");++ expect(harness.faux.state.callCount).toBe(1);+ expect(harness.eventsOfType("auto_retry_start")).toEqual([]);+ expect(harness.session.isRetrying).toBe(false);+ });++ it("does not retry local agent lifecycle listener failures", async () => {const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });harnesses.push(harness);- harness.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "invalid_api_key" })]);+ let unsubscribe = () => {};+ unsubscribe = harness.session.agent.subscribe((event) => {+ if (event.type === "message_start" && event.message.role === "assistant") {+ unsubscribe();+ throw new Error("local listener failed");+ }+ });+ harness.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("retry should not happen")]);await harness.session.prompt("test");+ const lastMessage = harness.session.messages.at(-1);expect(harness.faux.state.callCount).toBe(1);expect(harness.eventsOfType("auto_retry_start")).toEqual([]);+ expect(harness.session.isRetrying).toBe(false);+ expect(lastMessage?.role).toBe("assistant");+ if (lastMessage?.role === "assistant") {+ expect(lastMessage.diagnostics?.some((diagnostic) => diagnostic.type === "agent_lifecycle_failure")).toBe(+ true,+ );+ }+ });++ it("retries generic provider errors", async () => {+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });+ harnesses.push(harness);+ harness.setResponses([+ fauxAssistantMessage("", { stopReason: "error", errorMessage: "invalid_api_key" }),+ fauxAssistantMessage("recovered"),+ ]);++ await harness.session.prompt("test");++ expect(harness.faux.state.callCount).toBe(2);+ expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1]);+ expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([true]);+ });++ for (const kind of ["auth", "invalid_request", "refusal"] as const) {+ it(`retries structured permanent provider ${kind} failures once`, async () => {+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });+ harnesses.push(harness);+ harness.setResponses([+ structuredProviderFailure(kind),+ structuredProviderFailure(kind),+ fauxAssistantMessage("unused"),+ ]);++ await harness.session.prompt("test");++ expect(harness.faux.state.callCount).toBe(2);+ expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1]);+ expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([false]);+ expect(harness.session.isRetrying).toBe(false);+ });+ }++ it("keeps retry state active when overflow compaction will retry", async () => {+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });+ harnesses.push(harness);+ const internals = harness.session as unknown as SessionRetryCompactionInternals;+ const originalCheckCompaction = internals._checkCompaction.bind(harness.session);+ const overflowMessage = fauxAssistantMessage("", {+ stopReason: "error",+ errorMessage: "prompt is too long",+ });+ internals._retryAttempt = 1;+ internals._retryPromise = new Promise<void>((resolve) => {+ internals._retryResolve = resolve;+ });+ internals._checkCompaction = async () => true;++ try {+ await internals._processAgentEvent({ type: "agent_end", messages: [overflowMessage] } as AgentEvent);++ expect(internals._retryAttempt).toBe(1);+ expect(harness.session.isRetrying).toBe(true);+ expect(harness.eventsOfType("auto_retry_end")).toEqual([]);+ } finally {+ internals._checkCompaction = originalCheckCompaction;+ harness.session.abortRetry();+ }+ });++ it("cancels overflow-compaction retry continuation when abortRetry is called", async () => {+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });+ harnesses.push(harness);+ const internals = harness.session as unknown as SessionRetryCompactionInternals;+ const compactionAbortController = new AbortController();+ internals._retryAttempt = 1;+ internals._retryPromise = new Promise<void>((resolve) => {+ internals._retryResolve = resolve;+ });+ internals._autoCompactionAbortController = compactionAbortController;+ internals._schedulePostCompactionContinue();++ try {+ expect(internals._postCompactionContinuationScheduled).toBe(true);++ harness.session.abortRetry();++ expect(compactionAbortController.signal.aborted).toBe(true);+ expect(internals._postCompactionContinuationScheduled).toBe(false);+ expect(internals._retryAttempt).toBe(0);+ expect(harness.session.isRetrying).toBe(false);+ expect(harness.eventsOfType("auto_retry_end").at(-1)).toMatchObject({+ success: false,+ attempt: 1,+ finalError: "Retry cancelled",+ });+ } finally {+ internals._autoCompactionAbortController = undefined;+ internals._cancelPostCompactionContinue();+ }});it("cancels retry sleep when abortRetry is called", async () => {@@ -221,6 +368,7 @@ describe("AgentSession retry and event characterization", () => {expect(harness.faux.state.callCount).toBe(3);expect(toolRuns).toEqual(["hello"]);expect(harness.session.isStreaming).toBe(false);+ harness.appendResponses([fauxAssistantMessage("follow-up answer")]);await harness.session.prompt("follow-up");expect(harness.faux.state.callCount).toBe(4);});@@ -347,7 +495,7 @@ describe("AgentSession retry and event characterization", () => {});it("emits agent_end for error responses", async () => {- const harness = await createHarness();+ const harness = await createHarness({ settings: { retry: { enabled: false } } });harnesses.push(harness);harness.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "broken" })]);diff --git a/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.ts b/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.tsindex 062a70eb13..1aed13d7e1 100644--- a/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.ts+++ b/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.ts@@ -258,7 +258,7 @@ describe("ENG-4482 heartbeat injected prompt UI", () => {).toBe(true);});- it("returns labeled previews when clearing queued heartbeat prompts", async () => {+ it("returns raw text when clearing queued heartbeat prompts while previews remain labeled", async () => {let releaseToolExecution: (() => void) | undefined;const toolRelease = new Promise<void>((resolve) => {releaseToolExecution = resolve;@@ -278,7 +278,8 @@ describe("ENG-4482 heartbeat injected prompt UI", () => {};const harness = await createHarness({ tools: [waitTool] });harnesses.push(harness);- const heartbeatPreview = "Heartbeat prompt: Check whether the long-running task needs another step.";+ const heartbeatText = "Check whether the long-running task needs another step.";+ const heartbeatPreview = `Heartbeat prompt: ${heartbeatText}`;harness.setResponses([fauxAssistantMessage(fauxToolCall("wait", {}), { stopReason: "toolUse" }),fauxAssistantMessage("original turn complete"),@@ -297,7 +298,7 @@ describe("ENG-4482 heartbeat injected prompt UI", () => {await harness.session.promptHeartbeat(createHeartbeat(), { streamingBehavior: "followUp" });expect(harness.session.getFollowUpMessagePreviews()).toEqual([heartbeatPreview]);- expect(harness.session.clearQueue()).toEqual({ steering: [], followUp: [heartbeatPreview] });+ expect(harness.session.clearQueue()).toEqual({ steering: [], followUp: [heartbeatText] });releaseToolExecution?.();await promptPromise;diff --git a/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.ts b/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.tsindex 6c2e474fe1..48fc2a7ace 100644--- a/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.ts+++ b/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.ts@@ -51,7 +51,7 @@ describe("issue #4491 provider stale after repeated 401", () => {}});- it("retries concrete provider auth failures, then marks current auth stale", async () => {+ it("retries structured provider auth failures once, then marks current auth stale", async () => {const harness = await createHarness({settings: { retry: { enabled: true, maxRetries: 2, baseDelayMs: 1 } },});@@ -60,8 +60,8 @@ describe("issue #4491 provider stale after repeated 401", () => {await harness.session.prompt("hello");- expect(harness.faux.state.callCount).toBe(3);- expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1, 2]);+ expect(harness.faux.state.callCount).toBe(2);+ expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1]);expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([false]);expect(harness.eventsOfType("auth_stale")).toHaveLength(1);@@ -125,6 +125,24 @@ describe("issue #4491 provider stale after repeated 401", () => {harness.session.abortRetry();});+ it("creates retry promises for exhausted structured auth failures so cleanup is awaited", async () => {+ const harness = await createHarness({+ settings: { retry: { enabled: true, maxRetries: 2, baseDelayMs: 1 } },+ });+ harnesses.push(harness);+ const event = { type: "agent_end", messages: [provider401Message()] } as AgentEvent;+ const session = harness.session as unknown as {+ _retryAttempt: number;+ _createRetryPromiseForAgentEnd(event: AgentEvent): void;+ };+ session._retryAttempt = 1;++ session._createRetryPromiseForAgentEnd(event);++ expect(harness.session.isRetrying).toBe(true);+ harness.session.abortRetry();+ });+it("marks captured auth failures stale when retry backoff is cancelled", async () => {const harness = await createHarness({settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 100 } },@@ -186,7 +204,7 @@ describe("issue #4491 provider stale after repeated 401", () => {settings: { retry: { enabled: true, maxRetries: 2, baseDelayMs: 1 } },});harnesses.push(harness);- harness.setResponses([provider401Message(), provider401Message(), provider500Message()]);+ harness.setResponses([provider401Message(), provider500Message(), provider500Message()]);await harness.session.prompt("hello");