Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

retry provider errors before surfacing them (#347)

kt645f5147e617
diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md
index bf87c1b681..ee6a729b24 100644
--- a/packages/agent/CHANGELOG.md
+++ b/packages/agent/CHANGELOG.md
@@ -2,6 +2,8 @@
## [Unreleased]
+- Fixed run-lifecycle failures carrying no diagnostics, so hosts can distinguish local failures from provider failures.
+
## [0.2.6] - 2026-07-06
## [0.2.5] - 2026-07-06
diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts
index bc75d86c54..141eca0e9c 100644
--- a/packages/agent/src/agent.ts
+++ b/packages/agent/src/agent.ts
@@ -1,4 +1,5 @@
import {
+ createAssistantMessageDiagnostic,
type ImageContent,
type Message,
type Model,
@@ -502,6 +503,9 @@ export class Agent {
usage: EMPTY_USAGE,
stopReason: aborted ? "aborted" : "error",
errorMessage: error instanceof Error ? error.message : String(error),
+ diagnostics: aborted
+ ? undefined
+ : [createAssistantMessageDiagnostic("agent_lifecycle_failure", error, { source: "run_with_lifecycle" })],
timestamp: Date.now(),
} satisfies AgentMessage;
this._state.messages.push(failureMessage);
diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md
index 3c7a921a1c..a59b23f22e 100644
--- a/packages/coding-agent/CHANGELOG.md
+++ b/packages/coding-agent/CHANGELOG.md
@@ -10,6 +10,7 @@
- Fixed provider auth failures leaving stale credentials shown as connected in `/login` ([ENG-4491](https://linear.app/primeintellect/issue/ENG-4491/mark-provider-stale-after-repeated-401s)).
- Fixed typing into the prompt after highlighting an inline subagent ([ENG-4494](https://linear.app/primeintellect/issue/ENG-4494/allow-typing-after-highlighting-a-subagent)).
- Fixed session-targeted heartbeat jobs staying scheduled after sessions are killed or saved sessions are deleted ([#332](https://github.com/PrimeIntellect-ai/prime-agent/pull/332)).
+- Fixed provider errors being surfaced instead of retried within the retry budget ([ENG-4503](https://linear.app/primeintellect/issue/ENG-4503/restarting-old-session-returns-empty-model-response)).
## [0.2.6] - 2026-07-06
diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts
index af46064b4d..37d03ecbf6 100644
--- a/packages/coding-agent/src/core/agent-session.ts
+++ b/packages/coding-agent/src/core/agent-session.ts
@@ -1871,19 +1871,25 @@ export class AgentSession {
// Check for retryable errors first (overloaded, rate limit, server errors)
const concreteAuthFailure = this._isConcreteProviderAuthFailure(msg);
- if (this._isRetryableError(msg) || concreteAuthFailure) {
- if (concreteAuthFailure) {
+ const retryConcreteAuthFailure =
+ concreteAuthFailure && !this._isStructuredPermanentProviderRetryExhausted(msg);
+ if (this._isRetryableError(msg) || retryConcreteAuthFailure) {
+ if (retryConcreteAuthFailure) {
this._captureRetryAuthFailureSource(msg);
}
const didRetry = await this._handleRetryableError(msg, {
- markAuthStaleOnFailure: concreteAuthFailure,
- authSourceTokens: concreteAuthFailure ? this._retryAuthFailureSources : undefined,
+ markAuthStaleOnFailure: retryConcreteAuthFailure,
+ authSourceTokens: retryConcreteAuthFailure ? this._retryAuthFailureSources : undefined,
});
if (didRetry) return; // Retry was initiated, don't proceed to compaction
}
- this._resolveRetry();
const compactionWillRetry = await this._checkCompaction(msg);
+ if (compactionWillRetry && this._retryAttempt > 0) {
+ return;
+ }
+ this._finishActiveRetryWithFailure(msg);
+ this._resolveRetry();
if (!compactionWillRetry) {
this._finishGoalForTerminalAssistantMessage(msg);
this._scheduleAutoRefineAfterAgentEnd();
@@ -3222,8 +3228,8 @@ export class AgentSession {
* @returns Object with steering and followUp arrays
*/
clearQueue(): { steering: string[]; followUp: string[] } {
- const steering = this._steeringMessages.map(queuedMessagePreview);
- const followUp = this._followUpMessages.map(queuedMessagePreview);
+ const steering = this._steeringMessages.map((message) => message.text);
+ const followUp = this._followUpMessages.map((message) => message.text);
this._rejectQueuedAgentMessageDeliveries(new Error("Queued agent message was cleared before delivery."));
this._steeringMessages = [];
this._followUpMessages = [];
@@ -5626,34 +5632,64 @@ export class AgentSession {
const contextWindow = this.model?.contextWindow ?? 0;
if (isContextOverflow(message, contextWindow)) return false;
- // Structured classification from the provider (stream-failure.ts) beats
- // message-text matching. Safety/malformed/unknown kinds fall through to
- // the regex, which distinguishes transient content_filter cases.
- const failure = message.diagnostics?.find((d) => d.type === "provider_stream_failure");
- const kind = failure?.details?.kind;
- if (kind === "overloaded" || kind === "rate_limit" || kind === "server_error") return true;
- if (kind === "refusal" || kind === "auth" || kind === "invalid_request") return false;
-
- const err = message.errorMessage;
- // Match: overloaded_error, provider returned error, rate limit, 429, 500, 502, 503, 504, service unavailable, network/connection errors (including connection lost), WebSocket transport closes/errors, fetch failed, request ended without sending chunks, HTTP/2 closed before response, terminated, retry delay exceeded, transient content_filter finish_reason, provider policy-flag prose, and prose-form transient 5xx ("an error occurred while processing your request. you can retry")
- return /overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|http2 request did not get a response|timed? out|timeout|terminated|retry delay|content.?filter|flagged .*(cybersecurity|usage polic|violat)|error occurred while processing|you can retry/i.test(
- err,
- );
+ if (this._isFauxProviderQueueExhausted(message)) {
+ return false;
+ }
+
+ if (this._isAgentLifecycleFailure(message)) {
+ return false;
+ }
+
+ if (this._isStructuredPermanentProviderRetryExhausted(message)) {
+ return false;
+ }
+
+ return true;
}
- private _getProviderStreamFailureAuthStatus(message: AssistantMessage): number | undefined {
+ private _isFauxProviderQueueExhausted(message: AssistantMessage): boolean {
+ return message.provider === "faux" && message.errorMessage === "No more faux responses queued";
+ }
+
+ private _isAgentLifecycleFailure(message: AssistantMessage): boolean {
+ return message.diagnostics?.some((diagnostic) => diagnostic.type === "agent_lifecycle_failure") ?? false;
+ }
+
+ private _getProviderStreamFailureDetails(message: AssistantMessage): Record<string, unknown> | undefined {
const failure = message.diagnostics?.find((diagnostic) => diagnostic.type === "provider_stream_failure");
const details = failure?.details;
if (!details || typeof details !== "object") {
return undefined;
}
+ return details;
+ }
- const kind = "kind" in details ? details.kind : undefined;
+ private _getProviderStreamFailureKind(message: AssistantMessage): string | undefined {
+ const kind = this._getProviderStreamFailureDetails(message)?.kind;
+ return typeof kind === "string" ? kind : undefined;
+ }
+
+ private _isStructuredPermanentProviderFailure(message: AssistantMessage): boolean {
+ const kind = this._getProviderStreamFailureKind(message);
+ return kind === "auth" || kind === "invalid_request" || kind === "refusal";
+ }
+
+ private _isStructuredPermanentProviderRetryExhausted(message: AssistantMessage): boolean {
+ return this._retryAttempt > 0 && this._isStructuredPermanentProviderFailure(message);
+ }
+
+ private _getProviderStreamFailureAuthStatus(message: AssistantMessage): number | undefined {
+ const details = this._getProviderStreamFailureDetails(message);
+ if (!details) {
+ return undefined;
+ }
+
+ const kind = details.kind;
if (kind !== "auth") {
return undefined;
}
- const status = "status" in details ? details.status : undefined;
+ const status = details.status;
if (typeof status === "number") {
return status;
}
@@ -5735,6 +5771,21 @@ export class AgentSession {
return false;
}
+ private _finishActiveRetryWithFailure(message: AssistantMessage): void {
+ if (this._retryAttempt === 0) {
+ return;
+ }
+ this._markProviderAuthStaleForRetryFailure(message);
+ this._emit({
+ type: "auto_retry_end",
+ success: false,
+ attempt: this._retryAttempt,
+ finalError: message.errorMessage,
+ });
+ this._retryAttempt = 0;
+ this._retryAuthFailureSources = [];
+ }
+
/**
* Handle retryable errors with exponential backoff.
* @returns true if retry was initiated, false if max retries exceeded or disabled
@@ -5832,6 +5883,17 @@ export class AgentSession {
this._retryAbortController.abort();
return;
}
+ if (this._retryAttempt > 0) {
+ this._autoCompactionAbortController?.abort();
+ this._cancelPostCompactionContinue();
+ this._emit({
+ type: "auto_retry_end",
+ success: false,
+ attempt: this._retryAttempt,
+ finalError: "Retry cancelled",
+ });
+ this._retryAttempt = 0;
+ }
this._retryAuthFailureSources = [];
this._resolveRetry();
}
diff --git a/packages/coding-agent/test/agent-session-retry.test.ts b/packages/coding-agent/test/agent-session-retry.test.ts
index 67717dc180..223c87adf1 100644
--- a/packages/coding-agent/test/agent-session-retry.test.ts
+++ b/packages/coding-agent/test/agent-session-retry.test.ts
@@ -363,9 +363,11 @@ describe("AgentSession retry", () => {
expect(callCount).toBe(4);
});
- // Transient provider stream artifacts the classifier previously missed (ENG-4390).
+ // Provider errors are retried within the bounded retry budget (ENG-4390, ENG-4503).
const transientProviderErrors: Array<[string, string]> = [
["content_filter finish_reason", "Provider finish_reason: content_filter"],
+ ["empty provider response", "Provider returned an empty response"],
+ ["plain invalid API key", "invalid_api_key"],
["cybersecurity policy flag", "Your request was flagged for cybersecurity risk and cannot be processed."],
["usage policy flag", "flagged as potentially violating our usage policy"],
["prose-form transient 5xx", "An error occurred while processing your request. You can retry your request."],
diff --git a/packages/coding-agent/test/suite/agent-session-retry-events.test.ts b/packages/coding-agent/test/suite/agent-session-retry-events.test.ts
index a6d61b3d67..e14901b296 100644
--- a/packages/coding-agent/test/suite/agent-session-retry-events.test.ts
+++ b/packages/coding-agent/test/suite/agent-session-retry-events.test.ts
@@ -1,5 +1,5 @@
-import type { AgentTool } from "@earendil-works/pi-agent-core";
-import { fauxAssistantMessage, fauxThinking, fauxToolCall } from "@earendil-works/pi-ai";
+import type { AgentEvent, AgentTool } from "@earendil-works/pi-agent-core";
+import { type AssistantMessage, fauxAssistantMessage, fauxThinking, fauxToolCall } from "@earendil-works/pi-ai";
import { Type } from "typebox";
import { afterEach, describe, expect, it } from "vitest";
import { createHarness, type Harness } from "./harness.js";
@@ -21,6 +21,34 @@ function normalizeEventOrder(events: Harness["events"]): string[] {
return normalized;
}
+function structuredProviderFailure(kind: "auth" | "invalid_request" | "refusal"): AssistantMessage {
+ return {
+ ...fauxAssistantMessage("", {
+ stopReason: "error",
+ errorMessage: `provider ${kind} failure`,
+ }),
+ diagnostics: [
+ {
+ type: "provider_stream_failure",
+ timestamp: Date.now(),
+ details: { kind },
+ },
+ ],
+ };
+}
+
+type SessionRetryCompactionInternals = {
+ _retryAttempt: number;
+ _retryPromise: Promise<void> | undefined;
+ _retryResolve: (() => void) | undefined;
+ _autoCompactionAbortController: AbortController | undefined;
+ _postCompactionContinuationScheduled: boolean;
+ _processAgentEvent: (event: AgentEvent) => Promise<void>;
+ _checkCompaction: (message: AssistantMessage) => Promise<boolean>;
+ _schedulePostCompactionContinue: () => void;
+ _cancelPostCompactionContinue: () => void;
+};
+
describe("AgentSession retry and event characterization", () => {
const harnesses: Harness[] = [];
@@ -157,15 +185,134 @@ describe("AgentSession retry and event characterization", () => {
expect(harness.eventsOfType("auto_retry_start")).toEqual([]);
});
- it("does not retry non-retryable errors", async () => {
+ it("does not retry faux provider queue exhaustion", async () => {
+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
+ harnesses.push(harness);
+
+ await harness.session.prompt("test");
+
+ expect(harness.faux.state.callCount).toBe(1);
+ expect(harness.eventsOfType("auto_retry_start")).toEqual([]);
+ expect(harness.session.isRetrying).toBe(false);
+ });
+
+ it("does not retry local agent lifecycle listener failures", async () => {
const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
harnesses.push(harness);
- harness.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "invalid_api_key" })]);
+ let unsubscribe = () => {};
+ unsubscribe = harness.session.agent.subscribe((event) => {
+ if (event.type === "message_start" && event.message.role === "assistant") {
+ unsubscribe();
+ throw new Error("local listener failed");
+ }
+ });
+ harness.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("retry should not happen")]);
await harness.session.prompt("test");
+ const lastMessage = harness.session.messages.at(-1);
expect(harness.faux.state.callCount).toBe(1);
expect(harness.eventsOfType("auto_retry_start")).toEqual([]);
+ expect(harness.session.isRetrying).toBe(false);
+ expect(lastMessage?.role).toBe("assistant");
+ if (lastMessage?.role === "assistant") {
+ expect(lastMessage.diagnostics?.some((diagnostic) => diagnostic.type === "agent_lifecycle_failure")).toBe(
+ true,
+ );
+ }
+ });
+
+ it("retries generic provider errors", async () => {
+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
+ harnesses.push(harness);
+ harness.setResponses([
+ fauxAssistantMessage("", { stopReason: "error", errorMessage: "invalid_api_key" }),
+ fauxAssistantMessage("recovered"),
+ ]);
+
+ await harness.session.prompt("test");
+
+ expect(harness.faux.state.callCount).toBe(2);
+ expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1]);
+ expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([true]);
+ });
+
+ for (const kind of ["auth", "invalid_request", "refusal"] as const) {
+ it(`retries structured permanent provider ${kind} failures once`, async () => {
+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
+ harnesses.push(harness);
+ harness.setResponses([
+ structuredProviderFailure(kind),
+ structuredProviderFailure(kind),
+ fauxAssistantMessage("unused"),
+ ]);
+
+ await harness.session.prompt("test");
+
+ expect(harness.faux.state.callCount).toBe(2);
+ expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1]);
+ expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([false]);
+ expect(harness.session.isRetrying).toBe(false);
+ });
+ }
+
+ it("keeps retry state active when overflow compaction will retry", async () => {
+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
+ harnesses.push(harness);
+ const internals = harness.session as unknown as SessionRetryCompactionInternals;
+ const originalCheckCompaction = internals._checkCompaction.bind(harness.session);
+ const overflowMessage = fauxAssistantMessage("", {
+ stopReason: "error",
+ errorMessage: "prompt is too long",
+ });
+ internals._retryAttempt = 1;
+ internals._retryPromise = new Promise<void>((resolve) => {
+ internals._retryResolve = resolve;
+ });
+ internals._checkCompaction = async () => true;
+
+ try {
+ await internals._processAgentEvent({ type: "agent_end", messages: [overflowMessage] } as AgentEvent);
+
+ expect(internals._retryAttempt).toBe(1);
+ expect(harness.session.isRetrying).toBe(true);
+ expect(harness.eventsOfType("auto_retry_end")).toEqual([]);
+ } finally {
+ internals._checkCompaction = originalCheckCompaction;
+ harness.session.abortRetry();
+ }
+ });
+
+ it("cancels overflow-compaction retry continuation when abortRetry is called", async () => {
+ const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
+ harnesses.push(harness);
+ const internals = harness.session as unknown as SessionRetryCompactionInternals;
+ const compactionAbortController = new AbortController();
+ internals._retryAttempt = 1;
+ internals._retryPromise = new Promise<void>((resolve) => {
+ internals._retryResolve = resolve;
+ });
+ internals._autoCompactionAbortController = compactionAbortController;
+ internals._schedulePostCompactionContinue();
+
+ try {
+ expect(internals._postCompactionContinuationScheduled).toBe(true);
+
+ harness.session.abortRetry();
+
+ expect(compactionAbortController.signal.aborted).toBe(true);
+ expect(internals._postCompactionContinuationScheduled).toBe(false);
+ expect(internals._retryAttempt).toBe(0);
+ expect(harness.session.isRetrying).toBe(false);
+ expect(harness.eventsOfType("auto_retry_end").at(-1)).toMatchObject({
+ success: false,
+ attempt: 1,
+ finalError: "Retry cancelled",
+ });
+ } finally {
+ internals._autoCompactionAbortController = undefined;
+ internals._cancelPostCompactionContinue();
+ }
});
it("cancels retry sleep when abortRetry is called", async () => {
@@ -221,6 +368,7 @@ describe("AgentSession retry and event characterization", () => {
expect(harness.faux.state.callCount).toBe(3);
expect(toolRuns).toEqual(["hello"]);
expect(harness.session.isStreaming).toBe(false);
+ harness.appendResponses([fauxAssistantMessage("follow-up answer")]);
await harness.session.prompt("follow-up");
expect(harness.faux.state.callCount).toBe(4);
});
@@ -347,7 +495,7 @@ describe("AgentSession retry and event characterization", () => {
});
it("emits agent_end for error responses", async () => {
- const harness = await createHarness();
+ const harness = await createHarness({ settings: { retry: { enabled: false } } });
harnesses.push(harness);
harness.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "broken" })]);
diff --git a/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.ts b/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.ts
index 062a70eb13..1aed13d7e1 100644
--- a/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4482-heartbeat-injected-prompt.test.ts
@@ -258,7 +258,7 @@ describe("ENG-4482 heartbeat injected prompt UI", () => {
).toBe(true);
});
- it("returns labeled previews when clearing queued heartbeat prompts", async () => {
+ it("returns raw text when clearing queued heartbeat prompts while previews remain labeled", async () => {
let releaseToolExecution: (() => void) | undefined;
const toolRelease = new Promise<void>((resolve) => {
releaseToolExecution = resolve;
@@ -278,7 +278,8 @@ describe("ENG-4482 heartbeat injected prompt UI", () => {
};
const harness = await createHarness({ tools: [waitTool] });
harnesses.push(harness);
- const heartbeatPreview = "Heartbeat prompt: Check whether the long-running task needs another step.";
+ const heartbeatText = "Check whether the long-running task needs another step.";
+ const heartbeatPreview = `Heartbeat prompt: ${heartbeatText}`;
harness.setResponses([
fauxAssistantMessage(fauxToolCall("wait", {}), { stopReason: "toolUse" }),
fauxAssistantMessage("original turn complete"),
@@ -297,7 +298,7 @@ describe("ENG-4482 heartbeat injected prompt UI", () => {
await harness.session.promptHeartbeat(createHeartbeat(), { streamingBehavior: "followUp" });
expect(harness.session.getFollowUpMessagePreviews()).toEqual([heartbeatPreview]);
- expect(harness.session.clearQueue()).toEqual({ steering: [], followUp: [heartbeatPreview] });
+ expect(harness.session.clearQueue()).toEqual({ steering: [], followUp: [heartbeatText] });
releaseToolExecution?.();
await promptPromise;
diff --git a/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.ts b/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.ts
index 6c2e474fe1..48fc2a7ace 100644
--- a/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4491-provider-stale-after-401.test.ts
@@ -51,7 +51,7 @@ describe("issue #4491 provider stale after repeated 401", () => {
}
});
- it("retries concrete provider auth failures, then marks current auth stale", async () => {
+ it("retries structured provider auth failures once, then marks current auth stale", async () => {
const harness = await createHarness({
settings: { retry: { enabled: true, maxRetries: 2, baseDelayMs: 1 } },
});
@@ -60,8 +60,8 @@ describe("issue #4491 provider stale after repeated 401", () => {
await harness.session.prompt("hello");
- expect(harness.faux.state.callCount).toBe(3);
- expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1, 2]);
+ expect(harness.faux.state.callCount).toBe(2);
+ expect(harness.eventsOfType("auto_retry_start").map((event) => event.attempt)).toEqual([1]);
expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([false]);
expect(harness.eventsOfType("auth_stale")).toHaveLength(1);
@@ -125,6 +125,24 @@ describe("issue #4491 provider stale after repeated 401", () => {
harness.session.abortRetry();
});
+ it("creates retry promises for exhausted structured auth failures so cleanup is awaited", async () => {
+ const harness = await createHarness({
+ settings: { retry: { enabled: true, maxRetries: 2, baseDelayMs: 1 } },
+ });
+ harnesses.push(harness);
+ const event = { type: "agent_end", messages: [provider401Message()] } as AgentEvent;
+ const session = harness.session as unknown as {
+ _retryAttempt: number;
+ _createRetryPromiseForAgentEnd(event: AgentEvent): void;
+ };
+ session._retryAttempt = 1;
+
+ session._createRetryPromiseForAgentEnd(event);
+
+ expect(harness.session.isRetrying).toBe(true);
+ harness.session.abortRetry();
+ });
+
it("marks captured auth failures stale when retry backoff is cancelled", async () => {
const harness = await createHarness({
settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 100 } },
@@ -186,7 +204,7 @@ describe("issue #4491 provider stale after repeated 401", () => {
settings: { retry: { enabled: true, maxRetries: 2, baseDelayMs: 1 } },
});
harnesses.push(harness);
- harness.setResponses([provider401Message(), provider401Message(), provider500Message()]);
+ harness.setResponses([provider401Message(), provider500Message(), provider500Message()]);
await harness.session.prompt("hello");