Skip to content

A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.

TypeScript

fix: retry transient provider errors instead of failing tasks (#18)

Lucky Reddy1194b26c09fe
diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts
index 0b25b3c..5df6ecc 100644
--- a/apps/server/src/config.ts
+++ b/apps/server/src/config.ts
@@ -52,6 +52,14 @@ export function assertApiDeploymentConfig(
);
}
+// Provider SDKs retry transient failures before the response starts, with
+// exponential backoff: OpenAI and Anthropic retry HTTP 408, 409, 429, 5xx and
+// connection errors and honor retry-after; Gemini retries 408, 429, 500, 502,
+// 503 and 504. Other 4xx responses such as 400, 401 and 403 fail on the first
+// attempt, and a stream that fails after it starts is not retried. External
+// writes never re-fire here: they are dispatched outside the model loop through
+// reviewed, idempotency-keyed actions.
+export const MODEL_MAX_RETRIES = 2;
export function readConfig(): Config {
const mode = process.env.WORKSPACE_MODE ?? "sample";
if (mode !== "sample" && mode !== "live")
diff --git a/apps/server/src/engine/tanstack-agent.ts b/apps/server/src/engine/tanstack-agent.ts
index 59067dc..d76aa13 100644
--- a/apps/server/src/engine/tanstack-agent.ts
+++ b/apps/server/src/engine/tanstack-agent.ts
@@ -12,9 +12,10 @@ import { type GeminiTextModel, geminiText } from "@tanstack/ai-gemini";
import { type OpenAIChatModel, openaiText } from "@tanstack/ai-openai";
import { map, type Observable } from "rxjs";
import { z } from "zod";
+import { MODEL_MAX_RETRIES } from "../config.ts";
// Same "provider/model" strings, env vars and base URL formats as the AI SDK resolver in
-// @copilotkit/runtime. Retries are off, like the old `maxRetries: 0`.
+// @copilotkit/runtime. Each provider SDK retries transient failures up to MODEL_MAX_RETRIES times.
function adapter(spec: string) {
const [, provider = "", model = ""] = spec.trim().match(/^([^/:]*)[/:](.*)$/) ?? [];
if (!provider || !model.trim())
@@ -26,13 +27,13 @@ function adapter(spec: string) {
case "openai":
return openaiText(id as OpenAIChatModel, {
baseURL: process.env.OPENAI_BASE_URL,
- maxRetries: 0,
+ maxRetries: MODEL_MAX_RETRIES,
});
case "anthropic":
// The AI SDK base URL ends in /v1; the Anthropic SDK adds /v1 itself.
return anthropicText(id as AnthropicChatModel, {
baseURL: process.env.ANTHROPIC_BASE_URL?.replace(/\/v1\/?$/, ""),
- maxRetries: 0,
+ maxRetries: MODEL_MAX_RETRIES,
});
case "google":
case "gemini":
@@ -41,7 +42,8 @@ function adapter(spec: string) {
return geminiText(id as GeminiTextModel, {
httpOptions: {
baseUrl: process.env.GOOGLE_GENERATIVE_AI_BASE_URL?.replace(/\/v1beta\/?$/, ""),
- retryOptions: { attempts: 1 },
+ // @google/genai counts the first call in `attempts`.
+ retryOptions: { attempts: MODEL_MAX_RETRIES + 1 },
},
});
default:
diff --git a/tests/helpers/model.ts b/tests/helpers/model.ts
index 0b64425..303bfce 100644
--- a/tests/helpers/model.ts
+++ b/tests/helpers/model.ts
@@ -9,13 +9,101 @@ type ModelCall = { name: string; arguments: object };
export async function modelFixture(
t: TestContext,
reply: (index: number) => ModelCall | undefined | Promise<ModelCall | undefined>,
+ options: {
+ errorStatus?: (index: number) => number | undefined;
+ dropAfterStart?: (index: number) => boolean;
+ dropAfterText?: (index: number) => boolean;
+ errorPart?: (index: number) => boolean;
+ } = {},
) {
+ const { errorStatus, dropAfterStart, dropAfterText, errorPart } = options;
const requests: { path: string; body: string }[] = [];
const server = createServer(async (request, response) => {
let body = "";
for await (const chunk of request) body += chunk;
const index = requests.length;
requests.push({ path: request.url ?? "", body });
+ const status = errorStatus?.(index);
+ if (status !== undefined) {
+ response.writeHead(status, { "Content-Type": "application/json" });
+ response.end(
+ JSON.stringify({
+ error: { message: "Fixture provider failure", type: "server_error" },
+ }),
+ );
+ return;
+ }
+ if (dropAfterStart?.(index)) {
+ // Deliver a valid stream start, then fail the connection before any
+ // assistant output reaches the client.
+ response.writeHead(200, { "Content-Type": "text/event-stream" });
+ response.write(
+ `data: ${JSON.stringify({
+ type: "response.created",
+ response: {
+ id: `drop-${index}`,
+ created_at: 1000,
+ model: "fixture",
+ status: "in_progress",
+ },
+ })}\n\n`,
+ );
+ setTimeout(() => response.socket?.destroy(), 120);
+ return;
+ }
+ if (dropAfterText?.(index)) {
+ // Deliver real assistant output, then fail the connection. A retry
+ // must not replay output the client already received.
+ response.writeHead(200, { "Content-Type": "text/event-stream" });
+ response.write(
+ `data: ${JSON.stringify({
+ type: "response.created",
+ response: {
+ id: `drop-text-${index}`,
+ created_at: 1000,
+ model: "fixture",
+ status: "in_progress",
+ },
+ })}\n\n`,
+ );
+ response.write(
+ `data: ${JSON.stringify({
+ type: "response.output_item.added",
+ output_index: 0,
+ item: {
+ id: `msg-${index}`,
+ type: "message",
+ role: "assistant",
+ status: "in_progress",
+ content: [],
+ },
+ })}\n\n`,
+ );
+ response.write(
+ `data: ${JSON.stringify({
+ type: "response.output_text.delta",
+ item_id: `msg-${index}`,
+ output_index: 0,
+ delta: "Hello partial ",
+ })}\n\n`,
+ );
+ setTimeout(() => response.socket?.destroy(), 120);
+ return;
+ }
+ if (errorPart?.(index)) {
+ response.writeHead(200, { "Content-Type": "text/event-stream" });
+ response.write(
+ `data: ${JSON.stringify({
+ type: "response.failed",
+ sequence_number: 1,
+ response: {
+ error: { code: "server_error", message: "Provider reported response.failed" },
+ },
+ })}\n\n`,
+ );
+ response.end("data: [DONE]\n\n");
+ return;
+ }
const call = await reply(index);
response.writeHead(200, { "Content-Type": "text/event-stream" });
const emit = (type: string, value: object) =>
diff --git a/tests/model-retry.test.ts b/tests/model-retry.test.ts
new file mode 100644
index 0000000..a2b05fb
--- /dev/null
+++ b/tests/model-retry.test.ts
@@ -0,0 +1,147 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { EventType, type RunAgentInput } from "@ag-ui/core";
+import { type BuiltInAgent, defineTool } from "@copilotkit/runtime/v2";
+import { z } from "zod";
+import { MODEL_MAX_RETRIES } from "../apps/server/src/config.ts";
+import { tanstackAgent } from "../apps/server/src/engine/tanstack-agent.ts";
+import { modelFixture } from "./helpers/model.ts";
+
+const run = (agent: BuiltInAgent) => {
+ const input: RunAgentInput = {
+ threadId: "retry-fixture",
+ runId: "retry-fixture-run",
+ messages: [{ id: "m1", role: "user", content: "Reply briefly." }],
+ state: {},
+ tools: [],
+ context: [],
+ forwardedProps: {},
+ };
+ return new Promise<{ error?: string; finished: boolean; text: string }>((resolve) => {
+ let error: string | undefined;
+ let finished = false;
+ let text = "";
+ agent.run(input).subscribe({
+ next: (event) => {
+ if (
+ (event.type === EventType.TEXT_MESSAGE_CHUNK ||
+ event.type === EventType.TEXT_MESSAGE_CONTENT) &&
+ "delta" in event &&
+ typeof event.delta === "string"
+ )
+ text += event.delta;
+ if (event.type === EventType.RUN_ERROR && "message" in event) error = String(event.message);
+ if (event.type === EventType.RUN_FINISHED) finished = true;
+ },
+ error: (cause) => {
+ // An erroring observable never completes, so resolve here.
+ if (error === undefined) error = String(cause);
+ resolve({ error, finished: false, text });
+ },
+ complete: () => resolve({ error, finished, text }),
+ });
+ });
+};
+
+const agent = () =>
+ tanstackAgent({
+ model: "openai/fixture",
+ maxSteps: 2,
+ tools: [],
+ prompt: "Reply briefly.",
+ });
+
+test("a transient provider failure is retried and the run completes", async (t) => {
+ const { requests } = await modelFixture(t, () => undefined, {
+ errorStatus: (index) => (index === 0 ? 500 : undefined),
+ });
+ const outcome = await run(agent());
+ assert.equal(outcome.error, undefined);
+ assert.equal(outcome.finished, true);
+ assert.equal(requests.length, 2, "exactly one retry follows the transient failure");
+ assert.equal(requests[1].body, requests[0].body, "the retry replays the same model request");
+});
+
+test("a non-retryable provider failure fails fast without a retry", async (t) => {
+ const { requests } = await modelFixture(t, () => undefined, { errorStatus: () => 400 });
+ const outcome = await run(agent());
+ assert.equal(outcome.finished, false);
+ assert.match(outcome.error ?? "", /Fixture provider failure/);
+ assert.equal(requests.length, 1, "a 400 must never be retried");
+});
+
+test("retries give up after the configured attempts", async (t) => {
+ const { requests } = await modelFixture(t, () => undefined, { errorStatus: () => 500 });
+ const outcome = await run(agent());
+ assert.equal(outcome.finished, false);
+ assert.equal(requests.length, MODEL_MAX_RETRIES + 1, "retries are bounded");
+});
+
+// Provider SDKs retry only until the response starts. A failure after the stream
+// has started ends the run, even when a second attempt would succeed.
+test("a connection drop after the stream starts is not retried", async (t) => {
+ const { requests } = await modelFixture(t, () => undefined, {
+ dropAfterStart: (index) => index === 0,
+ });
+ const outcome = await run(agent());
+ assert.equal(outcome.finished, false);
+ assert.ok(outcome.error, "the run reports the dropped stream");
+ assert.equal(requests.length, 1, "a started stream is not retried");
+});
+
+test("a provider error part after the stream starts is not retried", async (t) => {
+ const { requests } = await modelFixture(t, () => undefined, {
+ errorPart: (index) => index === 0,
+ });
+ const outcome = await run(agent());
+ assert.equal(outcome.finished, false);
+ assert.match(outcome.error ?? "", /Provider reported response.failed/);
+ assert.equal(requests.length, 1, "an error part in a started stream is not retried");
+});
+
+test("a committed tool result is not re-executed when the next model call fails", async (t) => {
+ let toolRuns = 0;
+ const { requests } = await modelFixture(
+ t,
+ (index) =>
+ index === 0 ? { name: "note_step", arguments: { note: "step one done" } } : undefined,
+ { errorStatus: (index) => (index === 1 ? 500 : undefined) },
+ );
+ const stepAgent = tanstackAgent({
+ model: "openai/fixture",
+ maxSteps: 4,
+ tools: [
+ defineTool({
+ name: "note_step",
+ description: "Record a step note",
+ parameters: z.object({ note: z.string() }),
+ execute: async ({ note }) => {
+ toolRuns++;
+ return { recorded: note };
+ },
+ }),
+ ],
+ prompt: "Use the tool once, then finish.",
+ });
+ const outcome = await run(stepAgent);
+ assert.equal(outcome.error, undefined);
+ assert.equal(outcome.finished, true);
+ assert.equal(requests.length, 3, "the retry replays the failed model call only");
+ assert.equal(
+ requests[2].body,
+ requests[1].body,
+ "the retry resent the failed step, not a restarted run",
+ );
+ assert.equal(toolRuns, 1, "committed tool work is never re-run by a retry");
+});
+
+test("assistant output already delivered is never replayed by a retry", async (t) => {
+ const { requests } = await modelFixture(t, () => undefined, {
+ dropAfterText: (index) => index === 0,
+ });
+ const outcome = await run(agent());
+ assert.equal(outcome.finished, false);
+ assert.ok(outcome.error, "the run reports the dropped stream");
+ assert.equal(requests.length, 1, "a stream with visible output is not retried");
+ assert.equal(outcome.text, "Hello partial ", "the client saw the delivered delta exactly once");
+});