A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.
TypeScript
fix: retry transient provider errors instead of failing tasks (#18)
Lucky Reddy1194b26c09fe
diff --git a/apps/server/src/config.ts b/apps/server/src/config.tsindex 0b25b3c..5df6ecc 100644--- a/apps/server/src/config.ts+++ b/apps/server/src/config.ts@@ -52,6 +52,14 @@ export function assertApiDeploymentConfig();}+// Provider SDKs retry transient failures before the response starts, with+// exponential backoff: OpenAI and Anthropic retry HTTP 408, 409, 429, 5xx and+// connection errors and honor retry-after; Gemini retries 408, 429, 500, 502,+// 503 and 504. Other 4xx responses such as 400, 401 and 403 fail on the first+// attempt, and a stream that fails after it starts is not retried. External+// writes never re-fire here: they are dispatched outside the model loop through+// reviewed, idempotency-keyed actions.+export const MODEL_MAX_RETRIES = 2;export function readConfig(): Config {const mode = process.env.WORKSPACE_MODE ?? "sample";if (mode !== "sample" && mode !== "live")diff --git a/apps/server/src/engine/tanstack-agent.ts b/apps/server/src/engine/tanstack-agent.tsindex 59067dc..d76aa13 100644--- a/apps/server/src/engine/tanstack-agent.ts+++ b/apps/server/src/engine/tanstack-agent.ts@@ -12,9 +12,10 @@ import { type GeminiTextModel, geminiText } from "@tanstack/ai-gemini";import { type OpenAIChatModel, openaiText } from "@tanstack/ai-openai";import { map, type Observable } from "rxjs";import { z } from "zod";+import { MODEL_MAX_RETRIES } from "../config.ts";// Same "provider/model" strings, env vars and base URL formats as the AI SDK resolver in-// @copilotkit/runtime. Retries are off, like the old `maxRetries: 0`.+// @copilotkit/runtime. Each provider SDK retries transient failures up to MODEL_MAX_RETRIES times.function adapter(spec: string) {const [, provider = "", model = ""] = spec.trim().match(/^([^/:]*)[/:](.*)$/) ?? [];if (!provider || !model.trim())@@ -26,13 +27,13 @@ function adapter(spec: string) {case "openai":return openaiText(id as OpenAIChatModel, {baseURL: process.env.OPENAI_BASE_URL,- maxRetries: 0,+ maxRetries: MODEL_MAX_RETRIES,});case "anthropic":// The AI SDK base URL ends in /v1; the Anthropic SDK adds /v1 itself.return anthropicText(id as AnthropicChatModel, {baseURL: process.env.ANTHROPIC_BASE_URL?.replace(/\/v1\/?$/, ""),- maxRetries: 0,+ maxRetries: MODEL_MAX_RETRIES,});case "google":case "gemini":@@ -41,7 +42,8 @@ function adapter(spec: string) {return geminiText(id as GeminiTextModel, {httpOptions: {baseUrl: process.env.GOOGLE_GENERATIVE_AI_BASE_URL?.replace(/\/v1beta\/?$/, ""),- retryOptions: { attempts: 1 },+ // @google/genai counts the first call in `attempts`.+ retryOptions: { attempts: MODEL_MAX_RETRIES + 1 },},});default:diff --git a/tests/helpers/model.ts b/tests/helpers/model.tsindex 0b64425..303bfce 100644--- a/tests/helpers/model.ts+++ b/tests/helpers/model.ts@@ -9,13 +9,101 @@ type ModelCall = { name: string; arguments: object };export async function modelFixture(t: TestContext,reply: (index: number) => ModelCall | undefined | Promise<ModelCall | undefined>,+ options: {+ errorStatus?: (index: number) => number | undefined;+ dropAfterStart?: (index: number) => boolean;+ dropAfterText?: (index: number) => boolean;+ errorPart?: (index: number) => boolean;+ } = {},) {+ const { errorStatus, dropAfterStart, dropAfterText, errorPart } = options;const requests: { path: string; body: string }[] = [];const server = createServer(async (request, response) => {let body = "";for await (const chunk of request) body += chunk;const index = requests.length;requests.push({ path: request.url ?? "", body });+ const status = errorStatus?.(index);+ if (status !== undefined) {+ response.writeHead(status, { "Content-Type": "application/json" });+ response.end(+ JSON.stringify({+ error: { message: "Fixture provider failure", type: "server_error" },+ }),+ );+ return;+ }+ if (dropAfterStart?.(index)) {+ // Deliver a valid stream start, then fail the connection before any+ // assistant output reaches the client.+ response.writeHead(200, { "Content-Type": "text/event-stream" });+ response.write(+ `data: ${JSON.stringify({+ type: "response.created",+ response: {+ id: `drop-${index}`,+ created_at: 1000,+ model: "fixture",+ status: "in_progress",+ },+ })}\n\n`,+ );+ setTimeout(() => response.socket?.destroy(), 120);+ return;+ }+ if (dropAfterText?.(index)) {+ // Deliver real assistant output, then fail the connection. A retry+ // must not replay output the client already received.+ response.writeHead(200, { "Content-Type": "text/event-stream" });+ response.write(+ `data: ${JSON.stringify({+ type: "response.created",+ response: {+ id: `drop-text-${index}`,+ created_at: 1000,+ model: "fixture",+ status: "in_progress",+ },+ })}\n\n`,+ );+ response.write(+ `data: ${JSON.stringify({+ type: "response.output_item.added",+ output_index: 0,+ item: {+ id: `msg-${index}`,+ type: "message",+ role: "assistant",+ status: "in_progress",+ content: [],+ },+ })}\n\n`,+ );+ response.write(+ `data: ${JSON.stringify({+ type: "response.output_text.delta",+ item_id: `msg-${index}`,+ output_index: 0,+ delta: "Hello partial ",+ })}\n\n`,+ );+ setTimeout(() => response.socket?.destroy(), 120);+ return;+ }+ if (errorPart?.(index)) {+ response.writeHead(200, { "Content-Type": "text/event-stream" });+ response.write(+ `data: ${JSON.stringify({+ type: "response.failed",+ sequence_number: 1,+ response: {+ error: { code: "server_error", message: "Provider reported response.failed" },+ },+ })}\n\n`,+ );+ response.end("data: [DONE]\n\n");+ return;+ }const call = await reply(index);response.writeHead(200, { "Content-Type": "text/event-stream" });const emit = (type: string, value: object) =>diff --git a/tests/model-retry.test.ts b/tests/model-retry.test.tsnew file mode 100644index 0000000..a2b05fb--- /dev/null+++ b/tests/model-retry.test.ts@@ -0,0 +1,147 @@+import assert from "node:assert/strict";+import { test } from "node:test";+import { EventType, type RunAgentInput } from "@ag-ui/core";+import { type BuiltInAgent, defineTool } from "@copilotkit/runtime/v2";+import { z } from "zod";+import { MODEL_MAX_RETRIES } from "../apps/server/src/config.ts";+import { tanstackAgent } from "../apps/server/src/engine/tanstack-agent.ts";+import { modelFixture } from "./helpers/model.ts";++const run = (agent: BuiltInAgent) => {+ const input: RunAgentInput = {+ threadId: "retry-fixture",+ runId: "retry-fixture-run",+ messages: [{ id: "m1", role: "user", content: "Reply briefly." }],+ state: {},+ tools: [],+ context: [],+ forwardedProps: {},+ };+ return new Promise<{ error?: string; finished: boolean; text: string }>((resolve) => {+ let error: string | undefined;+ let finished = false;+ let text = "";+ agent.run(input).subscribe({+ next: (event) => {+ if (+ (event.type === EventType.TEXT_MESSAGE_CHUNK ||+ event.type === EventType.TEXT_MESSAGE_CONTENT) &&+ "delta" in event &&+ typeof event.delta === "string"+ )+ text += event.delta;+ if (event.type === EventType.RUN_ERROR && "message" in event) error = String(event.message);+ if (event.type === EventType.RUN_FINISHED) finished = true;+ },+ error: (cause) => {+ // An erroring observable never completes, so resolve here.+ if (error === undefined) error = String(cause);+ resolve({ error, finished: false, text });+ },+ complete: () => resolve({ error, finished, text }),+ });+ });+};++const agent = () =>+ tanstackAgent({+ model: "openai/fixture",+ maxSteps: 2,+ tools: [],+ prompt: "Reply briefly.",+ });++test("a transient provider failure is retried and the run completes", async (t) => {+ const { requests } = await modelFixture(t, () => undefined, {+ errorStatus: (index) => (index === 0 ? 500 : undefined),+ });+ const outcome = await run(agent());+ assert.equal(outcome.error, undefined);+ assert.equal(outcome.finished, true);+ assert.equal(requests.length, 2, "exactly one retry follows the transient failure");+ assert.equal(requests[1].body, requests[0].body, "the retry replays the same model request");+});++test("a non-retryable provider failure fails fast without a retry", async (t) => {+ const { requests } = await modelFixture(t, () => undefined, { errorStatus: () => 400 });+ const outcome = await run(agent());+ assert.equal(outcome.finished, false);+ assert.match(outcome.error ?? "", /Fixture provider failure/);+ assert.equal(requests.length, 1, "a 400 must never be retried");+});++test("retries give up after the configured attempts", async (t) => {+ const { requests } = await modelFixture(t, () => undefined, { errorStatus: () => 500 });+ const outcome = await run(agent());+ assert.equal(outcome.finished, false);+ assert.equal(requests.length, MODEL_MAX_RETRIES + 1, "retries are bounded");+});++// Provider SDKs retry only until the response starts. A failure after the stream+// has started ends the run, even when a second attempt would succeed.+test("a connection drop after the stream starts is not retried", async (t) => {+ const { requests } = await modelFixture(t, () => undefined, {+ dropAfterStart: (index) => index === 0,+ });+ const outcome = await run(agent());+ assert.equal(outcome.finished, false);+ assert.ok(outcome.error, "the run reports the dropped stream");+ assert.equal(requests.length, 1, "a started stream is not retried");+});++test("a provider error part after the stream starts is not retried", async (t) => {+ const { requests } = await modelFixture(t, () => undefined, {+ errorPart: (index) => index === 0,+ });+ const outcome = await run(agent());+ assert.equal(outcome.finished, false);+ assert.match(outcome.error ?? "", /Provider reported response.failed/);+ assert.equal(requests.length, 1, "an error part in a started stream is not retried");+});++test("a committed tool result is not re-executed when the next model call fails", async (t) => {+ let toolRuns = 0;+ const { requests } = await modelFixture(+ t,+ (index) =>+ index === 0 ? { name: "note_step", arguments: { note: "step one done" } } : undefined,+ { errorStatus: (index) => (index === 1 ? 500 : undefined) },+ );+ const stepAgent = tanstackAgent({+ model: "openai/fixture",+ maxSteps: 4,+ tools: [+ defineTool({+ name: "note_step",+ description: "Record a step note",+ parameters: z.object({ note: z.string() }),+ execute: async ({ note }) => {+ toolRuns++;+ return { recorded: note };+ },+ }),+ ],+ prompt: "Use the tool once, then finish.",+ });+ const outcome = await run(stepAgent);+ assert.equal(outcome.error, undefined);+ assert.equal(outcome.finished, true);+ assert.equal(requests.length, 3, "the retry replays the failed model call only");+ assert.equal(+ requests[2].body,+ requests[1].body,+ "the retry resent the failed step, not a restarted run",+ );+ assert.equal(toolRuns, 1, "committed tool work is never re-run by a retry");+});++test("assistant output already delivered is never replayed by a retry", async (t) => {+ const { requests } = await modelFixture(t, () => undefined, {+ dropAfterText: (index) => index === 0,+ });+ const outcome = await run(agent());+ assert.equal(outcome.finished, false);+ assert.ok(outcome.error, "the run reports the dropped stream");+ assert.equal(requests.length, 1, "a stream with visible output is not retried");+ assert.equal(outcome.text, "Hello partial ", "the client saw the delivered delta exactly once");+});