Multiplayer agent harness for work.
Let the run queue retry transient provider errors (#1731)
Joshua France9cf0ee7d08d6
diff --git a/src/core/turn-error.ts b/src/core/turn-error.tsindex a36ca1484..9f316475a 100644--- a/src/core/turn-error.ts+++ b/src/core/turn-error.ts@@ -7,6 +7,13 @@ export class NonRetryableTurnError extends Error {}}+export class ProviderTurnError extends Error {+ constructor(message: string) {+ super(message);+ this.name = "ProviderTurnError";+ }+}+export class TitleRejected extends Error {readonly rule: string;constructor(rule: string, sample: string) {@@ -21,5 +28,7 @@ export type TurnFailurePayload = { kind: "turn_failure"; message: string; runId?const GENERIC_TURN_FAILURE = "That turn failed and couldn't be completed. The details are in the operator error log.";export function turnFailureMessage(err: unknown): string {- return err instanceof NonRetryableTurnError && err.message.trim() ? err.message : GENERIC_TURN_FAILURE;+ return (err instanceof NonRetryableTurnError || err instanceof ProviderTurnError) && err.message.trim()+ ? err.message+ : GENERIC_TURN_FAILURE;}diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex 9b08cff98..96a85fe45 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -16,7 +16,10 @@ import {import {calculateCost,InMemoryCredentialStore,+ isContextOverflow,+ isRetryableAssistantError,type Api,+ type AssistantMessage,type Context,type Model,type ModelThinkingLevel,@@ -51,7 +54,7 @@ import type {TapeRecord,} from "../sessions/session-store.ts";import { tapeCheckpointPayload, tapeEntryMirrorRecord } from "../sessions/session-store.ts";-import { NonRetryableTurnError, TitleRejected } from "../core/turn-error.ts";+import { NonRetryableTurnError, ProviderTurnError, TitleRejected } from "../core/turn-error.ts";import { MAX_LLM_REQUEST_BYTES } from "../core/attachments.ts";import { asError, swallow, swallowAs } from "../util/errors.ts";import {@@ -925,38 +928,64 @@ export function textFromContent(content: unknown): string {type AssistantTextSession = Pick<AgentSession, "getLastAssistantText" | "messages">;+function parseProviderError(message: string): { type: string; message: string } | null {+ const jsonAt = message.indexOf("{");+ if (jsonAt < 0) return null;+ try {+ const parsed = JSON.parse(message.slice(jsonAt)) as {+ type?: unknown;+ message?: unknown;+ error?: { type?: unknown; message?: unknown };+ };+ const body = parsed.error && typeof parsed.error === "object" ? parsed.error : parsed;+ const providerMessage = typeof body.message === "string" ? body.message.trim() : "";+ const providerType = typeof body.type === "string" ? body.type.trim() : "";+ return providerMessage || providerType ? { type: providerType, message: providerMessage } : null;+ } catch (e) {+ swallow("pi: assistant error json parse", e);+ return null;+ }+}+function formatPiAssistantError(raw: string | undefined): string {const message = raw?.trim();if (!message) return "Pi agent stopped with an error";+ const provider = parseProviderError(message);+ if (!provider?.message) return message;+ return provider.type+ ? `Model provider API error (${provider.type}): ${provider.message}`+ : `Model provider API error: ${provider.message}`;+}- const jsonAt = message.indexOf("{");- if (jsonAt >= 0) {- try {- const parsed = JSON.parse(message.slice(jsonAt)) as { error?: { type?: unknown; message?: unknown } };- const providerMessage = typeof parsed.error?.message === "string" ? parsed.error.message.trim() : "";- const providerType = typeof parsed.error?.type === "string" ? parsed.error.type.trim() : "";- if (providerMessage)- return providerType- ? `Model provider API error (${providerType}): ${providerMessage}`- : `Model provider API error: ${providerMessage}`;- } catch (e) {- swallow("pi: assistant error json parse", e);- }- }+const TRANSIENT_PROVIDER_ERROR_TYPES = new Set(["overloaded_error", "api_error", "rate_limit_error", "timeout_error"]);- return message;+function piErrorRetryable(failed: AssistantMessage): boolean {+ const providerType = failed.errorMessage ? parseProviderError(failed.errorMessage)?.type : undefined;+ if (providerType && !TRANSIENT_PROVIDER_ERROR_TYPES.has(providerType)) return false;+ return isRetryableAssistantError(failed) && !isContextOverflow(failed);}-function piAssistantError(session: AssistantTextSession): string | null {+function piFailedAssistant(session: AssistantTextSession): AssistantMessage | undefined {const lastAssistant = [...session.messages].reverse().find((m) => m.role === "assistant") as- { stopReason?: string; errorMessage?: string } | undefined;- if (lastAssistant?.stopReason !== "error") return null;- return formatPiAssistantError(lastAssistant.errorMessage);+ AssistantMessage | undefined;+ return lastAssistant?.stopReason === "error" ? lastAssistant : undefined;+}++function piAssistantError(session: AssistantTextSession): string | null {+ const failed = piFailedAssistant(session);+ return failed ? formatPiAssistantError(failed.errorMessage) : null;+}++function piAssistantFailure(session: AssistantTextSession): Error | null {+ const failed = piFailedAssistant(session);+ if (!failed) return null;+ const message = formatPiAssistantError(failed.errorMessage);+ return piErrorRetryable(failed) ? new ProviderTurnError(message) : new NonRetryableTurnError(message);}export function piLastAssistantTextOrThrow(session: AssistantTextSession): string | undefined {- const err = piAssistantError(session);- if (err) throw new NonRetryableTurnError(err);+ const err = piAssistantFailure(session);+ if (err) throw err;return session.getLastAssistantText();}@@ -965,8 +994,8 @@ export function piTurnError(session: AssistantTextSession, thrown: unknown, messmessagesBefore === undefined? session: ({ messages: session.messages.slice(messagesBefore) } as AssistantTextSession);- const detailed = piAssistantError(fresh);- if (detailed) return new NonRetryableTurnError(detailed);+ const detailed = piAssistantFailure(fresh);+ if (detailed) return detailed;return thrown instanceof Error ? thrown : new Error(String(thrown));}diff --git a/test/pi-harness-oneshot.test.ts b/test/pi-harness-oneshot.test.tsindex dd41004f5..7424e1b4c 100644--- a/test/pi-harness-oneshot.test.ts+++ b/test/pi-harness-oneshot.test.ts@@ -32,6 +32,7 @@ import { modelGatewayRequest } from "../src/model/provider-endpoints.ts";import { reconstructMessagesFromHistory } from "../src/harness/replay.ts";import type { SessionEntry } from "../src/types.ts";import { testConfig } from "./support/test-config.ts";+import { NonRetryableTurnError, ProviderTurnError, turnFailureMessage } from "../src/core/turn-error.ts";function countTempDirs(prefix: string): number {return readdirSync(tmpdir()).filter((name) => name.startsWith(prefix)).length;@@ -612,6 +613,39 @@ test("piTurnError recovers the session's structured error when the agent loop reassert.match(err.message, /Model provider API error \(overloaded_error\): Overloaded/);});+test("transient Pi provider errors stay retryable and keep their message for the final failure", () => {+ const failed = (errorMessage: string) =>+ ({+ getLastAssistantText: () => undefined,+ messages: [{ role: "assistant", stopReason: "error", errorMessage, content: [] }],+ }) as unknown as Parameters<typeof piTurnError>[0];++ for (const errorMessage of [+ '529 {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}',+ '500 {"type":"error","error":{"type":"api_error","message":"Internal server error"}}',+ "429 rate limit exceeded",+ "fetch failed",+ ]) {+ const err = piTurnError(failed(errorMessage), new Error("An unknown error occurred"));+ assert.ok(err instanceof ProviderTurnError, errorMessage);+ assert.ok(!(err instanceof NonRetryableTurnError), errorMessage);+ assert.equal(turnFailureMessage(err), err.message);+ assert.throws(() => piLastAssistantTextOrThrow(failed(errorMessage)), ProviderTurnError);+ }++ for (const errorMessage of [+ '400 {"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low."}}',+ "429 insufficient_quota",+ '400 {"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 215003 tokens > 200000 maximum"}}',+ "prompt is too long: 215003 tokens > 200000 maximum",+ '400 {"type":"error","error":{"type":"invalid_request_error","message":"thinking.budget_tokens must be at least 1500"}}',+ '400 {"type":"invalid_request_error","message":"max_tokens must be at most 50000"}',+ '429 {"type":"error","error":{"type":"rate_limit_error","message":"quota exceeded"}}',+ ]) {+ assert.ok(piTurnError(failed(errorMessage), new Error("x")) instanceof NonRetryableTurnError, errorMessage);+ }+});+test("piTurnError falls back to the thrown error when the session has no structured error", () => {const session = {getLastAssistantText: () => undefined,