Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Let the run queue retry transient provider errors (#1731)

Joshua France9cf0ee7d08d6
diff --git a/src/core/turn-error.ts b/src/core/turn-error.ts
index a36ca1484..9f316475a 100644
--- a/src/core/turn-error.ts
+++ b/src/core/turn-error.ts
@@ -7,6 +7,13 @@ export class NonRetryableTurnError extends Error {
}
}
+export class ProviderTurnError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = "ProviderTurnError";
+ }
+}
+
export class TitleRejected extends Error {
readonly rule: string;
constructor(rule: string, sample: string) {
@@ -21,5 +28,7 @@ export type TurnFailurePayload = { kind: "turn_failure"; message: string; runId?
const GENERIC_TURN_FAILURE = "That turn failed and couldn't be completed. The details are in the operator error log.";
export function turnFailureMessage(err: unknown): string {
- return err instanceof NonRetryableTurnError && err.message.trim() ? err.message : GENERIC_TURN_FAILURE;
+ return (err instanceof NonRetryableTurnError || err instanceof ProviderTurnError) && err.message.trim()
+ ? err.message
+ : GENERIC_TURN_FAILURE;
}
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index 9b08cff98..96a85fe45 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -16,7 +16,10 @@ import {
import {
calculateCost,
InMemoryCredentialStore,
+ isContextOverflow,
+ isRetryableAssistantError,
type Api,
+ type AssistantMessage,
type Context,
type Model,
type ModelThinkingLevel,
@@ -51,7 +54,7 @@ import type {
TapeRecord,
} from "../sessions/session-store.ts";
import { tapeCheckpointPayload, tapeEntryMirrorRecord } from "../sessions/session-store.ts";
-import { NonRetryableTurnError, TitleRejected } from "../core/turn-error.ts";
+import { NonRetryableTurnError, ProviderTurnError, TitleRejected } from "../core/turn-error.ts";
import { MAX_LLM_REQUEST_BYTES } from "../core/attachments.ts";
import { asError, swallow, swallowAs } from "../util/errors.ts";
import {
@@ -925,38 +928,64 @@ export function textFromContent(content: unknown): string {
type AssistantTextSession = Pick<AgentSession, "getLastAssistantText" | "messages">;
+function parseProviderError(message: string): { type: string; message: string } | null {
+ const jsonAt = message.indexOf("{");
+ if (jsonAt < 0) return null;
+ try {
+ const parsed = JSON.parse(message.slice(jsonAt)) as {
+ type?: unknown;
+ message?: unknown;
+ error?: { type?: unknown; message?: unknown };
+ };
+ const body = parsed.error && typeof parsed.error === "object" ? parsed.error : parsed;
+ const providerMessage = typeof body.message === "string" ? body.message.trim() : "";
+ const providerType = typeof body.type === "string" ? body.type.trim() : "";
+ return providerMessage || providerType ? { type: providerType, message: providerMessage } : null;
+ } catch (e) {
+ swallow("pi: assistant error json parse", e);
+ return null;
+ }
+}
+
function formatPiAssistantError(raw: string | undefined): string {
const message = raw?.trim();
if (!message) return "Pi agent stopped with an error";
+ const provider = parseProviderError(message);
+ if (!provider?.message) return message;
+ return provider.type
+ ? `Model provider API error (${provider.type}): ${provider.message}`
+ : `Model provider API error: ${provider.message}`;
+}
- const jsonAt = message.indexOf("{");
- if (jsonAt >= 0) {
- try {
- const parsed = JSON.parse(message.slice(jsonAt)) as { error?: { type?: unknown; message?: unknown } };
- const providerMessage = typeof parsed.error?.message === "string" ? parsed.error.message.trim() : "";
- const providerType = typeof parsed.error?.type === "string" ? parsed.error.type.trim() : "";
- if (providerMessage)
- return providerType
- ? `Model provider API error (${providerType}): ${providerMessage}`
- : `Model provider API error: ${providerMessage}`;
- } catch (e) {
- swallow("pi: assistant error json parse", e);
- }
- }
+const TRANSIENT_PROVIDER_ERROR_TYPES = new Set(["overloaded_error", "api_error", "rate_limit_error", "timeout_error"]);
- return message;
+function piErrorRetryable(failed: AssistantMessage): boolean {
+ const providerType = failed.errorMessage ? parseProviderError(failed.errorMessage)?.type : undefined;
+ if (providerType && !TRANSIENT_PROVIDER_ERROR_TYPES.has(providerType)) return false;
+ return isRetryableAssistantError(failed) && !isContextOverflow(failed);
}
-function piAssistantError(session: AssistantTextSession): string | null {
+function piFailedAssistant(session: AssistantTextSession): AssistantMessage | undefined {
const lastAssistant = [...session.messages].reverse().find((m) => m.role === "assistant") as
- { stopReason?: string; errorMessage?: string } | undefined;
- if (lastAssistant?.stopReason !== "error") return null;
- return formatPiAssistantError(lastAssistant.errorMessage);
+ AssistantMessage | undefined;
+ return lastAssistant?.stopReason === "error" ? lastAssistant : undefined;
+}
+
+function piAssistantError(session: AssistantTextSession): string | null {
+ const failed = piFailedAssistant(session);
+ return failed ? formatPiAssistantError(failed.errorMessage) : null;
+}
+
+function piAssistantFailure(session: AssistantTextSession): Error | null {
+ const failed = piFailedAssistant(session);
+ if (!failed) return null;
+ const message = formatPiAssistantError(failed.errorMessage);
+ return piErrorRetryable(failed) ? new ProviderTurnError(message) : new NonRetryableTurnError(message);
}
export function piLastAssistantTextOrThrow(session: AssistantTextSession): string | undefined {
- const err = piAssistantError(session);
- if (err) throw new NonRetryableTurnError(err);
+ const err = piAssistantFailure(session);
+ if (err) throw err;
return session.getLastAssistantText();
}
@@ -965,8 +994,8 @@ export function piTurnError(session: AssistantTextSession, thrown: unknown, mess
messagesBefore === undefined
? session
: ({ messages: session.messages.slice(messagesBefore) } as AssistantTextSession);
- const detailed = piAssistantError(fresh);
- if (detailed) return new NonRetryableTurnError(detailed);
+ const detailed = piAssistantFailure(fresh);
+ if (detailed) return detailed;
return thrown instanceof Error ? thrown : new Error(String(thrown));
}
diff --git a/test/pi-harness-oneshot.test.ts b/test/pi-harness-oneshot.test.ts
index dd41004f5..7424e1b4c 100644
--- a/test/pi-harness-oneshot.test.ts
+++ b/test/pi-harness-oneshot.test.ts
@@ -32,6 +32,7 @@ import { modelGatewayRequest } from "../src/model/provider-endpoints.ts";
import { reconstructMessagesFromHistory } from "../src/harness/replay.ts";
import type { SessionEntry } from "../src/types.ts";
import { testConfig } from "./support/test-config.ts";
+import { NonRetryableTurnError, ProviderTurnError, turnFailureMessage } from "../src/core/turn-error.ts";
function countTempDirs(prefix: string): number {
return readdirSync(tmpdir()).filter((name) => name.startsWith(prefix)).length;
@@ -612,6 +613,39 @@ test("piTurnError recovers the session's structured error when the agent loop re
assert.match(err.message, /Model provider API error \(overloaded_error\): Overloaded/);
});
+test("transient Pi provider errors stay retryable and keep their message for the final failure", () => {
+ const failed = (errorMessage: string) =>
+ ({
+ getLastAssistantText: () => undefined,
+ messages: [{ role: "assistant", stopReason: "error", errorMessage, content: [] }],
+ }) as unknown as Parameters<typeof piTurnError>[0];
+
+ for (const errorMessage of [
+ '529 {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}',
+ '500 {"type":"error","error":{"type":"api_error","message":"Internal server error"}}',
+ "429 rate limit exceeded",
+ "fetch failed",
+ ]) {
+ const err = piTurnError(failed(errorMessage), new Error("An unknown error occurred"));
+ assert.ok(err instanceof ProviderTurnError, errorMessage);
+ assert.ok(!(err instanceof NonRetryableTurnError), errorMessage);
+ assert.equal(turnFailureMessage(err), err.message);
+ assert.throws(() => piLastAssistantTextOrThrow(failed(errorMessage)), ProviderTurnError);
+ }
+
+ for (const errorMessage of [
+ '400 {"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low."}}',
+ "429 insufficient_quota",
+ '400 {"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 215003 tokens > 200000 maximum"}}',
+ "prompt is too long: 215003 tokens > 200000 maximum",
+ '400 {"type":"error","error":{"type":"invalid_request_error","message":"thinking.budget_tokens must be at least 1500"}}',
+ '400 {"type":"invalid_request_error","message":"max_tokens must be at most 50000"}',
+ '429 {"type":"error","error":{"type":"rate_limit_error","message":"quota exceeded"}}',
+ ]) {
+ assert.ok(piTurnError(failed(errorMessage), new Error("x")) instanceof NonRetryableTurnError, errorMessage);
+ }
+});
+
test("piTurnError falls back to the thrown error when the session has no structured error", () => {
const session = {
getLastAssistantText: () => undefined,