Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

fix(llm): retry a 200 that carries an error, and fall back when a forced tool_choice isn't honoured (#502)

Alex Matthewsafdaea9f6f71
diff --git a/src/ai/llm/openai.ts b/src/ai/llm/openai.ts
index c23840d6..28bd3632 100644
--- a/src/ai/llm/openai.ts
+++ b/src/ai/llm/openai.ts
@@ -8,15 +8,74 @@
* is forwarded, cache breakpoints become `cache_control` content parts (which
* OpenRouter forwards to Anthropic), and cached tokens are subtracted out of the
* input count so {@link Usage.input} is uncached-only.
+ *
+ * Past that, everything here is tolerance rather than preference. An
+ * OpenAI-compatible endpoint is a gateway whose own upstream calls can fail
+ * while the HTTP status stays 200, and whose support for a forced `tool_choice`
+ * — the one structured-output mechanism graft can rely on across providers —
+ * ranges from exact to none. Each way that goes wrong gets its own narrow
+ * recovery: {@link ProviderResponseError} for a 200 that is not a completion,
+ * and a bounded rung ladder in {@link OpenAIChatModel.structured} for a
+ * structured reply the endpoint answers some other way. None of it changes the
+ * bytes on the wire for an endpoint that behaves.
*/
import OpenAI from "openai";
import { transportRetries } from "./types.js";
+import { unwrapMarkdownFence } from "./recover-tool.js";
import type { ChatModel, ChatRequest, ChatResponse, Message, ToolCall, ToolSpec, Usage } from "./types.js";
const PROVIDER = "openai";
/** Synthetic tool used to coerce a plain JSON object out of `{ kind: "json" }`. */
const JSON_TOOL = "emit_json";
+/** How one attempt asks for the tool the caller needs. See the rung ladder. */
+type ToolChoiceMode = "forced" | "required" | "auto";
+
+/** Backoff before re-sending a 200 that carried an error instead of a completion. */
+const EMPTY_RESPONSE_BACKOFF_MS = [500, 1000, 2000, 4000, 8000];
+
+/** How many nested emulated `{name, parameters}` wrappers to unwrap before stopping. */
+const MAX_UNWRAP_DEPTH = 2;
+
+/** Stand-in id for a payload lifted out of `content` — no wire call ever had one. */
+const CONTENT_CALL_ID = "from-content";
+
+/**
+ * Endpoints already seen to answer a forced `tool_choice` with something other
+ * than a tool call, keyed `<baseUrl>|<model>`. Process-local on purpose: it
+ * describes one gateway's current behaviour, the rung that works costs at most
+ * one extra call, and the next run re-probes for free.
+ *
+ * Two observations are proof, because neither survives a retry: a reply the
+ * endpoint did serve that just doesn't carry the asked-for call (prose where a
+ * tool call was forced), and — counted over {@link forcedFallbackThreshold}
+ * separate calls by {@link forcedToolChoiceRescues} — errored bodies on every
+ * forced-family rung of a call that a lower rung then answered.
+ */
+const forcedToolChoiceIgnored = new Set<string>();
+
+/**
+ * Rescued calls per `<baseUrl>|<model>`: structured calls where every
+ * forced-family rung failed with an errored body and the final "auto" rung then
+ * answered. One such call is a hiccup — the upstream behind the gateway may
+ * already be back, and a healthy endpoint must not spend the rest of the run
+ * on the slow rungs — so the downgrade waits for
+ * {@link forcedFallbackThreshold} of them, and a forced call that succeeds
+ * resets the count.
+ */
+const forcedToolChoiceRescues = new Map<string, number>();
+
+/**
+ * Rescued calls it takes to stop probing the forced rung on an endpoint +
+ * model (`GRAFT_LLM_FORCED_RESCUES`). Two by default: the first errored body
+ * says the endpoint's upstream blinked; a second call walking the same ladder
+ * to the same answer says the endpoint itself cannot serve the ask.
+ */
+function forcedFallbackThreshold(): number {
+ const raw = Number(process.env.GRAFT_LLM_FORCED_RESCUES);
+ return Number.isFinite(raw) && raw >= 1 ? Math.floor(raw) : 2;
+}
+
export interface OpenAIChatModelOptions {
apiKey: string;
model: string;
@@ -27,6 +86,8 @@ export interface OpenAIChatModelOptions {
headers?: Record<string, string>;
/** Inject a pre-built client (tests pass a stub; production omits it). */
client?: OpenAI;
+ /** Wait between retries of a 200-that-is-an-error (tests pass a no-op). */
+ sleep?: (ms: number) => Promise<void>;
}
type ChatParams = OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming;
@@ -130,14 +191,143 @@ function isRejectedToolsWithReasoning(err: unknown): boolean {
);
}
+/**
+ * A 200 that is not a completion.
+ *
+ * A proxy in front of a model can answer HTTP 200 with
+ * `{"error": {...}}` and no `choices` when the upstream call fails, because the
+ * failure belongs to a server the proxy itself did not talk to. The SDK's retry
+ * policy only ever sees the status, so it returns that body as a success, and
+ * the first `choices[0]` used to turn it into a TypeError naming nothing a user
+ * could act on. Retryable by construction — the request was never served — and
+ * carrying the provider's own message and code so the failure that does reach
+ * the caller says what the endpoint said.
+ */
+export class ProviderResponseError extends Error {
+ /** The status the provider actually sent. 200: that is the whole surprise. */
+ readonly status = 200;
+ /** Always true — the same request can succeed on a later attempt. */
+ readonly retryable = true;
+ constructor(
+ message: string,
+ /** The provider's own error code, when the body carried one. */
+ readonly code?: string | number,
+ ) {
+ super(message);
+ this.name = "ProviderResponseError";
+ }
+}
+
+/**
+ * The completion inside a 200 body, or a typed error saying what came back
+ * instead. The only gate onto the response fields: no caller downstream of it
+ * may touch `choices` before this has decided the body holds one.
+ */
+function completionOrThrow(resp: unknown): OpenAI.Chat.Completions.ChatCompletion {
+ const body = resp as { choices?: unknown; error?: unknown } | null | undefined;
+ const err = body?.error as { message?: unknown; code?: unknown } | undefined;
+ if (err) {
+ const message =
+ typeof err.message === "string" && err.message.trim()
+ ? err.message
+ : "the provider returned an error body with no choices";
+ const code = typeof err.code === "string" || typeof err.code === "number" ? err.code : undefined;
+ throw new ProviderResponseError(message, code);
+ }
+ if (!Array.isArray(body?.choices) || body.choices.length === 0) {
+ throw new ProviderResponseError("the provider returned a successful response with no choices");
+ }
+ return body as OpenAI.Chat.Completions.ChatCompletion;
+}
+
+/**
+ * A JSON value the model wrote into `content` instead of a tool call: fenced or
+ * bare, or wrapped in a sentence around it. Object before array, the same order
+ * the content recovery in ./recover-tool.ts uses, so both agree on what a
+ * half-written reply contains.
+ */
+function jsonFromContent(text: string): unknown {
+ const raw = unwrapMarkdownFence((text ?? "").trim());
+ if (!raw) return undefined;
+ try {
+ return JSON.parse(raw);
+ } catch {
+ /* a sentence around the payload — try the outermost braces/brackets */
+ }
+ for (const [open, close] of [
+ ["{", "}"],
+ ["[", "]"],
+ ] as const) {
+ const start = raw.indexOf(open);
+ const end = raw.lastIndexOf(close);
+ if (start < 0 || end <= start) continue;
+ try {
+ return JSON.parse(raw.slice(start, end + 1));
+ } catch {
+ /* not a single well-formed value — the caller judges it as prose */
+ }
+ }
+ return undefined;
+}
+
+/**
+ * The `{name, parameters|arguments|args}` call a model wrote into `content`
+ * instead of `tool_calls` (#129) — alone, emulated in an array, or wrapped in a
+ * sentence — and failing that a plain JSON object, which is what a gateway that
+ * ignored a forced `tool_choice` writes when one tool was offered.
+ *
+ * Deliberately conservative: only the tool the caller named is accepted, and
+ * the caller still validates the shape it asked for.
+ */
+function payloadFromContent(text: string, toolName: string, depth = 0): Record<string, unknown> | undefined {
+ const value = jsonFromContent(text);
+ if (!value || typeof value !== "object") return undefined;
+ if (Array.isArray(value)) {
+ for (const item of value) {
+ const hit = item && typeof item === "object" ? payloadOf(item as Record<string, unknown>, toolName, depth) : undefined;
+ if (hit) return hit;
+ }
+ return undefined;
+ }
+ return payloadOf(value as Record<string, unknown>, toolName, depth);
+}
+
+/** One emulated call object, at whatever depth of nesting it sits. */
+function payloadOf(obj: Record<string, unknown>, toolName: string, depth: number): Record<string, unknown> | undefined {
+ // A wrapper naming a different tool is not this caller's payload.
+ if (typeof obj.name === "string" && obj.name !== toolName) return undefined;
+ const params = obj.parameters ?? obj.arguments ?? obj.args;
+ if (params === undefined) return Object.keys(obj).length ? obj : undefined;
+ const parsed = typeof params === "string" ? tryParse(params) : params;
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
+ return depth < MAX_UNWRAP_DEPTH
+ ? payloadOf(parsed as Record<string, unknown>, toolName, depth + 1)
+ : (parsed as Record<string, unknown>);
+}
+
+function tryParse(s: string): unknown {
+ try {
+ return JSON.parse(s);
+ } catch {
+ return undefined;
+ }
+}
+
export class OpenAIChatModel implements ChatModel {
readonly label: string;
private client: OpenAI;
private model: string;
+ private sleep: (ms: number) => Promise<void>;
+ /** Key into {@link forcedToolChoiceIgnored}: one entry per endpoint + model. */
+ private choiceKey: string;
+ private forcedIgnored: boolean;
constructor(opts: OpenAIChatModelOptions) {
this.model = opts.model;
this.label = opts.label ?? `${PROVIDER}:${opts.model}`;
+ this.sleep = opts.sleep ?? ((ms) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+ this.choiceKey = `${opts.baseUrl ?? ""}|${opts.model}`;
+ this.forcedIgnored = forcedToolChoiceIgnored.has(this.choiceKey);
this.client =
opts.client ??
new OpenAI({
@@ -156,6 +346,8 @@ export class OpenAIChatModel implements ChatModel {
if (req.maxTokens !== undefined) params.max_tokens = req.maxTokens;
const fmt = req.responseFormat ?? { kind: "text" };
+ // The tool whose call answers the request, and the format that needs one.
+ let structured: { tool: string; kind: "json" | "tool" } | null = null;
if (fmt.kind === "json") {
// Coerce JSON via a forced synthetic tool — the one structured-output
// mechanism shared with Anthropic (no reliance on `response_format`).
@@ -163,16 +355,150 @@ export class OpenAIChatModel implements ChatModel {
...(tools ?? []),
{ type: "function", function: { name: JSON_TOOL, description: "Return the answer as a JSON object.", parameters: { type: "object", additionalProperties: true } } },
];
- params.tool_choice = { type: "function", function: { name: JSON_TOOL } };
+ structured = { tool: JSON_TOOL, kind: "json" };
} else if (fmt.kind === "tool") {
params.tools = tools;
- params.tool_choice = { type: "function", function: { name: fmt.name } };
+ structured = { tool: fmt.name, kind: "tool" };
} else if (tools) {
params.tools = tools;
}
- const resp = await this.createChatCompletion(params);
- return this.fromResponse(resp, fmt.kind);
+ if (!structured) return this.fromResponse(await this.complete(params, true), "text");
+ return this.structured(params, structured.tool, structured.kind);
+ }
+
+ /**
+ * A structured reply, however the endpoint is willing to produce one.
+ *
+ * A forced `tool_choice` is the only structured-output mechanism graft can
+ * rely on across providers, and it is the least portable corner of the
+ * OpenAI-compatible spec: an endpoint can reject the object form (handled one
+ * layer down), answer 200 with an error object, or quietly answer in prose.
+ * So walk a bounded ladder — the object form, then `"required"`, then
+ * `"auto"` with the tool named in the prompt — and take the payload from
+ * `content` if that is where it landed. `"required"` only stands in for the
+ * object form when one tool was offered, the same rule the 400 fallback
+ * applies: with several tools it would let the model pick freely rather than
+ * honour the caller's choice.
+ *
+ * An endpoint that honours the forced form never sees rung two — the first
+ * attempt answers the question and the ladder ends.
+ *
+ * An errored body is a weaker signal than a served-but-wrong reply, since it
+ * may belong to an upstream that a retry would reach, so it never downgrades
+ * on its own: it is counted instead. A call where every forced-family rung
+ * failed that way and the final "auto" rung then answered is a rescue, and
+ * the second rescue with no successful forced call in between is the
+ * endpoint's pattern, not its upstream's — that, and only that, switches
+ * later calls straight to "auto".
+ */
+ private async structured(base: ChatParams, tool: string, kind: "json" | "tool"): Promise<ChatResponse> {
+ const rungs: ToolChoiceMode[] = this.forcedIgnored
+ ? ["auto"]
+ : base.tools?.length === 1
+ ? ["forced", "required", "auto"]
+ : ["forced", "auto"];
+
+ // How many rungs of this call have answered with an errored body so far.
+ // The rescue judgement needs the whole call: forced-family rungs errored,
+ // and the rung that finally answered being the one below all of them.
+ let erroredAbove = 0;
+
+ for (const rung of rungs.slice(0, -1)) {
+ let res: ChatResponse;
+ try {
+ // No empty-response retry on a rung with a successor: changing the ask
+ // is a better next move than re-sending a request the endpoint cannot
+ // serve, and it costs the same one call.
+ res = this.fromResponse(await this.complete(this.attempt(base, rung, tool), false), kind);
+ } catch (err) {
+ if (!(err instanceof ProviderResponseError)) throw err;
+ erroredAbove += 1;
+ continue;
+ }
+ if (this.delivered(res, kind, tool)) {
+ // The forced rung answering is the strongest counter-evidence there
+ // is: whatever errored bodies came before, this endpoint serves the
+ // ask, and the count starts over.
+ if (rung === "forced") forcedToolChoiceRescues.delete(this.choiceKey);
+ return fromContentAsPayload(res, kind, tool);
+ }
+ this.rememberForcedIgnored();
+ }
+
+ // Last rung: "auto" plus the tool named in the prompt, retried on an
+ // errored body. Whatever comes back is the answer — the payload from
+ // `content` when that is where it landed, the raw reply otherwise, so the
+ // caller's own miss classification still sees exactly what the model said.
+ const last = this.fromResponse(await this.complete(this.attempt(base, "auto", tool), true), kind);
+ if (this.delivered(last, kind, tool) && erroredAbove === rungs.length - 1) this.noteForcedRescue();
+ return fromContentAsPayload(last, kind, tool);
+ }
+
+ /** One rung: the forced object form, its equivalent string form, or "auto" plus a prompt. */
+ private attempt(base: ChatParams, rung: ToolChoiceMode, tool: string): ChatParams {
+ if (rung === "forced") return { ...base, tool_choice: { type: "function", function: { name: tool } } };
+ if (rung === "required") return { ...base, tool_choice: "required" };
+ return { ...base, tool_choice: "auto", messages: withToolInstruction(base.messages, tool) };
+ }
+
+ /** Did the reply carry the payload the caller forced a tool for? */
+ private delivered(res: ChatResponse, kind: "json" | "tool", tool: string): boolean {
+ if (kind === "json") return jsonFromContent(res.text) !== undefined;
+ return res.toolCalls.some((c) => c.name === tool) || payloadFromContent(res.text, tool) !== undefined;
+ }
+
+ /**
+ * One more rescued call for this endpoint + model. When the count reaches
+ * the threshold with no successful forced call in between, the endpoint is
+ * marked the same way a served-but-wrong reply marks it.
+ */
+ private noteForcedRescue(): void {
+ if (this.forcedIgnored) return;
+ const rescues = (forcedToolChoiceRescues.get(this.choiceKey) ?? 0) + 1;
+ forcedToolChoiceRescues.set(this.choiceKey, rescues);
+ if (rescues >= forcedFallbackThreshold()) this.rememberForcedIgnored();
+ }
+
+ /**
+ * Remember, for this process, that the endpoint did not answer a forced
+ * `tool_choice` with a tool call, so later requests go straight to the rung
+ * that works. Logged once per endpoint + model — the set is shared by every
+ * instance, so a second adapter for the same endpoint never repeats it: it
+ * changes the bytes on the wire, which is exactly what someone reading a
+ * failing build needs told.
+ */
+ private rememberForcedIgnored(): void {
+ this.forcedIgnored = true;
+ if (forcedToolChoiceIgnored.has(this.choiceKey)) return;
+ forcedToolChoiceIgnored.add(this.choiceKey);
+ console.error(
+ `⚠ ${this.label}: the endpoint did not honor a forced tool_choice — asking for the tool in the prompt instead (tool_choice "auto") for the rest of this run.`,
+ );
+ }
+
+ /**
+ * One completion, with the transport's retry policy extended to a 200 that
+ * carried an error instead of one. The SDK retries 429, 5xx and connection
+ * failures but never a successful status, and that body is the endpoint
+ * reporting a failed upstream call — the same judgement, so the same
+ * `GRAFT_LLM_RETRIES` budget and backoff apply, and the typed error is what
+ * reaches the caller once that budget is spent.
+ */
+ private async complete(params: ChatParams, retryErroredBody: boolean): Promise<OpenAI.Chat.Completions.ChatCompletion> {
+ const retries = retryErroredBody ? transportRetries() : 0;
+ for (let attempt = 0; ; attempt++) {
+ let failure: ProviderResponseError;
+ try {
+ return completionOrThrow(await this.createChatCompletion(params));
+ } catch (err) {
+ if (!(err instanceof ProviderResponseError)) throw err;
+ failure = err;
+ }
+ if (attempt >= retries) throw failure;
+ const wait = EMPTY_RESPONSE_BACKOFF_MS[Math.min(attempt, EMPTY_RESPONSE_BACKOFF_MS.length - 1)]!;
+ await this.sleep(wait);
+ }
}
/**
@@ -245,6 +571,9 @@ export class OpenAIChatModel implements ChatModel {
resp: OpenAI.Chat.Completions.ChatCompletion,
format: "text" | "json" | "tool",
): ChatResponse {
+ // `resp` came through `completionOrThrow`, so `choices` holds at least one
+ // entry by now: this is a peek at the first one, not a dereference of a
+ // possibly-missing array.
const choice = resp.choices[0];
const msg = choice?.message;
const rawCalls = (msg?.tool_calls ?? []).filter(
@@ -288,6 +617,48 @@ export class OpenAIChatModel implements ChatModel {
}
}
+/**
+ * Name the tool in a system turn of its own, so an "auto" rung has something to
+ * act on. A separate message leaves every part the caller wrote — including a
+ * cache breakpoint — byte-for-byte as it was, so this costs no cache.
+ */
+function withToolInstruction(messages: ChatMessage[], tool: string): ChatMessage[] {
+ const line: ChatMessage = {
+ role: "system",
+ content: `Answer by calling the ${tool} tool. Put your entire answer in that tool's arguments and reply with nothing else.`,
+ } as ChatMessage;
+ // After the last system turn, or first when the caller sent none.
+ const at = messages.map((m) => m.role).lastIndexOf("system") + 1;
+ return [...messages.slice(0, at), line, ...messages.slice(at)];
+}
+
+/**
+ * An accepted rung's answer, put in the shape the caller asked for. A model that
+ * ignored a forced `tool_choice` often produced the payload anyway, as text
+ * (#129) — so surface it as the JSON text a `{ kind: "json" }` caller expects,
+ * or as a call to the named tool a `{ kind: "tool" }` caller expects, rather
+ * than reporting an empty turn. A reply with no recognizable payload is passed
+ * through untouched, so the caller's own miss classification still sees exactly
+ * what the model said.
+ */
+function fromContentAsPayload(res: ChatResponse, kind: "json" | "tool", tool: string): ChatResponse {
+ if (kind === "json") {
+ const value = jsonFromContent(res.text);
+ if (value === undefined) return res;
+ const text = JSON.stringify(value);
+ return { ...res, text, assistant: { ...res.assistant, content: text } };
+ }
+ if (res.toolCalls.some((c) => c.name === tool)) return res;
+ const args = payloadFromContent(res.text, tool);
+ if (!args) return res;
+ const call: ToolCall = { id: CONTENT_CALL_ID, name: tool, args };
+ return {
+ ...res,
+ toolCalls: [...res.toolCalls, call],
+ assistant: { ...res.assistant, toolCalls: [...(res.assistant.toolCalls ?? []), call] },
+ };
+}
+
/** Cached tokens are inside `prompt_tokens`; subtract so `input` is uncached-only. */
function normalizeUsage(u: OpenAI.Completions.CompletionUsage | undefined): Usage {
const prompt = u?.prompt_tokens ?? 0;
diff --git a/src/ai/llm/recover-tool.ts b/src/ai/llm/recover-tool.ts
index 625a91bd..63fcfdad 100644
--- a/src/ai/llm/recover-tool.ts
+++ b/src/ai/llm/recover-tool.ts
@@ -12,7 +12,7 @@
*/
/** Leading/trailing ```json fence, linear in `raw.length` — no quantified-whitespace regex. */
-function unwrapMarkdownFence(raw: string): string {
+export function unwrapMarkdownFence(raw: string): string {
let s = raw;
if (s.startsWith("```")) {
s = s.slice(3);
diff --git a/test/llm-structured-fallback.test.ts b/test/llm-structured-fallback.test.ts
new file mode 100644
index 00000000..5aaa98a1
--- /dev/null
+++ b/test/llm-structured-fallback.test.ts
@@ -0,0 +1,398 @@
+/**
+ * An OpenAI-compatible endpoint that does not behave like the spec: a 200 whose
+ * body is an error object (or has no `choices` at all), and a forced
+ * `tool_choice` that comes back as something other than a tool call.
+ *
+ * Network-free — every test drives a STUB client that replays the exact bodies
+ * such an endpoint produces, and records what went out on the wire, so the
+ * recovery ladder is asserted step by step.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type OpenAI from "openai";
+import { OpenAIChatModel } from "../src/ai/llm/openai.js";
+import type { ChatRequest } from "../src/ai/llm/types.js";
+
+/** A stub client that answers each call with the next queued body. */
+function stubClient(...bodies: unknown[]): { client: OpenAI; calls: any[] } {
+ const calls: any[] = [];
+ const client = {
+ chat: {
+ completions: {
+ create: async (params: any) => {
+ calls.push(params);
+ const body = bodies[Math.min(calls.length - 1, bodies.length - 1)];
+ if (body instanceof Error) throw body;
+ return body;
+ },
+ },
+ },
+ } as unknown as OpenAI;
+ return { client, calls };
+}
+
+/** No-op backoff, recording the waits so the retry ladder is observable. */
+function instantSleep(): { sleep: (ms: number) => Promise<void>; waits: number[] } {
+ const waits: number[] = [];
+ return { sleep: async (ms) => void waits.push(ms), waits };
+}
+
+/** The 200-with-an-error-body an unhappy gateway answers with. */
+function erroredBody(message = "Provider returned an empty response", code: string | number = 502) {
+ return { error: { message, code } };
+}
+
+function textBody(content: string) {
+ return {
+ choices: [{ message: { content, tool_calls: [] }, finish_reason: "stop" }],
+ usage: { prompt_tokens: 10, completion_tokens: 5 },
+ };
+}
+
+function toolBody(name: string, args: unknown) {
+ return {
+ choices: [
+ {
+ message: {
+ content: null,
+ tool_calls: [{ id: "c1", type: "function", function: { name, arguments: JSON.stringify(args) } }],
+ },
+ finish_reason: "tool_calls",
+ },
+ ],
+ usage: { prompt_tokens: 10, completion_tokens: 5 },
+ };
+}
+
+const TOOL_REQ: ChatRequest = {
+ messages: [{ role: "system", content: "sys" }, { role: "user", content: "go" }],
+ tools: [{ name: "record_graph", description: "d", parameters: { type: "object" } }],
+ responseFormat: { kind: "tool", name: "record_graph" },
+};
+
+/** Capture console.error so the one-off notices neither leak nor collide. */
+async function withCapturedError<T>(fn: () => Promise<T>): Promise<{ result: T; err: string[] }> {
+ const err: string[] = [];
+ const orig = console.error;
+ console.error = (...args: unknown[]) => void err.push(args.map(String).join(" "));
+ try {
+ return { result: await fn(), err };
+ } finally {
+ console.error = orig;
+ }
+}
+
+async function withRetries<T>(value: string, fn: () => Promise<T>): Promise<T> {
+ return withEnv("GRAFT_LLM_RETRIES", value, fn);
+}
+
+/** Set one env var for the duration of `fn`, restoring whatever it held. */
+async function withEnv<T>(name: string, value: string, fn: () => Promise<T>): Promise<T> {
+ const orig = process.env[name];
+ process.env[name] = value;
+ try {
+ return await fn();
+ } finally {
+ if (orig === undefined) delete process.env[name];
+ else process.env[name] = orig;
+ }
+}
+
+/**
+ * Matched by name, not by `instanceof`, so this file also loads against a
+ * build of the adapter that has no such class — the tests then fail on their
+ * own assertions instead of on an import.
+ */
+function isProviderResponseError(e: unknown): e is Error & { code?: unknown; retryable?: boolean } {
+ return e instanceof Error && e.name === "ProviderResponseError";
+}
+
+test("openai: a 200 carrying an error object is retried, and the retry succeeds", async () => {
+ const { client, calls } = stubClient(erroredBody(), textBody("hello"));
+ const { sleep, waits } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "errored-then-fine", client, sleep });
+
+ const res = await withRetries("2", () => m.create({ messages: [{ role: "user", content: "hi" }] }));
+
+ assert.equal(calls.length, 2);
+ assert.deepEqual(waits, [500]); // one backoff, not a tight retry loop
+ assert.equal(res.text, "hello");
+});
+
+test("openai: an errored 200 spends the retry budget, then throws a typed retryable error", async () => {
+ const { client, calls } = stubClient(erroredBody("upstream refused", "bad_gateway"));
+ const { sleep, waits } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "always-errored", client, sleep });
+
+ const err = await withRetries("1", () =>
+ m.create({ messages: [{ role: "user", content: "hi" }] }).then(
+ () => assert.fail("expected the errored body to surface"),
+ (e: unknown) => e,
+ ),
+ );
+
+ assert.equal(calls.length, 2); // GRAFT_LLM_RETRIES=1 → one retry
+ assert.deepEqual(waits, [500]);
+ assert.ok(isProviderResponseError(err), `expected a ProviderResponseError, got ${String(err)}`);
+ assert.equal(err.message, "upstream refused"); // the provider's own words, not ours
+ assert.equal(err.code, "bad_gateway");
+ assert.equal((err as { retryable?: boolean }).retryable, true);
+ assert.equal(err instanceof TypeError, false);
+});
+
+test("openai: a 200 with no choices is a typed error, never a TypeError from choices[0]", async () => {
+ for (const [what, body] of [
+ ["an empty object", {}],
+ ["choices but none in it", { choices: [] }],
+ ["a body that is not JSON at all", "gateway timeout"],
+ ] as const) {
+ const { client, calls } = stubClient(body);
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: `no-choices-${String(what.length)}`, client, sleep });
+
+ const err = await withRetries("0", () =>
+ m.create({ messages: [{ role: "user", content: "hi" }] }).then(
+ () => assert.fail(`expected ${what} to surface as an error`),
+ (e: unknown) => e,
+ ),
+ );
+
+ assert.equal(calls.length, 1);
+ assert.ok(isProviderResponseError(err), `${what}: got ${String(err)}`);
+ assert.equal(err instanceof TypeError, false);
+ }
+});
+
+test("openai: forced → required → auto fallback reaches a tool call", async () => {
+ const { client, calls } = stubClient(erroredBody(), textBody("I would rather explain it"), toolBody("record_graph", { nodes: [1, 2] }));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "ladder-model", client, sleep });
+
+ const { result, err } = await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.deepEqual(
+ calls.map((c) => c.tool_choice),
+ [{ type: "function", function: { name: "record_graph" } }, "required", "auto"],
+ );
+ assert.deepEqual(calls.map((c) => c.messages.length), [2, 2, 3]); // the auto rung names the tool
+ assert.match(String(calls[2].messages[1].content), /record_graph/);
+ assert.equal(result.toolCalls.length, 1);
+ assert.deepEqual(result.toolCalls[0].args, { nodes: [1, 2] });
+ assert.equal(err.length, 1); // the downgrade is announced once
+});
+
+test("openai: JSON written into content is returned as the JSON text json mode asked for", async () => {
+ const fenced = "```json\n{\"correct\":true}\n```";
+ const { client, calls } = stubClient(textBody("Sure, here you go"), textBody("Still explaining."), textBody(fenced));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "json-in-content", client, sleep });
+
+ const { result } = await withCapturedError(() =>
+ m.create({ messages: [{ role: "user", content: "grade" }], responseFormat: { kind: "json" } }),
+ );
+
+ assert.equal(calls.length, 3);
+ assert.equal(calls.at(-1)?.tool_choice, "auto");
+ assert.deepEqual(JSON.parse(result.text), { correct: true }); // fence stripped, not passed through
+ assert.equal(result.toolCalls.length, 0); // the synthetic tool stays hidden
+});
+
+test("openai: a tool payload left in content surfaces as a call to the named tool", async () => {
+ const { client, calls } = stubClient(textBody('Here is the graph: {"nodes":[1]}'));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "tool-in-content", client, sleep });
+
+ const { result } = await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(calls.length, 1); // the payload was in the reply; no second ask
+ assert.equal(result.toolCalls.length, 1);
+ assert.equal(result.toolCalls[0].name, "record_graph");
+ assert.deepEqual(result.toolCalls[0].args, { nodes: [1] });
+});
+
+test("openai: an emulated tool call in content on the last rung is unwrapped", async () => {
+ const emulated = '[{"name":"record_graph","parameters":{"nodes":[2]}}]';
+ const { client, calls } = stubClient(textBody("prose"), textBody("more prose"), textBody(emulated));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "emulated-last-rung", client, sleep });
+
+ const { result } = await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(calls.at(-1)?.tool_choice, "auto");
+ assert.equal(result.toolCalls.length, 1);
+ assert.deepEqual(result.toolCalls[0].args, { nodes: [2] });
+});
+
+test("openai: a reply with no payload at all is handed back untouched, for the caller to judge", async () => {
+ const { client, calls } = stubClient(textBody("I am not going to call a tool."));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "pure-prose", client, sleep });
+
+ const { result } = await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(calls.length, 3);
+ assert.equal(result.toolCalls.length, 0);
+ assert.equal(result.text, "I am not going to call a tool."); // nothing invented, nothing hidden
+});
+
+test("openai: a payload already in content is not re-requested on another rung", async () => {
+ const { client, calls } = stubClient(toolBody("record_graph", { nodes: [7] }));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "emulated-wrapper", client, sleep });
+
+ const { result } = await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(calls.length, 1);
+ assert.deepEqual(result.toolCalls[0].args, { nodes: [7] });
+});
+
+test("openai: once a model is known to ignore a forced tool_choice, later calls go straight to auto", async () => {
+ const { client, calls } = stubClient(textBody("here is a prose answer"));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "remembered-model", client, sleep });
+
+ const first = await withCapturedError(() => m.create(TOOL_REQ));
+ assert.equal(calls.length, 3); // forced, required, auto
+ assert.equal(first.err.length, 1);
+ assert.match(first.err[0]!, /remembered-model/);
+
+ calls.length = 0;
+ const second = await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0].tool_choice, "auto");
+ assert.equal(second.err.length, 0); // noted once, not once per call
+});
+
+test("openai: a second instance of the same endpoint + model skips the forced rung too", async () => {
+ const first = stubClient(textBody("prose"));
+ const { sleep } = instantSleep();
+ await withCapturedError(
+ () => new OpenAIChatModel({ apiKey: "x", model: "shared-endpoint-model", baseUrl: "http://stub", client: first.client, sleep }).create(TOOL_REQ),
+ );
+
+ const second = stubClient(textBody("prose"));
+ const m = new OpenAIChatModel({ apiKey: "x", model: "shared-endpoint-model", baseUrl: "http://stub", client: second.client, sleep });
+ await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(first.calls.length, 3);
+ assert.equal(second.calls.length, 1);
+ assert.equal(second.calls[0].tool_choice, "auto");
+});
+
+const MULTI_TOOL_REQ: ChatRequest = {
+ messages: [{ role: "user", content: "go" }],
+ tools: [
+ { name: "record_graph", description: "d", parameters: { type: "object" } },
+ { name: "draw_diagram", description: "d", parameters: { type: "object" } },
+ ],
+ responseFormat: { kind: "tool", name: "record_graph" },
+};
+
+test("openai: two ladder-rescued calls teach the model to go straight to auto", async () => {
+ // A rescued call: every forced-family rung answered with an errored body and
+ // the "auto" rung then answered. One of those is a hiccup; the pattern
+ // repeating on a second call is the endpoint, not the upstream.
+ const { client, calls } = stubClient(
+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // call 1: rescued
+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [2] }), // call 2: rescued again
+ toolBody("record_graph", { nodes: [3] }), // call 3: no ladder left to walk
+ );
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "learns-from-errors", client, sleep });
+
+ const first = await withCapturedError(() => m.create(TOOL_REQ));
+ assert.equal(calls.length, 3); // still probing: one rescued call proves nothing
+ assert.equal(first.err.length, 0);
+
+ const second = await withCapturedError(() => m.create(TOOL_REQ));
+ assert.equal(calls.length, 6); // the second rescue is what makes the pattern
+ assert.equal(second.err.length, 1); // the switch is announced, once, here
+
+ const third = await withCapturedError(() => m.create(TOOL_REQ));
+ assert.equal(calls.length, 7); // the third call asks once
+ assert.equal(calls[6].tool_choice, "auto");
+ assert.deepEqual(third.result.toolCalls[0].args, { nodes: [3] });
+ assert.equal(third.err.length, 0);
+});
+
+test("openai: one rescued call plus one clean forced success does not switch the model", async () => {
+ const { client, calls } = stubClient(
+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // rescued once
+ toolBody("record_graph", { nodes: [2] }), // forced works again — the hiccup is forgiven
+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [3] }), // this rescue is the first again
+ );
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "reset-by-success", client, sleep });
+
+ const first = await withCapturedError(() => m.create(TOOL_REQ));
+ assert.equal(calls.length, 3);
+ assert.equal(first.err.length, 0); // one rescued call is not a verdict
+
+ const second = await withCapturedError(() => m.create(TOOL_REQ));
+ assert.equal(calls.length, 4); // the forced rung answered; no ask was changed
+ assert.deepEqual(calls[3].tool_choice, { type: "function", function: { name: "record_graph" } });
+
+ const third = await withCapturedError(() => m.create(TOOL_REQ));
+ assert.equal(calls.length, 7); // the success reset the count, so this rescue is the first, not the second
+ assert.equal(third.err.length, 0); // and no downgrade was announced
+});
+
+test("openai: a transient errored body followed by a forced success on retry does not count", async () => {
+ // Several tools, so the ladder is forced → auto with no required rung: the
+ // errored body on the first call is rescued, the retry (the next call) is
+ // answered by the forced rung itself, and nothing may be held against the
+ // endpoint afterwards.
+ const { client, calls } = stubClient(
+ erroredBody(), toolBody("record_graph", { nodes: [1] }), // rescued past the errored body
+ toolBody("record_graph", { nodes: [2] }), // the retry: forced succeeds
+ erroredBody(), toolBody("record_graph", { nodes: [3] }), // rescued again — still only once on the count
+ );
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "transient-then-fine", client, sleep });
+
+ const first = await withCapturedError(() => m.create(MULTI_TOOL_REQ));
+ assert.deepEqual(calls.map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }, "auto"]);
+ assert.equal(first.err.length, 0);
+
+ const second = await withCapturedError(() => m.create(MULTI_TOOL_REQ));
+ assert.deepEqual(calls.slice(2).map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }]);
+ assert.deepEqual(second.result.toolCalls[0].args, { nodes: [2] });
+
+ const third = await withCapturedError(() => m.create(MULTI_TOOL_REQ));
+ assert.deepEqual(calls.slice(3).map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }, "auto"]);
+ assert.deepEqual(third.result.toolCalls[0].args, { nodes: [3] });
+ assert.equal(first.err.length + second.err.length + third.err.length, 0); // never marked
+});
+
+test("openai: GRAFT_LLM_FORCED_RESCUES sets how many rescued calls it takes to switch", async () => {
+ const { client, calls } = stubClient(
+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // the one rescued call
+ toolBody("record_graph", { nodes: [2] }), // every later call asks once, in auto
+ );
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "threshold-one", client, sleep });
+
+ const first = await withEnv("GRAFT_LLM_FORCED_RESCUES", "1", () => withCapturedError(() => m.create(TOOL_REQ)));
+ assert.equal(calls.length, 3);
+ assert.equal(first.err.length, 1); // a single rescued call was enough
+
+ await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(calls.length, 4);
+ assert.equal(calls[3].tool_choice, "auto");
+});
+
+test("openai: an endpoint that honors the forced tool_choice sees exactly one call, unchanged", async () => {
+ const { client, calls } = stubClient(toolBody("record_graph", { nodes: [1] }));
+ const { sleep } = instantSleep();
+ const m = new OpenAIChatModel({ apiKey: "x", model: "well-behaved-model", client, sleep });
+
+ const { result, err } = await withCapturedError(() => m.create(TOOL_REQ));
+
+ assert.equal(calls.length, 1);
+ assert.deepEqual(calls[0].tool_choice, { type: "function", function: { name: "record_graph" } });
+ assert.deepEqual(calls[0].messages, TOOL_REQ.messages.map((msg) => ({ role: msg.role, content: msg.content })));
+ assert.deepEqual(result.toolCalls[0].args, { nodes: [1] });
+ assert.equal(err.length, 0);
+});