Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
fix(llm): retry a 200 that carries an error, and fall back when a forced tool_choice isn't honoured (#502)
Alex Matthewsafdaea9f6f71
diff --git a/src/ai/llm/openai.ts b/src/ai/llm/openai.tsindex c23840d6..28bd3632 100644--- a/src/ai/llm/openai.ts+++ b/src/ai/llm/openai.ts@@ -8,15 +8,74 @@* is forwarded, cache breakpoints become `cache_control` content parts (which* OpenRouter forwards to Anthropic), and cached tokens are subtracted out of the* input count so {@link Usage.input} is uncached-only.+ *+ * Past that, everything here is tolerance rather than preference. An+ * OpenAI-compatible endpoint is a gateway whose own upstream calls can fail+ * while the HTTP status stays 200, and whose support for a forced `tool_choice`+ * — the one structured-output mechanism graft can rely on across providers —+ * ranges from exact to none. Each way that goes wrong gets its own narrow+ * recovery: {@link ProviderResponseError} for a 200 that is not a completion,+ * and a bounded rung ladder in {@link OpenAIChatModel.structured} for a+ * structured reply the endpoint answers some other way. None of it changes the+ * bytes on the wire for an endpoint that behaves.*/import OpenAI from "openai";import { transportRetries } from "./types.js";+import { unwrapMarkdownFence } from "./recover-tool.js";import type { ChatModel, ChatRequest, ChatResponse, Message, ToolCall, ToolSpec, Usage } from "./types.js";const PROVIDER = "openai";/** Synthetic tool used to coerce a plain JSON object out of `{ kind: "json" }`. */const JSON_TOOL = "emit_json";+/** How one attempt asks for the tool the caller needs. See the rung ladder. */+type ToolChoiceMode = "forced" | "required" | "auto";++/** Backoff before re-sending a 200 that carried an error instead of a completion. */+const EMPTY_RESPONSE_BACKOFF_MS = [500, 1000, 2000, 4000, 8000];++/** How many nested emulated `{name, parameters}` wrappers to unwrap before stopping. */+const MAX_UNWRAP_DEPTH = 2;++/** Stand-in id for a payload lifted out of `content` — no wire call ever had one. */+const CONTENT_CALL_ID = "from-content";++/**+ * Endpoints already seen to answer a forced `tool_choice` with something other+ * than a tool call, keyed `<baseUrl>|<model>`. Process-local on purpose: it+ * describes one gateway's current behaviour, the rung that works costs at most+ * one extra call, and the next run re-probes for free.+ *+ * Two observations are proof, because neither survives a retry: a reply the+ * endpoint did serve that just doesn't carry the asked-for call (prose where a+ * tool call was forced), and — counted over {@link forcedFallbackThreshold}+ * separate calls by {@link forcedToolChoiceRescues} — errored bodies on every+ * forced-family rung of a call that a lower rung then answered.+ */+const forcedToolChoiceIgnored = new Set<string>();++/**+ * Rescued calls per `<baseUrl>|<model>`: structured calls where every+ * forced-family rung failed with an errored body and the final "auto" rung then+ * answered. One such call is a hiccup — the upstream behind the gateway may+ * already be back, and a healthy endpoint must not spend the rest of the run+ * on the slow rungs — so the downgrade waits for+ * {@link forcedFallbackThreshold} of them, and a forced call that succeeds+ * resets the count.+ */+const forcedToolChoiceRescues = new Map<string, number>();++/**+ * Rescued calls it takes to stop probing the forced rung on an endpoint ++ * model (`GRAFT_LLM_FORCED_RESCUES`). Two by default: the first errored body+ * says the endpoint's upstream blinked; a second call walking the same ladder+ * to the same answer says the endpoint itself cannot serve the ask.+ */+function forcedFallbackThreshold(): number {+ const raw = Number(process.env.GRAFT_LLM_FORCED_RESCUES);+ return Number.isFinite(raw) && raw >= 1 ? Math.floor(raw) : 2;+}+export interface OpenAIChatModelOptions {apiKey: string;model: string;@@ -27,6 +86,8 @@ export interface OpenAIChatModelOptions {headers?: Record<string, string>;/** Inject a pre-built client (tests pass a stub; production omits it). */client?: OpenAI;+ /** Wait between retries of a 200-that-is-an-error (tests pass a no-op). */+ sleep?: (ms: number) => Promise<void>;}type ChatParams = OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming;@@ -130,14 +191,143 @@ function isRejectedToolsWithReasoning(err: unknown): boolean {);}+/**+ * A 200 that is not a completion.+ *+ * A proxy in front of a model can answer HTTP 200 with+ * `{"error": {...}}` and no `choices` when the upstream call fails, because the+ * failure belongs to a server the proxy itself did not talk to. The SDK's retry+ * policy only ever sees the status, so it returns that body as a success, and+ * the first `choices[0]` used to turn it into a TypeError naming nothing a user+ * could act on. Retryable by construction — the request was never served — and+ * carrying the provider's own message and code so the failure that does reach+ * the caller says what the endpoint said.+ */+export class ProviderResponseError extends Error {+ /** The status the provider actually sent. 200: that is the whole surprise. */+ readonly status = 200;+ /** Always true — the same request can succeed on a later attempt. */+ readonly retryable = true;+ constructor(+ message: string,+ /** The provider's own error code, when the body carried one. */+ readonly code?: string | number,+ ) {+ super(message);+ this.name = "ProviderResponseError";+ }+}++/**+ * The completion inside a 200 body, or a typed error saying what came back+ * instead. The only gate onto the response fields: no caller downstream of it+ * may touch `choices` before this has decided the body holds one.+ */+function completionOrThrow(resp: unknown): OpenAI.Chat.Completions.ChatCompletion {+ const body = resp as { choices?: unknown; error?: unknown } | null | undefined;+ const err = body?.error as { message?: unknown; code?: unknown } | undefined;+ if (err) {+ const message =+ typeof err.message === "string" && err.message.trim()+ ? err.message+ : "the provider returned an error body with no choices";+ const code = typeof err.code === "string" || typeof err.code === "number" ? err.code : undefined;+ throw new ProviderResponseError(message, code);+ }+ if (!Array.isArray(body?.choices) || body.choices.length === 0) {+ throw new ProviderResponseError("the provider returned a successful response with no choices");+ }+ return body as OpenAI.Chat.Completions.ChatCompletion;+}++/**+ * A JSON value the model wrote into `content` instead of a tool call: fenced or+ * bare, or wrapped in a sentence around it. Object before array, the same order+ * the content recovery in ./recover-tool.ts uses, so both agree on what a+ * half-written reply contains.+ */+function jsonFromContent(text: string): unknown {+ const raw = unwrapMarkdownFence((text ?? "").trim());+ if (!raw) return undefined;+ try {+ return JSON.parse(raw);+ } catch {+ /* a sentence around the payload — try the outermost braces/brackets */+ }+ for (const [open, close] of [+ ["{", "}"],+ ["[", "]"],+ ] as const) {+ const start = raw.indexOf(open);+ const end = raw.lastIndexOf(close);+ if (start < 0 || end <= start) continue;+ try {+ return JSON.parse(raw.slice(start, end + 1));+ } catch {+ /* not a single well-formed value — the caller judges it as prose */+ }+ }+ return undefined;+}++/**+ * The `{name, parameters|arguments|args}` call a model wrote into `content`+ * instead of `tool_calls` (#129) — alone, emulated in an array, or wrapped in a+ * sentence — and failing that a plain JSON object, which is what a gateway that+ * ignored a forced `tool_choice` writes when one tool was offered.+ *+ * Deliberately conservative: only the tool the caller named is accepted, and+ * the caller still validates the shape it asked for.+ */+function payloadFromContent(text: string, toolName: string, depth = 0): Record<string, unknown> | undefined {+ const value = jsonFromContent(text);+ if (!value || typeof value !== "object") return undefined;+ if (Array.isArray(value)) {+ for (const item of value) {+ const hit = item && typeof item === "object" ? payloadOf(item as Record<string, unknown>, toolName, depth) : undefined;+ if (hit) return hit;+ }+ return undefined;+ }+ return payloadOf(value as Record<string, unknown>, toolName, depth);+}++/** One emulated call object, at whatever depth of nesting it sits. */+function payloadOf(obj: Record<string, unknown>, toolName: string, depth: number): Record<string, unknown> | undefined {+ // A wrapper naming a different tool is not this caller's payload.+ if (typeof obj.name === "string" && obj.name !== toolName) return undefined;+ const params = obj.parameters ?? obj.arguments ?? obj.args;+ if (params === undefined) return Object.keys(obj).length ? obj : undefined;+ const parsed = typeof params === "string" ? tryParse(params) : params;+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;+ return depth < MAX_UNWRAP_DEPTH+ ? payloadOf(parsed as Record<string, unknown>, toolName, depth + 1)+ : (parsed as Record<string, unknown>);+}++function tryParse(s: string): unknown {+ try {+ return JSON.parse(s);+ } catch {+ return undefined;+ }+}+export class OpenAIChatModel implements ChatModel {readonly label: string;private client: OpenAI;private model: string;+ private sleep: (ms: number) => Promise<void>;+ /** Key into {@link forcedToolChoiceIgnored}: one entry per endpoint + model. */+ private choiceKey: string;+ private forcedIgnored: boolean;constructor(opts: OpenAIChatModelOptions) {this.model = opts.model;this.label = opts.label ?? `${PROVIDER}:${opts.model}`;+ this.sleep = opts.sleep ?? ((ms) => new Promise<void>((resolve) => setTimeout(resolve, ms)));+ this.choiceKey = `${opts.baseUrl ?? ""}|${opts.model}`;+ this.forcedIgnored = forcedToolChoiceIgnored.has(this.choiceKey);this.client =opts.client ??new OpenAI({@@ -156,6 +346,8 @@ export class OpenAIChatModel implements ChatModel {if (req.maxTokens !== undefined) params.max_tokens = req.maxTokens;const fmt = req.responseFormat ?? { kind: "text" };+ // The tool whose call answers the request, and the format that needs one.+ let structured: { tool: string; kind: "json" | "tool" } | null = null;if (fmt.kind === "json") {// Coerce JSON via a forced synthetic tool — the one structured-output// mechanism shared with Anthropic (no reliance on `response_format`).@@ -163,16 +355,150 @@ export class OpenAIChatModel implements ChatModel {...(tools ?? []),{ type: "function", function: { name: JSON_TOOL, description: "Return the answer as a JSON object.", parameters: { type: "object", additionalProperties: true } } },];- params.tool_choice = { type: "function", function: { name: JSON_TOOL } };+ structured = { tool: JSON_TOOL, kind: "json" };} else if (fmt.kind === "tool") {params.tools = tools;- params.tool_choice = { type: "function", function: { name: fmt.name } };+ structured = { tool: fmt.name, kind: "tool" };} else if (tools) {params.tools = tools;}- const resp = await this.createChatCompletion(params);- return this.fromResponse(resp, fmt.kind);+ if (!structured) return this.fromResponse(await this.complete(params, true), "text");+ return this.structured(params, structured.tool, structured.kind);+ }++ /**+ * A structured reply, however the endpoint is willing to produce one.+ *+ * A forced `tool_choice` is the only structured-output mechanism graft can+ * rely on across providers, and it is the least portable corner of the+ * OpenAI-compatible spec: an endpoint can reject the object form (handled one+ * layer down), answer 200 with an error object, or quietly answer in prose.+ * So walk a bounded ladder — the object form, then `"required"`, then+ * `"auto"` with the tool named in the prompt — and take the payload from+ * `content` if that is where it landed. `"required"` only stands in for the+ * object form when one tool was offered, the same rule the 400 fallback+ * applies: with several tools it would let the model pick freely rather than+ * honour the caller's choice.+ *+ * An endpoint that honours the forced form never sees rung two — the first+ * attempt answers the question and the ladder ends.+ *+ * An errored body is a weaker signal than a served-but-wrong reply, since it+ * may belong to an upstream that a retry would reach, so it never downgrades+ * on its own: it is counted instead. A call where every forced-family rung+ * failed that way and the final "auto" rung then answered is a rescue, and+ * the second rescue with no successful forced call in between is the+ * endpoint's pattern, not its upstream's — that, and only that, switches+ * later calls straight to "auto".+ */+ private async structured(base: ChatParams, tool: string, kind: "json" | "tool"): Promise<ChatResponse> {+ const rungs: ToolChoiceMode[] = this.forcedIgnored+ ? ["auto"]+ : base.tools?.length === 1+ ? ["forced", "required", "auto"]+ : ["forced", "auto"];++ // How many rungs of this call have answered with an errored body so far.+ // The rescue judgement needs the whole call: forced-family rungs errored,+ // and the rung that finally answered being the one below all of them.+ let erroredAbove = 0;++ for (const rung of rungs.slice(0, -1)) {+ let res: ChatResponse;+ try {+ // No empty-response retry on a rung with a successor: changing the ask+ // is a better next move than re-sending a request the endpoint cannot+ // serve, and it costs the same one call.+ res = this.fromResponse(await this.complete(this.attempt(base, rung, tool), false), kind);+ } catch (err) {+ if (!(err instanceof ProviderResponseError)) throw err;+ erroredAbove += 1;+ continue;+ }+ if (this.delivered(res, kind, tool)) {+ // The forced rung answering is the strongest counter-evidence there+ // is: whatever errored bodies came before, this endpoint serves the+ // ask, and the count starts over.+ if (rung === "forced") forcedToolChoiceRescues.delete(this.choiceKey);+ return fromContentAsPayload(res, kind, tool);+ }+ this.rememberForcedIgnored();+ }++ // Last rung: "auto" plus the tool named in the prompt, retried on an+ // errored body. Whatever comes back is the answer — the payload from+ // `content` when that is where it landed, the raw reply otherwise, so the+ // caller's own miss classification still sees exactly what the model said.+ const last = this.fromResponse(await this.complete(this.attempt(base, "auto", tool), true), kind);+ if (this.delivered(last, kind, tool) && erroredAbove === rungs.length - 1) this.noteForcedRescue();+ return fromContentAsPayload(last, kind, tool);+ }++ /** One rung: the forced object form, its equivalent string form, or "auto" plus a prompt. */+ private attempt(base: ChatParams, rung: ToolChoiceMode, tool: string): ChatParams {+ if (rung === "forced") return { ...base, tool_choice: { type: "function", function: { name: tool } } };+ if (rung === "required") return { ...base, tool_choice: "required" };+ return { ...base, tool_choice: "auto", messages: withToolInstruction(base.messages, tool) };+ }++ /** Did the reply carry the payload the caller forced a tool for? */+ private delivered(res: ChatResponse, kind: "json" | "tool", tool: string): boolean {+ if (kind === "json") return jsonFromContent(res.text) !== undefined;+ return res.toolCalls.some((c) => c.name === tool) || payloadFromContent(res.text, tool) !== undefined;+ }++ /**+ * One more rescued call for this endpoint + model. When the count reaches+ * the threshold with no successful forced call in between, the endpoint is+ * marked the same way a served-but-wrong reply marks it.+ */+ private noteForcedRescue(): void {+ if (this.forcedIgnored) return;+ const rescues = (forcedToolChoiceRescues.get(this.choiceKey) ?? 0) + 1;+ forcedToolChoiceRescues.set(this.choiceKey, rescues);+ if (rescues >= forcedFallbackThreshold()) this.rememberForcedIgnored();+ }++ /**+ * Remember, for this process, that the endpoint did not answer a forced+ * `tool_choice` with a tool call, so later requests go straight to the rung+ * that works. Logged once per endpoint + model — the set is shared by every+ * instance, so a second adapter for the same endpoint never repeats it: it+ * changes the bytes on the wire, which is exactly what someone reading a+ * failing build needs told.+ */+ private rememberForcedIgnored(): void {+ this.forcedIgnored = true;+ if (forcedToolChoiceIgnored.has(this.choiceKey)) return;+ forcedToolChoiceIgnored.add(this.choiceKey);+ console.error(+ `⚠ ${this.label}: the endpoint did not honor a forced tool_choice — asking for the tool in the prompt instead (tool_choice "auto") for the rest of this run.`,+ );+ }++ /**+ * One completion, with the transport's retry policy extended to a 200 that+ * carried an error instead of one. The SDK retries 429, 5xx and connection+ * failures but never a successful status, and that body is the endpoint+ * reporting a failed upstream call — the same judgement, so the same+ * `GRAFT_LLM_RETRIES` budget and backoff apply, and the typed error is what+ * reaches the caller once that budget is spent.+ */+ private async complete(params: ChatParams, retryErroredBody: boolean): Promise<OpenAI.Chat.Completions.ChatCompletion> {+ const retries = retryErroredBody ? transportRetries() : 0;+ for (let attempt = 0; ; attempt++) {+ let failure: ProviderResponseError;+ try {+ return completionOrThrow(await this.createChatCompletion(params));+ } catch (err) {+ if (!(err instanceof ProviderResponseError)) throw err;+ failure = err;+ }+ if (attempt >= retries) throw failure;+ const wait = EMPTY_RESPONSE_BACKOFF_MS[Math.min(attempt, EMPTY_RESPONSE_BACKOFF_MS.length - 1)]!;+ await this.sleep(wait);+ }}/**@@ -245,6 +571,9 @@ export class OpenAIChatModel implements ChatModel {resp: OpenAI.Chat.Completions.ChatCompletion,format: "text" | "json" | "tool",): ChatResponse {+ // `resp` came through `completionOrThrow`, so `choices` holds at least one+ // entry by now: this is a peek at the first one, not a dereference of a+ // possibly-missing array.const choice = resp.choices[0];const msg = choice?.message;const rawCalls = (msg?.tool_calls ?? []).filter(@@ -288,6 +617,48 @@ export class OpenAIChatModel implements ChatModel {}}+/**+ * Name the tool in a system turn of its own, so an "auto" rung has something to+ * act on. A separate message leaves every part the caller wrote — including a+ * cache breakpoint — byte-for-byte as it was, so this costs no cache.+ */+function withToolInstruction(messages: ChatMessage[], tool: string): ChatMessage[] {+ const line: ChatMessage = {+ role: "system",+ content: `Answer by calling the ${tool} tool. Put your entire answer in that tool's arguments and reply with nothing else.`,+ } as ChatMessage;+ // After the last system turn, or first when the caller sent none.+ const at = messages.map((m) => m.role).lastIndexOf("system") + 1;+ return [...messages.slice(0, at), line, ...messages.slice(at)];+}++/**+ * An accepted rung's answer, put in the shape the caller asked for. A model that+ * ignored a forced `tool_choice` often produced the payload anyway, as text+ * (#129) — so surface it as the JSON text a `{ kind: "json" }` caller expects,+ * or as a call to the named tool a `{ kind: "tool" }` caller expects, rather+ * than reporting an empty turn. A reply with no recognizable payload is passed+ * through untouched, so the caller's own miss classification still sees exactly+ * what the model said.+ */+function fromContentAsPayload(res: ChatResponse, kind: "json" | "tool", tool: string): ChatResponse {+ if (kind === "json") {+ const value = jsonFromContent(res.text);+ if (value === undefined) return res;+ const text = JSON.stringify(value);+ return { ...res, text, assistant: { ...res.assistant, content: text } };+ }+ if (res.toolCalls.some((c) => c.name === tool)) return res;+ const args = payloadFromContent(res.text, tool);+ if (!args) return res;+ const call: ToolCall = { id: CONTENT_CALL_ID, name: tool, args };+ return {+ ...res,+ toolCalls: [...res.toolCalls, call],+ assistant: { ...res.assistant, toolCalls: [...(res.assistant.toolCalls ?? []), call] },+ };+}+/** Cached tokens are inside `prompt_tokens`; subtract so `input` is uncached-only. */function normalizeUsage(u: OpenAI.Completions.CompletionUsage | undefined): Usage {const prompt = u?.prompt_tokens ?? 0;diff --git a/src/ai/llm/recover-tool.ts b/src/ai/llm/recover-tool.tsindex 625a91bd..63fcfdad 100644--- a/src/ai/llm/recover-tool.ts+++ b/src/ai/llm/recover-tool.ts@@ -12,7 +12,7 @@*//** Leading/trailing ```json fence, linear in `raw.length` — no quantified-whitespace regex. */-function unwrapMarkdownFence(raw: string): string {+export function unwrapMarkdownFence(raw: string): string {let s = raw;if (s.startsWith("```")) {s = s.slice(3);diff --git a/test/llm-structured-fallback.test.ts b/test/llm-structured-fallback.test.tsnew file mode 100644index 00000000..5aaa98a1--- /dev/null+++ b/test/llm-structured-fallback.test.ts@@ -0,0 +1,398 @@+/**+ * An OpenAI-compatible endpoint that does not behave like the spec: a 200 whose+ * body is an error object (or has no `choices` at all), and a forced+ * `tool_choice` that comes back as something other than a tool call.+ *+ * Network-free — every test drives a STUB client that replays the exact bodies+ * such an endpoint produces, and records what went out on the wire, so the+ * recovery ladder is asserted step by step.+ */+import { test } from "node:test";+import assert from "node:assert/strict";+import type OpenAI from "openai";+import { OpenAIChatModel } from "../src/ai/llm/openai.js";+import type { ChatRequest } from "../src/ai/llm/types.js";++/** A stub client that answers each call with the next queued body. */+function stubClient(...bodies: unknown[]): { client: OpenAI; calls: any[] } {+ const calls: any[] = [];+ const client = {+ chat: {+ completions: {+ create: async (params: any) => {+ calls.push(params);+ const body = bodies[Math.min(calls.length - 1, bodies.length - 1)];+ if (body instanceof Error) throw body;+ return body;+ },+ },+ },+ } as unknown as OpenAI;+ return { client, calls };+}++/** No-op backoff, recording the waits so the retry ladder is observable. */+function instantSleep(): { sleep: (ms: number) => Promise<void>; waits: number[] } {+ const waits: number[] = [];+ return { sleep: async (ms) => void waits.push(ms), waits };+}++/** The 200-with-an-error-body an unhappy gateway answers with. */+function erroredBody(message = "Provider returned an empty response", code: string | number = 502) {+ return { error: { message, code } };+}++function textBody(content: string) {+ return {+ choices: [{ message: { content, tool_calls: [] }, finish_reason: "stop" }],+ usage: { prompt_tokens: 10, completion_tokens: 5 },+ };+}++function toolBody(name: string, args: unknown) {+ return {+ choices: [+ {+ message: {+ content: null,+ tool_calls: [{ id: "c1", type: "function", function: { name, arguments: JSON.stringify(args) } }],+ },+ finish_reason: "tool_calls",+ },+ ],+ usage: { prompt_tokens: 10, completion_tokens: 5 },+ };+}++const TOOL_REQ: ChatRequest = {+ messages: [{ role: "system", content: "sys" }, { role: "user", content: "go" }],+ tools: [{ name: "record_graph", description: "d", parameters: { type: "object" } }],+ responseFormat: { kind: "tool", name: "record_graph" },+};++/** Capture console.error so the one-off notices neither leak nor collide. */+async function withCapturedError<T>(fn: () => Promise<T>): Promise<{ result: T; err: string[] }> {+ const err: string[] = [];+ const orig = console.error;+ console.error = (...args: unknown[]) => void err.push(args.map(String).join(" "));+ try {+ return { result: await fn(), err };+ } finally {+ console.error = orig;+ }+}++async function withRetries<T>(value: string, fn: () => Promise<T>): Promise<T> {+ return withEnv("GRAFT_LLM_RETRIES", value, fn);+}++/** Set one env var for the duration of `fn`, restoring whatever it held. */+async function withEnv<T>(name: string, value: string, fn: () => Promise<T>): Promise<T> {+ const orig = process.env[name];+ process.env[name] = value;+ try {+ return await fn();+ } finally {+ if (orig === undefined) delete process.env[name];+ else process.env[name] = orig;+ }+}++/**+ * Matched by name, not by `instanceof`, so this file also loads against a+ * build of the adapter that has no such class — the tests then fail on their+ * own assertions instead of on an import.+ */+function isProviderResponseError(e: unknown): e is Error & { code?: unknown; retryable?: boolean } {+ return e instanceof Error && e.name === "ProviderResponseError";+}++test("openai: a 200 carrying an error object is retried, and the retry succeeds", async () => {+ const { client, calls } = stubClient(erroredBody(), textBody("hello"));+ const { sleep, waits } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "errored-then-fine", client, sleep });++ const res = await withRetries("2", () => m.create({ messages: [{ role: "user", content: "hi" }] }));++ assert.equal(calls.length, 2);+ assert.deepEqual(waits, [500]); // one backoff, not a tight retry loop+ assert.equal(res.text, "hello");+});++test("openai: an errored 200 spends the retry budget, then throws a typed retryable error", async () => {+ const { client, calls } = stubClient(erroredBody("upstream refused", "bad_gateway"));+ const { sleep, waits } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "always-errored", client, sleep });++ const err = await withRetries("1", () =>+ m.create({ messages: [{ role: "user", content: "hi" }] }).then(+ () => assert.fail("expected the errored body to surface"),+ (e: unknown) => e,+ ),+ );++ assert.equal(calls.length, 2); // GRAFT_LLM_RETRIES=1 → one retry+ assert.deepEqual(waits, [500]);+ assert.ok(isProviderResponseError(err), `expected a ProviderResponseError, got ${String(err)}`);+ assert.equal(err.message, "upstream refused"); // the provider's own words, not ours+ assert.equal(err.code, "bad_gateway");+ assert.equal((err as { retryable?: boolean }).retryable, true);+ assert.equal(err instanceof TypeError, false);+});++test("openai: a 200 with no choices is a typed error, never a TypeError from choices[0]", async () => {+ for (const [what, body] of [+ ["an empty object", {}],+ ["choices but none in it", { choices: [] }],+ ["a body that is not JSON at all", "gateway timeout"],+ ] as const) {+ const { client, calls } = stubClient(body);+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: `no-choices-${String(what.length)}`, client, sleep });++ const err = await withRetries("0", () =>+ m.create({ messages: [{ role: "user", content: "hi" }] }).then(+ () => assert.fail(`expected ${what} to surface as an error`),+ (e: unknown) => e,+ ),+ );++ assert.equal(calls.length, 1);+ assert.ok(isProviderResponseError(err), `${what}: got ${String(err)}`);+ assert.equal(err instanceof TypeError, false);+ }+});++test("openai: forced → required → auto fallback reaches a tool call", async () => {+ const { client, calls } = stubClient(erroredBody(), textBody("I would rather explain it"), toolBody("record_graph", { nodes: [1, 2] }));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "ladder-model", client, sleep });++ const { result, err } = await withCapturedError(() => m.create(TOOL_REQ));++ assert.deepEqual(+ calls.map((c) => c.tool_choice),+ [{ type: "function", function: { name: "record_graph" } }, "required", "auto"],+ );+ assert.deepEqual(calls.map((c) => c.messages.length), [2, 2, 3]); // the auto rung names the tool+ assert.match(String(calls[2].messages[1].content), /record_graph/);+ assert.equal(result.toolCalls.length, 1);+ assert.deepEqual(result.toolCalls[0].args, { nodes: [1, 2] });+ assert.equal(err.length, 1); // the downgrade is announced once+});++test("openai: JSON written into content is returned as the JSON text json mode asked for", async () => {+ const fenced = "```json\n{\"correct\":true}\n```";+ const { client, calls } = stubClient(textBody("Sure, here you go"), textBody("Still explaining."), textBody(fenced));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "json-in-content", client, sleep });++ const { result } = await withCapturedError(() =>+ m.create({ messages: [{ role: "user", content: "grade" }], responseFormat: { kind: "json" } }),+ );++ assert.equal(calls.length, 3);+ assert.equal(calls.at(-1)?.tool_choice, "auto");+ assert.deepEqual(JSON.parse(result.text), { correct: true }); // fence stripped, not passed through+ assert.equal(result.toolCalls.length, 0); // the synthetic tool stays hidden+});++test("openai: a tool payload left in content surfaces as a call to the named tool", async () => {+ const { client, calls } = stubClient(textBody('Here is the graph: {"nodes":[1]}'));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "tool-in-content", client, sleep });++ const { result } = await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(calls.length, 1); // the payload was in the reply; no second ask+ assert.equal(result.toolCalls.length, 1);+ assert.equal(result.toolCalls[0].name, "record_graph");+ assert.deepEqual(result.toolCalls[0].args, { nodes: [1] });+});++test("openai: an emulated tool call in content on the last rung is unwrapped", async () => {+ const emulated = '[{"name":"record_graph","parameters":{"nodes":[2]}}]';+ const { client, calls } = stubClient(textBody("prose"), textBody("more prose"), textBody(emulated));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "emulated-last-rung", client, sleep });++ const { result } = await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(calls.at(-1)?.tool_choice, "auto");+ assert.equal(result.toolCalls.length, 1);+ assert.deepEqual(result.toolCalls[0].args, { nodes: [2] });+});++test("openai: a reply with no payload at all is handed back untouched, for the caller to judge", async () => {+ const { client, calls } = stubClient(textBody("I am not going to call a tool."));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "pure-prose", client, sleep });++ const { result } = await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(calls.length, 3);+ assert.equal(result.toolCalls.length, 0);+ assert.equal(result.text, "I am not going to call a tool."); // nothing invented, nothing hidden+});++test("openai: a payload already in content is not re-requested on another rung", async () => {+ const { client, calls } = stubClient(toolBody("record_graph", { nodes: [7] }));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "emulated-wrapper", client, sleep });++ const { result } = await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(calls.length, 1);+ assert.deepEqual(result.toolCalls[0].args, { nodes: [7] });+});++test("openai: once a model is known to ignore a forced tool_choice, later calls go straight to auto", async () => {+ const { client, calls } = stubClient(textBody("here is a prose answer"));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "remembered-model", client, sleep });++ const first = await withCapturedError(() => m.create(TOOL_REQ));+ assert.equal(calls.length, 3); // forced, required, auto+ assert.equal(first.err.length, 1);+ assert.match(first.err[0]!, /remembered-model/);++ calls.length = 0;+ const second = await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(calls.length, 1);+ assert.equal(calls[0].tool_choice, "auto");+ assert.equal(second.err.length, 0); // noted once, not once per call+});++test("openai: a second instance of the same endpoint + model skips the forced rung too", async () => {+ const first = stubClient(textBody("prose"));+ const { sleep } = instantSleep();+ await withCapturedError(+ () => new OpenAIChatModel({ apiKey: "x", model: "shared-endpoint-model", baseUrl: "http://stub", client: first.client, sleep }).create(TOOL_REQ),+ );++ const second = stubClient(textBody("prose"));+ const m = new OpenAIChatModel({ apiKey: "x", model: "shared-endpoint-model", baseUrl: "http://stub", client: second.client, sleep });+ await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(first.calls.length, 3);+ assert.equal(second.calls.length, 1);+ assert.equal(second.calls[0].tool_choice, "auto");+});++const MULTI_TOOL_REQ: ChatRequest = {+ messages: [{ role: "user", content: "go" }],+ tools: [+ { name: "record_graph", description: "d", parameters: { type: "object" } },+ { name: "draw_diagram", description: "d", parameters: { type: "object" } },+ ],+ responseFormat: { kind: "tool", name: "record_graph" },+};++test("openai: two ladder-rescued calls teach the model to go straight to auto", async () => {+ // A rescued call: every forced-family rung answered with an errored body and+ // the "auto" rung then answered. One of those is a hiccup; the pattern+ // repeating on a second call is the endpoint, not the upstream.+ const { client, calls } = stubClient(+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // call 1: rescued+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [2] }), // call 2: rescued again+ toolBody("record_graph", { nodes: [3] }), // call 3: no ladder left to walk+ );+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "learns-from-errors", client, sleep });++ const first = await withCapturedError(() => m.create(TOOL_REQ));+ assert.equal(calls.length, 3); // still probing: one rescued call proves nothing+ assert.equal(first.err.length, 0);++ const second = await withCapturedError(() => m.create(TOOL_REQ));+ assert.equal(calls.length, 6); // the second rescue is what makes the pattern+ assert.equal(second.err.length, 1); // the switch is announced, once, here++ const third = await withCapturedError(() => m.create(TOOL_REQ));+ assert.equal(calls.length, 7); // the third call asks once+ assert.equal(calls[6].tool_choice, "auto");+ assert.deepEqual(third.result.toolCalls[0].args, { nodes: [3] });+ assert.equal(third.err.length, 0);+});++test("openai: one rescued call plus one clean forced success does not switch the model", async () => {+ const { client, calls } = stubClient(+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // rescued once+ toolBody("record_graph", { nodes: [2] }), // forced works again — the hiccup is forgiven+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [3] }), // this rescue is the first again+ );+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "reset-by-success", client, sleep });++ const first = await withCapturedError(() => m.create(TOOL_REQ));+ assert.equal(calls.length, 3);+ assert.equal(first.err.length, 0); // one rescued call is not a verdict++ const second = await withCapturedError(() => m.create(TOOL_REQ));+ assert.equal(calls.length, 4); // the forced rung answered; no ask was changed+ assert.deepEqual(calls[3].tool_choice, { type: "function", function: { name: "record_graph" } });++ const third = await withCapturedError(() => m.create(TOOL_REQ));+ assert.equal(calls.length, 7); // the success reset the count, so this rescue is the first, not the second+ assert.equal(third.err.length, 0); // and no downgrade was announced+});++test("openai: a transient errored body followed by a forced success on retry does not count", async () => {+ // Several tools, so the ladder is forced → auto with no required rung: the+ // errored body on the first call is rescued, the retry (the next call) is+ // answered by the forced rung itself, and nothing may be held against the+ // endpoint afterwards.+ const { client, calls } = stubClient(+ erroredBody(), toolBody("record_graph", { nodes: [1] }), // rescued past the errored body+ toolBody("record_graph", { nodes: [2] }), // the retry: forced succeeds+ erroredBody(), toolBody("record_graph", { nodes: [3] }), // rescued again — still only once on the count+ );+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "transient-then-fine", client, sleep });++ const first = await withCapturedError(() => m.create(MULTI_TOOL_REQ));+ assert.deepEqual(calls.map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }, "auto"]);+ assert.equal(first.err.length, 0);++ const second = await withCapturedError(() => m.create(MULTI_TOOL_REQ));+ assert.deepEqual(calls.slice(2).map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }]);+ assert.deepEqual(second.result.toolCalls[0].args, { nodes: [2] });++ const third = await withCapturedError(() => m.create(MULTI_TOOL_REQ));+ assert.deepEqual(calls.slice(3).map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }, "auto"]);+ assert.deepEqual(third.result.toolCalls[0].args, { nodes: [3] });+ assert.equal(first.err.length + second.err.length + third.err.length, 0); // never marked+});++test("openai: GRAFT_LLM_FORCED_RESCUES sets how many rescued calls it takes to switch", async () => {+ const { client, calls } = stubClient(+ erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // the one rescued call+ toolBody("record_graph", { nodes: [2] }), // every later call asks once, in auto+ );+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "threshold-one", client, sleep });++ const first = await withEnv("GRAFT_LLM_FORCED_RESCUES", "1", () => withCapturedError(() => m.create(TOOL_REQ)));+ assert.equal(calls.length, 3);+ assert.equal(first.err.length, 1); // a single rescued call was enough++ await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(calls.length, 4);+ assert.equal(calls[3].tool_choice, "auto");+});++test("openai: an endpoint that honors the forced tool_choice sees exactly one call, unchanged", async () => {+ const { client, calls } = stubClient(toolBody("record_graph", { nodes: [1] }));+ const { sleep } = instantSleep();+ const m = new OpenAIChatModel({ apiKey: "x", model: "well-behaved-model", client, sleep });++ const { result, err } = await withCapturedError(() => m.create(TOOL_REQ));++ assert.equal(calls.length, 1);+ assert.deepEqual(calls[0].tool_choice, { type: "function", function: { name: "record_graph" } });+ assert.deepEqual(calls[0].messages, TOOL_REQ.messages.map((msg) => ({ role: msg.role, content: msg.content })));+ assert.deepEqual(result.toolCalls[0].args, { nodes: [1] });+ assert.equal(err.length, 0);+});