Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
/**
* An OpenAI-compatible endpoint that does not behave like the spec: a 200 whose
* body is an error object (or has no `choices` at all), and a forced
* `tool_choice` that comes back as something other than a tool call.
*
* Network-free — every test drives a STUB client that replays the exact bodies
* such an endpoint produces, and records what went out on the wire, so the
* recovery ladder is asserted step by step.
*/
import { test } from "node:test";
import assert from "node:assert/strict";
import type OpenAI from "openai";
import { OpenAIChatModel } from "../src/ai/llm/openai.js";
import type { ChatRequest } from "../src/ai/llm/types.js";
/** A stub client that answers each call with the next queued body. */
function stubClient(...bodies: unknown[]): { client: OpenAI; calls: any[] } {
const calls: any[] = [];
const client = {
chat: {
completions: {
create: async (params: any) => {
calls.push(params);
const body = bodies[Math.min(calls.length - 1, bodies.length - 1)];
if (body instanceof Error) throw body;
return body;
},
},
},
} as unknown as OpenAI;
return { client, calls };
}
/** No-op backoff, recording the waits so the retry ladder is observable. */
function instantSleep(): { sleep: (ms: number) => Promise<void>; waits: number[] } {
const waits: number[] = [];
return { sleep: async (ms) => void waits.push(ms), waits };
}
/** The 200-with-an-error-body an unhappy gateway answers with. */
function erroredBody(message = "Provider returned an empty response", code: string | number = 502) {
return { error: { message, code } };
}
function textBody(content: string) {
return {
choices: [{ message: { content, tool_calls: [] }, finish_reason: "stop" }],
usage: { prompt_tokens: 10, completion_tokens: 5 },
};
}
function toolBody(name: string, args: unknown) {
return {
choices: [
{
message: {
content: null,
tool_calls: [{ id: "c1", type: "function", function: { name, arguments: JSON.stringify(args) } }],
},
finish_reason: "tool_calls",
},
],
usage: { prompt_tokens: 10, completion_tokens: 5 },
};
}
const TOOL_REQ: ChatRequest = {
messages: [{ role: "system", content: "sys" }, { role: "user", content: "go" }],
tools: [{ name: "record_graph", description: "d", parameters: { type: "object" } }],
responseFormat: { kind: "tool", name: "record_graph" },
};
/** Capture console.error so the one-off notices neither leak nor collide. */
async function withCapturedError<T>(fn: () => Promise<T>): Promise<{ result: T; err: string[] }> {
const err: string[] = [];
const orig = console.error;
console.error = (...args: unknown[]) => void err.push(args.map(String).join(" "));
try {
return { result: await fn(), err };
} finally {
console.error = orig;
}
}
async function withRetries<T>(value: string, fn: () => Promise<T>): Promise<T> {
return withEnv("GRAFT_LLM_RETRIES", value, fn);
}
/** Set one env var for the duration of `fn`, restoring whatever it held. */
async function withEnv<T>(name: string, value: string, fn: () => Promise<T>): Promise<T> {
const orig = process.env[name];
process.env[name] = value;
try {
return await fn();
} finally {
if (orig === undefined) delete process.env[name];
else process.env[name] = orig;
}
}
/**
* Matched by name, not by `instanceof`, so this file also loads against a
* build of the adapter that has no such class — the tests then fail on their
* own assertions instead of on an import.
*/
function isProviderResponseError(e: unknown): e is Error & { code?: unknown; retryable?: boolean } {
return e instanceof Error && e.name === "ProviderResponseError";
}
test("openai: a 200 carrying an error object is retried, and the retry succeeds", async () => {
const { client, calls } = stubClient(erroredBody(), textBody("hello"));
const { sleep, waits } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "errored-then-fine", client, sleep });
const res = await withRetries("2", () => m.create({ messages: [{ role: "user", content: "hi" }] }));
assert.equal(calls.length, 2);
assert.deepEqual(waits, [500]); // one backoff, not a tight retry loop
assert.equal(res.text, "hello");
});
test("openai: an errored 200 spends the retry budget, then throws a typed retryable error", async () => {
const { client, calls } = stubClient(erroredBody("upstream refused", "bad_gateway"));
const { sleep, waits } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "always-errored", client, sleep });
const err = await withRetries("1", () =>
m.create({ messages: [{ role: "user", content: "hi" }] }).then(
() => assert.fail("expected the errored body to surface"),
(e: unknown) => e,
),
);
assert.equal(calls.length, 2); // GRAFT_LLM_RETRIES=1 → one retry
assert.deepEqual(waits, [500]);
assert.ok(isProviderResponseError(err), `expected a ProviderResponseError, got ${String(err)}`);
assert.equal(err.message, "upstream refused"); // the provider's own words, not ours
assert.equal(err.code, "bad_gateway");
assert.equal((err as { retryable?: boolean }).retryable, true);
assert.equal(err instanceof TypeError, false);
});
test("openai: a 200 with no choices is a typed error, never a TypeError from choices[0]", async () => {
for (const [what, body] of [
["an empty object", {}],
["choices but none in it", { choices: [] }],
["a body that is not JSON at all", "gateway timeout"],
] as const) {
const { client, calls } = stubClient(body);
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: `no-choices-${String(what.length)}`, client, sleep });
const err = await withRetries("0", () =>
m.create({ messages: [{ role: "user", content: "hi" }] }).then(
() => assert.fail(`expected ${what} to surface as an error`),
(e: unknown) => e,
),
);
assert.equal(calls.length, 1);
assert.ok(isProviderResponseError(err), `${what}: got ${String(err)}`);
assert.equal(err instanceof TypeError, false);
}
});
test("openai: forced → required → auto fallback reaches a tool call", async () => {
const { client, calls } = stubClient(erroredBody(), textBody("I would rather explain it"), toolBody("record_graph", { nodes: [1, 2] }));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "ladder-model", client, sleep });
const { result, err } = await withCapturedError(() => m.create(TOOL_REQ));
assert.deepEqual(
calls.map((c) => c.tool_choice),
[{ type: "function", function: { name: "record_graph" } }, "required", "auto"],
);
assert.deepEqual(calls.map((c) => c.messages.length), [2, 2, 3]); // the auto rung names the tool
assert.match(String(calls[2].messages[1].content), /record_graph/);
assert.equal(result.toolCalls.length, 1);
assert.deepEqual(result.toolCalls[0].args, { nodes: [1, 2] });
assert.equal(err.length, 1); // the downgrade is announced once
});
test("openai: JSON written into content is returned as the JSON text json mode asked for", async () => {
const fenced = "```json\n{\"correct\":true}\n```";
const { client, calls } = stubClient(textBody("Sure, here you go"), textBody("Still explaining."), textBody(fenced));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "json-in-content", client, sleep });
const { result } = await withCapturedError(() =>
m.create({ messages: [{ role: "user", content: "grade" }], responseFormat: { kind: "json" } }),
);
assert.equal(calls.length, 3);
assert.equal(calls.at(-1)?.tool_choice, "auto");
assert.deepEqual(JSON.parse(result.text), { correct: true }); // fence stripped, not passed through
assert.equal(result.toolCalls.length, 0); // the synthetic tool stays hidden
});
test("openai: a tool payload left in content surfaces as a call to the named tool", async () => {
const { client, calls } = stubClient(textBody('Here is the graph: {"nodes":[1]}'));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "tool-in-content", client, sleep });
const { result } = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 1); // the payload was in the reply; no second ask
assert.equal(result.toolCalls.length, 1);
assert.equal(result.toolCalls[0].name, "record_graph");
assert.deepEqual(result.toolCalls[0].args, { nodes: [1] });
});
test("openai: an emulated tool call in content on the last rung is unwrapped", async () => {
const emulated = '[{"name":"record_graph","parameters":{"nodes":[2]}}]';
const { client, calls } = stubClient(textBody("prose"), textBody("more prose"), textBody(emulated));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "emulated-last-rung", client, sleep });
const { result } = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.at(-1)?.tool_choice, "auto");
assert.equal(result.toolCalls.length, 1);
assert.deepEqual(result.toolCalls[0].args, { nodes: [2] });
});
test("openai: a reply with no payload at all is handed back untouched, for the caller to judge", async () => {
const { client, calls } = stubClient(textBody("I am not going to call a tool."));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "pure-prose", client, sleep });
const { result } = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 3);
assert.equal(result.toolCalls.length, 0);
assert.equal(result.text, "I am not going to call a tool."); // nothing invented, nothing hidden
});
test("openai: a payload already in content is not re-requested on another rung", async () => {
const { client, calls } = stubClient(toolBody("record_graph", { nodes: [7] }));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "emulated-wrapper", client, sleep });
const { result } = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 1);
assert.deepEqual(result.toolCalls[0].args, { nodes: [7] });
});
test("openai: once a model is known to ignore a forced tool_choice, later calls go straight to auto", async () => {
const { client, calls } = stubClient(textBody("here is a prose answer"));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "remembered-model", client, sleep });
const first = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 3); // forced, required, auto
assert.equal(first.err.length, 1);
assert.match(first.err[0]!, /remembered-model/);
calls.length = 0;
const second = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 1);
assert.equal(calls[0].tool_choice, "auto");
assert.equal(second.err.length, 0); // noted once, not once per call
});
test("openai: a second instance of the same endpoint + model skips the forced rung too", async () => {
const first = stubClient(textBody("prose"));
const { sleep } = instantSleep();
await withCapturedError(
() => new OpenAIChatModel({ apiKey: "x", model: "shared-endpoint-model", baseUrl: "http://stub", client: first.client, sleep }).create(TOOL_REQ),
);
const second = stubClient(textBody("prose"));
const m = new OpenAIChatModel({ apiKey: "x", model: "shared-endpoint-model", baseUrl: "http://stub", client: second.client, sleep });
await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(first.calls.length, 3);
assert.equal(second.calls.length, 1);
assert.equal(second.calls[0].tool_choice, "auto");
});
const MULTI_TOOL_REQ: ChatRequest = {
messages: [{ role: "user", content: "go" }],
tools: [
{ name: "record_graph", description: "d", parameters: { type: "object" } },
{ name: "draw_diagram", description: "d", parameters: { type: "object" } },
],
responseFormat: { kind: "tool", name: "record_graph" },
};
test("openai: two ladder-rescued calls teach the model to go straight to auto", async () => {
// A rescued call: every forced-family rung answered with an errored body and
// the "auto" rung then answered. One of those is a hiccup; the pattern
// repeating on a second call is the endpoint, not the upstream.
const { client, calls } = stubClient(
erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // call 1: rescued
erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [2] }), // call 2: rescued again
toolBody("record_graph", { nodes: [3] }), // call 3: no ladder left to walk
);
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "learns-from-errors", client, sleep });
const first = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 3); // still probing: one rescued call proves nothing
assert.equal(first.err.length, 0);
const second = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 6); // the second rescue is what makes the pattern
assert.equal(second.err.length, 1); // the switch is announced, once, here
const third = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 7); // the third call asks once
assert.equal(calls[6].tool_choice, "auto");
assert.deepEqual(third.result.toolCalls[0].args, { nodes: [3] });
assert.equal(third.err.length, 0);
});
test("openai: one rescued call plus one clean forced success does not switch the model", async () => {
const { client, calls } = stubClient(
erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // rescued once
toolBody("record_graph", { nodes: [2] }), // forced works again — the hiccup is forgiven
erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [3] }), // this rescue is the first again
);
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "reset-by-success", client, sleep });
const first = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 3);
assert.equal(first.err.length, 0); // one rescued call is not a verdict
const second = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 4); // the forced rung answered; no ask was changed
assert.deepEqual(calls[3].tool_choice, { type: "function", function: { name: "record_graph" } });
const third = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 7); // the success reset the count, so this rescue is the first, not the second
assert.equal(third.err.length, 0); // and no downgrade was announced
});
test("openai: a transient errored body followed by a forced success on retry does not count", async () => {
// Several tools, so the ladder is forced → auto with no required rung: the
// errored body on the first call is rescued, the retry (the next call) is
// answered by the forced rung itself, and nothing may be held against the
// endpoint afterwards.
const { client, calls } = stubClient(
erroredBody(), toolBody("record_graph", { nodes: [1] }), // rescued past the errored body
toolBody("record_graph", { nodes: [2] }), // the retry: forced succeeds
erroredBody(), toolBody("record_graph", { nodes: [3] }), // rescued again — still only once on the count
);
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "transient-then-fine", client, sleep });
const first = await withCapturedError(() => m.create(MULTI_TOOL_REQ));
assert.deepEqual(calls.map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }, "auto"]);
assert.equal(first.err.length, 0);
const second = await withCapturedError(() => m.create(MULTI_TOOL_REQ));
assert.deepEqual(calls.slice(2).map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }]);
assert.deepEqual(second.result.toolCalls[0].args, { nodes: [2] });
const third = await withCapturedError(() => m.create(MULTI_TOOL_REQ));
assert.deepEqual(calls.slice(3).map((c) => c.tool_choice), [{ type: "function", function: { name: "record_graph" } }, "auto"]);
assert.deepEqual(third.result.toolCalls[0].args, { nodes: [3] });
assert.equal(first.err.length + second.err.length + third.err.length, 0); // never marked
});
test("openai: GRAFT_LLM_FORCED_RESCUES sets how many rescued calls it takes to switch", async () => {
const { client, calls } = stubClient(
erroredBody(), erroredBody(), toolBody("record_graph", { nodes: [1] }), // the one rescued call
toolBody("record_graph", { nodes: [2] }), // every later call asks once, in auto
);
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "threshold-one", client, sleep });
const first = await withEnv("GRAFT_LLM_FORCED_RESCUES", "1", () => withCapturedError(() => m.create(TOOL_REQ)));
assert.equal(calls.length, 3);
assert.equal(first.err.length, 1); // a single rescued call was enough
await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 4);
assert.equal(calls[3].tool_choice, "auto");
});
test("openai: an endpoint that honors the forced tool_choice sees exactly one call, unchanged", async () => {
const { client, calls } = stubClient(toolBody("record_graph", { nodes: [1] }));
const { sleep } = instantSleep();
const m = new OpenAIChatModel({ apiKey: "x", model: "well-behaved-model", client, sleep });
const { result, err } = await withCapturedError(() => m.create(TOOL_REQ));
assert.equal(calls.length, 1);
assert.deepEqual(calls[0].tool_choice, { type: "function", function: { name: "record_graph" } });
assert.deepEqual(calls[0].messages, TOOL_REQ.messages.map((msg) => ({ role: msg.role, content: msg.content })));
assert.deepEqual(result.toolCalls[0].args, { nodes: [1] });
assert.equal(err.length, 0);
});