7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
// Mock only routeRequest so we don't need real provider keys; keep the rest of
// the router module (recordSuccess / recordRateLimitHit) intact.
const { mockRouteRequest } = vi.hoisted(() => ({ mockRouteRequest: vi.fn() }));
vi.mock('../../services/router.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../services/router.js')>();
return { ...actual, routeRequest: mockRouteRequest };
});
import type { Express } from 'express';
import { createApp } from '../../app.js';
import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
import { setStickyModel } from '../../routes/proxy.js';
function fakeRoute(provider: any) {
return { provider, modelId: 'fake-model', modelDbId: 9999, apiKey: 'k', keyId: 1, platform: 'fake', displayName: 'Fake Model' };
}
async function post(app: Express, path: string, body: any, key?: string, headers: Record<string, string> = {}) {
const server = app.listen(0, '127.0.0.1');
if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
const addr = server.address() as any;
const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', ...(key ? { Authorization: `Bearer ${key}` } : {}), ...headers },
body: JSON.stringify(body),
});
const text = await res.text();
server.close();
return { status: res.status, text, contentType: res.headers.get('content-type') ?? '', headers: res.headers };
}
describe('POST /v1/responses (#96)', () => {
let app: Express;
let key: string;
beforeAll(() => {
process.env.ENCRYPTION_KEY = '0'.repeat(64);
initDb(':memory:');
app = createApp();
key = getUnifiedApiKey();
});
it('rejects requests without a valid unified key (401)', async () => {
expect((await post(app, '/v1/responses', { input: 'hi' })).status).toBe(401);
expect((await post(app, '/v1/responses', { input: 'hi' }, 'wrong')).status).toBe(401);
});
it('rejects an invalid body (missing input) with 400', async () => {
expect((await post(app, '/v1/responses', { model: 'auto' }, key)).status).toBe(400);
});
it('accepts Codex additional_tools metadata items', async () => {
mockRouteRequest.mockClear();
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status, text } = await post(app, '/v1/responses', {
input: [
{ type: 'additional_tools', id: 'at_1', role: 'developer', tools: [{ type: 'shell' }] },
{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hello' }] },
],
stream: false,
}, key);
expect(status).toBe(200);
expect(JSON.parse(text).output_text).toBe('ok');
});
// #118: image parts now translate to image_url content blocks, so an image
// request must be routed with requireVision=true (only vision-capable models
// are candidates; a text-only pinned model is skipped, falling back to a
// vision-capable peer). The old unconditional 422 is gone.
it('routes an image request through requireVision (vision-capable model)', async () => {
mockRouteRequest.mockClear();
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'a red circle' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status, text } = await post(app, '/v1/responses', {
input: [{
role: 'user',
content: [
{ type: 'input_text', text: 'what is this?' },
{ type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgo=' },
],
}],
}, key);
expect(status).toBe(200);
expect(JSON.parse(text).output_text).toBe('a red circle');
// routeRequest arg [3] is requireVision.
expect(mockRouteRequest.mock.calls.at(-1)?.[3]).toBe(true);
});
it('forwards a declared x-freellm-task-type to the router as the task arg (#1127)', async () => {
mockRouteRequest.mockClear();
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status } = await post(app, '/v1/responses', { input: 'refactor this function' }, key, { 'x-freellm-task-type': 'code' });
expect(status).toBe(200);
// routeRequest arg [10] is the task type.
expect(mockRouteRequest.mock.calls.at(-1)?.[10]).toBe('code');
});
it('leaves the task arg undefined when no task header is sent (#1127)', async () => {
mockRouteRequest.mockClear();
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status } = await post(app, '/v1/responses', { input: 'hello there' }, key);
expect(status).toBe(200);
// routeRequest arg [10] is the task type; absent header ⇒ undefined ⇒ preset weights untouched.
expect(mockRouteRequest.mock.calls.at(-1)?.[10]).toBeUndefined();
});
it('rejects a file_id-only input_image with 422 before routing (no Files backend)', async () => {
mockRouteRequest.mockClear();
const { status, text } = await post(app, '/v1/responses', {
input: [{
role: 'user',
content: [
{ type: 'input_text', text: 'what is this?' },
{ type: 'input_image', file_id: 'file_abc123' },
],
}],
}, key);
expect(status).toBe(422);
const body = JSON.parse(text);
expect(body.error.code).toBe('unsupported_image_input');
expect(body.error.type).toBe('invalid_request_error');
expect(mockRouteRequest).not.toHaveBeenCalled();
});
it('rejects an input_image with no resolvable url instead of answering blind', async () => {
mockRouteRequest.mockClear();
// The lenient schema accepts this body; without the pre-check the image
// part would translate to nothing and the request would route as plain
// text — answering blind to an image the client believes was sent.
const { status, text } = await post(app, '/v1/responses', {
input: [{
role: 'user',
content: [
{ type: 'input_text', text: 'what is this?' },
{ type: 'input_image', detail: 'high' },
],
}],
}, key);
expect(status).toBe(422);
expect(JSON.parse(text).error.code).toBe('unsupported_image_input');
expect(mockRouteRequest).not.toHaveBeenCalled();
});
it('rejects an image request with 422 no_vision_model when no vision model is enabled', async () => {
mockRouteRequest.mockClear();
getDb().prepare('UPDATE models SET enabled = 0 WHERE supports_vision = 1').run();
try {
const { status, text } = await post(app, '/v1/responses', {
input: [{
role: 'user',
content: [{ type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgo=' }],
}],
}, key);
expect(status).toBe(422);
expect(JSON.parse(text).error.code).toBe('no_vision_model');
expect(mockRouteRequest).not.toHaveBeenCalled();
} finally {
getDb().prepare('UPDATE models SET enabled = 1 WHERE supports_vision = 1').run();
}
});
// #103: the x-api-key header (Anthropic wire format) must authenticate here
// too, not just on /v1/chat/completions.
it('accepts the unified key via the x-api-key header', async () => {
const server = app.listen(0, '127.0.0.1');
if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
const addr = server.address() as any;
const res = await fetch(`http://127.0.0.1:${addr.port}/v1/responses`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'x-api-key': key },
body: JSON.stringify({ input: 'hi' }),
});
server.close();
// Auth passes (not 401); body validity / routing is covered elsewhere.
expect(res.status).not.toBe(401);
});
it('non-stream: returns a completed Responses object with usage', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'Hello from fake' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status, text, contentType } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);
expect(status).toBe(200);
expect(contentType).toContain('application/json');
expect(text.length).toBeGreaterThan(0);
const body = JSON.parse(text);
expect(body.object).toBe('response');
expect(body.status).toBe('completed');
expect(body.output_text).toBe('Hello from fake');
expect(body.output[0]).toMatchObject({ type: 'message', role: 'assistant' });
expect(body.usage.total_tokens).toBe(7);
});
it('stream: emits the Responses SSE event sequence', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() { throw new Error('should not be called'); },
async *streamChatCompletion() {
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { role: 'assistant', content: 'Hel' }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { content: 'lo' }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] };
},
}));
const { status, text, contentType } = await post(app, '/v1/responses', { input: 'hi', stream: true }, key);
expect(status).toBe(200);
expect(contentType).toContain('text/event-stream');
for (const ev of ['response.created', 'response.output_item.added', 'response.content_part.added',
'response.output_text.delta', 'response.output_text.done', 'response.output_item.done', 'response.completed']) {
expect(text).toContain(`event: ${ev}`);
}
expect(text).toContain('"delta":"Hel"');
expect(text).toContain('"delta":"lo"');
// the terminal event carries the assembled text
const completed = text.split('event: response.completed')[1];
expect(completed).toContain('"output_text":"Hello"');
});
it('stream: a turn cut off at the token limit ends as response.incomplete', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() { throw new Error('should not be called'); },
async *streamChatCompletion() {
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { role: 'assistant', content: 'Partial answ' }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'length' }] };
},
}));
const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: true }, key);
expect(status).toBe(200);
expect(text).not.toContain('event: response.completed');
const terminal = JSON.parse(text.split('event: response.incomplete\ndata: ')[1].split('\n')[0]);
expect(terminal.response).toMatchObject({
status: 'incomplete',
incomplete_details: { reason: 'max_output_tokens' },
output_text: 'Partial answ',
});
expect(terminal.response.output[0].status).toBe('incomplete');
});
it('non-stream: a turn cut off at the token limit is incomplete', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'x', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'Partial answ' }, finish_reason: 'length' }],
usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);
expect(status).toBe(200);
expect(JSON.parse(text)).toMatchObject({ status: 'incomplete', incomplete_details: { reason: 'max_output_tokens' } });
});
it('stream: records ttfb on the first reasoning token, not the first visible text (#764)', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() { throw new Error('should not be called'); },
async *streamChatCompletion() {
// Reasoning models stream thinking before any visible text. ttfb must
// count the first token of ANY kind; a real delay between the
// reasoning frame and the first text frame makes the two moments
// measurable in the requests row.
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { reasoning_content: 'thinking hard…', content: null }, finish_reason: null }] };
await new Promise(r => setTimeout(r, 400));
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { content: 'Hello' }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] };
},
}));
const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: true }, key);
expect(status).toBe(200);
expect(text).toContain('event: response.completed');
const rows = getDb().prepare("SELECT status, latency_ms, ttfb_ms FROM requests ORDER BY id DESC LIMIT 1").all() as any[];
expect(rows[0].status).toBe('success');
expect(rows[0].ttfb_ms).not.toBeNull();
// ttfb must be the reasoning-head moment (well before the text token that
// triggers the commit), not the flush time ≈ latency.
expect(rows[0].ttfb_ms).toBeLessThan(rows[0].latency_ms - 250);
// The completed Response reports the thinking tokens truthfully instead of
// a hardcoded 0: 'thinking hard…' is 14 chars → ceil(14/4) = 4.
const completed = text.split('event: response.completed')[1];
const payload = JSON.parse(completed.slice(completed.indexOf('{')));
expect(payload.response.usage.output_tokens_details.reasoning_tokens).toBe(4);
});
it('non-stream: reports reasoning_tokens from the provider usage when advertised', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'answer', reasoning_content: 'thinking' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 3, completion_tokens: 6, total_tokens: 9, completion_tokens_details: { reasoning_tokens: 2 } },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);
expect(status).toBe(200);
const body = JSON.parse(text);
expect(body.usage.output_tokens_details.reasoning_tokens).toBe(2);
});
it('stream: tool-call deltas produce function_call events with assembled arguments', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() { throw new Error('nope'); },
async *streamChatCompletion() {
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{"ci' } }] }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, type: 'function', function: { arguments: 'ty":"SF"}' } }] }, finish_reason: 'tool_calls' }] };
},
}));
const { text } = await post(app, '/v1/responses', { input: 'weather?', stream: true }, key);
expect(text).toContain('"type":"function_call"');
expect(text).toContain('event: response.function_call_arguments.delta');
expect(text).toContain('event: response.function_call_arguments.done');
expect(text).toContain('"arguments":"{\\"city\\":\\"SF\\"}"');
});
// The Responses streaming SDK indexes snapshot.output by output_index and
// crashes on a collision (an output_text delta with the same index as a
// function_call item). Models that stream tool_calls BEFORE any closing text
// (GLM/Qwen on this surface) used to leave the text item stuck at index 0
// while the function_call owned 0 too. Every output item must claim a dense,
// unique index.
it('stream: keeps output_index unique when tool_calls precede the text item', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() { throw new Error('nope'); },
async *streamChatCompletion() {
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{"ci' } }] }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, type: 'function', function: { arguments: 'ty":"SF"}' } }] }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: { content: 'done' }, finish_reason: null }] };
yield { id: 'c', object: 'chat.completion.chunk', created: 0, model: 'fake-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] };
},
}));
const { text } = await post(app, '/v1/responses', { input: 'do it', stream: true }, key);
const indices: number[] = [...text.matchAll(/"output_index":(\d+)/g)].map((m) => Number(m[1]));
expect(indices.length).toBeGreaterThan(0);
// tool_calls + text = two distinct items, so both 0 AND 1 must appear and
// the function_call item must not share the text item's index.
expect(new Set(indices)).toContain(0);
expect(new Set(indices)).toContain(1);
// the closing text delta must reference the text item (index 1), not 0
const textDelta = text.split('event: response.output_text.delta')[1];
expect(textDelta).toContain('"output_index":1');
expect(textDelta).toContain('"delta":"done"');
});
it('rejects computer-use requests with a clear 422 before translation', async () => {
mockRouteRequest.mockClear();
const { status, text } = await post(app, '/v1/responses', {
input: 'control the computer',
tools: [{ type: 'computer_use_preview', name: 'computer' }],
}, key);
expect(status).toBe(422);
const body = JSON.parse(text);
expect(body.error.code).toBe('no_computer_use_model');
expect(body.error.type).toBe('invalid_request_error');
expect(mockRouteRequest).not.toHaveBeenCalled();
});
it('accepts computer_call_output round-trip items without 400 (validation leniency)', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status, text } = await post(app, '/v1/responses', {
input: [
{ type: 'reasoning', summary: [{ type: 'summary_text', text: 'planning' }] },
{ type: 'message', role: 'user', content: 'what is the weather?' },
],
}, key);
expect(status).toBe(200);
expect(JSON.parse(text).output_text).toBe('ok');
});
it('accepts built-in tool call items (web_search_call, mcp_call) without 400', async () => {
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status, text } = await post(app, '/v1/responses', {
input: [
{ type: 'web_search_call', id: 'ws_1', status: 'completed' },
{ type: 'mcp_call', id: 'mcp_1', name: 'lookup', arguments: '{}' },
{ type: 'message', role: 'user', content: 'summarize the results' },
],
}, key);
expect(status).toBe(200);
expect(JSON.parse(text).output_text).toBe('ok');
});
it('routes built-in Responses tools through tool-capable models', async () => {
mockRouteRequest.mockClear();
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
const { status } = await post(app, '/v1/responses', {
input: 'say hello',
tools: [{
type: 'local_shell',
name: 'exec_command',
description: 'Run a local shell command',
parameters: { type: 'object', properties: { cmd: { type: 'string' } } },
}],
}, key);
expect(status).toBe(200);
const lastCall = mockRouteRequest.mock.calls.at(-1);
expect(lastCall?.[4]).toBe(true);
});
it('non-stream: returns invalid_request_error when provider API 400s exhaust routing', async () => {
mockRouteRequest.mockImplementation((_estimated, skipKeys) => {
if (skipKeys?.size) {
throw Object.assign(new Error('All models exhausted'), { status: 429 });
}
return fakeRoute({
async chatCompletion() {
throw Object.assign(
new Error('Google API error 400: Invalid JSON payload received. Unknown name "x-google-enum-descriptions"'),
{ status: 400 },
);
},
async *streamChatCompletion() { /* unused */ },
});
});
const { status, text } = await post(app, '/v1/responses', { input: 'hi', stream: false }, key);
const body = JSON.parse(text);
expect(status).toBe(400);
expect(body.error.type).toBe('invalid_request_error');
expect(body.error.message).toContain('rejected the request as invalid');
});
});
// #579 gave this endpoint the same model resolution /v1/chat/completions has —
// explicit model > sticky session > auto — but shipped without a regression
// test. This is it. routeRequest is mocked, so the assertions read the routing
// DECISION off its arguments: [2] is the preferred (pinned/sticky) model db id
// and [6] is the strict chain an explicit model resolves to.
describe('POST /v1/responses model routing priority (#579)', () => {
let app: Express;
let key: string;
let pinnedId = 0;
let stickyId = 0;
let disabledId = 0;
// Unique model ids make each model its own unify group, so an explicit
// request resolves to exactly one row and the chain assertion is unambiguous.
function addModel(modelId: string, priority: number, enabled = true): number {
const db = getDb();
const inserted = db.prepare(`
INSERT INTO models (
platform, model_id, display_name, intelligence_rank, speed_rank, size_label,
rpm_limit, rpd_limit, tpm_limit, tpd_limit, monthly_token_budget,
context_window, enabled, supports_vision, supports_tools
)
VALUES ('groq', ?, ?, ?, ?, 'Small', NULL, NULL, NULL, NULL, '~1M', 128000, ?, 0, 1)
`).run(modelId, `Responses Priority ${modelId}`, priority, priority, enabled ? 1 : 0);
const id = Number(inserted.lastInsertRowid);
db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(id, priority);
const profile = db.prepare("SELECT value FROM settings WHERE key = 'active_profile_id'").get() as { value: string } | undefined;
if (profile) {
db.prepare('INSERT INTO profile_models (profile_id, model_db_id, priority, enabled) VALUES (?, ?, ?, 1)')
.run(Number(profile.value), id, priority);
}
return id;
}
beforeAll(() => {
process.env.ENCRYPTION_KEY = '0'.repeat(64);
initDb(':memory:');
app = createApp();
key = getUnifiedApiKey();
const db = getDb();
db.prepare('DELETE FROM profile_models').run();
db.prepare('DELETE FROM fallback_config').run();
db.prepare('DELETE FROM models').run();
pinnedId = addModel('pinned-responses-model', 1);
stickyId = addModel('sticky-responses-model', 2);
disabledId = addModel('disabled-responses-model', 3, false);
});
beforeEach(() => {
mockRouteRequest.mockReset();
mockRouteRequest.mockReturnValue(fakeRoute({
async chatCompletion() {
return {
id: 'c', object: 'chat.completion', created: 0, model: 'fake-model',
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
},
async *streamChatCompletion() { /* unused */ },
}));
});
// With a session header the sticky key is the header itself, so the message
// list here is irrelevant — but the REQUEST must carry an assistant turn for
// getStickyModel to read the map at all.
const followUp = [
{ role: 'user', content: 'hello' },
{ role: 'assistant', content: 'ok' },
{ role: 'user', content: 'again' },
];
function routingCall() {
const call = mockRouteRequest.mock.calls.at(-1);
return { preferredModelDbId: call?.[2] as number | undefined, chain: call?.[6] as { model_db_id: number }[] | undefined };
}
it('an explicit model wins over a sticky session', async () => {
setStickyModel([], stickyId, 'sess-explicit-wins');
const { status } = await post(app, '/v1/responses', {
model: 'pinned-responses-model',
input: followUp,
}, key, { 'x-session-id': 'sess-explicit-wins' });
expect(status).toBe(200);
const { preferredModelDbId, chain } = routingCall();
expect(chain?.map(row => row.model_db_id)).toEqual([pinnedId]);
expect(preferredModelDbId).not.toBe(stickyId);
});
it('a sticky session wins over auto routing when no model is sent', async () => {
setStickyModel([], stickyId, 'sess-sticky-wins');
const { status } = await post(app, '/v1/responses', {
input: followUp,
}, key, { 'x-session-id': 'sess-sticky-wins' });
expect(status).toBe(200);
const { preferredModelDbId, chain } = routingCall();
expect(preferredModelDbId).toBe(stickyId);
// Plain auto now threads the resolved active chain (parity with
// /v1/chat/completions), not a singleton pin.
expect(chain?.map(row => row.model_db_id)).toEqual(expect.arrayContaining([pinnedId, stickyId]));
expect(chain?.map(row => row.model_db_id)).not.toEqual([pinnedId]);
});
it('auto routing applies when there is neither an explicit model nor a sticky session', async () => {
const { status } = await post(app, '/v1/responses', {
input: followUp,
}, key, { 'x-session-id': 'sess-no-sticky' });
expect(status).toBe(200);
const { preferredModelDbId, chain } = routingCall();
expect(preferredModelDbId).toBeUndefined();
expect(chain?.map(row => row.model_db_id)).toEqual(expect.arrayContaining([pinnedId, stickyId]));
expect(chain?.map(row => row.model_db_id)).not.toEqual([pinnedId]);
});
it.each([
['unknown', 'no-such-model-anywhere'],
['disabled', 'disabled-responses-model'],
])('returns 400 model_not_found for an %s explicit model', async (_flavor, model) => {
void disabledId;
const { status, text } = await post(app, '/v1/responses', { model, input: 'hi' }, key);
expect(status).toBe(400);
const body = JSON.parse(text);
expect(body.error.code).toBe('model_not_found');
expect(body.error.type).toBe('invalid_request_error');
expect(mockRouteRequest).not.toHaveBeenCalled();
});
});