Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
25.9 KB
Raw Blame
import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
import type { Express } from 'express';
import { createApp } from '../../app.js';
import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
import { getStickyModel, setStickyModel } from '../../routes/proxy.js';
import { mintDashboardToken } from '../helpers/auth.js';

// Stream turn-integrity (#231 audit): the proxy must deliver agent-usable
// TURNS, not transport bytes. These tests feed crafted upstream SSE bodies
// through a mocked fetch and assert the failure modes observed live are
// either failed over (before headers) or surfaced honestly (after).

async function request(app: Express, path: string, body: any, extraHeaders: Record<string, string> = {}) {
  const server = app.listen(0, '127.0.0.1');
  if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
  const addr = server.address() as any;
  const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Authorization: `Bearer ${getUnifiedApiKey()}`,
      ...extraHeaders,
    },
    body: JSON.stringify(body),
  });
  const text = await res.text();
  server.close();
  let json: any = null;
  try { json = JSON.parse(text); } catch { /* SSE body */ }
  return { status: res.status, headers: res.headers, text, body: json };
}

/** Parse an SSE response body into JSON frames (excluding [DONE]). */
function frames(text: string): any[] {
  return text.split('\n')
    .filter(l => l.startsWith('data: ') && l.trim() !== 'data: [DONE]')
    .map(l => JSON.parse(l.slice(6)));
}

const sse = (...payloads: (object | string)[]) =>
  payloads.map(p => `data: ${typeof p === 'string' ? p : JSON.stringify(p)}\n\n`).join('');

const roleChunk = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', content: null }, finish_reason: null }] };
const textChunk = (s: string, finish: string | null = null) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { content: s }, finish_reason: finish }] });
const finishChunk = (reason: string) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: {}, finish_reason: reason }] });

const TOOLS = [{ type: 'function', function: { name: 'Read', description: 'read a file', parameters: { type: 'object', properties: { file_path: { type: 'string' } }, required: ['file_path'] } } }];

// Sequential upstream responder: call N gets script[N] (last entry repeats).
function mockUpstream(script: Array<{ body: string; status?: number }>) {
  const origFetch = global.fetch;
  let call = 0;
  const seen: Array<{ model: string; streamOptions?: { include_usage?: boolean } }> = [];
  vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
    const urlStr = typeof url === 'string' ? url : url.toString();
    // Only intercept provider upstreams; the test's own localhost request
    // and anything else goes through.
    if (!/api\.groq\.com|openrouter\.ai|api\.cohere|generativelanguage|integrate\.api\.nvidia|api\.cerebras|api\.b\.ai|developer\.amd\.com\.cn|api\.mistral|router\.huggingface|api\.cloudflare|models\.github|open\.bigmodel|api\.llm7|api\.kilo|gen\.pollinations|ollama\.com|opencode\.ai|api\.aionlabs\.ai|router\.requesty\.ai|api\.navy|router\.bynara\.id/.test(urlStr)) {
      return origFetch(url as any, init);
    }
    const reqBody = JSON.parse(String((init as RequestInit).body));
    seen.push({ model: reqBody.model, streamOptions: reqBody.stream_options });
    const step = script[Math.min(call++, script.length - 1)];
    return new Response(step.body, {
      status: step.status ?? 200,
      headers: { 'Content-Type': 'text/event-stream' },
    });
  });
  return { seen, calls: () => call };
}

describe('proxy stream turn-integrity', () => {
  let app: Express;
  let dashToken = '';

  beforeAll(() => {
    process.env.ENCRYPTION_KEY = '0'.repeat(64);
    initDb(':memory:');
    app = createApp();
    dashToken = mintDashboardToken();
  });

  beforeEach(async () => {
    const db = getDb();
    db.prepare('DELETE FROM api_keys').run();
    db.prepare('DELETE FROM requests').run();
    db.prepare('DELETE FROM rate_limit_cooldowns').run();
    db.prepare('DELETE FROM rate_limit_usage').run();
    // One groq key: the seeded chain has several tool-capable groq models, so
    // failover hops between groq models while staying on this mock.
    const { status } = await request(app, '/api/keys',
      { platform: 'groq', key: 'gsk_stream_integrity', label: 't' },
      { Authorization: `Bearer ${dashToken}` });
    expect(status).toBe(201);
  });

  afterEach(() => {
    vi.restoreAllMocks();
  });

  it('fails over when the upstream stream opens with an in-band error frame (Groq tool_use_failed)', async () => {
    const up = mockUpstream([
      { body: sse({ error: { message: "Failed to call a function. Please adjust your prompt.", type: 'invalid_request_error', code: 'tool_use_failed', status_code: 400 } }, '[DONE]') },
      { body: sse(roleChunk, textChunk('All good from the next model.'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, messages: [{ role: 'user', content: 'in-band error failover test' }],
    });
    expect(r.status).toBe(200);
    expect(up.calls()).toBe(2);
    expect(r.headers.get('x-fallback-attempts')).toBe('1');
    const fs = frames(r.text);
    expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('All good'))).toBe(true);
    expect(fs.some(f => f.error)).toBe(false);
    expect(r.headers.get('x-request-id')).toMatch(/\S+/);

    const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];
    expect(rows[0].status).toBe('error');
    expect(rows[0].error).toMatch(/in-band provider error/);
    expect(rows[1].status).toBe('success');
  });

  it('synthesizes finish_reason tool_calls and ids for a stream that ends without a terminal reason', async () => {
    // minimax/command-r live shape: valid tool_call deltas, then [DONE] with
    // no finish_reason chunk at all.
    const tcChunk = (frag: object) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { tool_calls: [frag] }, finish_reason: null }] });
    mockUpstream([{
      body: sse(
        roleChunk,
        tcChunk({ index: 0, function: { name: 'Read', arguments: '{"file_pa' } }),
        tcChunk({ index: 0, function: { arguments: 'th":"/tmp/a"}' } }),
        '[DONE]',
      ),
    }]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'finish synthesis test' }],
    });
    expect(r.status).toBe(200);
    const fs = frames(r.text);
    const tcFrame = fs.find(f => f.choices?.[0]?.delta?.tool_calls);
    expect(tcFrame).toBeDefined();
    const call = tcFrame.choices[0].delta.tool_calls[0];
    expect(call.function.name).toBe('Read');
    expect(JSON.parse(call.function.arguments)).toEqual({ file_path: '/tmp/a' });
    expect(call.id).toBe('call_stream_1'); // synthesized — upstream sent none
    const finishes = fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean);
    expect(finishes).toEqual(['tool_calls']);
  });

  it('rescues a streamed inline dialect turn into structured tool_calls', async () => {
    mockUpstream([{
      body: sse(
        roleChunk,
        textChunk('<|tool_calls_sec'),
        textChunk('tion_begin|><|tool_call_begin|>functions.Read:0<|tool_call_argument_begin|>{"file_path":"/a"}<|tool_call_end|><|tool_calls_section_end|>'),
        finishChunk('stop'),
        '[DONE]',
      ),
    }]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'dialect stream rescue test' }],
    });
    expect(r.status).toBe(200);
    const fs = frames(r.text);
    // The raw dialect text never reaches the client...
    expect(fs.some(f => typeof f.choices?.[0]?.delta?.content === 'string' && f.choices[0].delta.content.includes('<|tool_call'))).toBe(false);
    // ...a structured call does, with the correct terminal reason.
    const tcFrame = fs.find(f => f.choices?.[0]?.delta?.tool_calls);
    expect(tcFrame.choices[0].delta.tool_calls[0].function.name).toBe('Read');
    expect(tcFrame.choices[0].delta.tool_calls[0].id).toBe('call_rescued_1');
    expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['tool_calls']);
  });

  it('fails over an unparseable dialect turn (degraded id token) before headers', async () => {
    const up = mockUpstream([
      { body: sse(roleChunk, textChunk('<|tool_call_begin|> chatcmpl-tool-bde5 <|tool_call_argument_begin|> {"file_path": "/a"}'), finishChunk('stop'), '[DONE]') },
      { body: sse(roleChunk, textChunk('Recovered by the next model.'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'unparseable dialect failover test' }],
    });
    expect(r.status).toBe(200);
    expect(up.calls()).toBe(2);
    expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Recovered'))).toBe(true);
  });

  it('fails over an abrupt EOF that happens before any payload', async () => {
    const up = mockUpstream([
      { body: sse(roleChunk) /* EOF: no [DONE], no finish_reason, no payload */ },
      { body: sse(roleChunk, textChunk('Second model answers.'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, messages: [{ role: 'user', content: 'pre-payload truncation test' }],
    });
    expect(r.status).toBe(200);
    expect(up.calls()).toBe(2);
    expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Second model'))).toBe(true);
    const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];
    expect(rows[0].status).toBe('error');
    expect(rows[0].error).toMatch(/stream ended unexpectedly/);
  });

  it('surfaces an honest error frame when truncation happens after payload reached the client', async () => {
    mockUpstream([
      { body: sse(roleChunk, textChunk('Partial ans')) /* EOF mid-generation */ },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, messages: [{ role: 'user', content: 'post-payload truncation test' }],
    });
    expect(r.status).toBe(200);
    const fs = frames(r.text);
    expect(fs.some(f => f.choices?.[0]?.delta?.content === 'Partial ans')).toBe(true);
    expect(fs.some(f => f.error?.type === 'stream_error')).toBe(true);
    const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];
    expect(rows[0].status).toBe('error'); // truncation is never a success
  });

  it('fails over a stream that completes with no content and no tool calls', async () => {
    const up = mockUpstream([
      { body: sse(roleChunk, finishChunk('stop'), '[DONE]') },
      { body: sse(roleChunk, textChunk('Non-empty.'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, messages: [{ role: 'user', content: 'empty stream failover test' }],
    });
    expect(r.status).toBe(200);
    expect(up.calls()).toBe(2);
    expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Non-empty'))).toBe(true);
  });

  it('fails over a stream whose model answers in prose despite response_format json_object (#933)', async () => {
    // The whole point of #933: a streamed structured-output request must not
    // ship an essay as a "success". The first model ignores the format; the
    // second honors it. Headers are held in 'json' hold mode, so the first
    // attempt can still fail over invisibly.
    const up = mockUpstream([
      { body: sse(roleChunk, textChunk('Here is your answer in prose form, sorry.'), finishChunk('stop'), '[DONE]') },
      { body: sse(roleChunk, textChunk('{"city":"Paris"}'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true,
      messages: [{ role: 'user', content: 'give me json' }],
      response_format: { type: 'json_object' },
    });
    expect(r.status).toBe(200);
    expect(up.calls()).toBe(2);
    expect(r.headers.get('x-fallback-attempts')).toBe('1');
    const fs = frames(r.text);
    expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('prose form'))).toBe(false);
    expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('Paris'))).toBe(true);
    const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];
    expect(rows[0].status).toBe('error');
    expect(rows[0].error).toMatch(/ignored response_format/);
    expect(rows[1].status).toBe('success');
  });

  it('heals a fenced JSON block streamed for a json_object request (#933)', async () => {
    // Prose-wrapped/fenced output is the common "almost right" shape: heal it
    // in place and deliver only the JSON, same as the non-stream path.
    mockUpstream([
      { body: sse(roleChunk, textChunk('```json\n{"answer": 42}\n```'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true,
      messages: [{ role: 'user', content: 'give me json' }],
      response_format: { type: 'json_object' },
    });
    expect(r.status).toBe(200);
    const fs = frames(r.text);
    const content = fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('');
    expect(content).toContain('"answer"');
    expect(content).not.toContain('```');
    const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];
    expect(rows[0].status).toBe('success');
  });

  it('passes a clean JSON stream through untouched for a json_object request (#933)', async () => {
    mockUpstream([
      { body: sse(roleChunk, textChunk('{"ok":true,"items":[1,2,3]}'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true,
      messages: [{ role: 'user', content: 'give me json' }],
      response_format: { type: 'json_object' },
    });
    expect(r.status).toBe(200);
    const content = frames(r.text).map(f => f.choices?.[0]?.delta?.content ?? '').join('');
    expect(content).toBe('{"ok":true,"items":[1,2,3]}');
    const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];
    expect(rows[0].status).toBe('success');
  });

  it('never leaks raw tool_call deltas riding on role/reasoning chunks (OpenRouter shape)', async () => {
    // OpenRouter attaches tool_call fragments to chunks that also carry a
    // role or reasoning key. Those must be accumulated, not forwarded raw —
    // forwarding both the fragments AND the assembled call duplicates it.
    mockUpstream([{
      body: sse(
        { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', reasoning: '', tool_calls: [{ index: 0, id: 'or_1', function: { name: 'Read', arguments: '{"file_' } }] }, finish_reason: null }] },
        { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { reasoning: null, tool_calls: [{ index: 0, function: { arguments: 'path":"/x"}' } }] }, finish_reason: null }] },
        finishChunk('tool_calls'),
        '[DONE]',
      ),
    }]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'delta leak test' }],
    });
    expect(r.status).toBe(200);
    const fs = frames(r.text);
    const tcFrames = fs.filter(f => f.choices?.[0]?.delta?.tool_calls);
    expect(tcFrames).toHaveLength(1); // exactly one complete emission, no raw fragments
    const call = tcFrames[0].choices[0].delta.tool_calls[0];
    expect(call.id).toBe('or_1');
    expect(JSON.parse(call.function.arguments)).toEqual({ file_path: '/x' });
  });

  it('streams ordinary text through unmodified, always ending in a terminal finish_reason', async () => {
    const up = mockUpstream([
      { body: sse(roleChunk, textChunk('Hello'), textChunk(' world'), finishChunk('stop'), '[DONE]') },
    ]);
    const r = await request(app, '/v1/chat/completions', {
      stream: true, messages: [{ role: 'user', content: 'plain passthrough test' }],
    });
    const fs = frames(r.text);
    const text = fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('');
    expect(text).toBe('Hello world');
    expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['stop']);
    // The crafted upstream frames all advertise model "m". Public frames must
    // instead identify the concrete model selected by the router (#568).
    expect(fs.every(f => f.error || f.model === up.seen[0].model)).toBe(true);
    expect(r.text.trim().endsWith('data: [DONE]')).toBe(true);
  });

  it('forwards stream_options.include_usage and records upstream token usage', async () => {
    const usage = { prompt_tokens: 3263, completion_tokens: 15, total_tokens: 3278 };
    const up = mockUpstream([{
      body: sse(
        roleChunk,
        textChunk('Hello'),
        finishChunk('stop'),
        { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage },
        '[DONE]',
      ),
    }]);

    const r = await request(app, '/v1/chat/completions', {
      stream: true,
      stream_options: { include_usage: true },
      messages: [{ role: 'user', content: 'real streaming usage test' }],
    });

    expect(r.status).toBe(200);
    expect(up.seen[0].streamOptions).toEqual({ include_usage: true });
    expect(frames(r.text).find(f => f.usage)?.usage).toEqual(usage);

    const row = getDb().prepare(
      "SELECT input_tokens, output_tokens FROM requests WHERE status = 'success' ORDER BY id DESC LIMIT 1",
    ).get() as { input_tokens: number; output_tokens: number };
    expect(row).toEqual({ input_tokens: 3263, output_tokens: 15 });

    const ledger = getDb().prepare(
      "SELECT COALESCE(SUM(tokens), 0) AS total FROM rate_limit_usage WHERE kind = 'tokens'",
    ).get() as { total: number };
    expect(ledger.total).toBe(3278);
  });

  // Not every provider sends usage on a frame of its own: several attach it to
  // the last choice-bearing frame instead. Reading it only off choice-less
  // frames silently discarded those real counts and billed the chars/4
  // estimate.
  it('records usage bundled onto the final choice-bearing frame', async () => {
    const usage = { prompt_tokens: 1111, completion_tokens: 22, total_tokens: 1133 };
    mockUpstream([{
      body: sse(
        roleChunk,
        textChunk('Hello'),
        { ...finishChunk('stop'), usage },
        '[DONE]',
      ),
    }]);

    const r = await request(app, '/v1/chat/completions', {
      stream: true,
      stream_options: { include_usage: true },
      messages: [{ role: 'user', content: 'bundled usage on the finish frame' }],
    });

    expect(r.status).toBe(200);
    const fs = frames(r.text);
    // Exactly one usage frame, emitted last, and it carries no duplicated turn.
    const withUsage = fs.filter(f => f.usage);
    expect(withUsage).toHaveLength(1);
    expect(withUsage[0].usage).toEqual(usage);
    expect(fs[fs.length - 1]).toBe(withUsage[0]);
    expect(withUsage[0].choices).toEqual([]);
    // The turn itself is unchanged: one terminal finish_reason, text intact.
    expect(fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('')).toBe('Hello');
    expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['stop']);

    const row = getDb().prepare(
      "SELECT input_tokens, output_tokens FROM requests WHERE status = 'success' ORDER BY id DESC LIMIT 1",
    ).get() as { input_tokens: number; output_tokens: number };
    expect(row).toEqual({ input_tokens: 1111, output_tokens: 22 });

    const ledger = getDb().prepare(
      "SELECT COALESCE(SUM(tokens), 0) AS total FROM rate_limit_usage WHERE kind = 'tokens'",
    ).get() as { total: number };
    expect(ledger.total).toBe(1133);
  });

  it('never emits usage twice when it rides a forwarded content frame', async () => {
    const usage = { prompt_tokens: 7, completion_tokens: 8, total_tokens: 15 };
    mockUpstream([{
      body: sse(
        roleChunk,
        textChunk('Hello'),
        { ...textChunk(' world'), usage },
        finishChunk('stop'),
        '[DONE]',
      ),
    }]);

    const r = await request(app, '/v1/chat/completions', {
      stream: true,
      stream_options: { include_usage: true },
      messages: [{ role: 'user', content: 'usage riding a content frame' }],
    });

    const fs = frames(r.text);
    expect(fs.filter(f => f.usage)).toHaveLength(1);
    expect(fs[fs.length - 1].usage).toEqual(usage);
    // The content frame it arrived on is forwarded WITHOUT usage — the held
    // copy is the single, last one the client sees (OpenAI ordering).
    const contentFrame = fs.find(f => f.choices?.[0]?.delta?.content === ' world');
    expect(contentFrame).toBeDefined();
    expect(contentFrame).not.toHaveProperty('usage');
    expect(fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('')).toBe('Hello world');
  });

  it('rescues a non-streaming inline dialect answer into structured tool_calls', async () => {
    const origFetch = global.fetch;
    vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
      const urlStr = typeof url === 'string' ? url : url.toString();
      if (!urlStr.includes('api.groq.com')) return origFetch(url as any, init);
      return new Response(JSON.stringify({
        id: 'r1', object: 'chat.completion', created: 1, model: 'm',
        choices: [{ index: 0, message: { role: 'assistant', content: '<function=Read{"file_path": "/tmp/a"}</function>' }, finish_reason: 'stop' }],
        usage: { prompt_tokens: 5, completion_tokens: 9, total_tokens: 14 },
      }), { status: 200, headers: { 'Content-Type': 'application/json' } });
    });
    const r = await request(app, '/v1/chat/completions', {
      stream: false, tools: TOOLS, messages: [{ role: 'user', content: 'non-stream dialect rescue test' }],
    });
    expect(r.status).toBe(200);
    const msg = r.body.choices[0].message;
    expect(msg.tool_calls).toHaveLength(1);
    expect(msg.tool_calls[0].function.name).toBe('Read');
    expect(msg.tool_calls[0].id).toBe('call_rescued_1');
    expect(msg.content).toBeNull();
    expect(r.body.choices[0].finish_reason).toBe('tool_calls');
  });

  it('records ttfb on the first reasoning_content token, not the first visible text (#764)', async () => {
    // Reasoning models stream thinking (reasoning_content) long before the
    // first answer token. The regression: ttfb was recorded at header flush,
    // i.e. the END of the thinking phase, so the Analytics speed shown was the
    // reasoning tail — or NULL on turns that never flushed. It must count the
    // first token of ANY kind. We interleave a real delay between the
    // reasoning frame and the first text frame to make the two moments
    // measurable.
    const origFetch = global.fetch;
    const enc = new TextEncoder();
    vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
      const urlStr = typeof url === 'string' ? url : url.toString();
      if (!urlStr.includes('api.groq.com')) return origFetch(url as any, init);
      const stream = new ReadableStream({
        async start(controller) {
          controller.enqueue(enc.encode(sse(
            roleChunk,
            { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { reasoning_content: 'thinking hard…', content: null }, finish_reason: null }] },
          )));
          await new Promise(r => setTimeout(r, 400));
          controller.enqueue(enc.encode(sse(textChunk('Hello world'), finishChunk('stop'), '[DONE]')));
          controller.close();
        },
      });
      return new Response(stream, { status: 200, headers: { 'Content-Type': 'text/event-stream' } });
    });
    const r = await request(app, '/v1/chat/completions', {
      stream: true, messages: [{ role: 'user', content: 'ttfb reasoning test' }],
    });
    expect(r.status).toBe(200);
    const rows = getDb().prepare("SELECT status, latency_ms, ttfb_ms FROM requests ORDER BY id").all() as any[];
    expect(rows[0].status).toBe('success');
    expect(rows[0].ttfb_ms).not.toBeNull();
    // ttfb must be the reasoning-head moment (well before the text token that
    // triggers header flush), not the flush time ≈ latency.
    expect(rows[0].ttfb_ms).toBeLessThan(rows[0].latency_ms - 250);
  });
});

describe('sticky session integrity', () => {
  beforeAll(() => {
    process.env.ENCRYPTION_KEY = '0'.repeat(64);
    initDb(':memory:');
  });

  it('keeps affinity from turn 1 to turn 2 (the old single/multi key gap)', () => {
    const t1 = [
      { role: 'system' as const, content: 'sys' },
      { role: 'user' as const, content: 'sticky turn gap probe' },
    ];
    setStickyModel(t1, 42);
    const t2 = [...t1,
      { role: 'assistant' as const, content: 'answer' },
      { role: 'user' as const, content: 'follow-up' },
    ];
    expect(getStickyModel(t2)).toBe(42);
  });

  it('applies to array-of-blocks content (opencode-style agents)', () => {
    const t1 = [
      { role: 'user' as const, content: [{ type: 'text', text: 'array content sticky probe' }] as any },
    ];
    setStickyModel(t1, 7);
    const t2 = [...t1,
      { role: 'assistant' as const, content: 'ok' },
      { role: 'user' as const, content: [{ type: 'text', text: 'next' }] as any },
    ];
    expect(getStickyModel(t2)).toBe(7);
  });

  it('honors an explicit x-session-id over message hashing', () => {
    const conv1 = [{ role: 'user' as const, content: 'conversation one' }];
    setStickyModel(conv1, 11, 'session-abc');
    const conv2 = [
      { role: 'user' as const, content: 'completely different opener' },
      { role: 'assistant' as const, content: 'hi' },
      { role: 'user' as const, content: 'next' },
    ];
    expect(getStickyModel(conv2, 'session-abc')).toBe(11);
    expect(getStickyModel(conv2)).toBeUndefined();
  });
});