Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
14.7 KB
Raw Blame
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import type { Express } from 'express';
import { createApp } from '../../app.js';
import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
import { clearCache } from '../../services/cache.js';
import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';

let dashToken = '';

async function request(app: Express, method: string, path: string, body?: any, headers: Record<string, string> = {}) {
  const server = app.listen(0, '127.0.0.1');
  if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
  const addr = server.address() as any;
  const url = `http://127.0.0.1:${addr.port}${path}`;
  const res = await fetch(url, {
    method,
    headers: {
      ...(body ? { 'Content-Type': 'application/json' } : {}),
      ...(isGatedApiPath(path) && !('Authorization' in headers) ? { Authorization: `Bearer ${dashToken}` } : {}),
      ...headers,
    },
    body: body ? JSON.stringify(body) : undefined,
  });
  const raw = await res.text();
  server.close();
  let json: any = null;
  try { json = JSON.parse(raw); } catch { /* SSE or empty body */ }
  return { status: res.status, body: json, headers: res.headers, raw };
}

function authHeaders() {
  return { Authorization: `Bearer ${getUnifiedApiKey()}` };
}

// Mock every groq chat-completion call, counting how many actually reach the
// provider (separately for streaming vs non-streaming). The whole point of the
// cache is that a repeat non-streaming request does NOT reach the provider.
function mockGroq(content: string, streamOpts: { finish?: string; failMidStream?: boolean } = {}) {
  const origFetch = global.fetch;
  const counter = { calls: 0, streamCalls: 0 };
  vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
    const urlStr = typeof url === 'string' ? url : url.toString();
    if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
      const reqBody = JSON.parse(String((init as RequestInit).body));
      if (reqBody.stream) {
        counter.streamCalls++;
        const sse = [
          { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: { role: 'assistant', content: null }, finish_reason: null }] },
          { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: { content }, finish_reason: null }] },
          { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: {}, finish_reason: streamOpts.finish ?? 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 } },
        ].map(f => `data: ${JSON.stringify(f)}\n\n`).join('') + 'data: [DONE]\n\n';
        if (streamOpts.failMidStream) {
          // Upstream dies partway through: the client keeps the frames it
          // already got, but the turn never completed, so nothing may be
          // cached. Pull-driven so the proxy really has flushed headers and
          // forwarded a content chunk before the error lands.
          const enc = new TextEncoder();
          const frame = (delta: Record<string, unknown>) =>
            enc.encode(`data: ${JSON.stringify({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta, finish_reason: null }] })}\n\n`);
          let pulls = 0;
          const body = new ReadableStream<Uint8Array>({
            async pull(controller) {
              pulls++;
              if (pulls === 1) return void controller.enqueue(frame({ role: 'assistant', content: null }));
              if (pulls === 2) return void controller.enqueue(frame({ content }));
              await new Promise(resolve => setTimeout(resolve, 5));
              controller.error(new Error('upstream stream broke'));
            },
          });
          return new Response(body, { status: 200, headers: { 'Content-Type': 'text/event-stream' } }) as any;
        }
        return new Response(sse, { status: 200, headers: { 'Content-Type': 'text/event-stream' } }) as any;
      }
      counter.calls++;
      return {
        ok: true,
        json: () => Promise.resolve({
          id: 'chatcmpl-cache-it',
          object: 'chat.completion',
          created: 123,
          model: 'openai/gpt-oss-120b',
          choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],
          usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 },
        }),
      } as any;
    }
    return origFetch(url, init);
  });
  return counter;
}

describe('Response cache (proxy integration)', () => {
  let app: Express;

  beforeAll(() => {
    process.env.ENCRYPTION_KEY = '0'.repeat(64);
    process.env.RESPONSE_CACHE = 'on';
    initDb(':memory:');
    app = createApp();
    dashToken = mintDashboardToken();
  });

  afterAll(() => {
    delete process.env.RESPONSE_CACHE;
  });

  beforeEach(async () => {
    const db = getDb();
    db.prepare('DELETE FROM api_keys').run();
    db.prepare('DELETE FROM requests').run();
    db.prepare('DELETE FROM rate_limit_usage').run();
    clearCache();
    const addKey = await request(app, 'POST', '/api/keys', { platform: 'groq', key: 'gsk_cache_test', label: 'cache' });
    expect(addKey.status).toBe(201);
  });

  afterEach(() => {
    vi.restoreAllMocks();
    clearCache();
  });

  const chat = (extra: Record<string, unknown> = {}, headers: Record<string, string> = {}) =>
    request(app, 'POST', '/v1/chat/completions', {
      model: 'auto',
      messages: [{ role: 'user', content: 'what is 2+2?' }],
      ...extra,
    }, { ...authHeaders(), ...headers });

  it.each([
    { response_format: { type: 'json_object' } },
    { seed: 42 },
    { stop: 'END' },
    { parallel_tool_calls: false },
  ])('rejects idempotent retries with changed generation options: %j', async (extra) => {
    getDb().prepare('DELETE FROM idempotency_claims').run();
    const counter = mockGroq('plain text');
    const headers = { 'Idempotency-Key': 'generation-options', 'X-FreeLLM-Cache': 'off' };
    const first = await chat({}, headers);
    expect(first.status).toBe(200);
    const replay = await chat({}, headers);
    expect(replay.status).toBe(200);
    expect(replay.headers.get('x-routed-via')).toBe('idempotency');
    const second = await chat(extra, headers);
    expect(second.status).toBe(409);
    expect(second.body.error.message).toBe('idempotency_key_conflict');
    expect(counter.calls).toBe(1);
  });

  it('serves an identical second request from cache without calling the provider', async () => {
    const counter = mockGroq('the answer is four');

    const first = await chat();
    expect(first.status).toBe(200);
    expect(first.body.choices[0].message.content).toBe('the answer is four');
    expect(first.headers.get('x-freellm-cache')).toBe('MISS');
    expect(counter.calls).toBe(1);

    const second = await chat();
    expect(second.status).toBe(200);
    expect(second.body.choices[0].message.content).toBe('the answer is four');
    expect(second.headers.get('x-freellm-cache')).toBe('HIT');
    expect(second.headers.get('x-routed-via')).toBe('cache');
    // The provider was NOT hit again; the whole point.
    expect(counter.calls).toBe(1);
  });

  it('does not record rate-limit usage for a cache hit (quota preserved)', async () => {
    mockGroq('four');
    await chat();
    const usageAfterMiss = (getDb().prepare("SELECT COUNT(*) AS c FROM rate_limit_usage WHERE kind='request'").get() as { c: number }).c;
    await chat(); // hit
    const usageAfterHit = (getDb().prepare("SELECT COUNT(*) AS c FROM rate_limit_usage WHERE kind='request'").get() as { c: number }).c;
    expect(usageAfterHit).toBe(usageAfterMiss); // no new provider request recorded
  });

  it('treats a different prompt as a miss and calls the provider again', async () => {
    const counter = mockGroq('depends on the prompt');
    await chat({ messages: [{ role: 'user', content: 'first question' }] });
    await chat({ messages: [{ role: 'user', content: 'second question' }] });
    expect(counter.calls).toBe(2);
  });

  it('requests differing only in response_format, seed, or stop never collide', async () => {
    // A JSON string literal: valid JSON, so the response_format variant below
    // survives the structured-output enforcement instead of failing over.
    const counter = mockGroq('"knob-sensitive"');
    await chat(); // plain → MISS, populates
    // Same prompt with response_format json_object must be a MISS, not a
    // replay of the cached plain-text answer.
    const jsonReq = await chat({ response_format: { type: 'json_object' } });
    expect(jsonReq.headers.get('x-freellm-cache')).toBe('MISS');
    expect(counter.calls).toBe(2);
    // Same prompt, different seed / stop → also fresh generations.
    await chat({ seed: 42 });
    await chat({ stop: 'END' });
    expect(counter.calls).toBe(4);
    // But an exact repeat (with a knob present) is still a HIT.
    const repeat = await chat({ seed: 42 });
    expect(repeat.headers.get('x-freellm-cache')).toBe('HIT');
    expect(counter.calls).toBe(4);
  });

  it('X-FreeLLM-Cache: off bypasses the cache on both read and write', async () => {
    const counter = mockGroq('bypassed');
    await chat({}, { 'X-FreeLLM-Cache': 'off' });
    await chat({}, { 'X-FreeLLM-Cache': 'off' });
    // Neither call read nor populated the cache → provider hit twice.
    expect(counter.calls).toBe(2);
  });

  it('X-FreeLLM-Cache: on forces caching even when the cache is globally off', async () => {
    process.env.RESPONSE_CACHE = '';
    try {
      const counter = mockGroq('forced-on');
      const first = await chat({}, { 'X-FreeLLM-Cache': 'on' });
      expect(first.headers.get('x-freellm-cache')).toBe('MISS');
      const second = await chat({}, { 'X-FreeLLM-Cache': 'on' });
      expect(second.headers.get('x-freellm-cache')).toBe('HIT');
      expect(counter.calls).toBe(1);
    } finally {
      process.env.RESPONSE_CACHE = 'on';
    }
  });

  it('does not cache a high-temperature request', async () => {
    process.env.RESPONSE_CACHE_MAX_TEMPERATURE = '0.2';
    try {
      const counter = mockGroq('varies');
      await chat({ temperature: 0.9 });
      await chat({ temperature: 0.9 });
      expect(counter.calls).toBe(2); // above the cap → never cached
    } finally {
      delete process.env.RESPONSE_CACHE_MAX_TEMPERATURE;
    }
  });

  it('replays a second identical streaming request from cache, byte for byte', async () => {
    const counter = mockGroq('streamed four');

    const first = await chat({ stream: true });
    expect(first.status).toBe(200);
    expect(first.headers.get('x-freellm-cache')).toBe('MISS');
    expect(first.raw).toContain('streamed four');
    expect(first.raw.endsWith('data: [DONE]\n\n')).toBe(true);
    expect(counter.streamCalls).toBe(1);

    const second = await chat({ stream: true });
    expect(second.status).toBe(200);
    expect(second.headers.get('x-freellm-cache')).toBe('HIT');
    expect(second.headers.get('x-routed-via')).toBe('cache');
    expect(second.headers.get('content-type')).toContain('text/event-stream');
    // Identical SSE bytes, and no second provider stream.
    expect(second.raw).toBe(first.raw);
    expect(counter.streamCalls).toBe(1);
  });

  it('keeps the streaming and non-streaming entries for one prompt apart', async () => {
    const counter = mockGroq('four');
    await chat();                  // JSON MISS, populates the JSON store
    const streamed = await chat({ stream: true });
    // The JSON entry is never replayed as SSE: the stream is a MISS of its own.
    expect(streamed.headers.get('x-freellm-cache')).toBe('MISS');
    expect(counter.streamCalls).toBe(1);
    // ...and storing it did not disturb the JSON entry.
    const json = await chat();
    expect(json.headers.get('x-freellm-cache')).toBe('HIT');
    expect(json.body.choices[0].message.content).toBe('four');
    expect(counter.calls).toBe(1);
  });

  it('does not cache a truncated stream (finish_reason length)', async () => {
    const counter = mockGroq('cut off mid-', { finish: 'length' });
    const first = await chat({ stream: true });
    expect(first.status).toBe(200);
    expect(counter.streamCalls).toBe(1);
    // Nothing was stored, so the repeat goes back to the provider.
    const second = await chat({ stream: true });
    expect(second.headers.get('x-freellm-cache')).toBe('MISS');
    expect(counter.streamCalls).toBe(2);
  });

  it('does not cache a stream that broke mid-flight', async () => {
    const counter = mockGroq('half an ans', { failMidStream: true });
    const first = await chat({ stream: true });
    // Headers were already flushed and content forwarded before the break.
    expect(first.headers.get('x-freellm-cache')).toBe('MISS');
    expect(first.raw).toContain('half an ans');
    expect(first.raw).toContain('stream_error'); // the break is surfaced to the client
    const before = counter.streamCalls;
    // Nothing was stored, so the repeat goes back to the provider.
    const second = await chat({ stream: true });
    expect(second.headers.get('x-freellm-cache')).toBe('MISS');
    expect(counter.streamCalls).toBeGreaterThan(before);
  });

  it('retains nothing when the cache is off: a stream buffers no frames', async () => {
    process.env.RESPONSE_CACHE = '';
    try {
      const counter = mockGroq('uncached');
      const first = await chat({ stream: true });
      expect(first.headers.get('x-freellm-cache')).toBe('OFF');
      const second = await chat({ stream: true });
      expect(second.headers.get('x-freellm-cache')).toBe('OFF');
      expect(counter.streamCalls).toBe(2);
      const stats = await request(app, 'GET', '/api/cache/stats');
      expect(stats.body.entries).toBe(0);
    } finally {
      process.env.RESPONSE_CACHE = 'on';
    }
  });

  it('exposes cache stats and savings on the admin endpoint', async () => {
    mockGroq('four');
    await chat();        // MISS, populates
    await chat();        // HIT (1 hit, saves 5 prompt + 3 completion tokens)
    const stats = await request(app, 'GET', '/api/cache/stats');
    expect(stats.status).toBe(200);
    expect(stats.body.enabled).toBe(true);
    expect(stats.body.entries).toBe(1);
    expect(stats.body.totalHits).toBe(1);
    expect(stats.body.savedTokens).toBe(8); // 5 + 3
  });

  it('DELETE /api/cache flushes stored entries', async () => {
    mockGroq('four');
    await chat();
    let stats = await request(app, 'GET', '/api/cache/stats');
    expect(stats.body.entries).toBe(1);
    const cleared = await request(app, 'DELETE', '/api/cache');
    expect(cleared.status).toBe(200);
    expect(cleared.body.cleared).toBe(1);
    stats = await request(app, 'GET', '/api/cache/stats');
    expect(stats.body.entries).toBe(0);
  });
});