Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
4.7 KB
Raw Blame
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { initDb } from '../../db/index.js';
import {
  acquireLease,
  releaseLease,
  resetLeases,
  inFlightForKey,
  canUseKeyConcurrency,
  getKeyConcurrencyLimit,
} from '../../services/ratelimit.js';

/**
 * In-flight leases make the window between key selection and usage recording
 * visible. They back the per-key concurrency cap, which stops parallel streams
 * from all picking one key and 429-ing each other on providers that meter
 * concurrency per credential.
 */
describe('in-flight leases', () => {
  beforeEach(() => {
    initDb(':memory:');
    resetLeases();
  });

  afterEach(() => {
    resetLeases();
    delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;
    delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ;
    vi.useRealTimers();
  });

  it('counts only the matching platform and key', () => {
    acquireLease('groq', 'llama-3.3-70b', 1, 100);
    acquireLease('groq', 'llama-3.1-8b', 1, 100);
    acquireLease('groq', 'llama-3.3-70b', 2, 100);
    acquireLease('cerebras', 'qwen-3-coder', 1, 100);

    expect(inFlightForKey('groq', 1)).toBe(2);
    expect(inFlightForKey('groq', 2)).toBe(1);
    expect(inFlightForKey('cerebras', 1)).toBe(1);
    expect(inFlightForKey('groq', 99)).toBe(0);
  });

  it('release frees the slot and is idempotent', () => {
    const id = acquireLease('groq', 'llama-3.3-70b', 1, 100);
    expect(inFlightForKey('groq', 1)).toBe(1);

    releaseLease(id);
    expect(inFlightForKey('groq', 1)).toBe(0);
    // Overlapping cleanup paths must not double-decrement someone else's slot.
    releaseLease(id);
    releaseLease(id);
    expect(inFlightForKey('groq', 1)).toBe(0);
  });

  it('ages out a leaked lease so a key cannot be blocked forever', () => {
    vi.useFakeTimers();
    acquireLease('groq', 'llama-3.3-70b', 1, 100);
    expect(inFlightForKey('groq', 1)).toBe(1);

    // Past LEASE_MAX_AGE_MS (2 min) — the backstop for a release that never came.
    vi.advanceTimersByTime(3 * 60 * 1000);
    expect(inFlightForKey('groq', 1)).toBe(0);
  });
});

describe('per-key concurrency cap', () => {
  beforeEach(() => {
    initDb(':memory:');
    resetLeases();
  });

  afterEach(() => {
    resetLeases();
    delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY;
    delete process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ;
  });

  it('is unlimited by default — capping every provider would cost throughput', () => {
    expect(getKeyConcurrencyLimit('groq')).toBeNull();
    for (let i = 0; i < 25; i++) acquireLease('groq', 'llama-3.3-70b', 1, 100);
    expect(canUseKeyConcurrency('groq', 1)).toBe(true);
  });

  it('reads a per-platform env cap ahead of the global one', () => {
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY = '5';
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '2';
    expect(getKeyConcurrencyLimit('groq')).toBe(2);
    expect(getKeyConcurrencyLimit('cerebras')).toBe(5);
  });

  it('ignores a malformed cap rather than blocking everything', () => {
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = 'not-a-number';
    expect(getKeyConcurrencyLimit('groq')).toBeNull();
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '0';
    expect(getKeyConcurrencyLimit('groq')).toBeNull();
  });

  it('blocks a key at its cap and frees it on release', () => {
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';

    expect(canUseKeyConcurrency('groq', 1)).toBe(true);
    const id = acquireLease('groq', 'llama-3.3-70b', 1, 100);
    expect(canUseKeyConcurrency('groq', 1)).toBe(false);

    releaseLease(id);
    expect(canUseKeyConcurrency('groq', 1)).toBe(true);
  });

  it('caps each key independently, so a sibling key stays available', () => {
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';
    acquireLease('groq', 'llama-3.3-70b', 1, 100);

    // This is the point: the request fails over to key 2 instead of queueing
    // behind key 1 and collecting a 429.
    expect(canUseKeyConcurrency('groq', 1)).toBe(false);
    expect(canUseKeyConcurrency('groq', 2)).toBe(true);
  });

  it('counts a key busy on one model against a request for another', () => {
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '1';
    acquireLease('groq', 'llama-3.3-70b', 1, 100);
    // Concurrency is metered per credential, not per model.
    expect(canUseKeyConcurrency('groq', 1)).toBe(false);
  });

  it('allows up to the cap and no further', () => {
    process.env.MAX_CONCURRENT_REQUESTS_PER_KEY_GROQ = '3';
    const ids = [0, 1, 2].map(() => acquireLease('groq', 'llama-3.3-70b', 1, 100));
    expect(inFlightForKey('groq', 1)).toBe(3);
    expect(canUseKeyConcurrency('groq', 1)).toBe(false);

    releaseLease(ids[0]!);
    expect(canUseKeyConcurrency('groq', 1)).toBe(true);
  });
});