Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
4.2 KB
Raw Blame
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { initDb, getDb } from '../../db/index.js';
import {
  canUseProviderMinute,
  providerMinuteRequestCount,
  getProviderMinuteRequestCap,
} from '../../services/ratelimit.js';

/**
 * Account-wide per-minute caps. The per-model rpm_limit cannot express these:
 * NVIDIA NIM meters ~40 req/min across the whole account, so every model on it
 * draws from one bucket, and a model row with rpm_limit NULL isn't pre-throttled
 * at all. Without this gate the router keeps dispatching and eats real 429s.
 */
function seedRequests(platform: string, modelId: string, keyId: number, count: number, atMs: number) {
  const ins = getDb().prepare(`
    INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
    VALUES (?, ?, ?, 'request', 0, ?)
  `);
  for (let i = 0; i < count; i++) ins.run(platform, modelId, keyId, atMs);
}

describe('provider-wide per-minute request cap', () => {
  let keyId: number;

  beforeEach(() => {
    initDb(':memory:');
    keyId = Math.floor(Math.random() * 1_000_000);
  });

  afterEach(() => {
    delete process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA;
    delete process.env.PROVIDER_MINUTE_REQUEST_CAP_GROQ;
    delete process.env.PROVIDER_MINUTE_REQUEST_CAP_APINEX;
  });

  it('ships a default cap for nvidia and none for a provider without one', () => {
    expect(getProviderMinuteRequestCap('nvidia')).toBe(40);
    expect(getProviderMinuteRequestCap('groq')).toBeNull();
  });

  it('applies the APINex 5 RPM cap across different free models, per key', () => {
    const now = Date.now();
    expect(getProviderMinuteRequestCap('apinex')).toBe(5);
    seedRequests('apinex', 'free/deepseek-v4.1-flash', keyId, 3, now - 20_000);
    seedRequests('apinex', 'free/gpt-6-luna', keyId, 2, now - 10_000);
    expect(providerMinuteRequestCount('apinex', keyId, now)).toBe(5);
    expect(canUseProviderMinute('apinex', keyId, now)).toBe(false);
    expect(canUseProviderMinute('apinex', keyId + 1, now)).toBe(true);
    expect(canUseProviderMinute('apinex', keyId, now + 61_000)).toBe(true);
  });

  it('honours an env override and treats 0 as disabled', () => {
    process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '5';
    expect(getProviderMinuteRequestCap('nvidia')).toBe(5);
    process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '0';
    expect(getProviderMinuteRequestCap('nvidia')).toBeNull();
  });

  it('sums usage across every model on the same account+key', () => {
    const now = Date.now();
    // The whole point: three different models, one shared budget.
    seedRequests('nvidia', 'glm-4.7', keyId, 10, now - 5_000);
    seedRequests('nvidia', 'minimax-m3', keyId, 10, now - 4_000);
    seedRequests('nvidia', 'deepseek-v4', keyId, 10, now - 3_000);

    expect(providerMinuteRequestCount('nvidia', keyId, now)).toBe(30);
  });

  it('blocks once the shared budget is spent, even for an untouched model', () => {
    process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';
    const now = Date.now();
    seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 10_000);

    // deepseek-v4 has made zero requests of its own and may well have a NULL
    // rpm_limit, but the account budget is gone.
    expect(canUseProviderMinute('nvidia', keyId, now)).toBe(false);
  });

  it('allows again once the minute window rolls past', () => {
    process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';
    const now = Date.now();
    seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 61_000);

    expect(providerMinuteRequestCount('nvidia', keyId, now)).toBe(0);
    expect(canUseProviderMinute('nvidia', keyId, now)).toBe(true);
  });

  it('does not gate a provider with no configured cap', () => {
    const now = Date.now();
    seedRequests('groq', 'llama-3.3-70b', keyId, 500, now - 1_000);
    expect(canUseProviderMinute('groq', keyId, now)).toBe(true);
  });

  it('keeps separate budgets per key on the same platform', () => {
    process.env.PROVIDER_MINUTE_REQUEST_CAP_NVIDIA = '40';
    const now = Date.now();
    const other = keyId + 1;
    seedRequests('nvidia', 'glm-4.7', keyId, 40, now - 5_000);

    expect(canUseProviderMinute('nvidia', keyId, now)).toBe(false);
    expect(canUseProviderMinute('nvidia', other, now)).toBe(true);
  });
});