Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
16.7 KB
Raw Blame
import { describe, it, expect } from 'vitest';
import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError } from '../../routes/proxy.js';
import { isProviderBadRequestError } from '../../lib/error-classify.js';
import { cooldownDecisionForError } from '../../lib/fallback-loop.js';
import { MODEL_FORBIDDEN_COOLDOWN_MS } from '../../services/ratelimit.js';

describe('isModelAccessForbiddenError (403 model-not-on-tier, drives whole-model skip — issue #256)', () => {
  it('flags a 403 reaching the proxy by message or attached status', () => {
    // GitHub Models / Cloudflare 403 a model the key's free tier can't reach.
    expect(isModelAccessForbiddenError(new Error('GitHub Models API error 403: Model not available on your plan'))).toBe(true);
    expect(isModelAccessForbiddenError(new Error('Cloudflare API error 403: this model requires a subscription'))).toBe(true);
    expect(isModelAccessForbiddenError(new Error('Forbidden'))).toBe(true);
    // #261 attaches the upstream status to the thrown error; honor it even if
    // the message phrasing omits the code.
    expect(isModelAccessForbiddenError(Object.assign(new Error('access denied'), { status: 403 }))).toBe(true);
  });

  it('does not flag rate limits, 404s, or payment errors', () => {
    expect(isModelAccessForbiddenError(new Error('429 Too Many Requests'))).toBe(false);
    expect(isModelAccessForbiddenError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(false);
    expect(isModelAccessForbiddenError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);
  });

  // Issue #618: some providers deny model access with a 400 (or 401) instead of
  // a 403. Classified transient, those got the 90s bench — and the auto-router
  // re-picked the same unreachable model the moment it expired, forever, with
  // an escalating penalty. They must classify as model-inaccessible-for-this-key.
  it('flags a 400/401 whose body says the key may not access the model (#618)', () => {
    expect(isModelAccessForbiddenError(Object.assign(
      new Error('API error 400: user is not allowed to access model kat-coder-pro-v2.5'),
      { status: 400 },
    ))).toBe(true);
    expect(isModelAccessForbiddenError(Object.assign(
      new Error('API error 400: action plan limited, please upgrade'),
      { status: 400 },
    ))).toBe(true);
    expect(isModelAccessForbiddenError(Object.assign(
      new Error('API error 401: you do not have access to this model'),
      { status: 401 },
    ))).toBe(true);
    expect(isModelAccessForbiddenError(
      new Error('user is not allowed to access model kat-coder-pro-v2.5'),
    )).toBe(true);
  });

  it('still leaves ordinary 400/401 rejections alone', () => {
    expect(isModelAccessForbiddenError(Object.assign(new Error('Bad Request'), { status: 400 }))).toBe(false);
    expect(isModelAccessForbiddenError(Object.assign(new Error('Unauthorized'), { status: 401 }))).toBe(false);
    expect(isModelAccessForbiddenError(new Error('401 Unauthorized'))).toBe(false);
    expect(isModelAccessForbiddenError(new Error('400 Bad Request'))).toBe(false);
    expect(isModelAccessForbiddenError(Object.assign(
      new Error('Groq API error 400: Failed to call a function. Please adjust your prompt.'),
      { status: 400 },
    ))).toBe(false);
  });
});

// The loop-level consequence of the classification above: a "not allowed to
// access" 400 must take the model-forbidden bench (a day, source 'tier'), not
// the 90s transient one that made the router re-pick it every 90 seconds.
describe('cooldownDecisionForError on a 400 not-allowed body (#618)', () => {
  const route = {
    platform: 'kat', modelId: 'kat-coder-pro-v2.5', modelDbId: 1, keyId: 1,
    rpdLimit: null, tpdLimit: null,
  } as any;

  it('benches the model like a 403 instead of the transient ladder', () => {
    const err = Object.assign(
      new Error('API error 400: user is not allowed to access model kat-coder-pro-v2.5'),
      { status: 400 },
    );
    const decision = cooldownDecisionForError(route, err);
    expect(decision.durationMs).toBe(MODEL_FORBIDDEN_COOLDOWN_MS);
    expect(decision.source).toBe('tier');
  });
});

describe('isModelNotFoundError (drives whole-model skip within a request)', () => {
  it('flags 404 / not-found / no-endpoints phrasings', () => {
    expect(isModelNotFoundError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(true);
    expect(isModelNotFoundError(new Error('Model not found'))).toBe(true);
    expect(isModelNotFoundError(new Error('No endpoints found for openrouter/minimax/minimax-m2.5:free'))).toBe(true);
  });

  it('flags a stale/removed model reported as a 400 "No model found" (Routeway) — MODEL-level, not request shape', () => {
    // "No model found" does NOT contain the substring "not found" (words are
    // no/model/found), so before the phrase list it slipped through to
    // isProviderBadRequestError and surfaced as a request-blaming 400.
    expect(isModelNotFoundError(Object.assign(new Error('Routeway API error 400: No model found: llama-3.3-70b-instruct:free'), { status: 400 }))).toBe(true);
    expect(isModelNotFoundError(new Error('Groq API error 400: model not found'))).toBe(true);
    expect(isModelNotFoundError(new Error('Provider API error 400: unknown model'))).toBe(true);
    expect(isModelNotFoundError(new Error('API error 400: model does not exist'))).toBe(true);
    expect(isModelNotFoundError(new Error('API error 404: no such model'))).toBe(true);
  });

  it('flags 410 Gone (model pulled upstream) by message or attached status — #339', () => {
    expect(isModelNotFoundError(new Error('Ollama Cloud API error 410: Gone'))).toBe(true);
    expect(isModelNotFoundError(Object.assign(new Error('Gone'), { status: 410 }))).toBe(true);
  });

  it('does not flag rate limits, 5xx, or payment errors', () => {
    expect(isModelNotFoundError(new Error('429 Too Many Requests'))).toBe(false);
    expect(isModelNotFoundError(new Error('503 Service Unavailable'))).toBe(false);
    expect(isModelNotFoundError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);
  });
});

describe('isRetryableError', () => {
  describe('413 Payload Too Large', () => {
    it('treats explicit "413" in the error message as retryable', () => {
      expect(isRetryableError(new Error('GitHub Models API error 413: Request body too large'))).toBe(true);
      expect(isRetryableError(new Error('Cloudflare API error 413: Payload Too Large'))).toBe(true);
    });

    it('treats common 413 phrasings (no status code) as retryable', () => {
      expect(isRetryableError(new Error('Payload Too Large'))).toBe(true);
      expect(isRetryableError(new Error('Request body too large for this model'))).toBe(true);
      expect(isRetryableError(new Error('Request entity too large'))).toBe(true);
      expect(isRetryableError(new Error('Content too large'))).toBe(true);
    });
  });

  describe('404 model removed / not found (the bug #66 fixes)', () => {
    it('treats explicit "404" in the error message as retryable', () => {
      expect(isRetryableError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(true);
      expect(isRetryableError(new Error('Groq API error 404: model not found'))).toBe(true);
    });

    it('catches OpenRouter\'s "No endpoints found" phrasing for deprecated models', () => {
      expect(isRetryableError(new Error('No endpoints found for openrouter/minimax/minimax-m2.5:free'))).toBe(true);
    });

    it('catches bare "not found" phrasing (any provider, any case)', () => {
      expect(isRetryableError(new Error('Model not found'))).toBe(true);
      expect(isRetryableError(new Error('The requested model was not found'))).toBe(true);
    });
  });

  describe('provider tool-call generation 400s fail over (#168)', () => {
    // Groq (and every other openai-compat provider) throws its errors as
    // `${name} API error ${status}: ${msg}`, so a tool-call-generation failure
    // surfaces as "Groq API error 400: Failed to call a function...". That
    // matches the "api error 400" rule, so it's ALREADY retryable and fails
    // over to the next provider — #168 is covered by existing behavior.
    it('treats a Groq failed_generation 400 as retryable', () => {
      expect(isRetryableError(new Error(
        "Groq API error 400: Failed to call a function. Please adjust your prompt. See 'failed_generation' for more details.",
      ))).toBe(true);
    });

    it('treats any openai-compat "API error 400" as retryable (one provider rejects params another accepts)', () => {
      expect(isRetryableError(new Error('Cerebras API error 400: tool schema not supported'))).toBe(true);
    });

    it('but a bare validation "400 Bad Request" (our own schema) is still NOT retryable', () => {
      expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);
    });

    it('flags provider API 400s for invalid-request exhaustion reporting', () => {
      const err = Object.assign(
        new Error('Google API error 400: Invalid JSON payload received. Unknown name "x-google-enum-descriptions"'),
        { status: 400 },
      );
      expect(isProviderBadRequestError(err)).toBe(true);
      expect(isProviderBadRequestError(new Error('400 Bad Request'))).toBe(false);
      expect(isProviderBadRequestError(Object.assign(new Error('Bad Request'), { status: 400 }))).toBe(false);
    });

    it('treats provider API 422s like Mistral validation rejects: retryable, then invalid-request on exhaustion', () => {
      const err = Object.assign(
        new Error('Mistral API error 422: Unprocessable Entity'),
        { status: 422 },
      );
      expect(isRetryableError(err)).toBe(true);
      expect(isProviderBadRequestError(err)).toBe(true);
      expect(isRetryableError(new Error('Mistral API error 422: tool messages failed validation'))).toBe(true);
      expect(isProviderBadRequestError(new Error('Mistral API error 422: tool messages failed validation'))).toBe(true);
    });
  });

  describe('403 model not on this key\'s tier fails over instead of 502 (issue #256)', () => {
    it('treats a 403 from GitHub Models / Cloudflare as retryable', () => {
      expect(isRetryableError(new Error('GitHub Models API error 403: Model not available on your plan'))).toBe(true);
      expect(isRetryableError(new Error('Cloudflare API error 403: this model requires a subscription'))).toBe(true);
    });

    it('treats a bare "Forbidden" / attached 403 status as retryable', () => {
      expect(isRetryableError(new Error('Forbidden'))).toBe(true);
      expect(isRetryableError(Object.assign(new Error('access denied'), { status: 403 }))).toBe(true);
    });

    it('still treats a bare 400 validation error as non-retryable', () => {
      expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);
    });
  });

  describe('402 Payment Required out-of-credits fails over (graceful degradation)', () => {
    it('treats a HuggingFace Router 402 as retryable (same model lives on other providers)', () => {
      expect(isRetryableError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(true);
    });

    it('catches common out-of-credits phrasings', () => {
      expect(isRetryableError(new Error('Payment Required'))).toBe(true);
      expect(isRetryableError(new Error('You exceeded your current quota: insufficient_quota'))).toBe(true);
      expect(isRetryableError(new Error('Insufficient credit for this request'))).toBe(true);
      expect(isRetryableError(new Error('Insufficient balance'))).toBe(true);
    });

    it('isPaymentRequiredError flags 402 (drives the long bench) but not a 429', () => {
      expect(isPaymentRequiredError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(true);
      expect(isPaymentRequiredError(new Error('429 Too Many Requests'))).toBe(false);
      expect(isPaymentRequiredError(new Error('503 Service Unavailable'))).toBe(false);
    });

    // #1277 follow-up: the digits 402 inside a token count or id are not a
    // status. The 402 bench takes the key off every model of the platform for
    // a day, so these false positives emptied whole providers.
    it('isPaymentRequiredError ignores 402 inside other numbers and under another status', () => {
      for (const message of [
        'groq API error 413: Request too large. Limit 30000, Requested 34026',
        'openrouter API error 429: rate limit, 14023 tokens used',
        'provider API error 500: upstream request id 8f402ab',
        'ACLIDE API error 400: max_tokens 402 is below the minimum',
        'timeout after 4.402s',
      ]) {
        expect(isPaymentRequiredError(new Error(message)), message).toBe(false);
      }
      expect(isPaymentRequiredError(Object.assign(new Error('Requested 402 tokens'), { status: 413 }))).toBe(false);
    });

    it('isPaymentRequiredError still catches every real out-of-credits shape', () => {
      expect(isPaymentRequiredError(new Error('402 Payment Required'))).toBe(true);
      expect(isPaymentRequiredError(new Error('upstream returned 402'))).toBe(true);
      expect(isPaymentRequiredError(Object.assign(new Error('no credits left'), { status: 402 }))).toBe(true);
      expect(isPaymentRequiredError(new Error('Pollinations API error 402: insufficient credit'))).toBe(true);
      expect(isPaymentRequiredError(new Error('provider API error 429: insufficient balance (1008)'))).toBe(true);
      expect(isPaymentRequiredError(new Error('openai API error 429: insufficient_quota'))).toBe(true);
    });
  });

  describe('410 Gone & un-enumerated upstream statuses fail over instead of 502 (#337/#339)', () => {
    // The headline bug: a provider error whose HTTP status the substring allowlist
    // never enumerated (410 Gone, 502, 504, 408 …) used to abort the whole chain
    // with a 502 — stranding the healthy paid routes still queued later in the
    // fallback order. It must rotate to the next route instead.
    it('treats an Ollama "410: Gone" as retryable, by message and by attached status', () => {
      // openai-compat throws via providerHttpError, so the real error carries both.
      expect(isRetryableError(new Error('Ollama Cloud API error 410: Gone'))).toBe(true);
      expect(isRetryableError(Object.assign(new Error('Ollama Cloud API error 410: Gone'), { status: 410 }))).toBe(true);
    });

    it('fails over on any 5xx the substring rules never listed, via the structured status', () => {
      // No '502'/'504'/'507' substring rule exists; the err.status catch-all covers them.
      expect(isRetryableError(Object.assign(new Error('Bad Gateway'), { status: 502 }))).toBe(true);
      expect(isRetryableError(Object.assign(new Error('Gateway Timeout'), { status: 504 }))).toBe(true);
      expect(isRetryableError(Object.assign(new Error('Insufficient Storage'), { status: 507 }))).toBe(true);
    });

    it('fails over on 408 request-timeout / 409 conflict by status', () => {
      expect(isRetryableError(Object.assign(new Error('Request Timeout'), { status: 408 }))).toBe(true);
      expect(isRetryableError(Object.assign(new Error('Conflict'), { status: 409 }))).toBe(true);
    });

    it('still treats genuinely-fatal 400/401 as NON-retryable even with an attached status', () => {
      // The structured catch-all must not swallow client-fatal errors — they fail on
      // every provider identically, so aborting the request is the correct behavior.
      expect(isRetryableError(Object.assign(new Error('Bad Request'), { status: 400 }))).toBe(false);
      expect(isRetryableError(Object.assign(new Error('Unauthorized'), { status: 401 }))).toBe(false);
    });
  });

  describe('existing categories still classify correctly', () => {
    it('429 / rate limits are retryable', () => {
      expect(isRetryableError(new Error('429 Too Many Requests'))).toBe(true);
      expect(isRetryableError(new Error('rate limit exceeded'))).toBe(true);
      expect(isRetryableError(new Error('quota exhausted'))).toBe(true);
    });

    it('5xx and network errors are retryable', () => {
      expect(isRetryableError(new Error('503 Service Unavailable'))).toBe(true);
      expect(isRetryableError(new Error('500 Internal Server Error'))).toBe(true);
      expect(isRetryableError(new Error('ETIMEDOUT'))).toBe(true);
      expect(isRetryableError(new Error('ECONNREFUSED'))).toBe(true);
    });

    it('401 / bare-400 auth & validation errors are NOT retryable', () => {
      expect(isRetryableError(new Error('401 Unauthorized'))).toBe(false);
      expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);
      expect(isRetryableError(new Error('Invalid API key'))).toBe(false);
      // 403 is deliberately NOT here anymore: a request-time 403 on a key that
      // passed validateKey is a model-not-on-tier gate, so it fails over to the
      // next model rather than 502-ing the request (issue #256). The 403 cases
      // are covered in the dedicated describe block above.
    });
  });
});