Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
10.5 KB
Raw Blame
import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
import { initDb, getDb } from '../../db/index.js';
import { modelRetirementSignal } from '../../lib/error-classify.js';
import {
  noteModelRetirementSignal,
  resetModelRetirementObservations,
  RETIREMENT_CONFIRMATIONS_REQUIRED,
} from '../../services/model-retirement.js';
import { getCatalogModelTombstone, recordCatalogModelTombstone } from '../../services/model-state.js';
import { applyCatalog } from '../../services/catalog-sync.js';

// #634: NVIDIA retires a model upstream ("has reached its end of life"), and
// every subsequent request burns a fallback slot on the corpse because nothing
// persisted the 404/410. These tests lock the two halves of the fix: a
// conservative end-of-life CLASSIFIER (transient 404s must never match), and a
// corroboration threshold before the model is actually auto-disabled.

const PLATFORM = 'groq';
const MODEL_ID = 'eol-test-model';

beforeAll(() => {
  process.env.ENCRYPTION_KEY = '0'.repeat(64);
  initDb(':memory:');
});

function seedModel(modelId = MODEL_ID): number {
  const db = getDb();
  db.prepare(`
    INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled, source)
    VALUES (?, ?, 'EOL Test', 50, 50, 1, 'catalog')
    ON CONFLICT(platform, model_id, endpoint_scope) DO UPDATE SET enabled = 1
  `).run(PLATFORM, modelId);
  const row = db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
    .get(PLATFORM, modelId) as { id: number };
  db.prepare(`
    INSERT INTO fallback_config (model_db_id, priority, enabled)
    VALUES (?, 999, 1)
    ON CONFLICT(model_db_id) DO UPDATE SET enabled = 1
  `).run(row.id);
  return row.id;
}

function isRoutable(modelDbId: number): boolean {
  const row = getDb()
    .prepare('SELECT enabled FROM fallback_config WHERE model_db_id = ?')
    .get(modelDbId) as { enabled: number } | undefined;
  return row?.enabled === 1;
}

function routeFor(modelDbId: number, modelId = MODEL_ID) {
  return { modelDbId, platform: PLATFORM, modelId };
}

// The exact NVIDIA body from issue #634.
const NVIDIA_EOL = Object.assign(
  new Error(
    "NVIDIA NIM API error 410: The model 'minimaxai/minimax-m2.7' has reached its end of life "
    + 'on 2026-07-27T00:00:00Z and is no longer available.',
  ),
  { status: 410 },
);

// The transient shape from the same issue thread: a load balancer that cannot
// find the function for the account right now. Says nothing about the model.
const TRANSIENT_404 = Object.assign(
  new Error(
    "NVIDIA NIM API error 404: Function '23d4f03a-b8a6-4adb-a183-7daa083a09cc': Not found for account 'BDnmPUMI'",
  ),
  { status: 404 },
);

describe('modelRetirementSignal (conservative end-of-life detection — #634)', () => {
  it('treats an explicit 410 / end-of-life body as definitive', () => {
    expect(modelRetirementSignal(NVIDIA_EOL)).toBe('definitive');
    expect(modelRetirementSignal(Object.assign(new Error('Gone'), { status: 410 }))).toBe('definitive');
    expect(modelRetirementSignal(new Error('Ollama Cloud API error 410: Gone'))).toBe('definitive');
    // The wording alone is definitive even when the provider picks a 404.
    expect(modelRetirementSignal(Object.assign(
      new Error('API error 404: model has reached its end of life'),
      { status: 404 },
    ))).toBe('definitive');
  });

  // 'definitive' skips the corroboration gate and disables the model on ONE
  // response, so the phrase must not be load-bearing by itself: a rate-limit
  // or server-error body can quote a retirement notice (about this model or
  // another) without the model being gone.
  it('ignores an end-of-life phrase when the status does not agree the model is gone', () => {
    expect(modelRetirementSignal(Object.assign(
      new Error('Rate limit exceeded. Note: llama-3.1-8b reaches end of life on 2026-12-01'),
      { status: 429 },
    ))).toBeNull();
    expect(modelRetirementSignal(Object.assign(
      new Error('Internal server error: upstream pool has been decommissioned'),
      { status: 500 },
    ))).toBeNull();
    expect(modelRetirementSignal(Object.assign(
      new Error('Bad request: the v1 endpoint has been sunset, use v2'),
      { status: 400 },
    ))).toBeNull();
  });

  it('still fires when a gone-shaped status carries the phrase', () => {
    expect(modelRetirementSignal(Object.assign(
      new Error('API error 410: this model has been retired'),
      { status: 410 },
    ))).toBe('definitive');
    expect(modelRetirementSignal(Object.assign(
      new Error('API error 404: this model has been decommissioned'),
      { status: 404 },
    ))).toBe('definitive');
  });

  it('treats a 404 that clearly says the model is gone as probable (needs corroboration)', () => {
    expect(modelRetirementSignal(Object.assign(
      new Error('OpenRouter API error 404: this model is no longer available'),
      { status: 404 },
    ))).toBe('probable');
    expect(modelRetirementSignal(Object.assign(
      new Error('Groq API error 404: model has been removed'),
      { status: 404 },
    ))).toBe('probable');
  });

  it('never fires on transient 404s, empty not-found bodies, or unrelated failures', () => {
    expect(modelRetirementSignal(TRANSIENT_404)).toBeNull();
    expect(modelRetirementSignal(new Error('OpenRouter API error 404: Provider returned error'))).toBeNull();
    expect(modelRetirementSignal(new Error('Model not found'))).toBeNull();
    // OpenRouter says this while a model is merely unserved right now.
    expect(modelRetirementSignal(new Error('No endpoints found for openrouter/minimax/minimax-m2.5:free'))).toBeNull();
    expect(modelRetirementSignal(new Error('429 Too Many Requests'))).toBeNull();
    expect(modelRetirementSignal(new Error('503 Service Unavailable'))).toBeNull();
    // A digit run that merely contains 410 is not a 410.
    expect(modelRetirementSignal(new Error('Groq API error 400: max_tokens 41000 exceeds 32768'))).toBeNull();
  });
});

describe('noteModelRetirementSignal (auto-disable with a reason — #634)', () => {
  beforeEach(() => {
    resetModelRetirementObservations();
    getDb().prepare("DELETE FROM catalog_model_tombstones WHERE model_id LIKE 'eol-test-%'").run();
  });

  it('auto-disables the model on a single explicit 410 end-of-life response', () => {
    const id = seedModel();
    expect(noteModelRetirementSignal(routeFor(id), NVIDIA_EOL, {})).toBe(true);
    expect(isRoutable(id)).toBe(false);

    const tombstone = getCatalogModelTombstone(getDb(), 'chat', PLATFORM, MODEL_ID);
    expect(tombstone?.source).toBe('upstream_eol');
    expect(tombstone?.reason).toContain('end of life');
  });

  it('leaves a transient 404 alone no matter how often it repeats', () => {
    const id = seedModel();
    for (let i = 0; i < 5; i++) {
      expect(noteModelRetirementSignal(routeFor(id), TRANSIENT_404, {})).toBe(false);
    }
    expect(isRoutable(id)).toBe(true);
    expect(getCatalogModelTombstone(getDb(), 'chat', PLATFORM, MODEL_ID)).toBeUndefined();
  });

  it('requires corroboration from distinct requests before disabling on a probable 404', () => {
    const id = seedModel();
    const err = Object.assign(new Error('API error 404: this model is no longer available'), { status: 404 });

    expect(RETIREMENT_CONFIRMATIONS_REQUIRED).toBeGreaterThanOrEqual(2);
    expect(noteModelRetirementSignal(routeFor(id), err, {})).toBe(false);
    expect(isRoutable(id)).toBe(true);
    expect(noteModelRetirementSignal(routeFor(id), err, {})).toBe(true);
    expect(isRoutable(id)).toBe(false);
  });

  it('does not let one flaky request corroborate itself across sibling keys', () => {
    const id = seedModel();
    const err = Object.assign(new Error('API error 404: this model is no longer available'), { status: 404 });
    const oneRequest = {};

    for (let i = 0; i < 4; i++) {
      expect(noteModelRetirementSignal(routeFor(id), err, oneRequest)).toBe(false);
    }
    expect(isRoutable(id)).toBe(true);
  });

  it('never touches user-added models (they are not catalog state)', () => {
    const db = getDb();
    db.prepare(`
      INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled, source)
      VALUES ('custom', 'eol-test-custom', 'Custom EOL', 50, 50, 1, 'user')
      ON CONFLICT(platform, model_id, endpoint_scope) DO UPDATE SET enabled = 1
    `).run();
    const row = db.prepare("SELECT id FROM models WHERE platform = 'custom' AND model_id = 'eol-test-custom'")
      .get() as { id: number };
    db.prepare(`
      INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 998, 1)
      ON CONFLICT(model_db_id) DO UPDATE SET enabled = 1
    `).run(row.id);

    expect(noteModelRetirementSignal(
      { modelDbId: row.id, platform: 'custom', modelId: 'eol-test-custom' },
      NVIDIA_EOL,
      {},
    )).toBe(false);
    expect(isRoutable(row.id)).toBe(true);
  });
});

describe('catalog sync vs. upstream retirement (#634)', () => {
  function catalogWith(modelId: string, enabled: boolean) {
    return {
      version: '2099.01.01',
      generatedAt: new Date().toISOString(),
      tier: 'live' as const,
      models: [{
        platform: PLATFORM,
        modelId,
        displayName: 'EOL Test',
        intelligenceRank: 50,
        speedRank: 50,
        sizeLabel: 'Medium',
        limits: { rpm: 30, rpd: 1000, tpm: 6000, tpd: null },
        monthlyTokenBudget: '~1M',
        contextWindow: 8192,
        enabled,
        supportsVision: false,
        supportsTools: true,
      }],
      quirks: [],
    } as unknown as Parameters<typeof applyCatalog>[1];
  }

  it('re-enables an auto-retired model when a later catalog still lists it', () => {
    resetModelRetirementObservations();
    const id = seedModel('eol-test-relisted');
    noteModelRetirementSignal(routeFor(id, 'eol-test-relisted'), NVIDIA_EOL, {});
    expect(isRoutable(id)).toBe(false);

    applyCatalog(getDb(), catalogWith('eol-test-relisted', true));

    expect(isRoutable(id)).toBe(true);
    expect(getCatalogModelTombstone(getDb(), 'chat', PLATFORM, 'eol-test-relisted')).toBeUndefined();
    expect(getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
      .get(PLATFORM, 'eol-test-relisted')).toBeDefined();
  });

  it('still deletes models the USER tombstoned (unchanged behavior)', () => {
    seedModel('eol-test-user-deleted');
    recordCatalogModelTombstone(getDb(), 'chat', PLATFORM, 'eol-test-user-deleted');
    applyCatalog(getDb(), catalogWith('eol-test-user-deleted', true));
    expect(getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
      .get(PLATFORM, 'eol-test-user-deleted')).toBeUndefined();
  });
});