7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): rerank and moderation honour the per-key model scope (#657) (#1677)
tashdroidabf78d93744a
diff --git a/server/src/__tests__/routes/moderation.test.ts b/server/src/__tests__/routes/moderation.test.tsindex 4d176c20..dc7d7cfc 100644--- a/server/src/__tests__/routes/moderation.test.ts+++ b/server/src/__tests__/routes/moderation.test.ts@@ -30,12 +30,12 @@ async function postModeration(body: unknown, token?: string) {return { status: res.status, body: json as any };}-function insertKey(platform: string): number {+function insertKey(platform: string, modelScopeJson?: string): number {const { encrypted, iv, authTag } = encrypt('sk-test');const result = getDb().prepare(`- INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)- VALUES (?, 'test', ?, ?, ?, 'healthy', 1)- `).run(platform, encrypted, iv, authTag);+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, model_scope_json)+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1, ?)+ `).run(platform, encrypted, iv, authTag, modelScopeJson ?? null);return Number(result.lastInsertRowid);}@@ -105,6 +105,27 @@ describe('moderation service — model resolution', () => {expect(listModerationModels()).toHaveLength(0);});+ it('skips a key scoped to another model and takes the next key on the platform (#657)', () => {+ // First key is a relay chat-scope key — moderation must NOT pick it.+ insertKey('openai', '["gpt-4o-mini"]');+ const unscoped = insertKey('openai');+ const models = listModerationModels();+ expect(models.map(m => m.platform)).toEqual(['openai']);+ expect(models[0].keyId).toBe(unscoped);+ });++ it('lists no platform when its only key is scoped out', () => {+ insertKey('openai', '["gpt-4o-mini"]');+ expect(listModerationModels()).toHaveLength(0);+ });++ it('a key scoped exactly to the moderation model serves it', () => {+ const scoped = insertKey('openai', '["omni-moderation-latest"]');+ const models = listModerationModels();+ expect(models.map(m => m.platform)).toEqual(['openai']);+ expect(models[0].keyId).toBe(scoped);+ });+it('resolveModel maps OpenAI legacy names and unknown models to the default', () => {insertKey('openai');insertKey('nvidia');diff --git a/server/src/__tests__/services/rerank.test.ts b/server/src/__tests__/services/rerank.test.tsindex e6044df7..ef9f8232 100644--- a/server/src/__tests__/services/rerank.test.ts+++ b/server/src/__tests__/services/rerank.test.ts@@ -1,17 +1,20 @@import { describe, expect, it, vi, beforeEach } from 'vitest';import { runRerank, RerankError } from '../../services/rerank.js';+// Mutable so the scope tests can hand the service different key rows.+let mockRows: Array<{ id: number; platform: string; base_url: string | null; encrypted_key: string; iv: string; auth_tag: string; model_scope_json?: string | null }> = [+ // cohere first in insertion order; the service must still try the+ // custom endpoint first.+ { id: 1, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },+ { id: 2, platform: 'custom', base_url: 'https://relay.example/v1', encrypted_key: 'e', iv: 'i', auth_tag: 'a' },+ // a custom row without a base_url is not a candidate+ { id: 3, platform: 'custom', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },+];+vi.mock('../../db/index.js', () => ({getDb: () => ({prepare: () => ({- all: () => [- // cohere first in insertion order; the service must still try the- // custom endpoint first.- { id: 1, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },- { id: 2, platform: 'custom', base_url: 'https://relay.example/v1', encrypted_key: 'e', iv: 'i', auth_tag: 'a' },- // a custom row without a base_url is not a candidate- { id: 3, platform: 'custom', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },- ],+ all: () => mockRows,// The monthly-budget gate (#1158) reads api_keys caps and the ledger;// no rows here means every key is uncapped (0 = unlimited).get: () => undefined,@@ -41,4 +44,36 @@ describe('runRerank', () => {vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 9, relevance_score: 1 }]));await expect(runRerank(undefined, 'q', ['a'])).rejects.toBeInstanceOf(RerankError);});++ it('never picks a key scoped to a different model (#657)', async () => {+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 1 }]));+ mockRows = [+ // scoped to a chat model — must be skipped for the default rerank model+ { id: 4, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a', model_scope_json: '["llama-3.1-8b"]' },+ { id: 5, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },+ ];+ const out = await runRerank(undefined, 'q', ['a']);+ expect(out.platform).toBe('cohere');+ // Only the unscoped key may be called — one attempt, not two.+ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it('serves a requested model through a key scoped exactly to it', async () => {+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 1 }]));+ mockRows = [+ { id: 6, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a', model_scope_json: '["rerank-v3.5"]' },+ ];+ const out = await runRerank('rerank-v3.5', 'q', ['a']);+ expect(out.modelId).toBe('rerank-v3.5');+ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it('reports no capable key (503) when every key is scoped out', async () => {+ const fetch = vi.spyOn(global, 'fetch');+ mockRows = [+ { id: 7, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a', model_scope_json: '["some-other-model"]' },+ ];+ await expect(runRerank(undefined, 'q', ['a'])).rejects.toMatchObject({ status: 503 });+ expect(fetch).not.toHaveBeenCalled();+ });});diff --git a/server/src/services/moderation.ts b/server/src/services/moderation.tsindex dc1afb59..d132ab51 100644--- a/server/src/services/moderation.ts+++ b/server/src/services/moderation.ts@@ -14,6 +14,7 @@import { getDb } from '../db/index.js';import { getClientContext } from '../lib/client-context.js';import { decrypt } from '../lib/crypto.js';+import { parseModelScope, scopeAllows } from '../lib/model-scope.js';import { proxyFetch } from '../lib/proxy.js';import { checkMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';import { noteBudgetAlerts } from './budget-alerts.js';@@ -76,15 +77,19 @@ export function listModerationModels(): ModerationModelRow[] {const rows: ModerationModelRow[] = [];for (const platform of PLATFORM_ORDER) {const ep = MODERATION_MODELS[platform];- const key = db.prepare(`- SELECT id, encrypted_key, iv, auth_tag+ const keys = db.prepare(`+ SELECT id, encrypted_key, iv, auth_tag, model_scope_jsonFROM api_keysWHERE platform = ?AND enabled = 1AND status IN ('healthy', 'unknown')ORDER BY id- LIMIT 1- `).get(platform) as { id: number; encrypted_key: string; iv: string; auth_tag: string } | undefined;+ `).all(platform) as { id: number; encrypted_key: string; iv: string; auth_tag: string; model_scope_json: string | null }[];+ // Per-key model scope (#657): a relay key scoped to a chat model group must+ // never be picked to serve the moderation model — the same wall the chat+ // router and search pools honour. Walk the platform's keys in the old+ // first-key order and take the first one the scope admits.+ const key = keys.find(k => scopeAllows(parseModelScope(k.model_scope_json), ep.modelId));if (key) {rows.push({platform,diff --git a/server/src/services/rerank.ts b/server/src/services/rerank.tsindex cf4018f5..a9e59431 100644--- a/server/src/services/rerank.ts+++ b/server/src/services/rerank.ts@@ -1,6 +1,7 @@import { getDb } from '../db/index.js';import { decrypt } from '../lib/crypto.js';import { parseRetryAfterMs } from '../providers/base.js';+import { parseModelScope, scopeAllows } from '../lib/model-scope.js';import { checkMonthlyBudget, reserveMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';import { noteBudgetAlerts } from './budget-alerts.js';import { markKeyHealthyFromRequest } from './health.js';@@ -35,6 +36,14 @@ interface RerankCandidate {platform: string;key: string;baseUrl: string | null;+ /** Parsed api_keys.model_scope_json (#657) — null = unscoped, serves any rerank model. */+ scope: Set<string> | null;+}++/** The model a candidate actually calls for a requested model (same resolution+ * as callProvider), so the scope gate checks the id the key really serves. */+function effectiveModel(cand: Pick<RerankCandidate, 'platform'>, model?: string): string {+ return model || (cand.platform === 'cohere' ? 'rerank-v3.5' : 'rerank');}const FETCH_TIMEOUT_MS = 30_000;@@ -42,13 +51,13 @@ const MAX_DOCUMENTS = 100;function candidates(): RerankCandidate[] {const rows = getDb().prepare(- "SELECT id, platform, base_url, encrypted_key, iv, auth_tag FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') AND platform IN ('cohere', 'custom') ORDER BY platform, created_at",- ).all() as { id: number; platform: string; base_url: string | null; encrypted_key: string; iv: string; auth_tag: string }[];+ "SELECT id, platform, base_url, encrypted_key, iv, auth_tag, model_scope_json FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') AND platform IN ('cohere', 'custom') ORDER BY platform, created_at",+ ).all() as { id: number; platform: string; base_url: string | null; encrypted_key: string; iv: string; auth_tag: string; model_scope_json: string | null }[];const out: RerankCandidate[] = [];for (const row of rows) {if (row.platform === 'custom' && !row.base_url) continue;try {- out.push({ id: row.id, platform: row.platform, key: decrypt(row.encrypted_key, row.iv, row.auth_tag), baseUrl: row.base_url });+ out.push({ id: row.id, platform: row.platform, key: decrypt(row.encrypted_key, row.iv, row.auth_tag), baseUrl: row.base_url, scope: parseModelScope(row.model_scope_json) });} catch { /* undecryptable row is not a candidate */ }}// Custom endpoints (the operator's explicit rerank relays) go first.@@ -117,7 +126,12 @@ export async function runRerank(model: string | undefined, query: string, documeif (documents.length > MAX_DOCUMENTS) {throw new RerankError(`Too many documents: ${documents.length} (max ${MAX_DOCUMENTS})`, 400);}- const chain = candidates();+ // Per-key model scope (#657), the same wall the chat router and search key+ // pools honour: a relay key scoped to a model group must never be picked to+ // serve a model outside its list. Filter up front so a pool where every key+ // is scoped out reports "no capable key configured" (503) instead of+ // pretending the providers were tried and failed (502).+ const chain = candidates().filter(c => scopeAllows(c.scope, effectiveModel(c, model)));if (chain.length === 0) {throw new RerankError('No rerank-capable key configured. Add a Cohere key or a custom rerank endpoint.', 503);}