Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): rerank and moderation honour the per-key model scope (#657) (#1677)

tashdroidabf78d93744a
diff --git a/server/src/__tests__/routes/moderation.test.ts b/server/src/__tests__/routes/moderation.test.ts
index 4d176c20..dc7d7cfc 100644
--- a/server/src/__tests__/routes/moderation.test.ts
+++ b/server/src/__tests__/routes/moderation.test.ts
@@ -30,12 +30,12 @@ async function postModeration(body: unknown, token?: string) {
return { status: res.status, body: json as any };
}
-function insertKey(platform: string): number {
+function insertKey(platform: string, modelScopeJson?: string): number {
const { encrypted, iv, authTag } = encrypt('sk-test');
const result = getDb().prepare(`
- INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
- VALUES (?, 'test', ?, ?, ?, 'healthy', 1)
- `).run(platform, encrypted, iv, authTag);
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, model_scope_json)
+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1, ?)
+ `).run(platform, encrypted, iv, authTag, modelScopeJson ?? null);
return Number(result.lastInsertRowid);
}
@@ -105,6 +105,27 @@ describe('moderation service — model resolution', () => {
expect(listModerationModels()).toHaveLength(0);
});
+ it('skips a key scoped to another model and takes the next key on the platform (#657)', () => {
+ // First key is a relay chat-scope key — moderation must NOT pick it.
+ insertKey('openai', '["gpt-4o-mini"]');
+ const unscoped = insertKey('openai');
+ const models = listModerationModels();
+ expect(models.map(m => m.platform)).toEqual(['openai']);
+ expect(models[0].keyId).toBe(unscoped);
+ });
+
+ it('lists no platform when its only key is scoped out', () => {
+ insertKey('openai', '["gpt-4o-mini"]');
+ expect(listModerationModels()).toHaveLength(0);
+ });
+
+ it('a key scoped exactly to the moderation model serves it', () => {
+ const scoped = insertKey('openai', '["omni-moderation-latest"]');
+ const models = listModerationModels();
+ expect(models.map(m => m.platform)).toEqual(['openai']);
+ expect(models[0].keyId).toBe(scoped);
+ });
+
it('resolveModel maps OpenAI legacy names and unknown models to the default', () => {
insertKey('openai');
insertKey('nvidia');
diff --git a/server/src/__tests__/services/rerank.test.ts b/server/src/__tests__/services/rerank.test.ts
index e6044df7..ef9f8232 100644
--- a/server/src/__tests__/services/rerank.test.ts
+++ b/server/src/__tests__/services/rerank.test.ts
@@ -1,17 +1,20 @@
import { describe, expect, it, vi, beforeEach } from 'vitest';
import { runRerank, RerankError } from '../../services/rerank.js';
+// Mutable so the scope tests can hand the service different key rows.
+let mockRows: Array<{ id: number; platform: string; base_url: string | null; encrypted_key: string; iv: string; auth_tag: string; model_scope_json?: string | null }> = [
+ // cohere first in insertion order; the service must still try the
+ // custom endpoint first.
+ { id: 1, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
+ { id: 2, platform: 'custom', base_url: 'https://relay.example/v1', encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
+ // a custom row without a base_url is not a candidate
+ { id: 3, platform: 'custom', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
+];
+
vi.mock('../../db/index.js', () => ({
getDb: () => ({
prepare: () => ({
- all: () => [
- // cohere first in insertion order; the service must still try the
- // custom endpoint first.
- { id: 1, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
- { id: 2, platform: 'custom', base_url: 'https://relay.example/v1', encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
- // a custom row without a base_url is not a candidate
- { id: 3, platform: 'custom', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
- ],
+ all: () => mockRows,
// The monthly-budget gate (#1158) reads api_keys caps and the ledger;
// no rows here means every key is uncapped (0 = unlimited).
get: () => undefined,
@@ -41,4 +44,36 @@ describe('runRerank', () => {
vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 9, relevance_score: 1 }]));
await expect(runRerank(undefined, 'q', ['a'])).rejects.toBeInstanceOf(RerankError);
});
+
+ it('never picks a key scoped to a different model (#657)', async () => {
+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 1 }]));
+ mockRows = [
+ // scoped to a chat model — must be skipped for the default rerank model
+ { id: 4, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a', model_scope_json: '["llama-3.1-8b"]' },
+ { id: 5, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a' },
+ ];
+ const out = await runRerank(undefined, 'q', ['a']);
+ expect(out.platform).toBe('cohere');
+ // Only the unscoped key may be called — one attempt, not two.
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it('serves a requested model through a key scoped exactly to it', async () => {
+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(ok([{ index: 0, relevance_score: 1 }]));
+ mockRows = [
+ { id: 6, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a', model_scope_json: '["rerank-v3.5"]' },
+ ];
+ const out = await runRerank('rerank-v3.5', 'q', ['a']);
+ expect(out.modelId).toBe('rerank-v3.5');
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it('reports no capable key (503) when every key is scoped out', async () => {
+ const fetch = vi.spyOn(global, 'fetch');
+ mockRows = [
+ { id: 7, platform: 'cohere', base_url: null, encrypted_key: 'e', iv: 'i', auth_tag: 'a', model_scope_json: '["some-other-model"]' },
+ ];
+ await expect(runRerank(undefined, 'q', ['a'])).rejects.toMatchObject({ status: 503 });
+ expect(fetch).not.toHaveBeenCalled();
+ });
});
diff --git a/server/src/services/moderation.ts b/server/src/services/moderation.ts
index dc1afb59..d132ab51 100644
--- a/server/src/services/moderation.ts
+++ b/server/src/services/moderation.ts
@@ -14,6 +14,7 @@
import { getDb } from '../db/index.js';
import { getClientContext } from '../lib/client-context.js';
import { decrypt } from '../lib/crypto.js';
+import { parseModelScope, scopeAllows } from '../lib/model-scope.js';
import { proxyFetch } from '../lib/proxy.js';
import { checkMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';
import { noteBudgetAlerts } from './budget-alerts.js';
@@ -76,15 +77,19 @@ export function listModerationModels(): ModerationModelRow[] {
const rows: ModerationModelRow[] = [];
for (const platform of PLATFORM_ORDER) {
const ep = MODERATION_MODELS[platform];
- const key = db.prepare(`
- SELECT id, encrypted_key, iv, auth_tag
+ const keys = db.prepare(`
+ SELECT id, encrypted_key, iv, auth_tag, model_scope_json
FROM api_keys
WHERE platform = ?
AND enabled = 1
AND status IN ('healthy', 'unknown')
ORDER BY id
- LIMIT 1
- `).get(platform) as { id: number; encrypted_key: string; iv: string; auth_tag: string } | undefined;
+ `).all(platform) as { id: number; encrypted_key: string; iv: string; auth_tag: string; model_scope_json: string | null }[];
+ // Per-key model scope (#657): a relay key scoped to a chat model group must
+ // never be picked to serve the moderation model — the same wall the chat
+ // router and search pools honour. Walk the platform's keys in the old
+ // first-key order and take the first one the scope admits.
+ const key = keys.find(k => scopeAllows(parseModelScope(k.model_scope_json), ep.modelId));
if (key) {
rows.push({
platform,
diff --git a/server/src/services/rerank.ts b/server/src/services/rerank.ts
index cf4018f5..a9e59431 100644
--- a/server/src/services/rerank.ts
+++ b/server/src/services/rerank.ts
@@ -1,6 +1,7 @@
import { getDb } from '../db/index.js';
import { decrypt } from '../lib/crypto.js';
import { parseRetryAfterMs } from '../providers/base.js';
+import { parseModelScope, scopeAllows } from '../lib/model-scope.js';
import { checkMonthlyBudget, reserveMonthlyBudget, secondsUntilNextMonth } from './key-budget.js';
import { noteBudgetAlerts } from './budget-alerts.js';
import { markKeyHealthyFromRequest } from './health.js';
@@ -35,6 +36,14 @@ interface RerankCandidate {
platform: string;
key: string;
baseUrl: string | null;
+ /** Parsed api_keys.model_scope_json (#657) — null = unscoped, serves any rerank model. */
+ scope: Set<string> | null;
+}
+
+/** The model a candidate actually calls for a requested model (same resolution
+ * as callProvider), so the scope gate checks the id the key really serves. */
+function effectiveModel(cand: Pick<RerankCandidate, 'platform'>, model?: string): string {
+ return model || (cand.platform === 'cohere' ? 'rerank-v3.5' : 'rerank');
}
const FETCH_TIMEOUT_MS = 30_000;
@@ -42,13 +51,13 @@ const MAX_DOCUMENTS = 100;
function candidates(): RerankCandidate[] {
const rows = getDb().prepare(
- "SELECT id, platform, base_url, encrypted_key, iv, auth_tag FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') AND platform IN ('cohere', 'custom') ORDER BY platform, created_at",
- ).all() as { id: number; platform: string; base_url: string | null; encrypted_key: string; iv: string; auth_tag: string }[];
+ "SELECT id, platform, base_url, encrypted_key, iv, auth_tag, model_scope_json FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown') AND platform IN ('cohere', 'custom') ORDER BY platform, created_at",
+ ).all() as { id: number; platform: string; base_url: string | null; encrypted_key: string; iv: string; auth_tag: string; model_scope_json: string | null }[];
const out: RerankCandidate[] = [];
for (const row of rows) {
if (row.platform === 'custom' && !row.base_url) continue;
try {
- out.push({ id: row.id, platform: row.platform, key: decrypt(row.encrypted_key, row.iv, row.auth_tag), baseUrl: row.base_url });
+ out.push({ id: row.id, platform: row.platform, key: decrypt(row.encrypted_key, row.iv, row.auth_tag), baseUrl: row.base_url, scope: parseModelScope(row.model_scope_json) });
} catch { /* undecryptable row is not a candidate */ }
}
// Custom endpoints (the operator's explicit rerank relays) go first.
@@ -117,7 +126,12 @@ export async function runRerank(model: string | undefined, query: string, docume
if (documents.length > MAX_DOCUMENTS) {
throw new RerankError(`Too many documents: ${documents.length} (max ${MAX_DOCUMENTS})`, 400);
}
- const chain = candidates();
+ // Per-key model scope (#657), the same wall the chat router and search key
+ // pools honour: a relay key scoped to a model group must never be picked to
+ // serve a model outside its list. Filter up front so a pool where every key
+ // is scoped out reports "no capable key configured" (503) instead of
+ // pretending the providers were tried and failed (502).
+ const chain = candidates().filter(c => scopeAllows(c.scope, effectiveModel(c, model)));
if (chain.length === 0) {
throw new RerankError('No rerank-capable key configured. Add a Cohere key or a custom rerank endpoint.', 503);
}