Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix: chain-switch stall and cache poisoning (#1047); classify non-chat discovered models (#1051) (#1090)

Tashfeen560d687fa5e4
diff --git a/client/src/App.tsx b/client/src/App.tsx
index 7a3bc715..8e0b1131 100644
--- a/client/src/App.tsx
+++ b/client/src/App.tsx
@@ -47,6 +47,12 @@ import AgentsPage from '@/pages/AgentsPage'
// silently. A page that already shows the failure inline can opt out with
// `meta: { silenceToast: true }` on the mutation.
const queryClient = new QueryClient({
+ // A short staleTime dedupes the mount storm (#1047): the Models page alone
+ // mounts ~13 queries, several of them the same endpoint from different
+ // components, and staleTime 0 refetched every one of them on every
+ // navigation and window focus. Pollers use refetchInterval and mutations
+ // invalidate explicitly, so nothing user-visible goes stale.
+ defaultOptions: { queries: { staleTime: 5_000 } },
mutationCache: new MutationCache({
onError: (error, _variables, _context, mutation) => {
if (mutation.meta?.silenceToast) return
diff --git a/client/src/components/chain-manager.tsx b/client/src/components/chain-manager.tsx
index a0d42dab..00d68d18 100644
--- a/client/src/components/chain-manager.tsx
+++ b/client/src/components/chain-manager.tsx
@@ -62,12 +62,16 @@ export function ChainManager() {
queryFn: () => apiFetch('/api/profiles/active'),
})
- const invalidate = () => {
+ const invalidate = async () => {
+ // Resolve WHICH chain is active before refreshing anything keyed on it
+ // (#1047): invalidating ['fallback'] first refetched the chain table under
+ // the old profile id — a wasted full-catalog round trip, and the write that
+ // poisoned the old chain's cache entry with the new chain's rows.
+ await queryClient.refetchQueries({ queryKey: ['profiles', 'active'], exact: true })
queryClient.invalidateQueries({ queryKey: ['profiles'] })
- queryClient.invalidateQueries({ queryKey: ['profiles', 'active'] })
- // The fallback table renders the active chain; refresh it too.
+ // The fallback table renders the active chain; refresh it too. The prefix
+ // covers the chain, routing, token-usage and rate-limit queries.
queryClient.invalidateQueries({ queryKey: ['fallback'] })
- queryClient.invalidateQueries({ queryKey: ['fallback', 'routing'] })
}
const createChain = useMutation({
diff --git a/client/src/components/keys/discover-models-dialog.tsx b/client/src/components/keys/discover-models-dialog.tsx
index 1501b725..4e0e2c03 100644
--- a/client/src/components/keys/discover-models-dialog.tsx
+++ b/client/src/components/keys/discover-models-dialog.tsx
@@ -25,6 +25,9 @@ export interface DiscoveredModel {
isFree?: boolean
/** True when the upstream advertises image input. */
vision?: boolean
+ /** Present only when the model is discernibly NOT a chat model (#1051). The
+ * server routes it to the matching table on register; video is skipped. */
+ kind?: 'embedding' | 'image' | 'audio' | 'transcription' | 'video'
}
interface DiscoverResponse {
@@ -170,6 +173,13 @@ export function DiscoverModelsDialog({
{t('models.vision')}
</span>
)}
+ {/* #1051: a non-chat model registers into its own table (or,
+ for video, is skipped), so say what the row IS up front. */}
+ {model.kind && (
+ <span className="shrink-0 rounded-full bg-violet-600/15 px-1.5 py-0.5 text-[10px] text-violet-700 dark:bg-violet-400/15 dark:text-violet-400">
+ {model.kind}
+ </span>
+ )}
{model.contextWindow !== undefined && model.contextWindow > 0 && (
<span title={t('models.ctxTitle')} className="shrink-0 rounded-full bg-muted px-1.5 py-0.5 text-[10px] text-muted-foreground tabular-nums">
{t('models.ctxBadge', { size: formatContext(model.contextWindow) })}
diff --git a/client/src/pages/FallbackPage.tsx b/client/src/pages/FallbackPage.tsx
index 62cbc3f0..67e3e53e 100644
--- a/client/src/pages/FallbackPage.tsx
+++ b/client/src/pages/FallbackPage.tsx
@@ -118,11 +118,22 @@ export default function FallbackPage() {
const { data: entries = [], isLoading: entriesLoading } = useQuery<FallbackEntry[]>({
queryKey: ['fallback', 'chain', activeProfileId],
- queryFn: () => apiFetch('/api/fallback'),
+ // The chain id rides in the request itself (#1047): keyed-but-unpinned, a
+ // refetch racing an activation fetched "whichever chain is active by now"
+ // into the OLD chain's cache entry, and switching A→B→A then rendered (and
+ // could save) B's rows under A's name until a hard refresh.
+ queryFn: () => apiFetch(activeProfileId != null ? `/api/fallback?profile=${activeProfileId}` : '/api/fallback'),
enabled: !activePending,
})
const isLoading = activePending || entriesLoading
+ // Staged edits are DISCARDED when the active chain changes, not just hidden
+ // (#1047): merely masking them meant switching A→B→A resurrected A's stale
+ // unsaved rows over freshly fetched data, with only a refresh clearing them.
+ useEffect(() => {
+ setStaged(prev => (prev && prev.profileId !== activeProfileId ? null : prev))
+ }, [activeProfileId])
+
const localEntries = staged && staged.profileId === activeProfileId ? staged.entries : null
const setLocalEntries = (entries: FallbackEntry[] | null) =>
setStaged(entries === null ? null : { profileId: activeProfileId, entries })
@@ -174,15 +185,27 @@ export default function FallbackPage() {
const keySelection: KeySelectionStrategy = routing?.keySelectionStrategy ?? 'auto'
const isManual = strategy === 'priority'
- // Merge fallback metadata with live scores, keyed by model.
- const scoreById = new Map((routing?.scores ?? []).map(s => [s.modelDbId, s]))
- const allEntries = localEntries ?? entries
- const configured = allEntries.filter(e => e.keyCount > 0)
- const unconfiguredPlatforms = [...new Set(allEntries.filter(e => e.keyCount === 0).map(e => e.platform))]
+ // Merge fallback metadata with live scores, keyed by model. Memoized (#1047):
+ // recomputing these over the whole catalog on every render — and the page
+ // renders once per landing query plus once per 15s poll — was a large part of
+ // the "absurdly slow" feel on big catalogs.
+ const scoreById = useMemo(
+ () => new Map((routing?.scores ?? []).map(s => [s.modelDbId, s])),
+ [routing?.scores],
+ )
+ const allEntries = useMemo(() => localEntries ?? entries, [localEntries, entries])
+ const configured = useMemo(() => allEntries.filter(e => e.keyCount > 0), [allEntries])
+ const unconfiguredPlatforms = useMemo(
+ () => [...new Set(allEntries.filter(e => e.keyCount === 0).map(e => e.platform))],
+ [allEntries],
+ )
// Entry fields win on overlap: the routing snapshot also carries `enabled`
// (and identity fields), which would otherwise clobber unsaved local toggles.
- const rows: Row[] = configured.map(e => ({ ...(scoreById.get(e.modelDbId) ?? {}), ...e }))
+ const rows: Row[] = useMemo(
+ () => configured.map(e => ({ ...(scoreById.get(e.modelDbId) ?? {}), ...e })),
+ [configured, scoreById],
+ )
const sensors = useSensors(
useSensor(PointerSensor),
@@ -205,16 +228,16 @@ export default function FallbackPage() {
// ── Model unification: a model served by several providers is always shown as
// one logical row that links to its own page (the on/off toggle was removed). ─
- const orderedGroups = buildGroups(rows, isManual)
+ const orderedGroups = useMemo(() => buildGroups(rows, isManual), [rows, isManual])
// Catalog search + filters (#343). Filtering operates on whole logical-model
// groups; rank stays the model's position in the full chain so the numbers
// don't renumber as you filter. Drag-to-reorder is only offered over the full,
// unfiltered manual chain (reordering a filtered subset would be ambiguous).
- const rankByKey = new Map(orderedGroups.map((g, i) => [g.key, i + 1]))
+ const rankByKey = useMemo(() => new Map(orderedGroups.map((g, i) => [g.key, i + 1])), [orderedGroups])
const query = search.trim().toLowerCase()
const filtersActive = query !== '' || filterVision || filterTools || minContext > 0
- const visibleGroups = orderedGroups.filter(g => {
+ const visibleGroups = useMemo(() => orderedGroups.filter(g => {
if (filterVision && !g.members.some(m => m.supportsVision)) return false
if (filterTools && !g.members.some(m => m.supportsTools)) return false
if (minContext > 0 && groupMaxContext(g.members) < minContext) return false
@@ -229,7 +252,7 @@ export default function FallbackPage() {
if (!hay.includes(query)) return false
}
return true
- })
+ }), [orderedGroups, filterVision, filterTools, minContext, query])
const draggable = isManual && !filtersActive
// Progressive rendering: grow the row budget whenever the sentinel below the
diff --git a/server/src/__tests__/routes/custom-modalities.test.ts b/server/src/__tests__/routes/custom-modalities.test.ts
index 3dd13fe1..744bd80e 100644
--- a/server/src/__tests__/routes/custom-modalities.test.ts
+++ b/server/src/__tests__/routes/custom-modalities.test.ts
@@ -363,4 +363,54 @@ describe('custom provider modalities', () => {
).get() as any;
expect(row.display_name).toBe('My Voice');
});
+
+ it('classifies mixed ids posted to /api/keys/custom into the right tables (#1051)', async () => {
+ const fetchMock = vi.fn(async () => embeddingResponse(4));
+ globalThis.fetch = fetchMock as any;
+
+ const { status, body } = await post(app, '/api/keys/custom', {
+ baseUrl: 'http://127.0.0.1:8484/v1',
+ apiKey: 'mix-secret',
+ models: [
+ 'glm-chat-mini',
+ 'stable-diffusion-3-5-large',
+ 'whisper-large-v3',
+ 'CosyVoice2-0.5B',
+ 'Wan2.2-T2V-A14B',
+ 'nomic-embed-mini',
+ ],
+ });
+
+ expect(status).toBe(201);
+ // Only the chat model lands in the chat table.
+ const chatIds = (getDb().prepare("SELECT model_id FROM models WHERE platform = 'custom'").all() as any[])
+ .map(r => r.model_id).sort();
+ expect(chatIds).toEqual(['glm-chat-mini']);
+ // Media ids land in media_models with the inferred modality.
+ const mediaRows = (getDb().prepare("SELECT model_id, modality FROM media_models WHERE platform = 'custom'").all() as any[])
+ .map(r => `${r.modality}:${r.model_id}`).sort();
+ expect(mediaRows).toEqual([
+ 'audio:CosyVoice2-0.5B',
+ 'image:stable-diffusion-3-5-large',
+ 'transcription:whisper-large-v3',
+ ]);
+ // The embedding id is probed and lands in embedding_models.
+ const embedRow = getDb().prepare("SELECT dimensions FROM embedding_models WHERE platform = 'custom' AND model_id = 'nomic-embed-mini'").get() as any;
+ expect(embedRow?.dimensions).toBe(4);
+ // Video has no custom serving path: reported, never registered.
+ expect(body.videoSkipped).toEqual(['Wan2.2-T2V-A14B']);
+ expect(body.created).toBe(5);
+ });
+
+ it('an explicit capability flag keeps a media-looking id in the chat table (#1051)', async () => {
+ const { status } = await post(app, '/api/keys/custom', {
+ baseUrl: 'http://127.0.0.1:8585/v1',
+ apiKey: 'override-secret',
+ models: [{ model: 'my-sd-chat-relay', supportsTools: true }],
+ });
+ expect(status).toBe(201);
+ const row = getDb().prepare("SELECT model_id FROM models WHERE platform = 'custom' AND model_id = 'my-sd-chat-relay'").get();
+ expect(row).toBeTruthy();
+ expect(getDb().prepare("SELECT 1 FROM media_models WHERE model_id = 'my-sd-chat-relay'").get()).toBeUndefined();
+ });
});
diff --git a/server/src/__tests__/services/custom-model-sync.test.ts b/server/src/__tests__/services/custom-model-sync.test.ts
index e83e2d43..3b6d514e 100644
--- a/server/src/__tests__/services/custom-model-sync.test.ts
+++ b/server/src/__tests__/services/custom-model-sync.test.ts
@@ -246,4 +246,20 @@ describe('custom model sync', () => {
expect(result.skipped).toBe(1);
expect(customModelIds()).toEqual(['model-a']);
});
+
+ it('skips discovered media/embedding models instead of registering them as chat (#1051)', async () => {
+ addCustomEndpoint('http://localhost:9999');
+ (discoverEndpointModels as unknown as ReturnType<typeof vi.fn>).mockResolvedValue([
+ { id: 'glm-chat-mini', ownedBy: null },
+ { id: 'stable-diffusion-3-5-large', ownedBy: null, kind: 'image' },
+ { id: 'whisper-large-v3', ownedBy: null, kind: 'transcription' },
+ { id: 'Wan2.2-T2V-A14B', ownedBy: null, kind: 'video' },
+ { id: 'text-embedding-3-small', ownedBy: null, kind: 'embedding' },
+ ]);
+
+ const result = await runCustomModelSync(getDb());
+ expect(result.added).toBe(1);
+ expect(result.nonChatSkipped).toBe(4);
+ expect(customModelIds()).toEqual(['glm-chat-mini']);
+ });
});
diff --git a/server/src/__tests__/services/model-discovery.test.ts b/server/src/__tests__/services/model-discovery.test.ts
index f145741c..97da8e40 100644
--- a/server/src/__tests__/services/model-discovery.test.ts
+++ b/server/src/__tests__/services/model-discovery.test.ts
@@ -247,3 +247,77 @@ describe('vision inferred from the model id (#1051)', () => {
expect(m.vision).toBe(true);
});
});
+
+// #1051, the non-vision half: SiliconFlow-style upstreams answer /v1/models
+// with bare ids, so a diffusion, whisper or video model used to register as a
+// chat model and 404 forever. The id (or explicit type metadata) now yields a
+// `kind`; absent kind still means "chat as far as anyone can tell".
+describe('non-chat kind inferred from the model id (#1051)', () => {
+ const kindOf = (id: string) => parseModelCatalog({ data: [{ id }] })[0]!.kind;
+
+ it('classifies the SiliconFlow ids from the issue', () => {
+ expect(kindOf('stabilityai/stable-diffusion-3-5-large')).toBe('image');
+ expect(kindOf('black-forest-labs/FLUX.1-dev')).toBe('image');
+ expect(kindOf('Kwai-Kolors/Kolors')).toBe('image');
+ expect(kindOf('FunAudioLLM/SenseVoiceSmall')).toBe('transcription');
+ expect(kindOf('FunAudioLLM/CosyVoice2-0.5B')).toBe('audio');
+ expect(kindOf('Wan-AI/Wan2.2-T2V-A14B')).toBe('video');
+ expect(kindOf('Wan-AI/Wan2.2-I2V-A14B')).toBe('video');
+ });
+
+ it('classifies the common whisper/tts/video families', () => {
+ expect(kindOf('whisper-large-v3')).toBe('transcription');
+ expect(kindOf('openai/whisper-large-v3-turbo')).toBe('transcription');
+ expect(kindOf('gpt-tts-1')).toBe('audio');
+ expect(kindOf('Lightricks/LTX-Video-0.9.5')).toBe('video');
+ expect(kindOf('tencent/HunyuanVideo')).toBe('video');
+ expect(kindOf('sdxl-turbo')).toBe('image');
+ expect(kindOf('sd3-medium')).toBe('image');
+ expect(kindOf('dall-e-3')).toBe('image');
+ });
+
+ it('classifies embedding ids, including bare `embed` tokens', () => {
+ expect(kindOf('text-embedding-3-small')).toBe('embedding');
+ expect(kindOf('nomic-embed-text')).toBe('embedding');
+ expect(kindOf('BAAI/bge-m3')).toBe('embedding');
+ });
+
+ it('leaves chat models without a kind key at all', () => {
+ expect(parseModelCatalog({ data: [{ id: 'qwen3-4b' }] })).toEqual([
+ { id: 'qwen3-4b', ownedBy: null },
+ ]);
+ expect(kindOf('deepseek-chat')).toBeUndefined();
+ expect(kindOf('claude-sonnet-5')).toBeUndefined();
+ });
+
+ it('a VL model stays a chat model that sees, never a media model', () => {
+ const [m] = parseModelCatalog({ data: [{ id: 'Qwen/Qwen2.5-VL-72B-Instruct' }] });
+ expect(m.vision).toBe(true);
+ expect(m.kind).toBeUndefined();
+ });
+
+ it('does not read markers out of longer tokens', () => {
+ expect(kindOf('sdk-helper-7b')).toBeUndefined(); // sdk is not sd
+ expect(kindOf('tootsie-8b')).toBeUndefined(); // no tts token
+ expect(kindOf('wandering-llama-3b')).toBeUndefined(); // wandering is not wan
+ expect(kindOf('videosaurus')).toBe('video'); // but a video token anywhere counts
+ });
+
+ it('classifies bare string entries the same way', () => {
+ expect(parseModelCatalog(['whisper-1'])).toEqual([
+ { id: 'whisper-1', ownedBy: null, kind: 'transcription' },
+ ]);
+ });
+
+ it('prefers explicit upstream type metadata over the id', () => {
+ const [m] = parseModelCatalog({ data: [{ id: 'mystery-model-9', type: 'text-to-image' }] });
+ expect(m.kind).toBe('image');
+ const [n] = parseModelCatalog({ data: [{ id: 'plain-9', object: 'embedding' }] });
+ expect(n.kind).toBe('embedding');
+ });
+
+ it("ignores OpenAI's uninformative object:'model'", () => {
+ const [m] = parseModelCatalog({ data: [{ id: 'gpt-x', object: 'model' }] });
+ expect(m.kind).toBeUndefined();
+ });
+});
diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.ts
index 5c4875ef..45fba9aa 100644
--- a/server/src/routes/fallback.ts
+++ b/server/src/routes/fallback.ts
@@ -189,9 +189,24 @@ function orderProfileRows(rows: any[]): any[] {
}
// Get fallback chain (with dynamic penalties)
-fallbackRouter.get('/', (_req: Request, res: Response) => {
+fallbackRouter.get('/', (req: Request, res: Response) => {
const db = getDb();
- const activeProfileId = getActiveProfileId(db);
+ // #1047: an explicit ?profile= pins the read to that chain, so a client
+ // caching per-chain can never have "the active chain, whichever that is right
+ // now" written into a specific chain's cache entry. During an activation the
+ // active id changes between the client's two fetches; that race is how chain
+ // B's rows ended up rendered (and nearly saved) under chain A's name.
+ let activeProfileId = getActiveProfileId(db);
+ const requestedRaw = req.query.profile;
+ if (requestedRaw !== undefined) {
+ const requested = Number.parseInt(String(requestedRaw), 10);
+ if (!Number.isInteger(requested) || requested <= 0
+ || !db.prepare('SELECT 1 FROM profiles WHERE id = ?').get(requested)) {
+ res.status(404).json({ error: { message: `no such profile: ${String(requestedRaw)}` } });
+ return;
+ }
+ activeProfileId = requested;
+ }
// `fallback_config` is the chain only for an install that has no profiles at
// all. Once one is active it is authoritative even when it is empty — falling
// through to the global table there is what made two different empty chains
diff --git a/server/src/routes/keys.ts b/server/src/routes/keys.ts
index 170fac76..b104ca1c 100644
--- a/server/src/routes/keys.ts
+++ b/server/src/routes/keys.ts
@@ -14,7 +14,8 @@ import { ensureModelInProfiles } from '../services/profile-models.js';
import { resolveCustomEndpointKey, customEndpointKeyIds, siblingEndpointKeyId, endpointHasCredential } from '../services/custom-endpoint.js';
import { customModelSeed } from '../services/custom-model-seed.js';
import { registerCustomModels, registerCustomChatModels } from '../services/custom-model-register.js';
-import { discoverEndpointModels, probeEndpointModel, ModelDiscoveryError } from '../services/model-discovery.js';
+import { registerCustomMediaModel } from '../services/custom-media-register.js';
+import { discoverEndpointModels, probeEndpointModel, classifyModelId, ModelDiscoveryError } from '../services/model-discovery.js';
import { probeEmbeddingDimensions, registerCustomEmbeddingModel } from '../services/embeddings.js';
import { endpointScopeForBaseUrl, normalizeBaseUrl } from '../lib/endpoint-scope.js';
import { recordCustomModelTombstone } from '../services/custom-model-tombstone.js';
@@ -729,8 +730,18 @@ async function registerImportedModels(
models: Array<{ id: string; supportsTools?: boolean; supportsVision?: boolean }>,
errors: Array<{ key: string; error: string }>,
): Promise<number> {
- const chat = models.filter(m => !/embedding/i.test(m.id));
- const embeds = models.filter(m => /embedding/i.test(m.id));
+ // #1051: classify by id, not just /embedding/. A whisper, diffusion or video
+ // id registered as a chat model 404s in every chain; they go to their own
+ // tables (or, for video, are skipped — nothing can serve a custom video
+ // model yet).
+ const kindOf = (id: string) => /embedding/i.test(id) ? 'embedding' : classifyModelId(id);
+ const chat = models.filter(m => kindOf(m.id) === undefined);
+ const embeds = models.filter(m => kindOf(m.id) === 'embedding');
+ const media = models.filter(m => {
+ const kind = kindOf(m.id);
+ return kind === 'image' || kind === 'audio' || kind === 'transcription';
+ });
+ const video = models.filter(m => kindOf(m.id) === 'video');
let registered = 0;
if (chat.length > 0) {
@@ -743,6 +754,22 @@ async function registerImportedModels(
registered += db.transaction(() => registerCustomChatModels(db, baseUrl, keyId, entries))().length;
}
+ for (const m of media) {
+ try {
+ registerCustomMediaModel(db, keyId, {
+ modelId: m.id,
+ displayName: null,
+ modality: kindOf(m.id) as 'image' | 'audio' | 'transcription',
+ });
+ registered++;
+ } catch (err: any) {
+ errors.push({ key: `${keyName}: ${m.id}`, error: err?.message ?? 'media registration failed' });
+ }
+ }
+ for (const m of video) {
+ errors.push({ key: `${keyName}: ${m.id}`, error: 'video generation models are not supported on custom endpoints yet; skipped' });
+ }
+
for (const m of embeds) {
try {
const existing = db.prepare(
@@ -1045,27 +1072,91 @@ keysRouter.post('/custom', async (req: Request, res: Response) => {
return;
}
+ // #1051: a whisper, diffusion or video id submitted here used to register as
+ // a chat model and then 404 in every chain. Classify by id and route each
+ // entry to the table that can actually serve it. An explicit capability flag
+ // on the entry is an operator override: someone who says "this is a chat
+ // model with vision/tools" wins over the keyword guess.
+ const overridden = (e: typeof entries[number]) => e.supportsTools !== undefined || e.supportsVision !== undefined;
+ const kindOf = (e: typeof entries[number]) => overridden(e) ? undefined : classifyModelId(e.modelId);
+ const chatEntries = entries.filter(e => kindOf(e) === undefined || kindOf(e) === 'embedding');
+ const embedEntries = chatEntries.filter(e => kindOf(e) === 'embedding');
+ const pureChatEntries = chatEntries.filter(e => kindOf(e) !== 'embedding');
+ const mediaEntries = entries.filter(e => {
+ const kind = kindOf(e);
+ return kind === 'image' || kind === 'audio' || kind === 'transcription';
+ });
+ const videoEntries = entries.filter(e => kindOf(e) === 'video');
+
const { keyId, storedKey, registered } = registerCustomModels(
- db, baseUrl, providedKey, label, endpoint.keyId ?? undefined, entries,
+ db, baseUrl, providedKey, label, endpoint.keyId ?? undefined, pureChatEntries,
);
+
+ const mediaRegistered: Array<{ modelDbId: number; model: string; modality: string; created: boolean }> = [];
+ const perModelErrors: Array<{ model: string; error: string }> = [];
+ for (const e of mediaEntries) {
+ try {
+ mediaRegistered.push(db.transaction(() => registerCustomMediaModel(db, keyId, {
+ modelId: e.modelId,
+ displayName: e.displayName,
+ modality: kindOf(e) as 'image' | 'audio' | 'transcription',
+ }))());
+ } catch (err: any) {
+ perModelErrors.push({ model: e.modelId, error: err?.message ?? 'media registration failed' });
+ }
+ }
+
+ const embeddingsRegistered: Array<{ model: string }> = [];
+ for (const e of embedEntries) {
+ try {
+ const existing = db.prepare(
+ "SELECT dimensions FROM embedding_models WHERE platform = 'custom' AND model_id = ?",
+ ).get(e.modelId) as { dimensions: number } | undefined;
+ const dimensions = existing?.dimensions ?? await probeEmbeddingDimensions(baseUrl, providedKey ?? storedKey, e.modelId);
+ registerCustomEmbeddingModel(db, {
+ keyId,
+ modelId: e.modelId,
+ displayName: e.displayName,
+ family: e.modelId,
+ dimensions,
+ maxInputTokens: null,
+ quotaLabel: 'custom endpoint',
+ });
+ embeddingsRegistered.push({ model: e.modelId });
+ } catch (err: any) {
+ perModelErrors.push({ model: e.modelId, error: err?.message ?? 'embedding registration failed' });
+ }
+ }
+
// `model`/`displayName`/`modelDbId` echo the first model for older clients;
// `models` carries the full set registered in this call.
- const first = registered[0]!;
+ const first = registered[0];
res.status(201).json({
success: true,
keyId,
- modelDbId: first.modelDbId,
+ ...(first ? {
+ modelDbId: first.modelDbId,
+ model: first.model,
+ displayName: first.displayName,
+ supportsTools: first.supportsTools,
+ supportsVision: first.supportsVision,
+ } : {}),
platform: 'custom',
baseUrl,
- model: first.model,
- displayName: first.displayName,
- supportsTools: first.supportsTools,
- supportsVision: first.supportsVision,
models: registered,
+ media: mediaRegistered,
+ embeddings: embeddingsRegistered,
+ // Video generation has no custom-endpoint serving path yet; saying so
+ // beats registering a chat model that can never answer.
+ videoSkipped: videoEntries.map(e => e.modelId),
+ modelErrors: perModelErrors,
// Bulk registration (#488) needs to tell the user what actually changed:
// picking a whole discovered list re-submits ids that are already there.
- created: registered.filter(m => m.created).length,
- alreadyRegistered: registered.filter(m => !m.created).length,
+ created: registered.filter(m => m.created).length
+ + mediaRegistered.filter(m => m.created).length
+ + embeddingsRegistered.length,
+ alreadyRegistered: registered.filter(m => !m.created).length
+ + mediaRegistered.filter(m => !m.created).length,
maskedKey: maskKey(storedKey),
});
});
diff --git a/server/src/routes/media.ts b/server/src/routes/media.ts
index 65b62359..76a3f35a 100644
--- a/server/src/routes/media.ts
+++ b/server/src/routes/media.ts
@@ -4,7 +4,8 @@ import { z } from 'zod';
import { getDb } from '../db/index.js';
import { maskKey } from '../lib/crypto.js';
import { deleteUnusedCustomEndpointKey } from '../lib/custom-provider-cleanup.js';
-import { resolveCustomEndpointKey, customEndpointKeyIds } from '../services/custom-endpoint.js';
+import { resolveCustomEndpointKey } from '../services/custom-endpoint.js';
+import { registerCustomMediaModel } from '../services/custom-media-register.js';
import { listAllMediaModels } from '../services/media.js';
export const mediaRouter = Router();
@@ -135,46 +136,16 @@ mediaRouter.post('/custom', (req: Request, res: Response) => {
const upsert = db.transaction(() => {
// A new secret for a known endpoint is an ADDITIONAL credential, never a
- // replacement for the stored one (#619).
+ // replacement for the stored one (#619). The upsert itself is shared with
+ // the classified branch of POST /api/keys/custom (#1051).
const { keyId, storedKey: storedKeyForMask } = resolveCustomEndpointKey(db, baseUrl, providedKey, label);
- const endpointKeyIds = customEndpointKeyIds(db, keyId);
-
- const existingModel = db.prepare(`
- SELECT id, modality, priority, key_id
- FROM media_models
- WHERE platform = 'custom' AND model_id = ?
- LIMIT 1
- `).get(modelId) as { id: number; modality: string; priority: number; key_id: number | null } | undefined;
- // A model already on this endpoint keeps the key it has; only a move to a
- // different endpoint re-binds it.
- const bindKeyId = existingModel?.key_id != null && endpointKeyIds.has(existingModel.key_id)
- ? existingModel.key_id
- : keyId;
- const priority = existingModel && existingModel.modality === parsed.data.modality
- ? existingModel.priority
- : (db.prepare('SELECT COALESCE(MAX(priority), 0) AS maxPriority FROM media_models WHERE modality = ?')
- .get(parsed.data.modality) as { maxPriority: number }).maxPriority + 1;
-
- if (existingModel) {
- db.prepare(`
- UPDATE media_models
- SET display_name = COALESCE(?, display_name),
- modality = ?,
- priority = ?,
- enabled = 1,
- quota_label = ?,
- key_id = ?
- WHERE id = ?
- `).run(submittedName, parsed.data.modality, priority, quotaLabel, bindKeyId, existingModel.id);
- return { modelDbId: existingModel.id, keyId, storedKeyForMask };
- }
-
- const model = db.prepare(`
- INSERT INTO media_models
- (platform, model_id, display_name, modality, priority, enabled, quota_label, key_id)
- VALUES ('custom', ?, ?, ?, ?, 1, ?, ?)
- `).run(modelId, submittedName ?? modelId, parsed.data.modality, priority, quotaLabel, bindKeyId);
- return { modelDbId: Number(model.lastInsertRowid), keyId, storedKeyForMask };
+ const { modelDbId } = registerCustomMediaModel(db, keyId, {
+ modelId,
+ displayName: submittedName,
+ modality: parsed.data.modality,
+ quotaLabel,
+ });
+ return { modelDbId, keyId, storedKeyForMask };
});
const result = upsert();
diff --git a/server/src/services/catalog-sync.ts b/server/src/services/catalog-sync.ts
index eb1e20e9..bb33354f 100644
--- a/server/src/services/catalog-sync.ts
+++ b/server/src/services/catalog-sync.ts
@@ -273,6 +273,15 @@ function routableContextWindow(platform: string, modelId: string, contextWindow:
* dead-model migrations do (fallback_config row first, FK order).
*/
export function applyCatalog(db: Db, catalog: Catalog): NonNullable<SyncResult['counts']> {
+ // One transaction for the whole apply (#1047): the body runs hundreds of
+ // individual statements, and under WAL each one outside a transaction is its
+ // own commit + fsync executed on the event loop. On slow storage that made
+ // the 12-hourly sync freeze every in-flight request — including the 3-query
+ // profile-activate POST — for minutes.
+ return db.transaction(() => applyCatalogInner(db, catalog))();
+}
+
+function applyCatalogInner(db: Db, catalog: Catalog): NonNullable<SyncResult['counts']> {
const counts = { updated: 0, inserted: 0, removed: 0, skippedUnknownPlatform: 0, quirks: 0 };
const selectModel = db.prepare('SELECT id, enabled, source FROM models WHERE platform = ? AND model_id = ?');
diff --git a/server/src/services/custom-media-register.ts b/server/src/services/custom-media-register.ts
new file mode 100644
index 00000000..72000265
--- /dev/null
+++ b/server/src/services/custom-media-register.ts
@@ -0,0 +1,76 @@
+import type { Db } from '../db/types.js';
+import { customEndpointKeyIds } from './custom-endpoint.js';
+
+// ── Shared custom MEDIA-model registration (#1051) ───────────────────────────
+//
+// POST /api/media/custom and the classified branch of POST /api/keys/custom
+// register image/audio/transcription models against a user's own endpoint
+// through the SAME upsert, exactly like custom-model-register.ts does for chat
+// models. This used to live inline in routes/media.ts.
+
+export type CustomMediaModality = 'image' | 'audio' | 'transcription';
+
+export interface CustomMediaEntry {
+ modelId: string;
+ displayName: string | null;
+ modality: CustomMediaModality;
+ quotaLabel?: string;
+}
+
+export interface RegisteredCustomMediaModel {
+ modelDbId: number;
+ model: string;
+ modality: CustomMediaModality;
+ created: boolean;
+}
+
+/**
+ * Upsert one media model row against an ALREADY-resolved endpoint key, inside
+ * the caller's transaction. media_models identity is (platform, model_id) with
+ * no endpoint_scope — a model already on this endpoint keeps the key it has;
+ * only a move to a different endpoint re-binds it (#619 semantics).
+ */
+export function registerCustomMediaModel(
+ db: Db,
+ keyId: number,
+ entry: CustomMediaEntry,
+): RegisteredCustomMediaModel {
+ const endpointKeyIds = customEndpointKeyIds(db, keyId);
+ const modelId = entry.modelId;
+ const quotaLabel = entry.quotaLabel?.trim() || 'custom endpoint';
+
+ const existingModel = db.prepare(`
+ SELECT id, modality, priority, key_id
+ FROM media_models
+ WHERE platform = 'custom' AND model_id = ?
+ LIMIT 1
+ `).get(modelId) as { id: number; modality: string; priority: number; key_id: number | null } | undefined;
+ const bindKeyId = existingModel?.key_id != null && endpointKeyIds.has(existingModel.key_id)
+ ? existingModel.key_id
+ : keyId;
+ const priority = existingModel && existingModel.modality === entry.modality
+ ? existingModel.priority
+ : (db.prepare('SELECT COALESCE(MAX(priority), 0) AS maxPriority FROM media_models WHERE modality = ?')
+ .get(entry.modality) as { maxPriority: number }).maxPriority + 1;
+
+ if (existingModel) {
+ db.prepare(`
+ UPDATE media_models
+ SET display_name = COALESCE(?, display_name),
+ modality = ?,
+ priority = ?,
+ enabled = 1,
+ quota_label = ?,
+ key_id = ?
+ WHERE id = ?
+ `).run(entry.displayName, entry.modality, priority, quotaLabel, bindKeyId, existingModel.id);
+ return { modelDbId: existingModel.id, model: modelId, modality: entry.modality, created: false };
+ }
+
+ const model = db.prepare(`
+ INSERT INTO media_models
+ (platform, model_id, display_name, modality, priority, enabled, quota_label, key_id)
+ VALUES ('custom', ?, ?, ?, ?, 1, ?, ?)
+ `).run(modelId, entry.displayName ?? modelId, entry.modality, priority, quotaLabel, bindKeyId);
+ return { modelDbId: Number(model.lastInsertRowid), model: modelId, modality: entry.modality, created: true };
+}
diff --git a/server/src/services/custom-model-sync.ts b/server/src/services/custom-model-sync.ts
index 23cfb2d2..a894454f 100644
--- a/server/src/services/custom-model-sync.ts
+++ b/server/src/services/custom-model-sync.ts
@@ -62,6 +62,10 @@ export interface CustomModelSyncResult {
paidSkipped: number;
/** Models skipped because the operator deleted them and they must stay deleted (#926). */
tombstoned: number;
+ /** Models skipped because they are discernibly not chat models — embedding,
+ * image, audio, transcription or video ids (#1051). Registering those as
+ * chat models is how they used to 404 forever. */
+ nonChatSkipped: number;
failures: Array<{ baseUrl: string; error: string }>;
}
@@ -69,7 +73,7 @@ export interface CustomModelSyncResult {
* that wants a manual pass) can run it directly without waiting for the timer. */
export async function runCustomModelSync(db: Db): Promise<CustomModelSyncResult> {
const endpoints = listCustomEndpoints(db);
- const result: CustomModelSyncResult = { endpoints: endpoints.length, added: 0, skipped: 0, paidSkipped: 0, tombstoned: 0, failures: [] };
+ const result: CustomModelSyncResult = { endpoints: endpoints.length, added: 0, skipped: 0, paidSkipped: 0, tombstoned: 0, nonChatSkipped: 0, failures: [] };
for (const endpoint of endpoints) {
try {
@@ -105,6 +109,14 @@ export async function runCustomModelSync(db: Db): Promise<CustomModelSyncResult>
result.paidSkipped += 1;
continue;
}
+ // #1051: a diffusion/whisper/video/embedding id is not a chat model,
+ // and registering it here is how it ends up 404ing in every chain.
+ // The sync only ADDS chat models; media and embedding models go
+ // through their own explicit registration paths.
+ if (model.kind !== undefined) {
+ result.nonChatSkipped += 1;
+ continue;
+ }
// Bare id only, like the dashboard's bulk "Fetch models" submit: no name
// or capability claims invented here, they resolve in SQL on insert.
fresh.push({ modelId: model.id, displayName: null });
diff --git a/server/src/services/model-discovery.ts b/server/src/services/model-discovery.ts
index 22cc74e0..27bcde87 100644
--- a/server/src/services/model-discovery.ts
+++ b/server/src/services/model-discovery.ts
@@ -27,6 +27,11 @@ export const MAX_DISCOVERED_MODELS = 500;
/** Ids longer than this are certainly not model ids; skip rather than store. */
const MAX_MODEL_ID_LENGTH = 256;
+/** What a discovered model IS, when it discernibly isn't a chat model (#1051).
+ * Absent means "chat as far as anyone can tell" — the field is emitted only
+ * when a non-chat kind is detected, so plain chat rows keep their shape. */
+export type DiscoveredModelKind = 'embedding' | 'image' | 'audio' | 'transcription' | 'video';
+
export interface DiscoveredModel {
id: string;
ownedBy: string | null;
@@ -42,6 +47,8 @@ export interface DiscoveredModel {
isFree?: boolean;
/** True when the upstream advertises image input (modalities/vision). */
vision?: boolean;
+ /** Present only when the model is discernibly NOT a chat model (#1051). */
+ kind?: DiscoveredModelKind;
}
/** Carries the HTTP status the route should answer with, so a relay's 401 stays
@@ -226,6 +233,82 @@ function visionFromId(id: string): true | undefined {
return undefined;
}
+// #1051, the non-vision half: an upstream that answers with bare ids gives us
+// no `object`/`type` to read (SiliconFlow ships none at all), so a diffusion,
+// whisper or video model registers as a chat model and then 404s forever. The
+// id is the only signal left. Same rules as VISION_ID_MARKERS: matched per
+// token (the id split on non-alphanumerics), never as a raw substring, so
+// `sdk`, `tootsie` or `pastta` can't smuggle a kind in. Chat is the default —
+// classifyModelId returns undefined unless a marker is unambiguous, because a
+// wrong "media" verdict makes a working chat model unreachable, which is worse
+// than the status quo.
+const KIND_ID_MARKERS: ReadonlyArray<readonly [DiscoveredModelKind, RegExp]> = [
+ // Embedding first: "embedding" ids sometimes also carry family tokens.
+ ['embedding', /^embed(ding)?s?$/],
+ ['embedding', /^bge$/],
+ // Speech-to-text families.
+ ['transcription', /^whisper$/],
+ ['transcription', /^(asr|stt)$/],
+ ['transcription', /^transcri(be|ption)$/],
+ ['transcription', /^sensevoice(small|large)?$/],
+ ['transcription', /^voxtral$/],
+ // Text-to-speech families.
+ ['audio', /^tts$/],
+ ['audio', /^cosyvoice\d*$/],
+ // Video generation: Wan2.2's own tokens (`wan2`, `t2v`, `i2v`) plus ids that
+ // simply say video (HunyuanVideo, LTX-Video, ...).
+ ['video', /^wan\d*$/],
+ ['video', /^[ti]2v$/],
+ ['video', /video/],
+ // Image generation: diffusion/SD/FLUX/DALL-E/Kolors families. `sd` only as a
+ // whole token with optional xl/digits (sd3, sdxl, sd35) so `sdk` stays out.
+ ['image', /^diffusion$/],
+ ['image', /^sd(xl)?\d*$/],
+ ['image', /^flux$/],
+ ['image', /^dall-?e?\d*$/],
+ ['image', /^dalle\d*$/],
+ ['image', /^kolors$/],
+ ['image', /^imagen\d*$/],
+ ['image', /^photomaker(v\d+)?$/],
+];
+
+// Upstream type metadata beats the id when it exists: a few relays ship a
+// `type`/`task`/`object` per entry ("embedding", "text-to-image", ...). OpenAI's
+// own `object: "model"` says nothing and maps to undefined here.
+const KIND_METADATA_KEYS = ['type', 'task', 'object', 'model_type', 'modelType'] as const;
+const KIND_METADATA_VALUES: ReadonlyArray<readonly [DiscoveredModelKind, RegExp]> = [
+ ['embedding', /embedding/],
+ ['transcription', /(speech-?to-?text|transcri|asr\b|\bstt\b)/],
+ ['audio', /(text-?to-?speech|\btts\b|speech-?synthesis)/],
+ ['video', /video/],
+ ['image', /(text-?to-?image|image-?generation|diffusion)/],
+];
+
+/** The model's kind read off its id, for upstreams that advertise no usable
+ * type metadata (#1051). Returns a kind or undefined and never 'chat': absence
+ * of a marker is not evidence of anything, and chat stays the default. */
+export function classifyModelId(id: string): DiscoveredModelKind | undefined {
+ for (const token of id.toLowerCase().split(/[^a-z0-9]+/)) {
+ for (const [kind, pattern] of KIND_ID_MARKERS) {
+ if (pattern.test(token)) return kind;
+ }
+ }
+ return undefined;
+}
+
+function kindFromMetadata(record: Record<string, unknown>): DiscoveredModelKind | undefined {
+ for (const key of KIND_METADATA_KEYS) {
+ const value = record[key];
+ if (typeof value !== 'string') continue;
+ const normalized = value.trim().toLowerCase();
+ if (!normalized || normalized === 'model') continue;
+ for (const [kind, pattern] of KIND_METADATA_VALUES) {
+ if (pattern.test(normalized)) return kind;
+ }
+ }
+ return undefined;
+}
+
function toDiscovered(entry: unknown): DiscoveredModel | null {
if (typeof entry === 'string') {
const id = entry.trim();
@@ -233,6 +316,10 @@ function toDiscovered(entry: unknown): DiscoveredModel | null {
const bare: DiscoveredModel = { id, ownedBy: null };
const bareVision = visionFromId(id);
if (bareVision !== undefined) bare.vision = bareVision;
+ if (bareVision !== true) {
+ const bareKind = classifyModelId(id);
+ if (bareKind !== undefined) bare.kind = bareKind;
+ }
return bare;
}
const record = asRecord(entry);
@@ -253,6 +340,12 @@ function toDiscovered(entry: unknown): DiscoveredModel | null {
// `??` not `||`: an upstream that explicitly says false keeps saying false.
const vision = visionOf(record) ?? visionFromId(id);
if (vision !== undefined) model.vision = vision;
+ // Metadata wins over the id; a vision verdict wins over both — a VL model is
+ // a chat model that sees, not a media model (#1051).
+ if (vision !== true) {
+ const kind = kindFromMetadata(record) ?? classifyModelId(id);
+ if (kind !== undefined) model.kind = kind;
+ }
return model;
}
diff --git a/server/src/services/model-groups.ts b/server/src/services/model-groups.ts
index 9417ae45..bebadaf8 100644
--- a/server/src/services/model-groups.ts
+++ b/server/src/services/model-groups.ts
@@ -360,6 +360,15 @@ export function resolveRequestedIdForDispatch(
}
// ── DB convenience ───────────────────────────────────────────────────────────
+
+// #1047: GET /api/fallback calls this on every request, and grouping the whole
+// catalog (regex + canonical-id assignment over hundreds of rows) dominated the
+// handler once the chain read went catalog-wide (#1023). The SELECT itself is a
+// cheap pk scan, so run it every time and memoize only the expensive grouping,
+// keyed on a fingerprint of the exact inputs — any row change, in place or not,
+// and any override change recomputes, so this can never serve stale groups.
+let groupsCache: { fingerprint: string; groups: ModelGroup[] } | null = null;
+
/**
* Group the whole catalog (enabled + disabled rows so availability can be shown
* and resolution is complete), applying the persisted overrides.
@@ -372,5 +381,13 @@ export function getModelGroups(): ModelGroup[] {
FROM models m
ORDER BY m.id
`).all() as GroupableRow[];
- return groupRows(rows, getUnifyOverrides());
+ const overrides = getUnifyOverrides();
+ let fingerprint = JSON.stringify(overrides);
+ for (const r of rows) {
+ fingerprint += `${r.model_db_id}${r.platform}${r.model_id}${r.display_name}${r.intelligence_rank}${r.endpoint_scope ?? ''}`;
+ }
+ if (groupsCache?.fingerprint === fingerprint) return groupsCache.groups;
+ const groups = groupRows(rows, overrides);
+ groupsCache = { fingerprint, groups };
+ return groups;
}
diff --git a/server/src/services/profile-models.ts b/server/src/services/profile-models.ts
index df4e7109..038d11fa 100644
--- a/server/src/services/profile-models.ts
+++ b/server/src/services/profile-models.ts
@@ -47,12 +47,18 @@ export function ensureAllModelsInProfiles(db: Db): void {
const maxPriority = db.prepare('SELECT COALESCE(MAX(priority), 0) AS max_priority FROM profile_models WHERE profile_id = ?');
const insert = db.prepare('INSERT INTO profile_models (profile_id, model_db_id, priority, enabled) VALUES (?, ?, ?, ?)');
- for (const profile of profiles) {
- const rows = missing.all(profile.id) as { id: number; enabled: number }[];
- if (rows.length === 0) continue;
- const max = maxPriority.get(profile.id) as { max_priority: number };
- rows.forEach((row, index) => {
- insert.run(profile.id, row.id, max.max_priority + index + 1, row.enabled);
- });
- }
+ // One transaction for the whole backfill (#1047): under WAL each bare
+ // insert.run() is its own commit + fsync, and profiles × missing models of
+ // those, run on the event loop, is how a catalog sync froze every request
+ // on the box for minutes on slow storage.
+ db.transaction(() => {
+ for (const profile of profiles) {
+ const rows = missing.all(profile.id) as { id: number; enabled: number }[];
+ if (rows.length === 0) continue;
+ const max = maxPriority.get(profile.id) as { max_priority: number };
+ rows.forEach((row, index) => {
+ insert.run(profile.id, row.id, max.max_priority + index + 1, row.enabled);
+ });
+ }
+ })();
}
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index b87a861e..65743ce5 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -2081,7 +2081,28 @@ export function getRoutingScores(): { strategy: RoutingStrategy; keySelectionStr
const strategy = getRoutingStrategy();
refreshStatsCache(db);
- const chain = getActiveChain(db);
+ // Score the whole enabled catalog, not just the active chain (#1047). Since
+ // #1023 the dashboard table lists every catalog row through the chain, and
+ // merging it against chain-only scores left every not-yet-opted-in row with
+ // blank reliability/speed — which read as "each new chain starts from
+ // scratch" even though the underlying per-model stats are global. Rows the
+ // chain names keep its enabled flag; the rest score as disabled. Display
+ // only: routeRequest still walks getActiveChain.
+ const profileId = getActiveProfileId(db);
+ const chain = db.prepare(`
+ SELECT m.id AS model_db_id, COALESCE(c.priority, 0) AS priority, COALESCE(c.enabled, 0) AS enabled,
+ m.platform, m.model_id, m.display_name, m.intelligence_rank,
+ m.size_label, m.monthly_token_budget,
+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,
+ m.supports_tools, m.context_window, m.key_id, m.endpoint_scope
+ FROM models m
+ LEFT JOIN ${profileId != null
+ ? '(SELECT model_db_id, priority, enabled FROM profile_models WHERE profile_id = ?) c'
+ : '(SELECT model_db_id, priority, enabled FROM fallback_config) c'}
+ ON c.model_db_id = m.id
+ WHERE m.enabled = 1
+ ORDER BY c.priority ASC
+ `).all(...(profileId != null ? [profileId] : [])) as ChainRow[];
// For display we score under 'balanced' weights when in priority mode, so the
// table still shows a meaningful ranking even with the bandit turned off.