7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(routing): honor auto:<profile> chains on /v1/messages and /v1/responses (#1224)
rome-xicd5b83b6fbc2
diff --git a/server/src/__tests__/routes/models-management.test.ts b/server/src/__tests__/routes/models-management.test.tsindex d8a11f19..74290ddb 100644--- a/server/src/__tests__/routes/models-management.test.ts+++ b/server/src/__tests__/routes/models-management.test.ts@@ -514,4 +514,83 @@ describe('Model management API', () => {expect(missing.status).toBe(400);expect(JSON.stringify(missing.body)).toContain('no-such-group');});++ function mockGroqChat(modelId: string, content: string) {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const u = typeof url === 'string' ? url : url.toString();+ if (u.includes('api.groq.com')) {+ return {+ ok: true,+ status: 200,+ headers: new Headers(),+ json: () => Promise.resolve({+ id: 'chatcmpl-chain', object: 'chat.completion', created: 1, model: modelId,+ choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },+ }),+ } as any;+ }+ return origFetch(url, init);+ });+ }++ function lastServedModel(): string | undefined {+ return (getDb().prepare('SELECT model_id FROM requests ORDER BY id DESC LIMIT 1')+ .get() as { model_id: string } | undefined)?.model_id;+ }++ it('routes /v1/messages to model auto:<name> through that named chain (#1170)', async () => {+ seedGroqKey();+ const chainOnlyId = seedChainOnlyModel('named-chain-messages', 128000);+ addToProfile(newProfile('messages-group', 8), chainOnlyId);+ mockGroqChat('named-chain-messages', 'routed via messages-group');++ const routed = await request(app, 'POST', '/v1/messages', {+ model: 'auto:messages-group',+ max_tokens: 64,+ messages: [{ role: 'user', content: 'hi' }],+ });+ expect(routed.status).toBe(200);+ expect(lastServedModel()).toBe('named-chain-messages');+ });++ it('rejects unknown auto:<name> on /v1/messages with 400 (#1170)', async () => {+ seedGroqKey();+ mockGroqChat('should-not-serve', 'silent pool fallback');++ const missing = await request(app, 'POST', '/v1/messages', {+ model: 'auto:no-such-group',+ max_tokens: 64,+ messages: [{ role: 'user', content: 'hi' }],+ });+ expect(missing.status).toBe(400);+ expect(JSON.stringify(missing.body)).toContain('no-such-group');+ });++ it('routes /v1/responses to model auto:<name> through that named chain (#1170)', async () => {+ seedGroqKey();+ const chainOnlyId = seedChainOnlyModel('named-chain-responses', 128000);+ addToProfile(newProfile('responses-group', 9), chainOnlyId);+ mockGroqChat('named-chain-responses', 'routed via responses-group');++ const routed = await request(app, 'POST', '/v1/responses', {+ model: 'auto:responses-group',+ input: 'hi',+ });+ expect(routed.status).toBe(200);+ expect(lastServedModel()).toBe('named-chain-responses');+ });++ it('rejects unknown auto:<name> on /v1/responses with 400 (#1170)', async () => {+ seedGroqKey();+ mockGroqChat('should-not-serve', 'silent pool fallback');++ const missing = await request(app, 'POST', '/v1/responses', {+ model: 'auto:no-such-group',+ input: 'hi',+ });+ expect(missing.status).toBe(400);+ expect(JSON.stringify(missing.body)).toContain('no-such-group');+ });});diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.tsindex 4e82323e..109cea01 100644--- a/server/src/__tests__/routes/responses.test.ts+++ b/server/src/__tests__/routes/responses.test.ts@@ -567,7 +567,10 @@ describe('POST /v1/responses model routing priority (#579)', () => {expect(status).toBe(200);const { preferredModelDbId, chain } = routingCall();expect(preferredModelDbId).toBe(stickyId);- expect(chain).toBeUndefined();+ // Plain auto now threads the resolved active chain (parity with+ // /v1/chat/completions), not a singleton pin.+ expect(chain?.map(row => row.model_db_id)).toEqual(expect.arrayContaining([pinnedId, stickyId]));+ expect(chain?.map(row => row.model_db_id)).not.toEqual([pinnedId]);});it('auto routing applies when there is neither an explicit model nor a sticky session', async () => {@@ -578,7 +581,8 @@ describe('POST /v1/responses model routing priority (#579)', () => {expect(status).toBe(200);const { preferredModelDbId, chain } = routingCall();expect(preferredModelDbId).toBeUndefined();- expect(chain).toBeUndefined();+ expect(chain?.map(row => row.model_db_id)).toEqual(expect.arrayContaining([pinnedId, stickyId]));+ expect(chain?.map(row => row.model_db_id)).not.toEqual([pinnedId]);});it.each([diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 581e7551..a714e53c 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -9,7 +9,7 @@ import type {ChatToolChoice,ChatContentBlock,} from '@freellmapi/shared/types.js';-import { routeRequest, resolveModelGroupCandidates, resolveStickyPreference, routingReserveTokens, type RouteResult, type ChainRow } from '../services/router.js';+import { routeRequest, resolveModelGroupCandidates, resolveRoutingChain, resolveStickyPreference, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';import { getSetting, getUnifiedApiKey } from '../db/index.js';import { contentToString } from '../lib/content.js';import { resolveTaskType } from '../lib/task-type.js';@@ -45,6 +45,14 @@ import { normalizeMessageImages } from '../lib/image-normalize.js';// (already handled by extractApiToken) as well as a bearer token.export const anthropicRouter = Router();+const AUTO_MODEL_ID = 'auto';++function isAutoModel(modelId: string | undefined): boolean {+ if (!modelId) return true;+ const lower = modelId.toLowerCase();+ return lower === AUTO_MODEL_ID || lower.startsWith(`${AUTO_MODEL_ID}:`);+}+const MAX_RETRIES = 20;// Anthropic requires `max_tokens`; mirror the OpenAI route's routing-budget// default when a client somehow omits it.@@ -531,6 +539,29 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {const resolved = resolveAnthropicModel(routedModel);const pinnedModelId = resolved.pinned ? (body.model ?? null) : null;+ // Named `auto:<profile>` chains (and plain `auto`) must go through the same+ // resolver as /v1/chat/completions. resolveAnthropicModel treats unknown ids+ // as unpinned auto-route, which used to silently walk the active pool.+ let resolvedChain: ResolvedChain | undefined;+ let strategyKey: string | undefined;+ const autoModelString = isAutoModel(requestedModel)+ ? requestedModel+ : (isAutoModel(routedModel) ? routedModel : undefined);+ if (!resolved.pinned && autoModelString !== undefined) {+ try {+ resolvedChain = resolveRoutingChain(autoModelString);+ // Named auto:<profile> / auto:smart get their own sticky bucket. Plain+ // `auto` stays unscoped so Claude Code session affinity is unchanged.+ strategyKey = resolvedChain.strategyKey === 'auto' ? undefined : resolvedChain.strategyKey;+ } catch (err: any) {+ if (err?.status === 400) {+ sendError(res, 400, 'invalid_request_error', err.message);+ return;+ }+ throw err;+ }+ }+// Session affinity: Claude Code stamps every request in a session with// X-Claude-Code-Session-Id. When auto-routing, stick the whole session to one// model so it doesn't flap between free providers mid-conversation.@@ -577,7 +608,9 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {const sticky = getStickyModel(messages, sessionId, stickyScope);preferredModel = (sticky != null && groupChain.some(r => r.model_db_id === sticky)) ? sticky : undefined;}- if (preferredModel == null && !groupChain) preferredModel = resolveStickyPreference(getStickyModel(messages, sessionId));+ if (preferredModel == null && !groupChain) {+ preferredModel = resolveStickyPreference(getStickyModel(messages, sessionId, strategyKey), resolvedChain?.chain);+ }// Thin adapter over the shared fallback loop (lib/fallback-loop.ts): the// cooldown/skip/penalty/exhaustion machinery is shared, only the Anthropic@@ -622,14 +655,14 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {// size latched onto state so the existing size gates in router.ts skip// low-TPM / small-context models on retry.const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);- return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);+ return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);},dispatch: async (route, attempt, dispatchCtx) => {if (stream) {try {await streamCompletion(res, route, messages, dispatchOptions, {start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,- sessionId, pinned: resolved.pinned, stickyScope, disarmHedge: dispatchCtx.disarmHedge,+ sessionId, pinned: resolved.pinned, stickyScope, strategyKey, disarmHedge: dispatchCtx.disarmHedge,});return 'done';} catch (err: any) {@@ -710,7 +743,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {// Remember this model for the rest of the auto-routed session. A pin used// to make this a no-op (the pin fixed the model); a group pin still has a// provider choice to remember, recorded under the group's own scope.- if (!resolved.pinned || stickyScope) setStickyModel(messages, route.modelDbId, sessionId, stickyScope);+ if (!resolved.pinned || stickyScope) setStickyModel(messages, route.modelDbId, sessionId, stickyScope ?? strategyKey);const anthropicResponse: AnthropicMessageResponse = {id: newMessageId(),@@ -765,6 +798,8 @@ interface StreamCtx {pinned: boolean;// Sticky bucket for a group-pinned request; undefined for auto routing.stickyScope?: string;+ // Named auto:<profile> sticky bucket; undefined for plain auto.+ strategyKey?: string;/** Cancel this attempt's time-budget hedge once the stream commits. */disarmHedge: () => void;}@@ -1021,7 +1056,7 @@ async function streamCompletion(res.end();recordUpstreamSuccess(route, ctx.estimatedInputTokens + outputTokens);- if (!ctx.pinned || ctx.stickyScope) setStickyModel(messages, route.modelDbId, ctx.sessionId, ctx.stickyScope);+ if (!ctx.pinned || ctx.stickyScope) setStickyModel(messages, route.modelDbId, ctx.sessionId, ctx.stickyScope ?? ctx.strategyKey);logRequest(route.platform, route.modelId, route.keyId, 'success', ctx.estimatedInputTokens, outputTokens, Date.now() - ctx.start, null, null, ctx.pinnedModelId, null, 'http');} catch (err: any) {if (err instanceof StreamAlreadyStarted) throw err;diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 4da0d5b8..7dfd961f 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -9,7 +9,7 @@ import type {ChatToolChoice,Platform,} from '@freellmapi/shared/types.js';-import { routeRequest, hasEnabledVisionModel, hasEnabledToolsModel, resolveStickyPreference, routingReserveTokens, resolveModelGroupCandidates, type RouteResult, type ChainRow } from '../services/router.js';+import { routeRequest, hasEnabledVisionModel, hasEnabledToolsModel, resolveStickyPreference, routingReserveTokens, resolveModelGroupCandidates, resolveRoutingChain, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';import { getDb } from '../db/index.js';import { resolveAuth, prependSystemPrompt } from '../lib/system-prompt.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from '../services/model-groups.js';@@ -781,9 +781,29 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// Priority: explicit model > sticky session > auto routing.let preferredModel: number | undefined;let groupChain: ChainRow[] | undefined;+ let resolvedChain: ResolvedChain | undefined;++ // Sticky scope for a named auto:<profile> (or auto:smart) chain. Plain `auto`+ // stays on the unscoped key so existing session affinity is unchanged.+ let stickyStrategyKey: string | undefined;if (isAutoModel(requestedModelLabel)) {- preferredModel = resolveStickyPreference(getStickyModel(messages, sessionIdHeader));+ try {+ resolvedChain = resolveRoutingChain(requestedModelLabel);+ } catch (err: any) {+ if (err?.status === 400) {+ res.status(400).json({+ error: { message: err.message, type: 'invalid_request_error' },+ });+ return;+ }+ throw err;+ }+ stickyStrategyKey = resolvedChain.strategyKey === 'auto' ? undefined : resolvedChain.strategyKey;+ preferredModel = resolveStickyPreference(+ getStickyModel(messages, sessionIdHeader, stickyStrategyKey),+ resolvedChain.chain,+ );} else if (isFusionModel(requestedModelLabel)) {// Fusion is a virtual model, not a row in the catalog. Its panel and// judge each resolve through the normal router inside runFusion below.@@ -1116,7 +1136,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// size latched onto state so the existing size gates in router.ts skip// low-TPM / small-context models on retry.const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);- return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);+ return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);},dispatch: async (route, attempt, ctx) => {traceRouteEvent('Responses', {@@ -1404,7 +1424,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {res.end();recordUpstreamSuccess(route, estimatedInputTokens + totalOutputTokens);- setStickyModel(messages, route.modelDbId, sessionIdHeader);+ setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);traceRouteEvent('Responses', {event: 'ok',requestId: requestGroupId,@@ -1529,7 +1549,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// tokens against the rate-limit ledger; promptTokens/completionTokens// above already carry the chars/4 estimate.recordUpstreamSuccess(route, result.usage?.total_tokens ?? (promptTokens + completionTokens));- setStickyModel(messages, route.modelDbId, sessionIdHeader);+ setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));setFallbackHeaders(res, attempt, attemptLog);