Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(routing): honor auto:<profile> chains on /v1/messages and /v1/responses (#1224)

rome-xicd5b83b6fbc2
diff --git a/server/src/__tests__/routes/models-management.test.ts b/server/src/__tests__/routes/models-management.test.ts
index d8a11f19..74290ddb 100644
--- a/server/src/__tests__/routes/models-management.test.ts
+++ b/server/src/__tests__/routes/models-management.test.ts
@@ -514,4 +514,83 @@ describe('Model management API', () => {
expect(missing.status).toBe(400);
expect(JSON.stringify(missing.body)).toContain('no-such-group');
});
+
+ function mockGroqChat(modelId: string, content: string) {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const u = typeof url === 'string' ? url : url.toString();
+ if (u.includes('api.groq.com')) {
+ return {
+ ok: true,
+ status: 200,
+ headers: new Headers(),
+ json: () => Promise.resolve({
+ id: 'chatcmpl-chain', object: 'chat.completion', created: 1, model: modelId,
+ choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
+ }),
+ } as any;
+ }
+ return origFetch(url, init);
+ });
+ }
+
+ function lastServedModel(): string | undefined {
+ return (getDb().prepare('SELECT model_id FROM requests ORDER BY id DESC LIMIT 1')
+ .get() as { model_id: string } | undefined)?.model_id;
+ }
+
+ it('routes /v1/messages to model auto:<name> through that named chain (#1170)', async () => {
+ seedGroqKey();
+ const chainOnlyId = seedChainOnlyModel('named-chain-messages', 128000);
+ addToProfile(newProfile('messages-group', 8), chainOnlyId);
+ mockGroqChat('named-chain-messages', 'routed via messages-group');
+
+ const routed = await request(app, 'POST', '/v1/messages', {
+ model: 'auto:messages-group',
+ max_tokens: 64,
+ messages: [{ role: 'user', content: 'hi' }],
+ });
+ expect(routed.status).toBe(200);
+ expect(lastServedModel()).toBe('named-chain-messages');
+ });
+
+ it('rejects unknown auto:<name> on /v1/messages with 400 (#1170)', async () => {
+ seedGroqKey();
+ mockGroqChat('should-not-serve', 'silent pool fallback');
+
+ const missing = await request(app, 'POST', '/v1/messages', {
+ model: 'auto:no-such-group',
+ max_tokens: 64,
+ messages: [{ role: 'user', content: 'hi' }],
+ });
+ expect(missing.status).toBe(400);
+ expect(JSON.stringify(missing.body)).toContain('no-such-group');
+ });
+
+ it('routes /v1/responses to model auto:<name> through that named chain (#1170)', async () => {
+ seedGroqKey();
+ const chainOnlyId = seedChainOnlyModel('named-chain-responses', 128000);
+ addToProfile(newProfile('responses-group', 9), chainOnlyId);
+ mockGroqChat('named-chain-responses', 'routed via responses-group');
+
+ const routed = await request(app, 'POST', '/v1/responses', {
+ model: 'auto:responses-group',
+ input: 'hi',
+ });
+ expect(routed.status).toBe(200);
+ expect(lastServedModel()).toBe('named-chain-responses');
+ });
+
+ it('rejects unknown auto:<name> on /v1/responses with 400 (#1170)', async () => {
+ seedGroqKey();
+ mockGroqChat('should-not-serve', 'silent pool fallback');
+
+ const missing = await request(app, 'POST', '/v1/responses', {
+ model: 'auto:no-such-group',
+ input: 'hi',
+ });
+ expect(missing.status).toBe(400);
+ expect(JSON.stringify(missing.body)).toContain('no-such-group');
+ });
});
diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.ts
index 4e82323e..109cea01 100644
--- a/server/src/__tests__/routes/responses.test.ts
+++ b/server/src/__tests__/routes/responses.test.ts
@@ -567,7 +567,10 @@ describe('POST /v1/responses model routing priority (#579)', () => {
expect(status).toBe(200);
const { preferredModelDbId, chain } = routingCall();
expect(preferredModelDbId).toBe(stickyId);
- expect(chain).toBeUndefined();
+ // Plain auto now threads the resolved active chain (parity with
+ // /v1/chat/completions), not a singleton pin.
+ expect(chain?.map(row => row.model_db_id)).toEqual(expect.arrayContaining([pinnedId, stickyId]));
+ expect(chain?.map(row => row.model_db_id)).not.toEqual([pinnedId]);
});
it('auto routing applies when there is neither an explicit model nor a sticky session', async () => {
@@ -578,7 +581,8 @@ describe('POST /v1/responses model routing priority (#579)', () => {
expect(status).toBe(200);
const { preferredModelDbId, chain } = routingCall();
expect(preferredModelDbId).toBeUndefined();
- expect(chain).toBeUndefined();
+ expect(chain?.map(row => row.model_db_id)).toEqual(expect.arrayContaining([pinnedId, stickyId]));
+ expect(chain?.map(row => row.model_db_id)).not.toEqual([pinnedId]);
});
it.each([
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 581e7551..a714e53c 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -9,7 +9,7 @@ import type {
ChatToolChoice,
ChatContentBlock,
} from '@freellmapi/shared/types.js';
-import { routeRequest, resolveModelGroupCandidates, resolveStickyPreference, routingReserveTokens, type RouteResult, type ChainRow } from '../services/router.js';
+import { routeRequest, resolveModelGroupCandidates, resolveRoutingChain, resolveStickyPreference, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
import { getSetting, getUnifiedApiKey } from '../db/index.js';
import { contentToString } from '../lib/content.js';
import { resolveTaskType } from '../lib/task-type.js';
@@ -45,6 +45,14 @@ import { normalizeMessageImages } from '../lib/image-normalize.js';
// (already handled by extractApiToken) as well as a bearer token.
export const anthropicRouter = Router();
+const AUTO_MODEL_ID = 'auto';
+
+function isAutoModel(modelId: string | undefined): boolean {
+ if (!modelId) return true;
+ const lower = modelId.toLowerCase();
+ return lower === AUTO_MODEL_ID || lower.startsWith(`${AUTO_MODEL_ID}:`);
+}
+
const MAX_RETRIES = 20;
// Anthropic requires `max_tokens`; mirror the OpenAI route's routing-budget
// default when a client somehow omits it.
@@ -531,6 +539,29 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
const resolved = resolveAnthropicModel(routedModel);
const pinnedModelId = resolved.pinned ? (body.model ?? null) : null;
+ // Named `auto:<profile>` chains (and plain `auto`) must go through the same
+ // resolver as /v1/chat/completions. resolveAnthropicModel treats unknown ids
+ // as unpinned auto-route, which used to silently walk the active pool.
+ let resolvedChain: ResolvedChain | undefined;
+ let strategyKey: string | undefined;
+ const autoModelString = isAutoModel(requestedModel)
+ ? requestedModel
+ : (isAutoModel(routedModel) ? routedModel : undefined);
+ if (!resolved.pinned && autoModelString !== undefined) {
+ try {
+ resolvedChain = resolveRoutingChain(autoModelString);
+ // Named auto:<profile> / auto:smart get their own sticky bucket. Plain
+ // `auto` stays unscoped so Claude Code session affinity is unchanged.
+ strategyKey = resolvedChain.strategyKey === 'auto' ? undefined : resolvedChain.strategyKey;
+ } catch (err: any) {
+ if (err?.status === 400) {
+ sendError(res, 400, 'invalid_request_error', err.message);
+ return;
+ }
+ throw err;
+ }
+ }
+
// Session affinity: Claude Code stamps every request in a session with
// X-Claude-Code-Session-Id. When auto-routing, stick the whole session to one
// model so it doesn't flap between free providers mid-conversation.
@@ -577,7 +608,9 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
const sticky = getStickyModel(messages, sessionId, stickyScope);
preferredModel = (sticky != null && groupChain.some(r => r.model_db_id === sticky)) ? sticky : undefined;
}
- if (preferredModel == null && !groupChain) preferredModel = resolveStickyPreference(getStickyModel(messages, sessionId));
+ if (preferredModel == null && !groupChain) {
+ preferredModel = resolveStickyPreference(getStickyModel(messages, sessionId, strategyKey), resolvedChain?.chain);
+ }
// Thin adapter over the shared fallback loop (lib/fallback-loop.ts): the
// cooldown/skip/penalty/exhaustion machinery is shared, only the Anthropic
@@ -622,14 +655,14 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
// size latched onto state so the existing size gates in router.ts skip
// low-TPM / small-context models on retry.
const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);
- return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
+ return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
},
dispatch: async (route, attempt, dispatchCtx) => {
if (stream) {
try {
await streamCompletion(res, route, messages, dispatchOptions, {
start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,
- sessionId, pinned: resolved.pinned, stickyScope, disarmHedge: dispatchCtx.disarmHedge,
+ sessionId, pinned: resolved.pinned, stickyScope, strategyKey, disarmHedge: dispatchCtx.disarmHedge,
});
return 'done';
} catch (err: any) {
@@ -710,7 +743,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
// Remember this model for the rest of the auto-routed session. A pin used
// to make this a no-op (the pin fixed the model); a group pin still has a
// provider choice to remember, recorded under the group's own scope.
- if (!resolved.pinned || stickyScope) setStickyModel(messages, route.modelDbId, sessionId, stickyScope);
+ if (!resolved.pinned || stickyScope) setStickyModel(messages, route.modelDbId, sessionId, stickyScope ?? strategyKey);
const anthropicResponse: AnthropicMessageResponse = {
id: newMessageId(),
@@ -765,6 +798,8 @@ interface StreamCtx {
pinned: boolean;
// Sticky bucket for a group-pinned request; undefined for auto routing.
stickyScope?: string;
+ // Named auto:<profile> sticky bucket; undefined for plain auto.
+ strategyKey?: string;
/** Cancel this attempt's time-budget hedge once the stream commits. */
disarmHedge: () => void;
}
@@ -1021,7 +1056,7 @@ async function streamCompletion(
res.end();
recordUpstreamSuccess(route, ctx.estimatedInputTokens + outputTokens);
- if (!ctx.pinned || ctx.stickyScope) setStickyModel(messages, route.modelDbId, ctx.sessionId, ctx.stickyScope);
+ if (!ctx.pinned || ctx.stickyScope) setStickyModel(messages, route.modelDbId, ctx.sessionId, ctx.stickyScope ?? ctx.strategyKey);
logRequest(route.platform, route.modelId, route.keyId, 'success', ctx.estimatedInputTokens, outputTokens, Date.now() - ctx.start, null, null, ctx.pinnedModelId, null, 'http');
} catch (err: any) {
if (err instanceof StreamAlreadyStarted) throw err;
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 4da0d5b8..7dfd961f 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -9,7 +9,7 @@ import type {
ChatToolChoice,
Platform,
} from '@freellmapi/shared/types.js';
-import { routeRequest, hasEnabledVisionModel, hasEnabledToolsModel, resolveStickyPreference, routingReserveTokens, resolveModelGroupCandidates, type RouteResult, type ChainRow } from '../services/router.js';
+import { routeRequest, hasEnabledVisionModel, hasEnabledToolsModel, resolveStickyPreference, routingReserveTokens, resolveModelGroupCandidates, resolveRoutingChain, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
import { getDb } from '../db/index.js';
import { resolveAuth, prependSystemPrompt } from '../lib/system-prompt.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from '../services/model-groups.js';
@@ -781,9 +781,29 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// Priority: explicit model > sticky session > auto routing.
let preferredModel: number | undefined;
let groupChain: ChainRow[] | undefined;
+ let resolvedChain: ResolvedChain | undefined;
+
+ // Sticky scope for a named auto:<profile> (or auto:smart) chain. Plain `auto`
+ // stays on the unscoped key so existing session affinity is unchanged.
+ let stickyStrategyKey: string | undefined;
if (isAutoModel(requestedModelLabel)) {
- preferredModel = resolveStickyPreference(getStickyModel(messages, sessionIdHeader));
+ try {
+ resolvedChain = resolveRoutingChain(requestedModelLabel);
+ } catch (err: any) {
+ if (err?.status === 400) {
+ res.status(400).json({
+ error: { message: err.message, type: 'invalid_request_error' },
+ });
+ return;
+ }
+ throw err;
+ }
+ stickyStrategyKey = resolvedChain.strategyKey === 'auto' ? undefined : resolvedChain.strategyKey;
+ preferredModel = resolveStickyPreference(
+ getStickyModel(messages, sessionIdHeader, stickyStrategyKey),
+ resolvedChain.chain,
+ );
} else if (isFusionModel(requestedModelLabel)) {
// Fusion is a virtual model, not a row in the catalog. Its panel and
// judge each resolve through the normal router inside runFusion below.
@@ -1116,7 +1136,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// size latched onto state so the existing size gates in router.ts skip
// low-TPM / small-context models on retry.
const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);
- return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
+ return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
},
dispatch: async (route, attempt, ctx) => {
traceRouteEvent('Responses', {
@@ -1404,7 +1424,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
res.end();
recordUpstreamSuccess(route, estimatedInputTokens + totalOutputTokens);
- setStickyModel(messages, route.modelDbId, sessionIdHeader);
+ setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);
traceRouteEvent('Responses', {
event: 'ok',
requestId: requestGroupId,
@@ -1529,7 +1549,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// tokens against the rate-limit ledger; promptTokens/completionTokens
// above already carry the chars/4 estimate.
recordUpstreamSuccess(route, result.usage?.total_tokens ?? (promptTokens + completionTokens));
- setStickyModel(messages, route.modelDbId, sessionIdHeader);
+ setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, attempt, attemptLog);