7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(embeddings): relay the upstream Retry-After to the client (#1311)
tashdroide41d1595fc6d
diff --git a/docs/en/api/01-rest-api.md b/docs/en/api/01-rest-api.mdindex 7d4d1f51..5a89946d 100644--- a/docs/en/api/01-rest-api.md+++ b/docs/en/api/01-rest-api.md@@ -476,3 +476,10 @@ pointing to the next UTC month. OpenAI-compatible endpoints also return`error.code: "quota_exceeded"`; other protocol adapters preserve their nativeerror format. Fusion subcalls obey the caps but retain Fusion's aggregate errorformat. If another key has capacity, normal fallback can use it.++Embeddings additionally relay the upstream providers' own back-off. A provider+that answers `429` is simply skipped for the next one in the family; only when+every provider in the family was rate limited and each stated a `Retry-After`+does the gateway's `429` carry one: the soonest of them, in whole seconds+(clamped to 24h). A chain that also hit a non-rate-limit failure returns `502`+without `Retry-After`, since waiting would not be a promise.diff --git a/server/src/__tests__/lib/module-purity.test.ts b/server/src/__tests__/lib/module-purity.test.tsindex 3bfd8d9f..82850dba 100644--- a/server/src/__tests__/lib/module-purity.test.ts+++ b/server/src/__tests__/lib/module-purity.test.ts@@ -36,6 +36,7 @@ const PURE_MODULES = ['header-value.ts','provider-identity.ts','provider-size-parser.ts',+ 'retry-hint.ts','structured-output.ts','tool-args.ts','tool-call-rescue.ts',diff --git a/server/src/__tests__/routes/embeddings-retry-after.test.ts b/server/src/__tests__/routes/embeddings-retry-after.test.tsnew file mode 100644index 00000000..1885005d--- /dev/null+++ b/server/src/__tests__/routes/embeddings-retry-after.test.ts@@ -0,0 +1,108 @@+// A provider 429 that states when to come back must reach the client as a+// Retry-After header on BOTH embedding surfaces (/v1/embeddings and the Ollama+// emulation /api/embed). Before this, only the LOCAL monthly-budget block set+// the header; an upstream `Retry-After: 17` came back as a bare 429, so SDK+// clients retried on their own aggressive defaults and burned the tier again.+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { once } from 'node:events';+import type { Server } from 'node:http';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey, setSetting } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';++const realFetch = globalThis.fetch;++let server: Server;+let baseUrl: string;+let token: string;++beforeEach(async () => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ const db = getDb();+ const secret = encrypt('embed-rate-test-key');+ const keyId = Number(db.prepare(`INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, enabled, status, base_url)+ VALUES ('custom', 'embed-rate', ?, ?, ?, 1, 'healthy', 'http://embeddings.test/v1')`)+ .run(secret.encrypted, secret.iv, secret.authTag).lastInsertRowid);+ db.prepare(`INSERT INTO embedding_models (family, platform, model_id, display_name, dimensions, priority, enabled, quota_label, key_id)+ VALUES ('rate-embed', 'custom', 'rate-embed', 'Rate embed', 2, 1, 1, '', ?)`).run(keyId);+ token = getUnifiedApiKey();+ // Ollama emulation is opt-in; /api/embed 404s until it's on.+ setSetting('ollama_emulation', 'key-required');+ server = createApp().listen(0, '127.0.0.1');+ await once(server, 'listening');+ baseUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}`;+});++afterEach(async () => {+ globalThis.fetch = realFetch;+ vi.restoreAllMocks();+ await new Promise<void>((resolve, reject) => server.close(err => (err ? reject(err) : resolve())));+});++function rateLimited(retryAfter: string) {+ return new Response(JSON.stringify({ error: { message: 'slow down', type: 'rate_limit_error' } }), {+ status: 429,+ headers: { 'content-type': 'application/json', 'Retry-After': retryAfter },+ });+}++async function postJson(path: string, body: unknown) {+ // realFetch: globalThis.fetch is replaced with the provider mock inside each+ // test, and the request to OUR server must not be swallowed by it.+ return realFetch(`${baseUrl}${path}`, {+ method: 'POST',+ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+}++describe('embedding rate-limit Retry-After propagation', () => {+ it.each(['/v1/embeddings', '/api/embed'])('%s relays the upstream Retry-After header', async (path) => {+ globalThis.fetch = vi.fn(async () => rateLimited('17')) as unknown as typeof fetch;++ const res = await postJson(path, { model: 'rate-embed', input: 'hello' });+ expect(res.status).toBe(429);+ expect(res.headers.get('Retry-After')).toBe('17');+ });++ it.each(['/v1/embeddings', '/api/embed'])('%s has no Retry-After when the upstream stated none', async (path) => {+ globalThis.fetch = vi.fn(async () => new Response('server exploded', { status: 500 })) as unknown as typeof fetch;++ const res = await postJson(path, { model: 'rate-embed', input: 'hello' });+ expect(res.status).toBe(502); // chain dry -> gateway error, not a fake 429+ expect(res.headers.get('Retry-After')).toBeNull();+ });++ it.each(['/v1/embeddings', '/api/embed'])('%s fails over past a rate-limited provider without relaying its Retry-After', async (path) => {+ // A second provider in the same family that answers fine.+ const db = getDb();+ const secret = encrypt('embed-ok-key');+ const okKey = Number(db.prepare(`INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, enabled, status, base_url)+ VALUES ('custom', 'embed-ok', ?, ?, ?, 1, 'healthy', 'http://embeddings-ok.test/v1')`)+ .run(secret.encrypted, secret.iv, secret.authTag).lastInsertRowid);+ db.prepare(`INSERT INTO embedding_models (family, platform, model_id, display_name, dimensions, priority, enabled, quota_label, key_id)+ VALUES ('rate-embed', 'custom', 'rate-embed-ok', 'Rate embed ok', 2, 2, 1, '', ?)`).run(okKey);+ globalThis.fetch = vi.fn(async (url: string) => String(url).includes('embeddings-ok.test')+ ? new Response(JSON.stringify({ data: [{ index: 0, embedding: [0.1, 0.2] }] }), { status: 200, headers: { 'content-type': 'application/json' } })+ : rateLimited('17')) as unknown as typeof fetch;++ const res = await postJson(path, { model: 'rate-embed', input: 'hello' });+ expect(res.status).toBe(200);+ expect(res.headers.get('Retry-After')).toBeNull();+ });++ it('relays only Retry-After, never other upstream headers', async () => {+ globalThis.fetch = vi.fn(async () => new Response('{}', {+ status: 429,+ headers: { 'Retry-After': '17', 'x-ratelimit-remaining-requests': '0', 'x-upstream-provider': 'secret-host', 'set-cookie': 'a=b' },+ })) as unknown as typeof fetch;++ const res = await postJson('/v1/embeddings', { model: 'rate-embed', input: 'hello' });+ expect(res.status).toBe(429);+ expect(res.headers.get('Retry-After')).toBe('17');+ expect(res.headers.get('x-ratelimit-remaining-requests')).toBeNull();+ expect(res.headers.get('x-upstream-provider')).toBeNull();+ expect(res.headers.get('set-cookie')).toBeNull();+ });+});diff --git a/server/src/__tests__/services/embeddings.test.ts b/server/src/__tests__/services/embeddings.test.tsindex 5771d16f..390c27a0 100644--- a/server/src/__tests__/services/embeddings.test.ts+++ b/server/src/__tests__/services/embeddings.test.ts@@ -168,6 +168,86 @@ describe('embeddings service', () => {await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello'])).rejects.toMatchObject({ status: 429 });});+ it('surfaces the upstream Retry-After on a fully-exhausted chain', async () => {+ addKey('nvidia');+ addKey('openrouter');+ mockFetch(async () => new Response('slow down', {+ status: 429, headers: { 'Retry-After': '17' },+ }));++ await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']))+ .rejects.toMatchObject({ status: 429, retryAfterMs: 17_000 });+ });++ it('surfaces the SOONEST back-off, not the last one, when every provider rate-limits', async () => {+ addKey('nvidia');+ addKey('openrouter');+ const fetchMock = mockFetch(async () => new Response('slow down', {+ status: 429, headers: { 'Retry-After': '60' },+ }));+ fetchMock.mockResolvedValueOnce(new Response('slow down', {+ status: 429, headers: { 'Retry-After': '5' },+ }));+ // nvidia comes back in 5s, openrouter in 60s: the client may retry in 5s.+ await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']))+ .rejects.toMatchObject({ status: 429, retryAfterMs: 5_000 });+ });++ it('parses an HTTP-date Retry-After', async () => {+ addKey('nvidia');+ addKey('openrouter');+ const when = new Date(Date.now() + 42_000).toUTCString();+ mockFetch(async () => new Response('slow down', { status: 429, headers: { 'Retry-After': when } }));+ const err = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']).catch(e => e);+ expect(err.status).toBe(429);+ expect(err.retryAfterMs).toBeGreaterThan(35_000);+ expect(err.retryAfterMs).toBeLessThanOrEqual(42_000);+ });++ it('clamps an absurd Retry-After to a day', async () => {+ addKey('nvidia');+ addKey('openrouter');+ mockFetch(async () => new Response('slow down', { status: 429, headers: { 'Retry-After': '99999999999' } }));+ await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']))+ .rejects.toMatchObject({ status: 429, retryAfterMs: 24 * 60 * 60 * 1000 });+ });++ it('drops the hint when a sibling failed for a non-rate-limit reason', async () => {+ addKey('nvidia');+ addKey('openrouter');+ const fetchMock = mockFetch(async () => new Response('boom', { status: 500 }));+ fetchMock.mockResolvedValueOnce(new Response('slow down', {+ status: 429, headers: { 'Retry-After': '30' },+ }));+ // nvidia 429s with a hint, openrouter 500s: waiting 30s is no promise.+ const err = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']).catch(e => e);+ expect(err.status).toBe(502);+ expect(err.retryAfterMs).toBeUndefined();+ });++ it('drops the hint when another rate-limited provider stated no delay', async () => {+ addKey('nvidia');+ addKey('openrouter');+ const fetchMock = mockFetch(async () => new Response('slow down', { status: 429 }));+ fetchMock.mockResolvedValueOnce(new Response('slow down', {+ status: 429, headers: { 'Retry-After': '30' },+ }));+ const err = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']).catch(e => e);+ expect(err.status).toBe(429);+ expect(err.retryAfterMs).toBeUndefined();+ });++ it('does not surface a 429 hint when a later provider succeeds', async () => {+ addKey('nvidia');+ addKey('openrouter');+ const fetchMock = mockFetch(async () => okEmbeddingResponse(2048));+ fetchMock.mockResolvedValueOnce(new Response('slow down', {+ status: 429, headers: { 'Retry-After': '30' },+ }));+ const out = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']);+ expect(out.platform).toBe('openrouter');+ });+it('throws 503 when the family has no enabled providers', async () => {getDb().prepare("UPDATE embedding_models SET enabled = 0 WHERE family = 'bge-m3'").run();await expect(runEmbeddings('bge-m3', ['hello'])).rejects.toMatchObject({ status: 503 });diff --git a/server/src/lib/retry-hint.ts b/server/src/lib/retry-hint.tsnew file mode 100644index 00000000..4cc8bb4b--- /dev/null+++ b/server/src/lib/retry-hint.ts@@ -0,0 +1,38 @@+// Aggregate the upstream back-off hints seen while a service walks its provider+// chain (embeddings, media) into the one Retry-After the gateway may honestly+// send once the chain is dry.+//+// Mirrors the chat exhaustion rule (lib/fallback-loop.ts exhaustedRetryError):+// a Retry-After is only a promise when EVERY attempted candidate was rate+// limited, and then it is the SOONEST moment any of them comes back — never the+// last provider's value. A single 429 whose sibling then 500s, or a 429 that+// stated no delay, means we cannot tell the client when to return, so no hint.++/** One chain walk's view of the upstream back-offs. */+export class RetryHintTracker {+ private minMs: number | undefined;+ private unknown = false;+ private attempts = 0;++ /** Record one failed candidate. `retryAfterMs` is the stated back-off, if any. */+ record(status: number, retryAfterMs: number | undefined): void {+ this.attempts++;+ if (status !== 429 || retryAfterMs === undefined) {+ this.unknown = true;+ return;+ }+ this.minMs = this.minMs === undefined ? retryAfterMs : Math.min(this.minMs, retryAfterMs);+ }++ /** The soonest stated back-off when every recorded failure was a 429 carrying+ * one; undefined otherwise (including when nothing was recorded). */+ retryAfterMs(): number | undefined {+ if (this.attempts === 0 || this.unknown) return undefined;+ return this.minMs;+ }+}++/** Milliseconds → whole Retry-After seconds (ceil, at least 1). */+export function retryAfterSeconds(ms: number): number {+ return Math.max(1, Math.ceil(ms / 1000));+}diff --git a/server/src/routes/ollama.ts b/server/src/routes/ollama.tsindex 32791637..69560944 100644--- a/server/src/routes/ollama.ts+++ b/server/src/routes/ollama.ts@@ -9,8 +9,7 @@ import { getSetting, getUnifiedApiKey } from '../db/index.js';import { buildModelListing } from '../services/model-listing.js';import { extractApiToken, timingSafeStringEqual } from './proxy.js';import { runInboundChat, type InboundChatResult, type InboundChatWire } from '../lib/inbound-chat.js';-import { secondsUntilNextMonth } from '../services/key-budget.js';-import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';+import { runEmbeddings, embeddingsRetryAfterSec, EmbeddingsError } from '../services/embeddings.js';import { validateSession } from '../services/auth.js';export const ollamaRouter = Router();@@ -585,8 +584,9 @@ async function handleEmbed(req: Request, res: Response, legacy: boolean): Promis}} catch (error: any) {const status = error instanceof EmbeddingsError ? error.status : 502;- if (error instanceof EmbeddingsError && error.code === 'quota_exceeded') {- res.setHeader('Retry-After', secondsUntilNextMonth());+ if (error instanceof EmbeddingsError) {+ const retrySec = embeddingsRetryAfterSec(error);+ if (retrySec !== undefined) res.setHeader('Retry-After', retrySec);}res.status(status).json({ error: error.message ?? 'embedding request failed' });}diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 5c95bc55..8a4a630b 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -5,7 +5,7 @@ import { z } from 'zod';import type { ChatMessage, ChatToolCall, TokenUsage } from '@freellmapi/shared/types.js';import { type RouteResult, type ResolvedChain, type ChainRow, routeRequest, resolveRoutingChain, resolveModelGroupCandidates, resolveStickyPreference, hasEnabledVisionModel, hasEnabledToolsModel, routingReserveTokens } from '../services/router.js';import { secondsUntilNextMonth } from '../services/key-budget.js';-import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';+import { runEmbeddings, embeddingsRetryAfterSec, EmbeddingsError } from '../services/embeddings.js';import { runImageGeneration, runVideoGeneration, runSpeech, runTranscription, MediaError, MAX_TRANSCRIPTION_BYTES } from '../services/media.js';import multer from 'multer';import { getDb } from '../db/index.js';@@ -691,6 +691,14 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {} catch (err: any) {const status = err instanceof EmbeddingsError ? err.status : 502;const code = err instanceof EmbeddingsError ? inferenceBudgetCode(err, res) : {};+ // Relay the chain's back-off only when every provider was rate limited+ // (soonest stated Retry-After, or the budget reset) — overriding the+ // month-long budget header inferenceBudgetCode set when a sibling comes+ // back sooner. A lone upstream 429 the chain failed over from never does.+ if (err instanceof EmbeddingsError) {+ const retrySec = embeddingsRetryAfterSec(err);+ if (retrySec !== undefined) res.setHeader('Retry-After', retrySec);+ }const type = status === 400 ? 'invalid_request_error' : status === 429 ? 'rate_limit_error' : 'server_error';res.status(status).json({ error: { message: `embedding error: ${err?.message ?? 'unknown'}`, type, ...code } });}diff --git a/server/src/services/embeddings.ts b/server/src/services/embeddings.tsindex 599ff21c..a1e35649 100644--- a/server/src/services/embeddings.ts+++ b/server/src/services/embeddings.ts@@ -8,6 +8,9 @@// always works: with one provider it just uses that one, with several it gets// cross-provider redundancy for free.import { getDb, getSetting } from '../db/index.js';+import { secondsUntilNextMonth } from './key-budget.js';+import { parseRetryAfterMs } from '../providers/base.js';+import { RetryHintTracker, retryAfterSeconds } from '../lib/retry-hint.js';import { getClientContext } from '../lib/client-context.js';import { reserveProviderCredential } from './provider-credential.js';import { proxyFetch } from '../lib/proxy.js';@@ -40,13 +43,32 @@ export interface EmbeddingsResult {export class EmbeddingsError extends Error {status: number;code?: string;- constructor(message: string, status: number, code?: string) {+ /** Back-off the upstream provider stated (`Retry-After` header or a+ * retry-delay field in the error body), in milliseconds. Kept separate from+ * the message so the route can answer the client with a real Retry-After+ * instead of a bare 429, the same way the chat router benches keys. */+ retryAfterMs?: number;+ constructor(message: string, status: number, code?: string, retryAfterMs?: number) {super(message);this.status = status;this.code = code;+ this.retryAfterMs = retryAfterMs;}}+/** EmbeddingsError for a non-OK upstream response: status, truncated body and+ * any stated back-off read off the `Retry-After` header before the body is+ * consumed. */+async function upstreamEmbeddingsError(r: Response): Promise<EmbeddingsError> {+ const retryAfterMs = parseRetryAfterMs(r.headers?.get('retry-after') ?? null);+ return new EmbeddingsError(+ `upstream ${r.status}: ${(await r.text()).slice(0, 200)}`,+ r.status,+ undefined,+ retryAfterMs,+ );+}+export function listEmbeddingModels(): EmbeddingModelRow[] {return getDb().prepare('SELECT * FROM embedding_models ORDER BY family, priority',@@ -121,7 +143,7 @@ async function openAiStyleEmbed(signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),}, platform, 'embedding', FETCH_TIMEOUT_MS);if (!r.ok) {- throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);+ throw await upstreamEmbeddingsError(r);}const j = (await r.json()) as {data?: { index?: number; embedding: number[] }[];@@ -261,7 +283,7 @@ async function callProvider(row: EmbeddingModelRow, credential: ProviderCredenti},row.platform, 'embedding', FETCH_TIMEOUT_MS,);- if (!r.ok) throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);+ if (!r.ok) throw await upstreamEmbeddingsError(r);const j = await r.json() as number[][] | number[];const vectors = Array.isArray(j[0]) ? (j as number[][]) : [j as number[]];return { vectors, inputTokens: null };@@ -278,7 +300,7 @@ async function callProvider(row: EmbeddingModelRow, credential: ProviderCredenti}),signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),}, row.platform, 'embedding', FETCH_TIMEOUT_MS);- if (!r.ok) throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);+ if (!r.ok) throw await upstreamEmbeddingsError(r);const j = (await r.json()) as { embeddings?: { float?: number[][] }; meta?: { billed_units?: { input_tokens?: number } } };return { vectors: j.embeddings?.float ?? [], inputTokens: j.meta?.billed_units?.input_tokens ?? null };}@@ -331,10 +353,16 @@ export async function runEmbeddings(model: string | undefined, inputs: string[],}let lastError: EmbeddingsError | null = null;+ // Upstream back-offs across the whole chain: only a chain that was rate+ // limited end to end earns a Retry-After, and then the soonest one.+ const hints = new RetryHintTracker();for (const row of chain) {const { credential, budgetBlocked } = reserveProviderCredential(row, estimateTokens(inputs));if (!credential) {- if (budgetBlocked) lastError = new EmbeddingsError('Monthly key budget exhausted', 429, 'quota_exceeded');+ if (budgetBlocked) {+ lastError = new EmbeddingsError('Monthly key budget exhausted', 429, 'quota_exceeded');+ hints.record(429, secondsUntilNextMonth() * 1000);+ }continue;}const started = Date.now();@@ -357,6 +385,7 @@ export async function runEmbeddings(model: string | undefined, inputs: string[],const e = err instanceof EmbeddingsError ? err : new EmbeddingsError(String(err?.message ?? err), 502);logEmbeddingRequest(row, credential.id, 'error', 0, Date.now() - started, e.message.slice(0, 300));lastError = e;+ hints.record(e.status, e.retryAfterMs);// fall through to the next provider in the family} finally {credential.release();@@ -367,5 +396,16 @@ export async function runEmbeddings(model: string | undefined, inputs: string[],`All providers for embedding family '${family}' failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,lastError && lastError.status === 429 ? 429 : 502,lastError?.code,+ lastError && lastError.status === 429 ? hints.retryAfterMs() : undefined,);}++/** Whole seconds the client should wait before retrying an embeddings request:+ * the soonest upstream back-off when the whole chain was rate limited, else the+ * monthly-budget reset for a local quota block. Undefined means the caller+ * shouldn't set the header. */+export function embeddingsRetryAfterSec(err: EmbeddingsError): number | undefined {+ if (err.retryAfterMs !== undefined) return retryAfterSeconds(err.retryAfterMs);+ if (err.code === 'quota_exceeded') return secondsUntilNextMonth();+ return undefined;+}