Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(embeddings): relay the upstream Retry-After to the client (#1311)

tashdroide41d1595fc6d
diff --git a/docs/en/api/01-rest-api.md b/docs/en/api/01-rest-api.md
index 7d4d1f51..5a89946d 100644
--- a/docs/en/api/01-rest-api.md
+++ b/docs/en/api/01-rest-api.md
@@ -476,3 +476,10 @@ pointing to the next UTC month. OpenAI-compatible endpoints also return
`error.code: "quota_exceeded"`; other protocol adapters preserve their native
error format. Fusion subcalls obey the caps but retain Fusion's aggregate error
format. If another key has capacity, normal fallback can use it.
+
+Embeddings additionally relay the upstream providers' own back-off. A provider
+that answers `429` is simply skipped for the next one in the family; only when
+every provider in the family was rate limited and each stated a `Retry-After`
+does the gateway's `429` carry one: the soonest of them, in whole seconds
+(clamped to 24h). A chain that also hit a non-rate-limit failure returns `502`
+without `Retry-After`, since waiting would not be a promise.
diff --git a/server/src/__tests__/lib/module-purity.test.ts b/server/src/__tests__/lib/module-purity.test.ts
index 3bfd8d9f..82850dba 100644
--- a/server/src/__tests__/lib/module-purity.test.ts
+++ b/server/src/__tests__/lib/module-purity.test.ts
@@ -36,6 +36,7 @@ const PURE_MODULES = [
'header-value.ts',
'provider-identity.ts',
'provider-size-parser.ts',
+ 'retry-hint.ts',
'structured-output.ts',
'tool-args.ts',
'tool-call-rescue.ts',
diff --git a/server/src/__tests__/routes/embeddings-retry-after.test.ts b/server/src/__tests__/routes/embeddings-retry-after.test.ts
new file mode 100644
index 00000000..1885005d
--- /dev/null
+++ b/server/src/__tests__/routes/embeddings-retry-after.test.ts
@@ -0,0 +1,108 @@
+// A provider 429 that states when to come back must reach the client as a
+// Retry-After header on BOTH embedding surfaces (/v1/embeddings and the Ollama
+// emulation /api/embed). Before this, only the LOCAL monthly-budget block set
+// the header; an upstream `Retry-After: 17` came back as a bare 429, so SDK
+// clients retried on their own aggressive defaults and burned the tier again.
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { once } from 'node:events';
+import type { Server } from 'node:http';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey, setSetting } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+
+const realFetch = globalThis.fetch;
+
+let server: Server;
+let baseUrl: string;
+let token: string;
+
+beforeEach(async () => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ const db = getDb();
+ const secret = encrypt('embed-rate-test-key');
+ const keyId = Number(db.prepare(`INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, enabled, status, base_url)
+ VALUES ('custom', 'embed-rate', ?, ?, ?, 1, 'healthy', 'http://embeddings.test/v1')`)
+ .run(secret.encrypted, secret.iv, secret.authTag).lastInsertRowid);
+ db.prepare(`INSERT INTO embedding_models (family, platform, model_id, display_name, dimensions, priority, enabled, quota_label, key_id)
+ VALUES ('rate-embed', 'custom', 'rate-embed', 'Rate embed', 2, 1, 1, '', ?)`).run(keyId);
+ token = getUnifiedApiKey();
+ // Ollama emulation is opt-in; /api/embed 404s until it's on.
+ setSetting('ollama_emulation', 'key-required');
+ server = createApp().listen(0, '127.0.0.1');
+ await once(server, 'listening');
+ baseUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}`;
+});
+
+afterEach(async () => {
+ globalThis.fetch = realFetch;
+ vi.restoreAllMocks();
+ await new Promise<void>((resolve, reject) => server.close(err => (err ? reject(err) : resolve())));
+});
+
+function rateLimited(retryAfter: string) {
+ return new Response(JSON.stringify({ error: { message: 'slow down', type: 'rate_limit_error' } }), {
+ status: 429,
+ headers: { 'content-type': 'application/json', 'Retry-After': retryAfter },
+ });
+}
+
+async function postJson(path: string, body: unknown) {
+ // realFetch: globalThis.fetch is replaced with the provider mock inside each
+ // test, and the request to OUR server must not be swallowed by it.
+ return realFetch(`${baseUrl}${path}`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+}
+
+describe('embedding rate-limit Retry-After propagation', () => {
+ it.each(['/v1/embeddings', '/api/embed'])('%s relays the upstream Retry-After header', async (path) => {
+ globalThis.fetch = vi.fn(async () => rateLimited('17')) as unknown as typeof fetch;
+
+ const res = await postJson(path, { model: 'rate-embed', input: 'hello' });
+ expect(res.status).toBe(429);
+ expect(res.headers.get('Retry-After')).toBe('17');
+ });
+
+ it.each(['/v1/embeddings', '/api/embed'])('%s has no Retry-After when the upstream stated none', async (path) => {
+ globalThis.fetch = vi.fn(async () => new Response('server exploded', { status: 500 })) as unknown as typeof fetch;
+
+ const res = await postJson(path, { model: 'rate-embed', input: 'hello' });
+ expect(res.status).toBe(502); // chain dry -> gateway error, not a fake 429
+ expect(res.headers.get('Retry-After')).toBeNull();
+ });
+
+ it.each(['/v1/embeddings', '/api/embed'])('%s fails over past a rate-limited provider without relaying its Retry-After', async (path) => {
+ // A second provider in the same family that answers fine.
+ const db = getDb();
+ const secret = encrypt('embed-ok-key');
+ const okKey = Number(db.prepare(`INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, enabled, status, base_url)
+ VALUES ('custom', 'embed-ok', ?, ?, ?, 1, 'healthy', 'http://embeddings-ok.test/v1')`)
+ .run(secret.encrypted, secret.iv, secret.authTag).lastInsertRowid);
+ db.prepare(`INSERT INTO embedding_models (family, platform, model_id, display_name, dimensions, priority, enabled, quota_label, key_id)
+ VALUES ('rate-embed', 'custom', 'rate-embed-ok', 'Rate embed ok', 2, 2, 1, '', ?)`).run(okKey);
+ globalThis.fetch = vi.fn(async (url: string) => String(url).includes('embeddings-ok.test')
+ ? new Response(JSON.stringify({ data: [{ index: 0, embedding: [0.1, 0.2] }] }), { status: 200, headers: { 'content-type': 'application/json' } })
+ : rateLimited('17')) as unknown as typeof fetch;
+
+ const res = await postJson(path, { model: 'rate-embed', input: 'hello' });
+ expect(res.status).toBe(200);
+ expect(res.headers.get('Retry-After')).toBeNull();
+ });
+
+ it('relays only Retry-After, never other upstream headers', async () => {
+ globalThis.fetch = vi.fn(async () => new Response('{}', {
+ status: 429,
+ headers: { 'Retry-After': '17', 'x-ratelimit-remaining-requests': '0', 'x-upstream-provider': 'secret-host', 'set-cookie': 'a=b' },
+ })) as unknown as typeof fetch;
+
+ const res = await postJson('/v1/embeddings', { model: 'rate-embed', input: 'hello' });
+ expect(res.status).toBe(429);
+ expect(res.headers.get('Retry-After')).toBe('17');
+ expect(res.headers.get('x-ratelimit-remaining-requests')).toBeNull();
+ expect(res.headers.get('x-upstream-provider')).toBeNull();
+ expect(res.headers.get('set-cookie')).toBeNull();
+ });
+});
diff --git a/server/src/__tests__/services/embeddings.test.ts b/server/src/__tests__/services/embeddings.test.ts
index 5771d16f..390c27a0 100644
--- a/server/src/__tests__/services/embeddings.test.ts
+++ b/server/src/__tests__/services/embeddings.test.ts
@@ -168,6 +168,86 @@ describe('embeddings service', () => {
await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello'])).rejects.toMatchObject({ status: 429 });
});
+ it('surfaces the upstream Retry-After on a fully-exhausted chain', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ mockFetch(async () => new Response('slow down', {
+ status: 429, headers: { 'Retry-After': '17' },
+ }));
+
+ await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']))
+ .rejects.toMatchObject({ status: 429, retryAfterMs: 17_000 });
+ });
+
+ it('surfaces the SOONEST back-off, not the last one, when every provider rate-limits', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = mockFetch(async () => new Response('slow down', {
+ status: 429, headers: { 'Retry-After': '60' },
+ }));
+ fetchMock.mockResolvedValueOnce(new Response('slow down', {
+ status: 429, headers: { 'Retry-After': '5' },
+ }));
+ // nvidia comes back in 5s, openrouter in 60s: the client may retry in 5s.
+ await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']))
+ .rejects.toMatchObject({ status: 429, retryAfterMs: 5_000 });
+ });
+
+ it('parses an HTTP-date Retry-After', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const when = new Date(Date.now() + 42_000).toUTCString();
+ mockFetch(async () => new Response('slow down', { status: 429, headers: { 'Retry-After': when } }));
+ const err = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']).catch(e => e);
+ expect(err.status).toBe(429);
+ expect(err.retryAfterMs).toBeGreaterThan(35_000);
+ expect(err.retryAfterMs).toBeLessThanOrEqual(42_000);
+ });
+
+ it('clamps an absurd Retry-After to a day', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ mockFetch(async () => new Response('slow down', { status: 429, headers: { 'Retry-After': '99999999999' } }));
+ await expect(runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']))
+ .rejects.toMatchObject({ status: 429, retryAfterMs: 24 * 60 * 60 * 1000 });
+ });
+
+ it('drops the hint when a sibling failed for a non-rate-limit reason', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = mockFetch(async () => new Response('boom', { status: 500 }));
+ fetchMock.mockResolvedValueOnce(new Response('slow down', {
+ status: 429, headers: { 'Retry-After': '30' },
+ }));
+ // nvidia 429s with a hint, openrouter 500s: waiting 30s is no promise.
+ const err = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']).catch(e => e);
+ expect(err.status).toBe(502);
+ expect(err.retryAfterMs).toBeUndefined();
+ });
+
+ it('drops the hint when another rate-limited provider stated no delay', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = mockFetch(async () => new Response('slow down', { status: 429 }));
+ fetchMock.mockResolvedValueOnce(new Response('slow down', {
+ status: 429, headers: { 'Retry-After': '30' },
+ }));
+ const err = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']).catch(e => e);
+ expect(err.status).toBe(429);
+ expect(err.retryAfterMs).toBeUndefined();
+ });
+
+ it('does not surface a 429 hint when a later provider succeeds', async () => {
+ addKey('nvidia');
+ addKey('openrouter');
+ const fetchMock = mockFetch(async () => okEmbeddingResponse(2048));
+ fetchMock.mockResolvedValueOnce(new Response('slow down', {
+ status: 429, headers: { 'Retry-After': '30' },
+ }));
+ const out = await runEmbeddings('llama-nemotron-embed-vl-1b-v2', ['hello']);
+ expect(out.platform).toBe('openrouter');
+ });
+
it('throws 503 when the family has no enabled providers', async () => {
getDb().prepare("UPDATE embedding_models SET enabled = 0 WHERE family = 'bge-m3'").run();
await expect(runEmbeddings('bge-m3', ['hello'])).rejects.toMatchObject({ status: 503 });
diff --git a/server/src/lib/retry-hint.ts b/server/src/lib/retry-hint.ts
new file mode 100644
index 00000000..4cc8bb4b
--- /dev/null
+++ b/server/src/lib/retry-hint.ts
@@ -0,0 +1,38 @@
+// Aggregate the upstream back-off hints seen while a service walks its provider
+// chain (embeddings, media) into the one Retry-After the gateway may honestly
+// send once the chain is dry.
+//
+// Mirrors the chat exhaustion rule (lib/fallback-loop.ts exhaustedRetryError):
+// a Retry-After is only a promise when EVERY attempted candidate was rate
+// limited, and then it is the SOONEST moment any of them comes back — never the
+// last provider's value. A single 429 whose sibling then 500s, or a 429 that
+// stated no delay, means we cannot tell the client when to return, so no hint.
+
+/** One chain walk's view of the upstream back-offs. */
+export class RetryHintTracker {
+ private minMs: number | undefined;
+ private unknown = false;
+ private attempts = 0;
+
+ /** Record one failed candidate. `retryAfterMs` is the stated back-off, if any. */
+ record(status: number, retryAfterMs: number | undefined): void {
+ this.attempts++;
+ if (status !== 429 || retryAfterMs === undefined) {
+ this.unknown = true;
+ return;
+ }
+ this.minMs = this.minMs === undefined ? retryAfterMs : Math.min(this.minMs, retryAfterMs);
+ }
+
+ /** The soonest stated back-off when every recorded failure was a 429 carrying
+ * one; undefined otherwise (including when nothing was recorded). */
+ retryAfterMs(): number | undefined {
+ if (this.attempts === 0 || this.unknown) return undefined;
+ return this.minMs;
+ }
+}
+
+/** Milliseconds → whole Retry-After seconds (ceil, at least 1). */
+export function retryAfterSeconds(ms: number): number {
+ return Math.max(1, Math.ceil(ms / 1000));
+}
diff --git a/server/src/routes/ollama.ts b/server/src/routes/ollama.ts
index 32791637..69560944 100644
--- a/server/src/routes/ollama.ts
+++ b/server/src/routes/ollama.ts
@@ -9,8 +9,7 @@ import { getSetting, getUnifiedApiKey } from '../db/index.js';
import { buildModelListing } from '../services/model-listing.js';
import { extractApiToken, timingSafeStringEqual } from './proxy.js';
import { runInboundChat, type InboundChatResult, type InboundChatWire } from '../lib/inbound-chat.js';
-import { secondsUntilNextMonth } from '../services/key-budget.js';
-import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
+import { runEmbeddings, embeddingsRetryAfterSec, EmbeddingsError } from '../services/embeddings.js';
import { validateSession } from '../services/auth.js';
export const ollamaRouter = Router();
@@ -585,8 +584,9 @@ async function handleEmbed(req: Request, res: Response, legacy: boolean): Promis
}
} catch (error: any) {
const status = error instanceof EmbeddingsError ? error.status : 502;
- if (error instanceof EmbeddingsError && error.code === 'quota_exceeded') {
- res.setHeader('Retry-After', secondsUntilNextMonth());
+ if (error instanceof EmbeddingsError) {
+ const retrySec = embeddingsRetryAfterSec(error);
+ if (retrySec !== undefined) res.setHeader('Retry-After', retrySec);
}
res.status(status).json({ error: error.message ?? 'embedding request failed' });
}
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 5c95bc55..8a4a630b 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -5,7 +5,7 @@ import { z } from 'zod';
import type { ChatMessage, ChatToolCall, TokenUsage } from '@freellmapi/shared/types.js';
import { type RouteResult, type ResolvedChain, type ChainRow, routeRequest, resolveRoutingChain, resolveModelGroupCandidates, resolveStickyPreference, hasEnabledVisionModel, hasEnabledToolsModel, routingReserveTokens } from '../services/router.js';
import { secondsUntilNextMonth } from '../services/key-budget.js';
-import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
+import { runEmbeddings, embeddingsRetryAfterSec, EmbeddingsError } from '../services/embeddings.js';
import { runImageGeneration, runVideoGeneration, runSpeech, runTranscription, MediaError, MAX_TRANSCRIPTION_BYTES } from '../services/media.js';
import multer from 'multer';
import { getDb } from '../db/index.js';
@@ -691,6 +691,14 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {
} catch (err: any) {
const status = err instanceof EmbeddingsError ? err.status : 502;
const code = err instanceof EmbeddingsError ? inferenceBudgetCode(err, res) : {};
+ // Relay the chain's back-off only when every provider was rate limited
+ // (soonest stated Retry-After, or the budget reset) — overriding the
+ // month-long budget header inferenceBudgetCode set when a sibling comes
+ // back sooner. A lone upstream 429 the chain failed over from never does.
+ if (err instanceof EmbeddingsError) {
+ const retrySec = embeddingsRetryAfterSec(err);
+ if (retrySec !== undefined) res.setHeader('Retry-After', retrySec);
+ }
const type = status === 400 ? 'invalid_request_error' : status === 429 ? 'rate_limit_error' : 'server_error';
res.status(status).json({ error: { message: `embedding error: ${err?.message ?? 'unknown'}`, type, ...code } });
}
diff --git a/server/src/services/embeddings.ts b/server/src/services/embeddings.ts
index 599ff21c..a1e35649 100644
--- a/server/src/services/embeddings.ts
+++ b/server/src/services/embeddings.ts
@@ -8,6 +8,9 @@
// always works: with one provider it just uses that one, with several it gets
// cross-provider redundancy for free.
import { getDb, getSetting } from '../db/index.js';
+import { secondsUntilNextMonth } from './key-budget.js';
+import { parseRetryAfterMs } from '../providers/base.js';
+import { RetryHintTracker, retryAfterSeconds } from '../lib/retry-hint.js';
import { getClientContext } from '../lib/client-context.js';
import { reserveProviderCredential } from './provider-credential.js';
import { proxyFetch } from '../lib/proxy.js';
@@ -40,13 +43,32 @@ export interface EmbeddingsResult {
export class EmbeddingsError extends Error {
status: number;
code?: string;
- constructor(message: string, status: number, code?: string) {
+ /** Back-off the upstream provider stated (`Retry-After` header or a
+ * retry-delay field in the error body), in milliseconds. Kept separate from
+ * the message so the route can answer the client with a real Retry-After
+ * instead of a bare 429, the same way the chat router benches keys. */
+ retryAfterMs?: number;
+ constructor(message: string, status: number, code?: string, retryAfterMs?: number) {
super(message);
this.status = status;
this.code = code;
+ this.retryAfterMs = retryAfterMs;
}
}
+/** EmbeddingsError for a non-OK upstream response: status, truncated body and
+ * any stated back-off read off the `Retry-After` header before the body is
+ * consumed. */
+async function upstreamEmbeddingsError(r: Response): Promise<EmbeddingsError> {
+ const retryAfterMs = parseRetryAfterMs(r.headers?.get('retry-after') ?? null);
+ return new EmbeddingsError(
+ `upstream ${r.status}: ${(await r.text()).slice(0, 200)}`,
+ r.status,
+ undefined,
+ retryAfterMs,
+ );
+}
+
export function listEmbeddingModels(): EmbeddingModelRow[] {
return getDb().prepare(
'SELECT * FROM embedding_models ORDER BY family, priority',
@@ -121,7 +143,7 @@ async function openAiStyleEmbed(
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
}, platform, 'embedding', FETCH_TIMEOUT_MS);
if (!r.ok) {
- throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);
+ throw await upstreamEmbeddingsError(r);
}
const j = (await r.json()) as {
data?: { index?: number; embedding: number[] }[];
@@ -261,7 +283,7 @@ async function callProvider(row: EmbeddingModelRow, credential: ProviderCredenti
},
row.platform, 'embedding', FETCH_TIMEOUT_MS,
);
- if (!r.ok) throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);
+ if (!r.ok) throw await upstreamEmbeddingsError(r);
const j = await r.json() as number[][] | number[];
const vectors = Array.isArray(j[0]) ? (j as number[][]) : [j as number[]];
return { vectors, inputTokens: null };
@@ -278,7 +300,7 @@ async function callProvider(row: EmbeddingModelRow, credential: ProviderCredenti
}),
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
}, row.platform, 'embedding', FETCH_TIMEOUT_MS);
- if (!r.ok) throw new EmbeddingsError(`upstream ${r.status}: ${(await r.text()).slice(0, 200)}`, r.status);
+ if (!r.ok) throw await upstreamEmbeddingsError(r);
const j = (await r.json()) as { embeddings?: { float?: number[][] }; meta?: { billed_units?: { input_tokens?: number } } };
return { vectors: j.embeddings?.float ?? [], inputTokens: j.meta?.billed_units?.input_tokens ?? null };
}
@@ -331,10 +353,16 @@ export async function runEmbeddings(model: string | undefined, inputs: string[],
}
let lastError: EmbeddingsError | null = null;
+ // Upstream back-offs across the whole chain: only a chain that was rate
+ // limited end to end earns a Retry-After, and then the soonest one.
+ const hints = new RetryHintTracker();
for (const row of chain) {
const { credential, budgetBlocked } = reserveProviderCredential(row, estimateTokens(inputs));
if (!credential) {
- if (budgetBlocked) lastError = new EmbeddingsError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ if (budgetBlocked) {
+ lastError = new EmbeddingsError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ hints.record(429, secondsUntilNextMonth() * 1000);
+ }
continue;
}
const started = Date.now();
@@ -357,6 +385,7 @@ export async function runEmbeddings(model: string | undefined, inputs: string[],
const e = err instanceof EmbeddingsError ? err : new EmbeddingsError(String(err?.message ?? err), 502);
logEmbeddingRequest(row, credential.id, 'error', 0, Date.now() - started, e.message.slice(0, 300));
lastError = e;
+ hints.record(e.status, e.retryAfterMs);
// fall through to the next provider in the family
} finally {
credential.release();
@@ -367,5 +396,16 @@ export async function runEmbeddings(model: string | undefined, inputs: string[],
`All providers for embedding family '${family}' failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,
lastError && lastError.status === 429 ? 429 : 502,
lastError?.code,
+ lastError && lastError.status === 429 ? hints.retryAfterMs() : undefined,
);
}
+
+/** Whole seconds the client should wait before retrying an embeddings request:
+ * the soonest upstream back-off when the whole chain was rate limited, else the
+ * monthly-budget reset for a local quota block. Undefined means the caller
+ * shouldn't set the header. */
+export function embeddingsRetryAfterSec(err: EmbeddingsError): number | undefined {
+ if (err.retryAfterMs !== undefined) return retryAfterSeconds(err.retryAfterMs);
+ if (err.code === 'quota_exceeded') return secondsUntilNextMonth();
+ return undefined;
+}