Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(media): relay the upstream Retry-After on image/video/speech/transcription (#1312)

tashdroid1346b7d39ecb
diff --git a/server/src/__tests__/routes/media-retry-after.test.ts b/server/src/__tests__/routes/media-retry-after.test.ts
new file mode 100644
index 00000000..b015fd54
--- /dev/null
+++ b/server/src/__tests__/routes/media-retry-after.test.ts
@@ -0,0 +1,99 @@
+// Media routes relay an upstream Retry-After only once the provider chain is
+// rate limited end to end (the soonest stated delay). A single provider's 429
+// is failed over and never reaches the client as a back-off.
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { once } from 'node:events';
+import type { Server } from 'node:http';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+
+const realFetch = globalThis.fetch;
+
+let server: Server;
+let baseUrl: string;
+let token: string;
+
+function addKey(platform: string) {
+ const s = encrypt(`${platform}-media-key`);
+ getDb().prepare(`INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1)`).run(platform, s.encrypted, s.iv, s.authTag);
+}
+
+function addMedia(platform: string, modelId: string, priority: number) {
+ getDb().prepare(`INSERT INTO media_models (platform, model_id, display_name, modality, priority, enabled, quota_label, key_id, meta_json)
+ VALUES (?, ?, ?, 'image', ?, 1, '', NULL, NULL)`).run(platform, modelId, modelId, priority);
+}
+
+beforeEach(async () => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ addMedia('nvidia', 'black-forest-labs/flux.1-schnell', 1);
+ addMedia('siliconflow', 'black-forest-labs/FLUX.1-schnell', 2);
+ addKey('nvidia');
+ addKey('siliconflow');
+ token = getUnifiedApiKey();
+ server = createApp().listen(0, '127.0.0.1');
+ await once(server, 'listening');
+ baseUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}`;
+});
+
+afterEach(async () => {
+ globalThis.fetch = realFetch;
+ vi.restoreAllMocks();
+ await new Promise<void>((resolve, reject) => server.close(err => (err ? reject(err) : resolve())));
+});
+
+const limited = (ra?: string) => new Response('{"error":"slow down"}', {
+ status: 429,
+ headers: { ...(ra ? { 'Retry-After': ra } : {}), 'x-upstream-provider': 'secret-host', 'set-cookie': 'a=b' },
+});
+
+function upstream(nvidia: () => Response, siliconflow: () => Response) {
+ globalThis.fetch = vi.fn(async (url: string) =>
+ (String(url).includes('siliconflow') ? siliconflow() : nvidia())) as unknown as typeof fetch;
+}
+
+function generate() {
+ return realFetch(`${baseUrl}/v1/images/generations`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ prompt: 'a cat' }),
+ });
+}
+
+describe('image generation Retry-After relay', () => {
+ it('sends the soonest Retry-After when every provider is rate limited, and no other upstream headers', async () => {
+ upstream(() => limited('60'), () => limited(new Date(Date.now() + 20_000).toUTCString()));
+ const res = await generate();
+ expect(res.status).toBe(429);
+ const ra = Number(res.headers.get('Retry-After'));
+ expect(ra).toBeGreaterThanOrEqual(18);
+ expect(ra).toBeLessThanOrEqual(20);
+ expect(res.headers.get('x-upstream-provider')).toBeNull();
+ expect(res.headers.get('set-cookie')).toBeNull();
+ });
+
+ it('clamps an absurd Retry-After to a day', async () => {
+ upstream(() => limited('99999999999'), () => limited('99999999999'));
+ const res = await generate();
+ expect(res.status).toBe(429);
+ expect(res.headers.get('Retry-After')).toBe('86400');
+ });
+
+ it('fails over past a rate-limited provider without relaying its Retry-After', async () => {
+ upstream(() => limited('17'), () => new Response(JSON.stringify({ images: [{ url: 'https://x/y.png' }] }), {
+ status: 200, headers: { 'content-type': 'application/json' },
+ }));
+ const res = await generate();
+ expect(res.status).toBe(200);
+ expect(res.headers.get('Retry-After')).toBeNull();
+ });
+
+ it('sends no Retry-After when the chain also hit a non-rate-limit failure', async () => {
+ upstream(() => limited('17'), () => new Response('boom', { status: 500 }));
+ const res = await generate();
+ expect(res.status).toBe(502);
+ expect(res.headers.get('Retry-After')).toBeNull();
+ });
+});
diff --git a/server/src/__tests__/services/media.test.ts b/server/src/__tests__/services/media.test.ts
index ee972417..fac9a4c1 100644
--- a/server/src/__tests__/services/media.test.ts
+++ b/server/src/__tests__/services/media.test.ts
@@ -88,6 +88,44 @@ describe('media service', () => {
});
describe('image generation', () => {
+ function twoImageProviders() {
+ addMedia('nvidia', 'black-forest-labs/flux.1-schnell', 'image', 1);
+ addMedia('siliconflow', 'black-forest-labs/FLUX.1-schnell', 'image', 2);
+ addKey('nvidia');
+ addKey('siliconflow');
+ }
+ const limited = (ra?: string) => new Response('slow down', { status: 429, headers: ra ? { 'retry-after': ra } : {} });
+ const byHost = (nvidia: () => Response, siliconflow: () => Response) =>
+ vi.fn(async (url: string) => (String(url).includes('siliconflow') ? siliconflow() : nvidia())) as any;
+
+ it('relays the SOONEST Retry-After when every provider is rate-limited', async () => {
+ twoImageProviders();
+ globalThis.fetch = byHost(() => limited('5'), () => limited('60'));
+ await expect(runImageGeneration('auto', { prompt: 'a cat' })).rejects.toMatchObject({ status: 429, retryAfterMs: 5_000 });
+ });
+
+ it('drops the hint when a rate-limited provider stated no delay', async () => {
+ twoImageProviders();
+ globalThis.fetch = byHost(() => limited('17'), () => limited());
+ const err = await runImageGeneration('auto', { prompt: 'a cat' }).catch(e => e);
+ expect(err.status).toBe(429);
+ expect(err.retryAfterMs).toBeUndefined();
+ });
+
+ it('drops the hint when the chain ends in a non-rate-limit failure', async () => {
+ twoImageProviders();
+ globalThis.fetch = byHost(() => limited('17'), () => new Response('boom', { status: 500 }));
+ const err = await runImageGeneration('auto', { prompt: 'a cat' }).catch(e => e);
+ expect(err.status).toBe(502);
+ expect(err.retryAfterMs).toBeUndefined();
+ });
+
+ it('fails over past a rate-limited provider', async () => {
+ twoImageProviders();
+ globalThis.fetch = byHost(() => limited('17'), () => jsonResponse({ images: [{ url: 'https://x/y.png' }] }));
+ await expect(runImageGeneration('auto', { prompt: 'a cat' })).resolves.toMatchObject({ platform: 'siliconflow' });
+ });
+
it('NVIDIA: maps artifacts[].base64 → b64_json', async () => {
addMedia('nvidia', 'black-forest-labs/flux.1-schnell', 'image');
addKey('nvidia');
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 8a4a630b..6d7204c3 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -5,7 +5,8 @@ import { z } from 'zod';
import type { ChatMessage, ChatToolCall, TokenUsage } from '@freellmapi/shared/types.js';
import { type RouteResult, type ResolvedChain, type ChainRow, routeRequest, resolveRoutingChain, resolveModelGroupCandidates, resolveStickyPreference, hasEnabledVisionModel, hasEnabledToolsModel, routingReserveTokens } from '../services/router.js';
import { secondsUntilNextMonth } from '../services/key-budget.js';
-import { runEmbeddings, embeddingsRetryAfterSec, EmbeddingsError } from '../services/embeddings.js';
+import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
+import { retryAfterSeconds } from '../lib/retry-hint.js';
import { runImageGeneration, runVideoGeneration, runSpeech, runTranscription, MediaError, MAX_TRANSCRIPTION_BYTES } from '../services/media.js';
import multer from 'multer';
import { getDb } from '../db/index.js';
@@ -691,14 +692,6 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {
} catch (err: any) {
const status = err instanceof EmbeddingsError ? err.status : 502;
const code = err instanceof EmbeddingsError ? inferenceBudgetCode(err, res) : {};
- // Relay the chain's back-off only when every provider was rate limited
- // (soonest stated Retry-After, or the budget reset) — overriding the
- // month-long budget header inferenceBudgetCode set when a sibling comes
- // back sooner. A lone upstream 429 the chain failed over from never does.
- if (err instanceof EmbeddingsError) {
- const retrySec = embeddingsRetryAfterSec(err);
- if (retrySec !== undefined) res.setHeader('Retry-After', retrySec);
- }
const type = status === 400 ? 'invalid_request_error' : status === 429 ? 'rate_limit_error' : 'server_error';
res.status(status).json({ error: { message: `embedding error: ${err?.message ?? 'unknown'}`, type, ...code } });
}
@@ -716,8 +709,12 @@ const ImageBody = z.object({
response_format: z.enum(['url', 'b64_json']).optional(),
});
-function inferenceBudgetCode(error: { code?: string }, res: Response): { code?: string } {
- if (error.code === 'quota_exceeded') res.setHeader('Retry-After', secondsUntilNextMonth());
+function inferenceBudgetCode(error: { code?: string; retryAfterMs?: number }, res: Response): { code?: string } {
+ // retryAfterMs is set by the embeddings/media services only when the whole
+ // chain was rate limited (soonest stated back-off, budget resets included),
+ // so it wins over the month-long budget reset when a sibling returns sooner.
+ if (error.retryAfterMs !== undefined) res.setHeader('Retry-After', retryAfterSeconds(error.retryAfterMs));
+ else if (error.code === 'quota_exceeded') res.setHeader('Retry-After', secondsUntilNextMonth());
return error.code ? { code: error.code } : {};
}
diff --git a/server/src/services/media.ts b/server/src/services/media.ts
index 5353e648..d55db584 100644
--- a/server/src/services/media.ts
+++ b/server/src/services/media.ts
@@ -8,6 +8,9 @@
// published catalog and arrive via catalog-sync (premium on the live tier within
// ~12h, free once each model is 30 days old) — never seeded by migrations.
import { getDb } from '../db/index.js';
+import { parseRetryAfterMs } from '../providers/base.js';
+import { RetryHintTracker } from '../lib/retry-hint.js';
+import { secondsUntilNextMonth } from './key-budget.js';
import { getClientContext } from '../lib/client-context.js';
import { reserveProviderCredential } from './provider-credential.js';
import { proxyFetch } from '../lib/proxy.js';
@@ -56,10 +59,15 @@ export class MediaError extends Error {
status: number;
/** Optional machine-readable error code surfaced in the OpenAI-shaped body. */
code?: string;
- constructor(message: string, status: number, code?: string) {
+ /** Back-off the upstream provider stated (`Retry-After` header), in
+ * milliseconds — relayed to the client so an SDK sleeps the stated amount
+ * instead of hammering the chain, mirroring the embeddings path. */
+ retryAfterMs?: number;
+ constructor(message: string, status: number, code?: string, retryAfterMs?: number) {
super(message);
this.status = status;
this.code = code;
+ this.retryAfterMs = retryAfterMs;
}
}
@@ -237,7 +245,12 @@ async function mediaFetch(
);
if (!r.ok) {
const body = await r.text().catch(() => '');
- throw new MediaError(`${platform} ${r.status}: ${body.slice(0, 200)}`, r.status);
+ throw new MediaError(
+ `${platform} ${r.status}: ${body.slice(0, 200)}`,
+ r.status,
+ undefined,
+ parseRetryAfterMs(r.headers?.get('retry-after') ?? null),
+ );
}
return r;
}
@@ -725,7 +738,7 @@ function logMedia(row: Pick<MediaModelRow, 'platform' | 'model_id' | 'modality'>
}
}
-function chainError(modality: MediaModality, lastError: MediaError | null): MediaError {
+function chainError(modality: MediaModality, lastError: MediaError | null, hints: RetryHintTracker): MediaError {
// Only statuses the CALLER can act on are passed through. 400 (bad prompt,
// duration the model does not accept) and 413 (upload too large) describe
// the caller's own request, and 429 is the long-standing rate-limit signal.
@@ -740,6 +753,10 @@ function chainError(modality: MediaModality, lastError: MediaError | null): Medi
`All ${modality} providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,
status,
lastError?.code,
+ // A 429 from one provider is failed over, so its Retry-After is only the
+ // client's business when EVERY provider was rate limited with a stated
+ // delay — then the soonest one (same rule as the chat exhaustion path).
+ status === 429 ? hints.retryAfterMs() : undefined,
);
}
@@ -747,12 +764,17 @@ function chainError(modality: MediaModality, lastError: MediaError | null): Medi
export async function runImageGeneration(model: string | undefined, params: ImageParams): Promise<ImageResult> {
const chain = resolveMediaChain(model, 'image');
let lastError: MediaError | null = null;
+ // Only a chain rate limited end to end earns a Retry-After (the soonest).
+ const hints = new RetryHintTracker();
for (const row of chain) {
const { credential, budgetBlocked } = KEYLESS_CAPABLE.has(row.platform)
? { credential: { id: null, key: null, baseUrl: null, release: () => {} }, budgetBlocked: false }
: reserveProviderCredential(row, 0);
if (!credential) {
- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ if (budgetBlocked) {
+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ hints.record(429, secondsUntilNextMonth() * 1000);
+ }
continue;
}
const started = Date.now();
@@ -767,11 +789,12 @@ export async function runImageGeneration(model: string | undefined, params: Imag
const e = err instanceof MediaError ? err : new MediaError(String(err?.message ?? err), 502);
logMedia(row, credential.id, 'error', Date.now() - started, e.message.slice(0, 300));
lastError = e;
+ hints.record(e.status, e.retryAfterMs);
} finally {
credential.release();
}
}
- throw chainError('image', lastError);
+ throw chainError('image', lastError, hints);
}
/** Generate a video, failing over across catalogued text-to-video providers.
@@ -784,12 +807,17 @@ export async function runVideoGeneration(
): Promise<VideoResult> {
const chain = resolveMediaChain(model, 'video');
let lastError: MediaError | null = null;
+ // Only a chain rate limited end to end earns a Retry-After (the soonest).
+ const hints = new RetryHintTracker();
for (const row of chain) {
throwIfClientGone(clientSignal);
const { credential, budgetBlocked } = reserveProviderCredential(row, 0,
keyId => isOnCooldown(row.platform, row.model_id, keyId));
if (!credential) {
- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ if (budgetBlocked) {
+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ hints.record(429, secondsUntilNextMonth() * 1000);
+ }
continue;
}
const started = Date.now();
@@ -803,13 +831,14 @@ export async function runVideoGeneration(
logMedia(row, credential.id, 'error', Date.now() - started, e.message.slice(0, 300));
if (e.status === 429 && credential.id != null) setCooldown(row.platform, row.model_id, credential.id);
lastError = e;
+ hints.record(e.status, e.retryAfterMs);
// A caller that hung up gets no second generation charged to its account.
throwIfClientGone(clientSignal);
} finally {
credential.release();
}
}
- throw chainError('video', lastError);
+ throw chainError('video', lastError, hints);
}
// ---------------------------------------------------------------------------
@@ -1056,11 +1085,16 @@ export async function runTranscription(model: string | undefined, p: Transcripti
throw new MediaError(`Audio file exceeds the ${maxMb} MB provider upload limit.`, 413);
}
let lastError: MediaError | null = null;
+ // Only a chain rate limited end to end earns a Retry-After (the soonest).
+ const hints = new RetryHintTracker();
for (const m of usable) {
const { credential, budgetBlocked } = reserveProviderCredential({ platform: m.platform, key_id: m.keyId }, 0,
keyId => isOnCooldown(m.platform, m.modelId, keyId));
if (!credential) {
- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ if (budgetBlocked) {
+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ hints.record(429, secondsUntilNextMonth() * 1000);
+ }
continue;
}
const logRow = { platform: m.platform, model_id: m.modelId, modality: 'transcription' as const };
@@ -1079,23 +1113,29 @@ export async function runTranscription(model: string | undefined, p: Transcripti
setCooldown(m.platform, m.modelId, credential.id);
}
lastError = e;
+ hints.record(e.status, e.retryAfterMs);
} finally {
credential.release();
}
}
- throw chainError('transcription', lastError);
+ throw chainError('transcription', lastError, hints);
}
/** Synthesize speech, failing over across providers serving the modality. */
export async function runSpeech(model: string | undefined, params: SpeechParams): Promise<SpeechResult> {
const chain = resolveMediaChain(model, 'audio');
let lastError: MediaError | null = null;
+ // Only a chain rate limited end to end earns a Retry-After (the soonest).
+ const hints = new RetryHintTracker();
for (const row of chain) {
const { credential, budgetBlocked } = KEYLESS_CAPABLE.has(row.platform)
? { credential: { id: null, key: null, baseUrl: null, release: () => {} }, budgetBlocked: false }
: reserveProviderCredential(row, 0);
if (!credential) {
- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ if (budgetBlocked) {
+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');
+ hints.record(429, secondsUntilNextMonth() * 1000);
+ }
continue;
}
const started = Date.now();
@@ -1108,9 +1148,10 @@ export async function runSpeech(model: string | undefined, params: SpeechParams)
const e = err instanceof MediaError ? err : new MediaError(String(err?.message ?? err), 502);
logMedia(row, credential.id, 'error', Date.now() - started, e.message.slice(0, 300));
lastError = e;
+ hints.record(e.status, e.retryAfterMs);
} finally {
credential.release();
}
}
- throw chainError('audio', lastError);
+ throw chainError('audio', lastError, hints);
}