7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(media): relay the upstream Retry-After on image/video/speech/transcription (#1312)
tashdroid1346b7d39ecb
diff --git a/server/src/__tests__/routes/media-retry-after.test.ts b/server/src/__tests__/routes/media-retry-after.test.tsnew file mode 100644index 00000000..b015fd54--- /dev/null+++ b/server/src/__tests__/routes/media-retry-after.test.ts@@ -0,0 +1,99 @@+// Media routes relay an upstream Retry-After only once the provider chain is+// rate limited end to end (the soonest stated delay). A single provider's 429+// is failed over and never reaches the client as a back-off.+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { once } from 'node:events';+import type { Server } from 'node:http';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';++const realFetch = globalThis.fetch;++let server: Server;+let baseUrl: string;+let token: string;++function addKey(platform: string) {+ const s = encrypt(`${platform}-media-key`);+ getDb().prepare(`INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1)`).run(platform, s.encrypted, s.iv, s.authTag);+}++function addMedia(platform: string, modelId: string, priority: number) {+ getDb().prepare(`INSERT INTO media_models (platform, model_id, display_name, modality, priority, enabled, quota_label, key_id, meta_json)+ VALUES (?, ?, ?, 'image', ?, 1, '', NULL, NULL)`).run(platform, modelId, modelId, priority);+}++beforeEach(async () => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ addMedia('nvidia', 'black-forest-labs/flux.1-schnell', 1);+ addMedia('siliconflow', 'black-forest-labs/FLUX.1-schnell', 2);+ addKey('nvidia');+ addKey('siliconflow');+ token = getUnifiedApiKey();+ server = createApp().listen(0, '127.0.0.1');+ await once(server, 'listening');+ baseUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}`;+});++afterEach(async () => {+ globalThis.fetch = realFetch;+ vi.restoreAllMocks();+ await new Promise<void>((resolve, reject) => server.close(err => (err ? reject(err) : resolve())));+});++const limited = (ra?: string) => new Response('{"error":"slow down"}', {+ status: 429,+ headers: { ...(ra ? { 'Retry-After': ra } : {}), 'x-upstream-provider': 'secret-host', 'set-cookie': 'a=b' },+});++function upstream(nvidia: () => Response, siliconflow: () => Response) {+ globalThis.fetch = vi.fn(async (url: string) =>+ (String(url).includes('siliconflow') ? siliconflow() : nvidia())) as unknown as typeof fetch;+}++function generate() {+ return realFetch(`${baseUrl}/v1/images/generations`, {+ method: 'POST',+ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },+ body: JSON.stringify({ prompt: 'a cat' }),+ });+}++describe('image generation Retry-After relay', () => {+ it('sends the soonest Retry-After when every provider is rate limited, and no other upstream headers', async () => {+ upstream(() => limited('60'), () => limited(new Date(Date.now() + 20_000).toUTCString()));+ const res = await generate();+ expect(res.status).toBe(429);+ const ra = Number(res.headers.get('Retry-After'));+ expect(ra).toBeGreaterThanOrEqual(18);+ expect(ra).toBeLessThanOrEqual(20);+ expect(res.headers.get('x-upstream-provider')).toBeNull();+ expect(res.headers.get('set-cookie')).toBeNull();+ });++ it('clamps an absurd Retry-After to a day', async () => {+ upstream(() => limited('99999999999'), () => limited('99999999999'));+ const res = await generate();+ expect(res.status).toBe(429);+ expect(res.headers.get('Retry-After')).toBe('86400');+ });++ it('fails over past a rate-limited provider without relaying its Retry-After', async () => {+ upstream(() => limited('17'), () => new Response(JSON.stringify({ images: [{ url: 'https://x/y.png' }] }), {+ status: 200, headers: { 'content-type': 'application/json' },+ }));+ const res = await generate();+ expect(res.status).toBe(200);+ expect(res.headers.get('Retry-After')).toBeNull();+ });++ it('sends no Retry-After when the chain also hit a non-rate-limit failure', async () => {+ upstream(() => limited('17'), () => new Response('boom', { status: 500 }));+ const res = await generate();+ expect(res.status).toBe(502);+ expect(res.headers.get('Retry-After')).toBeNull();+ });+});diff --git a/server/src/__tests__/services/media.test.ts b/server/src/__tests__/services/media.test.tsindex ee972417..fac9a4c1 100644--- a/server/src/__tests__/services/media.test.ts+++ b/server/src/__tests__/services/media.test.ts@@ -88,6 +88,44 @@ describe('media service', () => {});describe('image generation', () => {+ function twoImageProviders() {+ addMedia('nvidia', 'black-forest-labs/flux.1-schnell', 'image', 1);+ addMedia('siliconflow', 'black-forest-labs/FLUX.1-schnell', 'image', 2);+ addKey('nvidia');+ addKey('siliconflow');+ }+ const limited = (ra?: string) => new Response('slow down', { status: 429, headers: ra ? { 'retry-after': ra } : {} });+ const byHost = (nvidia: () => Response, siliconflow: () => Response) =>+ vi.fn(async (url: string) => (String(url).includes('siliconflow') ? siliconflow() : nvidia())) as any;++ it('relays the SOONEST Retry-After when every provider is rate-limited', async () => {+ twoImageProviders();+ globalThis.fetch = byHost(() => limited('5'), () => limited('60'));+ await expect(runImageGeneration('auto', { prompt: 'a cat' })).rejects.toMatchObject({ status: 429, retryAfterMs: 5_000 });+ });++ it('drops the hint when a rate-limited provider stated no delay', async () => {+ twoImageProviders();+ globalThis.fetch = byHost(() => limited('17'), () => limited());+ const err = await runImageGeneration('auto', { prompt: 'a cat' }).catch(e => e);+ expect(err.status).toBe(429);+ expect(err.retryAfterMs).toBeUndefined();+ });++ it('drops the hint when the chain ends in a non-rate-limit failure', async () => {+ twoImageProviders();+ globalThis.fetch = byHost(() => limited('17'), () => new Response('boom', { status: 500 }));+ const err = await runImageGeneration('auto', { prompt: 'a cat' }).catch(e => e);+ expect(err.status).toBe(502);+ expect(err.retryAfterMs).toBeUndefined();+ });++ it('fails over past a rate-limited provider', async () => {+ twoImageProviders();+ globalThis.fetch = byHost(() => limited('17'), () => jsonResponse({ images: [{ url: 'https://x/y.png' }] }));+ await expect(runImageGeneration('auto', { prompt: 'a cat' })).resolves.toMatchObject({ platform: 'siliconflow' });+ });+it('NVIDIA: maps artifacts[].base64 → b64_json', async () => {addMedia('nvidia', 'black-forest-labs/flux.1-schnell', 'image');addKey('nvidia');diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 8a4a630b..6d7204c3 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -5,7 +5,8 @@ import { z } from 'zod';import type { ChatMessage, ChatToolCall, TokenUsage } from '@freellmapi/shared/types.js';import { type RouteResult, type ResolvedChain, type ChainRow, routeRequest, resolveRoutingChain, resolveModelGroupCandidates, resolveStickyPreference, hasEnabledVisionModel, hasEnabledToolsModel, routingReserveTokens } from '../services/router.js';import { secondsUntilNextMonth } from '../services/key-budget.js';-import { runEmbeddings, embeddingsRetryAfterSec, EmbeddingsError } from '../services/embeddings.js';+import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';+import { retryAfterSeconds } from '../lib/retry-hint.js';import { runImageGeneration, runVideoGeneration, runSpeech, runTranscription, MediaError, MAX_TRANSCRIPTION_BYTES } from '../services/media.js';import multer from 'multer';import { getDb } from '../db/index.js';@@ -691,14 +692,6 @@ proxyRouter.post('/embeddings', async (req: Request, res: Response) => {} catch (err: any) {const status = err instanceof EmbeddingsError ? err.status : 502;const code = err instanceof EmbeddingsError ? inferenceBudgetCode(err, res) : {};- // Relay the chain's back-off only when every provider was rate limited- // (soonest stated Retry-After, or the budget reset) — overriding the- // month-long budget header inferenceBudgetCode set when a sibling comes- // back sooner. A lone upstream 429 the chain failed over from never does.- if (err instanceof EmbeddingsError) {- const retrySec = embeddingsRetryAfterSec(err);- if (retrySec !== undefined) res.setHeader('Retry-After', retrySec);- }const type = status === 400 ? 'invalid_request_error' : status === 429 ? 'rate_limit_error' : 'server_error';res.status(status).json({ error: { message: `embedding error: ${err?.message ?? 'unknown'}`, type, ...code } });}@@ -716,8 +709,12 @@ const ImageBody = z.object({response_format: z.enum(['url', 'b64_json']).optional(),});-function inferenceBudgetCode(error: { code?: string }, res: Response): { code?: string } {- if (error.code === 'quota_exceeded') res.setHeader('Retry-After', secondsUntilNextMonth());+function inferenceBudgetCode(error: { code?: string; retryAfterMs?: number }, res: Response): { code?: string } {+ // retryAfterMs is set by the embeddings/media services only when the whole+ // chain was rate limited (soonest stated back-off, budget resets included),+ // so it wins over the month-long budget reset when a sibling returns sooner.+ if (error.retryAfterMs !== undefined) res.setHeader('Retry-After', retryAfterSeconds(error.retryAfterMs));+ else if (error.code === 'quota_exceeded') res.setHeader('Retry-After', secondsUntilNextMonth());return error.code ? { code: error.code } : {};}diff --git a/server/src/services/media.ts b/server/src/services/media.tsindex 5353e648..d55db584 100644--- a/server/src/services/media.ts+++ b/server/src/services/media.ts@@ -8,6 +8,9 @@// published catalog and arrive via catalog-sync (premium on the live tier within// ~12h, free once each model is 30 days old) — never seeded by migrations.import { getDb } from '../db/index.js';+import { parseRetryAfterMs } from '../providers/base.js';+import { RetryHintTracker } from '../lib/retry-hint.js';+import { secondsUntilNextMonth } from './key-budget.js';import { getClientContext } from '../lib/client-context.js';import { reserveProviderCredential } from './provider-credential.js';import { proxyFetch } from '../lib/proxy.js';@@ -56,10 +59,15 @@ export class MediaError extends Error {status: number;/** Optional machine-readable error code surfaced in the OpenAI-shaped body. */code?: string;- constructor(message: string, status: number, code?: string) {+ /** Back-off the upstream provider stated (`Retry-After` header), in+ * milliseconds — relayed to the client so an SDK sleeps the stated amount+ * instead of hammering the chain, mirroring the embeddings path. */+ retryAfterMs?: number;+ constructor(message: string, status: number, code?: string, retryAfterMs?: number) {super(message);this.status = status;this.code = code;+ this.retryAfterMs = retryAfterMs;}}@@ -237,7 +245,12 @@ async function mediaFetch();if (!r.ok) {const body = await r.text().catch(() => '');- throw new MediaError(`${platform} ${r.status}: ${body.slice(0, 200)}`, r.status);+ throw new MediaError(+ `${platform} ${r.status}: ${body.slice(0, 200)}`,+ r.status,+ undefined,+ parseRetryAfterMs(r.headers?.get('retry-after') ?? null),+ );}return r;}@@ -725,7 +738,7 @@ function logMedia(row: Pick<MediaModelRow, 'platform' | 'model_id' | 'modality'>}}-function chainError(modality: MediaModality, lastError: MediaError | null): MediaError {+function chainError(modality: MediaModality, lastError: MediaError | null, hints: RetryHintTracker): MediaError {// Only statuses the CALLER can act on are passed through. 400 (bad prompt,// duration the model does not accept) and 413 (upload too large) describe// the caller's own request, and 429 is the long-standing rate-limit signal.@@ -740,6 +753,10 @@ function chainError(modality: MediaModality, lastError: MediaError | null): Medi`All ${modality} providers failed${lastError ? ` (last: ${lastError.message.slice(0, 160)})` : ' (no usable keys)'}.`,status,lastError?.code,+ // A 429 from one provider is failed over, so its Retry-After is only the+ // client's business when EVERY provider was rate limited with a stated+ // delay — then the soonest one (same rule as the chat exhaustion path).+ status === 429 ? hints.retryAfterMs() : undefined,);}@@ -747,12 +764,17 @@ function chainError(modality: MediaModality, lastError: MediaError | null): Mediexport async function runImageGeneration(model: string | undefined, params: ImageParams): Promise<ImageResult> {const chain = resolveMediaChain(model, 'image');let lastError: MediaError | null = null;+ // Only a chain rate limited end to end earns a Retry-After (the soonest).+ const hints = new RetryHintTracker();for (const row of chain) {const { credential, budgetBlocked } = KEYLESS_CAPABLE.has(row.platform)? { credential: { id: null, key: null, baseUrl: null, release: () => {} }, budgetBlocked: false }: reserveProviderCredential(row, 0);if (!credential) {- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ if (budgetBlocked) {+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ hints.record(429, secondsUntilNextMonth() * 1000);+ }continue;}const started = Date.now();@@ -767,11 +789,12 @@ export async function runImageGeneration(model: string | undefined, params: Imagconst e = err instanceof MediaError ? err : new MediaError(String(err?.message ?? err), 502);logMedia(row, credential.id, 'error', Date.now() - started, e.message.slice(0, 300));lastError = e;+ hints.record(e.status, e.retryAfterMs);} finally {credential.release();}}- throw chainError('image', lastError);+ throw chainError('image', lastError, hints);}/** Generate a video, failing over across catalogued text-to-video providers.@@ -784,12 +807,17 @@ export async function runVideoGeneration(): Promise<VideoResult> {const chain = resolveMediaChain(model, 'video');let lastError: MediaError | null = null;+ // Only a chain rate limited end to end earns a Retry-After (the soonest).+ const hints = new RetryHintTracker();for (const row of chain) {throwIfClientGone(clientSignal);const { credential, budgetBlocked } = reserveProviderCredential(row, 0,keyId => isOnCooldown(row.platform, row.model_id, keyId));if (!credential) {- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ if (budgetBlocked) {+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ hints.record(429, secondsUntilNextMonth() * 1000);+ }continue;}const started = Date.now();@@ -803,13 +831,14 @@ export async function runVideoGeneration(logMedia(row, credential.id, 'error', Date.now() - started, e.message.slice(0, 300));if (e.status === 429 && credential.id != null) setCooldown(row.platform, row.model_id, credential.id);lastError = e;+ hints.record(e.status, e.retryAfterMs);// A caller that hung up gets no second generation charged to its account.throwIfClientGone(clientSignal);} finally {credential.release();}}- throw chainError('video', lastError);+ throw chainError('video', lastError, hints);}// ---------------------------------------------------------------------------@@ -1056,11 +1085,16 @@ export async function runTranscription(model: string | undefined, p: Transcriptithrow new MediaError(`Audio file exceeds the ${maxMb} MB provider upload limit.`, 413);}let lastError: MediaError | null = null;+ // Only a chain rate limited end to end earns a Retry-After (the soonest).+ const hints = new RetryHintTracker();for (const m of usable) {const { credential, budgetBlocked } = reserveProviderCredential({ platform: m.platform, key_id: m.keyId }, 0,keyId => isOnCooldown(m.platform, m.modelId, keyId));if (!credential) {- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ if (budgetBlocked) {+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ hints.record(429, secondsUntilNextMonth() * 1000);+ }continue;}const logRow = { platform: m.platform, model_id: m.modelId, modality: 'transcription' as const };@@ -1079,23 +1113,29 @@ export async function runTranscription(model: string | undefined, p: TranscriptisetCooldown(m.platform, m.modelId, credential.id);}lastError = e;+ hints.record(e.status, e.retryAfterMs);} finally {credential.release();}}- throw chainError('transcription', lastError);+ throw chainError('transcription', lastError, hints);}/** Synthesize speech, failing over across providers serving the modality. */export async function runSpeech(model: string | undefined, params: SpeechParams): Promise<SpeechResult> {const chain = resolveMediaChain(model, 'audio');let lastError: MediaError | null = null;+ // Only a chain rate limited end to end earns a Retry-After (the soonest).+ const hints = new RetryHintTracker();for (const row of chain) {const { credential, budgetBlocked } = KEYLESS_CAPABLE.has(row.platform)? { credential: { id: null, key: null, baseUrl: null, release: () => {} }, budgetBlocked: false }: reserveProviderCredential(row, 0);if (!credential) {- if (budgetBlocked) lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ if (budgetBlocked) {+ lastError = new MediaError('Monthly key budget exhausted', 429, 'quota_exceeded');+ hints.record(429, secondsUntilNextMonth() * 1000);+ }continue;}const started = Date.now();@@ -1108,9 +1148,10 @@ export async function runSpeech(model: string | undefined, params: SpeechParams)const e = err instanceof MediaError ? err : new MediaError(String(err?.message ?? err), 502);logMedia(row, credential.id, 'error', Date.now() - started, e.message.slice(0, 300));lastError = e;+ hints.record(e.status, e.retryAfterMs);} finally {credential.release();}}- throw chainError('audio', lastError);+ throw chainError('audio', lastError, hints);}