7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Fix/cloudflare account daily quota (#1622)
Vladislavc2cf3adbaa49
diff --git a/server/src/__tests__/services/cloudflare-daily-quota.test.ts b/server/src/__tests__/services/cloudflare-daily-quota.test.tsnew file mode 100644index 00000000..5a266755--- /dev/null+++ b/server/src/__tests__/services/cloudflare-daily-quota.test.ts@@ -0,0 +1,148 @@+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { benchCloudflareAccount, cloudflareAccountId, isCloudflareNeuronsExhausted } from '../../services/cloudflare-daily-quota.js';+import { clearCooldownsForKey, getActiveCooldownsForKeys, isOnCooldown, clearAllCooldowns } from '../../services/ratelimit.js';++// Cloudflare Workers AI's free 10K neurons/day is shared by every model and+// every token of one ACCOUNT, so a daily neurons 429 benches the whole account+// (as ordinary, visible, clearable cooldown rows) while a different Cloudflare+// account stays available for fallback.++const NEURONS_MSG = 'Cloudflare API error 429: you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare\'s Workers Paid plan';+const DAILY = { status: 429, message: NEURONS_MSG };+const HOUR = 60 * 60_000;++function insertKey(value: string): number {+ const { encrypted, iv, authTag } = encrypt(value);+ return Number(getDb().prepare(+ "INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('cloudflare', 'test', ?, ?, ?, 'healthy', 1)",+ ).run(encrypted, iv, authTag).lastInsertRowid);+}++function cloudflareModels(): string[] {+ return (getDb().prepare("SELECT model_id FROM models WHERE platform = 'cloudflare' AND enabled = 1").all() as { model_id: string }[])+ .map(r => r.model_id);+}++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++beforeEach(() => {+ getDb().prepare("DELETE FROM api_keys WHERE platform = 'cloudflare'").run();+ clearAllCooldowns();+});++afterEach(() => {+ vi.restoreAllMocks();+});++describe('Cloudflare shared daily neurons allowance', () => {+ it('parses the account id and only matches a daily neurons 429', () => {+ expect(cloudflareAccountId('ACCOUNT-A:token')).toBe('account-a');+ expect(cloudflareAccountId('no-separator')).toBeNull();+ expect(cloudflareAccountId(':token')).toBeNull();+ expect(isCloudflareNeuronsExhausted(DAILY)).toBe(true);+ expect(isCloudflareNeuronsExhausted({ status: 429, message: 'Too Many Requests' })).toBe(false);+ expect(isCloudflareNeuronsExhausted({ status: 402, message: NEURONS_MSG })).toBe(false);+ });++ it('benches every model on every token of the same account, not another account', () => {+ const a1 = insertKey('account-a:token-one');+ const a2 = insertKey('account-a:token-two');+ const b1 = insertKey('account-b:token-three');+ const models = cloudflareModels();+ expect(models.length).toBeGreaterThan(1);++ expect(benchCloudflareAccount('account-a:token-one', DAILY, HOUR, 'authoritative')).toBe(models.length * 2);++ for (const m of models) {+ expect(isOnCooldown('cloudflare', m, a1)).toBe(true);+ expect(isOnCooldown('cloudflare', m, a2)).toBe(true);+ expect(isOnCooldown('cloudflare', m, b1)).toBe(false);+ }+ });++ it('ignores transient 429s and other errors', () => {+ const a2 = insertKey('account-a:token-two');+ expect(benchCloudflareAccount('account-a:token-one', { status: 429, message: 'Too Many Requests' }, HOUR, 'heuristic')).toBe(0);+ expect(benchCloudflareAccount('account-a:token-one', { status: 500, message: NEURONS_MSG }, HOUR, 'heuristic')).toBe(0);+ expect(getActiveCooldownsForKeys([a2]).get(a2) ?? []).toEqual([]);+ });++ it('uses the duration it is given, shows on the key, and Clear cooldowns lifts it', () => {+ const a2 = insertKey('account-a:token-two');+ const before = Date.now();+ benchCloudflareAccount('account-a:token-one', DAILY, HOUR, 'authoritative');++ const active = getActiveCooldownsForKeys([a2]).get(a2) ?? [];+ expect(active.length).toBe(cloudflareModels().length);+ for (const c of active) {+ expect(c.expiresAtMs).toBeGreaterThanOrEqual(before + HOUR);+ expect(c.expiresAtMs).toBeLessThanOrEqual(Date.now() + HOUR);+ }++ expect(clearCooldownsForKey(a2)).toBe(active.length);+ expect(getActiveCooldownsForKeys([a2]).get(a2) ?? []).toEqual([]);+ });+});++describe('Cloudflare account bench through /v1/chat/completions', () => {+ let app: Express;++ async function post(body: unknown) {+ const server = app.listen(0, '127.0.0.1');+ if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));+ const addr = server.address() as { port: number };+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getUnifiedApiKey()}` },+ body: JSON.stringify(body),+ });+ const json = await res.json().catch(() => null);+ server.close();+ return { status: res.status, body: json };+ }++ beforeAll(() => {+ app = createApp();+ });++ it('a neurons 429 on account A benches all of A and the request is served by account B', async () => {+ const a1 = insertKey('account-a:token-one');+ const a2 = insertKey('account-a:token-two');+ const b1 = insertKey('account-b:token-three');+ const model = cloudflareModels()[0];+ const realFetch = globalThis.fetch;+ const upstream: string[] = [];+ vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => {+ const url = String(input instanceof Request ? input.url : input);+ if (!url.startsWith('https://api.cloudflare.com/')) return realFetch(input, init);+ upstream.push(url);+ if (url.includes('/accounts/account-a/')) {+ return new Response(JSON.stringify({ errors: [{ message: 'you have used up your daily free allocation of 10,000 neurons' }] }), { status: 429 });+ }+ return new Response(JSON.stringify({+ id: 'x', object: 'chat.completion', created: 0, model,+ choices: [{ index: 0, message: { role: 'assistant', content: 'from account b' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 3, completion_tokens: 3, total_tokens: 6 },+ }), { status: 200, headers: { 'Content-Type': 'application/json' } });+ });++ const { status, body } = await post({ model, messages: [{ role: 'user', content: 'hi' }] });+ expect(status).toBe(200);+ expect(body.choices[0].message.content).toBe('from account b');+ // Account A was tried exactly once: its sibling token was benched with it.+ expect(upstream.filter(u => u.includes('/accounts/account-a/'))).toHaveLength(1);++ for (const m of cloudflareModels()) {+ expect(isOnCooldown('cloudflare', m, a1)).toBe(true);+ expect(isOnCooldown('cloudflare', m, a2)).toBe(true);+ expect(isOnCooldown('cloudflare', m, b1)).toBe(false);+ }+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 9e2f32e5..2b9f9770 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -1,4 +1,5 @@import { nextMonthResetAt } from '../services/key-budget.js';+import { benchCloudflareAccount } from '../services/cloudflare-daily-quota.js';// One shared provider retry/fallback loop for every OpenAI-, Responses- and// Anthropic-shaped chat surface (routes/proxy.ts legacy /completions and// /chat/completions, routes/responses.ts, routes/anthropic.ts). Each surface@@ -541,6 +542,9 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fallif (consumeSkipBenchExemption(route, err)) return true;const decision = cooldownDecisionForError(route, err);setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);+ // Workers AI's daily neurons pool is per ACCOUNT: bench every model on every+ // token of the same account with this route's own duration and provenance.+ if (route.platform === 'cloudflare') benchCloudflareAccount(route.apiKey, err, decision.durationMs, decision.source);// A truncated stream (200 + partial SSE, no [DONE]/finish_reason) on the// SAME route repeatedly is a sick edge, not bad luck — but a single one is// common enough on flaky free gateways that benching on first sight woulddiff --git a/server/src/services/cloudflare-daily-quota.ts b/server/src/services/cloudflare-daily-quota.tsnew file mode 100644index 00000000..cccfedd7--- /dev/null+++ b/server/src/services/cloudflare-daily-quota.ts@@ -0,0 +1,71 @@+import { getDb } from '../db/index.js';+import { decrypt } from '../lib/crypto.js';+import { setCooldown, type CooldownSource } from './ratelimit.js';++// Cloudflare Workers AI meters its free 10K neurons/day per ACCOUNT: every+// @cf/* model and every API token of one account draw on the same pool. The+// normal daily-quota bench only parks the one model+key that hit the 429, so+// the router kept walking the account's sibling models and tokens into the+// same "daily free allocation" wall all day.+//+// When that 429 arrives we bench every enabled Cloudflare model on every key+// of the same account, with the same duration and provenance the failing route+// just got (cooldownDecisionForError: next UTC midnight, an explicit+// Retry-After, the operator's ceiling). They are ordinary cooldown rows, so+// they show on the Keys page, "Clear cooldowns" lifts them, and keys of a+// different Cloudflare account stay available for fallback.++/** Account id of a Cloudflare key ("account_id:api_token"), or null. */+export function cloudflareAccountId(apiKey: string): string | null {+ const colon = apiKey.indexOf(':');+ if (colon <= 0 || colon === apiKey.length - 1) return null;+ return apiKey.slice(0, colon).toLowerCase();+}++/** Only a Workers AI daily *neurons* 429 means the whole account is spent. */+export function isCloudflareNeuronsExhausted(err: { status?: number; message?: string } | null | undefined): boolean {+ const msg = err?.message ?? '';+ return err?.status === 429 && /daily free allocation/i.test(msg) && /neurons/i.test(msg);+}++/**+ * Bench the whole Cloudflare account behind `apiKey` after a daily neurons+ * 429. Returns the number of model+key cooldowns written (0 when the error is+ * not a neurons exhaustion or the key has no account id).+ */+export function benchCloudflareAccount(+ apiKey: string,+ err: { status?: number; message?: string } | null | undefined,+ durationMs: number,+ source: CooldownSource,+): number {+ if (!isCloudflareNeuronsExhausted(err)) return 0;+ const account = cloudflareAccountId(apiKey);+ if (!account) return 0;++ const db = getDb();+ const keys = db.prepare(+ "SELECT id, encrypted_key, iv, auth_tag FROM api_keys WHERE platform = 'cloudflare' AND enabled = 1",+ ).all() as { id: number; encrypted_key: string; iv: string; auth_tag: string }[];+ const keyIds = keys.filter(k => {+ try {+ return cloudflareAccountId(decrypt(k.encrypted_key, k.iv, k.auth_tag)) === account;+ } catch {+ return false; // undecryptable row: not ours to bench+ }+ }).map(k => k.id);+ if (keyIds.length === 0) return 0;++ const models = db.prepare(+ "SELECT model_id FROM models WHERE platform = 'cloudflare' AND enabled = 1",+ ).all() as { model_id: string }[];++ let benched = 0;+ for (const keyId of keyIds) {+ for (const { model_id } of models) {+ setCooldown('cloudflare', model_id, keyId, durationMs, source);+ benched++;+ }+ }+ return benched;+}