Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Fix/cloudflare account daily quota (#1622)

Vladislavc2cf3adbaa49
diff --git a/server/src/__tests__/services/cloudflare-daily-quota.test.ts b/server/src/__tests__/services/cloudflare-daily-quota.test.ts
new file mode 100644
index 00000000..5a266755
--- /dev/null
+++ b/server/src/__tests__/services/cloudflare-daily-quota.test.ts
@@ -0,0 +1,148 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { benchCloudflareAccount, cloudflareAccountId, isCloudflareNeuronsExhausted } from '../../services/cloudflare-daily-quota.js';
+import { clearCooldownsForKey, getActiveCooldownsForKeys, isOnCooldown, clearAllCooldowns } from '../../services/ratelimit.js';
+
+// Cloudflare Workers AI's free 10K neurons/day is shared by every model and
+// every token of one ACCOUNT, so a daily neurons 429 benches the whole account
+// (as ordinary, visible, clearable cooldown rows) while a different Cloudflare
+// account stays available for fallback.
+
+const NEURONS_MSG = 'Cloudflare API error 429: you have used up your daily free allocation of 10,000 neurons, please upgrade to Cloudflare\'s Workers Paid plan';
+const DAILY = { status: 429, message: NEURONS_MSG };
+const HOUR = 60 * 60_000;
+
+function insertKey(value: string): number {
+ const { encrypted, iv, authTag } = encrypt(value);
+ return Number(getDb().prepare(
+ "INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('cloudflare', 'test', ?, ?, ?, 'healthy', 1)",
+ ).run(encrypted, iv, authTag).lastInsertRowid);
+}
+
+function cloudflareModels(): string[] {
+ return (getDb().prepare("SELECT model_id FROM models WHERE platform = 'cloudflare' AND enabled = 1").all() as { model_id: string }[])
+ .map(r => r.model_id);
+}
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+beforeEach(() => {
+ getDb().prepare("DELETE FROM api_keys WHERE platform = 'cloudflare'").run();
+ clearAllCooldowns();
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe('Cloudflare shared daily neurons allowance', () => {
+ it('parses the account id and only matches a daily neurons 429', () => {
+ expect(cloudflareAccountId('ACCOUNT-A:token')).toBe('account-a');
+ expect(cloudflareAccountId('no-separator')).toBeNull();
+ expect(cloudflareAccountId(':token')).toBeNull();
+ expect(isCloudflareNeuronsExhausted(DAILY)).toBe(true);
+ expect(isCloudflareNeuronsExhausted({ status: 429, message: 'Too Many Requests' })).toBe(false);
+ expect(isCloudflareNeuronsExhausted({ status: 402, message: NEURONS_MSG })).toBe(false);
+ });
+
+ it('benches every model on every token of the same account, not another account', () => {
+ const a1 = insertKey('account-a:token-one');
+ const a2 = insertKey('account-a:token-two');
+ const b1 = insertKey('account-b:token-three');
+ const models = cloudflareModels();
+ expect(models.length).toBeGreaterThan(1);
+
+ expect(benchCloudflareAccount('account-a:token-one', DAILY, HOUR, 'authoritative')).toBe(models.length * 2);
+
+ for (const m of models) {
+ expect(isOnCooldown('cloudflare', m, a1)).toBe(true);
+ expect(isOnCooldown('cloudflare', m, a2)).toBe(true);
+ expect(isOnCooldown('cloudflare', m, b1)).toBe(false);
+ }
+ });
+
+ it('ignores transient 429s and other errors', () => {
+ const a2 = insertKey('account-a:token-two');
+ expect(benchCloudflareAccount('account-a:token-one', { status: 429, message: 'Too Many Requests' }, HOUR, 'heuristic')).toBe(0);
+ expect(benchCloudflareAccount('account-a:token-one', { status: 500, message: NEURONS_MSG }, HOUR, 'heuristic')).toBe(0);
+ expect(getActiveCooldownsForKeys([a2]).get(a2) ?? []).toEqual([]);
+ });
+
+ it('uses the duration it is given, shows on the key, and Clear cooldowns lifts it', () => {
+ const a2 = insertKey('account-a:token-two');
+ const before = Date.now();
+ benchCloudflareAccount('account-a:token-one', DAILY, HOUR, 'authoritative');
+
+ const active = getActiveCooldownsForKeys([a2]).get(a2) ?? [];
+ expect(active.length).toBe(cloudflareModels().length);
+ for (const c of active) {
+ expect(c.expiresAtMs).toBeGreaterThanOrEqual(before + HOUR);
+ expect(c.expiresAtMs).toBeLessThanOrEqual(Date.now() + HOUR);
+ }
+
+ expect(clearCooldownsForKey(a2)).toBe(active.length);
+ expect(getActiveCooldownsForKeys([a2]).get(a2) ?? []).toEqual([]);
+ });
+});
+
+describe('Cloudflare account bench through /v1/chat/completions', () => {
+ let app: Express;
+
+ async function post(body: unknown) {
+ const server = app.listen(0, '127.0.0.1');
+ if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
+ const addr = server.address() as { port: number };
+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getUnifiedApiKey()}` },
+ body: JSON.stringify(body),
+ });
+ const json = await res.json().catch(() => null);
+ server.close();
+ return { status: res.status, body: json };
+ }
+
+ beforeAll(() => {
+ app = createApp();
+ });
+
+ it('a neurons 429 on account A benches all of A and the request is served by account B', async () => {
+ const a1 = insertKey('account-a:token-one');
+ const a2 = insertKey('account-a:token-two');
+ const b1 = insertKey('account-b:token-three');
+ const model = cloudflareModels()[0];
+ const realFetch = globalThis.fetch;
+ const upstream: string[] = [];
+ vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => {
+ const url = String(input instanceof Request ? input.url : input);
+ if (!url.startsWith('https://api.cloudflare.com/')) return realFetch(input, init);
+ upstream.push(url);
+ if (url.includes('/accounts/account-a/')) {
+ return new Response(JSON.stringify({ errors: [{ message: 'you have used up your daily free allocation of 10,000 neurons' }] }), { status: 429 });
+ }
+ return new Response(JSON.stringify({
+ id: 'x', object: 'chat.completion', created: 0, model,
+ choices: [{ index: 0, message: { role: 'assistant', content: 'from account b' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 3, completion_tokens: 3, total_tokens: 6 },
+ }), { status: 200, headers: { 'Content-Type': 'application/json' } });
+ });
+
+ const { status, body } = await post({ model, messages: [{ role: 'user', content: 'hi' }] });
+ expect(status).toBe(200);
+ expect(body.choices[0].message.content).toBe('from account b');
+ // Account A was tried exactly once: its sibling token was benched with it.
+ expect(upstream.filter(u => u.includes('/accounts/account-a/'))).toHaveLength(1);
+
+ for (const m of cloudflareModels()) {
+ expect(isOnCooldown('cloudflare', m, a1)).toBe(true);
+ expect(isOnCooldown('cloudflare', m, a2)).toBe(true);
+ expect(isOnCooldown('cloudflare', m, b1)).toBe(false);
+ }
+ });
+});
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 9e2f32e5..2b9f9770 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -1,4 +1,5 @@
import { nextMonthResetAt } from '../services/key-budget.js';
+import { benchCloudflareAccount } from '../services/cloudflare-daily-quota.js';
// One shared provider retry/fallback loop for every OpenAI-, Responses- and
// Anthropic-shaped chat surface (routes/proxy.ts legacy /completions and
// /chat/completions, routes/responses.ts, routes/anthropic.ts). Each surface
@@ -541,6 +542,9 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
if (consumeSkipBenchExemption(route, err)) return true;
const decision = cooldownDecisionForError(route, err);
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
+ // Workers AI's daily neurons pool is per ACCOUNT: bench every model on every
+ // token of the same account with this route's own duration and provenance.
+ if (route.platform === 'cloudflare') benchCloudflareAccount(route.apiKey, err, decision.durationMs, decision.source);
// A truncated stream (200 + partial SSE, no [DONE]/finish_reason) on the
// SAME route repeatedly is a sick edge, not bad luck — but a single one is
// common enough on flaky free gateways that benching on first sight would
diff --git a/server/src/services/cloudflare-daily-quota.ts b/server/src/services/cloudflare-daily-quota.ts
new file mode 100644
index 00000000..cccfedd7
--- /dev/null
+++ b/server/src/services/cloudflare-daily-quota.ts
@@ -0,0 +1,71 @@
+import { getDb } from '../db/index.js';
+import { decrypt } from '../lib/crypto.js';
+import { setCooldown, type CooldownSource } from './ratelimit.js';
+
+// Cloudflare Workers AI meters its free 10K neurons/day per ACCOUNT: every
+// @cf/* model and every API token of one account draw on the same pool. The
+// normal daily-quota bench only parks the one model+key that hit the 429, so
+// the router kept walking the account's sibling models and tokens into the
+// same "daily free allocation" wall all day.
+//
+// When that 429 arrives we bench every enabled Cloudflare model on every key
+// of the same account, with the same duration and provenance the failing route
+// just got (cooldownDecisionForError: next UTC midnight, an explicit
+// Retry-After, the operator's ceiling). They are ordinary cooldown rows, so
+// they show on the Keys page, "Clear cooldowns" lifts them, and keys of a
+// different Cloudflare account stay available for fallback.
+
+/** Account id of a Cloudflare key ("account_id:api_token"), or null. */
+export function cloudflareAccountId(apiKey: string): string | null {
+ const colon = apiKey.indexOf(':');
+ if (colon <= 0 || colon === apiKey.length - 1) return null;
+ return apiKey.slice(0, colon).toLowerCase();
+}
+
+/** Only a Workers AI daily *neurons* 429 means the whole account is spent. */
+export function isCloudflareNeuronsExhausted(err: { status?: number; message?: string } | null | undefined): boolean {
+ const msg = err?.message ?? '';
+ return err?.status === 429 && /daily free allocation/i.test(msg) && /neurons/i.test(msg);
+}
+
+/**
+ * Bench the whole Cloudflare account behind `apiKey` after a daily neurons
+ * 429. Returns the number of model+key cooldowns written (0 when the error is
+ * not a neurons exhaustion or the key has no account id).
+ */
+export function benchCloudflareAccount(
+ apiKey: string,
+ err: { status?: number; message?: string } | null | undefined,
+ durationMs: number,
+ source: CooldownSource,
+): number {
+ if (!isCloudflareNeuronsExhausted(err)) return 0;
+ const account = cloudflareAccountId(apiKey);
+ if (!account) return 0;
+
+ const db = getDb();
+ const keys = db.prepare(
+ "SELECT id, encrypted_key, iv, auth_tag FROM api_keys WHERE platform = 'cloudflare' AND enabled = 1",
+ ).all() as { id: number; encrypted_key: string; iv: string; auth_tag: string }[];
+ const keyIds = keys.filter(k => {
+ try {
+ return cloudflareAccountId(decrypt(k.encrypted_key, k.iv, k.auth_tag)) === account;
+ } catch {
+ return false; // undecryptable row: not ours to bench
+ }
+ }).map(k => k.id);
+ if (keyIds.length === 0) return 0;
+
+ const models = db.prepare(
+ "SELECT model_id FROM models WHERE platform = 'cloudflare' AND enabled = 1",
+ ).all() as { model_id: string }[];
+
+ let benched = 0;
+ for (const keyId of keyIds) {
+ for (const { model_id } of models) {
+ setCooldown('cloudflare', model_id, keyId, durationMs, source);
+ benched++;
+ }
+ }
+ return benched;
+}