7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix: default output floor on Cloudflare + pace the health probe pass (#553) (#644)
Tashfeen75f0498015c9
diff --git a/server/src/__tests__/providers/default-max-tokens.test.ts b/server/src/__tests__/providers/default-max-tokens.test.tsnew file mode 100644index 00000000..2494683f--- /dev/null+++ b/server/src/__tests__/providers/default-max-tokens.test.ts@@ -0,0 +1,123 @@+import { describe, it, expect, vi, afterEach } from 'vitest';+import { CloudflareProvider } from '../../providers/cloudflare.js';+import { OpenAICompatProvider } from '../../providers/openai-compat.js';+import { defaultMaxTokensFor, resolveMaxTokens } from '../../lib/sampling-params.js';++// #553: Workers AI applies its own (small) server-side max_tokens when the+// request omits one, so a client that never sets max_tokens — the OpenAI SDK+// default — gets a response cut off mid-sentence a couple of seconds in. The+// gateway therefore sends a generous per-platform floor when, and only when,+// the client omitted the parameter.++function jsonResponse(): any {+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-1',+ object: 'chat.completion',+ created: 1,+ model: 'm',+ choices: [{ index: 0, message: { role: 'assistant', content: 'hi' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }),+ headers: new Headers(),+ };+}++function sseResponse(): any {+ const stream = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new TextEncoder().encode('data: [DONE]\n\n'));+ controller.close();+ },+ });+ return { ok: true, body: stream, headers: new Headers() };+}++function captureBody(response: () => any): { body: () => any } {+ let captured: any = null;+ vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {+ captured = JSON.parse((init as any).body);+ return response() as any;+ });+ return { body: () => captured };+}++afterEach(() => {+ vi.restoreAllMocks();+});++describe('resolveMaxTokens', () => {+ it('applies the platform default only when the client omitted max_tokens', () => {+ const cfDefault = defaultMaxTokensFor('cloudflare');+ expect(cfDefault).toBeGreaterThan(0);++ expect(resolveMaxTokens('cloudflare', undefined)).toBe(cfDefault);+ expect(resolveMaxTokens('cloudflare', 64)).toBe(64);+ // A client value above the default is never clamped down.+ expect(resolveMaxTokens('cloudflare', 100000)).toBe(100000);+ });++ it('leaves platforms without a default untouched', () => {+ expect(defaultMaxTokensFor('groq')).toBeUndefined();+ expect(resolveMaxTokens('groq', undefined)).toBeUndefined();+ expect(resolveMaxTokens('groq', 32)).toBe(32);+ expect(resolveMaxTokens('unknown-platform', undefined)).toBeUndefined();+ });+});++describe('Cloudflare default max_tokens', () => {+ const provider = new CloudflareProvider();+ const messages = [{ role: 'user' as const, content: 'Hi' }];++ it('sends the default when the client omitted max_tokens (non-streaming)', async () => {+ const cap = captureBody(jsonResponse);+ await provider.chatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b');+ expect(cap.body().max_tokens).toBe(defaultMaxTokensFor('cloudflare'));+ });++ it('sends the default when the client omitted max_tokens (streaming)', async () => {+ const cap = captureBody(sseResponse);+ const gen = provider.streamChatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b');+ for await (const _chunk of gen) { /* drain */ }+ expect(cap.body().max_tokens).toBe(defaultMaxTokensFor('cloudflare'));+ });++ it('passes a client-supplied max_tokens through untouched', async () => {+ const cap = captureBody(jsonResponse);+ await provider.chatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b', { max_tokens: 16 });+ expect(cap.body().max_tokens).toBe(16);+ });++ it('passes a client-supplied max_tokens larger than the default through untouched', async () => {+ const cap = captureBody(jsonResponse);+ await provider.chatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b', { max_tokens: 120000 });+ expect(cap.body().max_tokens).toBe(120000);+ });+});++describe('other platforms are unchanged', () => {+ const messages = [{ role: 'user' as const, content: 'Hi' }];++ it('omits max_tokens entirely when the client did (groq)', async () => {+ const provider = new OpenAICompatProvider({+ platform: 'groq',+ name: 'Groq',+ baseUrl: 'https://api.test.com/v1',+ });+ const cap = captureBody(jsonResponse);+ await provider.chatCompletion('sk-test', messages, 'llama-3.3-70b');+ expect(cap.body().max_tokens).toBeUndefined();+ });++ it('still forwards a client-supplied max_tokens (groq)', async () => {+ const provider = new OpenAICompatProvider({+ platform: 'groq',+ name: 'Groq',+ baseUrl: 'https://api.test.com/v1',+ });+ const cap = captureBody(jsonResponse);+ await provider.chatCompletion('sk-test', messages, 'llama-3.3-70b', { max_tokens: 77 });+ expect(cap.body().max_tokens).toBe(77);+ });+});diff --git a/server/src/__tests__/services/health-probe-pacing.test.ts b/server/src/__tests__/services/health-probe-pacing.test.tsnew file mode 100644index 00000000..aa7a8b7e--- /dev/null+++ b/server/src/__tests__/services/health-probe-pacing.test.ts@@ -0,0 +1,203 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';+import type { Scheduler } from '../../lib/scheduler.js';++const { initDb, getDb } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const {+ checkAllKeys,+ nextHealthCheckDelayMs,+ startHealthChecker,+ stopHealthChecker,+ HEALTH_CHECK_INTERVAL_MS,+ HEALTH_PASS_TIME_BUDGET_MS,+ RECENT_CHECK_SKIP_MS,+} = await import('../../services/health.js');++// #553: every enabled key was probed in raw DB order behind a concurrency-8+// pool every 5 minutes with no spacing, so a fleet with many keys on one+// provider fired them all back to back from one IP — which reads as abuse to+// the provider. The pass now interleaves providers, spaces same-provider+// probes, skips keys validated recently, and jitters its own interval.++let nextId = 20000;++function seedKey(platform: string, opts: { lastCheckedAt?: string; status?: string } = {}): number {+ const id = ++nextId;+ const enc = encrypt(`pacing-${id}`);+ getDb().prepare(`+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_checked_at)+ VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)+ `).run(id, platform, `pacing-${id}`, enc.encrypted, enc.iv, enc.authTag, opts.status ?? 'healthy', opts.lastCheckedAt ?? null);+ return id;+}++/** Fake clock driven by the injected sleep, so a pass costs no real time. */+function fakeClock() {+ const state = { t: 0 };+ return {+ now: () => state.t,+ sleep: async (ms: number) => { state.t += ms; },+ };+}++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++beforeEach(() => {+ getDb().prepare('DELETE FROM api_keys').run();+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+});++describe('health pass ordering', () => {+ it('interleaves platforms instead of probing one provider back to back', async () => {+ // Inserted provider-clustered, exactly as the old raw-DB-order pass read them.+ const groq = [seedKey('groq'), seedKey('groq'), seedKey('groq')];+ const cf = [seedKey('cloudflare'), seedKey('cloudflare')];+ const mistral = [seedKey('mistral')];++ const clock = fakeClock();+ const platformOf = new Map<number, string>([+ ...groq.map(id => [id, 'groq'] as const),+ ...cf.map(id => [id, 'cloudflare'] as const),+ ...mistral.map(id => [id, 'mistral'] as const),+ ]);+ const order: string[] = [];++ await checkAllKeys({+ now: clock.now,+ sleep: clock.sleep,+ concurrency: 1,+ check: async (keyId: number) => { order.push(platformOf.get(keyId)!); },+ });++ expect(order).toEqual(['groq', 'cloudflare', 'mistral', 'groq', 'cloudflare', 'groq']);+ });++ it('spaces consecutive probes of the same provider', async () => {+ const ids = [seedKey('groq'), seedKey('groq'), seedKey('groq'), seedKey('cloudflare')];+ const clock = fakeClock();+ const spacingMs = 1000;+ const startedAt = new Map<number, number>();++ await checkAllKeys({+ now: clock.now,+ sleep: clock.sleep,+ concurrency: 1,+ minSpacingMs: spacingMs,+ check: async (keyId: number) => { startedAt.set(keyId, clock.now()); },+ });++ const groqTimes = ids.slice(0, 3).map(id => startedAt.get(id)!);+ expect(groqTimes).toHaveLength(3);+ for (let i = 1; i < groqTimes.length; i++) {+ expect(groqTimes[i]! - groqTimes[i - 1]!).toBeGreaterThanOrEqual(spacingMs);+ }+ // The other provider is not held behind groq's spacing.+ expect(startedAt.get(ids[3]!)).toBe(0);+ });++ it('keeps the whole pass inside its time budget even for a big single-provider fleet', async () => {+ for (let i = 0; i < 400; i++) seedKey('groq');++ const clock = fakeClock();+ await checkAllKeys({+ now: clock.now,+ sleep: clock.sleep,+ concurrency: 8,+ check: async () => {},+ });++ expect(clock.now()).toBeLessThanOrEqual(HEALTH_PASS_TIME_BUDGET_MS);+ expect(HEALTH_PASS_TIME_BUDGET_MS).toBeLessThan(HEALTH_CHECK_INTERVAL_MS);+ });+});++describe('recently-validated keys', () => {+ it('skips keys checked inside the recency window', async () => {+ const fresh = seedKey('groq');+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now') WHERE id = ?").run(fresh);+ const stale = seedKey('groq');+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now', '-30 minutes') WHERE id = ?").run(stale);+ const never = seedKey('groq');++ const clock = fakeClock();+ const checked: number[] = [];+ const result = await checkAllKeys({+ now: clock.now,+ sleep: clock.sleep,+ concurrency: 1,+ check: async (keyId: number) => { checked.push(keyId); },+ });++ expect(checked.sort()).toEqual([stale, never].sort());+ expect(result.skippedKeyIds).toEqual([fresh]);+ expect(RECENT_CHECK_SKIP_MS).toBeLessThan(HEALTH_CHECK_INTERVAL_MS);+ });++ it('never skips a key sitting at status=error', async () => {+ const errored = seedKey('groq', { status: 'error' });+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now') WHERE id = ?").run(errored);++ const clock = fakeClock();+ const checked: number[] = [];+ await checkAllKeys({+ now: clock.now, sleep: clock.sleep, concurrency: 1,+ check: async (keyId: number) => { checked.push(keyId); },+ });++ expect(checked).toEqual([errored]);+ });++ it('force ignores both the recency skip and the spacing (manual / post-wake pass)', async () => {+ const a = seedKey('groq');+ const b = seedKey('groq');+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now')").run();++ const clock = fakeClock();+ const checked: number[] = [];+ const result = await checkAllKeys({+ now: clock.now, sleep: clock.sleep, concurrency: 1, force: true,+ check: async (keyId: number) => { checked.push(keyId); },+ });++ expect(checked.sort()).toEqual([a, b].sort());+ expect(result.skippedKeyIds).toEqual([]);+ expect(clock.now()).toBe(0);+ });+});++describe('interval jitter', () => {+ it('spreads the next pass around the base interval', () => {+ expect(nextHealthCheckDelayMs(() => 0)).toBeLessThan(HEALTH_CHECK_INTERVAL_MS);+ expect(nextHealthCheckDelayMs(() => 0.999)).toBeGreaterThan(HEALTH_CHECK_INTERVAL_MS);+ for (const r of [0, 0.25, 0.5, 0.75, 0.999]) {+ const delay = nextHealthCheckDelayMs(() => r);+ expect(delay).toBeGreaterThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 0.5);+ expect(delay).toBeLessThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 1.5);+ // A jittered pass must never start before the recency window has closed,+ // or the skip filter would swallow whole passes.+ expect(delay).toBeGreaterThan(RECENT_CHECK_SKIP_MS);+ }+ });++ it('reschedules itself with a fresh delay after each pass', async () => {+ const scheduled: { ms: number; fn: () => void | Promise<void> }[] = [];+ const scheduler: Scheduler = {+ every: () => vi.fn(),+ after(ms, fn) { scheduled.push({ ms, fn }); return vi.fn(); },+ };++ startHealthChecker(scheduler);+ expect(scheduled).toHaveLength(1);+ await scheduled[0]!.fn();+ expect(scheduled).toHaveLength(2);+ stopHealthChecker();+ await scheduled[1]!.fn();+ // Stopped: the pass must not queue another timer.+ expect(scheduled).toHaveLength(2);+ });+});diff --git a/server/src/__tests__/services/health-scheduler.test.ts b/server/src/__tests__/services/health-scheduler.test.tsindex 4dee9e09..b2095b92 100644--- a/server/src/__tests__/services/health-scheduler.test.ts+++ b/server/src/__tests__/services/health-scheduler.test.ts@@ -1,23 +1,23 @@import { describe, it, expect, beforeAll, afterEach, vi } from 'vitest';import { initDb } from '../../db/index.js';-import { startHealthChecker, stopHealthChecker } from '../../services/health.js';+import { startHealthChecker, stopHealthChecker, HEALTH_CHECK_INTERVAL_MS } from '../../services/health.js';import type { Scheduler } from '../../lib/scheduler.js';function makeScheduler() {- const every: { ms: number; fn: () => void | Promise<void> }[] = [];+ const after: { ms: number; fn: () => void | Promise<void> }[] = [];const cancels: ReturnType<typeof vi.fn>[] = [];const scheduler: Scheduler = {- every(ms, fn) {+ every(_ms, _fn) {+ return vi.fn();+ },+ after(ms, fn) {const cancel = vi.fn();- every.push({ ms, fn });+ after.push({ ms, fn });cancels.push(cancel);return cancel;},- after(_ms, _fn) {- return vi.fn();- },};- return { scheduler, every, cancels };+ return { scheduler, after, cancels };}describe('startHealthChecker / stopHealthChecker', () => {@@ -30,18 +30,22 @@ describe('startHealthChecker / stopHealthChecker', () => {stopHealthChecker();});- it('registers one every-5-minute job', () => {- const { scheduler, every } = makeScheduler();+ // The pass re-arms itself after each run with a fresh jittered delay (#553)+ // instead of sitting on a fixed 5-minute interval, so a fleet's probes don't+ // stay phase-locked to the same instant on every provider.+ it('arms one jittered timer around the 5-minute base interval', () => {+ const { scheduler, after } = makeScheduler();startHealthChecker(scheduler);- expect(every).toHaveLength(1);- expect(every[0].ms).toBe(5 * 60 * 1000);+ expect(after).toHaveLength(1);+ expect(after[0].ms).toBeGreaterThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 0.8);+ expect(after[0].ms).toBeLessThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 1.2);});it('is idempotent — double-start registers only one job', () => {- const { scheduler, every } = makeScheduler();+ const { scheduler, after } = makeScheduler();startHealthChecker(scheduler);startHealthChecker(scheduler);- expect(every).toHaveLength(1);+ expect(after).toHaveLength(1);});it('stop invokes the cancel handle', () => {@@ -56,14 +60,14 @@ describe('startHealthChecker / stopHealthChecker', () => {startHealthChecker(s1);stopHealthChecker();- const { scheduler: s2, every } = makeScheduler();+ const { scheduler: s2, after } = makeScheduler();startHealthChecker(s2);- expect(every).toHaveLength(1);+ expect(after).toHaveLength(1);});it('the registered job runs checkAllKeys without throwing', async () => {- const { scheduler, every } = makeScheduler();+ const { scheduler, after } = makeScheduler();startHealthChecker(scheduler);- await expect(every[0].fn()).resolves.toBeUndefined();+ await expect(after[0].fn()).resolves.toBeUndefined();});});diff --git a/server/src/index.ts b/server/src/index.tsindex 4bfc6fec..4403b8ea 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -76,7 +76,9 @@ async function main() {console.log(`[wake] resumed after ~${idle}s (${event.reason}${event.signal ? `:${event.signal}` : ''}) — flushing stale sockets, re-probing keys`);flushProxyCache();try {- await checkAllKeys();+ // Forced: every status predates the sleep, so the recency skip and+ // provider spacing of a scheduled pass would only delay the picture.+ await checkAllKeys({ force: true });} catch (err: any) {console.error(`[wake] post-wake key re-probe failed: ${err?.message ?? err}`);}diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.tsindex 4358b306..0ded96e5 100644--- a/server/src/lib/sampling-params.ts+++ b/server/src/lib/sampling-params.ts@@ -208,6 +208,15 @@ export interface PlatformParamPolicy {// supported value instead of being forwarded into a 400 (#619). Omitted =// forward whatever the client asked for.reasoningEfforts?: readonly ReasoningEffort[];+ // Output-token floor sent when — and ONLY when — the client omitted+ // max_tokens entirely (#553). Some providers apply a tiny server-side+ // default in that case and cut the answer off mid-sentence, which clients+ // experience as a broken stream rather than as a truncation. A+ // client-supplied value always wins, larger or smaller. Applied by the+ // adapters through resolveMaxTokens(), so a platform added here only takes+ // effect once its adapter routes max_tokens through that helper+ // (openai-compat + cloudflare already do).+ defaultMaxTokens?: number;}// Keyed by Platform (not string) so a typo'd platform id fails tsc instead of@@ -238,8 +247,18 @@ export const PLATFORM_PARAM_POLICIES: Partial<Record<Platform, PlatformParamPoli// `thinking` object) have no mapping there.cohere: { drop: ['top_k', 'min_p', 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs', 'reasoning_effort'] },// Workers AI's OpenAI-compat endpoint parses a known subset; send only what- // it understands.- cloudflare: { drop: ['min_p', 'logit_bias', 'logprobs', 'top_logprobs', 'reasoning_effort'] },+ // it understands. It also applies a small server-side max_tokens default+ // when the request omits one, so an SDK client that never sets max_tokens+ // (the common case) sees answers stop a couple of seconds in — reported on+ // @cf/openai/gpt-oss-120b, where the model's own reasoning trace eats the+ // default before any visible text lands (#553). 8192 is the floor we send+ // instead: generous for a chat turn plus reasoning, and small enough to fit+ // inside the smallest context window in the Workers AI catalog (24K on+ // @cf/meta/llama-3.3-70b-instruct-fp8-fast) with room left for the prompt.+ cloudflare: {+ drop: ['min_p', 'logit_bias', 'logprobs', 'top_logprobs', 'reasoning_effort'],+ defaultMaxTokens: 8192,+ },// AI Horde builds its own payload format; none of the extended set maps.aihorde: { drop: [...EXTENDED_SAMPLING_KEYS] },// Kilo's anonymous gateway 400s ("Provider returned error") whenever@@ -288,6 +307,23 @@ export function extendedBodyParams(platform: string, options: ExtendedSamplingOpreturn out;}+/** The output-token floor this platform sends for a request that carries no+ * max_tokens at all, or undefined when the provider's own default is fine. */+export function defaultMaxTokensFor(platform: string): number | undefined {+ return PLATFORM_PARAM_POLICIES[platform as Platform]?.defaultMaxTokens;+}++/**+ * The max_tokens to put on the wire for one request: whatever the client asked+ * for, or the platform's floor when the client asked for nothing (#553).+ * Never clamps — a client-set value passes through untouched in both+ * directions, and the gateway's own guardrails (token budget, routing reserve)+ * have already had their say by the time an adapter calls this.+ */+export function resolveMaxTokens(platform: string, requested: number | undefined): number | undefined {+ return requested ?? defaultMaxTokensFor(platform);+}+/** True when this platform's policy strips response_format before send — the* router uses it to skip such platforms for structured-output requests. */export function platformDropsResponseFormat(platform: string): boolean {diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex baa5cf4d..db76f2eb 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -4,7 +4,7 @@ import type {ChatCompletionChunk,} from '@freellmapi/shared/types.js';import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValidationResult, type KeyValidationFailure } from './base.js';-import { extendedBodyParams } from '../lib/sampling-params.js';+import { extendedBodyParams, resolveMaxTokens } from '../lib/sampling-params.js';import { contentToString } from '../lib/content.js';import { extractThinkFromMessage } from '../lib/think-tags.js';import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';@@ -69,7 +69,7 @@ export class CloudflareProvider extends BaseProvider {model: modelId,messages: this.normalizeMessages(messages),temperature: options?.temperature,- max_tokens: options?.max_tokens,+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),top_p: options?.top_p,stop: options?.stop,tools: options?.tools,@@ -125,7 +125,7 @@ export class CloudflareProvider extends BaseProvider {model: modelId,messages: this.normalizeMessages(messages),temperature: options?.temperature,- max_tokens: options?.max_tokens,+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),top_p: options?.top_p,stop: options?.stop,tools: options?.tools,diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex 21752b53..a80b1a83 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -6,7 +6,7 @@ import type {Platform,} from '@freellmapi/shared/types.js';import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValidationResult } from './base.js';-import { extendedBodyParams } from '../lib/sampling-params.js';+import { extendedBodyParams, resolveMaxTokens } from '../lib/sampling-params.js';import { rescueInlineToolCalls } from '../lib/tool-call-rescue.js';import { extractThinkFromMessage } from '../lib/think-tags.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';@@ -187,7 +187,7 @@ export class OpenAICompatProvider extends BaseProvider {model: modelId,messages: this.messagesForPlatform(messages),temperature: sampling.temperature,- max_tokens: options?.max_tokens,+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),top_p: sampling.topP,stop: options?.stop,tools: options?.tools,@@ -306,7 +306,7 @@ export class OpenAICompatProvider extends BaseProvider {model: modelId,messages: this.messagesForPlatform(messages),temperature: sampling.temperature,- max_tokens: options?.max_tokens,+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),top_p: sampling.topP,stop: options?.stop,tools: options?.tools,diff --git a/server/src/routes/health.ts b/server/src/routes/health.tsindex 7e24d5fc..74ae3141 100644--- a/server/src/routes/health.ts+++ b/server/src/routes/health.ts@@ -69,8 +69,10 @@ healthRouter.post('/check/:keyId', async (req: Request, res: Response) => {res.json({ keyId, status });});-// Check all keys+// Check all keys. Forced: someone pressed the button and wants an answer about+// every key now, so the scheduled pass' recency skip and provider spacing (#553)+// don't apply.healthRouter.post('/check-all', async (_req: Request, res: Response) => {- await checkAllKeys();+ await checkAllKeys({ force: true });res.json({ success: true });});diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex cb8cd8c0..e749619e 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -9,16 +9,49 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';const CHECK_INTERVAL_MS = 5 * 60 * 1000; // 5 minutesconst CONSECUTIVE_FAILURES_TO_DISABLE = 3;const DEFAULT_HEALTH_CHECK_CONCURRENCY = 8;+const DEFAULT_MIN_SPACING_MS = 1000;++/** Base cadence of the scheduled pass (jittered per run, see+ * nextHealthCheckDelayMs). Exported for tests. */+export const HEALTH_CHECK_INTERVAL_MS = CHECK_INTERVAL_MS;++// Jitter applied to every scheduled interval, ±20% (4–6 minutes). Two gateways+// started by the same deploy — or one gateway restarted on a cron — otherwise+// probe the same provider on the same phase forever; a per-run offset breaks+// that lock-step and keeps the average cadence at 5 minutes.+const CHECK_INTERVAL_JITTER = 0.2;++// A key validated more recently than this is skipped by the scheduled pass:+// re-asking a provider about a credential it just answered for is exactly the+// traffic #553 is about. Below the jittered minimum interval (4 minutes) so a+// short interval can never skip a whole pass' worth of keys.+export const RECENT_CHECK_SKIP_MS = 3.5 * 60 * 1000;++// Wall-clock ceiling for one pass. Per-provider spacing is compressed rather+// than allowed to push a pass past this, so a large fleet still finishes well+// inside the interval it is scheduled on and the dashboard never shows+// statuses from two passes ago.+export const HEALTH_PASS_TIME_BUDGET_MS = CHECK_INTERVAL_MS / 2;/** Parallel key probes per health pass. Tunable because the right number depends* on how many keys share one provider; 0 or a bad value falls back to the default. */function getHealthCheckConcurrency(): number {- const raw = process.env.HEALTH_CHECK_CONCURRENCY;+ return positiveIntEnv(process.env.HEALTH_CHECK_CONCURRENCY, DEFAULT_HEALTH_CHECK_CONCURRENCY);+}++/** Minimum gap between two probes aimed at the SAME provider (per platform ++ * base_url). Tunable for operators who know their provider tolerates more or+ * want to be gentler still; 0 or a bad value falls back to the default. */+function getMinSpacingMs(): number {+ return positiveIntEnv(process.env.HEALTH_CHECK_MIN_SPACING_MS, DEFAULT_MIN_SPACING_MS);+}++function positiveIntEnv(raw: string | undefined, fallback: number): number {if (raw !== undefined && raw.trim() !== '') {const n = Number(raw);if (Number.isInteger(n) && n > 0) return n;}- return DEFAULT_HEALTH_CHECK_CONCURRENCY;+ return fallback;}// Track consecutive failures per key@@ -165,55 +198,200 @@ export function markKeyHealthyFromRequest(keyId: number): void {// same genuinely-bad key's failureCount, reaching the auto-disable threshold// in fewer wall-clock checks than "3 consecutive checks" intends. A second// caller joins the in-flight pass instead of starting another.-let checkAllInFlight: Promise<void> | null = null;+let checkAllInFlight: Promise<HealthPassResult> | null = null;-export function checkAllKeys(): Promise<void> {- if (checkAllInFlight) return checkAllInFlight;- checkAllInFlight = (async () => {- const db = getDb();- const keys = db.prepare('SELECT id, platform FROM api_keys WHERE enabled = 1').all() as { id: number; platform: string }[];-- console.log(`[Health] Checking ${keys.length} keys...`);-- // Bounded worker pool rather than a sequential await. validateKey allows up- // to 30s per key (some provider /models endpoints are genuinely that slow),- // so a serial pass over a large key fleet can outlast the 5-minute interval- // it is scheduled on and leave the dashboard showing stale statuses. The cap- // keeps us from opening one socket per key against the same provider.- const concurrency = getHealthCheckConcurrency();- let cursor = 0;- const workers = Array.from({ length: Math.min(concurrency, keys.length) }, async () => {- while (cursor < keys.length) {- const key = keys[cursor++]!;- try {- await checkKeyHealth(key.id);- } catch (err) {- // checkKeyHealth handles its own errors; this is a backstop so one- // rejection cannot abandon the rest of the pass.- console.error(`[Health] Key ${key.id} check threw:`, err);- }- }- });- await Promise.all(workers);+interface HealthKeyRow {+ id: number;+ platform: string;+ base_url: string | null;+ status: string;+ age_ms: number | null;+}- console.log(`[Health] Check complete.`);- })().finally(() => {+export interface HealthPassOptions {+ /** Probe every enabled key now: no recency skip, no per-provider spacing.+ * Used by the dashboard's "check all" button and the post-wake re-probe,+ * where the point is an immediate, complete picture. */+ force?: boolean;+ /** Test seams: injectable clock, sleep, probe and pool size, mirroring the+ * cooldown-probe pass so a pacing test needs no real timers. */+ now?: () => number;+ sleep?: (ms: number) => Promise<void>;+ check?: (keyId: number) => Promise<unknown>;+ concurrency?: number;+ minSpacingMs?: number;+}++export interface HealthPassResult {+ /** Keys probed this pass, in the order the pass started them. */+ checkedKeyIds: number[];+ /** Enabled keys left alone because they were validated recently. */+ skippedKeyIds: number[];+}++/** The provider a probe actually lands on: two openai-compat keys pointed at+ * different base URLs are different hosts and need no spacing between them. */+function providerBucket(row: HealthKeyRow): string {+ return row.base_url ? `${row.platform}|${row.base_url}` : row.platform;+}++/**+ * Round-robin the queue across providers: one key from each provider, then the+ * next from each, and so on. Raw DB order is provider-clustered (keys are added+ * a provider at a time), which is what turned a fleet with 40 keys on one+ * provider into 40 back-to-back requests from one IP (#553). Exported for tests.+ */+export function interleaveByProvider<T>(rows: T[], bucketOf: (row: T) => string): T[] {+ const buckets = new Map<string, T[]>();+ for (const row of rows) {+ const key = bucketOf(row);+ const list = buckets.get(key);+ if (list) list.push(row);+ else buckets.set(key, [row]);+ }+ const lists = [...buckets.values()];+ const out: T[] = [];+ for (let i = 0; out.length < rows.length; i++) {+ for (const list of lists) {+ if (i < list.length) out.push(list[i]!);+ }+ }+ return out;+}++const realSleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));++export function checkAllKeys(opts: HealthPassOptions = {}): Promise<HealthPassResult> {+ // A caller that arrives mid-pass joins it (including a forced one — the+ // in-flight pass is already probing, and stacking a second set of validates+ // on the same providers is the exact behaviour this guard exists to prevent).+ if (checkAllInFlight) return checkAllInFlight;+ checkAllInFlight = runHealthPass(opts).finally(() => {checkAllInFlight = null;});return checkAllInFlight;}+async function runHealthPass(opts: HealthPassOptions): Promise<HealthPassResult> {+ const now = opts.now ?? Date.now;+ const sleep = opts.sleep ?? realSleep;+ const check = opts.check ?? checkKeyHealth;+ const db = getDb();++ // age_ms comes from the DB clock (last_checked_at is written as+ // datetime('now')); the injected `now` above only drives pacing.+ const rows = db.prepare(`+ SELECT id, platform, base_url, status,+ CAST((julianday('now') - julianday(last_checked_at)) * 86400000 AS INTEGER) AS age_ms+ FROM api_keys WHERE enabled = 1+ `).all() as HealthKeyRow[];++ const skippedKeyIds: number[] = [];+ const due = rows.filter(row => {+ if (opts.force) return true;+ // A key parked at 'error' is out of rotation until a probe says otherwise+ // (the router writes that status, and last_checked_at with it), so it is+ // never skipped — it is the one key whose verdict is worth re-asking for.+ if (row.status === 'error') return true;+ if (row.age_ms !== null && row.age_ms < RECENT_CHECK_SKIP_MS) {+ skippedKeyIds.push(row.id);+ return false;+ }+ return true;+ });++ const queue = interleaveByProvider(due, providerBucket);+ const perBucket = new Map<string, number>();+ for (const row of queue) {+ const bucket = providerBucket(row);+ perBucket.set(bucket, (perBucket.get(bucket) ?? 0) + 1);+ }+ const largestBucket = Math.max(0, ...perBucket.values());+ // Spacing shrinks rather than letting a big fleet run past the budget: the+ // provider with the most keys sets the pass length, so its gaps are what the+ // budget has to divide.+ const requestedSpacing = opts.minSpacingMs ?? getMinSpacingMs();+ const spacingMs = opts.force || largestBucket < 2+ ? 0+ : Math.min(requestedSpacing, Math.floor(HEALTH_PASS_TIME_BUDGET_MS / (largestBucket - 1)));++ console.log(+ `[Health] Checking ${queue.length} keys` ++ (skippedKeyIds.length > 0 ? ` (${skippedKeyIds.length} checked recently, skipped)` : '') ++ (spacingMs > 0 ? ` — ${spacingMs}ms between probes of the same provider` : '') + '...',+ );++ // Bounded worker pool rather than a sequential await. validateKey allows up+ // to 30s per key (some provider /models endpoints are genuinely that slow),+ // so a serial pass over a large key fleet can outlast the 5-minute interval+ // it is scheduled on and leave the dashboard showing stale statuses. The cap+ // keeps us from opening one socket per key against the same provider.+ const concurrency = opts.concurrency ?? getHealthCheckConcurrency();+ const nextAllowedAt = new Map<string, number>();+ const checkedKeyIds: number[] = [];+ let cursor = 0;+ const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {+ while (cursor < queue.length) {+ const key = queue[cursor++]!;+ if (spacingMs > 0) {+ // Reserve the slot BEFORE awaiting, so two workers that pick up keys on+ // the same provider queue behind each other instead of both waiting for+ // the same instant and firing together.+ const bucket = providerBucket(key);+ const slot = Math.max(nextAllowedAt.get(bucket) ?? 0, now());+ nextAllowedAt.set(bucket, slot + spacingMs);+ const wait = slot - now();+ if (wait > 0) await sleep(wait);+ }+ checkedKeyIds.push(key.id);+ try {+ await check(key.id);+ } catch (err) {+ // checkKeyHealth handles its own errors; this is a backstop so one+ // rejection cannot abandon the rest of the pass.+ console.error(`[Health] Key ${key.id} check threw:`, err);+ }+ }+ });+ await Promise.all(workers);++ console.log(`[Health] Check complete.`);+ return { checkedKeyIds, skippedKeyIds };+}++/** Delay until the next scheduled pass: the base interval ±20%, so restarts and+ * co-deployed gateways don't stay phase-locked on the same providers. */+export function nextHealthCheckDelayMs(jitter: () => number = Math.random): number {+ return Math.round(CHECK_INTERVAL_MS * (1 + (jitter() * 2 - 1) * CHECK_INTERVAL_JITTER));+}+let cancelHealthCheck: (() => void) | null = null;+let healthCheckerRunning = false;export function startHealthChecker(scheduler: Scheduler): void {- if (cancelHealthCheck) return;- console.log(`[Health] Starting health checker (every ${CHECK_INTERVAL_MS / 1000}s)`);- cancelHealthCheck = scheduler.every(CHECK_INTERVAL_MS, () =>- checkAllKeys().catch(err => console.error('[Health] Check failed:', err)),+ if (healthCheckerRunning) return;+ healthCheckerRunning = true;+ console.log(+ `[Health] Starting health checker (every ~${CHECK_INTERVAL_MS / 1000}s ±${CHECK_INTERVAL_JITTER * 100}%)`,);+ // Self-rescheduling one-shot timers rather than a fixed interval: each pass+ // picks its own jittered delay (see nextHealthCheckDelayMs), and the next one+ // is only armed once the current pass has finished.+ const scheduleNext = (): void => {+ cancelHealthCheck = scheduler.after(nextHealthCheckDelayMs(), async () => {+ try {+ await checkAllKeys();+ } catch (err) {+ console.error('[Health] Check failed:', err);+ }+ if (healthCheckerRunning) scheduleNext();+ });+ };+ scheduleNext();}export function stopHealthChecker(): void {+ healthCheckerRunning = false;if (cancelHealthCheck) {cancelHealthCheck();cancelHealthCheck = null;