Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix: default output floor on Cloudflare + pace the health probe pass (#553) (#644)

Tashfeen75f0498015c9
diff --git a/server/src/__tests__/providers/default-max-tokens.test.ts b/server/src/__tests__/providers/default-max-tokens.test.ts
new file mode 100644
index 00000000..2494683f
--- /dev/null
+++ b/server/src/__tests__/providers/default-max-tokens.test.ts
@@ -0,0 +1,123 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { CloudflareProvider } from '../../providers/cloudflare.js';
+import { OpenAICompatProvider } from '../../providers/openai-compat.js';
+import { defaultMaxTokensFor, resolveMaxTokens } from '../../lib/sampling-params.js';
+
+// #553: Workers AI applies its own (small) server-side max_tokens when the
+// request omits one, so a client that never sets max_tokens — the OpenAI SDK
+// default — gets a response cut off mid-sentence a couple of seconds in. The
+// gateway therefore sends a generous per-platform floor when, and only when,
+// the client omitted the parameter.
+
+function jsonResponse(): any {
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'chatcmpl-1',
+ object: 'chat.completion',
+ created: 1,
+ model: 'm',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'hi' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ }),
+ headers: new Headers(),
+ };
+}
+
+function sseResponse(): any {
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(new TextEncoder().encode('data: [DONE]\n\n'));
+ controller.close();
+ },
+ });
+ return { ok: true, body: stream, headers: new Headers() };
+}
+
+function captureBody(response: () => any): { body: () => any } {
+ let captured: any = null;
+ vi.spyOn(global, 'fetch').mockImplementation(async (_url, init) => {
+ captured = JSON.parse((init as any).body);
+ return response() as any;
+ });
+ return { body: () => captured };
+}
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe('resolveMaxTokens', () => {
+ it('applies the platform default only when the client omitted max_tokens', () => {
+ const cfDefault = defaultMaxTokensFor('cloudflare');
+ expect(cfDefault).toBeGreaterThan(0);
+
+ expect(resolveMaxTokens('cloudflare', undefined)).toBe(cfDefault);
+ expect(resolveMaxTokens('cloudflare', 64)).toBe(64);
+ // A client value above the default is never clamped down.
+ expect(resolveMaxTokens('cloudflare', 100000)).toBe(100000);
+ });
+
+ it('leaves platforms without a default untouched', () => {
+ expect(defaultMaxTokensFor('groq')).toBeUndefined();
+ expect(resolveMaxTokens('groq', undefined)).toBeUndefined();
+ expect(resolveMaxTokens('groq', 32)).toBe(32);
+ expect(resolveMaxTokens('unknown-platform', undefined)).toBeUndefined();
+ });
+});
+
+describe('Cloudflare default max_tokens', () => {
+ const provider = new CloudflareProvider();
+ const messages = [{ role: 'user' as const, content: 'Hi' }];
+
+ it('sends the default when the client omitted max_tokens (non-streaming)', async () => {
+ const cap = captureBody(jsonResponse);
+ await provider.chatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b');
+ expect(cap.body().max_tokens).toBe(defaultMaxTokensFor('cloudflare'));
+ });
+
+ it('sends the default when the client omitted max_tokens (streaming)', async () => {
+ const cap = captureBody(sseResponse);
+ const gen = provider.streamChatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b');
+ for await (const _chunk of gen) { /* drain */ }
+ expect(cap.body().max_tokens).toBe(defaultMaxTokensFor('cloudflare'));
+ });
+
+ it('passes a client-supplied max_tokens through untouched', async () => {
+ const cap = captureBody(jsonResponse);
+ await provider.chatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b', { max_tokens: 16 });
+ expect(cap.body().max_tokens).toBe(16);
+ });
+
+ it('passes a client-supplied max_tokens larger than the default through untouched', async () => {
+ const cap = captureBody(jsonResponse);
+ await provider.chatCompletion('acct:token', messages, '@cf/openai/gpt-oss-120b', { max_tokens: 120000 });
+ expect(cap.body().max_tokens).toBe(120000);
+ });
+});
+
+describe('other platforms are unchanged', () => {
+ const messages = [{ role: 'user' as const, content: 'Hi' }];
+
+ it('omits max_tokens entirely when the client did (groq)', async () => {
+ const provider = new OpenAICompatProvider({
+ platform: 'groq',
+ name: 'Groq',
+ baseUrl: 'https://api.test.com/v1',
+ });
+ const cap = captureBody(jsonResponse);
+ await provider.chatCompletion('sk-test', messages, 'llama-3.3-70b');
+ expect(cap.body().max_tokens).toBeUndefined();
+ });
+
+ it('still forwards a client-supplied max_tokens (groq)', async () => {
+ const provider = new OpenAICompatProvider({
+ platform: 'groq',
+ name: 'Groq',
+ baseUrl: 'https://api.test.com/v1',
+ });
+ const cap = captureBody(jsonResponse);
+ await provider.chatCompletion('sk-test', messages, 'llama-3.3-70b', { max_tokens: 77 });
+ expect(cap.body().max_tokens).toBe(77);
+ });
+});
diff --git a/server/src/__tests__/services/health-probe-pacing.test.ts b/server/src/__tests__/services/health-probe-pacing.test.ts
new file mode 100644
index 00000000..aa7a8b7e
--- /dev/null
+++ b/server/src/__tests__/services/health-probe-pacing.test.ts
@@ -0,0 +1,203 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import type { Scheduler } from '../../lib/scheduler.js';
+
+const { initDb, getDb } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const {
+ checkAllKeys,
+ nextHealthCheckDelayMs,
+ startHealthChecker,
+ stopHealthChecker,
+ HEALTH_CHECK_INTERVAL_MS,
+ HEALTH_PASS_TIME_BUDGET_MS,
+ RECENT_CHECK_SKIP_MS,
+} = await import('../../services/health.js');
+
+// #553: every enabled key was probed in raw DB order behind a concurrency-8
+// pool every 5 minutes with no spacing, so a fleet with many keys on one
+// provider fired them all back to back from one IP — which reads as abuse to
+// the provider. The pass now interleaves providers, spaces same-provider
+// probes, skips keys validated recently, and jitters its own interval.
+
+let nextId = 20000;
+
+function seedKey(platform: string, opts: { lastCheckedAt?: string; status?: string } = {}): number {
+ const id = ++nextId;
+ const enc = encrypt(`pacing-${id}`);
+ getDb().prepare(`
+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status, last_checked_at)
+ VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)
+ `).run(id, platform, `pacing-${id}`, enc.encrypted, enc.iv, enc.authTag, opts.status ?? 'healthy', opts.lastCheckedAt ?? null);
+ return id;
+}
+
+/** Fake clock driven by the injected sleep, so a pass costs no real time. */
+function fakeClock() {
+ const state = { t: 0 };
+ return {
+ now: () => state.t,
+ sleep: async (ms: number) => { state.t += ms; },
+ };
+}
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+beforeEach(() => {
+ getDb().prepare('DELETE FROM api_keys').run();
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+});
+
+describe('health pass ordering', () => {
+ it('interleaves platforms instead of probing one provider back to back', async () => {
+ // Inserted provider-clustered, exactly as the old raw-DB-order pass read them.
+ const groq = [seedKey('groq'), seedKey('groq'), seedKey('groq')];
+ const cf = [seedKey('cloudflare'), seedKey('cloudflare')];
+ const mistral = [seedKey('mistral')];
+
+ const clock = fakeClock();
+ const platformOf = new Map<number, string>([
+ ...groq.map(id => [id, 'groq'] as const),
+ ...cf.map(id => [id, 'cloudflare'] as const),
+ ...mistral.map(id => [id, 'mistral'] as const),
+ ]);
+ const order: string[] = [];
+
+ await checkAllKeys({
+ now: clock.now,
+ sleep: clock.sleep,
+ concurrency: 1,
+ check: async (keyId: number) => { order.push(platformOf.get(keyId)!); },
+ });
+
+ expect(order).toEqual(['groq', 'cloudflare', 'mistral', 'groq', 'cloudflare', 'groq']);
+ });
+
+ it('spaces consecutive probes of the same provider', async () => {
+ const ids = [seedKey('groq'), seedKey('groq'), seedKey('groq'), seedKey('cloudflare')];
+ const clock = fakeClock();
+ const spacingMs = 1000;
+ const startedAt = new Map<number, number>();
+
+ await checkAllKeys({
+ now: clock.now,
+ sleep: clock.sleep,
+ concurrency: 1,
+ minSpacingMs: spacingMs,
+ check: async (keyId: number) => { startedAt.set(keyId, clock.now()); },
+ });
+
+ const groqTimes = ids.slice(0, 3).map(id => startedAt.get(id)!);
+ expect(groqTimes).toHaveLength(3);
+ for (let i = 1; i < groqTimes.length; i++) {
+ expect(groqTimes[i]! - groqTimes[i - 1]!).toBeGreaterThanOrEqual(spacingMs);
+ }
+ // The other provider is not held behind groq's spacing.
+ expect(startedAt.get(ids[3]!)).toBe(0);
+ });
+
+ it('keeps the whole pass inside its time budget even for a big single-provider fleet', async () => {
+ for (let i = 0; i < 400; i++) seedKey('groq');
+
+ const clock = fakeClock();
+ await checkAllKeys({
+ now: clock.now,
+ sleep: clock.sleep,
+ concurrency: 8,
+ check: async () => {},
+ });
+
+ expect(clock.now()).toBeLessThanOrEqual(HEALTH_PASS_TIME_BUDGET_MS);
+ expect(HEALTH_PASS_TIME_BUDGET_MS).toBeLessThan(HEALTH_CHECK_INTERVAL_MS);
+ });
+});
+
+describe('recently-validated keys', () => {
+ it('skips keys checked inside the recency window', async () => {
+ const fresh = seedKey('groq');
+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now') WHERE id = ?").run(fresh);
+ const stale = seedKey('groq');
+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now', '-30 minutes') WHERE id = ?").run(stale);
+ const never = seedKey('groq');
+
+ const clock = fakeClock();
+ const checked: number[] = [];
+ const result = await checkAllKeys({
+ now: clock.now,
+ sleep: clock.sleep,
+ concurrency: 1,
+ check: async (keyId: number) => { checked.push(keyId); },
+ });
+
+ expect(checked.sort()).toEqual([stale, never].sort());
+ expect(result.skippedKeyIds).toEqual([fresh]);
+ expect(RECENT_CHECK_SKIP_MS).toBeLessThan(HEALTH_CHECK_INTERVAL_MS);
+ });
+
+ it('never skips a key sitting at status=error', async () => {
+ const errored = seedKey('groq', { status: 'error' });
+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now') WHERE id = ?").run(errored);
+
+ const clock = fakeClock();
+ const checked: number[] = [];
+ await checkAllKeys({
+ now: clock.now, sleep: clock.sleep, concurrency: 1,
+ check: async (keyId: number) => { checked.push(keyId); },
+ });
+
+ expect(checked).toEqual([errored]);
+ });
+
+ it('force ignores both the recency skip and the spacing (manual / post-wake pass)', async () => {
+ const a = seedKey('groq');
+ const b = seedKey('groq');
+ getDb().prepare("UPDATE api_keys SET last_checked_at = datetime('now')").run();
+
+ const clock = fakeClock();
+ const checked: number[] = [];
+ const result = await checkAllKeys({
+ now: clock.now, sleep: clock.sleep, concurrency: 1, force: true,
+ check: async (keyId: number) => { checked.push(keyId); },
+ });
+
+ expect(checked.sort()).toEqual([a, b].sort());
+ expect(result.skippedKeyIds).toEqual([]);
+ expect(clock.now()).toBe(0);
+ });
+});
+
+describe('interval jitter', () => {
+ it('spreads the next pass around the base interval', () => {
+ expect(nextHealthCheckDelayMs(() => 0)).toBeLessThan(HEALTH_CHECK_INTERVAL_MS);
+ expect(nextHealthCheckDelayMs(() => 0.999)).toBeGreaterThan(HEALTH_CHECK_INTERVAL_MS);
+ for (const r of [0, 0.25, 0.5, 0.75, 0.999]) {
+ const delay = nextHealthCheckDelayMs(() => r);
+ expect(delay).toBeGreaterThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 0.5);
+ expect(delay).toBeLessThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 1.5);
+ // A jittered pass must never start before the recency window has closed,
+ // or the skip filter would swallow whole passes.
+ expect(delay).toBeGreaterThan(RECENT_CHECK_SKIP_MS);
+ }
+ });
+
+ it('reschedules itself with a fresh delay after each pass', async () => {
+ const scheduled: { ms: number; fn: () => void | Promise<void> }[] = [];
+ const scheduler: Scheduler = {
+ every: () => vi.fn(),
+ after(ms, fn) { scheduled.push({ ms, fn }); return vi.fn(); },
+ };
+
+ startHealthChecker(scheduler);
+ expect(scheduled).toHaveLength(1);
+ await scheduled[0]!.fn();
+ expect(scheduled).toHaveLength(2);
+ stopHealthChecker();
+ await scheduled[1]!.fn();
+ // Stopped: the pass must not queue another timer.
+ expect(scheduled).toHaveLength(2);
+ });
+});
diff --git a/server/src/__tests__/services/health-scheduler.test.ts b/server/src/__tests__/services/health-scheduler.test.ts
index 4dee9e09..b2095b92 100644
--- a/server/src/__tests__/services/health-scheduler.test.ts
+++ b/server/src/__tests__/services/health-scheduler.test.ts
@@ -1,23 +1,23 @@
import { describe, it, expect, beforeAll, afterEach, vi } from 'vitest';
import { initDb } from '../../db/index.js';
-import { startHealthChecker, stopHealthChecker } from '../../services/health.js';
+import { startHealthChecker, stopHealthChecker, HEALTH_CHECK_INTERVAL_MS } from '../../services/health.js';
import type { Scheduler } from '../../lib/scheduler.js';
function makeScheduler() {
- const every: { ms: number; fn: () => void | Promise<void> }[] = [];
+ const after: { ms: number; fn: () => void | Promise<void> }[] = [];
const cancels: ReturnType<typeof vi.fn>[] = [];
const scheduler: Scheduler = {
- every(ms, fn) {
+ every(_ms, _fn) {
+ return vi.fn();
+ },
+ after(ms, fn) {
const cancel = vi.fn();
- every.push({ ms, fn });
+ after.push({ ms, fn });
cancels.push(cancel);
return cancel;
},
- after(_ms, _fn) {
- return vi.fn();
- },
};
- return { scheduler, every, cancels };
+ return { scheduler, after, cancels };
}
describe('startHealthChecker / stopHealthChecker', () => {
@@ -30,18 +30,22 @@ describe('startHealthChecker / stopHealthChecker', () => {
stopHealthChecker();
});
- it('registers one every-5-minute job', () => {
- const { scheduler, every } = makeScheduler();
+ // The pass re-arms itself after each run with a fresh jittered delay (#553)
+ // instead of sitting on a fixed 5-minute interval, so a fleet's probes don't
+ // stay phase-locked to the same instant on every provider.
+ it('arms one jittered timer around the 5-minute base interval', () => {
+ const { scheduler, after } = makeScheduler();
startHealthChecker(scheduler);
- expect(every).toHaveLength(1);
- expect(every[0].ms).toBe(5 * 60 * 1000);
+ expect(after).toHaveLength(1);
+ expect(after[0].ms).toBeGreaterThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 0.8);
+ expect(after[0].ms).toBeLessThanOrEqual(HEALTH_CHECK_INTERVAL_MS * 1.2);
});
it('is idempotent — double-start registers only one job', () => {
- const { scheduler, every } = makeScheduler();
+ const { scheduler, after } = makeScheduler();
startHealthChecker(scheduler);
startHealthChecker(scheduler);
- expect(every).toHaveLength(1);
+ expect(after).toHaveLength(1);
});
it('stop invokes the cancel handle', () => {
@@ -56,14 +60,14 @@ describe('startHealthChecker / stopHealthChecker', () => {
startHealthChecker(s1);
stopHealthChecker();
- const { scheduler: s2, every } = makeScheduler();
+ const { scheduler: s2, after } = makeScheduler();
startHealthChecker(s2);
- expect(every).toHaveLength(1);
+ expect(after).toHaveLength(1);
});
it('the registered job runs checkAllKeys without throwing', async () => {
- const { scheduler, every } = makeScheduler();
+ const { scheduler, after } = makeScheduler();
startHealthChecker(scheduler);
- await expect(every[0].fn()).resolves.toBeUndefined();
+ await expect(after[0].fn()).resolves.toBeUndefined();
});
});
diff --git a/server/src/index.ts b/server/src/index.ts
index 4bfc6fec..4403b8ea 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -76,7 +76,9 @@ async function main() {
console.log(`[wake] resumed after ~${idle}s (${event.reason}${event.signal ? `:${event.signal}` : ''}) — flushing stale sockets, re-probing keys`);
flushProxyCache();
try {
- await checkAllKeys();
+ // Forced: every status predates the sleep, so the recency skip and
+ // provider spacing of a scheduled pass would only delay the picture.
+ await checkAllKeys({ force: true });
} catch (err: any) {
console.error(`[wake] post-wake key re-probe failed: ${err?.message ?? err}`);
}
diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.ts
index 4358b306..0ded96e5 100644
--- a/server/src/lib/sampling-params.ts
+++ b/server/src/lib/sampling-params.ts
@@ -208,6 +208,15 @@ export interface PlatformParamPolicy {
// supported value instead of being forwarded into a 400 (#619). Omitted =
// forward whatever the client asked for.
reasoningEfforts?: readonly ReasoningEffort[];
+ // Output-token floor sent when — and ONLY when — the client omitted
+ // max_tokens entirely (#553). Some providers apply a tiny server-side
+ // default in that case and cut the answer off mid-sentence, which clients
+ // experience as a broken stream rather than as a truncation. A
+ // client-supplied value always wins, larger or smaller. Applied by the
+ // adapters through resolveMaxTokens(), so a platform added here only takes
+ // effect once its adapter routes max_tokens through that helper
+ // (openai-compat + cloudflare already do).
+ defaultMaxTokens?: number;
}
// Keyed by Platform (not string) so a typo'd platform id fails tsc instead of
@@ -238,8 +247,18 @@ export const PLATFORM_PARAM_POLICIES: Partial<Record<Platform, PlatformParamPoli
// `thinking` object) have no mapping there.
cohere: { drop: ['top_k', 'min_p', 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs', 'reasoning_effort'] },
// Workers AI's OpenAI-compat endpoint parses a known subset; send only what
- // it understands.
- cloudflare: { drop: ['min_p', 'logit_bias', 'logprobs', 'top_logprobs', 'reasoning_effort'] },
+ // it understands. It also applies a small server-side max_tokens default
+ // when the request omits one, so an SDK client that never sets max_tokens
+ // (the common case) sees answers stop a couple of seconds in — reported on
+ // @cf/openai/gpt-oss-120b, where the model's own reasoning trace eats the
+ // default before any visible text lands (#553). 8192 is the floor we send
+ // instead: generous for a chat turn plus reasoning, and small enough to fit
+ // inside the smallest context window in the Workers AI catalog (24K on
+ // @cf/meta/llama-3.3-70b-instruct-fp8-fast) with room left for the prompt.
+ cloudflare: {
+ drop: ['min_p', 'logit_bias', 'logprobs', 'top_logprobs', 'reasoning_effort'],
+ defaultMaxTokens: 8192,
+ },
// AI Horde builds its own payload format; none of the extended set maps.
aihorde: { drop: [...EXTENDED_SAMPLING_KEYS] },
// Kilo's anonymous gateway 400s ("Provider returned error") whenever
@@ -288,6 +307,23 @@ export function extendedBodyParams(platform: string, options: ExtendedSamplingOp
return out;
}
+/** The output-token floor this platform sends for a request that carries no
+ * max_tokens at all, or undefined when the provider's own default is fine. */
+export function defaultMaxTokensFor(platform: string): number | undefined {
+ return PLATFORM_PARAM_POLICIES[platform as Platform]?.defaultMaxTokens;
+}
+
+/**
+ * The max_tokens to put on the wire for one request: whatever the client asked
+ * for, or the platform's floor when the client asked for nothing (#553).
+ * Never clamps — a client-set value passes through untouched in both
+ * directions, and the gateway's own guardrails (token budget, routing reserve)
+ * have already had their say by the time an adapter calls this.
+ */
+export function resolveMaxTokens(platform: string, requested: number | undefined): number | undefined {
+ return requested ?? defaultMaxTokensFor(platform);
+}
+
/** True when this platform's policy strips response_format before send — the
* router uses it to skip such platforms for structured-output requests. */
export function platformDropsResponseFormat(platform: string): boolean {
diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.ts
index baa5cf4d..db76f2eb 100644
--- a/server/src/providers/cloudflare.ts
+++ b/server/src/providers/cloudflare.ts
@@ -4,7 +4,7 @@ import type {
ChatCompletionChunk,
} from '@freellmapi/shared/types.js';
import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValidationResult, type KeyValidationFailure } from './base.js';
-import { extendedBodyParams } from '../lib/sampling-params.js';
+import { extendedBodyParams, resolveMaxTokens } from '../lib/sampling-params.js';
import { contentToString } from '../lib/content.js';
import { extractThinkFromMessage } from '../lib/think-tags.js';
import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';
@@ -69,7 +69,7 @@ export class CloudflareProvider extends BaseProvider {
model: modelId,
messages: this.normalizeMessages(messages),
temperature: options?.temperature,
- max_tokens: options?.max_tokens,
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
top_p: options?.top_p,
stop: options?.stop,
tools: options?.tools,
@@ -125,7 +125,7 @@ export class CloudflareProvider extends BaseProvider {
model: modelId,
messages: this.normalizeMessages(messages),
temperature: options?.temperature,
- max_tokens: options?.max_tokens,
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
top_p: options?.top_p,
stop: options?.stop,
tools: options?.tools,
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 21752b53..a80b1a83 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -6,7 +6,7 @@ import type {
Platform,
} from '@freellmapi/shared/types.js';
import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValidationResult } from './base.js';
-import { extendedBodyParams } from '../lib/sampling-params.js';
+import { extendedBodyParams, resolveMaxTokens } from '../lib/sampling-params.js';
import { rescueInlineToolCalls } from '../lib/tool-call-rescue.js';
import { extractThinkFromMessage } from '../lib/think-tags.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
@@ -187,7 +187,7 @@ export class OpenAICompatProvider extends BaseProvider {
model: modelId,
messages: this.messagesForPlatform(messages),
temperature: sampling.temperature,
- max_tokens: options?.max_tokens,
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
top_p: sampling.topP,
stop: options?.stop,
tools: options?.tools,
@@ -306,7 +306,7 @@ export class OpenAICompatProvider extends BaseProvider {
model: modelId,
messages: this.messagesForPlatform(messages),
temperature: sampling.temperature,
- max_tokens: options?.max_tokens,
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
top_p: sampling.topP,
stop: options?.stop,
tools: options?.tools,
diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts
index 7e24d5fc..74ae3141 100644
--- a/server/src/routes/health.ts
+++ b/server/src/routes/health.ts
@@ -69,8 +69,10 @@ healthRouter.post('/check/:keyId', async (req: Request, res: Response) => {
res.json({ keyId, status });
});
-// Check all keys
+// Check all keys. Forced: someone pressed the button and wants an answer about
+// every key now, so the scheduled pass' recency skip and provider spacing (#553)
+// don't apply.
healthRouter.post('/check-all', async (_req: Request, res: Response) => {
- await checkAllKeys();
+ await checkAllKeys({ force: true });
res.json({ success: true });
});
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index cb8cd8c0..e749619e 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -9,16 +9,49 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
const CHECK_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
const CONSECUTIVE_FAILURES_TO_DISABLE = 3;
const DEFAULT_HEALTH_CHECK_CONCURRENCY = 8;
+const DEFAULT_MIN_SPACING_MS = 1000;
+
+/** Base cadence of the scheduled pass (jittered per run, see
+ * nextHealthCheckDelayMs). Exported for tests. */
+export const HEALTH_CHECK_INTERVAL_MS = CHECK_INTERVAL_MS;
+
+// Jitter applied to every scheduled interval, ±20% (4–6 minutes). Two gateways
+// started by the same deploy — or one gateway restarted on a cron — otherwise
+// probe the same provider on the same phase forever; a per-run offset breaks
+// that lock-step and keeps the average cadence at 5 minutes.
+const CHECK_INTERVAL_JITTER = 0.2;
+
+// A key validated more recently than this is skipped by the scheduled pass:
+// re-asking a provider about a credential it just answered for is exactly the
+// traffic #553 is about. Below the jittered minimum interval (4 minutes) so a
+// short interval can never skip a whole pass' worth of keys.
+export const RECENT_CHECK_SKIP_MS = 3.5 * 60 * 1000;
+
+// Wall-clock ceiling for one pass. Per-provider spacing is compressed rather
+// than allowed to push a pass past this, so a large fleet still finishes well
+// inside the interval it is scheduled on and the dashboard never shows
+// statuses from two passes ago.
+export const HEALTH_PASS_TIME_BUDGET_MS = CHECK_INTERVAL_MS / 2;
/** Parallel key probes per health pass. Tunable because the right number depends
* on how many keys share one provider; 0 or a bad value falls back to the default. */
function getHealthCheckConcurrency(): number {
- const raw = process.env.HEALTH_CHECK_CONCURRENCY;
+ return positiveIntEnv(process.env.HEALTH_CHECK_CONCURRENCY, DEFAULT_HEALTH_CHECK_CONCURRENCY);
+}
+
+/** Minimum gap between two probes aimed at the SAME provider (per platform +
+ * base_url). Tunable for operators who know their provider tolerates more or
+ * want to be gentler still; 0 or a bad value falls back to the default. */
+function getMinSpacingMs(): number {
+ return positiveIntEnv(process.env.HEALTH_CHECK_MIN_SPACING_MS, DEFAULT_MIN_SPACING_MS);
+}
+
+function positiveIntEnv(raw: string | undefined, fallback: number): number {
if (raw !== undefined && raw.trim() !== '') {
const n = Number(raw);
if (Number.isInteger(n) && n > 0) return n;
}
- return DEFAULT_HEALTH_CHECK_CONCURRENCY;
+ return fallback;
}
// Track consecutive failures per key
@@ -165,55 +198,200 @@ export function markKeyHealthyFromRequest(keyId: number): void {
// same genuinely-bad key's failureCount, reaching the auto-disable threshold
// in fewer wall-clock checks than "3 consecutive checks" intends. A second
// caller joins the in-flight pass instead of starting another.
-let checkAllInFlight: Promise<void> | null = null;
+let checkAllInFlight: Promise<HealthPassResult> | null = null;
-export function checkAllKeys(): Promise<void> {
- if (checkAllInFlight) return checkAllInFlight;
- checkAllInFlight = (async () => {
- const db = getDb();
- const keys = db.prepare('SELECT id, platform FROM api_keys WHERE enabled = 1').all() as { id: number; platform: string }[];
-
- console.log(`[Health] Checking ${keys.length} keys...`);
-
- // Bounded worker pool rather than a sequential await. validateKey allows up
- // to 30s per key (some provider /models endpoints are genuinely that slow),
- // so a serial pass over a large key fleet can outlast the 5-minute interval
- // it is scheduled on and leave the dashboard showing stale statuses. The cap
- // keeps us from opening one socket per key against the same provider.
- const concurrency = getHealthCheckConcurrency();
- let cursor = 0;
- const workers = Array.from({ length: Math.min(concurrency, keys.length) }, async () => {
- while (cursor < keys.length) {
- const key = keys[cursor++]!;
- try {
- await checkKeyHealth(key.id);
- } catch (err) {
- // checkKeyHealth handles its own errors; this is a backstop so one
- // rejection cannot abandon the rest of the pass.
- console.error(`[Health] Key ${key.id} check threw:`, err);
- }
- }
- });
- await Promise.all(workers);
+interface HealthKeyRow {
+ id: number;
+ platform: string;
+ base_url: string | null;
+ status: string;
+ age_ms: number | null;
+}
- console.log(`[Health] Check complete.`);
- })().finally(() => {
+export interface HealthPassOptions {
+ /** Probe every enabled key now: no recency skip, no per-provider spacing.
+ * Used by the dashboard's "check all" button and the post-wake re-probe,
+ * where the point is an immediate, complete picture. */
+ force?: boolean;
+ /** Test seams: injectable clock, sleep, probe and pool size, mirroring the
+ * cooldown-probe pass so a pacing test needs no real timers. */
+ now?: () => number;
+ sleep?: (ms: number) => Promise<void>;
+ check?: (keyId: number) => Promise<unknown>;
+ concurrency?: number;
+ minSpacingMs?: number;
+}
+
+export interface HealthPassResult {
+ /** Keys probed this pass, in the order the pass started them. */
+ checkedKeyIds: number[];
+ /** Enabled keys left alone because they were validated recently. */
+ skippedKeyIds: number[];
+}
+
+/** The provider a probe actually lands on: two openai-compat keys pointed at
+ * different base URLs are different hosts and need no spacing between them. */
+function providerBucket(row: HealthKeyRow): string {
+ return row.base_url ? `${row.platform}|${row.base_url}` : row.platform;
+}
+
+/**
+ * Round-robin the queue across providers: one key from each provider, then the
+ * next from each, and so on. Raw DB order is provider-clustered (keys are added
+ * a provider at a time), which is what turned a fleet with 40 keys on one
+ * provider into 40 back-to-back requests from one IP (#553). Exported for tests.
+ */
+export function interleaveByProvider<T>(rows: T[], bucketOf: (row: T) => string): T[] {
+ const buckets = new Map<string, T[]>();
+ for (const row of rows) {
+ const key = bucketOf(row);
+ const list = buckets.get(key);
+ if (list) list.push(row);
+ else buckets.set(key, [row]);
+ }
+ const lists = [...buckets.values()];
+ const out: T[] = [];
+ for (let i = 0; out.length < rows.length; i++) {
+ for (const list of lists) {
+ if (i < list.length) out.push(list[i]!);
+ }
+ }
+ return out;
+}
+
+const realSleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));
+
+export function checkAllKeys(opts: HealthPassOptions = {}): Promise<HealthPassResult> {
+ // A caller that arrives mid-pass joins it (including a forced one — the
+ // in-flight pass is already probing, and stacking a second set of validates
+ // on the same providers is the exact behaviour this guard exists to prevent).
+ if (checkAllInFlight) return checkAllInFlight;
+ checkAllInFlight = runHealthPass(opts).finally(() => {
checkAllInFlight = null;
});
return checkAllInFlight;
}
+async function runHealthPass(opts: HealthPassOptions): Promise<HealthPassResult> {
+ const now = opts.now ?? Date.now;
+ const sleep = opts.sleep ?? realSleep;
+ const check = opts.check ?? checkKeyHealth;
+ const db = getDb();
+
+ // age_ms comes from the DB clock (last_checked_at is written as
+ // datetime('now')); the injected `now` above only drives pacing.
+ const rows = db.prepare(`
+ SELECT id, platform, base_url, status,
+ CAST((julianday('now') - julianday(last_checked_at)) * 86400000 AS INTEGER) AS age_ms
+ FROM api_keys WHERE enabled = 1
+ `).all() as HealthKeyRow[];
+
+ const skippedKeyIds: number[] = [];
+ const due = rows.filter(row => {
+ if (opts.force) return true;
+ // A key parked at 'error' is out of rotation until a probe says otherwise
+ // (the router writes that status, and last_checked_at with it), so it is
+ // never skipped — it is the one key whose verdict is worth re-asking for.
+ if (row.status === 'error') return true;
+ if (row.age_ms !== null && row.age_ms < RECENT_CHECK_SKIP_MS) {
+ skippedKeyIds.push(row.id);
+ return false;
+ }
+ return true;
+ });
+
+ const queue = interleaveByProvider(due, providerBucket);
+ const perBucket = new Map<string, number>();
+ for (const row of queue) {
+ const bucket = providerBucket(row);
+ perBucket.set(bucket, (perBucket.get(bucket) ?? 0) + 1);
+ }
+ const largestBucket = Math.max(0, ...perBucket.values());
+ // Spacing shrinks rather than letting a big fleet run past the budget: the
+ // provider with the most keys sets the pass length, so its gaps are what the
+ // budget has to divide.
+ const requestedSpacing = opts.minSpacingMs ?? getMinSpacingMs();
+ const spacingMs = opts.force || largestBucket < 2
+ ? 0
+ : Math.min(requestedSpacing, Math.floor(HEALTH_PASS_TIME_BUDGET_MS / (largestBucket - 1)));
+
+ console.log(
+ `[Health] Checking ${queue.length} keys` +
+ (skippedKeyIds.length > 0 ? ` (${skippedKeyIds.length} checked recently, skipped)` : '') +
+ (spacingMs > 0 ? ` — ${spacingMs}ms between probes of the same provider` : '') + '...',
+ );
+
+ // Bounded worker pool rather than a sequential await. validateKey allows up
+ // to 30s per key (some provider /models endpoints are genuinely that slow),
+ // so a serial pass over a large key fleet can outlast the 5-minute interval
+ // it is scheduled on and leave the dashboard showing stale statuses. The cap
+ // keeps us from opening one socket per key against the same provider.
+ const concurrency = opts.concurrency ?? getHealthCheckConcurrency();
+ const nextAllowedAt = new Map<string, number>();
+ const checkedKeyIds: number[] = [];
+ let cursor = 0;
+ const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+ while (cursor < queue.length) {
+ const key = queue[cursor++]!;
+ if (spacingMs > 0) {
+ // Reserve the slot BEFORE awaiting, so two workers that pick up keys on
+ // the same provider queue behind each other instead of both waiting for
+ // the same instant and firing together.
+ const bucket = providerBucket(key);
+ const slot = Math.max(nextAllowedAt.get(bucket) ?? 0, now());
+ nextAllowedAt.set(bucket, slot + spacingMs);
+ const wait = slot - now();
+ if (wait > 0) await sleep(wait);
+ }
+ checkedKeyIds.push(key.id);
+ try {
+ await check(key.id);
+ } catch (err) {
+ // checkKeyHealth handles its own errors; this is a backstop so one
+ // rejection cannot abandon the rest of the pass.
+ console.error(`[Health] Key ${key.id} check threw:`, err);
+ }
+ }
+ });
+ await Promise.all(workers);
+
+ console.log(`[Health] Check complete.`);
+ return { checkedKeyIds, skippedKeyIds };
+}
+
+/** Delay until the next scheduled pass: the base interval ±20%, so restarts and
+ * co-deployed gateways don't stay phase-locked on the same providers. */
+export function nextHealthCheckDelayMs(jitter: () => number = Math.random): number {
+ return Math.round(CHECK_INTERVAL_MS * (1 + (jitter() * 2 - 1) * CHECK_INTERVAL_JITTER));
+}
+
let cancelHealthCheck: (() => void) | null = null;
+let healthCheckerRunning = false;
export function startHealthChecker(scheduler: Scheduler): void {
- if (cancelHealthCheck) return;
- console.log(`[Health] Starting health checker (every ${CHECK_INTERVAL_MS / 1000}s)`);
- cancelHealthCheck = scheduler.every(CHECK_INTERVAL_MS, () =>
- checkAllKeys().catch(err => console.error('[Health] Check failed:', err)),
+ if (healthCheckerRunning) return;
+ healthCheckerRunning = true;
+ console.log(
+ `[Health] Starting health checker (every ~${CHECK_INTERVAL_MS / 1000}s ±${CHECK_INTERVAL_JITTER * 100}%)`,
);
+ // Self-rescheduling one-shot timers rather than a fixed interval: each pass
+ // picks its own jittered delay (see nextHealthCheckDelayMs), and the next one
+ // is only armed once the current pass has finished.
+ const scheduleNext = (): void => {
+ cancelHealthCheck = scheduler.after(nextHealthCheckDelayMs(), async () => {
+ try {
+ await checkAllKeys();
+ } catch (err) {
+ console.error('[Health] Check failed:', err);
+ }
+ if (healthCheckerRunning) scheduleNext();
+ });
+ };
+ scheduleNext();
}
export function stopHealthChecker(): void {
+ healthCheckerRunning = false;
if (cancelHealthCheck) {
cancelHealthCheck();
cancelHealthCheck = null;