7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
import { getDb } from '../db/index.js';
import { resolveProvider } from '../providers/index.js';
import { BaseProvider } from '../providers/base.js';
import { decrypt } from '../lib/crypto.js';
import { decryptProxyUrl } from '../lib/key-proxy.js';
import { withKeyProxy } from '../lib/proxy.js';
import type { Platform, KeyStatus } from '@freellmapi/shared/types.js';
import { inferQuotaPoolKey } from './provider-quota.js';
import { updateDegradationState } from './degradation.js';
import type { Scheduler } from '../lib/scheduler.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { providerLog } from '../lib/server-logs.js';
import { triggerBuiltinModelDiscovery } from './builtin-model-discovery.js';
const DEFAULT_CHECK_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
// Floor of 60s: a misconfigured tiny interval turns the health pass into a
// self-inflicted abuse-shaped flood, which is the exact traffic #553/#1599 are
// about. Anything below the floor falls back to the default.
const MIN_CHECK_INTERVAL_MS = 60 * 1000;
const CONSECUTIVE_FAILURES_TO_DISABLE = 3;
const DEFAULT_HEALTH_CHECK_CONCURRENCY = 8;
const DEFAULT_MIN_SPACING_MS = 1000;
/** Base cadence of the scheduled pass (jittered per run, see
* nextHealthCheckDelayMs), tunable via HEALTH_CHECK_INTERVAL_MS (#1593):
* fleets on providers that dislike frequent key checks
* want a slower pass; a fresh install being shaken out wants a faster one.
* Read once at module load — the scheduler re-arms from the same constant. */
function getCheckIntervalMs(): number {
const n = positiveIntEnv(process.env.HEALTH_CHECK_INTERVAL_MS, DEFAULT_CHECK_INTERVAL_MS);
return n < MIN_CHECK_INTERVAL_MS ? DEFAULT_CHECK_INTERVAL_MS : n;
}
const CHECK_INTERVAL_MS = getCheckIntervalMs();
/** Base cadence of the scheduled pass (jittered per run, see
* nextHealthCheckDelayMs). Exported for tests. */
export const HEALTH_CHECK_INTERVAL_MS = CHECK_INTERVAL_MS;
// Jitter applied to every scheduled interval, ±20% (4–6 minutes). Two gateways
// started by the same deploy — or one gateway restarted on a cron — otherwise
// probe the same provider on the same phase forever; a per-run offset breaks
// that lock-step and keeps the average cadence at 5 minutes.
const CHECK_INTERVAL_JITTER = 0.2;
// A key validated more recently than this is skipped by the scheduled pass:
// re-asking a provider about a credential it just answered for is exactly the
// traffic #553 is about. Capped at 70% of the configured interval so a faster
// HEALTH_CHECK_INTERVAL_MS can never skip a whole pass' worth of keys (the
// invariant nextHealthCheckDelayMs asserts).
export const RECENT_CHECK_SKIP_MS = Math.min(3.5 * 60 * 1000, CHECK_INTERVAL_MS * 0.7);
// #1599: how often the scheduled pass re-asks about a key parked at 'error' on
// a provider that must not be polled (supportsKeyPolling === false). Healthy
// and invalid keys on those providers are never re-validated on a schedule.
export const NO_POLL_ERROR_REPROBE_MS = 60 * 60 * 1000;
// Wall-clock ceiling for one pass. Per-provider spacing is compressed rather
// than allowed to push a pass past this, so a large fleet still finishes well
// inside the interval it is scheduled on and the dashboard never shows
// statuses from two passes ago.
export const HEALTH_PASS_TIME_BUDGET_MS = CHECK_INTERVAL_MS / 2;
/** Parallel key probes per health pass. Tunable because the right number depends
* on how many keys share one provider; 0 or a bad value falls back to the default. */
function getHealthCheckConcurrency(): number {
return positiveIntEnv(process.env.HEALTH_CHECK_CONCURRENCY, DEFAULT_HEALTH_CHECK_CONCURRENCY);
}
/** Minimum gap between two probes aimed at the SAME provider (per platform +
* base_url). Tunable for operators who know their provider tolerates more or
* want to be gentler still; 0 or a bad value falls back to the default. */
function getMinSpacingMs(): number {
return positiveIntEnv(process.env.HEALTH_CHECK_MIN_SPACING_MS, DEFAULT_MIN_SPACING_MS);
}
function positiveIntEnv(raw: string | undefined, fallback: number): number {
if (raw !== undefined && raw.trim() !== '') {
const n = Number(raw);
if (Number.isInteger(n) && n > 0) return n;
}
return fallback;
}
// Track consecutive failures per key
const failureCount = new Map<number, number>();
// #1510: consecutive transport errors per key. A key whose provider has been
// unreachable (DNS, connection refused, TCP timeout) for this many probes in a
// row is parked at status='error'. Unlike 'invalid' this is NOT an auto-disable:
// 'error' keys are re-probed on every pass (never recency-skipped — see the
// checkAllKeys due filter) and any live request that serves restores them
// (markKeyHealthyFromRequest), so a provider outage that heals needs no
// operator action. At the 5-minute cadence the threshold is ~30 minutes of
// total unreachability — far past any single blip, and the point where a key
// still marked 'healthy' is provably worse than no key: the router keeps
// handing it traffic, every attempt dies, and failover costs the request its
// latency budget (#1510 measured 264 failures vs 8 successes in a day on a
// 39-key install with zero keys ever showing 'error').
//
// Opt-in: HEALTH_TRANSPORT_PARK_AFTER sets the streak length (6 is the
// suggested value, ~30 minutes at the default cadence). Unset or 0 keeps the
// default "keep trying" behaviour: a transport error never takes a key out of
// rotation, because a key the router stops using can only come back through a
// probe, and a provider whose validation endpoint is down while chat still
// works would otherwise lose every key at once.
export const SUGGESTED_TRANSPORT_PARK_AFTER = 6;
/** Streak length that parks a key, or 0 when parking is off (the default). */
export function getTransportParkAfter(): number {
return positiveIntEnv(process.env.HEALTH_TRANSPORT_PARK_AFTER, 0);
}
const transportFailures = new Map<number, number>();
/** Test seam: drop transport-parking state. */
export function clearTransportFailureState(): void {
transportFailures.clear();
}
/** Increment and apply the park rule. A 429 never reaches here — the caller
* treats it as proof the provider is reachable. Returns true when this call
* parked the key (status written to 'error'). */
function recordTransportFailure(keyId: number, platform: string | undefined): boolean {
const parkAfter = getTransportParkAfter();
if (parkAfter === 0) return false;
const count = (transportFailures.get(keyId) ?? 0) + 1;
transportFailures.set(keyId, count);
if (count < parkAfter) return false;
// Park: 'error' is out of rotation (the router only routes healthy/unknown)
// but stays eligible for re-probing and self-heals. Never touch 'invalid' —
// a confirmed-bad credential verdict outranks network evidence in both
// directions, and only a successful validateKey clears it.
const db = getDb();
const res = db
.prepare("UPDATE api_keys SET status = 'error' WHERE id = ? AND enabled = 1 AND status IN ('healthy', 'unknown')")
.run(keyId);
if (res.changes > 0) {
providerLog(
'warn',
`[Health] Key ${keyId} (${platform}) parked as 'error' after ${count} consecutive transport failures`,
{ provider: platform, event: 'key_transport_park' },
);
}
return res.changes > 0;
}
// A key whose last validation was rate-limited (429) is skipped by the
// scheduled pass until the skip window passes: re-asking a provider that just
// told us to slow down is how validation traffic becomes a ban. LLMTR's
// validator is a POST to /chat/completions with a nonexistent model —
// shaped exactly like the bot traffic their abuse system hunts — so an
// unpaced re-ask every 5 minutes per key kept accounts flagged (#1369).
// Forced passes (dashboard check-all, wake re-probe) still probe: explicit
// operator intent wins over pacing. A later successful validation clears the
// mark; validation that never ran leaves no mark.
const validationRateLimitedAt = new Map<number, number>();
const DEFAULT_VALIDATION_RATE_LIMIT_SKIP_MS = 60 * 60 * 1000;
/** How long a 429ed key stays out of the scheduled pass. Tunable like the
* other pacing knobs; 0 or a bad value falls back to the default. */
function getValidationRateLimitSkipMs(): number {
return positiveIntEnv(process.env.HEALTH_CHECK_RATE_LIMIT_SKIP_MS, DEFAULT_VALIDATION_RATE_LIMIT_SKIP_MS);
}
/** Record that validating this key was just rate-limited. Exported for tests;
* production calls it from checkKeyHealth's 429 path. */
export function recordValidationRateLimited(keyId: number, atMs = Date.now()): void {
validationRateLimitedAt.set(keyId, atMs);
}
/** Test seam: drop all validation-429 pacing state. */
export function clearValidationRateLimitState(): void {
validationRateLimitedAt.clear();
}
function recordInvalidFailure(keyId: number, platform?: string): void {
const count = (failureCount.get(keyId) ?? 0) + 1;
failureCount.set(keyId, count);
if (count >= CONSECUTIVE_FAILURES_TO_DISABLE) {
getDb().prepare('UPDATE api_keys SET enabled = 0 WHERE id = ?').run(keyId);
// providerLog, not console.log: losing a key is the event an operator is
// most likely to be looking for after the fact, so it goes to the dashboard
// log viewer (where warn/error survive a restart) as well as to stdout —
// which providerLog still writes, so nothing here is only visible behind a
// login. Raised to warn for the same reason.
providerLog(
'warn',
`[Health] Auto-disabled key ${keyId} after ${count} consecutive failures`,
{ provider: platform, event: 'key_auto_disabled' },
);
}
}
/**
* Provider-reported quota polling (#1403 phase 1). Providers with a key-info
* endpoint (see BaseProvider.fetchQuota) get polled right after a health pass
* confirms the key works, so the dashboard shows the provider's own balance
* instead of a regex guess over catalog text. Throttled per key: a poll is
* only worth taking when the last quota_api observation is older than
* QUOTA_POLL_INTERVAL_MS (or none exists). Fire-and-forget and failure-silent:
* a quota probe says nothing about key validity and must never bend a health
* verdict or the pass budget.
*/
export const QUOTA_POLL_INTERVAL_MS = 15 * 60 * 1000;
function quotaPollDue(keyId: number, now = Date.now()): boolean {
const row = getDb().prepare(`
SELECT created_at AS seenAt FROM provider_quota_observations
WHERE key_id = ? AND source = 'quota_api'
ORDER BY created_at DESC LIMIT 1
`).get(keyId) as { seenAt: string | null } | undefined;
if (!row?.seenAt) return true;
const seen = Date.parse(row.seenAt.includes('T') ? row.seenAt : row.seenAt.replace(' ', 'T') + 'Z');
if (!Number.isFinite(seen)) return true;
return now - seen >= QUOTA_POLL_INTERVAL_MS;
}
export async function pollKeyQuota(
keyId: number,
// The api_keys row as stored (encrypted key columns + the proxy columns
// decryptProxyUrl reads). `any` mirrors how checkKeyHealth carries it.
row: any,
provider: BaseProvider,
): Promise<boolean> {
try {
const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);
return await withKeyProxy(decryptProxyUrl(row), () => provider.fetchQuota(apiKey, {
platform: row.platform as Platform,
keyId,
quotaPoolKey: inferQuotaPoolKey(row.platform as Platform, null),
endpoint: 'quota_api',
origin: 'health',
}));
} catch {
// Inconclusive by construction: never throws into the health pass.
return false;
}
}
/** True when this provider actually has a quota endpoint to probe (#1403).
* Uses the provider's own hasQuotaProbe getter: OpenAICompatProvider
* overrides fetchQuota for every instance, spec or not, so a method-identity
* check would claim quota support for specless platforms. */
export function supportsQuotaPolling(provider: BaseProvider): boolean {
return provider.hasQuotaProbe;
}
/** Throttled, self-contained quota poll for one key — the entry point other
* services (or a future scheduler) can use without duplicating the gate. */
export async function maybePollKeyQuota(keyId: number): Promise<boolean> {
try {
const db = getDb();
const row = db.prepare('SELECT * FROM api_keys WHERE id = ? AND enabled = 1').get(keyId) as any;
if (!row) return false;
const provider = resolveProvider(row.platform as Platform, row.base_url);
if (!provider || !supportsQuotaPolling(provider)) return false;
if (!quotaPollDue(keyId)) return false;
return await pollKeyQuota(keyId, row, provider);
} catch {
return false;
}
}
export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
const db = getDb();
const row = db.prepare('SELECT * FROM api_keys WHERE id = ?').get(keyId) as any;
if (!row) return 'error';
const provider = resolveProvider(row.platform as Platform, row.base_url);
if (!provider) return 'error';
// #1459: a provider that forbids automated key checks gets no probe. Its
// status is left to real traffic (markKeyHealthyFromRequest), and any reason
// an older probe stored is cleared so the dashboard stops showing it.
if (provider.supportsKeyValidation === false) {
if (row.last_health_error != null) {
db.prepare('UPDATE api_keys SET last_health_error = NULL WHERE id = ?').run(keyId);
}
return row.status as KeyStatus;
}
try {
const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);
// #590: probe the key from the same exit its traffic uses. A key that is
// only reachable through its own proxy (the reason to set one) would
// otherwise be validated direct, fail, and be auto-disabled after three
// checks while real requests through the proxy were working fine.
const validation = await withKeyProxy(decryptProxyUrl(row), () => provider.validateKey(apiKey, {
platform: row.platform as Platform,
keyId,
quotaPoolKey: inferQuotaPoolKey(row.platform as Platform, null),
endpoint: 'models',
origin: 'health',
}));
const isValid = typeof validation === 'boolean' ? validation : validation.valid;
const lastError = isValid
? null
: sanitizeProviderErrorMessage(
typeof validation === 'boolean'
? `${provider.name} rejected the API key`
: validation.error,
);
const status: KeyStatus = isValid ? 'healthy' : 'invalid';
// #1493: stamp last_healthy_at only on a VALID verdict. A transport error
// (the catch below) refreshes last_checked_at but records no success, so
// "checked recently" and "worked recently" stay distinguishable.
db.prepare(isValid
? "UPDATE api_keys SET status = ?, last_health_error = ?, last_checked_at = datetime('now'), last_healthy_at = datetime('now') WHERE id = ?"
: "UPDATE api_keys SET status = ?, last_health_error = ?, last_checked_at = datetime('now') WHERE id = ?")
.run(status, lastError, keyId);
if (isValid) {
failureCount.delete(keyId);
// #1510: the provider answered — whatever unreachable streak this key had
// is over, so the transport counter starts from zero again.
transportFailures.delete(keyId);
// The provider answered again: whatever rate-limit told us to back off
// is over, so the key rejoins the normal cadence.
validationRateLimitedAt.delete(keyId);
// #1348: a healthy key on a built-in platform the catalog does not carry
// serves nothing until its models are discovered. Fire-and-forget; the
// trigger is a no-op for every other platform, and throttled.
triggerBuiltinModelDiscovery(db, row.platform, 'healthy');
// #1403: piggyback provider-reported quota polling on the healthy
// verdict — but only for providers with a quota endpoint, and at most
// once per QUOTA_POLL_INTERVAL_MS per key. Fire-and-forget, silent.
if (supportsQuotaPolling(provider) && quotaPollDue(keyId)) {
void pollKeyQuota(keyId, row, provider).catch(() => {});
}
} else {
providerLog(
'warn',
`[Health] Key ${keyId} (${row.platform}, base=${row.base_url ?? 'default'}) invalid: ${lastError}`,
{ provider: row.platform, event: 'key_invalid' },
);
recordInvalidFailure(keyId, row.platform);
}
return status;
} catch (err: any) {
// Transport errors (DNS/timeout/TLS) — provider unreachable, not necessarily
// a bad key. Mark status='error' but do NOT increment failure counter — auto-
// disable is reserved for confirmed 401/403 (returned by validateKey as false).
// Include platform + base_url so a flapping CloudFront edge or DNS failure is
// attributable to the responsible provider in one log read. The leading
// "[Health] Key N (" prefix is preserved so the 12-hourly crash watchdog
// (cron bff5ae167d28) that scrapes /tmp/freellmapi.log for these lines
// continues to match unchanged.
// A 429 here means the provider rate-limited the validation itself:
// remember it so the scheduled pass backs off instead of re-asking every
// 5 minutes (#1369). Other transport errors carry no such instruction.
if (err?.status === 429) recordValidationRateLimited(keyId);
const lastError = sanitizeProviderErrorMessage(err?.message ?? err);
console.error(
`[Health] Key ${keyId} (${row.platform}, base=${row.base_url ?? 'default'}) ` +
`transport error: ${lastError} — status preserved as '${row.status}'`,
);
// Do NOT write status='error' on the first transport failure. selectKeyForModel
// only considers keys with status IN ('healthy','unknown'), so demoting here
// silently removes the key's capacity for up to a full check interval — and a
// transport error is evidence about the network, not about the key. One flaky
// DNS lookup, a laptop suspended for thirty seconds, or a provider edge outage
// would take every key on that provider out of rotation at once. Record the
// diagnostic and the timestamp; leave the verdict to a probe that actually
// reached the provider. Confirmed 401/403 (the isValid=false path above) still
// demotes.
//
// #1510 adds the accumulation this path was missing (opt-in via
// HEALTH_TRANSPORT_PARK_AFTER): a key that has been unreachable that many
// probes in a row is parked at 'error' — still re-probed every pass, self-healing on the first answer
// or served request, but no longer first-class routing capacity. A 429 is
// proof the provider is reachable, so it counts as a break in the streak,
// not an increment.
let parked = false;
if (err?.status === 429) transportFailures.delete(keyId);
else parked = recordTransportFailure(keyId, row.platform);
db.prepare("UPDATE api_keys SET last_health_error = ?, last_checked_at = datetime('now') WHERE id = ?")
.run(lastError, keyId);
return (parked ? 'error' : row.status) as KeyStatus;
}
}
// ── Cooldown-probe validation (side-effect-free) ─────────────────────────────
// The cooldown-probe recovery job (services/cooldown-probe.ts) needs the same
// cheap validateKey call checkKeyHealth makes, WITHOUT any of its bookkeeping:
// no status writes, no last_checked_at, and above all no failureCount — probes
// run far more often than the 5-minute health pass, so letting them feed the
// consecutive-failure counter would auto-disable a genuinely-bad key in a
// fraction of the "3 consecutive checks" the threshold promises. A probe is a
// question, never a verdict.
export type KeyProbeOutcome = 'valid' | 'invalid' | 'error';
export async function probeKeyValidity(keyId: number): Promise<KeyProbeOutcome> {
try {
const db = getDb();
const row = db.prepare('SELECT * FROM api_keys WHERE id = ? AND enabled = 1').get(keyId) as any;
if (!row) return 'error';
const provider = resolveProvider(row.platform as Platform, row.base_url);
if (!provider) return 'error';
// #1459: no probe for a provider that forbids them; the cooldown simply
// runs out, as it would after an inconclusive probe.
if (provider.supportsKeyValidation === false) return 'error';
// #1599: the probe would be the same empty search the health pass no
// longer sends on a schedule; let the cooldown run out instead.
if (provider.supportsKeyPolling === false) return 'error';
const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);
// Same reasoning as checkKeyHealth: probe through the key's own proxy (#590).
const validation = await withKeyProxy(decryptProxyUrl(row), () => provider.validateKey(apiKey, {
platform: row.platform as Platform,
keyId,
quotaPoolKey: inferQuotaPoolKey(row.platform as Platform, null),
endpoint: 'models',
origin: 'probe',
}));
const isValid = typeof validation === 'boolean' ? validation : validation.valid;
return isValid ? 'valid' : 'invalid';
} catch {
// Transport error (DNS/timeout/TLS): inconclusive, same as checkKeyHealth's
// reasoning — evidence about the network, not the key.
return 'error';
}
}
/**
* #1493: throttle for the last_healthy_at write from live traffic. The probe
* path already writes the column at most once per health pass, but this
* function runs on every successful proxied request, and an unthrottled UPDATE
* would put a DB write on the hot path. One stamp per key per minute is plenty
* for "last worked N ago".
*/
const lastHealthyStampedAt = new Map<number, number>();
const LAST_HEALTHY_STAMP_INTERVAL_MS = 60_000;
/** Test seam: drop the stamp throttle so a test can observe consecutive writes. */
export function clearLastHealthyStampState(): void {
lastHealthyStampedAt.clear();
}
/**
* Promote a key out of 'error' or 'unknown' after it successfully served a live
* request.
*
* Serving traffic is stronger evidence than any probe, so a key stuck at 'error'
* from an earlier transport blip should not have to wait for the next health pass
* to become routable again, and a key never probed yet (or one whose provider
* forbids probes, #1459) has just proved itself. Deliberately narrow: 'invalid'
* means a provider confirmed the credential is bad, and only a real validateKey
* pass clears that.
*/
export function markKeyHealthyFromRequest(keyId: number): void {
try {
// #1493: a served request is the strongest "this key works" evidence there
// is — stamp last_healthy_at regardless of current status. This is the only
// success signal for providers with supportsKeyValidation === false (#1459),
// which never get a probe. Throttled in memory, not per request.
const now = Date.now();
const lastStamp = lastHealthyStampedAt.get(keyId);
if (lastStamp === undefined || now - lastStamp >= LAST_HEALTHY_STAMP_INTERVAL_MS) {
lastHealthyStampedAt.set(keyId, now);
getDb()
.prepare("UPDATE api_keys SET last_healthy_at = datetime('now') WHERE id = ?")
.run(keyId);
}
getDb()
.prepare("UPDATE api_keys SET status = 'healthy', last_health_error = NULL WHERE id = ? AND status IN ('error', 'unknown')")
.run(keyId);
failureCount.delete(keyId);
// #1510: a served request is the strongest reachability evidence there is —
// it also ends any transport-error streak, parked or not.
transportFailures.delete(keyId);
} catch {
// Never let health bookkeeping break a request that already succeeded.
}
}
// Overlap guard: the scheduled 5-minute pass and wake-recovery re-probes can
// coincide (or SIGCONT spam can queue several) — concurrent full passes
// multiply provider validate traffic and let two passes each increment the
// same genuinely-bad key's failureCount, reaching the auto-disable threshold
// in fewer wall-clock checks than "3 consecutive checks" intends. A second
// caller joins the in-flight pass instead of starting another.
let checkAllInFlight: Promise<HealthPassResult> | null = null;
interface HealthKeyRow {
id: number;
platform: string;
base_url: string | null;
status: string;
age_ms: number | null;
}
export interface HealthPassOptions {
/** Probe every enabled key now: no recency skip, no per-provider spacing.
* Used by the dashboard's "check all" button and the post-wake re-probe,
* where the point is an immediate, complete picture. */
force?: boolean;
/** Someone pressed "Check all": also validate keys on providers that must
* not be polled (#1599). The post-wake re-probe is forced but automatic, so
* it leaves them alone. */
manual?: boolean;
/** Test seams: injectable clock, sleep, probe and pool size, mirroring the
* cooldown-probe pass so a pacing test needs no real timers. */
now?: () => number;
sleep?: (ms: number) => Promise<void>;
check?: (keyId: number) => Promise<unknown>;
concurrency?: number;
minSpacingMs?: number;
}
export interface HealthPassResult {
/** Keys probed this pass, in the order the pass started them. */
checkedKeyIds: number[];
/** Enabled keys left alone because they were validated recently. */
skippedKeyIds: number[];
}
/** The provider a probe actually lands on: two openai-compat keys pointed at
* different base URLs are different hosts and need no spacing between them. */
function providerBucket(row: HealthKeyRow): string {
return row.base_url ? `${row.platform}|${row.base_url}` : row.platform;
}
/**
* Round-robin the queue across providers: one key from each provider, then the
* next from each, and so on. Raw DB order is provider-clustered (keys are added
* a provider at a time), which is what turned a fleet with 40 keys on one
* provider into 40 back-to-back requests from one IP (#553). Exported for tests.
*/
export function interleaveByProvider<T>(rows: T[], bucketOf: (row: T) => string): T[] {
const buckets = new Map<string, T[]>();
for (const row of rows) {
const key = bucketOf(row);
const list = buckets.get(key);
if (list) list.push(row);
else buckets.set(key, [row]);
}
const lists = [...buckets.values()];
const out: T[] = [];
for (let i = 0; out.length < rows.length; i++) {
for (const list of lists) {
if (i < list.length) out.push(list[i]!);
}
}
return out;
}
const realSleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));
export function checkAllKeys(opts: HealthPassOptions = {}): Promise<HealthPassResult> {
// A caller that arrives mid-pass joins it (including a forced one — the
// in-flight pass is already probing, and stacking a second set of validates
// on the same providers is the exact behaviour this guard exists to prevent).
if (checkAllInFlight) return checkAllInFlight;
checkAllInFlight = runHealthPass(opts).finally(() => {
checkAllInFlight = null;
});
// Keep the degraded-mode state machine in step with the latest verdicts
// (#904): a fleet-wide outage should flip the gateway into degraded mode
// shortly after the pass that observed it, not after the next request.
void checkAllInFlight.then(() => updateDegradationState());
return checkAllInFlight;
}
async function runHealthPass(opts: HealthPassOptions): Promise<HealthPassResult> {
const now = opts.now ?? Date.now;
const sleep = opts.sleep ?? realSleep;
const check = opts.check ?? checkKeyHealth;
const db = getDb();
// age_ms comes from the DB clock (last_checked_at is written as
// datetime('now')); the injected `now` above only drives pacing.
const rows = db.prepare(`
SELECT id, platform, base_url, status,
CAST((julianday('now') - julianday(last_checked_at)) * 86400000 AS INTEGER) AS age_ms
FROM api_keys WHERE enabled = 1
`).all() as HealthKeyRow[];
const skippedKeyIds: number[] = [];
const skipMs = opts.force ? 0 : getValidationRateLimitSkipMs();
const due = rows.filter(row => {
// #1599: a provider whose key check is an empty search gets one check per
// new key, not one every pass. Applies to forced automatic passes too.
if (!opts.manual && resolveProvider(row.platform as Platform, row.base_url)?.supportsKeyPolling === false) {
const floorMs = row.status === 'error' ? NO_POLL_ERROR_REPROBE_MS : Infinity;
if (row.age_ms !== null && row.age_ms < floorMs) {
skippedKeyIds.push(row.id);
return false;
}
return true;
}
if (opts.force) return true;
// A key the provider just rate-limited stays out until the skip window
// passes — even one parked at 'error'. A 429 is fresher, more specific
// evidence than the eagerness to re-ask an errored key (#1369).
const limitedAt = validationRateLimitedAt.get(row.id);
if (limitedAt !== undefined && now() - limitedAt < skipMs) {
skippedKeyIds.push(row.id);
return false;
}
// A key parked at 'error' is out of rotation until a probe says otherwise
// (the router writes that status, and last_checked_at with it), so it is
// never skipped — it is the one key whose verdict is worth re-asking for.
if (row.status === 'error') return true;
if (row.age_ms !== null && row.age_ms < RECENT_CHECK_SKIP_MS) {
skippedKeyIds.push(row.id);
return false;
}
return true;
});
const queue = interleaveByProvider(due, providerBucket);
const perBucket = new Map<string, number>();
for (const row of queue) {
const bucket = providerBucket(row);
perBucket.set(bucket, (perBucket.get(bucket) ?? 0) + 1);
}
const largestBucket = Math.max(0, ...perBucket.values());
// Spacing shrinks rather than letting a big fleet run past the budget: the
// provider with the most keys sets the pass length, so its gaps are what the
// budget has to divide.
const requestedSpacing = opts.minSpacingMs ?? getMinSpacingMs();
const spacingMs = opts.force || largestBucket < 2
? 0
: Math.min(requestedSpacing, Math.floor(HEALTH_PASS_TIME_BUDGET_MS / (largestBucket - 1)));
console.log(
`[Health] Checking ${queue.length} keys` +
(skippedKeyIds.length > 0 ? ` (${skippedKeyIds.length} checked recently, skipped)` : '') +
(spacingMs > 0 ? ` — ${spacingMs}ms between probes of the same provider` : '') + '...',
);
// Bounded worker pool rather than a sequential await. validateKey allows up
// to 30s per key (some provider /models endpoints are genuinely that slow),
// so a serial pass over a large key fleet can outlast the 5-minute interval
// it is scheduled on and leave the dashboard showing stale statuses. The cap
// keeps us from opening one socket per key against the same provider.
const concurrency = opts.concurrency ?? getHealthCheckConcurrency();
const nextAllowedAt = new Map<string, number>();
const checkedKeyIds: number[] = [];
let cursor = 0;
const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
while (cursor < queue.length) {
const key = queue[cursor++]!;
if (spacingMs > 0) {
// Reserve the slot BEFORE awaiting, so two workers that pick up keys on
// the same provider queue behind each other instead of both waiting for
// the same instant and firing together.
const bucket = providerBucket(key);
const slot = Math.max(nextAllowedAt.get(bucket) ?? 0, now());
nextAllowedAt.set(bucket, slot + spacingMs);
const wait = slot - now();
if (wait > 0) await sleep(wait);
}
checkedKeyIds.push(key.id);
try {
await check(key.id);
} catch (err) {
// checkKeyHealth handles its own errors; this is a backstop so one
// rejection cannot abandon the rest of the pass.
console.error(`[Health] Key ${key.id} check threw:`, err);
}
}
});
await Promise.all(workers);
console.log(`[Health] Check complete.`);
return { checkedKeyIds, skippedKeyIds };
}
/** Delay until the next scheduled pass: the base interval ±20%, so restarts and
* co-deployed gateways don't stay phase-locked on the same providers. */
export function nextHealthCheckDelayMs(jitter: () => number = Math.random): number {
return Math.round(CHECK_INTERVAL_MS * (1 + (jitter() * 2 - 1) * CHECK_INTERVAL_JITTER));
}
let cancelHealthCheck: (() => void) | null = null;
let healthCheckerRunning = false;
export function startHealthChecker(scheduler: Scheduler): void {
if (healthCheckerRunning) return;
healthCheckerRunning = true;
console.log(
`[Health] Starting health checker (every ~${CHECK_INTERVAL_MS / 1000}s ±${CHECK_INTERVAL_JITTER * 100}%)`,
);
// Self-rescheduling one-shot timers rather than a fixed interval: each pass
// picks its own jittered delay (see nextHealthCheckDelayMs), and the next one
// is only armed once the current pass has finished.
const scheduleNext = (): void => {
cancelHealthCheck = scheduler.after(nextHealthCheckDelayMs(), async () => {
try {
await checkAllKeys();
} catch (err) {
console.error('[Health] Check failed:', err);
}
if (healthCheckerRunning) scheduleNext();
});
};
scheduleNext();
}
export function stopHealthChecker(): void {
healthCheckerRunning = false;
if (cancelHealthCheck) {
cancelHealthCheck();
cancelHealthCheck = null;
}
}