Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(models): proactive per-model probing with explicit status labels (#1438) (#1492)

suantea (asuan_suan)6c021c19d6fc
diff --git a/.env.example b/.env.example
index f8b108a9..5e5c8245 100644
--- a/.env.example
+++ b/.env.example
@@ -162,6 +162,14 @@ PORT=3001
# which costs quota where a provider's key check is metered. Read at startup.
# HEALTH_CHECK_INTERVAL_MS=300000
+# Opt-in background model probes (#1438): every 30 minutes, send a tiny
+# "Reply with OK" chat to up to MODEL_PROBE_MAX_PER_PASS (default 3) model+key
+# pairs and record whether each model answers. Off by default because every
+# probe spends free-tier quota. POST /api/health/probe-models runs one pass on
+# demand either way.
+# MODEL_PROBE_ENABLED=1
+# MODEL_PROBE_MAX_PER_PASS=3
+
# Base wall-clock budget for provider failover, in milliseconds (default: 45000).
# The initial attempt and one failover hop are always allowed. After that,
# exhaustion stops new attempts and can cancel an in-flight attempt still
diff --git a/desktop/src/__tests__/server-host-boot.test.ts b/desktop/src/__tests__/server-host-boot.test.ts
index 56872bcb..193ed165 100644
--- a/desktop/src/__tests__/server-host-boot.test.ts
+++ b/desktop/src/__tests__/server-host-boot.test.ts
@@ -64,6 +64,12 @@ vi.mock('../../../server/src/services/cooldown-probe.js', () => ({
}),
}));
+vi.mock('../../../server/src/services/model-probe.js', () => ({
+ startModelProbe: vi.fn(() => {
+ calls.push('startModelProbe');
+ }),
+}));
+
vi.mock('../../../server/src/services/backups.js', () => ({
startBackupScheduler: vi.fn(() => {
calls.push('startBackupScheduler');
@@ -174,6 +180,7 @@ describe('desktop server boot sequence (#949)', () => {
'startHealthChecker',
'startCatalogSync',
'startCooldownProbe',
+ 'startModelProbe',
'startBackupScheduler',
'startCustomModelSync',
'startBuiltinModelDiscovery',
diff --git a/desktop/src/server-host.ts b/desktop/src/server-host.ts
index 735a159b..772812e8 100644
--- a/desktop/src/server-host.ts
+++ b/desktop/src/server-host.ts
@@ -15,6 +15,7 @@ import { restoreProxySettings, flushProxyCache, getProxyMode, getProxyUrl, isPro
import { startHealthChecker, checkAllKeys } from '../../server/src/services/health.js';
import { startCatalogSync } from '../../server/src/services/catalog-sync.js';
import { startCooldownProbe } from '../../server/src/services/cooldown-probe.js';
+import { startModelProbe } from '../../server/src/services/model-probe.js';
import { startCustomModelSync } from '../../server/src/services/custom-model-sync.js';
import { startBuiltinModelDiscovery } from '../../server/src/services/builtin-model-discovery.js';
import { startBackupScheduler, createBackup } from '../../server/src/services/backups.js';
@@ -154,6 +155,7 @@ export async function startServer(opts: StartOptions): Promise<ServerHandle> {
startHealthChecker(scheduler);
startCatalogSync(scheduler);
startCooldownProbe(scheduler);
+ startModelProbe(scheduler);
startBackupScheduler(scheduler);
startCustomModelSync(getDb(), scheduler);
startBuiltinModelDiscovery(getDb(), scheduler);
diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.ts
index 360fba4f..e9d936ae 100644
--- a/server/src/__tests__/db/migrate/roundtrip.test.ts
+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts
@@ -43,6 +43,7 @@ const REQUEST_MODEL_ATTRIBUTION_FILENAME = '20260913_000001_request_model_attrib
const KEY_MONTHLY_USAGE_FILENAME = '20260914_000001_key_monthly_usage.ts';
const QUOTA_SNAPSHOT_FRESHNESS_FILENAME = '20260915_000001_quota_snapshot_freshness.ts';
const CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME = '20260930_000001_custom_endpoint_group_label.ts';
+const MODEL_PROBE_STATUS_FILENAME = '20261007_000001_model_probe_status.ts';
const SEARCH_ENGINES_FILENAME = '20261013_000001_search_engines.ts';
const DECISION_REQUESTS_FILENAME = '20261016_000001_decision_requests.ts';
const LLMTR_REENABLE_KEYS_FILENAME = '20261017_000001_llmtr_reenable_keys.ts';
@@ -139,6 +140,7 @@ describe('migration round trip', () => {
KEY_MONTHLY_USAGE_FILENAME,
QUOTA_SNAPSHOT_FRESHNESS_FILENAME,
CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME,
+ MODEL_PROBE_STATUS_FILENAME,
SEARCH_ENGINES_FILENAME,
DECISION_REQUESTS_FILENAME,
LLMTR_REENABLE_KEYS_FILENAME,
diff --git a/server/src/__tests__/services/model-probe.test.ts b/server/src/__tests__/services/model-probe.test.ts
new file mode 100644
index 00000000..fb954df7
--- /dev/null
+++ b/server/src/__tests__/services/model-probe.test.ts
@@ -0,0 +1,210 @@
+// model-probe tests (#1438 PR 1): candidate selection (stalest-first, fresh
+// real-traffic skip), label recording, per-pass budget, restart stagger +
+// first-sighting scheduling, failed-probe backoff, transport-error
+// inconclusiveness (verdict preserved), and the kill switch.
+
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import {
+ selectProbeCandidates,
+ probeModel,
+ recordProbeResult,
+ runModelProbePass,
+ resetModelProbeState,
+ startModelProbe,
+ stopModelProbe,
+ MODEL_PROBE_STALE_MS,
+} from '../../services/model-probe.js';
+import type { ProbeCandidate } from '../../services/model-probe.js';
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+beforeEach(() => {
+ resetModelProbeState();
+ const db = getDb();
+ db.prepare('DELETE FROM model_probe_status').run();
+ db.prepare('DELETE FROM fallback_config').run();
+ db.prepare('DELETE FROM profile_models').run();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM models').run();
+ // Fresh healthy key the models can hang off.
+ db.prepare(`INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (1, 'probe-test', '', '', '', '', 'healthy', 1)`).run();
+});
+
+function addModel(id: number, modelId: string, opts: { enabled?: number } = {}): void {
+ getDb().prepare(`INSERT INTO models (id, platform, model_id, display_name, intelligence_rank, speed_rank, endpoint_scope, enabled)
+ VALUES (?, 'probe-test', ?, ?, 0, 0, '', ?)`).run(id, modelId, modelId, opts.enabled ?? 1);
+}
+
+const candidate = (over: Partial<ProbeCandidate> = {}): ProbeCandidate => ({
+ keyId: 1,
+ platform: 'probe-test',
+ modelId: 'm1',
+ modelDbId: 1,
+ endpointScope: '',
+ ...over,
+});
+
+// Seed one pass' worth of candidates so stagger/backoff choreography is
+// deterministic: first sighting schedules (no probe), next pass probes.
+function seedCandidates(n: number): ProbeCandidate[] {
+ const out: ProbeCandidate[] = [];
+ for (let i = 1; i <= n; i++) {
+ addModel(i, `m${i}`);
+ out.push(candidate({ modelId: `m${i}`, modelDbId: i }));
+ }
+ return out;
+}
+
+const selectFrom = (list: ProbeCandidate[]) => (limit: number) => list.slice(0, limit);
+const chatOk = () => vi.fn(async () => ({}));
+const chatFail = (message: string) => vi.fn(async () => { throw new Error(message); });
+const noJitter = () => 0;
+
+describe('candidate selection', () => {
+ it('probes the stalest evidence first, never-probed before stale', () => {
+ const db = getDb();
+ addModel(1, 'old');
+ addModel(2, 'fresh');
+ addModel(3, 'never');
+ recordProbeResult(db, candidate({ modelId: 'old', modelDbId: 1 }),
+ { status: 'working', latencyMs: 5 }, new Date(Date.now() - MODEL_PROBE_STALE_MS - 1000));
+ recordProbeResult(db, candidate({ modelId: 'fresh', modelDbId: 2 }),
+ { status: 'working', latencyMs: 5 }, new Date());
+
+ const picked = selectProbeCandidates(db, 10);
+ expect(picked.map(p => p.modelId)).toEqual(['never', 'old']);
+ });
+
+ it('skips models with fresh evidence (staleness window)', () => {
+ const db = getDb();
+ addModel(1, 'fresh');
+ recordProbeResult(db, candidate({ modelId: 'fresh', modelDbId: 1 }),
+ { status: 'working', latencyMs: 5 }, new Date());
+ expect(selectProbeCandidates(db, 10)).toEqual([]);
+ });
+
+ it('ignores disabled models and disabled/invalid keys', () => {
+ const db = getDb();
+ addModel(1, 'off-model', { enabled: 0 });
+ addModel(2, 'bad-key-model');
+ db.prepare("UPDATE api_keys SET status = 'invalid' WHERE id = 1").run();
+ expect(selectProbeCandidates(db, 10)).toEqual([]);
+ });
+});
+
+describe('probe outcome classification', () => {
+ it('a successful chat is working with latency', async () => {
+ const outcome = await probeModel(candidate(), { chat: chatOk() });
+ expect(outcome.status).toBe('working');
+ expect(outcome.latencyMs).toBeGreaterThanOrEqual(0);
+ });
+
+ it('a provider rejection is unavailable and carries the sanitized error', async () => {
+ const outcome = await probeModel(candidate(), { chat: chatFail('403: quota exhausted upstream') });
+ expect(outcome.status).toBe('unavailable');
+ expect(outcome.error).toContain('403');
+ });
+
+ it('a transport error is inconclusive (error), never unavailable', async () => {
+ const outcome = await probeModel(candidate(), { chat: chatFail('fetch failed: getaddrinfo ENOTFOUND api.example.com') });
+ expect(outcome.status).toBe('error');
+ });
+});
+
+describe('recording', () => {
+ it('writes working with latency and clears prior errors', () => {
+ const db = getDb();
+ recordProbeResult(db, candidate(), { status: 'unavailable', error: '403' });
+ recordProbeResult(db, candidate(), { status: 'working', latencyMs: 42 });
+ const row = db.prepare('SELECT * FROM model_probe_status').get() as any;
+ expect(row.status).toBe('working');
+ expect(row.latency_ms).toBe(42);
+ expect(row.last_error).toBeNull();
+ });
+
+ it('an inconclusive probe preserves the previous verdict', () => {
+ const db = getDb();
+ recordProbeResult(db, candidate(), { status: 'working', latencyMs: 10 });
+ recordProbeResult(db, candidate(), { status: 'error', error: 'fetch failed' });
+ const row = db.prepare('SELECT * FROM model_probe_status').get() as any;
+ expect(row.status).toBe('working');
+ expect(row.last_error).toBe('fetch failed');
+ });
+});
+
+describe('probe pass', () => {
+ it('first sighting schedules instead of probing; the next pass probes within budget', async () => {
+ const list = seedCandidates(5);
+ const chat = chatOk();
+ const pass1 = await runModelProbePass({ chat, jitter: noJitter, maxProbes: 3, select: selectFrom(list) });
+ expect(pass1.probed).toEqual([]);
+ expect(chat).not.toHaveBeenCalled();
+
+ const pass2 = await runModelProbePass({ chat, jitter: noJitter, maxProbes: 3, select: selectFrom(list) });
+ expect(pass2.probed).toHaveLength(3);
+ expect(chat).toHaveBeenCalledTimes(3);
+ });
+
+ it('a working probe clears pacing so the model is not re-probed until stale', async () => {
+ const list = seedCandidates(1);
+ const chat = chatOk;
+ await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });
+ await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });
+ // Third pass: the single candidate just worked; nothing is probeable.
+ const pass3 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });
+ expect(pass3.probed).toEqual([]);
+ });
+
+ it('an unavailable probe backs off instead of re-probing every pass', async () => {
+ const list = seedCandidates(1);
+ const chat = chatFail('403 quota');
+ // Pass 1: first sighting only schedules.
+ await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });
+ // Pass 2: the actual probe — fails, sets a 30m backoff.
+ const pass2 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });
+ expect(pass2.probed).toHaveLength(1);
+ // Pass 3 immediately after: still inside the backoff, nothing probed.
+ const pass3 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });
+ expect(pass3.probed).toEqual([]);
+ // Past the first backoff step (30m) it probes again.
+ const later = Date.now() + 31 * 60_000;
+ const pass4 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list), now: later });
+ expect(pass4.probed).toHaveLength(1);
+ });
+
+ it('records outcomes in the table', async () => {
+ const list = seedCandidates(2);
+ await runModelProbePass({ chat: chatOk(), jitter: noJitter, select: selectFrom(list) });
+ await runModelProbePass({ chat: chatOk(), jitter: noJitter, select: selectFrom(list) });
+ const rows = getDb().prepare('SELECT model_id, status FROM model_probe_status ORDER BY model_id').all() as any[];
+ expect(rows).toEqual([
+ { model_id: 'm1', status: 'working' },
+ { model_id: 'm2', status: 'working' },
+ ]);
+ });
+});
+
+describe('kill switch and lifecycle', () => {
+ it('is off by default: the scheduler does not start without MODEL_PROBE_ENABLED=1', () => {
+ delete process.env.MODEL_PROBE_ENABLED;
+ const scheduler = { every: vi.fn(() => () => undefined) };
+ startModelProbe(scheduler as any);
+ expect(scheduler.every).not.toHaveBeenCalled();
+ stopModelProbe();
+ });
+
+ it('MODEL_PROBE_ENABLED=1 schedules the job on the 30m cadence', () => {
+ process.env.MODEL_PROBE_ENABLED = '1';
+ const scheduler = { every: vi.fn(() => () => undefined) };
+ startModelProbe(scheduler as any);
+ expect(scheduler.every).toHaveBeenCalledTimes(1);
+ expect(scheduler.every.mock.calls[0][0]).toBe(30 * 60 * 1000);
+ delete process.env.MODEL_PROBE_ENABLED;
+ stopModelProbe();
+ });
+});
diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.ts
index b875446d..5222df2d 100644
--- a/server/src/db/migrate/defaults.ts
+++ b/server/src/db/migrate/defaults.ts
@@ -38,6 +38,7 @@ import * as keyMonthlyUsage from '../migrations/20260914_000001_key_monthly_usag
import * as quotaSnapshotFreshness from '../migrations/20260915_000001_quota_snapshot_freshness.js';
import * as requestModelAttribution from '../migrations/20260913_000001_request_model_attribution.js';
import * as customEndpointGroupLabel from '../migrations/20260930_000001_custom_endpoint_group_label.js';
+import * as modelProbeStatus from '../migrations/20261007_000001_model_probe_status.js';
import * as searchEngines from '../migrations/20261013_000001_search_engines.js';
import * as decisionRequests from '../migrations/20261016_000001_decision_requests.js';
import * as llmtrReenableKeys from '../migrations/20261017_000001_llmtr_reenable_keys.js';
@@ -93,6 +94,7 @@ export const RESPONSE_CACHE_FILENAME = '20260903_000002_response_cache.ts';
export const KEY_MONTHLY_BUDGET_FILENAME = '20260904_000001_key_monthly_budget.ts';
export const REQUEST_MODEL_ATTRIBUTION_FILENAME = '20260913_000001_request_model_attribution.ts';
export const CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME = '20260930_000001_custom_endpoint_group_label.ts';
+export const MODEL_PROBE_STATUS_FILENAME = '20261007_000001_model_probe_status.ts';
export const KEY_MONTHLY_USAGE_FILENAME = '20260914_000001_key_monthly_usage.ts';
export const QUOTA_SNAPSHOT_FRESHNESS_FILENAME = '20260915_000001_quota_snapshot_freshness.ts';
export const SEARCH_ENGINES_FILENAME = '20261013_000001_search_engines.ts';
@@ -143,6 +145,7 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: KEY_MONTHLY_USAGE_FILENAME, module: keyMonthlyUsage },
{ filename: QUOTA_SNAPSHOT_FRESHNESS_FILENAME, module: quotaSnapshotFreshness },
{ filename: CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME, module: customEndpointGroupLabel },
+ { filename: MODEL_PROBE_STATUS_FILENAME, module: modelProbeStatus },
{ filename: SEARCH_ENGINES_FILENAME, module: searchEngines },
{ filename: DECISION_REQUESTS_FILENAME, module: decisionRequests },
{ filename: LLMTR_REENABLE_KEYS_FILENAME, module: llmtrReenableKeys },
diff --git a/server/src/db/migrations/20261007_000001_model_probe_status.ts b/server/src/db/migrations/20261007_000001_model_probe_status.ts
new file mode 100644
index 00000000..ef5942af
--- /dev/null
+++ b/server/src/db/migrations/20261007_000001_model_probe_status.ts
@@ -0,0 +1,28 @@
+// model_probe_status — proactive per-model probe results (#1438 PR 1).
+//
+// One row per (platform, model_id, endpoint_scope, key_id): the outcome of the
+// most recent proactive probe for that routable model+key pair. Purely
+// observational — nothing in routing or scoring reads this table in this PR;
+// a follow-up consumes the labels. DOWN drops the table; probe data is a
+// cache of reality, not a record, so dropping it loses nothing.
+import type { Db } from '../types.js';
+
+export function up(db: Db): void {
+ db.exec(`CREATE TABLE IF NOT EXISTS model_probe_status (
+ platform TEXT NOT NULL,
+ model_id TEXT NOT NULL,
+ endpoint_scope TEXT,
+ key_id INTEGER NOT NULL,
+ status TEXT NOT NULL DEFAULT 'unknown',
+ last_probed_at TEXT,
+ last_error TEXT,
+ latency_ms INTEGER,
+ PRIMARY KEY (platform, model_id, endpoint_scope, key_id)
+ )`);
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_model_probe_status_probed_at
+ ON model_probe_status(last_probed_at)`);
+}
+
+export function down(db: Db): void {
+ db.exec('DROP TABLE IF EXISTS model_probe_status');
+}
diff --git a/server/src/index.ts b/server/src/index.ts
index b477a045..35f11e3e 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -6,6 +6,7 @@ import { restoreProxySettings, flushProxyCache } from './lib/proxy.js';
import { startWakeDetect } from './lib/wake-detect.js';
import { startCatalogSync } from './services/catalog-sync.js';
import { startCooldownProbe } from './services/cooldown-probe.js';
+import { startModelProbe } from './services/model-probe.js';
import { startCustomModelSync } from './services/custom-model-sync.js';
import { startBuiltinModelDiscovery } from './services/builtin-model-discovery.js';
import { installProcessSafetyNet } from './lib/process-safety-net.js';
@@ -84,6 +85,7 @@ async function main() {
startHealthChecker(scheduler);
startCatalogSync(scheduler);
startCooldownProbe(scheduler);
+ startModelProbe(scheduler);
startDbBackupPump(getDb(), scheduler, config.dbPath ?? undefined);
startBackupScheduler(scheduler);
startCustomModelSync(getDb(), scheduler);
diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts
index 0d6cc8a7..6939eb26 100644
--- a/server/src/routes/health.ts
+++ b/server/src/routes/health.ts
@@ -3,6 +3,7 @@ import type { Request, Response } from 'express';
import { getDb } from '../db/index.js';
import { checkKeyHealth, checkAllKeys, pollKeyQuota, supportsQuotaPolling } from '../services/health.js';
import { getDegradationStatus } from '../services/degradation.js';
+import { runModelProbePass } from '../services/model-probe.js';
import { hasProvider, resolveProvider } from '../providers/index.js';
import { getQuotaStateForKeys } from '../services/provider-quota.js';
import type { Platform } from '@freellmapi/shared/types.js';
@@ -112,3 +113,27 @@ healthRouter.post('/quota/:keyId', async (req: Request, res: Response) => {
);
res.json({ keyId, recorded, quotaStates: state });
});
+
+// Run one proactive model-probe pass now (#1438 PR 1). The scheduled pass runs
+// every 30 minutes; this is the manual override — an operator wanting fresh
+// labels immediately instead of waiting for the next pass. Same shape as
+// POST /check-all: the operator pressed the button, so run a full pass.
+healthRouter.post('/probe-models', async (_req: Request, res: Response) => {
+ const result = await runModelProbePass();
+ res.json({
+ probed: result.probed.map(c => ({
+ platform: c.platform,
+ modelId: c.modelId,
+ keyId: c.keyId,
+ endpointScope: c.endpointScope,
+ })),
+ outcomes: result.outcomes.map(o => ({
+ platform: o.platform,
+ modelId: o.modelId,
+ keyId: o.keyId,
+ status: o.outcome.status,
+ latencyMs: o.outcome.latencyMs ?? null,
+ error: o.outcome.error ?? null,
+ })),
+ });
+});
diff --git a/server/src/services/model-probe.ts b/server/src/services/model-probe.ts
new file mode 100644
index 00000000..8454d491
--- /dev/null
+++ b/server/src/services/model-probe.ts
@@ -0,0 +1,345 @@
+// model-probe — proactive per-model probes with explicit labels (#1438 PR 1).
+//
+// Key-level health says a credential works; it says nothing about whether a
+// specific MODEL behind that key actually answers. Free-tier aggregators
+// routinely list models that 403 on every real chat (upstream quota drained),
+// and the router only learns that after burning a real user turn (#1267 is
+// passive: it demotes a model after real traffic has already hit it).
+//
+// This job sends a minimal real chat request ("Reply with OK") to routable
+// model+key pairs that have not been observed recently, and writes an explicit
+// status row: 'working' | 'unavailable'. PR 1 is OBSERVATION ONLY — nothing in
+// routing, scoring or fallback reads the table yet; a follow-up consumes the
+// labels (tool-call variant, chat_only classification, dashboard badges).
+//
+// What it will never do:
+// - probe a model that just served real traffic successfully (serving is
+// stronger evidence than any probe, and probes cost free-tier quota);
+// - feed the health checker, cooldowns or the failure counters — a probe is
+// a question, never a verdict (same principle as probeKeyValidity);
+// - use the fallback/healing machinery: the probe measures the model as-is,
+// not the pipeline, so a probe never retries and never repairs;
+// - thundering-herd a provider after a restart: first-sighting staggering
+// plus a small per-pass budget keep probe traffic bounded.
+//
+// Cadence: one pass every 30 minutes, at most MODEL_PROBE_MAX_PER_PASS probes
+// per pass (default 3). OFF by default: probes spend the user's free-tier
+// quota and nothing consumes the labels yet, so the scheduled job only runs
+// with MODEL_PROBE_ENABLED=1. POST /api/health/probe-models always runs a pass.
+
+import type { Scheduler } from '../lib/scheduler.js';
+import type { Db } from '../db/types.js';
+import { getDb } from '../db/index.js';
+import { resolveProvider } from '../providers/index.js';
+import { decrypt } from '../lib/crypto.js';
+import { decryptProxyUrl } from '../lib/key-proxy.js';
+import { withKeyProxy } from '../lib/proxy.js';
+import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
+import { providerLog } from '../lib/server-logs.js';
+import { inferQuotaPoolKey } from './provider-quota.js';
+
+// One pass every 30 minutes. Probes are quota-costing real requests; the point
+// is freshness on the scale of "an operator glances at the dashboard", not
+// second-level liveness (the cooldown/health machinery owns that).
+export const MODEL_PROBE_INTERVAL_MS = 30 * 60 * 1000;
+
+// Stale rows are re-probed after this long; fresher evidence is trusted and
+// the model is skipped. Well above the pass cadence so a skipped model is not
+// re-considered every pass.
+export const MODEL_PROBE_STALE_MS = 6 * 60 * 60 * 1000;
+
+// A model+key that served a real request within this window does not need a
+// probe: real success is stronger evidence, and the probe would only spend
+// free-tier quota to re-learn it. Mirrors the RECENT_CHECK_SKIP_MS idea from
+// the health pass.
+export const RECENT_SERVE_SKIP_MS = 60 * 60 * 1000;
+
+const DEFAULT_MAX_PROBES_PER_PASS = 3;
+
+// Ceiling for one probe request, mirroring health's transport posture. A probe
+// that takes longer than this is treated as an error (inconclusive), not as
+// evidence the model is unavailable.
+const PROBE_TIMEOUT_MS = 60 * 1000;
+
+// max_tokens floor: some relays reject max_tokens <= 2 (#903). The probe never
+// relies on tiny ceilings, so a generous floor avoids that whole failure class.
+const PROBE_MAX_TOKENS = 64;
+
+/** Per-pass probe budget. Tunable; 0 or a bad value falls back to the default
+ * (mirrors HEALTH_CHECK_CONCURRENCY / COOLDOWN_PROBE_MAX_PER_PASS). */
+function getMaxProbesPerPass(): number {
+ const raw = process.env.MODEL_PROBE_MAX_PER_PASS;
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isInteger(n) && n > 0) return n;
+ }
+ return DEFAULT_MAX_PROBES_PER_PASS;
+}
+
+export type ModelProbeStatus = 'working' | 'unavailable';
+
+export interface ProbeCandidate {
+ keyId: number;
+ platform: string;
+ modelId: string;
+ modelDbId: number;
+ endpointScope: string;
+}
+
+export interface ProbeOutcome {
+ status: ModelProbeStatus | 'error';
+ latencyMs?: number;
+ error?: string;
+}
+
+export interface ProbePassResult {
+ probed: ProbeCandidate[];
+ outcomes: Array<ProbeCandidate & { outcome: ProbeOutcome }>;
+}
+
+// ── Candidate selection ──────────────────────────────────────────────────────
+// Routable pairs only: enabled model on an enabled, currently-healthy-or-
+// unknown key, with endpoint scope carried through (custom-endpoint identity,
+// migration 20260729_000001). Ordered to rotate fairly: stalest evidence
+// first, models never probed (NULL last_probed_at) before stale ones.
+
+interface ProbeCandidateRow extends ProbeCandidate {
+ lastProbedAt: string | null;
+}
+
+export function selectProbeCandidates(db: Db, limit: number, now = Date.now()): ProbeCandidateRow[] {
+ const rows = db.prepare(`
+ SELECT m.id AS modelDbId, m.platform, m.model_id AS modelId, m.endpoint_scope AS endpointScope,
+ k.id AS keyId, COALESCE(p.last_probed_at, '') AS lastProbedAt
+ FROM models m
+ JOIN api_keys k ON k.platform = m.platform AND k.enabled = 1 AND k.status IN ('healthy', 'unknown')
+ LEFT JOIN model_probe_status p
+ ON p.platform = m.platform AND p.model_id = m.model_id
+ AND p.endpoint_scope = m.endpoint_scope AND p.key_id = k.id
+ WHERE m.enabled = 1
+ ORDER BY (p.last_probed_at IS NULL) DESC, p.last_probed_at ASC, m.id ASC, k.id ASC
+ LIMIT ?
+ `).all(limit * 8) as Array<{
+ modelDbId: number; platform: string; modelId: string; endpointScope: string;
+ keyId: number; lastProbedAt: string;
+ }>;
+
+ const out: ProbeCandidateRow[] = [];
+ for (const row of rows) {
+ if (out.length >= limit) break;
+ if (row.lastProbedAt) {
+ const seen = Date.parse(row.lastProbedAt.includes('T') ? row.lastProbedAt : row.lastProbedAt.replace(' ', 'T') + 'Z');
+ if (Number.isFinite(seen) && now - seen < MODEL_PROBE_STALE_MS) continue;
+ }
+ out.push(row);
+ }
+ return out;
+}
+
+// ── The probe itself ─────────────────────────────────────────────────────────
+
+export interface ModelProbeRequestDeps {
+ // Test seam: the chat call the probe makes. Production uses the provider's
+ // own chatCompletion through the key's proxy.
+ chat?: (candidate: ProbeCandidate) => Promise<unknown>;
+}
+
+async function probeChat(candidate: ProbeCandidate): Promise<void> {
+ const db = getDb();
+ const row = db.prepare('SELECT * FROM api_keys WHERE id = ? AND enabled = 1').get(candidate.keyId) as any;
+ if (!row) throw new Error('key not found or disabled');
+ const provider = resolveProvider(candidate.platform as any, row.base_url);
+ if (!provider) throw new Error(`no provider adapter for ${candidate.platform}`);
+
+ const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);
+ // Same reasoning as checkKeyHealth (#590): probe through the key's own proxy,
+ // or a key only reachable via its proxy would fail the probe while real
+ // traffic through the proxy works fine.
+ await withKeyProxy(decryptProxyUrl(row), () => provider.chatCompletion(apiKey, [
+ { role: 'user', content: 'Reply with OK' },
+ ], candidate.modelId, {
+ max_tokens: PROBE_MAX_TOKENS,
+ timeoutMs: PROBE_TIMEOUT_MS,
+ } as any, {
+ platform: candidate.platform as any,
+ keyId: candidate.keyId,
+ quotaPoolKey: inferQuotaPoolKey(candidate.platform as any, null),
+ endpoint: 'chat',
+ origin: 'probe',
+ } as any));
+}
+
+/**
+ * Run the probe against one candidate: does this model answer at all?
+ * A transport error is 'error' (inconclusive — the network, not the model,
+ * failed); any provider rejection (quota, 403/404 on the model) is
+ * 'unavailable'. A success is 'working'.
+ *
+ * Capability classification (tool_capable / chat_only) is a follow-up PR; this
+ * pass establishes liveness only.
+ *
+ * Testable: `chat` replaces the wire call.
+ */
+export async function probeModel(candidate: ProbeCandidate, deps: ModelProbeRequestDeps = {}): Promise<ProbeOutcome> {
+ const started = Date.now();
+ const chat = deps.chat ?? probeChat;
+ try {
+ await chat(candidate);
+ return { status: 'working', latencyMs: Date.now() - started };
+ } catch (err: any) {
+ const message = sanitizeProviderErrorMessage(err?.message ?? err);
+ const transport = /(?:fetch failed|network|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|certificate|TLS|timeout)/i.test(message);
+ if (transport || err?.name === 'AbortError') return { status: 'error', error: message };
+ return { status: 'unavailable', error: message };
+ }
+}
+
+// ── Recording ────────────────────────────────────────────────────────────────
+
+export function recordProbeResult(db: Db, candidate: ProbeCandidate, outcome: ProbeOutcome, probedAt = new Date()): void {
+ if (outcome.status === 'error') {
+ // Inconclusive: do not overwrite the last real verdict, but keep the
+ // diagnostic and refresh the timestamp so pacing/backoff can see it.
+ db.prepare(`
+ UPDATE model_probe_status SET last_error = ?, last_probed_at = datetime('now')
+ WHERE platform = ? AND model_id = ? AND endpoint_scope = ? AND key_id = ?
+ `).run(outcome.error ?? 'probe error', candidate.platform, candidate.modelId, candidate.endpointScope, candidate.keyId);
+ return;
+ }
+ db.prepare(`
+ INSERT INTO model_probe_status (platform, model_id, endpoint_scope, key_id, status, last_probed_at, last_error, latency_ms)
+ VALUES (?, ?, ?, ?, ?, ?, NULL, ?)
+ ON CONFLICT(platform, model_id, endpoint_scope, key_id) DO UPDATE SET
+ status = excluded.status,
+ last_probed_at = excluded.last_probed_at,
+ last_error = NULL,
+ latency_ms = excluded.latency_ms
+ `).run(
+ candidate.platform, candidate.modelId, candidate.endpointScope, candidate.keyId,
+ outcome.status, probedAt.toISOString(), outcome.latencyMs ?? null,
+ );
+}
+
+// ── The pass ─────────────────────────────────────────────────────────────────
+
+// Failed-probe backoff per candidate: 30m, 1h, 2h, capped at 6h. A model that
+// stays unavailable converges to one probe per staleness window — re-probing
+// it harder only burns the free quota the probe is meant to conserve.
+const PROBE_BACKOFF_BASE_MS = 30 * 60 * 1000;
+const PROBE_BACKOFF_MAX_MS = 6 * 60 * 60 * 1000;
+
+// Stagger window for a candidate's FIRST probe after boot, so a restart with a
+// large catalog does not fire every probe at once (mirrors cooldown-probe).
+const FIRST_PROBE_STAGGER_MS = 60 * 1000;
+
+interface CandidateProbeState {
+ failures: number;
+ nextProbeAtMs: number;
+}
+
+// keyId -> pacing, in-memory only (same as cooldown-probe: restart forgetting
+// backoff is fine because first-sighting staggering re-spaces probes anyway).
+const probeState = new Map<string, CandidateProbeState>();
+
+function stateKey(candidate: ProbeCandidate): string {
+ return `${candidate.platform}:${candidate.modelId}:${candidate.endpointScope}:${candidate.keyId}`;
+}
+
+export interface ProbePassOptions extends ModelProbeRequestDeps {
+ now?: number;
+ jitter?: () => number; // [0, 1), defaults to Math.random
+ maxProbes?: number;
+ select?: (limit: number, now: number) => ProbeCandidate[];
+}
+
+/** One probe pass: probe up to maxProbes stale candidates, record the outcomes.
+ * Exported for tests; production runs it on the scheduler via startModelProbe. */
+export async function runModelProbePass(opts: ProbePassOptions = {}): Promise<ProbePassResult> {
+ const now = opts.now ?? Date.now();
+ const jitter = opts.jitter ?? Math.random;
+ const maxProbes = opts.maxProbes ?? getMaxProbesPerPass();
+ const select = opts.select ?? ((limit: number, at: number) => selectProbeCandidates(getDb(), limit, at));
+
+ const candidates = select(maxProbes, now);
+ const probed: ProbeCandidate[] = [];
+ const outcomes: ProbePassResult['outcomes'] = [];
+
+ for (const candidate of candidates) {
+ if (probed.length >= maxProbes) break;
+ const key = stateKey(candidate);
+ const state = probeState.get(key);
+ if (!state) {
+ // First sighting after boot: schedule, don't probe (restart stagger).
+ probeState.set(key, { failures: 0, nextProbeAtMs: now + Math.floor(jitter() * FIRST_PROBE_STAGGER_MS) });
+ continue;
+ }
+ if (now < state.nextProbeAtMs) continue;
+
+ probed.push(candidate);
+ const outcome = await probeModel(candidate, opts);
+ outcomes.push({ ...candidate, outcome });
+
+ if (outcome.status === 'unavailable') {
+ state.failures++;
+ const backoff = Math.min(PROBE_BACKOFF_BASE_MS * 2 ** (state.failures - 1), PROBE_BACKOFF_MAX_MS);
+ state.nextProbeAtMs = now + backoff;
+ providerLog(
+ 'warn',
+ `[ModelProbe] ${candidate.platform}/${candidate.modelId} (key ${candidate.keyId}) unavailable: ${outcome.error ?? 'unknown error'}`,
+ { provider: candidate.platform, event: 'model_probe_unavailable' },
+ );
+ } else {
+ // A model that answered is healthy, so its pacing resets: re-probing a
+ // working model harder would only burn the free quota the probe exists
+ // to conserve.
+ probeState.delete(key);
+ providerLog(
+ 'info',
+ `[ModelProbe] ${candidate.platform}/${candidate.modelId} (key ${candidate.keyId}) → ${outcome.status}` +
+ (outcome.latencyMs != null ? ` in ${outcome.latencyMs}ms` : ''),
+ { provider: candidate.platform, event: 'model_probe_ok' },
+ );
+ }
+ recordProbeResult(getDb(), candidate, outcome);
+ }
+
+ return { probed, outcomes };
+}
+
+// Overlap guard, same shape as checkAllKeys / runCooldownProbePass.
+let passInFlight: Promise<ProbePassResult> | null = null;
+
+function runGuardedPass(): Promise<ProbePassResult> {
+ if (passInFlight) return passInFlight;
+ passInFlight = runModelProbePass().finally(() => {
+ passInFlight = null;
+ });
+ return passInFlight;
+}
+
+let cancelProbeJob: (() => void) | null = null;
+
+export function startModelProbe(scheduler: Scheduler): void {
+ if (cancelProbeJob) return;
+ if (process.env.MODEL_PROBE_ENABLED !== '1') return;
+ console.log(`[ModelProbe] starting proactive model probes (every ${MODEL_PROBE_INTERVAL_MS / 60_000}m, max ${getMaxProbesPerPass()}/pass)`);
+ cancelProbeJob = scheduler.every(
+ MODEL_PROBE_INTERVAL_MS,
+ async () => {
+ await runGuardedPass().catch(err => console.error('[ModelProbe] pass failed:', err));
+ },
+ { name: 'model-probe' },
+ );
+}
+
+export function stopModelProbe(): void {
+ if (cancelProbeJob) {
+ cancelProbeJob();
+ cancelProbeJob = null;
+ }
+}
+
+/** Test seam: drop all probe pacing state. */
+export function resetModelProbeState(): void {
+ probeState.clear();
+}