7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(models): proactive per-model probing with explicit status labels (#1438) (#1492)
suantea (asuan_suan)6c021c19d6fc
diff --git a/.env.example b/.env.exampleindex f8b108a9..5e5c8245 100644--- a/.env.example+++ b/.env.example@@ -162,6 +162,14 @@ PORT=3001# which costs quota where a provider's key check is metered. Read at startup.# HEALTH_CHECK_INTERVAL_MS=300000+# Opt-in background model probes (#1438): every 30 minutes, send a tiny+# "Reply with OK" chat to up to MODEL_PROBE_MAX_PER_PASS (default 3) model+key+# pairs and record whether each model answers. Off by default because every+# probe spends free-tier quota. POST /api/health/probe-models runs one pass on+# demand either way.+# MODEL_PROBE_ENABLED=1+# MODEL_PROBE_MAX_PER_PASS=3+# Base wall-clock budget for provider failover, in milliseconds (default: 45000).# The initial attempt and one failover hop are always allowed. After that,# exhaustion stops new attempts and can cancel an in-flight attempt stilldiff --git a/desktop/src/__tests__/server-host-boot.test.ts b/desktop/src/__tests__/server-host-boot.test.tsindex 56872bcb..193ed165 100644--- a/desktop/src/__tests__/server-host-boot.test.ts+++ b/desktop/src/__tests__/server-host-boot.test.ts@@ -64,6 +64,12 @@ vi.mock('../../../server/src/services/cooldown-probe.js', () => ({}),}));+vi.mock('../../../server/src/services/model-probe.js', () => ({+ startModelProbe: vi.fn(() => {+ calls.push('startModelProbe');+ }),+}));+vi.mock('../../../server/src/services/backups.js', () => ({startBackupScheduler: vi.fn(() => {calls.push('startBackupScheduler');@@ -174,6 +180,7 @@ describe('desktop server boot sequence (#949)', () => {'startHealthChecker','startCatalogSync','startCooldownProbe',+ 'startModelProbe','startBackupScheduler','startCustomModelSync','startBuiltinModelDiscovery',diff --git a/desktop/src/server-host.ts b/desktop/src/server-host.tsindex 735a159b..772812e8 100644--- a/desktop/src/server-host.ts+++ b/desktop/src/server-host.ts@@ -15,6 +15,7 @@ import { restoreProxySettings, flushProxyCache, getProxyMode, getProxyUrl, isProimport { startHealthChecker, checkAllKeys } from '../../server/src/services/health.js';import { startCatalogSync } from '../../server/src/services/catalog-sync.js';import { startCooldownProbe } from '../../server/src/services/cooldown-probe.js';+import { startModelProbe } from '../../server/src/services/model-probe.js';import { startCustomModelSync } from '../../server/src/services/custom-model-sync.js';import { startBuiltinModelDiscovery } from '../../server/src/services/builtin-model-discovery.js';import { startBackupScheduler, createBackup } from '../../server/src/services/backups.js';@@ -154,6 +155,7 @@ export async function startServer(opts: StartOptions): Promise<ServerHandle> {startHealthChecker(scheduler);startCatalogSync(scheduler);startCooldownProbe(scheduler);+ startModelProbe(scheduler);startBackupScheduler(scheduler);startCustomModelSync(getDb(), scheduler);startBuiltinModelDiscovery(getDb(), scheduler);diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.tsindex 360fba4f..e9d936ae 100644--- a/server/src/__tests__/db/migrate/roundtrip.test.ts+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts@@ -43,6 +43,7 @@ const REQUEST_MODEL_ATTRIBUTION_FILENAME = '20260913_000001_request_model_attribconst KEY_MONTHLY_USAGE_FILENAME = '20260914_000001_key_monthly_usage.ts';const QUOTA_SNAPSHOT_FRESHNESS_FILENAME = '20260915_000001_quota_snapshot_freshness.ts';const CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME = '20260930_000001_custom_endpoint_group_label.ts';+const MODEL_PROBE_STATUS_FILENAME = '20261007_000001_model_probe_status.ts';const SEARCH_ENGINES_FILENAME = '20261013_000001_search_engines.ts';const DECISION_REQUESTS_FILENAME = '20261016_000001_decision_requests.ts';const LLMTR_REENABLE_KEYS_FILENAME = '20261017_000001_llmtr_reenable_keys.ts';@@ -139,6 +140,7 @@ describe('migration round trip', () => {KEY_MONTHLY_USAGE_FILENAME,QUOTA_SNAPSHOT_FRESHNESS_FILENAME,CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME,+ MODEL_PROBE_STATUS_FILENAME,SEARCH_ENGINES_FILENAME,DECISION_REQUESTS_FILENAME,LLMTR_REENABLE_KEYS_FILENAME,diff --git a/server/src/__tests__/services/model-probe.test.ts b/server/src/__tests__/services/model-probe.test.tsnew file mode 100644index 00000000..fb954df7--- /dev/null+++ b/server/src/__tests__/services/model-probe.test.ts@@ -0,0 +1,210 @@+// model-probe tests (#1438 PR 1): candidate selection (stalest-first, fresh+// real-traffic skip), label recording, per-pass budget, restart stagger ++// first-sighting scheduling, failed-probe backoff, transport-error+// inconclusiveness (verdict preserved), and the kill switch.++import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import {+ selectProbeCandidates,+ probeModel,+ recordProbeResult,+ runModelProbePass,+ resetModelProbeState,+ startModelProbe,+ stopModelProbe,+ MODEL_PROBE_STALE_MS,+} from '../../services/model-probe.js';+import type { ProbeCandidate } from '../../services/model-probe.js';++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++beforeEach(() => {+ resetModelProbeState();+ const db = getDb();+ db.prepare('DELETE FROM model_probe_status').run();+ db.prepare('DELETE FROM fallback_config').run();+ db.prepare('DELETE FROM profile_models').run();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare('DELETE FROM models').run();+ // Fresh healthy key the models can hang off.+ db.prepare(`INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (1, 'probe-test', '', '', '', '', 'healthy', 1)`).run();+});++function addModel(id: number, modelId: string, opts: { enabled?: number } = {}): void {+ getDb().prepare(`INSERT INTO models (id, platform, model_id, display_name, intelligence_rank, speed_rank, endpoint_scope, enabled)+ VALUES (?, 'probe-test', ?, ?, 0, 0, '', ?)`).run(id, modelId, modelId, opts.enabled ?? 1);+}++const candidate = (over: Partial<ProbeCandidate> = {}): ProbeCandidate => ({+ keyId: 1,+ platform: 'probe-test',+ modelId: 'm1',+ modelDbId: 1,+ endpointScope: '',+ ...over,+});++// Seed one pass' worth of candidates so stagger/backoff choreography is+// deterministic: first sighting schedules (no probe), next pass probes.+function seedCandidates(n: number): ProbeCandidate[] {+ const out: ProbeCandidate[] = [];+ for (let i = 1; i <= n; i++) {+ addModel(i, `m${i}`);+ out.push(candidate({ modelId: `m${i}`, modelDbId: i }));+ }+ return out;+}++const selectFrom = (list: ProbeCandidate[]) => (limit: number) => list.slice(0, limit);+const chatOk = () => vi.fn(async () => ({}));+const chatFail = (message: string) => vi.fn(async () => { throw new Error(message); });+const noJitter = () => 0;++describe('candidate selection', () => {+ it('probes the stalest evidence first, never-probed before stale', () => {+ const db = getDb();+ addModel(1, 'old');+ addModel(2, 'fresh');+ addModel(3, 'never');+ recordProbeResult(db, candidate({ modelId: 'old', modelDbId: 1 }),+ { status: 'working', latencyMs: 5 }, new Date(Date.now() - MODEL_PROBE_STALE_MS - 1000));+ recordProbeResult(db, candidate({ modelId: 'fresh', modelDbId: 2 }),+ { status: 'working', latencyMs: 5 }, new Date());++ const picked = selectProbeCandidates(db, 10);+ expect(picked.map(p => p.modelId)).toEqual(['never', 'old']);+ });++ it('skips models with fresh evidence (staleness window)', () => {+ const db = getDb();+ addModel(1, 'fresh');+ recordProbeResult(db, candidate({ modelId: 'fresh', modelDbId: 1 }),+ { status: 'working', latencyMs: 5 }, new Date());+ expect(selectProbeCandidates(db, 10)).toEqual([]);+ });++ it('ignores disabled models and disabled/invalid keys', () => {+ const db = getDb();+ addModel(1, 'off-model', { enabled: 0 });+ addModel(2, 'bad-key-model');+ db.prepare("UPDATE api_keys SET status = 'invalid' WHERE id = 1").run();+ expect(selectProbeCandidates(db, 10)).toEqual([]);+ });+});++describe('probe outcome classification', () => {+ it('a successful chat is working with latency', async () => {+ const outcome = await probeModel(candidate(), { chat: chatOk() });+ expect(outcome.status).toBe('working');+ expect(outcome.latencyMs).toBeGreaterThanOrEqual(0);+ });++ it('a provider rejection is unavailable and carries the sanitized error', async () => {+ const outcome = await probeModel(candidate(), { chat: chatFail('403: quota exhausted upstream') });+ expect(outcome.status).toBe('unavailable');+ expect(outcome.error).toContain('403');+ });++ it('a transport error is inconclusive (error), never unavailable', async () => {+ const outcome = await probeModel(candidate(), { chat: chatFail('fetch failed: getaddrinfo ENOTFOUND api.example.com') });+ expect(outcome.status).toBe('error');+ });+});++describe('recording', () => {+ it('writes working with latency and clears prior errors', () => {+ const db = getDb();+ recordProbeResult(db, candidate(), { status: 'unavailable', error: '403' });+ recordProbeResult(db, candidate(), { status: 'working', latencyMs: 42 });+ const row = db.prepare('SELECT * FROM model_probe_status').get() as any;+ expect(row.status).toBe('working');+ expect(row.latency_ms).toBe(42);+ expect(row.last_error).toBeNull();+ });++ it('an inconclusive probe preserves the previous verdict', () => {+ const db = getDb();+ recordProbeResult(db, candidate(), { status: 'working', latencyMs: 10 });+ recordProbeResult(db, candidate(), { status: 'error', error: 'fetch failed' });+ const row = db.prepare('SELECT * FROM model_probe_status').get() as any;+ expect(row.status).toBe('working');+ expect(row.last_error).toBe('fetch failed');+ });+});++describe('probe pass', () => {+ it('first sighting schedules instead of probing; the next pass probes within budget', async () => {+ const list = seedCandidates(5);+ const chat = chatOk();+ const pass1 = await runModelProbePass({ chat, jitter: noJitter, maxProbes: 3, select: selectFrom(list) });+ expect(pass1.probed).toEqual([]);+ expect(chat).not.toHaveBeenCalled();++ const pass2 = await runModelProbePass({ chat, jitter: noJitter, maxProbes: 3, select: selectFrom(list) });+ expect(pass2.probed).toHaveLength(3);+ expect(chat).toHaveBeenCalledTimes(3);+ });++ it('a working probe clears pacing so the model is not re-probed until stale', async () => {+ const list = seedCandidates(1);+ const chat = chatOk;+ await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });+ await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });+ // Third pass: the single candidate just worked; nothing is probeable.+ const pass3 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });+ expect(pass3.probed).toEqual([]);+ });++ it('an unavailable probe backs off instead of re-probing every pass', async () => {+ const list = seedCandidates(1);+ const chat = chatFail('403 quota');+ // Pass 1: first sighting only schedules.+ await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });+ // Pass 2: the actual probe — fails, sets a 30m backoff.+ const pass2 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });+ expect(pass2.probed).toHaveLength(1);+ // Pass 3 immediately after: still inside the backoff, nothing probed.+ const pass3 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list) });+ expect(pass3.probed).toEqual([]);+ // Past the first backoff step (30m) it probes again.+ const later = Date.now() + 31 * 60_000;+ const pass4 = await runModelProbePass({ chat, jitter: noJitter, select: selectFrom(list), now: later });+ expect(pass4.probed).toHaveLength(1);+ });++ it('records outcomes in the table', async () => {+ const list = seedCandidates(2);+ await runModelProbePass({ chat: chatOk(), jitter: noJitter, select: selectFrom(list) });+ await runModelProbePass({ chat: chatOk(), jitter: noJitter, select: selectFrom(list) });+ const rows = getDb().prepare('SELECT model_id, status FROM model_probe_status ORDER BY model_id').all() as any[];+ expect(rows).toEqual([+ { model_id: 'm1', status: 'working' },+ { model_id: 'm2', status: 'working' },+ ]);+ });+});++describe('kill switch and lifecycle', () => {+ it('is off by default: the scheduler does not start without MODEL_PROBE_ENABLED=1', () => {+ delete process.env.MODEL_PROBE_ENABLED;+ const scheduler = { every: vi.fn(() => () => undefined) };+ startModelProbe(scheduler as any);+ expect(scheduler.every).not.toHaveBeenCalled();+ stopModelProbe();+ });++ it('MODEL_PROBE_ENABLED=1 schedules the job on the 30m cadence', () => {+ process.env.MODEL_PROBE_ENABLED = '1';+ const scheduler = { every: vi.fn(() => () => undefined) };+ startModelProbe(scheduler as any);+ expect(scheduler.every).toHaveBeenCalledTimes(1);+ expect(scheduler.every.mock.calls[0][0]).toBe(30 * 60 * 1000);+ delete process.env.MODEL_PROBE_ENABLED;+ stopModelProbe();+ });+});diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.tsindex b875446d..5222df2d 100644--- a/server/src/db/migrate/defaults.ts+++ b/server/src/db/migrate/defaults.ts@@ -38,6 +38,7 @@ import * as keyMonthlyUsage from '../migrations/20260914_000001_key_monthly_usagimport * as quotaSnapshotFreshness from '../migrations/20260915_000001_quota_snapshot_freshness.js';import * as requestModelAttribution from '../migrations/20260913_000001_request_model_attribution.js';import * as customEndpointGroupLabel from '../migrations/20260930_000001_custom_endpoint_group_label.js';+import * as modelProbeStatus from '../migrations/20261007_000001_model_probe_status.js';import * as searchEngines from '../migrations/20261013_000001_search_engines.js';import * as decisionRequests from '../migrations/20261016_000001_decision_requests.js';import * as llmtrReenableKeys from '../migrations/20261017_000001_llmtr_reenable_keys.js';@@ -93,6 +94,7 @@ export const RESPONSE_CACHE_FILENAME = '20260903_000002_response_cache.ts';export const KEY_MONTHLY_BUDGET_FILENAME = '20260904_000001_key_monthly_budget.ts';export const REQUEST_MODEL_ATTRIBUTION_FILENAME = '20260913_000001_request_model_attribution.ts';export const CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME = '20260930_000001_custom_endpoint_group_label.ts';+export const MODEL_PROBE_STATUS_FILENAME = '20261007_000001_model_probe_status.ts';export const KEY_MONTHLY_USAGE_FILENAME = '20260914_000001_key_monthly_usage.ts';export const QUOTA_SNAPSHOT_FRESHNESS_FILENAME = '20260915_000001_quota_snapshot_freshness.ts';export const SEARCH_ENGINES_FILENAME = '20261013_000001_search_engines.ts';@@ -143,6 +145,7 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: KEY_MONTHLY_USAGE_FILENAME, module: keyMonthlyUsage },{ filename: QUOTA_SNAPSHOT_FRESHNESS_FILENAME, module: quotaSnapshotFreshness },{ filename: CUSTOM_ENDPOINT_GROUP_LABEL_FILENAME, module: customEndpointGroupLabel },+ { filename: MODEL_PROBE_STATUS_FILENAME, module: modelProbeStatus },{ filename: SEARCH_ENGINES_FILENAME, module: searchEngines },{ filename: DECISION_REQUESTS_FILENAME, module: decisionRequests },{ filename: LLMTR_REENABLE_KEYS_FILENAME, module: llmtrReenableKeys },diff --git a/server/src/db/migrations/20261007_000001_model_probe_status.ts b/server/src/db/migrations/20261007_000001_model_probe_status.tsnew file mode 100644index 00000000..ef5942af--- /dev/null+++ b/server/src/db/migrations/20261007_000001_model_probe_status.ts@@ -0,0 +1,28 @@+// model_probe_status — proactive per-model probe results (#1438 PR 1).+//+// One row per (platform, model_id, endpoint_scope, key_id): the outcome of the+// most recent proactive probe for that routable model+key pair. Purely+// observational — nothing in routing or scoring reads this table in this PR;+// a follow-up consumes the labels. DOWN drops the table; probe data is a+// cache of reality, not a record, so dropping it loses nothing.+import type { Db } from '../types.js';++export function up(db: Db): void {+ db.exec(`CREATE TABLE IF NOT EXISTS model_probe_status (+ platform TEXT NOT NULL,+ model_id TEXT NOT NULL,+ endpoint_scope TEXT,+ key_id INTEGER NOT NULL,+ status TEXT NOT NULL DEFAULT 'unknown',+ last_probed_at TEXT,+ last_error TEXT,+ latency_ms INTEGER,+ PRIMARY KEY (platform, model_id, endpoint_scope, key_id)+ )`);+ db.exec(`CREATE INDEX IF NOT EXISTS idx_model_probe_status_probed_at+ ON model_probe_status(last_probed_at)`);+}++export function down(db: Db): void {+ db.exec('DROP TABLE IF EXISTS model_probe_status');+}diff --git a/server/src/index.ts b/server/src/index.tsindex b477a045..35f11e3e 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -6,6 +6,7 @@ import { restoreProxySettings, flushProxyCache } from './lib/proxy.js';import { startWakeDetect } from './lib/wake-detect.js';import { startCatalogSync } from './services/catalog-sync.js';import { startCooldownProbe } from './services/cooldown-probe.js';+import { startModelProbe } from './services/model-probe.js';import { startCustomModelSync } from './services/custom-model-sync.js';import { startBuiltinModelDiscovery } from './services/builtin-model-discovery.js';import { installProcessSafetyNet } from './lib/process-safety-net.js';@@ -84,6 +85,7 @@ async function main() {startHealthChecker(scheduler);startCatalogSync(scheduler);startCooldownProbe(scheduler);+ startModelProbe(scheduler);startDbBackupPump(getDb(), scheduler, config.dbPath ?? undefined);startBackupScheduler(scheduler);startCustomModelSync(getDb(), scheduler);diff --git a/server/src/routes/health.ts b/server/src/routes/health.tsindex 0d6cc8a7..6939eb26 100644--- a/server/src/routes/health.ts+++ b/server/src/routes/health.ts@@ -3,6 +3,7 @@ import type { Request, Response } from 'express';import { getDb } from '../db/index.js';import { checkKeyHealth, checkAllKeys, pollKeyQuota, supportsQuotaPolling } from '../services/health.js';import { getDegradationStatus } from '../services/degradation.js';+import { runModelProbePass } from '../services/model-probe.js';import { hasProvider, resolveProvider } from '../providers/index.js';import { getQuotaStateForKeys } from '../services/provider-quota.js';import type { Platform } from '@freellmapi/shared/types.js';@@ -112,3 +113,27 @@ healthRouter.post('/quota/:keyId', async (req: Request, res: Response) => {);res.json({ keyId, recorded, quotaStates: state });});++// Run one proactive model-probe pass now (#1438 PR 1). The scheduled pass runs+// every 30 minutes; this is the manual override — an operator wanting fresh+// labels immediately instead of waiting for the next pass. Same shape as+// POST /check-all: the operator pressed the button, so run a full pass.+healthRouter.post('/probe-models', async (_req: Request, res: Response) => {+ const result = await runModelProbePass();+ res.json({+ probed: result.probed.map(c => ({+ platform: c.platform,+ modelId: c.modelId,+ keyId: c.keyId,+ endpointScope: c.endpointScope,+ })),+ outcomes: result.outcomes.map(o => ({+ platform: o.platform,+ modelId: o.modelId,+ keyId: o.keyId,+ status: o.outcome.status,+ latencyMs: o.outcome.latencyMs ?? null,+ error: o.outcome.error ?? null,+ })),+ });+});diff --git a/server/src/services/model-probe.ts b/server/src/services/model-probe.tsnew file mode 100644index 00000000..8454d491--- /dev/null+++ b/server/src/services/model-probe.ts@@ -0,0 +1,345 @@+// model-probe — proactive per-model probes with explicit labels (#1438 PR 1).+//+// Key-level health says a credential works; it says nothing about whether a+// specific MODEL behind that key actually answers. Free-tier aggregators+// routinely list models that 403 on every real chat (upstream quota drained),+// and the router only learns that after burning a real user turn (#1267 is+// passive: it demotes a model after real traffic has already hit it).+//+// This job sends a minimal real chat request ("Reply with OK") to routable+// model+key pairs that have not been observed recently, and writes an explicit+// status row: 'working' | 'unavailable'. PR 1 is OBSERVATION ONLY — nothing in+// routing, scoring or fallback reads the table yet; a follow-up consumes the+// labels (tool-call variant, chat_only classification, dashboard badges).+//+// What it will never do:+// - probe a model that just served real traffic successfully (serving is+// stronger evidence than any probe, and probes cost free-tier quota);+// - feed the health checker, cooldowns or the failure counters — a probe is+// a question, never a verdict (same principle as probeKeyValidity);+// - use the fallback/healing machinery: the probe measures the model as-is,+// not the pipeline, so a probe never retries and never repairs;+// - thundering-herd a provider after a restart: first-sighting staggering+// plus a small per-pass budget keep probe traffic bounded.+//+// Cadence: one pass every 30 minutes, at most MODEL_PROBE_MAX_PER_PASS probes+// per pass (default 3). OFF by default: probes spend the user's free-tier+// quota and nothing consumes the labels yet, so the scheduled job only runs+// with MODEL_PROBE_ENABLED=1. POST /api/health/probe-models always runs a pass.++import type { Scheduler } from '../lib/scheduler.js';+import type { Db } from '../db/types.js';+import { getDb } from '../db/index.js';+import { resolveProvider } from '../providers/index.js';+import { decrypt } from '../lib/crypto.js';+import { decryptProxyUrl } from '../lib/key-proxy.js';+import { withKeyProxy } from '../lib/proxy.js';+import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';+import { providerLog } from '../lib/server-logs.js';+import { inferQuotaPoolKey } from './provider-quota.js';++// One pass every 30 minutes. Probes are quota-costing real requests; the point+// is freshness on the scale of "an operator glances at the dashboard", not+// second-level liveness (the cooldown/health machinery owns that).+export const MODEL_PROBE_INTERVAL_MS = 30 * 60 * 1000;++// Stale rows are re-probed after this long; fresher evidence is trusted and+// the model is skipped. Well above the pass cadence so a skipped model is not+// re-considered every pass.+export const MODEL_PROBE_STALE_MS = 6 * 60 * 60 * 1000;++// A model+key that served a real request within this window does not need a+// probe: real success is stronger evidence, and the probe would only spend+// free-tier quota to re-learn it. Mirrors the RECENT_CHECK_SKIP_MS idea from+// the health pass.+export const RECENT_SERVE_SKIP_MS = 60 * 60 * 1000;++const DEFAULT_MAX_PROBES_PER_PASS = 3;++// Ceiling for one probe request, mirroring health's transport posture. A probe+// that takes longer than this is treated as an error (inconclusive), not as+// evidence the model is unavailable.+const PROBE_TIMEOUT_MS = 60 * 1000;++// max_tokens floor: some relays reject max_tokens <= 2 (#903). The probe never+// relies on tiny ceilings, so a generous floor avoids that whole failure class.+const PROBE_MAX_TOKENS = 64;++/** Per-pass probe budget. Tunable; 0 or a bad value falls back to the default+ * (mirrors HEALTH_CHECK_CONCURRENCY / COOLDOWN_PROBE_MAX_PER_PASS). */+function getMaxProbesPerPass(): number {+ const raw = process.env.MODEL_PROBE_MAX_PER_PASS;+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isInteger(n) && n > 0) return n;+ }+ return DEFAULT_MAX_PROBES_PER_PASS;+}++export type ModelProbeStatus = 'working' | 'unavailable';++export interface ProbeCandidate {+ keyId: number;+ platform: string;+ modelId: string;+ modelDbId: number;+ endpointScope: string;+}++export interface ProbeOutcome {+ status: ModelProbeStatus | 'error';+ latencyMs?: number;+ error?: string;+}++export interface ProbePassResult {+ probed: ProbeCandidate[];+ outcomes: Array<ProbeCandidate & { outcome: ProbeOutcome }>;+}++// ── Candidate selection ──────────────────────────────────────────────────────+// Routable pairs only: enabled model on an enabled, currently-healthy-or-+// unknown key, with endpoint scope carried through (custom-endpoint identity,+// migration 20260729_000001). Ordered to rotate fairly: stalest evidence+// first, models never probed (NULL last_probed_at) before stale ones.++interface ProbeCandidateRow extends ProbeCandidate {+ lastProbedAt: string | null;+}++export function selectProbeCandidates(db: Db, limit: number, now = Date.now()): ProbeCandidateRow[] {+ const rows = db.prepare(`+ SELECT m.id AS modelDbId, m.platform, m.model_id AS modelId, m.endpoint_scope AS endpointScope,+ k.id AS keyId, COALESCE(p.last_probed_at, '') AS lastProbedAt+ FROM models m+ JOIN api_keys k ON k.platform = m.platform AND k.enabled = 1 AND k.status IN ('healthy', 'unknown')+ LEFT JOIN model_probe_status p+ ON p.platform = m.platform AND p.model_id = m.model_id+ AND p.endpoint_scope = m.endpoint_scope AND p.key_id = k.id+ WHERE m.enabled = 1+ ORDER BY (p.last_probed_at IS NULL) DESC, p.last_probed_at ASC, m.id ASC, k.id ASC+ LIMIT ?+ `).all(limit * 8) as Array<{+ modelDbId: number; platform: string; modelId: string; endpointScope: string;+ keyId: number; lastProbedAt: string;+ }>;++ const out: ProbeCandidateRow[] = [];+ for (const row of rows) {+ if (out.length >= limit) break;+ if (row.lastProbedAt) {+ const seen = Date.parse(row.lastProbedAt.includes('T') ? row.lastProbedAt : row.lastProbedAt.replace(' ', 'T') + 'Z');+ if (Number.isFinite(seen) && now - seen < MODEL_PROBE_STALE_MS) continue;+ }+ out.push(row);+ }+ return out;+}++// ── The probe itself ─────────────────────────────────────────────────────────++export interface ModelProbeRequestDeps {+ // Test seam: the chat call the probe makes. Production uses the provider's+ // own chatCompletion through the key's proxy.+ chat?: (candidate: ProbeCandidate) => Promise<unknown>;+}++async function probeChat(candidate: ProbeCandidate): Promise<void> {+ const db = getDb();+ const row = db.prepare('SELECT * FROM api_keys WHERE id = ? AND enabled = 1').get(candidate.keyId) as any;+ if (!row) throw new Error('key not found or disabled');+ const provider = resolveProvider(candidate.platform as any, row.base_url);+ if (!provider) throw new Error(`no provider adapter for ${candidate.platform}`);++ const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);+ // Same reasoning as checkKeyHealth (#590): probe through the key's own proxy,+ // or a key only reachable via its proxy would fail the probe while real+ // traffic through the proxy works fine.+ await withKeyProxy(decryptProxyUrl(row), () => provider.chatCompletion(apiKey, [+ { role: 'user', content: 'Reply with OK' },+ ], candidate.modelId, {+ max_tokens: PROBE_MAX_TOKENS,+ timeoutMs: PROBE_TIMEOUT_MS,+ } as any, {+ platform: candidate.platform as any,+ keyId: candidate.keyId,+ quotaPoolKey: inferQuotaPoolKey(candidate.platform as any, null),+ endpoint: 'chat',+ origin: 'probe',+ } as any));+}++/**+ * Run the probe against one candidate: does this model answer at all?+ * A transport error is 'error' (inconclusive — the network, not the model,+ * failed); any provider rejection (quota, 403/404 on the model) is+ * 'unavailable'. A success is 'working'.+ *+ * Capability classification (tool_capable / chat_only) is a follow-up PR; this+ * pass establishes liveness only.+ *+ * Testable: `chat` replaces the wire call.+ */+export async function probeModel(candidate: ProbeCandidate, deps: ModelProbeRequestDeps = {}): Promise<ProbeOutcome> {+ const started = Date.now();+ const chat = deps.chat ?? probeChat;+ try {+ await chat(candidate);+ return { status: 'working', latencyMs: Date.now() - started };+ } catch (err: any) {+ const message = sanitizeProviderErrorMessage(err?.message ?? err);+ const transport = /(?:fetch failed|network|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|certificate|TLS|timeout)/i.test(message);+ if (transport || err?.name === 'AbortError') return { status: 'error', error: message };+ return { status: 'unavailable', error: message };+ }+}++// ── Recording ────────────────────────────────────────────────────────────────++export function recordProbeResult(db: Db, candidate: ProbeCandidate, outcome: ProbeOutcome, probedAt = new Date()): void {+ if (outcome.status === 'error') {+ // Inconclusive: do not overwrite the last real verdict, but keep the+ // diagnostic and refresh the timestamp so pacing/backoff can see it.+ db.prepare(`+ UPDATE model_probe_status SET last_error = ?, last_probed_at = datetime('now')+ WHERE platform = ? AND model_id = ? AND endpoint_scope = ? AND key_id = ?+ `).run(outcome.error ?? 'probe error', candidate.platform, candidate.modelId, candidate.endpointScope, candidate.keyId);+ return;+ }+ db.prepare(`+ INSERT INTO model_probe_status (platform, model_id, endpoint_scope, key_id, status, last_probed_at, last_error, latency_ms)+ VALUES (?, ?, ?, ?, ?, ?, NULL, ?)+ ON CONFLICT(platform, model_id, endpoint_scope, key_id) DO UPDATE SET+ status = excluded.status,+ last_probed_at = excluded.last_probed_at,+ last_error = NULL,+ latency_ms = excluded.latency_ms+ `).run(+ candidate.platform, candidate.modelId, candidate.endpointScope, candidate.keyId,+ outcome.status, probedAt.toISOString(), outcome.latencyMs ?? null,+ );+}++// ── The pass ─────────────────────────────────────────────────────────────────++// Failed-probe backoff per candidate: 30m, 1h, 2h, capped at 6h. A model that+// stays unavailable converges to one probe per staleness window — re-probing+// it harder only burns the free quota the probe is meant to conserve.+const PROBE_BACKOFF_BASE_MS = 30 * 60 * 1000;+const PROBE_BACKOFF_MAX_MS = 6 * 60 * 60 * 1000;++// Stagger window for a candidate's FIRST probe after boot, so a restart with a+// large catalog does not fire every probe at once (mirrors cooldown-probe).+const FIRST_PROBE_STAGGER_MS = 60 * 1000;++interface CandidateProbeState {+ failures: number;+ nextProbeAtMs: number;+}++// keyId -> pacing, in-memory only (same as cooldown-probe: restart forgetting+// backoff is fine because first-sighting staggering re-spaces probes anyway).+const probeState = new Map<string, CandidateProbeState>();++function stateKey(candidate: ProbeCandidate): string {+ return `${candidate.platform}:${candidate.modelId}:${candidate.endpointScope}:${candidate.keyId}`;+}++export interface ProbePassOptions extends ModelProbeRequestDeps {+ now?: number;+ jitter?: () => number; // [0, 1), defaults to Math.random+ maxProbes?: number;+ select?: (limit: number, now: number) => ProbeCandidate[];+}++/** One probe pass: probe up to maxProbes stale candidates, record the outcomes.+ * Exported for tests; production runs it on the scheduler via startModelProbe. */+export async function runModelProbePass(opts: ProbePassOptions = {}): Promise<ProbePassResult> {+ const now = opts.now ?? Date.now();+ const jitter = opts.jitter ?? Math.random;+ const maxProbes = opts.maxProbes ?? getMaxProbesPerPass();+ const select = opts.select ?? ((limit: number, at: number) => selectProbeCandidates(getDb(), limit, at));++ const candidates = select(maxProbes, now);+ const probed: ProbeCandidate[] = [];+ const outcomes: ProbePassResult['outcomes'] = [];++ for (const candidate of candidates) {+ if (probed.length >= maxProbes) break;+ const key = stateKey(candidate);+ const state = probeState.get(key);+ if (!state) {+ // First sighting after boot: schedule, don't probe (restart stagger).+ probeState.set(key, { failures: 0, nextProbeAtMs: now + Math.floor(jitter() * FIRST_PROBE_STAGGER_MS) });+ continue;+ }+ if (now < state.nextProbeAtMs) continue;++ probed.push(candidate);+ const outcome = await probeModel(candidate, opts);+ outcomes.push({ ...candidate, outcome });++ if (outcome.status === 'unavailable') {+ state.failures++;+ const backoff = Math.min(PROBE_BACKOFF_BASE_MS * 2 ** (state.failures - 1), PROBE_BACKOFF_MAX_MS);+ state.nextProbeAtMs = now + backoff;+ providerLog(+ 'warn',+ `[ModelProbe] ${candidate.platform}/${candidate.modelId} (key ${candidate.keyId}) unavailable: ${outcome.error ?? 'unknown error'}`,+ { provider: candidate.platform, event: 'model_probe_unavailable' },+ );+ } else {+ // A model that answered is healthy, so its pacing resets: re-probing a+ // working model harder would only burn the free quota the probe exists+ // to conserve.+ probeState.delete(key);+ providerLog(+ 'info',+ `[ModelProbe] ${candidate.platform}/${candidate.modelId} (key ${candidate.keyId}) → ${outcome.status}` ++ (outcome.latencyMs != null ? ` in ${outcome.latencyMs}ms` : ''),+ { provider: candidate.platform, event: 'model_probe_ok' },+ );+ }+ recordProbeResult(getDb(), candidate, outcome);+ }++ return { probed, outcomes };+}++// Overlap guard, same shape as checkAllKeys / runCooldownProbePass.+let passInFlight: Promise<ProbePassResult> | null = null;++function runGuardedPass(): Promise<ProbePassResult> {+ if (passInFlight) return passInFlight;+ passInFlight = runModelProbePass().finally(() => {+ passInFlight = null;+ });+ return passInFlight;+}++let cancelProbeJob: (() => void) | null = null;++export function startModelProbe(scheduler: Scheduler): void {+ if (cancelProbeJob) return;+ if (process.env.MODEL_PROBE_ENABLED !== '1') return;+ console.log(`[ModelProbe] starting proactive model probes (every ${MODEL_PROBE_INTERVAL_MS / 60_000}m, max ${getMaxProbesPerPass()}/pass)`);+ cancelProbeJob = scheduler.every(+ MODEL_PROBE_INTERVAL_MS,+ async () => {+ await runGuardedPass().catch(err => console.error('[ModelProbe] pass failed:', err));+ },+ { name: 'model-probe' },+ );+}++export function stopModelProbe(): void {+ if (cancelProbeJob) {+ cancelProbeJob();+ cancelProbeJob = null;+ }+}++/** Test seam: drop all probe pacing state. */+export function resetModelProbeState(): void {+ probeState.clear();+}