Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

14.0 KB
Raw Blame
// @ts-check
// Live LLM provider health — the truth behind "AI features are up".
//
// `llm.js` fails over silently across a free-first chain (Groq → OpenRouter →
// NVIDIA → paid Anthropic/OpenAI), which is exactly what keeps user-facing AI
// alive when a provider dies — and exactly what hides a dead provider from ops.
// The June 2026 outage took out all three paid providers at once while chat
// stayed (barely) up on a free-tier OpenRouter key; nobody saw it until users
// hit `:free`-only quality. This module probes the paid providers the platform
// depends on with a near-zero-cost ping (max_tokens: 1) so an outage is visible
// before it degrades the product.
//
// Probed providers mirror the paid tier of the routing chain:
//   • OpenRouter      (primary, env.OPENROUTER_API_KEY) — leads the paid path
//   • Anthropic       (env.ANTHROPIC_API_KEY)           — paid backstop
//   • OpenAI          (env.OPENAI_API_KEY)              — paid backstop
//   • Vertex Gemini   (GOOGLE_CLOUD_PROJECT)            — GCP-credit-billed anchor
//   • Vertex Anthropic (VERTEX_CLAUDE_ENABLED=1)        — GCP-credit-billed Claude
// A provider with no key/config is simply not probed (omitted from the report) —
// it is not "down", it is "not part of this deployment".
//
// The two Vertex rungs matter here specifically because llm.js's real chain
// (providerChain()) already reaches them whenever GOOGLE_CLOUD_PROJECT is set —
// they are the reliability anchor that survives a same-day outage of all three
// third-party paid keys above. Before these were added, this probe (and the
// /api/llm/health dashboard it backs) would report `down`/`degraded` during
// exactly the outage Vertex was already absorbing for real users — a false
// alarm hiding the fact that the platform was fine.
//
// Statuses per provider: 'ok' (2xx) | 'error' (timeout, unreachable, or non-2xx,
// e.g. a 402 out-of-credits or a 401 bad key). `overall` is 'ok' when every
// configured provider passes, 'down' when none do, 'degraded' in between, and
// 'unconfigured' when no paid key is set at all (the free-first chain still
// serves, so that is not an outage — see the overall calc below).
//
// Reused directly (not over HTTP) by api/_lib/forge-health.js and exposed,
// gated, at GET /api/llm/health.

import { env } from './env.js';
import { vertexClaudeEnabled, vertexMessagesUrl, vertexRequestHeaders, toVertexBody } from './vertex-claude.js';
import { OPENROUTER_ATTRIBUTION_HEADERS } from './openrouter-attribution.js';

const PROBE_TIMEOUT_MS = 5_000;

// Cheapest live model per provider — a max_tokens:1 completion costs a fraction
// of a cent and the probe only reads the HTTP status, not the body.
// Probe the SAME model the real chain leads its OpenRouter lane with
// (llm.js OPENROUTER_MODEL, the primary key's paid rung), not an unrelated id.
// The previous 'openai/gpt-5.4-nano' was a wrong-family guess that need not even
// exist on OpenRouter (a permanent 404). The paid 70B rung returns 200 on a
// funded key at ~$3e-6/probe (≈$0.03/yr hourly — negligible), so a 200 confirms
// the account actually has credits, and a 402 is then a TRUE, actionable "top up
// the OpenRouter key" signal rather than a false alarm. The :free variant is not
// used here: OpenRouter globally rate-limits free models (observed 429), which
// would itself masquerade as an outage.
const OPENROUTER_MODEL = 'meta-llama/llama-3.3-70b-instruct';
const ANTHROPIC_MODEL = 'claude-haiku-4-5-20251001';
const OPENAI_MODEL = 'gpt-5.4-nano';
// Same model ids llm.js's real chain uses for these two rungs — see
// vertexGeminiProvider()/vertexAnthropicProvider() there.
const VERTEX_GEMINI_MODEL = process.env.VERTEX_GEMINI_MODEL || 'google/gemini-2.5-flash';

// fetch with a hard 5s timeout; returns the Response (or null on transport
// error) plus the measured round-trip so the report can surface latency.
async function timedFetch(url, options) {
	const started = Date.now();
	try {
		const res = await fetch(url, { ...options, signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
		return { res, latencyMs: Date.now() - started };
	} catch (err) {
		return { res: null, latencyMs: Date.now() - started, err };
	}
}

// Turn a probe response into a provider verdict. A 2xx means the key authed and
// the account has quota; anything else (timeout, network, 401/402/429/5xx) is an
// error carrying a short, account-detail-free reason.
function judge({ res, latencyMs, err }, model) {
	if (!res) {
		const reason = err?.name === 'TimeoutError' ? `timed out after ${PROBE_TIMEOUT_MS}ms` : 'unreachable';
		return { status: 'error', error: reason, latencyMs };
	}
	if (res.status >= 200 && res.status < 300) {
		return { status: 'ok', model, latencyMs };
	}
	return { status: 'error', error: `${res.status} ${res.statusText || ''}`.trim(), latencyMs };
}

async function probeOpenAiCompat({ key, url, model, extraHeaders = {} }) {
	const r = await timedFetch(url, {
		method: 'POST',
		headers: { 'content-type': 'application/json', authorization: `Bearer ${key}`, ...extraHeaders },
		body: JSON.stringify({ model, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] }),
	});
	return judge(r, model);
}

async function probeAnthropic({ key, model }) {
	const r = await timedFetch('https://api.anthropic.com/v1/messages', {
		method: 'POST',
		headers: { 'content-type': 'application/json', 'x-api-key': key, 'anthropic-version': '2023-06-01' },
		body: JSON.stringify({ model, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] }),
	});
	return judge(r, model);
}

// Vertex probes authenticate with a GCP OAuth bearer token (vertexRequestHeaders)
// instead of a static API key, so a token-exchange failure (no service-account
// credentials, no aiplatform IAM) is itself a real, reportable 'error' verdict —
// caught here rather than left to throw uncaught.
async function probeVertexGemini() {
	const project = process.env.GOOGLE_CLOUD_PROJECT;
	const location = process.env.GOOGLE_CLOUD_LOCATION_GEMINI || 'global';
	const host = location === 'global' ? 'aiplatform.googleapis.com' : `${location}-aiplatform.googleapis.com`;
	const url = `https://${host}/v1beta1/projects/${project}/locations/${location}/endpoints/openapi/chat/completions`;
	try {
		const headers = { 'content-type': 'application/json', ...(await vertexRequestHeaders()) };
		const r = await timedFetch(url, {
			method: 'POST',
			headers,
			body: JSON.stringify({ model: VERTEX_GEMINI_MODEL, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] }),
		});
		return judge(r, VERTEX_GEMINI_MODEL);
	} catch (err) {
		return { status: 'error', error: `token exchange failed: ${err?.message || 'unknown'}`, latencyMs: 0 };
	}
}

async function probeVertexAnthropic({ model }) {
	try {
		const headers = await vertexRequestHeaders();
		const r = await timedFetch(vertexMessagesUrl(model, { stream: false }), {
			method: 'POST',
			headers,
			body: JSON.stringify(toVertexBody({ model, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] })),
		});
		return judge(r, model);
	} catch (err) {
		return { status: 'error', error: `token exchange failed: ${err?.message || 'unknown'}`, latencyMs: 0 };
	}
}

// Probe every configured paid provider in parallel and fold the per-provider
// verdicts into one `overall`. Returns { [provider]: verdict, ..., overall }.
export async function probeLlmHealth() {
	const probes = [];
	if (env.OPENROUTER_API_KEY) {
		probes.push([
			'openrouter',
			probeOpenAiCompat({
				key: env.OPENROUTER_API_KEY,
				url: 'https://openrouter.ai/api/v1/chat/completions',
				model: OPENROUTER_MODEL,
				extraHeaders: OPENROUTER_ATTRIBUTION_HEADERS,
			}),
		]);
	}
	if (env.ANTHROPIC_API_KEY) {
		probes.push(['anthropic', probeAnthropic({ key: env.ANTHROPIC_API_KEY, model: ANTHROPIC_MODEL })]);
	}
	if (env.OPENAI_API_KEY) {
		probes.push([
			'openai',
			probeOpenAiCompat({
				key: env.OPENAI_API_KEY,
				url: 'https://api.openai.com/v1/chat/completions',
				model: OPENAI_MODEL,
			}),
		]);
	}
	// Same gate llm.js's real chain uses (providerChain(): `if
	// (process.env.GOOGLE_CLOUD_PROJECT) chain.push(vertexGeminiProvider())`) —
	// probed whenever the chain would actually reach for it, not behind a
	// separate flag.
	if (process.env.GOOGLE_CLOUD_PROJECT) {
		probes.push(['vertex-gemini', probeVertexGemini()]);
	}
	if (vertexClaudeEnabled()) {
		probes.push(['vertex-anthropic', probeVertexAnthropic({ model: ANTHROPIC_MODEL })]);
	}

	const verdicts = await Promise.all(probes.map(async ([name, p]) => [name, await p]));
	const report = Object.fromEntries(verdicts);

	const total = verdicts.length;
	const passed = verdicts.filter(([, v]) => v.status === 'ok').length;
	let overall;
	// No paid key on this deployment is not an outage: the free-first chain
	// (Groq / OpenRouter-free / NVIDIA) still serves, so report 'unconfigured'
	// rather than 'down' — otherwise a free-only install would page ops forever
	// and permanently degrade forge?health against the platform's own policy.
	if (total === 0) overall = 'unconfigured';
	else if (passed === 0) overall = 'down';
	else if (passed === total) overall = 'ok';
	else overall = 'degraded';

	report.overall = overall;
	return report;
}

// ── Open-model roster health ─────────────────────────────────────────────────
// Per-model health for the model picker: every reachable route of every roster
// model (api/_lib/model-roster.js) gets the same max_tokens:1 ping, and the
// model is judged by its routes in order:
//   ok           the first reachable route answered
//   degraded     the first route failed but a later route of the SAME model
//                answered, so the model works on a failover lane
//   down         no route answered: messages on it fall through to the
//                platform chain and are answered by another model
//   unavailable  this deployment has no credential for any of its routes
// `latencyMs` is the round-trip of the route that would serve the next call.
// Results are cached (Redis when configured, memory otherwise) for five
// minutes and a probe already in flight is shared, so a busy picker costs one
// sweep per window, not one per page view.

const ROSTER_HEALTH_KEY = 'llm:roster-health:v1';
const ROSTER_HEALTH_TTL_S = 300;
const ROSTER_PROBE_TIMEOUT_MS = 10_000;
let rosterSweep = null;

async function probeRosterTransport(transport) {
	const { transportHeaders } = await import('./model-routes.js');
	// The rawPredict surface answers a non-streaming ping; the streaming URL
	// would hold the socket open for an SSE body the probe never reads.
	const url = transport.name === 'vertex-mistral' ? transport.url.replace(':streamRawPredict', ':rawPredict') : transport.url;
	let headers;
	try {
		headers = await transportHeaders(transport);
	} catch (err) {
		return { status: 'error', error: `auth failed: ${err?.message || 'unknown'}`, latencyMs: 0 };
	}
	const started = Date.now();
	try {
		const res = await fetch(url, {
			method: 'POST',
			headers,
			body: JSON.stringify({ model: transport.model, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] }),
			signal: AbortSignal.timeout(ROSTER_PROBE_TIMEOUT_MS),
		});
		const latencyMs = Date.now() - started;
		if (res.ok) return { status: 'ok', latencyMs };
		const detail = (await res.text().catch(() => '')).slice(0, 160).replace(/\s+/g, ' ');
		return { status: 'error', error: `${res.status}${detail ? ` ${detail}` : ''}`, latencyMs };
	} catch (err) {
		const latencyMs = Date.now() - started;
		return { status: 'error', error: err?.name === 'TimeoutError' ? `timed out after ${ROSTER_PROBE_TIMEOUT_MS}ms` : 'unreachable', latencyMs };
	}
}

/**
 * Fold one model's per-route verdicts (in route order) into its health.
 * Exported for tests.
 * @param {Array<{ lane: string, status: string, latencyMs: number, error?: string }>} routes
 */
export function judgeRosterModel(routes) {
	if (!routes.length) return { status: 'unavailable', latencyMs: null, lane: null, routes };
	const firstOk = routes.findIndex((r) => r.status === 'ok');
	if (firstOk === -1) return { status: 'down', latencyMs: null, lane: null, routes };
	return {
		status: firstOk === 0 ? 'ok' : 'degraded',
		latencyMs: routes[firstOk].latencyMs,
		lane: routes[firstOk].lane,
		routes,
	};
}

async function sweepRoster() {
	const [{ ROSTER }, { rosterTransports }] = await Promise.all([import('./model-roster.js'), import('./model-routes.js')]);
	const models = {};
	await Promise.all(
		ROSTER.map(async (m) => {
			// One probe per distinct route: the OpenRouter fan-out shares a model
			// id across keys, and the first key answers for the lane.
			const seen = new Set();
			const transports = rosterTransports(m.id).filter((t) => {
				const lane = t.name.split('#')[0];
				const k = `${lane}|${t.model}`;
				if (seen.has(k)) return false;
				seen.add(k);
				return true;
			});
			const verdicts = await Promise.all(
				transports.map(async (t) => ({ lane: t.name.split('#')[0], model: t.model, ...(await probeRosterTransport(t)) })),
			);
			models[m.id] = judgeRosterModel(verdicts);
		}),
	);
	return { checkedAt: new Date().toISOString(), models };
}

/**
 * Live health of every roster model, cached for five minutes.
 * @param {{ fresh?: boolean }} [opts] `fresh` skips the cache (ops dashboards)
 */
export async function probeRosterHealth({ fresh = false } = {}) {
	const { cacheGet, cacheSet } = await import('./cache.js');
	if (!fresh) {
		const cached = await cacheGet(ROSTER_HEALTH_KEY).catch(() => null);
		if (cached?.models) return cached;
	}
	if (!rosterSweep) {
		rosterSweep = sweepRoster()
			.then(async (report) => {
				await cacheSet(ROSTER_HEALTH_KEY, report, ROSTER_HEALTH_TTL_S).catch(() => {});
				return report;
			})
			.finally(() => {
				rosterSweep = null;
			});
	}
	return rosterSweep;
}

/** The cached roster health without probing, or null when nothing is cached. */
export async function cachedRosterHealth() {
	const { cacheGet } = await import('./cache.js');
	const cached = await cacheGet(ROSTER_HEALTH_KEY).catch(() => null);
	return cached?.models ? cached : null;
}