Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(okx-chat-bot): probe the AI credential at boot so a refused key cannot report ready
nirholas5c03a989d0e2
diff --git a/scripts/gcp-triage.mjs b/scripts/gcp-triage.mjsindex 0e0689ff8..5084ff3a4 100644--- a/scripts/gcp-triage.mjs+++ b/scripts/gcp-triage.mjs@@ -158,6 +158,13 @@ const KNOWN_SIGNATURES = [class: 'owner',action: `The OKX marketplace chat bot's wallet session expired, so every XMTP client is offline and buyer chat for agent #2632 is NOT delivered. Nothing here is config-fixable and a redeploy will not help: OKX requires a human to complete an email OTP as [email protected]. The host already minted the login URL and holds the exact three commands, so do not go hunting for them: curl -s https://okx-chat-bot-<hash>-uc.a.run.app/readyz | jq .remedy (or read the ops alert, which carries the same lines). Renewing the session restores delivery within a minute; the host's next probe flips healthz okx_chat_bot back to ok on its own. Left unfixed, OKX's own chat test times out at 30 minutes and flags the listing offline. See workers/okx-chat-bot/README.md.`,},+ {+ id: 'okx-bot-provider-unauthorized',+ match: /ai_provider_unauthorized|"msg":"provider credential".*"code":"unauthorized"/i,+ services: ['okx-chat-bot'],+ class: 'owner',+ action: `The chat bot's AI credential is configured but the provider refuses it, so buyer chat for agent #2632 arrives and is never answered. This is NOT the same as a missing key and no redeploy fixes it: the two credentials this project holds have each failed this way (Vertex answers "Lightning dunning decision is deny", a GCP billing hold that reads like an IAM error; the openai-api-key secret is valid but its account answers billing_not_active). Read the exact three ways out off the host rather than guessing: curl -s $OKX_BOT_URL/readyz | jq .remedy. Preferred is Vertex, because it authenticates with the runtime service account and there is no key to mint or rotate. See workers/okx-chat-bot/README.md.`,+ },{id: 'okx-bot-daemon-restart',match: /"msg":"daemon exited"/i,diff --git a/workers/okx-chat-bot/health-server.js b/workers/okx-chat-bot/health-server.jsindex f8d525b69..e53e7bbbd 100644--- a/workers/okx-chat-bot/health-server.js+++ b/workers/okx-chat-bot/health-server.js@@ -17,7 +17,7 @@// testable over real HTTP without a daemon, a wallet, or a network.import http from 'node:http';-import { loginInstructions } from './session.js';+import { loginInstructions, providerInstructions } from './session.js';import { log } from './log.js';/**@@ -34,7 +34,17 @@ export function statusBody(cfg, live, daemonStats, bootAt) {agentId: cfg.agentId,bootAt,host: { label: cfg.host, durable: cfg.hostDurable },- provider: { name: cfg.provider, credentialed: cfg.providerCredentialed, reason: cfg.providerReason },+ provider: {+ name: cfg.provider,+ transport: cfg.providerTransport,+ credentialed: cfg.providerCredentialed,+ reason: cfg.providerReason,+ // What the provider itself last answered, as opposed to what this host+ // merely has configured. null until the first probe returns.+ verdict: live.providerProbe?.code ?? null,+ verdictDetail: live.providerProbe?.detail ?? null,+ verdictAt: live.providerProbe?.checkedAt ?? null,+ },daemon: { status: live.daemon, ...daemonStats },session: live.wallet ? { loggedIn: live.wallet.loggedIn, email: live.wallet.email } : null,agents: live.agents,@@ -44,6 +54,7 @@ export function statusBody(cfg, live, daemonStats, bootAt) {checkedAt: live.checkedAt,};if (live.verdict.needsHumanLogin) body.remedy = loginInstructions(live.login?.loginUrl, live.login?.authSessionId);+ else if (live.verdict.reason === 'ai_provider_unauthorized') body.remedy = providerInstructions(live.providerProbe?.detail);return body;}diff --git a/workers/okx-chat-bot/index.js b/workers/okx-chat-bot/index.jsindex 6a3f8cb89..4e9ed2614 100644--- a/workers/okx-chat-bot/index.js+++ b/workers/okx-chat-bot/index.js@@ -25,6 +25,7 @@ import { sql } from '../../api/_lib/db.js';import { loadConfig, paths } from './config.js';import { agentRefresh, beginLogin, daemonStatus, exec, walletStatus } from './cli.js';import { startHealthServer } from './health-server.js';+import { loginCodex, probeProvider } from './provider.js';import { classify, loginInstructions } from './session.js';import { restoreState, snapshotState } from './state.js';import { createSupervisor } from './supervisor.js';@@ -44,6 +45,10 @@ const live = {login: null,workspace: null,stateRestore: null,+ // What the provider's own API last answered. Undefined until the first probe,+ // which is why classify() treats a missing verdict as "no opinion" rather than+ // as a failure: a host must not read red for a probe that has not run yet.+ providerProbe: null,};async function heartbeat(cfg, supervisor) {@@ -61,7 +66,9 @@ async function heartbeat(cfg, supervisor) {agentCount: live.agents.agentCount,activeClients: live.agents.activeClients,provider: cfg.provider,+ providerTransport: cfg.providerTransport,providerCredentialed: cfg.providerCredentialed,+ providerVerdict: live.providerProbe?.code ?? null,daemonRestarts: supervisor.stats().restarts,checkedAt: live.checkedAt,};@@ -106,6 +113,24 @@ async function maybeAlert(cfg, verdict) {}).catch((err) => log.warn('ops alert failed', { err: err?.message }));}+// Ask the provider whether it still serves this host. Kept off the session+// probe's path: it is a network call to a third party, and a slow one must not+// delay the verdict about whether XMTP is delivering. A transient `unreachable`+// answer is remembered as-is, because classify() deliberately does not fail+// readiness on it.+async function probeProviderCredential(cfg) {+ const before = live.providerProbe?.code ?? null;+ live.providerProbe = await probeProvider(cfg);+ if (live.providerProbe.code !== before) {+ log[live.providerProbe.code === 'unauthorized' ? 'error' : 'info']('provider credential', {+ provider: cfg.provider,+ transport: cfg.providerTransport,+ code: live.providerProbe.code,+ detail: live.providerProbe.detail,+ });+ }+}+// One probe at a time. A probe can legitimately outrun its own interval (the// three CLI calls are bounded at 15s + 30s + 90s), and two in flight would race// on the login mint below: both would see `live.login` empty and start a second@@ -128,7 +153,13 @@ async function runProbe(cfg, supervisor) {const daemon = await daemonStatus(cfg);const wallet = daemon.startsWith('running') ? await walletStatus(cfg) : null;const agents = wallet?.loggedIn ? await agentRefresh(cfg) : { agentCount: 0, activeClients: 0 };- const verdict = classify({ daemon, wallet, agents, providerCredentialed: cfg.providerCredentialed });+ const verdict = classify({+ daemon,+ wallet,+ agents,+ providerCredentialed: cfg.providerCredentialed,+ providerProbe: live.providerProbe,+ });// Mint a fresh login URL only when one is actually needed, and only when we// do not already hold an unused one: `login --phase init` starts a new auth@@ -173,7 +204,7 @@ async function main() {if (!cfg.providerCredentialed) {log.error('no AI-provider credential: the daemon will receive chat but cannot author replies', {provider: cfg.provider,- needs: 'ANTHROPIC_API_KEY (preferred) or OPENAI_API_KEY',+ needs: 'CLAUDE_CODE_USE_VERTEX=1 (preferred), ANTHROPIC_API_KEY, or OPENAI_API_KEY',});}@@ -190,6 +221,12 @@ async function main() {await exec(cfg, 'okx-a2a', ['config', 'provider', '--provider', cfg.provider], { timeoutMs: 30_000 });await exec(cfg, 'okx-a2a', ['config', 'permissions', '--preset', 'bypass'], { timeoutMs: 30_000 });+ // Codex reads ~/.codex/auth.json, never OPENAI_API_KEY from the environment,+ // so without this the key is present, the CLI spawns, and every reply dies on+ // a 401 the buyer only experiences as silence. No-op for every other provider.+ const codexLogin = await loginCodex(cfg);+ if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);+const supervisor = createSupervisor(cfg, p);supervisor.start();const server = startHealthServer(cfg, live, supervisor, BOOT_AT);@@ -208,6 +245,13 @@ async function main() {() => snapshotState(cfg, { reason: 'timer' }).catch(() => {}),cfg.snapshotMs,);+ const providerProbeTimer = setInterval(+ () => probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message })),+ cfg.providerProbeMs,+ );+ // Run the first credential probe immediately, but off the boot path: a slow+ // provider must delay the verdict, never the daemon that receives the chat.+ probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message }));// Give the daemon a moment to bind XMTP before the first verdict, so a normal// boot does not page as "down".setTimeout(probe, 10_000);@@ -220,6 +264,7 @@ async function main() {clearInterval(probeTimer);clearInterval(heartbeatTimer);clearInterval(snapshotTimer);+ clearInterval(providerProbeTimer);server?.close();// Order matters: stop the daemon FIRST so its sqlite files are quiesced,// then snapshot. A live-copy snapshot can tear; this one cannot.diff --git a/workers/okx-chat-bot/provider.js b/workers/okx-chat-bot/provider.jsnew file mode 100644index 000000000..429da54bc--- /dev/null+++ b/workers/okx-chat-bot/provider.js@@ -0,0 +1,190 @@+// okx-chat-bot: does the AI credential actually work, right now?+//+// config.js answers "is a credential configured". That is a weaker question than+// it looks, and on this project the two answers came apart in both directions on+// 2026-09-04: the GCP project's Vertex access answers+// `PERMISSION_DENIED: Lightning dunning decision is deny` (a billing hold that+// reads like an IAM problem), and the `openai-api-key` secret is a valid,+// well-formed key whose account answers `billing_not_active`. Either one would+// have satisfied a presence check, booted green, received a buyer's message and+// never answered it, which is precisely the silent failure this worker exists to+// kill, rebuilt one level up.+//+// So the credential is asked, not assumed. One tiny request to the provider's own+// API, classified into a verdict readiness is judged on:+//+// ok the provider answered; a reply can be authored+// unauthorized the provider refused the credential (expired, revoked, unpaid)+// unreachable the network or the provider blipped; not the credential's fault+// unprobed no cheap, honest probe exists for this transport+//+// `unreachable` deliberately does NOT fail readiness. A provider outage is real+// but transient and self-heals; treating it like a dead key would page a human+// for something no human can fix, and a page nobody can act on is how alerts get+// ignored.++import { execFile } from 'node:child_process';+import { getGcpAccessToken } from '../../api/_lib/gcp-auth.js';+import { cliEnv } from './cli.js';+import { log } from './log.js';++const PROBE_TIMEOUT_MS = 20_000;++// Small on purpose: the probe exists to exercise auth, not to generate text.+const VERTEX_PROBE_MODEL = 'claude-haiku-4-5@20251001';+const ANTHROPIC_PROBE_MODEL = 'claude-haiku-4-5-20251001';+const OPENAI_PROBE_MODEL = 'gpt-4o-mini';++/**+ * Turn an HTTP status into a credential verdict.+ *+ * 400 and 404 count as `ok`: the provider had to authenticate the caller before+ * it could object to the model id or the request shape, so a quibble about the+ * body is proof the credential was accepted. Only an explicit refusal+ * (401/403/402, or a 429 that names billing rather than rate) is the credential's+ * fault; everything else is the provider having a bad minute.+ */+export function classifyProbeStatus(status, body = '') {+ if (status >= 200 && status < 300) return { code: 'ok', detail: 'the provider answered' };+ if (status === 400 || status === 404) {+ return { code: 'ok', detail: `credential accepted (provider returned ${status} on the probe request itself)` };+ }+ if (status === 401 || status === 403 || status === 402) {+ return { code: 'unauthorized', detail: `the provider refused the credential (${status}): ${summarize(body)}` };+ }+ if (status === 429) {+ // A rate limit is transient and is not the credential's fault. A suspended+ // or unpaid account also answers 429, and that one no retry will fix.+ const billing = /billing|not active|quota|insufficient|suspend/i.test(body);+ return billing+ ? { code: 'unauthorized', detail: `the provider account cannot serve requests: ${summarize(body)}` }+ : { code: 'unreachable', detail: 'rate limited by the provider' };+ }+ return { code: 'unreachable', detail: `provider returned ${status}: ${summarize(body)}` };+}++function summarize(body) {+ const text = String(body || '').replace(/\s+/g, ' ').trim();+ try {+ const j = JSON.parse(text);+ return String(j?.error?.message || j?.error?.status || text).slice(0, 200);+ } catch {+ return text.slice(0, 200);+ }+}++async function postJson(url, headers, body) {+ const ctl = new AbortController();+ const timer = setTimeout(() => ctl.abort(), PROBE_TIMEOUT_MS);+ try {+ const res = await fetch(url, {+ method: 'POST',+ headers: { 'content-type': 'application/json', ...headers },+ body: JSON.stringify(body),+ signal: ctl.signal,+ });+ return classifyProbeStatus(res.status, await res.text());+ } catch (err) {+ return { code: 'unreachable', detail: `probe request failed: ${err?.message || err}` };+ } finally {+ clearTimeout(timer);+ }+}++function vertexLocation(env) {+ return (env.CLOUD_ML_REGION || env.GOOGLE_CLOUD_LOCATION_CLAUDE || 'global').trim();+}++async function probeVertex(env) {+ const project = env.ANTHROPIC_VERTEX_PROJECT_ID || env.GOOGLE_CLOUD_PROJECT;+ const location = vertexLocation(env);+ const model = env.OKX_BOT_VERTEX_PROBE_MODEL || VERTEX_PROBE_MODEL;+ let token;+ try {+ token = await getGcpAccessToken();+ } catch (err) {+ // No ADC at all is a configuration fault, not a blip: the CLI will fail the+ // same way on the buyer's first message.+ return { code: 'unauthorized', detail: `no GCP credentials for Vertex: ${err?.message || err}` };+ }+ const host = location === 'global' ? 'aiplatform.googleapis.com' : `${location}-aiplatform.googleapis.com`;+ return postJson(+ `https://${host}/v1/projects/${project}/locations/${location}/publishers/anthropic/models/${model}:rawPredict`,+ { authorization: `Bearer ${token}` },+ { anthropic_version: 'vertex-2023-10-16', max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },+ );+}++async function probeAnthropicKey(env) {+ return postJson(+ 'https://api.anthropic.com/v1/messages',+ { 'x-api-key': env.ANTHROPIC_API_KEY, 'anthropic-version': '2023-06-01' },+ { model: ANTHROPIC_PROBE_MODEL, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },+ );+}++async function probeOpenAiKey(env) {+ return postJson(+ 'https://api.openai.com/v1/chat/completions',+ { authorization: `Bearer ${env.OPENAI_API_KEY}` },+ { model: OPENAI_PROBE_MODEL, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },+ );+}++/**+ * Ask the configured provider whether it will serve this host.+ *+ * @param {ReturnType<import('./config.js').loadConfig>} cfg+ * @param {NodeJS.ProcessEnv} [env]+ * @returns {Promise<{ code: 'ok'|'unauthorized'|'unreachable'|'unprobed', detail: string, transport: string, checkedAt: number }>}+ */+export async function probeProvider(cfg, env = process.env) {+ const wrap = (r) => ({ ...r, transport: cfg.providerTransport, checkedAt: Date.now() });++ if (!cfg.providerCredentialed) {+ return wrap({ code: 'unauthorized', detail: 'no AI-provider credential is configured' });+ }+ if (cfg.providerTransport === 'vertex') return wrap(await probeVertex(env));+ if (cfg.provider === 'claude' && env.ANTHROPIC_API_KEY) return wrap(await probeAnthropicKey(env));+ if (cfg.provider === 'codex' && env.OPENAI_API_KEY) return wrap(await probeOpenAiKey(env));++ // An interactive OAuth grant and a CLAUDE_CODE_OAUTH_TOKEN are refreshed by+ // the CLI itself; there is no endpoint this worker can call that proves the+ // grant without reimplementing that refresh. Claiming a verdict here would be+ // inventing one, so the probe abstains and the presence check stands.+ return wrap({ code: 'unprobed', detail: `no credential probe exists for the ${cfg.providerTransport} transport` });+}++/**+ * Hand the codex CLI its API key.+ *+ * Codex >= 0.153 does NOT authenticate from `OPENAI_API_KEY` in the environment:+ * it reads ~/.codex/auth.json, and without it every request fails+ * `401 Missing bearer or basic authentication in header` while the env var sits+ * there looking correct. `codex login --with-api-key` reads the key from stdin+ * and writes that file, so this runs at boot, before the daemon can spawn a+ * subsession. Keyless providers are a no-op.+ *+ * @param {ReturnType<import('./config.js').loadConfig>} cfg+ * @param {NodeJS.ProcessEnv} [env]+ * @returns {Promise<{ ran: boolean, ok: boolean, detail: string }>}+ */+export function loginCodex(cfg, env = process.env) {+ if (cfg.provider !== 'codex') return Promise.resolve({ ran: false, ok: true, detail: 'provider is not codex' });+ const key = env.OPENAI_API_KEY;+ if (!key) return Promise.resolve({ ran: false, ok: false, detail: 'codex is the provider but OPENAI_API_KEY is unset' });++ return new Promise((resolve) => {+ const child = execFile(+ 'codex',+ ['login', '--with-api-key'],+ { env: cliEnv(cfg), timeout: 60_000, encoding: 'utf8' },+ (err, stdout, stderr) => {+ const detail = (stderr || stdout || '').trim().slice(0, 300);+ if (err) log.error('codex login failed', { detail });+ resolve({ ran: true, ok: !err, detail: detail || (err ? String(err.message) : 'logged in') });+ },+ );+ child.stdin?.end(key);+ });+}diff --git a/workers/okx-chat-bot/session.js b/workers/okx-chat-bot/session.jsindex 3632a7cb1..a5f07ff7b 100644--- a/workers/okx-chat-bot/session.js+++ b/workers/okx-chat-bot/session.js@@ -22,6 +22,32 @@ export function loginInstructions(loginUrl, authSessionId) {];}+/**+ * The exact commands that put a working AI credential on the service.+ *+ * A refused credential is the other failure only a human can clear, so it gets+ * the same treatment as an expired session: the fix travels with the status+ * rather than living in a runbook someone has to find. Vertex leads because it+ * is the only option with no secret to mint, rotate or forget, and it bills the+ * GCP credit pool the platform already prefers.+ */+export function providerInstructions(detail = '') {+ return [+ `The provider refused this host's credential: ${detail}`,+ '',+ '1. Preferred (no secret): clear the project billing hold, then',+ ' gcloud run services update okx-chat-bot --region us-central1 \\',+ " --update-env-vars=CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=aerial-vehicle-466722-p5,CLOUD_ML_REGION=global",+ '2. Or mint an Anthropic key and put it on the service through Secret Manager:',+ " printf '%s' \"$ANTHROPIC_API_KEY\" | gcloud secrets create anthropic-api-key --data-file=-",+ ' then add ANTHROPIC_API_KEY=anthropic-api-key:latest to --set-secrets in',+ ' workers/okx-chat-bot/cloudbuild.yaml and redeploy (a hand-patched secret is',+ ' stripped by the next deploy).',+ '3. Or reactivate billing on the OpenAI account behind the openai-api-key secret',+ ' and pin OKX_BOT_AI_PROVIDER=codex.',+ ];+}+/*** Roll the three raw probes into one verdict. Pure, so the state machine is* testable without a daemon, a wallet, or a network.@@ -31,10 +57,11 @@ export function loginInstructions(loginUrl, authSessionId) {* @param {{ loggedIn: boolean, email?: string }|null} probe.wallet null = CLI failed* @param {{ agentCount: number, activeClients: number }} probe.agents* @param {boolean} probe.providerCredentialed+ * @param {{ code: string, detail: string }} [probe.providerProbe] what the provider's own API answered* @returns {{ status: 'ok'|'degraded'|'down'|'unknown', ready: boolean, reason: string, detail: string,* needsHumanLogin: boolean }}*/-export function classify({ daemon, wallet, agents, providerCredentialed }) {+export function classify({ daemon, wallet, agents, providerCredentialed, providerProbe }) {const daemonRunning = typeof daemon === 'string' && daemon.startsWith('running');if (!daemonRunning) {@@ -88,7 +115,24 @@ export function classify({ daemon, wallet, agents, providerCredentialed }) {reason: 'ai_provider_uncredentialed',detail:'chat is delivered but no AI-provider credential is configured, so the spawned subsession cannot ' +- 'author a reply. Set ANTHROPIC_API_KEY (preferred) or OPENAI_API_KEY on the service.',+ 'author a reply. Set CLAUDE_CODE_USE_VERTEX=1 (preferred: no key to rotate), ANTHROPIC_API_KEY, ' ++ 'or OPENAI_API_KEY on the service.',+ needsHumanLogin: false,+ };+ }++ // A configured credential is not a working one. A key the provider refuses+ // looks identical from inside this process to a key that works, right up until+ // a buyer's message arrives and the subsession cannot author a reply, which is+ // the silent failure the whole worker exists to make loud.+ if (providerProbe?.code === 'unauthorized') {+ return {+ status: 'degraded',+ ready: false,+ reason: 'ai_provider_unauthorized',+ detail:+ `chat is delivered but the AI provider refuses this host's credential, so no reply can be authored: ` ++ `${providerProbe.detail}`,needsHumanLogin: false,};}