Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(okx-chat-bot): probe the AI credential at boot so a refused key cannot report ready

nirholas5c03a989d0e2
diff --git a/scripts/gcp-triage.mjs b/scripts/gcp-triage.mjs
index 0e0689ff8..5084ff3a4 100644
--- a/scripts/gcp-triage.mjs
+++ b/scripts/gcp-triage.mjs
@@ -158,6 +158,13 @@ const KNOWN_SIGNATURES = [
class: 'owner',
action: `The OKX marketplace chat bot's wallet session expired, so every XMTP client is offline and buyer chat for agent #2632 is NOT delivered. Nothing here is config-fixable and a redeploy will not help: OKX requires a human to complete an email OTP as [email protected]. The host already minted the login URL and holds the exact three commands, so do not go hunting for them: curl -s https://okx-chat-bot-<hash>-uc.a.run.app/readyz | jq .remedy (or read the ops alert, which carries the same lines). Renewing the session restores delivery within a minute; the host's next probe flips healthz okx_chat_bot back to ok on its own. Left unfixed, OKX's own chat test times out at 30 minutes and flags the listing offline. See workers/okx-chat-bot/README.md.`,
},
+ {
+ id: 'okx-bot-provider-unauthorized',
+ match: /ai_provider_unauthorized|"msg":"provider credential".*"code":"unauthorized"/i,
+ services: ['okx-chat-bot'],
+ class: 'owner',
+ action: `The chat bot's AI credential is configured but the provider refuses it, so buyer chat for agent #2632 arrives and is never answered. This is NOT the same as a missing key and no redeploy fixes it: the two credentials this project holds have each failed this way (Vertex answers "Lightning dunning decision is deny", a GCP billing hold that reads like an IAM error; the openai-api-key secret is valid but its account answers billing_not_active). Read the exact three ways out off the host rather than guessing: curl -s $OKX_BOT_URL/readyz | jq .remedy. Preferred is Vertex, because it authenticates with the runtime service account and there is no key to mint or rotate. See workers/okx-chat-bot/README.md.`,
+ },
{
id: 'okx-bot-daemon-restart',
match: /"msg":"daemon exited"/i,
diff --git a/workers/okx-chat-bot/health-server.js b/workers/okx-chat-bot/health-server.js
index f8d525b69..e53e7bbbd 100644
--- a/workers/okx-chat-bot/health-server.js
+++ b/workers/okx-chat-bot/health-server.js
@@ -17,7 +17,7 @@
// testable over real HTTP without a daemon, a wallet, or a network.
import http from 'node:http';
-import { loginInstructions } from './session.js';
+import { loginInstructions, providerInstructions } from './session.js';
import { log } from './log.js';
/**
@@ -34,7 +34,17 @@ export function statusBody(cfg, live, daemonStats, bootAt) {
agentId: cfg.agentId,
bootAt,
host: { label: cfg.host, durable: cfg.hostDurable },
- provider: { name: cfg.provider, credentialed: cfg.providerCredentialed, reason: cfg.providerReason },
+ provider: {
+ name: cfg.provider,
+ transport: cfg.providerTransport,
+ credentialed: cfg.providerCredentialed,
+ reason: cfg.providerReason,
+ // What the provider itself last answered, as opposed to what this host
+ // merely has configured. null until the first probe returns.
+ verdict: live.providerProbe?.code ?? null,
+ verdictDetail: live.providerProbe?.detail ?? null,
+ verdictAt: live.providerProbe?.checkedAt ?? null,
+ },
daemon: { status: live.daemon, ...daemonStats },
session: live.wallet ? { loggedIn: live.wallet.loggedIn, email: live.wallet.email } : null,
agents: live.agents,
@@ -44,6 +54,7 @@ export function statusBody(cfg, live, daemonStats, bootAt) {
checkedAt: live.checkedAt,
};
if (live.verdict.needsHumanLogin) body.remedy = loginInstructions(live.login?.loginUrl, live.login?.authSessionId);
+ else if (live.verdict.reason === 'ai_provider_unauthorized') body.remedy = providerInstructions(live.providerProbe?.detail);
return body;
}
diff --git a/workers/okx-chat-bot/index.js b/workers/okx-chat-bot/index.js
index 6a3f8cb89..4e9ed2614 100644
--- a/workers/okx-chat-bot/index.js
+++ b/workers/okx-chat-bot/index.js
@@ -25,6 +25,7 @@ import { sql } from '../../api/_lib/db.js';
import { loadConfig, paths } from './config.js';
import { agentRefresh, beginLogin, daemonStatus, exec, walletStatus } from './cli.js';
import { startHealthServer } from './health-server.js';
+import { loginCodex, probeProvider } from './provider.js';
import { classify, loginInstructions } from './session.js';
import { restoreState, snapshotState } from './state.js';
import { createSupervisor } from './supervisor.js';
@@ -44,6 +45,10 @@ const live = {
login: null,
workspace: null,
stateRestore: null,
+ // What the provider's own API last answered. Undefined until the first probe,
+ // which is why classify() treats a missing verdict as "no opinion" rather than
+ // as a failure: a host must not read red for a probe that has not run yet.
+ providerProbe: null,
};
async function heartbeat(cfg, supervisor) {
@@ -61,7 +66,9 @@ async function heartbeat(cfg, supervisor) {
agentCount: live.agents.agentCount,
activeClients: live.agents.activeClients,
provider: cfg.provider,
+ providerTransport: cfg.providerTransport,
providerCredentialed: cfg.providerCredentialed,
+ providerVerdict: live.providerProbe?.code ?? null,
daemonRestarts: supervisor.stats().restarts,
checkedAt: live.checkedAt,
};
@@ -106,6 +113,24 @@ async function maybeAlert(cfg, verdict) {
}).catch((err) => log.warn('ops alert failed', { err: err?.message }));
}
+// Ask the provider whether it still serves this host. Kept off the session
+// probe's path: it is a network call to a third party, and a slow one must not
+// delay the verdict about whether XMTP is delivering. A transient `unreachable`
+// answer is remembered as-is, because classify() deliberately does not fail
+// readiness on it.
+async function probeProviderCredential(cfg) {
+ const before = live.providerProbe?.code ?? null;
+ live.providerProbe = await probeProvider(cfg);
+ if (live.providerProbe.code !== before) {
+ log[live.providerProbe.code === 'unauthorized' ? 'error' : 'info']('provider credential', {
+ provider: cfg.provider,
+ transport: cfg.providerTransport,
+ code: live.providerProbe.code,
+ detail: live.providerProbe.detail,
+ });
+ }
+}
+
// One probe at a time. A probe can legitimately outrun its own interval (the
// three CLI calls are bounded at 15s + 30s + 90s), and two in flight would race
// on the login mint below: both would see `live.login` empty and start a second
@@ -128,7 +153,13 @@ async function runProbe(cfg, supervisor) {
const daemon = await daemonStatus(cfg);
const wallet = daemon.startsWith('running') ? await walletStatus(cfg) : null;
const agents = wallet?.loggedIn ? await agentRefresh(cfg) : { agentCount: 0, activeClients: 0 };
- const verdict = classify({ daemon, wallet, agents, providerCredentialed: cfg.providerCredentialed });
+ const verdict = classify({
+ daemon,
+ wallet,
+ agents,
+ providerCredentialed: cfg.providerCredentialed,
+ providerProbe: live.providerProbe,
+ });
// Mint a fresh login URL only when one is actually needed, and only when we
// do not already hold an unused one: `login --phase init` starts a new auth
@@ -173,7 +204,7 @@ async function main() {
if (!cfg.providerCredentialed) {
log.error('no AI-provider credential: the daemon will receive chat but cannot author replies', {
provider: cfg.provider,
- needs: 'ANTHROPIC_API_KEY (preferred) or OPENAI_API_KEY',
+ needs: 'CLAUDE_CODE_USE_VERTEX=1 (preferred), ANTHROPIC_API_KEY, or OPENAI_API_KEY',
});
}
@@ -190,6 +221,12 @@ async function main() {
await exec(cfg, 'okx-a2a', ['config', 'provider', '--provider', cfg.provider], { timeoutMs: 30_000 });
await exec(cfg, 'okx-a2a', ['config', 'permissions', '--preset', 'bypass'], { timeoutMs: 30_000 });
+ // Codex reads ~/.codex/auth.json, never OPENAI_API_KEY from the environment,
+ // so without this the key is present, the CLI spawns, and every reply dies on
+ // a 401 the buyer only experiences as silence. No-op for every other provider.
+ const codexLogin = await loginCodex(cfg);
+ if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);
+
const supervisor = createSupervisor(cfg, p);
supervisor.start();
const server = startHealthServer(cfg, live, supervisor, BOOT_AT);
@@ -208,6 +245,13 @@ async function main() {
() => snapshotState(cfg, { reason: 'timer' }).catch(() => {}),
cfg.snapshotMs,
);
+ const providerProbeTimer = setInterval(
+ () => probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message })),
+ cfg.providerProbeMs,
+ );
+ // Run the first credential probe immediately, but off the boot path: a slow
+ // provider must delay the verdict, never the daemon that receives the chat.
+ probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message }));
// Give the daemon a moment to bind XMTP before the first verdict, so a normal
// boot does not page as "down".
setTimeout(probe, 10_000);
@@ -220,6 +264,7 @@ async function main() {
clearInterval(probeTimer);
clearInterval(heartbeatTimer);
clearInterval(snapshotTimer);
+ clearInterval(providerProbeTimer);
server?.close();
// Order matters: stop the daemon FIRST so its sqlite files are quiesced,
// then snapshot. A live-copy snapshot can tear; this one cannot.
diff --git a/workers/okx-chat-bot/provider.js b/workers/okx-chat-bot/provider.js
new file mode 100644
index 000000000..429da54bc
--- /dev/null
+++ b/workers/okx-chat-bot/provider.js
@@ -0,0 +1,190 @@
+// okx-chat-bot: does the AI credential actually work, right now?
+//
+// config.js answers "is a credential configured". That is a weaker question than
+// it looks, and on this project the two answers came apart in both directions on
+// 2026-09-04: the GCP project's Vertex access answers
+// `PERMISSION_DENIED: Lightning dunning decision is deny` (a billing hold that
+// reads like an IAM problem), and the `openai-api-key` secret is a valid,
+// well-formed key whose account answers `billing_not_active`. Either one would
+// have satisfied a presence check, booted green, received a buyer's message and
+// never answered it, which is precisely the silent failure this worker exists to
+// kill, rebuilt one level up.
+//
+// So the credential is asked, not assumed. One tiny request to the provider's own
+// API, classified into a verdict readiness is judged on:
+//
+// ok the provider answered; a reply can be authored
+// unauthorized the provider refused the credential (expired, revoked, unpaid)
+// unreachable the network or the provider blipped; not the credential's fault
+// unprobed no cheap, honest probe exists for this transport
+//
+// `unreachable` deliberately does NOT fail readiness. A provider outage is real
+// but transient and self-heals; treating it like a dead key would page a human
+// for something no human can fix, and a page nobody can act on is how alerts get
+// ignored.
+
+import { execFile } from 'node:child_process';
+import { getGcpAccessToken } from '../../api/_lib/gcp-auth.js';
+import { cliEnv } from './cli.js';
+import { log } from './log.js';
+
+const PROBE_TIMEOUT_MS = 20_000;
+
+// Small on purpose: the probe exists to exercise auth, not to generate text.
+const VERTEX_PROBE_MODEL = 'claude-haiku-4-5@20251001';
+const ANTHROPIC_PROBE_MODEL = 'claude-haiku-4-5-20251001';
+const OPENAI_PROBE_MODEL = 'gpt-4o-mini';
+
+/**
+ * Turn an HTTP status into a credential verdict.
+ *
+ * 400 and 404 count as `ok`: the provider had to authenticate the caller before
+ * it could object to the model id or the request shape, so a quibble about the
+ * body is proof the credential was accepted. Only an explicit refusal
+ * (401/403/402, or a 429 that names billing rather than rate) is the credential's
+ * fault; everything else is the provider having a bad minute.
+ */
+export function classifyProbeStatus(status, body = '') {
+ if (status >= 200 && status < 300) return { code: 'ok', detail: 'the provider answered' };
+ if (status === 400 || status === 404) {
+ return { code: 'ok', detail: `credential accepted (provider returned ${status} on the probe request itself)` };
+ }
+ if (status === 401 || status === 403 || status === 402) {
+ return { code: 'unauthorized', detail: `the provider refused the credential (${status}): ${summarize(body)}` };
+ }
+ if (status === 429) {
+ // A rate limit is transient and is not the credential's fault. A suspended
+ // or unpaid account also answers 429, and that one no retry will fix.
+ const billing = /billing|not active|quota|insufficient|suspend/i.test(body);
+ return billing
+ ? { code: 'unauthorized', detail: `the provider account cannot serve requests: ${summarize(body)}` }
+ : { code: 'unreachable', detail: 'rate limited by the provider' };
+ }
+ return { code: 'unreachable', detail: `provider returned ${status}: ${summarize(body)}` };
+}
+
+function summarize(body) {
+ const text = String(body || '').replace(/\s+/g, ' ').trim();
+ try {
+ const j = JSON.parse(text);
+ return String(j?.error?.message || j?.error?.status || text).slice(0, 200);
+ } catch {
+ return text.slice(0, 200);
+ }
+}
+
+async function postJson(url, headers, body) {
+ const ctl = new AbortController();
+ const timer = setTimeout(() => ctl.abort(), PROBE_TIMEOUT_MS);
+ try {
+ const res = await fetch(url, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', ...headers },
+ body: JSON.stringify(body),
+ signal: ctl.signal,
+ });
+ return classifyProbeStatus(res.status, await res.text());
+ } catch (err) {
+ return { code: 'unreachable', detail: `probe request failed: ${err?.message || err}` };
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+function vertexLocation(env) {
+ return (env.CLOUD_ML_REGION || env.GOOGLE_CLOUD_LOCATION_CLAUDE || 'global').trim();
+}
+
+async function probeVertex(env) {
+ const project = env.ANTHROPIC_VERTEX_PROJECT_ID || env.GOOGLE_CLOUD_PROJECT;
+ const location = vertexLocation(env);
+ const model = env.OKX_BOT_VERTEX_PROBE_MODEL || VERTEX_PROBE_MODEL;
+ let token;
+ try {
+ token = await getGcpAccessToken();
+ } catch (err) {
+ // No ADC at all is a configuration fault, not a blip: the CLI will fail the
+ // same way on the buyer's first message.
+ return { code: 'unauthorized', detail: `no GCP credentials for Vertex: ${err?.message || err}` };
+ }
+ const host = location === 'global' ? 'aiplatform.googleapis.com' : `${location}-aiplatform.googleapis.com`;
+ return postJson(
+ `https://${host}/v1/projects/${project}/locations/${location}/publishers/anthropic/models/${model}:rawPredict`,
+ { authorization: `Bearer ${token}` },
+ { anthropic_version: 'vertex-2023-10-16', max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },
+ );
+}
+
+async function probeAnthropicKey(env) {
+ return postJson(
+ 'https://api.anthropic.com/v1/messages',
+ { 'x-api-key': env.ANTHROPIC_API_KEY, 'anthropic-version': '2023-06-01' },
+ { model: ANTHROPIC_PROBE_MODEL, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },
+ );
+}
+
+async function probeOpenAiKey(env) {
+ return postJson(
+ 'https://api.openai.com/v1/chat/completions',
+ { authorization: `Bearer ${env.OPENAI_API_KEY}` },
+ { model: OPENAI_PROBE_MODEL, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },
+ );
+}
+
+/**
+ * Ask the configured provider whether it will serve this host.
+ *
+ * @param {ReturnType<import('./config.js').loadConfig>} cfg
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {Promise<{ code: 'ok'|'unauthorized'|'unreachable'|'unprobed', detail: string, transport: string, checkedAt: number }>}
+ */
+export async function probeProvider(cfg, env = process.env) {
+ const wrap = (r) => ({ ...r, transport: cfg.providerTransport, checkedAt: Date.now() });
+
+ if (!cfg.providerCredentialed) {
+ return wrap({ code: 'unauthorized', detail: 'no AI-provider credential is configured' });
+ }
+ if (cfg.providerTransport === 'vertex') return wrap(await probeVertex(env));
+ if (cfg.provider === 'claude' && env.ANTHROPIC_API_KEY) return wrap(await probeAnthropicKey(env));
+ if (cfg.provider === 'codex' && env.OPENAI_API_KEY) return wrap(await probeOpenAiKey(env));
+
+ // An interactive OAuth grant and a CLAUDE_CODE_OAUTH_TOKEN are refreshed by
+ // the CLI itself; there is no endpoint this worker can call that proves the
+ // grant without reimplementing that refresh. Claiming a verdict here would be
+ // inventing one, so the probe abstains and the presence check stands.
+ return wrap({ code: 'unprobed', detail: `no credential probe exists for the ${cfg.providerTransport} transport` });
+}
+
+/**
+ * Hand the codex CLI its API key.
+ *
+ * Codex >= 0.153 does NOT authenticate from `OPENAI_API_KEY` in the environment:
+ * it reads ~/.codex/auth.json, and without it every request fails
+ * `401 Missing bearer or basic authentication in header` while the env var sits
+ * there looking correct. `codex login --with-api-key` reads the key from stdin
+ * and writes that file, so this runs at boot, before the daemon can spawn a
+ * subsession. Keyless providers are a no-op.
+ *
+ * @param {ReturnType<import('./config.js').loadConfig>} cfg
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {Promise<{ ran: boolean, ok: boolean, detail: string }>}
+ */
+export function loginCodex(cfg, env = process.env) {
+ if (cfg.provider !== 'codex') return Promise.resolve({ ran: false, ok: true, detail: 'provider is not codex' });
+ const key = env.OPENAI_API_KEY;
+ if (!key) return Promise.resolve({ ran: false, ok: false, detail: 'codex is the provider but OPENAI_API_KEY is unset' });
+
+ return new Promise((resolve) => {
+ const child = execFile(
+ 'codex',
+ ['login', '--with-api-key'],
+ { env: cliEnv(cfg), timeout: 60_000, encoding: 'utf8' },
+ (err, stdout, stderr) => {
+ const detail = (stderr || stdout || '').trim().slice(0, 300);
+ if (err) log.error('codex login failed', { detail });
+ resolve({ ran: true, ok: !err, detail: detail || (err ? String(err.message) : 'logged in') });
+ },
+ );
+ child.stdin?.end(key);
+ });
+}
diff --git a/workers/okx-chat-bot/session.js b/workers/okx-chat-bot/session.js
index 3632a7cb1..a5f07ff7b 100644
--- a/workers/okx-chat-bot/session.js
+++ b/workers/okx-chat-bot/session.js
@@ -22,6 +22,32 @@ export function loginInstructions(loginUrl, authSessionId) {
];
}
+/**
+ * The exact commands that put a working AI credential on the service.
+ *
+ * A refused credential is the other failure only a human can clear, so it gets
+ * the same treatment as an expired session: the fix travels with the status
+ * rather than living in a runbook someone has to find. Vertex leads because it
+ * is the only option with no secret to mint, rotate or forget, and it bills the
+ * GCP credit pool the platform already prefers.
+ */
+export function providerInstructions(detail = '') {
+ return [
+ `The provider refused this host's credential: ${detail}`,
+ '',
+ '1. Preferred (no secret): clear the project billing hold, then',
+ ' gcloud run services update okx-chat-bot --region us-central1 \\',
+ " --update-env-vars=CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=aerial-vehicle-466722-p5,CLOUD_ML_REGION=global",
+ '2. Or mint an Anthropic key and put it on the service through Secret Manager:',
+ " printf '%s' \"$ANTHROPIC_API_KEY\" | gcloud secrets create anthropic-api-key --data-file=-",
+ ' then add ANTHROPIC_API_KEY=anthropic-api-key:latest to --set-secrets in',
+ ' workers/okx-chat-bot/cloudbuild.yaml and redeploy (a hand-patched secret is',
+ ' stripped by the next deploy).',
+ '3. Or reactivate billing on the OpenAI account behind the openai-api-key secret',
+ ' and pin OKX_BOT_AI_PROVIDER=codex.',
+ ];
+}
+
/**
* Roll the three raw probes into one verdict. Pure, so the state machine is
* testable without a daemon, a wallet, or a network.
@@ -31,10 +57,11 @@ export function loginInstructions(loginUrl, authSessionId) {
* @param {{ loggedIn: boolean, email?: string }|null} probe.wallet null = CLI failed
* @param {{ agentCount: number, activeClients: number }} probe.agents
* @param {boolean} probe.providerCredentialed
+ * @param {{ code: string, detail: string }} [probe.providerProbe] what the provider's own API answered
* @returns {{ status: 'ok'|'degraded'|'down'|'unknown', ready: boolean, reason: string, detail: string,
* needsHumanLogin: boolean }}
*/
-export function classify({ daemon, wallet, agents, providerCredentialed }) {
+export function classify({ daemon, wallet, agents, providerCredentialed, providerProbe }) {
const daemonRunning = typeof daemon === 'string' && daemon.startsWith('running');
if (!daemonRunning) {
@@ -88,7 +115,24 @@ export function classify({ daemon, wallet, agents, providerCredentialed }) {
reason: 'ai_provider_uncredentialed',
detail:
'chat is delivered but no AI-provider credential is configured, so the spawned subsession cannot ' +
- 'author a reply. Set ANTHROPIC_API_KEY (preferred) or OPENAI_API_KEY on the service.',
+ 'author a reply. Set CLAUDE_CODE_USE_VERTEX=1 (preferred: no key to rotate), ANTHROPIC_API_KEY, ' +
+ 'or OPENAI_API_KEY on the service.',
+ needsHumanLogin: false,
+ };
+ }
+
+ // A configured credential is not a working one. A key the provider refuses
+ // looks identical from inside this process to a key that works, right up until
+ // a buyer's message arrives and the subsession cannot author a reply, which is
+ // the silent failure the whole worker exists to make loud.
+ if (providerProbe?.code === 'unauthorized') {
+ return {
+ status: 'degraded',
+ ready: false,
+ reason: 'ai_provider_unauthorized',
+ detail:
+ `chat is delivered but the AI provider refuses this host's credential, so no reply can be authored: ` +
+ `${providerProbe.detail}`,
needsHumanLogin: false,
};
}