Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// okx-chat-bot: always-on host for the OKX.AI marketplace chat bot (agent #2632).
//
// Marketplace chat for our listing is delivered over XMTP to a local `okx-a2a`
// daemon backed by an `onchainos` wallet session. Both used to live on a
// developer codespace, which cannot stay up: a rebuild wiped the CLIs and an
// idle nap killed the daemon (observed alive 21:09, dead by 03:13 the same
// night). OKX's chat test then reports "no delivery in 30 min" and the listing
// gets flagged offline. This worker is the durable host.
//
// What it owns:
// 1. State restore the wallet/XMTP identity from GCS, snapshot it back
// 2. Workspace rebuild the AI subsession's briefing + skills from the image
// 3. Daemon supervise `okx-a2a run` with capped backoff restarts
// 4. Health /readyz strict readiness, every other path liveness
// 5. Heartbeat bot_heartbeat row → /api/healthz subsystems → gcp-triage
// 6. Alerts a logged-out session pages a human WITH the exact commands
//
// Run: node workers/okx-chat-bot (npm run worker:okx-bot)
// Deploy: workers/okx-chat-bot/cloudbuild.yaml. See the README for the one-time
// bucket/secret setup and why --max-instances=1 is load-bearing.
import { mkdir } from 'node:fs/promises';
import { sendOpsAlert } from '../../api/_lib/alerts.js';
import { sql } from '../../api/_lib/db.js';
import { applyLane, loadConfig, paths } from './config.js';
import { agentRefresh, beginLogin, daemonStatus, exec, walletStatus } from './cli.js';
import { startHealthServer } from './health-server.js';
import { electProvider, loginCodex } from './provider.js';
import { classify, loginInstructions } from './session.js';
import { restoreState, snapshotState } from './state.js';
import { createSupervisor } from './supervisor.js';
import { createLease, sqlLeaseStore } from './lease.js';
import { buildWorkspace } from './workspace.js';
import { log } from './log.js';
const WORKER = 'okx-chat-bot';
const BOOT_AT = new Date().toISOString();
/**
* The single-writer lease (lease.js), or null on a host with no shared state.
* Held from before the state restore until after the final snapshot, so no two
* hosts ever restore, run or snapshot the bot's identity at the same time.
*/
let lease = null;
/** May this host write the shared state right now? */
const mayWrite = () => !lease || lease.fresh();
/** Live health, refreshed by the session probe and read by the HTTP handlers. */
const live = {
verdict: { status: 'unknown', ready: false, reason: 'booting', detail: 'probe has not run yet', needsHumanLogin: false },
checkedAt: 0,
daemon: 'unknown',
wallet: null,
agents: { agentCount: 0, activeClients: 0 },
login: null,
workspace: null,
stateRestore: null,
// What the provider's own API last answered. Null until the first probe lands,
// which is why classify() treats a missing verdict as "no opinion" rather than
// as a failure: a host must not read red for a probe that has not run yet.
providerProbe: null,
// What EVERY lane in the chain last answered. The elected lane's verdict is
// `providerProbe`; this is the whole picture, so a human reading /readyz can
// see which credential to fund rather than only that one of them was refused.
providerLanes: [],
// The single-writer lease's state (lease.js), null on a host without one.
lease: null,
};
async function heartbeat(cfg, supervisor) {
const meta = {
bootAt: BOOT_AT,
agentId: cfg.agentId,
host: cfg.host,
hostDurable: cfg.hostDurable,
health: live.verdict.status,
ready: live.verdict.ready,
reason: live.verdict.reason,
detail: live.verdict.detail,
needsHumanLogin: live.verdict.needsHumanLogin,
loggedIn: live.wallet?.loggedIn ?? null,
agentCount: live.agents.agentCount,
activeClients: live.agents.activeClients,
provider: cfg.provider,
providerTransport: cfg.providerTransport,
providerCredentialed: cfg.providerCredentialed,
providerLane: cfg.activeLane?.id ?? null,
providerVerdict: live.providerProbe?.code ?? null,
providerChain: live.providerLanes.map((l) => ({ lane: l.lane, code: l.code })),
daemonRestarts: supervisor.stats().restarts,
checkedAt: live.checkedAt,
// Marks this host as lease-aware. A successor treats a beat WITHOUT this
// field as a host that predates the lease and waits it out (lease.js).
leaseHolder: lease?.state.holder ?? null,
};
try {
await sql`
INSERT INTO bot_heartbeat (worker, mode, last_beat_at, meta)
VALUES (${WORKER}, ${cfg.provider}, now(), ${JSON.stringify(meta)}::jsonb)
ON CONFLICT (worker) DO UPDATE
SET mode = excluded.mode, last_beat_at = excluded.last_beat_at, meta = excluded.meta
`;
} catch (err) {
log.warn('heartbeat write failed', { err: err?.message });
}
}
// A session expiry is the one failure only a human can clear, so it must page
// loudly and carry the commands. Alert on the transition into the bad state,
// then at most every cfg.alertRepeatMs while it persists, so one overnight
// expiry does not become a hundred notifications.
let lastAlertReason = null;
let lastAlertAt = 0;
async function maybeAlert(cfg, verdict) {
if (verdict.status === 'ok' || verdict.status === 'unknown') {
if (lastAlertReason) {
lastAlertReason = null;
await sendOpsAlert('OKX chat bot recovered', `agent #${cfg.agentId} is reachable again: ${verdict.detail}`, {
severity: 'info',
}).catch(() => {});
}
return;
}
const now = Date.now();
const changed = verdict.reason !== lastAlertReason;
if (!changed && now - lastAlertAt < cfg.alertRepeatMs) return;
lastAlertReason = verdict.reason;
lastAlertAt = now;
const lines = [`agent #${cfg.agentId}: ${verdict.detail}`];
if (verdict.needsHumanLogin) lines.push('', 'Fix (needs a human):', ...loginInstructions(live.login?.loginUrl, live.login?.authSessionId));
await sendOpsAlert(`OKX chat bot ${verdict.status}: ${verdict.reason}`, lines.join('\n'), {
severity: verdict.status === 'down' ? 'critical' : 'warn',
}).catch((err) => log.warn('ops alert failed', { err: err?.message }));
}
// Ask every AI lane whether it still serves this host, and move to the best one.
//
// Kept off the session probe's path: these are network calls to third parties,
// and a slow one must not delay the verdict about whether XMTP is delivering. A
// transient `unreachable` answer is remembered as-is, because classify()
// deliberately does not fail readiness on it.
//
// Election is what makes a dead credential survivable. The host used to hold one
// lane, so the Vertex billing hold on 2026-09-04 left it unable to author a
// single reply even though other credentials were configured. Now the lane that
// answers is the lane that serves, and a lane that recovers is picked up on the
// next tick with no deploy.
async function electProviderLane(cfg, supervisor) {
const before = live.providerProbe?.code ?? null;
const beforeLane = cfg.activeLane?.id ?? null;
const { elected, verdict, lanes } = await electProvider(cfg.chain);
live.providerLanes = lanes;
if (verdict) live.providerProbe = verdict;
if (elected && elected.id !== beforeLane) {
applyLane(cfg, elected);
log.info('provider lane elected', { lane: elected.id, transport: elected.transport, from: beforeLane });
// The adapter reads the provider out of ~/.okx-agent-task/config.toml, and
// the spawned CLI reads its credentials out of the daemon's environment.
// Both are set at spawn time, so a lane change is only real once the daemon
// has been replaced with one that carries it.
await exec(cfg, 'okx-a2a', ['config', 'provider', '--provider', cfg.provider], { timeoutMs: 30_000 });
const codexLogin = await loginCodex(cfg);
if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);
await supervisor.restart(`ai lane ${beforeLane || 'none'} -> ${elected.id}`);
}
if (live.providerProbe && live.providerProbe.code !== before) {
log[live.providerProbe.code === 'unauthorized' ? 'error' : 'info']('provider credential', {
provider: cfg.provider,
transport: cfg.providerTransport,
lane: live.providerProbe.lane,
code: live.providerProbe.code,
detail: live.providerProbe.detail,
lanes: lanes.map((l) => `${l.lane}=${l.code}`).join(' '),
});
}
}
// One probe at a time. A probe can legitimately outrun its own interval (the
// three CLI calls are bounded at 15s + 30s + 90s), and two in flight would race
// on the login mint below: both would see `live.login` empty and start a second
// auth session, invalidating the URL a human is halfway through using.
let probing = false;
async function probeSession(cfg, supervisor) {
if (probing) {
log.warn('probe still running, skipping this tick', { sinceMs: Date.now() - live.checkedAt });
return;
}
probing = true;
try {
await runProbe(cfg, supervisor);
} finally {
probing = false;
}
}
async function runProbe(cfg, supervisor) {
const daemon = await daemonStatus(cfg);
const wallet = daemon.startsWith('running') ? await walletStatus(cfg) : null;
const agents = wallet?.loggedIn ? await agentRefresh(cfg) : { agentCount: 0, activeClients: 0 };
const verdict = classify({
daemon,
wallet,
agents,
providerCredentialed: cfg.providerCredentialed,
providerProbe: live.providerProbe,
daemonChild: supervisor.stats(),
});
// Mint a fresh login URL only when one is actually needed, and only when we
// do not already hold an unused one: `login --phase init` starts a new auth
// session server-side, so calling it every probe would invalidate the URL a
// human is halfway through using.
if (verdict.needsHumanLogin && !live.login) {
live.login = await beginLogin(cfg);
} else if (!verdict.needsHumanLogin) {
live.login = null;
}
const changed = verdict.reason !== live.verdict.reason;
live.daemon = daemon;
live.wallet = wallet;
live.agents = agents;
live.verdict = verdict;
live.checkedAt = Date.now();
if (changed) {
log[verdict.status === 'ok' ? 'info' : 'warn']('health changed', {
status: verdict.status,
reason: verdict.reason,
detail: verdict.detail,
});
}
await maybeAlert(cfg, verdict);
await heartbeat(cfg, supervisor);
}
async function main() {
const cfg = loadConfig();
const p = paths(cfg);
log.info('boot', {
home: cfg.home,
agentId: cfg.agentId,
host: cfg.host,
hostDurable: cfg.hostDurable,
provider: cfg.provider,
providerReason: cfg.providerReason,
chain: cfg.chain.map((l) => l.id).join(' > ') || 'none',
stateBucket: cfg.stateBucket || null,
});
if (!cfg.providerCredentialed) {
log.error('no AI-provider credential: the daemon will receive chat but cannot author replies', {
provider: cfg.provider,
needs: 'CLAUDE_CODE_USE_VERTEX=1 (preferred), ANTHROPIC_API_KEY, or OPENAI_API_KEY',
});
}
// The health server comes up first, before the lease wait. Cloud Run's
// startup probe has to pass for a rollout to move traffic and SIGTERM the old
// instance, and the old instance releasing the lease is exactly what this one
// is waiting for: holding the probe behind the lease would deadlock the deploy.
const supervisor = createSupervisor(cfg, p);
const server = startHealthServer(cfg, live, supervisor, BOOT_AT);
// A SIGTERM that lands while waiting has nothing to save: this host never
// restored, ran or snapshotted anything.
const waitAbort = new AbortController();
const onEarlySignal = (signal) => {
log.info('shutdown before taking the lease', { signal });
waitAbort.abort();
server?.close();
process.exit(0);
};
process.once('SIGTERM', onEarlySignal);
process.once('SIGINT', onEarlySignal);
if (cfg.leaseRequired) {
lease = createLease({
store: sqlLeaseStore(sql),
host: cfg.host,
ttlMs: cfg.leaseTtlMs,
renewMs: cfg.leaseRenewMs,
pollMs: cfg.leasePollMs,
legacyStaleMs: cfg.legacyBeatStaleMs,
onLost: (reason) => void fence(cfg, supervisor, reason),
});
live.lease = lease.state;
live.verdict = {
status: 'unknown',
ready: false,
reason: 'lease_wait',
detail: 'waiting for the single-writer lease before restoring the bot identity',
needsHumanLogin: false,
};
const got = await lease.acquire({
signal: waitAbort.signal,
onWait: (reason) => {
live.verdict = { ...live.verdict, detail: `waiting for the single-writer lease: ${reason}` };
},
});
if (!got) return;
lease.start();
live.verdict = { status: 'unknown', ready: false, reason: 'booting', detail: 'lease held, restoring state', needsHumanLogin: false };
} else {
log.warn('no state bucket: running without the single-writer lease (fine for a local run, never for Cloud Run)');
}
process.off('SIGTERM', onEarlySignal);
process.off('SIGINT', onEarlySignal);
await mkdir(p.agentTask, { recursive: true });
live.stateRestore = await restoreState(cfg);
await mkdir(p.logs, { recursive: true });
live.workspace = await buildWorkspace(cfg, p);
// Provider + permission preset are persisted in ~/.okx-agent-task/config.toml,
// which the restored snapshot may carry from an older posture. Re-assert both
// so the deployed configuration always wins over the snapshot. Bypass matters:
// without it the subsession stalls on a tool-approval prompt nobody is there to
// answer, which reads to the buyer as an unresponsive bot.
await exec(cfg, 'okx-a2a', ['config', 'provider', '--provider', cfg.provider], { timeoutMs: 30_000 });
await exec(cfg, 'okx-a2a', ['config', 'permissions', '--preset', 'bypass'], { timeoutMs: 30_000 });
// Codex reads ~/.codex/auth.json, never OPENAI_API_KEY from the environment,
// so without this the key is present, the CLI spawns, and every reply dies on
// a 401 the buyer only experiences as silence. No-op for every other provider.
const codexLogin = await loginCodex(cfg);
if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);
supervisor.start();
const probe = () => probeSession(cfg, supervisor).catch((err) => log.error('probe failed', { err: err?.message }));
const probeTimer = setInterval(probe, cfg.sessionProbeMs);
// The heartbeat runs on its own timer rather than only at the end of a probe.
// A probe can outlast the freshness window /api/healthz judges this host by,
// so tying the beat to it would let one slow CLI call read as "the host is
// gone" while the bot is perfectly online.
const heartbeatTimer = setInterval(
() => heartbeat(cfg, supervisor).catch((err) => log.warn('heartbeat failed', { err: err?.message })),
cfg.heartbeatMs,
);
const snapshotTimer = setInterval(() => {
if (!mayWrite()) return log.warn('snapshot skipped: the single-writer lease is not fresh', {});
snapshotState(cfg, { reason: 'timer' }).catch(() => {});
}, cfg.snapshotMs);
const providerProbeTimer = setInterval(
() => electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message })),
cfg.providerProbeMs,
);
// Run the first election immediately, but off the boot path: a slow provider
// must delay the verdict, never the daemon that receives the chat.
electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message }));
// A first verdict early enough to be useful. It can legitimately land before
// the daemon has claimed its lock, which classify() reports as
// `daemon_starting` rather than `daemon_down`, so an ordinary boot no longer
// pages critical and recovers a minute later.
setTimeout(probe, 10_000);
let draining = false;
const shutdown = async (signal) => {
if (draining) return;
draining = true;
log.info('shutdown', { signal });
clearInterval(probeTimer);
clearInterval(heartbeatTimer);
clearInterval(snapshotTimer);
clearInterval(providerProbeTimer);
server?.close();
// Order matters: stop the daemon FIRST so its sqlite files are quiesced,
// then snapshot. A live-copy snapshot can tear; this one cannot. The
// snapshot is written only while the lease is still ours, and the lease is
// released only after it, so the successor restores exactly this state.
await supervisor.stop();
if (mayWrite()) await snapshotState(cfg, { reason: signal });
else log.error('final snapshot skipped: the single-writer lease is not held', { signal });
await lease?.release();
log.info('bye', {});
process.exit(0);
};
process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('unhandledRejection', (err) => log.error('unhandledRejection', { err: err?.message }));
}
/**
* The lease is gone (another host took it, or it could not be renewed before it
* would expire). Another host may already be writing the identity, so this one
* stops its daemon and exits WITHOUT a snapshot: writing one now is exactly the
* clobber the lease exists to prevent. Cloud Run restarts the container, and the
* fresh process waits for the lease like any other.
*/
let fencing = false;
async function fence(cfg, supervisor, reason) {
if (fencing) return;
fencing = true;
live.verdict = { status: 'down', ready: false, reason: 'lease_lost', detail: `single-writer lease lost: ${reason}`, needsHumanLogin: false };
await sendOpsAlert(
'OKX chat bot fenced itself',
`agent #${cfg.agentId} on ${cfg.host} lost the single-writer lease (${reason}) and stopped its daemon without snapshotting. It restarts and waits for the lease; no action is needed unless this repeats.`,
{ severity: 'warn' },
).catch(() => {});
await supervisor.stop(5_000).catch(() => {});
process.exit(75);
}
main().catch((err) => {
log.error('fatal', { err: err?.message, stack: err?.stack });
process.exit(1);
});