Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(okx-bot): hold a single-writer lease across rollouts and ship the worker with one command

nirholasa71cc4425420
diff --git a/package.json b/package.json
index 34b753fd0..b3a72f259 100644
--- a/package.json
+++ b/package.json
@@ -453,6 +453,8 @@
"typecheck": "tsc -p jsconfig.json",
"okx:bot": "node scripts/okx-bot-revive.mjs",
"okx:bot:seed-state": "node scripts/okx-bot-seed-state.mjs",
+ "okx:bot:gateway": "node scripts/okx-bot-llm-gateway.mjs",
+ "okx:bot:deploy": "node scripts/okx-bot-deploy.mjs",
"okx:three-copy": "node scripts/okx-three-copy-check.mjs",
"okx:gauntlet": "node scripts/okx-e2e-gauntlet.mjs",
"check:conflicts": "node scripts/check-merge-conflicts.mjs",
diff --git a/scripts/okx-bot-deploy.mjs b/scripts/okx-bot-deploy.mjs
new file mode 100644
index 000000000..a5afc8cd3
--- /dev/null
+++ b/scripts/okx-bot-deploy.mjs
@@ -0,0 +1,211 @@
+#!/usr/bin/env node
+// Ship the OKX marketplace chat bot (workers/okx-chat-bot) in one command.
+//
+// npm run okx:bot:deploy # preflight, provision, build, submit, verify
+// npm run okx:bot:deploy -- --dry-run # preflight and provisioning plan only: writes nothing, submits nothing
+// npm run okx:bot:deploy -- --allow-dirty # submit the working tree instead of a clean worktree of HEAD
+//
+// What "one command" has to cover, in order:
+//
+// 1. gcloud is authenticated (every later step needs it).
+// 2. The worker is built from a CLEAN worktree of HEAD, never from the shared
+// working tree: the image copies all of api/, and other agents' in-flight
+// edits there would otherwise ship inside the bot. A dirty
+// workers/okx-chat-bot/ is refused, because that is uncommitted bot code the
+// clean build would silently leave out.
+// 3. The payment-free AI lane exists: scripts/okx-bot-llm-gateway.mjs --apply
+// (idempotent) provisions the metering agent, the service-account key and
+// the Secret Manager secret, then proves the lane with the bot's own probe.
+// Its agent id becomes the build's _GATEWAY_AGENT.
+// 4. Whether the live API already serves the agent-scoped SDK path the lane
+// calls. It is not a blocker: the lane is elected on a live probe every 15
+// minutes, so if the API deploy lands second the bot moves onto the lane by
+// itself. It is reported so nobody reads a Vertex-only bot as a failure.
+// 5. Submit workers/okx-chat-bot/cloudbuild.yaml.
+// 6. Verify on the platform's own health view (no gcloud, no secret): the
+// okx_chat_bot subsystem must report the NEW revision as its host. The new
+// instance first waits for the single-writer lease (workers/okx-chat-bot/
+// lease.js) while the old one drains, so this poll is expected to take a
+// couple of minutes, not seconds.
+//
+// Owner-gated: this deploys to production. Run it only on the owner's say-so.
+
+import { execFile, spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { promisify } from 'node:util';
+
+const run = promisify(execFile);
+
+const args = process.argv.slice(2);
+const DRY = args.includes('--dry-run');
+const ALLOW_DIRTY = args.includes('--allow-dirty');
+const PROJECT = 'aerial-vehicle-466722-p5';
+const REGION = 'us-central1';
+const SERVICE = 'okx-chat-bot';
+const ROOT = resolve(new URL('..', import.meta.url).pathname);
+const WORKTREE = resolve(ROOT, '..', '.okx-bot-deploy-wt');
+const HEALTHZ = 'https://three.ws/api/healthz';
+const VERSION = 'https://three.ws/api/version';
+// The commit that taught the proxy the SDK path and the Claude Code request
+// shape (system turns inside messages). The live API needs it for the lane.
+const SDK_PATH_COMMIT = 'd3074c1c8';
+const VERIFY_TIMEOUT_MS = 15 * 60_000;
+
+const say = (...m) => console.log(...m);
+const step = (n, title) => say(`\n[${n}] ${title}`);
+const die = (msg, code = 1) => {
+ console.error(`\n FAILED: ${msg}\n`);
+ process.exit(code);
+};
+
+const git = (argv, opts = {}) => run('git', argv, { cwd: ROOT, maxBuffer: 16 * 1024 * 1024, ...opts });
+
+function stream(cmd, argv, opts = {}) {
+ return new Promise((res, rej) => {
+ const child = spawn(cmd, argv, { stdio: ['ignore', 'pipe', 'inherit'], ...opts });
+ let out = '';
+ child.stdout.on('data', (d) => {
+ out += d;
+ process.stdout.write(d);
+ });
+ child.on('error', rej);
+ child.on('close', (code) => (code === 0 ? res(out) : rej(Object.assign(new Error(`${cmd} exited ${code}`), { code, out }))));
+ });
+}
+
+async function okxSubsystem() {
+ const res = await fetch(HEALTHZ, { signal: AbortSignal.timeout(20_000) });
+ const body = await res.json();
+ return (body?.subsystems?.subsystems || []).find((s) => s.name === 'okx_chat_bot') || null;
+}
+
+async function liveApiHasSdkPath() {
+ const v = await fetch(VERSION, { signal: AbortSignal.timeout(20_000) }).then((r) => r.json());
+ const live = v?.commit;
+ if (!live) return { ok: false, detail: '/api/version reported no commit' };
+ try {
+ await git(['merge-base', '--is-ancestor', SDK_PATH_COMMIT, live]);
+ return { ok: true, detail: `live API ${v.commitShort} (${v.runtime?.revision}) contains ${SDK_PATH_COMMIT}` };
+ } catch {
+ return { ok: false, detail: `live API ${v.commitShort} (${v.runtime?.revision}) predates ${SDK_PATH_COMMIT}` };
+ }
+}
+
+async function main() {
+ step(1, 'gcloud authentication');
+ try {
+ await run('gcloud', ['auth', 'print-access-token'], { timeout: 30_000 });
+ say(' ok');
+ } catch {
+ const msg = 'gcloud is not authenticated here: run `gcloud auth login` (and `gcloud auth application-default login`), then re-run';
+ if (!DRY) die(msg);
+ say(` WARNING: ${msg}`);
+ }
+
+ step(2, 'build source');
+ const { stdout: dirty } = await git(['status', '--porcelain', '--', 'workers/okx-chat-bot', 'scripts/okx-bot-llm-gateway.mjs', 'scripts/okx-bot-deploy.mjs']);
+ let context = WORKTREE;
+ if (ALLOW_DIRTY) {
+ context = ROOT;
+ say(' --allow-dirty: submitting the working tree as it is, including any uncommitted edits under api/');
+ } else if (dirty.trim() && DRY) {
+ say(` WARNING: uncommitted bot code; a real run refuses until it is committed:\n${dirty}`);
+ } else if (dirty.trim()) {
+ die(
+ `uncommitted bot code would be left out of a clean build:\n${dirty}\n` +
+ ' Commit it (the owner approves this content), or pass --allow-dirty to ship the working tree.',
+ );
+ } else {
+ const { stdout: head } = await git(['rev-parse', '--short', 'HEAD']);
+ say(` clean worktree of HEAD ${head.trim()} at ${WORKTREE}`);
+ }
+
+ step(3, 'AI gateway lane (metering agent, key, secret, live probe)');
+ const gw = await stream(process.execPath, ['scripts/okx-bot-llm-gateway.mjs', DRY ? '--verify' : '--apply'], { cwd: ROOT }).catch((err) => {
+ // Exit 2 is "provisioned, but the probe did not pass yet" (the SDK path is
+ // not live): the deploy still proceeds, see step 4.
+ if (err.code === 2) return err.out;
+ if (DRY) return err.out || '';
+ die('provisioning the gateway lane failed (output above)');
+ });
+ const agentMatch = /OKX_BOT_ANTHROPIC_BASE_URL=\S+\/agents\/([0-9a-f-]{36})/.exec(gw || '');
+ if (!agentMatch && !DRY) die('the gateway script did not report a metering agent id');
+ const agentId = agentMatch?.[1] || '<provisioned by --apply>';
+
+ step(4, 'live API serves the agent-scoped SDK path');
+ const api = await liveApiHasSdkPath();
+ say(` ${api.ok ? 'yes' : 'NOT YET'}: ${api.detail}`);
+ if (!api.ok) {
+ say(' The bot still deploys and keeps serving on its current lane. Deploy the API (`npm run deploy:gcp:full`)');
+ say(' and the next 15-minute election moves it onto the gateway lane with no second worker deploy.');
+ }
+
+ const substitutions = `SHORT_SHA=manual${Math.floor(Date.now() / 1000)},_GATEWAY_AGENT=${agentId}`;
+ const submitArgs = [
+ 'builds',
+ 'submit',
+ '--config',
+ 'workers/okx-chat-bot/cloudbuild.yaml',
+ '--region',
+ REGION,
+ '--project',
+ PROJECT,
+ `--substitutions=${substitutions}`,
+ '.',
+ ];
+ if (DRY) {
+ say(`\n dry run: would run in ${context}:\n gcloud ${submitArgs.join(' ')}\n`);
+ return;
+ }
+
+ const before = await okxSubsystem().catch(() => null);
+ say(`\n serving now: ${before?.host || 'unknown'} (${before?.status || 'unknown'})`);
+
+ step(5, 'build and deploy');
+ if (context === WORKTREE) {
+ if (existsSync(WORKTREE)) await git(['worktree', 'remove', '--force', WORKTREE]);
+ await git(['worktree', 'add', '--detach', WORKTREE, 'HEAD']);
+ }
+ try {
+ await stream('gcloud', submitArgs, { cwd: context });
+ } catch {
+ die('Cloud Build failed (log above). The serving revision is untouched: a failed deploy never moves traffic.');
+ } finally {
+ if (context === WORKTREE) await git(['worktree', 'remove', '--force', WORKTREE]).catch(() => {});
+ }
+
+ step(6, 'verify the rollout on /api/healthz');
+ const { stdout: rev } = await run(
+ 'gcloud',
+ ['run', 'services', 'describe', SERVICE, '--region', REGION, '--project', PROJECT, '--format=value(status.latestReadyRevisionName)'],
+ { timeout: 60_000 },
+ );
+ const revision = rev.trim();
+ say(` new revision ${revision}; waiting for it to take the single-writer lease and beat`);
+ const deadline = Date.now() + VERIFY_TIMEOUT_MS;
+ let last = '';
+ while (Date.now() < deadline) {
+ const s = await okxSubsystem().catch(() => null);
+ const line = s ? `${s.host} | ${s.status} | ${s.detail}` : 'healthz unreachable';
+ if (line !== last) {
+ say(` ${new Date().toISOString().slice(11, 19)} ${line}`);
+ last = line;
+ }
+ if (s?.host?.includes(revision) && (s.status === 'ok' || s.status === 'degraded')) {
+ say(`\n DEPLOYED: ${revision} holds the lease and is beating (${s.status}).`);
+ if (s.status !== 'ok') say(` Not fully green yet: ${s.detail}\n ${s.hint || ''}`);
+ say(' Lane detail: curl -s -H "Authorization: Bearer $(gcloud auth print-identity-token)" \\');
+ say(' https://okx-chat-bot-lp642k3kpa-uc.a.run.app/readyz | jq \'{lane: .provider.lane, chain: .provider.chain, lease: .lease}\'\n');
+ return;
+ }
+ await new Promise((r) => setTimeout(r, 15_000));
+ }
+ die(
+ `${revision} did not report in within ${VERIFY_TIMEOUT_MS / 60_000} minutes. Read its /readyz (lease.waitingOn names what it waits for) ` +
+ `and: gcloud logging read 'resource.labels.service_name="${SERVICE}"' --freshness=30m --limit 100`,
+ 3,
+ );
+}
+
+main().catch((err) => die(err?.message || String(err)));
diff --git a/scripts/okx-bot-llm-gateway.mjs b/scripts/okx-bot-llm-gateway.mjs
index 03adfadea..9e7f571de 100644
--- a/scripts/okx-bot-llm-gateway.mjs
+++ b/scripts/okx-bot-llm-gateway.mjs
@@ -161,8 +161,12 @@ async function activeKeys(sql, userId) {
async function secretState() {
try {
await gcloud(['secrets', 'describe', SECRET, '--format=value(name)']);
- } catch {
- return { exists: false, token: null };
+ } catch (err) {
+ // Only a NOT_FOUND means the secret is missing. Anything else (an expired
+ // gcloud login above all) means nobody could look, and reporting that as
+ // "missing" would send --apply off to create a secret that already exists.
+ if (/NOT_FOUND|not found/i.test(err?.stderr || err?.message || '')) return { exists: false, token: null };
+ return { exists: null, token: null, error: String(err?.message || err).split('\n')[0].slice(0, 200) };
}
try {
const { stdout } = await gcloud(['secrets', 'versions', 'access', 'latest', '--secret', SECRET]);
@@ -343,7 +347,8 @@ async function main() {
say(` metering agent ${agent ? `${agent.id} "${agent.name}"` : 'missing'}`);
say(` model ${MODEL}`);
say(` api keys ${keys.length} active`);
- say(` secret ${SECRET}: ${!secret.exists ? 'missing' : secretKey ? `holds active key ${secretKey.prefix}...` : 'holds no active key'}`);
+ say(` secret ${SECRET}: ${secret.exists === null ? `unknown (${secret.error})` : !secret.exists ? 'missing' : secretKey ? `holds active key ${secretKey.prefix}...` : 'holds no active key'}`);
+ if (secret.exists === null && (APPLY || VERIFY_ONLY)) die('gcloud cannot read Secret Manager here: run `gcloud auth login`, then re-run');
if (!APPLY && !VERIFY_ONLY) {
say('\n plan only: nothing was written. Re-run with --apply to provision, or --verify to prove an existing lane.\n');
diff --git a/tests/okx-chat-bot.test.js b/tests/okx-chat-bot.test.js
index 693c9c30f..064e10e0d 100644
--- a/tests/okx-chat-bot.test.js
+++ b/tests/okx-chat-bot.test.js
@@ -899,3 +899,208 @@ describe('okx_chat_bot subsystem identity fields', () => {
expect(s.hostDurable).toBeNull();
});
});
+
+// ── single-writer lease ─────────────────────────────────────────────────────
+//
+// The bot identity must never have two writers, and Cloud Run's max-instances is
+// per revision, so a rollout overlaps the old and new instance. These pin the
+// lease that sequences that overlap. The store below reproduces the SQL
+// contract of sqlLeaseStore (conditional upsert on holder / released / expiry,
+// all against one clock) so the state machine is exercised against the same
+// semantics the database enforces.
+
+import { createLease, legacyWriterBlocks, sqlLeaseStore } from '../workers/okx-chat-bot/lease.js';
+
+function memoryLeaseStore(clock) {
+ let row = null;
+ let beat = null;
+ return {
+ setBeat(b) {
+ beat = b;
+ },
+ row: () => row,
+ async acquire(holder, ttlSec, meta) {
+ const free = !row || row.holder === holder || row.released || clock.now - row.at > ttlSec * 1000;
+ if (free) {
+ row = { ...meta, holder, at: clock.now, released: false };
+ return { acquired: true };
+ }
+ return { acquired: false, holder: row.holder, host: row.host, ageSec: (clock.now - row.at) / 1000 };
+ },
+ async renew(holder) {
+ if (!row || row.holder !== holder || row.released) return false;
+ row.at = clock.now;
+ return true;
+ },
+ async release(holder) {
+ if (row && row.holder === holder) row = { ...row, released: true, at: clock.now };
+ },
+ async hostBeat() {
+ return beat ? { ...beat, ageSec: (clock.now - beat.at) / 1000 } : null;
+ },
+ };
+}
+
+function leaseFor(store, clock, host, extra = {}) {
+ return createLease({
+ store,
+ host,
+ holder: `${host}#test`,
+ ttlMs: 120_000,
+ renewMs: 20_000,
+ pollMs: 5_000,
+ legacyStaleMs: 90_000,
+ now: () => clock.now,
+ // Waiting advances the shared clock, so a blocked acquire() observes time
+ // passing exactly as a real poll loop would.
+ sleep: async (ms) => {
+ clock.now += ms;
+ clock.onTick?.();
+ },
+ ...extra,
+ });
+}
+
+describe('okx-chat-bot single-writer lease', () => {
+ it('lets the first host take a free lease immediately', async () => {
+ const clock = { now: 1_000_000 };
+ const store = memoryLeaseStore(clock);
+ const a = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-2)');
+ expect(await a.acquire()).toBe(true);
+ expect(a.state.held).toBe(true);
+ expect(a.fresh()).toBe(true);
+ });
+
+ it('makes a rollout wait for the old instance, then hands over on release', async () => {
+ const clock = { now: 1_000_000 };
+ const store = memoryLeaseStore(clock);
+ const old = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-1)');
+ await old.acquire();
+ const next = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-2)');
+ const waits = [];
+ // The old instance keeps renewing for 60s, then shuts down and releases.
+ clock.onTick = () => {
+ if (clock.now - 1_000_000 < 60_000) void old.renewOnce();
+ else if (old.state.held) void old.release();
+ };
+ expect(await next.acquire({ onWait: (r) => waits.push(r) })).toBe(true);
+ expect(waits[0]).toMatch(/rev-1\) holds the lease/);
+ // Handed over on the release, well before the 120s TTL would have lapsed.
+ expect(clock.now - 1_000_000).toBeLessThan(120_000);
+ expect(store.row().holder).toBe('cloudrun:okx-chat-bot (rev-2)#test');
+ });
+
+ it('waits out the TTL for a host that died holding the lease', async () => {
+ const clock = { now: 1_000_000 };
+ const store = memoryLeaseStore(clock);
+ await leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-1)').acquire();
+ const next = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-2)');
+ expect(await next.acquire()).toBe(true);
+ expect(clock.now - 1_000_000).toBeGreaterThan(120_000);
+ });
+
+ it('never lets two hosts hold it at once', async () => {
+ const clock = { now: 1_000_000 };
+ const store = memoryLeaseStore(clock);
+ const a = leaseFor(store, clock, 'host-a');
+ const b = leaseFor(store, clock, 'host-b');
+ await a.acquire();
+ const ctl = new AbortController();
+ clock.onTick = () => {
+ void a.renewOnce();
+ if (clock.now - 1_000_000 > 600_000) ctl.abort();
+ };
+ expect(await b.acquire({ signal: ctl.signal })).toBe(false);
+ expect(a.state.held).toBe(true);
+ expect(b.state.held).toBe(false);
+ });
+
+ it('waits for a host that predates the lease while its heartbeat is fresh', async () => {
+ const clock = { now: 1_000_000 };
+ const store = memoryLeaseStore(clock);
+ // The live revision today: beats every 30s, carries no leaseHolder.
+ store.setBeat({ host: 'cloudrun:okx-chat-bot (okx-chat-bot-00001-926)', leaseAware: false, at: clock.now });
+ const next = leaseFor(store, clock, 'cloudrun:okx-chat-bot (okx-chat-bot-00002-abc)');
+ const waits = [];
+ // It keeps beating for 45s, then Cloud Run stops it.
+ clock.onTick = () => {
+ if (clock.now - 1_000_000 < 45_000) store.setBeat({ host: 'cloudrun:okx-chat-bot (okx-chat-bot-00001-926)', leaseAware: false, at: clock.now });
+ };
+ expect(await next.acquire({ onWait: (r) => waits.push(r) })).toBe(true);
+ expect(waits[0]).toMatch(/00001-926\) predates the single-writer lease/);
+ // Its last beat landed at 40s; the lease is taken only once that is 90s stale.
+ expect(clock.now - 1_000_000).toBe(40_000 + 90_000);
+ });
+
+ it('ignores a heartbeat that cannot be a second writer', () => {
+ const opts = { selfHost: 'cloudrun:okx-chat-bot (rev-2)', staleSec: 90 };
+ expect(legacyWriterBlocks(null, opts)).toBeNull();
+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-1)', leaseAware: true, ageSec: 5 }, opts)).toBeNull();
+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-2)', leaseAware: false, ageSec: 5 }, opts)).toBeNull();
+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-1)', leaseAware: false, ageSec: 91 }, opts)).toBeNull();
+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-1)', leaseAware: false, ageSec: 30 }, opts)).toMatch(/predates/);
+ });
+
+ it('reports a lost lease once and stops calling itself fresh', async () => {
+ const clock = { now: 1_000_000 };
+ const store = memoryLeaseStore(clock);
+ const lost = [];
+ const a = leaseFor(store, clock, 'host-a', { onLost: (r) => lost.push(r) });
+ await a.acquire();
+ // Another host took it (the TTL lapsed while this one was frozen).
+ clock.now += 130_000;
+ await leaseFor(store, clock, 'host-b').acquire();
+ await a.renewOnce();
+ await a.renewOnce();
+ expect(lost).toEqual(['another host holds the lease']);
+ expect(a.fresh()).toBe(false);
+ });
+
+ it('fences itself before an unrenewable lease can expire under it', async () => {
+ const clock = { now: 1_000_000 };
+ const lost = [];
+ const store = memoryLeaseStore(clock);
+ const a = leaseFor(store, clock, 'host-a', { onLost: (r) => lost.push(r) });
+ await a.acquire();
+ store.renew = async () => {
+ throw new Error('database unreachable');
+ };
+ clock.now += 60_000;
+ await a.renewOnce();
+ expect(lost).toEqual([]);
+ // Not fresh any more once the fencing margin (ttl - renew) has passed.
+ clock.now += 40_000;
+ expect(a.fresh()).toBe(false);
+ await a.renewOnce();
+ expect(lost).toHaveLength(1);
+ expect(lost[0]).toMatch(/could not renew for 100s/);
+ });
+
+ it('refuses a renewal cadence that could let the lease lapse between renewals', () => {
+ const clock = { now: 0 };
+ expect(() => leaseFor(memoryLeaseStore(clock), clock, 'h', { renewMs: 70_000 })).toThrow(/under half/);
+ });
+
+ it('drives the lease with conditional statements against bot_heartbeat', async () => {
+ const calls = [];
+ const sql = async (strings, ...values) => {
+ calls.push({ text: strings.join('?'), values });
+ return calls.length === 1 ? [{ worker: 'okx-chat-bot:lease' }] : [];
+ };
+ const store = sqlLeaseStore(sql);
+ expect(await store.acquire('h#1', 120, { host: 'h' })).toEqual({ acquired: true });
+ expect(calls[0].text).toMatch(/ON CONFLICT \(worker\) DO UPDATE/);
+ expect(calls[0].text).toMatch(/meta->>'holder' = \?/);
+ expect(calls[0].text).toMatch(/meta->>'released' = 'true'/);
+ expect(calls[0].text).toMatch(/last_beat_at < now\(\)/);
+ expect(calls[0].values).toContain('okx-chat-bot:lease');
+ expect(JSON.parse(calls[0].values[1])).toEqual({ host: 'h', holder: 'h#1' });
+ expect(await store.renew('h#1')).toBe(false);
+ expect(calls[1].text).toMatch(/UPDATE bot_heartbeat SET last_beat_at = now\(\)/);
+ });
+
+ it('only requires the lease when state is shared through the bucket', () => {
+ expect(loadConfig({ OKX_BOT_STATE_BUCKET: 'three-ws-okx-bot-state' }).leaseRequired).toBe(true);
+ expect(loadConfig({}).leaseRequired).toBe(false);
+ });
+});
diff --git a/workers/okx-chat-bot/cloudbuild.yaml b/workers/okx-chat-bot/cloudbuild.yaml
index 7f9352187..9f57bffea 100644
--- a/workers/okx-chat-bot/cloudbuild.yaml
+++ b/workers/okx-chat-bot/cloudbuild.yaml
@@ -1,11 +1,19 @@
# Cloud Build → Cloud Run for the OKX.AI marketplace chat bot host
# (workers/okx-chat-bot). Build context is the REPO ROOT.
#
+# Deploy with ONE command, which provisions the gateway lane, builds from a
+# clean worktree, submits this config and verifies the rollout:
+#
+# npm run okx:bot:deploy
+#
+# (scripts/okx-bot-deploy.mjs). A hand submit needs _GATEWAY_AGENT, the metering
+# agent `node scripts/okx-bot-llm-gateway.mjs --verify` prints:
+#
# gcloud builds submit --config workers/okx-chat-bot/cloudbuild.yaml \
# --region us-central1 --project aerial-vehicle-466722-p5 \
-# --substitutions=SHORT_SHA=manual$(date +%s) .
+# --substitutions=SHORT_SHA=manual$(date +%s),_GATEWAY_AGENT=<agent-id> .
#
-# ── WHY --max-instances=1 IS LOAD-BEARING ────────────────────────────────────
+# ── WHY --max-instances=1 IS LOAD-BEARING, AND WHY IT IS NOT ENOUGH ──────────
# The bot's wallet/XMTP identity is a single GCS state object that this service
# reads on boot and overwrites on a timer. Two concurrent instances would
# interleave those writes and corrupt the identity, which costs a human OTP to
@@ -13,6 +21,13 @@
# daemon, not a request handler), and --no-cpu-throttling keeps it ticking when
# no HTTP request is in flight.
#
+# max-instances is PER REVISION, though, and a rollout runs the new revision's
+# instance beside the old one until it is Ready. The worker's single-writer
+# lease (workers/okx-chat-bot/lease.js) closes that window: the new instance
+# passes its startup probe, then waits to restore state or start the daemon
+# until the old one has snapshotted and released the lease (or, for a revision
+# built before the lease, until its heartbeat has gone quiet).
+#
# ── ONE-TIME SETUP: DONE (2026-09-02 / 2026-09-04) ───────────────────────────
# Nothing below needs running again; it is recorded so the next reader knows
# what the deploy depends on.
@@ -61,6 +76,12 @@ substitutions:
_REPO: workers
# Deliberately not parameterised: see the max-instances note above.
_STATE_BUCKET: three-ws-okx-bot-state
+ # The three.ws agent the gateway lane meters its AI calls against: the
+ # dedicated, platform-owned, unpublished agent scripts/okx-bot-llm-gateway.mjs
+ # provisions. Required: an empty value builds a base URL no probe accepts, so
+ # the lane would simply never be elected.
+ _GATEWAY_AGENT: ''
+ _GATEWAY_MODEL: nvidia/nemotron-3-super-120b-a12b
steps:
- id: build
@@ -111,23 +132,28 @@ steps:
# CLAUDE_CODE_USE_VERTEX points the subsession at Vertex AI, authenticated
# by three-ws@ through ADC (roles/aiplatform.user, already granted).
#
- # OKX_BOT_ANTHROPIC_* is the last rung of the AI chain (providerLanes in
- # config.js): an Anthropic-wire-format gateway the same `claude` CLI can be
- # pointed at, so a lane change never costs the task lifecycle. It is here
- # rather than in a runbook because the whole point of the chain is that a
- # funded lane is used WITHOUT a deploy: while the OpenRouter account has no
- # credit this lane simply probes 402 and is never elected, and the minute it
- # is funded the next 15-minute probe elects it on its own. The base URL is
- # the prefix the CLI appends /v1/messages to, so it ends at /api, NOT /api/v1
- # (measured 2026-09-09: one segment too deep answers 404 on every reply).
- - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global,OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6
+ # OKX_BOT_ANTHROPIC_* is the gateway rung of the AI chain (providerLanes in
+ # config.js): an Anthropic-wire-format endpoint the same `claude` CLI is
+ # pointed at, so a lane change never costs the task lifecycle. It points at
+ # three.ws's OWN we-pay proxy, at the agent-scoped base URL an SDK client
+ # appends /v1/messages to, which serves the free model chain: no GCP billing
+ # and no third-party account. Vertex still leads the chain, so the day the
+ # billing hold clears the next 15-minute election moves back to Claude on
+ # Vertex by itself. Measured 2026-09-18: the CLI's full agentic request (22
+ # tools, streaming, a real Bash tool call) answered through this proxy on
+ # nvidia/nemotron-3-super-120b-a12b.
+ - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global,OKX_BOT_ANTHROPIC_BASE_URL=https://three.ws/api/llm/anthropic/agents/${_GATEWAY_AGENT},OKX_BOT_ANTHROPIC_MODEL=${_GATEWAY_MODEL}
# An API key, if one is ever preferred over Vertex, belongs HERE and not
# patched on afterwards: --set-secrets replaces the whole secret set, so a
# hand-added key is silently stripped by the next deploy. Add
# ANTHROPIC_API_KEY=anthropic-api-key:latest (or, for codex,
# OPENAI_API_KEY=openai-api-key:latest) once the secret exists; the chain
# ranks Vertex above both, and pinning is only needed to force one lane.
- - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest,OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest
+ # The gateway token is a three.ws API key for a service account that owns
+ # only the metering agent, so the reply subsession (which can read its own
+ # environment) holds a credential worth one agent's AI budget and nothing
+ # else. scripts/okx-bot-llm-gateway.mjs mints it straight into this secret.
+ - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest,OKX_BOT_ANTHROPIC_AUTH_TOKEN=okx-chat-bot-llm-gateway-token:latest
# Liveness only. Readiness is deliberately NOT wired to Cloud Run: a
# logged-out session fails /readyz and no restart can fix it, so probing it
# here would put the service in a crash loop instead of paging a human.
diff --git a/workers/okx-chat-bot/config.js b/workers/okx-chat-bot/config.js
index 4025b817c..255fa67a6 100644
--- a/workers/okx-chat-bot/config.js
+++ b/workers/okx-chat-bot/config.js
@@ -30,10 +30,11 @@ const num = (v, fallback) => {
* No secret exists to leak, rotate or forget, and the spend
* lands on the pool the platform already prefers.
* anthropic-key a first-party ANTHROPIC_API_KEY.
- * anthropic-gateway any Anthropic-wire-format gateway (OpenRouter serves one
- * at /api/v1/messages). Opt-in: it only exists when an
- * operator sets its base URL and token, because it bills a
- * third-party account per token.
+ * anthropic-gateway any Anthropic-wire-format gateway. The deploy points it
+ * at three.ws's own we-pay proxy (the free model chain,
+ * metered against a dedicated platform agent); OpenRouter
+ * serves one too. Opt-in: it only exists when an operator
+ * sets its base URL and token.
* anthropic-oauth an interactive `claude` login on a developer host.
* openai-key the codex CLI on OPENAI_API_KEY.
*
@@ -286,6 +287,19 @@ export function loadConfig(env = process.env) {
repoRoot: env.OKX_BOT_REPO_ROOT || '/app',
heartbeatMs: num(env.OKX_BOT_HEARTBEAT_MS, 30_000),
+
+ // The single-writer lease (lease.js). Required whenever state is shared
+ // through the bucket, because that is when a second host could write it.
+ // The TTL bounds how long a host that died without releasing blocks its
+ // successor; renewal runs well inside it so a slow query never loses it.
+ leaseRequired: !!(env.OKX_BOT_STATE_BUCKET || '').trim(),
+ leaseTtlMs: num(env.OKX_BOT_LEASE_TTL_MS, 120_000),
+ leaseRenewMs: num(env.OKX_BOT_LEASE_RENEW_MS, 20_000),
+ leasePollMs: num(env.OKX_BOT_LEASE_POLL_MS, 5_000),
+ // A host built before the lease existed is recognised by its heartbeat and
+ // waited out until that beat is this old: three missed beats at the default
+ // cadence, so one slow write never reads as the host having gone.
+ legacyBeatStaleMs: num(env.OKX_BOT_LEGACY_BEAT_STALE_MS, 3 * num(env.OKX_BOT_HEARTBEAT_MS, 30_000)),
sessionProbeMs: num(env.OKX_BOT_SESSION_PROBE_MS, 60_000),
snapshotMs: num(env.OKX_BOT_SNAPSHOT_MS, 5 * 60_000),
// A logged-out session needs a human. Alert on the transition, then at most
diff --git a/workers/okx-chat-bot/health-server.js b/workers/okx-chat-bot/health-server.js
index 198267dbd..0a290bfc1 100644
--- a/workers/okx-chat-bot/health-server.js
+++ b/workers/okx-chat-bot/health-server.js
@@ -61,6 +61,18 @@ export function statusBody(cfg, live, daemonStats, bootAt) {
agents: live.agents,
workspace: live.workspace,
state: { bucket: cfg.stateBucket || null, restore: live.stateRestore },
+ // Who may write the bot identity. `waitingOn` names what a host that has
+ // not taken the lease yet is waiting for (a rollout's old instance, or a
+ // host that predates the lease).
+ lease: live.lease
+ ? {
+ held: live.lease.held,
+ holder: live.lease.holder,
+ acquiredAt: live.lease.acquiredAt || null,
+ waitingOn: live.lease.waitingOn,
+ lostReason: live.lease.lostReason,
+ }
+ : null,
health: live.verdict,
checkedAt: live.checkedAt,
};
diff --git a/workers/okx-chat-bot/index.js b/workers/okx-chat-bot/index.js
index 5c0503805..dc0c3bde5 100644
--- a/workers/okx-chat-bot/index.js
+++ b/workers/okx-chat-bot/index.js
@@ -29,12 +29,23 @@ import { electProvider, loginCodex } from './provider.js';
import { classify, loginInstructions } from './session.js';
import { restoreState, snapshotState } from './state.js';
import { createSupervisor } from './supervisor.js';
+import { createLease, sqlLeaseStore } from './lease.js';
import { buildWorkspace } from './workspace.js';
import { log } from './log.js';
const WORKER = 'okx-chat-bot';
const BOOT_AT = new Date().toISOString();
+/**
+ * The single-writer lease (lease.js), or null on a host with no shared state.
+ * Held from before the state restore until after the final snapshot, so no two
+ * hosts ever restore, run or snapshot the bot's identity at the same time.
+ */
+let lease = null;
+
+/** May this host write the shared state right now? */
+const mayWrite = () => !lease || lease.fresh();
+
/** Live health, refreshed by the session probe and read by the HTTP handlers. */
const live = {
verdict: { status: 'unknown', ready: false, reason: 'booting', detail: 'probe has not run yet', needsHumanLogin: false },
@@ -53,6 +64,8 @@ const live = {
// `providerProbe`; this is the whole picture, so a human reading /readyz can
// see which credential to fund rather than only that one of them was refused.
providerLanes: [],
+ // The single-writer lease's state (lease.js), null on a host without one.
+ lease: null,
};
async function heartbeat(cfg, supervisor) {
@@ -77,6 +90,9 @@ async function heartbeat(cfg, supervisor) {
providerChain: live.providerLanes.map((l) => ({ lane: l.lane, code: l.code })),
daemonRestarts: supervisor.stats().restarts,
checkedAt: live.checkedAt,
+ // Marks this host as lease-aware. A successor treats a beat WITHOUT this
+ // field as a host that predates the lease and waits it out (lease.js).
+ leaseHolder: lease?.state.holder ?? null,
};
try {
await sql`
@@ -242,6 +258,58 @@ async function main() {
});
}
+ // The health server comes up first, before the lease wait. Cloud Run's
+ // startup probe has to pass for a rollout to move traffic and SIGTERM the old
+ // instance, and the old instance releasing the lease is exactly what this one
+ // is waiting for: holding the probe behind the lease would deadlock the deploy.
+ const supervisor = createSupervisor(cfg, p);
+ const server = startHealthServer(cfg, live, supervisor, BOOT_AT);
+
+ // A SIGTERM that lands while waiting has nothing to save: this host never
+ // restored, ran or snapshotted anything.
+ const waitAbort = new AbortController();
+ const onEarlySignal = (signal) => {
+ log.info('shutdown before taking the lease', { signal });
+ waitAbort.abort();
+ server?.close();
+ process.exit(0);
+ };
+ process.once('SIGTERM', onEarlySignal);
+ process.once('SIGINT', onEarlySignal);
+
+ if (cfg.leaseRequired) {
+ lease = createLease({
+ store: sqlLeaseStore(sql),
+ host: cfg.host,
+ ttlMs: cfg.leaseTtlMs,
+ renewMs: cfg.leaseRenewMs,
+ pollMs: cfg.leasePollMs,
+ legacyStaleMs: cfg.legacyBeatStaleMs,
+ onLost: (reason) => void fence(cfg, supervisor, reason),
+ });
+ live.lease = lease.state;
+ live.verdict = {
+ status: 'unknown',
+ ready: false,
+ reason: 'lease_wait',
+ detail: 'waiting for the single-writer lease before restoring the bot identity',
+ needsHumanLogin: false,
+ };
+ const got = await lease.acquire({
+ signal: waitAbort.signal,
+ onWait: (reason) => {
+ live.verdict = { ...live.verdict, detail: `waiting for the single-writer lease: ${reason}` };
+ },
+ });
+ if (!got) return;
+ lease.start();
+ live.verdict = { status: 'unknown', ready: false, reason: 'booting', detail: 'lease held, restoring state', needsHumanLogin: false };
+ } else {
+ log.warn('no state bucket: running without the single-writer lease (fine for a local run, never for Cloud Run)');
+ }
+ process.off('SIGTERM', onEarlySignal);
+ process.off('SIGINT', onEarlySignal);
+
await mkdir(p.agentTask, { recursive: true });
live.stateRestore = await restoreState(cfg);
await mkdir(p.logs, { recursive: true });
@@ -261,9 +329,7 @@ async function main() {
const codexLogin = await loginCodex(cfg);
if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);
- const supervisor = createSupervisor(cfg, p);
supervisor.start();
- const server = startHealthServer(cfg, live, supervisor, BOOT_AT);
const probe = () => probeSession(cfg, supervisor).catch((err) => log.error('probe failed', { err: err?.message }));
const probeTimer = setInterval(probe, cfg.sessionProbeMs);
@@ -275,10 +341,10 @@ async function main() {
() => heartbeat(cfg, supervisor).catch((err) => log.warn('heartbeat failed', { err: err?.message })),
cfg.heartbeatMs,
);
- const snapshotTimer = setInterval(
- () => snapshotState(cfg, { reason: 'timer' }).catch(() => {}),
- cfg.snapshotMs,
- );
+ const snapshotTimer = setInterval(() => {
+ if (!mayWrite()) return log.warn('snapshot skipped: the single-writer lease is not fresh', {});
+ snapshotState(cfg, { reason: 'timer' }).catch(() => {});
+ }, cfg.snapshotMs);
const providerProbeTimer = setInterval(
() => electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message })),
cfg.providerProbeMs,
@@ -303,9 +369,13 @@ async function main() {
clearInterval(providerProbeTimer);
server?.close();
// Order matters: stop the daemon FIRST so its sqlite files are quiesced,
- // then snapshot. A live-copy snapshot can tear; this one cannot.
+ // then snapshot. A live-copy snapshot can tear; this one cannot. The
+ // snapshot is written only while the lease is still ours, and the lease is
+ // released only after it, so the successor restores exactly this state.
await supervisor.stop();
- await snapshotState(cfg, { reason: signal });
+ if (mayWrite()) await snapshotState(cfg, { reason: signal });
+ else log.error('final snapshot skipped: the single-writer lease is not held', { signal });
+ await lease?.release();
log.info('bye', {});
process.exit(0);
};
@@ -314,6 +384,27 @@ async function main() {
process.on('unhandledRejection', (err) => log.error('unhandledRejection', { err: err?.message }));
}
+/**
+ * The lease is gone (another host took it, or it could not be renewed before it
+ * would expire). Another host may already be writing the identity, so this one
+ * stops its daemon and exits WITHOUT a snapshot: writing one now is exactly the
+ * clobber the lease exists to prevent. Cloud Run restarts the container, and the
+ * fresh process waits for the lease like any other.
+ */
+let fencing = false;
+async function fence(cfg, supervisor, reason) {
+ if (fencing) return;
+ fencing = true;
+ live.verdict = { status: 'down', ready: false, reason: 'lease_lost', detail: `single-writer lease lost: ${reason}`, needsHumanLogin: false };
+ await sendOpsAlert(
+ 'OKX chat bot fenced itself',
+ `agent #${cfg.agentId} on ${cfg.host} lost the single-writer lease (${reason}) and stopped its daemon without snapshotting. It restarts and waits for the lease; no action is needed unless this repeats.`,
+ { severity: 'warn' },
+ ).catch(() => {});
+ await supervisor.stop(5_000).catch(() => {});
+ process.exit(75);
+}
+
main().catch((err) => {
log.error('fatal', { err: err?.message, stack: err?.stack });
process.exit(1);
diff --git a/workers/okx-chat-bot/lease.js b/workers/okx-chat-bot/lease.js
new file mode 100644
index 000000000..2b9a0d9af
--- /dev/null
+++ b/workers/okx-chat-bot/lease.js
@@ -0,0 +1,264 @@
+// okx-chat-bot: the single-writer lease.
+//
+// The bot's identity (the wallet keyring plus the XMTP client database) is one
+// state object with exactly one legal writer. `--max-instances=1` was the only
+// thing enforcing that, and it does not: Cloud Run applies max instances PER
+// REVISION, and a rollout deliberately runs the new revision's instance next to
+// the old one until the new one is Ready and traffic has moved. For that window
+// two daemons would hold the same XMTP installation, and the old instance's
+// SIGTERM snapshot would then overwrite whatever the new one had written. A torn
+// identity costs a human email OTP to recover.
+//
+// So a host must hold this lease before it restores state, starts the daemon or
+// writes a snapshot, and must keep renewing it to keep doing any of those. The
+// lease is a row in the `bot_heartbeat` table the worker already writes (worker
+// = LEASE_KEY), so it needs no new table, no migration and no new credential.
+// Every transition is one conditional statement, so two hosts racing for it
+// cannot both win.
+//
+// A rollout therefore sequences itself:
+//
+// 1. The new instance boots, serves /healthz (so Cloud Run's startup probe
+// passes and the rollout can proceed) and waits, reporting `lease_wait`.
+// 2. Cloud Run moves traffic and SIGTERMs the old instance, which stops its
+// daemon, writes its final snapshot while it still holds the lease, and
+// releases it.
+// 3. The new instance takes the lease, restores THAT snapshot, and only then
+// starts its daemon.
+//
+// A host that dies without releasing is waited out for LEASE_TTL. A host that
+// cannot renew (its database is unreachable) fences itself before the TTL can
+// expire under it: it stops the daemon and exits WITHOUT snapshotting, because
+// by then another host may legitimately be writing.
+//
+// Hosts built before this lease existed never take it. They are recognised by
+// their heartbeat, which carries no `leaseHolder`: while such a host is still
+// beating, nobody may take the lease. That is what makes the first rollout onto
+// this code as safe as every later one.
+
+import { randomUUID } from 'node:crypto';
+import { log } from './log.js';
+
+export const LEASE_KEY = 'okx-chat-bot:lease';
+const HEARTBEAT_WORKER = 'okx-chat-bot';
+
+/**
+ * The lease's storage: one `bot_heartbeat` row, driven by conditional upserts.
+ * Every clock comparison uses the database's now(), so two hosts with skewed
+ * clocks still agree on whether the lease has expired.
+ *
+ * @param {Function} sql tagged-template client (api/_lib/db.js)
+ * @param {string} [key]
+ */
+export function sqlLeaseStore(sql, key = LEASE_KEY) {
+ return {
+ /** Take the lease if it is free, expired, released, or already ours. */
+ async acquire(holder, ttlSec, meta = {}) {
+ const row = JSON.stringify({ ...meta, holder });
+ const won = await sql`
+ INSERT INTO bot_heartbeat (worker, mode, last_beat_at, meta)
+ VALUES (${key}, 'lease', now(), ${row}::jsonb)
+ ON CONFLICT (worker) DO UPDATE
+ SET last_beat_at = now(), meta = excluded.meta
+ WHERE bot_heartbeat.meta->>'holder' = ${holder}
+ OR bot_heartbeat.meta->>'released' = 'true'
+ OR bot_heartbeat.last_beat_at < now() - (${ttlSec}::int * interval '1 second')
+ RETURNING worker
+ `;
+ if (won.length) return { acquired: true };
+ const [cur] = await sql`
+ SELECT meta, extract(epoch FROM now() - last_beat_at)::float8 AS age_s
+ FROM bot_heartbeat WHERE worker = ${key}
+ `;
+ return {
+ acquired: false,
+ holder: cur?.meta?.holder ?? null,
+ host: cur?.meta?.host ?? null,
+ ageSec: cur ? Number(cur.age_s) : null,
+ };
+ },
+ /** Extend the lease. False means it is no longer ours. */
+ async renew(holder) {
+ const rows = await sql`
+ UPDATE bot_heartbeat SET last_beat_at = now()
+ WHERE worker = ${key} AND meta->>'holder' = ${holder}
+ AND coalesce(meta->>'released', '') <> 'true'
+ RETURNING worker
+ `;
+ return rows.length > 0;
+ },
+ /** Hand the lease to whoever is waiting, without waiting out the TTL. */
+ async release(holder) {
+ await sql`
+ UPDATE bot_heartbeat
+ SET meta = meta || '{"released": true}'::jsonb, last_beat_at = now()
+ WHERE worker = ${key} AND meta->>'holder' = ${holder}
+ `;
+ },
+ /** The serving host's own heartbeat, for recognising a pre-lease host. */
+ async hostBeat() {
+ const [row] = await sql`
+ SELECT meta->>'host' AS host, meta->>'leaseHolder' AS lease_holder,
+ extract(epoch FROM now() - last_beat_at)::float8 AS age_s
+ FROM bot_heartbeat WHERE worker = ${HEARTBEAT_WORKER}
+ `;
+ return row ? { host: row.host ?? null, leaseAware: !!row.lease_holder, ageSec: Number(row.age_s) } : null;
+ },
+ };
+}
+
+/**
+ * Does a host that predates the lease still look alive?
+ *
+ * Returns the reason to keep waiting, or null when the heartbeat is no obstacle:
+ * absent, stale, written by a lease-aware host (the lease itself then governs),
+ * or written by this same host label (a previous process of this revision, which
+ * is dead: a revision never runs two instances).
+ *
+ * @param {{ host: string|null, leaseAware: boolean, ageSec: number }|null} beat
+ * @param {{ selfHost: string, staleSec: number }} opts
+ * @returns {string|null}
+ */
+export function legacyWriterBlocks(beat, { selfHost, staleSec }) {
+ if (!beat) return null;
+ if (beat.leaseAware) return null;
+ if (beat.host && beat.host === selfHost) return null;
+ if (!Number.isFinite(beat.ageSec) || beat.ageSec >= staleSec) return null;
+ return `${beat.host || 'an unnamed host'} predates the single-writer lease and beat ${Math.round(beat.ageSec)}s ago`;
+}
+
+/**
+ * @param {object} opts
+ * @param {ReturnType<typeof sqlLeaseStore>} opts.store
+ * @param {string} opts.host this host's label (config.resolveHost)
+ * @param {number} opts.ttlMs how long an unrenewed lease stays taken
+ * @param {number} opts.renewMs renewal cadence; must be well inside ttlMs
+ * @param {number} opts.pollMs how often a waiting host asks again
+ * @param {number} opts.legacyStaleMs how old a pre-lease host's beat must be to ignore it
+ * @param {(reason: string) => void} [opts.onLost] called once when the lease is gone
+ * @param {() => number} [opts.now]
+ * @param {(ms: number) => Promise<void>} [opts.sleep]
+ * @param {string} [opts.holder]
+ */
+export function createLease({
+ store,
+ host,
+ ttlMs,
+ renewMs,
+ pollMs,
+ legacyStaleMs,
+ onLost = () => {},
+ now = Date.now,
+ sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+ holder = `${host}#${randomUUID()}`,
+}) {
+ if (!(renewMs * 2 < ttlMs)) throw new Error(`lease renewMs (${renewMs}) must be under half of ttlMs (${ttlMs})`);
+ const ttlSec = Math.ceil(ttlMs / 1000);
+ const state = { holder, held: false, acquiredAt: 0, lastRenewAt: 0, waitingOn: null, lostReason: null };
+ let timer = null;
+ let renewing = false;
+ let lostFired = false;
+
+ function lose(reason) {
+ if (lostFired) return;
+ lostFired = true;
+ state.held = false;
+ state.lostReason = reason;
+ if (timer) clearInterval(timer);
+ timer = null;
+ log.error('single-writer lease lost', { holder, reason });
+ onLost(reason);
+ }
+
+ async function renewOnce() {
+ if (!state.held || renewing) return;
+ renewing = true;
+ try {
+ if (await store.renew(holder)) state.lastRenewAt = now();
+ else lose('another host holds the lease');
+ } catch (err) {
+ // A database blip is not a lost lease, but an unrenewable one is about to
+ // expire under us. Fence before it can: the renewal margin is exactly the
+ // time left before another host may legitimately take it.
+ const silentMs = now() - state.lastRenewAt;
+ log.warn('lease renew failed', { holder, err: err?.message, silentMs });
+ if (silentMs >= ttlMs - renewMs) lose(`could not renew for ${Math.round(silentMs / 1000)}s`);
+ } finally {
+ renewing = false;
+ }
+ }
+
+ return {
+ state,
+
+ /**
+ * Block until this host holds the lease. Resolves false only when aborted.
+ * @param {{ signal?: AbortSignal, onWait?: (reason: string) => void }} [opts]
+ */
+ async acquire({ signal, onWait = () => {} } = {}) {
+ let lastLogged = null;
+ while (!signal?.aborted) {
+ let reason = null;
+ try {
+ reason = legacyWriterBlocks(await store.hostBeat(), { selfHost: host, staleSec: legacyStaleMs / 1000 });
+ if (!reason) {
+ const r = await store.acquire(holder, ttlSec, { host, since: new Date(now()).toISOString() });
+ if (r.acquired) {
+ state.held = true;
+ state.acquiredAt = now();
+ state.lastRenewAt = now();
+ state.waitingOn = null;
+ log.info('single-writer lease acquired', { holder });
+ return true;
+ }
+ reason = `${r.host || r.holder || 'another host'} holds the lease (renewed ${Math.round(r.ageSec ?? 0)}s ago)`;
+ }
+ } catch (err) {
+ reason = `lease store unreachable: ${err?.message || err}`;
+ }
+ state.waitingOn = reason;
+ onWait(reason);
+ if (reason !== lastLogged) {
+ log.warn('waiting for the single-writer lease', { holder, reason });
+ lastLogged = reason;
+ }
+ await sleep(pollMs);
+ }
+ return false;
+ },
+
+ /** Start renewing. Call once, right after acquire() resolves true. */
+ start() {
+ if (timer || !state.held) return;
+ timer = setInterval(() => void renewOnce(), renewMs);
+ if (typeof timer.unref === 'function') timer.unref();
+ },
+
+ /** Exposed for tests and for a caller that wants one renewal now. */
+ renewOnce,
+
+ /**
+ * True while this host may write shared state. Stricter than `held`: a
+ * lease whose last renewal is older than the fencing margin is treated as
+ * already gone, so no snapshot is ever written on a lease that may have
+ * expired in the database.
+ */
+ fresh() {
+ return state.held && now() - state.lastRenewAt < ttlMs - renewMs;
+ },
+
+ /** Stop renewing and hand the lease over. Best effort: a failure just leaves it to expire. */
+ async release() {
+ if (timer) clearInterval(timer);
+ timer = null;
+ if (!state.held) return;
+ state.held = false;
+ try {
+ await store.release(holder);
+ log.info('single-writer lease released', { holder });
+ } catch (err) {
+ log.warn('lease release failed: it will expire on its TTL', { holder, err: err?.message });
+ }
+ },
+ };
+}
diff --git a/workers/okx-chat-bot/session.js b/workers/okx-chat-bot/session.js
index 5fc154732..917b42c46 100644
--- a/workers/okx-chat-bot/session.js
+++ b/workers/okx-chat-bot/session.js
@@ -46,6 +46,12 @@ export function providerInstructions(detail = '', lanes = []) {
`The provider refused this host's credential: ${detail}`,
...chain,
'',
+ '0. Payment-free: the anthropic-gateway lane on three.ws\'s own proxy',
+ ' (/api/llm/anthropic/agents/<agent>, the free model chain). If it is in the',
+ ' chain above and refused, `npm run okx:bot:gateway -- --verify` says why;',
+ ' `-- --apply --rotate` re-mints its key; the API must be deployed with the',
+ ' agent-scoped SDK path (`npm run deploy:gcp:full`). If it is missing from',
+ ' the chain, redeploy the worker: `npm run okx:bot:deploy`.',
'1. Preferred (no secret): clear the project billing hold, then',
' gcloud run services update okx-chat-bot --region us-central1 \\',
" --update-env-vars=CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=aerial-vehicle-466722-p5,CLOUD_ML_REGION=global",
@@ -56,8 +62,9 @@ export function providerInstructions(detail = '', lanes = []) {
' stripped by the next deploy).',
'3. Or reactivate billing on the OpenAI account behind the openai-api-key secret',
' and pin OKX_BOT_AI_PROVIDER=codex.',
- '4. Or point the host at any funded Anthropic-wire-format gateway (config only,',
- ' pre-approved). OpenRouter serves one at /api/v1/messages:',
+ '4. Or point the gateway lane at a funded third-party Anthropic-wire-format',
+ ' gateway instead (config only, pre-approved). OpenRouter serves one at',
+ ' /api/v1/messages:',
' gcloud run services update okx-chat-bot --region us-central1 \\',
' --update-env-vars=OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6 \\',
' --update-secrets=OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest',