Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(okx-bot): hold a single-writer lease across rollouts and ship the worker with one command
nirholasa71cc4425420
diff --git a/package.json b/package.jsonindex 34b753fd0..b3a72f259 100644--- a/package.json+++ b/package.json@@ -453,6 +453,8 @@"typecheck": "tsc -p jsconfig.json","okx:bot": "node scripts/okx-bot-revive.mjs","okx:bot:seed-state": "node scripts/okx-bot-seed-state.mjs",+ "okx:bot:gateway": "node scripts/okx-bot-llm-gateway.mjs",+ "okx:bot:deploy": "node scripts/okx-bot-deploy.mjs","okx:three-copy": "node scripts/okx-three-copy-check.mjs","okx:gauntlet": "node scripts/okx-e2e-gauntlet.mjs","check:conflicts": "node scripts/check-merge-conflicts.mjs",diff --git a/scripts/okx-bot-deploy.mjs b/scripts/okx-bot-deploy.mjsnew file mode 100644index 000000000..a5afc8cd3--- /dev/null+++ b/scripts/okx-bot-deploy.mjs@@ -0,0 +1,211 @@+#!/usr/bin/env node+// Ship the OKX marketplace chat bot (workers/okx-chat-bot) in one command.+//+// npm run okx:bot:deploy # preflight, provision, build, submit, verify+// npm run okx:bot:deploy -- --dry-run # preflight and provisioning plan only: writes nothing, submits nothing+// npm run okx:bot:deploy -- --allow-dirty # submit the working tree instead of a clean worktree of HEAD+//+// What "one command" has to cover, in order:+//+// 1. gcloud is authenticated (every later step needs it).+// 2. The worker is built from a CLEAN worktree of HEAD, never from the shared+// working tree: the image copies all of api/, and other agents' in-flight+// edits there would otherwise ship inside the bot. A dirty+// workers/okx-chat-bot/ is refused, because that is uncommitted bot code the+// clean build would silently leave out.+// 3. The payment-free AI lane exists: scripts/okx-bot-llm-gateway.mjs --apply+// (idempotent) provisions the metering agent, the service-account key and+// the Secret Manager secret, then proves the lane with the bot's own probe.+// Its agent id becomes the build's _GATEWAY_AGENT.+// 4. Whether the live API already serves the agent-scoped SDK path the lane+// calls. It is not a blocker: the lane is elected on a live probe every 15+// minutes, so if the API deploy lands second the bot moves onto the lane by+// itself. It is reported so nobody reads a Vertex-only bot as a failure.+// 5. Submit workers/okx-chat-bot/cloudbuild.yaml.+// 6. Verify on the platform's own health view (no gcloud, no secret): the+// okx_chat_bot subsystem must report the NEW revision as its host. The new+// instance first waits for the single-writer lease (workers/okx-chat-bot/+// lease.js) while the old one drains, so this poll is expected to take a+// couple of minutes, not seconds.+//+// Owner-gated: this deploys to production. Run it only on the owner's say-so.++import { execFile, spawn } from 'node:child_process';+import { existsSync } from 'node:fs';+import { resolve } from 'node:path';+import { promisify } from 'node:util';++const run = promisify(execFile);++const args = process.argv.slice(2);+const DRY = args.includes('--dry-run');+const ALLOW_DIRTY = args.includes('--allow-dirty');+const PROJECT = 'aerial-vehicle-466722-p5';+const REGION = 'us-central1';+const SERVICE = 'okx-chat-bot';+const ROOT = resolve(new URL('..', import.meta.url).pathname);+const WORKTREE = resolve(ROOT, '..', '.okx-bot-deploy-wt');+const HEALTHZ = 'https://three.ws/api/healthz';+const VERSION = 'https://three.ws/api/version';+// The commit that taught the proxy the SDK path and the Claude Code request+// shape (system turns inside messages). The live API needs it for the lane.+const SDK_PATH_COMMIT = 'd3074c1c8';+const VERIFY_TIMEOUT_MS = 15 * 60_000;++const say = (...m) => console.log(...m);+const step = (n, title) => say(`\n[${n}] ${title}`);+const die = (msg, code = 1) => {+ console.error(`\n FAILED: ${msg}\n`);+ process.exit(code);+};++const git = (argv, opts = {}) => run('git', argv, { cwd: ROOT, maxBuffer: 16 * 1024 * 1024, ...opts });++function stream(cmd, argv, opts = {}) {+ return new Promise((res, rej) => {+ const child = spawn(cmd, argv, { stdio: ['ignore', 'pipe', 'inherit'], ...opts });+ let out = '';+ child.stdout.on('data', (d) => {+ out += d;+ process.stdout.write(d);+ });+ child.on('error', rej);+ child.on('close', (code) => (code === 0 ? res(out) : rej(Object.assign(new Error(`${cmd} exited ${code}`), { code, out }))));+ });+}++async function okxSubsystem() {+ const res = await fetch(HEALTHZ, { signal: AbortSignal.timeout(20_000) });+ const body = await res.json();+ return (body?.subsystems?.subsystems || []).find((s) => s.name === 'okx_chat_bot') || null;+}++async function liveApiHasSdkPath() {+ const v = await fetch(VERSION, { signal: AbortSignal.timeout(20_000) }).then((r) => r.json());+ const live = v?.commit;+ if (!live) return { ok: false, detail: '/api/version reported no commit' };+ try {+ await git(['merge-base', '--is-ancestor', SDK_PATH_COMMIT, live]);+ return { ok: true, detail: `live API ${v.commitShort} (${v.runtime?.revision}) contains ${SDK_PATH_COMMIT}` };+ } catch {+ return { ok: false, detail: `live API ${v.commitShort} (${v.runtime?.revision}) predates ${SDK_PATH_COMMIT}` };+ }+}++async function main() {+ step(1, 'gcloud authentication');+ try {+ await run('gcloud', ['auth', 'print-access-token'], { timeout: 30_000 });+ say(' ok');+ } catch {+ const msg = 'gcloud is not authenticated here: run `gcloud auth login` (and `gcloud auth application-default login`), then re-run';+ if (!DRY) die(msg);+ say(` WARNING: ${msg}`);+ }++ step(2, 'build source');+ const { stdout: dirty } = await git(['status', '--porcelain', '--', 'workers/okx-chat-bot', 'scripts/okx-bot-llm-gateway.mjs', 'scripts/okx-bot-deploy.mjs']);+ let context = WORKTREE;+ if (ALLOW_DIRTY) {+ context = ROOT;+ say(' --allow-dirty: submitting the working tree as it is, including any uncommitted edits under api/');+ } else if (dirty.trim() && DRY) {+ say(` WARNING: uncommitted bot code; a real run refuses until it is committed:\n${dirty}`);+ } else if (dirty.trim()) {+ die(+ `uncommitted bot code would be left out of a clean build:\n${dirty}\n` ++ ' Commit it (the owner approves this content), or pass --allow-dirty to ship the working tree.',+ );+ } else {+ const { stdout: head } = await git(['rev-parse', '--short', 'HEAD']);+ say(` clean worktree of HEAD ${head.trim()} at ${WORKTREE}`);+ }++ step(3, 'AI gateway lane (metering agent, key, secret, live probe)');+ const gw = await stream(process.execPath, ['scripts/okx-bot-llm-gateway.mjs', DRY ? '--verify' : '--apply'], { cwd: ROOT }).catch((err) => {+ // Exit 2 is "provisioned, but the probe did not pass yet" (the SDK path is+ // not live): the deploy still proceeds, see step 4.+ if (err.code === 2) return err.out;+ if (DRY) return err.out || '';+ die('provisioning the gateway lane failed (output above)');+ });+ const agentMatch = /OKX_BOT_ANTHROPIC_BASE_URL=\S+\/agents\/([0-9a-f-]{36})/.exec(gw || '');+ if (!agentMatch && !DRY) die('the gateway script did not report a metering agent id');+ const agentId = agentMatch?.[1] || '<provisioned by --apply>';++ step(4, 'live API serves the agent-scoped SDK path');+ const api = await liveApiHasSdkPath();+ say(` ${api.ok ? 'yes' : 'NOT YET'}: ${api.detail}`);+ if (!api.ok) {+ say(' The bot still deploys and keeps serving on its current lane. Deploy the API (`npm run deploy:gcp:full`)');+ say(' and the next 15-minute election moves it onto the gateway lane with no second worker deploy.');+ }++ const substitutions = `SHORT_SHA=manual${Math.floor(Date.now() / 1000)},_GATEWAY_AGENT=${agentId}`;+ const submitArgs = [+ 'builds',+ 'submit',+ '--config',+ 'workers/okx-chat-bot/cloudbuild.yaml',+ '--region',+ REGION,+ '--project',+ PROJECT,+ `--substitutions=${substitutions}`,+ '.',+ ];+ if (DRY) {+ say(`\n dry run: would run in ${context}:\n gcloud ${submitArgs.join(' ')}\n`);+ return;+ }++ const before = await okxSubsystem().catch(() => null);+ say(`\n serving now: ${before?.host || 'unknown'} (${before?.status || 'unknown'})`);++ step(5, 'build and deploy');+ if (context === WORKTREE) {+ if (existsSync(WORKTREE)) await git(['worktree', 'remove', '--force', WORKTREE]);+ await git(['worktree', 'add', '--detach', WORKTREE, 'HEAD']);+ }+ try {+ await stream('gcloud', submitArgs, { cwd: context });+ } catch {+ die('Cloud Build failed (log above). The serving revision is untouched: a failed deploy never moves traffic.');+ } finally {+ if (context === WORKTREE) await git(['worktree', 'remove', '--force', WORKTREE]).catch(() => {});+ }++ step(6, 'verify the rollout on /api/healthz');+ const { stdout: rev } = await run(+ 'gcloud',+ ['run', 'services', 'describe', SERVICE, '--region', REGION, '--project', PROJECT, '--format=value(status.latestReadyRevisionName)'],+ { timeout: 60_000 },+ );+ const revision = rev.trim();+ say(` new revision ${revision}; waiting for it to take the single-writer lease and beat`);+ const deadline = Date.now() + VERIFY_TIMEOUT_MS;+ let last = '';+ while (Date.now() < deadline) {+ const s = await okxSubsystem().catch(() => null);+ const line = s ? `${s.host} | ${s.status} | ${s.detail}` : 'healthz unreachable';+ if (line !== last) {+ say(` ${new Date().toISOString().slice(11, 19)} ${line}`);+ last = line;+ }+ if (s?.host?.includes(revision) && (s.status === 'ok' || s.status === 'degraded')) {+ say(`\n DEPLOYED: ${revision} holds the lease and is beating (${s.status}).`);+ if (s.status !== 'ok') say(` Not fully green yet: ${s.detail}\n ${s.hint || ''}`);+ say(' Lane detail: curl -s -H "Authorization: Bearer $(gcloud auth print-identity-token)" \\');+ say(' https://okx-chat-bot-lp642k3kpa-uc.a.run.app/readyz | jq \'{lane: .provider.lane, chain: .provider.chain, lease: .lease}\'\n');+ return;+ }+ await new Promise((r) => setTimeout(r, 15_000));+ }+ die(+ `${revision} did not report in within ${VERIFY_TIMEOUT_MS / 60_000} minutes. Read its /readyz (lease.waitingOn names what it waits for) ` ++ `and: gcloud logging read 'resource.labels.service_name="${SERVICE}"' --freshness=30m --limit 100`,+ 3,+ );+}++main().catch((err) => die(err?.message || String(err)));diff --git a/scripts/okx-bot-llm-gateway.mjs b/scripts/okx-bot-llm-gateway.mjsindex 03adfadea..9e7f571de 100644--- a/scripts/okx-bot-llm-gateway.mjs+++ b/scripts/okx-bot-llm-gateway.mjs@@ -161,8 +161,12 @@ async function activeKeys(sql, userId) {async function secretState() {try {await gcloud(['secrets', 'describe', SECRET, '--format=value(name)']);- } catch {- return { exists: false, token: null };+ } catch (err) {+ // Only a NOT_FOUND means the secret is missing. Anything else (an expired+ // gcloud login above all) means nobody could look, and reporting that as+ // "missing" would send --apply off to create a secret that already exists.+ if (/NOT_FOUND|not found/i.test(err?.stderr || err?.message || '')) return { exists: false, token: null };+ return { exists: null, token: null, error: String(err?.message || err).split('\n')[0].slice(0, 200) };}try {const { stdout } = await gcloud(['secrets', 'versions', 'access', 'latest', '--secret', SECRET]);@@ -343,7 +347,8 @@ async function main() {say(` metering agent ${agent ? `${agent.id} "${agent.name}"` : 'missing'}`);say(` model ${MODEL}`);say(` api keys ${keys.length} active`);- say(` secret ${SECRET}: ${!secret.exists ? 'missing' : secretKey ? `holds active key ${secretKey.prefix}...` : 'holds no active key'}`);+ say(` secret ${SECRET}: ${secret.exists === null ? `unknown (${secret.error})` : !secret.exists ? 'missing' : secretKey ? `holds active key ${secretKey.prefix}...` : 'holds no active key'}`);+ if (secret.exists === null && (APPLY || VERIFY_ONLY)) die('gcloud cannot read Secret Manager here: run `gcloud auth login`, then re-run');if (!APPLY && !VERIFY_ONLY) {say('\n plan only: nothing was written. Re-run with --apply to provision, or --verify to prove an existing lane.\n');diff --git a/tests/okx-chat-bot.test.js b/tests/okx-chat-bot.test.jsindex 693c9c30f..064e10e0d 100644--- a/tests/okx-chat-bot.test.js+++ b/tests/okx-chat-bot.test.js@@ -899,3 +899,208 @@ describe('okx_chat_bot subsystem identity fields', () => {expect(s.hostDurable).toBeNull();});});++// ── single-writer lease ─────────────────────────────────────────────────────+//+// The bot identity must never have two writers, and Cloud Run's max-instances is+// per revision, so a rollout overlaps the old and new instance. These pin the+// lease that sequences that overlap. The store below reproduces the SQL+// contract of sqlLeaseStore (conditional upsert on holder / released / expiry,+// all against one clock) so the state machine is exercised against the same+// semantics the database enforces.++import { createLease, legacyWriterBlocks, sqlLeaseStore } from '../workers/okx-chat-bot/lease.js';++function memoryLeaseStore(clock) {+ let row = null;+ let beat = null;+ return {+ setBeat(b) {+ beat = b;+ },+ row: () => row,+ async acquire(holder, ttlSec, meta) {+ const free = !row || row.holder === holder || row.released || clock.now - row.at > ttlSec * 1000;+ if (free) {+ row = { ...meta, holder, at: clock.now, released: false };+ return { acquired: true };+ }+ return { acquired: false, holder: row.holder, host: row.host, ageSec: (clock.now - row.at) / 1000 };+ },+ async renew(holder) {+ if (!row || row.holder !== holder || row.released) return false;+ row.at = clock.now;+ return true;+ },+ async release(holder) {+ if (row && row.holder === holder) row = { ...row, released: true, at: clock.now };+ },+ async hostBeat() {+ return beat ? { ...beat, ageSec: (clock.now - beat.at) / 1000 } : null;+ },+ };+}++function leaseFor(store, clock, host, extra = {}) {+ return createLease({+ store,+ host,+ holder: `${host}#test`,+ ttlMs: 120_000,+ renewMs: 20_000,+ pollMs: 5_000,+ legacyStaleMs: 90_000,+ now: () => clock.now,+ // Waiting advances the shared clock, so a blocked acquire() observes time+ // passing exactly as a real poll loop would.+ sleep: async (ms) => {+ clock.now += ms;+ clock.onTick?.();+ },+ ...extra,+ });+}++describe('okx-chat-bot single-writer lease', () => {+ it('lets the first host take a free lease immediately', async () => {+ const clock = { now: 1_000_000 };+ const store = memoryLeaseStore(clock);+ const a = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-2)');+ expect(await a.acquire()).toBe(true);+ expect(a.state.held).toBe(true);+ expect(a.fresh()).toBe(true);+ });++ it('makes a rollout wait for the old instance, then hands over on release', async () => {+ const clock = { now: 1_000_000 };+ const store = memoryLeaseStore(clock);+ const old = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-1)');+ await old.acquire();+ const next = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-2)');+ const waits = [];+ // The old instance keeps renewing for 60s, then shuts down and releases.+ clock.onTick = () => {+ if (clock.now - 1_000_000 < 60_000) void old.renewOnce();+ else if (old.state.held) void old.release();+ };+ expect(await next.acquire({ onWait: (r) => waits.push(r) })).toBe(true);+ expect(waits[0]).toMatch(/rev-1\) holds the lease/);+ // Handed over on the release, well before the 120s TTL would have lapsed.+ expect(clock.now - 1_000_000).toBeLessThan(120_000);+ expect(store.row().holder).toBe('cloudrun:okx-chat-bot (rev-2)#test');+ });++ it('waits out the TTL for a host that died holding the lease', async () => {+ const clock = { now: 1_000_000 };+ const store = memoryLeaseStore(clock);+ await leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-1)').acquire();+ const next = leaseFor(store, clock, 'cloudrun:okx-chat-bot (rev-2)');+ expect(await next.acquire()).toBe(true);+ expect(clock.now - 1_000_000).toBeGreaterThan(120_000);+ });++ it('never lets two hosts hold it at once', async () => {+ const clock = { now: 1_000_000 };+ const store = memoryLeaseStore(clock);+ const a = leaseFor(store, clock, 'host-a');+ const b = leaseFor(store, clock, 'host-b');+ await a.acquire();+ const ctl = new AbortController();+ clock.onTick = () => {+ void a.renewOnce();+ if (clock.now - 1_000_000 > 600_000) ctl.abort();+ };+ expect(await b.acquire({ signal: ctl.signal })).toBe(false);+ expect(a.state.held).toBe(true);+ expect(b.state.held).toBe(false);+ });++ it('waits for a host that predates the lease while its heartbeat is fresh', async () => {+ const clock = { now: 1_000_000 };+ const store = memoryLeaseStore(clock);+ // The live revision today: beats every 30s, carries no leaseHolder.+ store.setBeat({ host: 'cloudrun:okx-chat-bot (okx-chat-bot-00001-926)', leaseAware: false, at: clock.now });+ const next = leaseFor(store, clock, 'cloudrun:okx-chat-bot (okx-chat-bot-00002-abc)');+ const waits = [];+ // It keeps beating for 45s, then Cloud Run stops it.+ clock.onTick = () => {+ if (clock.now - 1_000_000 < 45_000) store.setBeat({ host: 'cloudrun:okx-chat-bot (okx-chat-bot-00001-926)', leaseAware: false, at: clock.now });+ };+ expect(await next.acquire({ onWait: (r) => waits.push(r) })).toBe(true);+ expect(waits[0]).toMatch(/00001-926\) predates the single-writer lease/);+ // Its last beat landed at 40s; the lease is taken only once that is 90s stale.+ expect(clock.now - 1_000_000).toBe(40_000 + 90_000);+ });++ it('ignores a heartbeat that cannot be a second writer', () => {+ const opts = { selfHost: 'cloudrun:okx-chat-bot (rev-2)', staleSec: 90 };+ expect(legacyWriterBlocks(null, opts)).toBeNull();+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-1)', leaseAware: true, ageSec: 5 }, opts)).toBeNull();+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-2)', leaseAware: false, ageSec: 5 }, opts)).toBeNull();+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-1)', leaseAware: false, ageSec: 91 }, opts)).toBeNull();+ expect(legacyWriterBlocks({ host: 'cloudrun:okx-chat-bot (rev-1)', leaseAware: false, ageSec: 30 }, opts)).toMatch(/predates/);+ });++ it('reports a lost lease once and stops calling itself fresh', async () => {+ const clock = { now: 1_000_000 };+ const store = memoryLeaseStore(clock);+ const lost = [];+ const a = leaseFor(store, clock, 'host-a', { onLost: (r) => lost.push(r) });+ await a.acquire();+ // Another host took it (the TTL lapsed while this one was frozen).+ clock.now += 130_000;+ await leaseFor(store, clock, 'host-b').acquire();+ await a.renewOnce();+ await a.renewOnce();+ expect(lost).toEqual(['another host holds the lease']);+ expect(a.fresh()).toBe(false);+ });++ it('fences itself before an unrenewable lease can expire under it', async () => {+ const clock = { now: 1_000_000 };+ const lost = [];+ const store = memoryLeaseStore(clock);+ const a = leaseFor(store, clock, 'host-a', { onLost: (r) => lost.push(r) });+ await a.acquire();+ store.renew = async () => {+ throw new Error('database unreachable');+ };+ clock.now += 60_000;+ await a.renewOnce();+ expect(lost).toEqual([]);+ // Not fresh any more once the fencing margin (ttl - renew) has passed.+ clock.now += 40_000;+ expect(a.fresh()).toBe(false);+ await a.renewOnce();+ expect(lost).toHaveLength(1);+ expect(lost[0]).toMatch(/could not renew for 100s/);+ });++ it('refuses a renewal cadence that could let the lease lapse between renewals', () => {+ const clock = { now: 0 };+ expect(() => leaseFor(memoryLeaseStore(clock), clock, 'h', { renewMs: 70_000 })).toThrow(/under half/);+ });++ it('drives the lease with conditional statements against bot_heartbeat', async () => {+ const calls = [];+ const sql = async (strings, ...values) => {+ calls.push({ text: strings.join('?'), values });+ return calls.length === 1 ? [{ worker: 'okx-chat-bot:lease' }] : [];+ };+ const store = sqlLeaseStore(sql);+ expect(await store.acquire('h#1', 120, { host: 'h' })).toEqual({ acquired: true });+ expect(calls[0].text).toMatch(/ON CONFLICT \(worker\) DO UPDATE/);+ expect(calls[0].text).toMatch(/meta->>'holder' = \?/);+ expect(calls[0].text).toMatch(/meta->>'released' = 'true'/);+ expect(calls[0].text).toMatch(/last_beat_at < now\(\)/);+ expect(calls[0].values).toContain('okx-chat-bot:lease');+ expect(JSON.parse(calls[0].values[1])).toEqual({ host: 'h', holder: 'h#1' });+ expect(await store.renew('h#1')).toBe(false);+ expect(calls[1].text).toMatch(/UPDATE bot_heartbeat SET last_beat_at = now\(\)/);+ });++ it('only requires the lease when state is shared through the bucket', () => {+ expect(loadConfig({ OKX_BOT_STATE_BUCKET: 'three-ws-okx-bot-state' }).leaseRequired).toBe(true);+ expect(loadConfig({}).leaseRequired).toBe(false);+ });+});diff --git a/workers/okx-chat-bot/cloudbuild.yaml b/workers/okx-chat-bot/cloudbuild.yamlindex 7f9352187..9f57bffea 100644--- a/workers/okx-chat-bot/cloudbuild.yaml+++ b/workers/okx-chat-bot/cloudbuild.yaml@@ -1,11 +1,19 @@# Cloud Build → Cloud Run for the OKX.AI marketplace chat bot host# (workers/okx-chat-bot). Build context is the REPO ROOT.#+# Deploy with ONE command, which provisions the gateway lane, builds from a+# clean worktree, submits this config and verifies the rollout:+#+# npm run okx:bot:deploy+#+# (scripts/okx-bot-deploy.mjs). A hand submit needs _GATEWAY_AGENT, the metering+# agent `node scripts/okx-bot-llm-gateway.mjs --verify` prints:+## gcloud builds submit --config workers/okx-chat-bot/cloudbuild.yaml \# --region us-central1 --project aerial-vehicle-466722-p5 \-# --substitutions=SHORT_SHA=manual$(date +%s) .+# --substitutions=SHORT_SHA=manual$(date +%s),_GATEWAY_AGENT=<agent-id> .#-# ── WHY --max-instances=1 IS LOAD-BEARING ────────────────────────────────────+# ── WHY --max-instances=1 IS LOAD-BEARING, AND WHY IT IS NOT ENOUGH ──────────# The bot's wallet/XMTP identity is a single GCS state object that this service# reads on boot and overwrites on a timer. Two concurrent instances would# interleave those writes and corrupt the identity, which costs a human OTP to@@ -13,6 +21,13 @@# daemon, not a request handler), and --no-cpu-throttling keeps it ticking when# no HTTP request is in flight.#+# max-instances is PER REVISION, though, and a rollout runs the new revision's+# instance beside the old one until it is Ready. The worker's single-writer+# lease (workers/okx-chat-bot/lease.js) closes that window: the new instance+# passes its startup probe, then waits to restore state or start the daemon+# until the old one has snapshotted and released the lease (or, for a revision+# built before the lease, until its heartbeat has gone quiet).+## ── ONE-TIME SETUP: DONE (2026-09-02 / 2026-09-04) ───────────────────────────# Nothing below needs running again; it is recorded so the next reader knows# what the deploy depends on.@@ -61,6 +76,12 @@ substitutions:_REPO: workers# Deliberately not parameterised: see the max-instances note above._STATE_BUCKET: three-ws-okx-bot-state+ # The three.ws agent the gateway lane meters its AI calls against: the+ # dedicated, platform-owned, unpublished agent scripts/okx-bot-llm-gateway.mjs+ # provisions. Required: an empty value builds a base URL no probe accepts, so+ # the lane would simply never be elected.+ _GATEWAY_AGENT: ''+ _GATEWAY_MODEL: nvidia/nemotron-3-super-120b-a12bsteps:- id: build@@ -111,23 +132,28 @@ steps:# CLAUDE_CODE_USE_VERTEX points the subsession at Vertex AI, authenticated# by three-ws@ through ADC (roles/aiplatform.user, already granted).#- # OKX_BOT_ANTHROPIC_* is the last rung of the AI chain (providerLanes in- # config.js): an Anthropic-wire-format gateway the same `claude` CLI can be- # pointed at, so a lane change never costs the task lifecycle. It is here- # rather than in a runbook because the whole point of the chain is that a- # funded lane is used WITHOUT a deploy: while the OpenRouter account has no- # credit this lane simply probes 402 and is never elected, and the minute it- # is funded the next 15-minute probe elects it on its own. The base URL is- # the prefix the CLI appends /v1/messages to, so it ends at /api, NOT /api/v1- # (measured 2026-09-09: one segment too deep answers 404 on every reply).- - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global,OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6+ # OKX_BOT_ANTHROPIC_* is the gateway rung of the AI chain (providerLanes in+ # config.js): an Anthropic-wire-format endpoint the same `claude` CLI is+ # pointed at, so a lane change never costs the task lifecycle. It points at+ # three.ws's OWN we-pay proxy, at the agent-scoped base URL an SDK client+ # appends /v1/messages to, which serves the free model chain: no GCP billing+ # and no third-party account. Vertex still leads the chain, so the day the+ # billing hold clears the next 15-minute election moves back to Claude on+ # Vertex by itself. Measured 2026-09-18: the CLI's full agentic request (22+ # tools, streaming, a real Bash tool call) answered through this proxy on+ # nvidia/nemotron-3-super-120b-a12b.+ - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global,OKX_BOT_ANTHROPIC_BASE_URL=https://three.ws/api/llm/anthropic/agents/${_GATEWAY_AGENT},OKX_BOT_ANTHROPIC_MODEL=${_GATEWAY_MODEL}# An API key, if one is ever preferred over Vertex, belongs HERE and not# patched on afterwards: --set-secrets replaces the whole secret set, so a# hand-added key is silently stripped by the next deploy. Add# ANTHROPIC_API_KEY=anthropic-api-key:latest (or, for codex,# OPENAI_API_KEY=openai-api-key:latest) once the secret exists; the chain# ranks Vertex above both, and pinning is only needed to force one lane.- - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest,OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest+ # The gateway token is a three.ws API key for a service account that owns+ # only the metering agent, so the reply subsession (which can read its own+ # environment) holds a credential worth one agent's AI budget and nothing+ # else. scripts/okx-bot-llm-gateway.mjs mints it straight into this secret.+ - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest,OKX_BOT_ANTHROPIC_AUTH_TOKEN=okx-chat-bot-llm-gateway-token:latest# Liveness only. Readiness is deliberately NOT wired to Cloud Run: a# logged-out session fails /readyz and no restart can fix it, so probing it# here would put the service in a crash loop instead of paging a human.diff --git a/workers/okx-chat-bot/config.js b/workers/okx-chat-bot/config.jsindex 4025b817c..255fa67a6 100644--- a/workers/okx-chat-bot/config.js+++ b/workers/okx-chat-bot/config.js@@ -30,10 +30,11 @@ const num = (v, fallback) => {* No secret exists to leak, rotate or forget, and the spend* lands on the pool the platform already prefers.* anthropic-key a first-party ANTHROPIC_API_KEY.- * anthropic-gateway any Anthropic-wire-format gateway (OpenRouter serves one- * at /api/v1/messages). Opt-in: it only exists when an- * operator sets its base URL and token, because it bills a- * third-party account per token.+ * anthropic-gateway any Anthropic-wire-format gateway. The deploy points it+ * at three.ws's own we-pay proxy (the free model chain,+ * metered against a dedicated platform agent); OpenRouter+ * serves one too. Opt-in: it only exists when an operator+ * sets its base URL and token.* anthropic-oauth an interactive `claude` login on a developer host.* openai-key the codex CLI on OPENAI_API_KEY.*@@ -286,6 +287,19 @@ export function loadConfig(env = process.env) {repoRoot: env.OKX_BOT_REPO_ROOT || '/app',heartbeatMs: num(env.OKX_BOT_HEARTBEAT_MS, 30_000),++ // The single-writer lease (lease.js). Required whenever state is shared+ // through the bucket, because that is when a second host could write it.+ // The TTL bounds how long a host that died without releasing blocks its+ // successor; renewal runs well inside it so a slow query never loses it.+ leaseRequired: !!(env.OKX_BOT_STATE_BUCKET || '').trim(),+ leaseTtlMs: num(env.OKX_BOT_LEASE_TTL_MS, 120_000),+ leaseRenewMs: num(env.OKX_BOT_LEASE_RENEW_MS, 20_000),+ leasePollMs: num(env.OKX_BOT_LEASE_POLL_MS, 5_000),+ // A host built before the lease existed is recognised by its heartbeat and+ // waited out until that beat is this old: three missed beats at the default+ // cadence, so one slow write never reads as the host having gone.+ legacyBeatStaleMs: num(env.OKX_BOT_LEGACY_BEAT_STALE_MS, 3 * num(env.OKX_BOT_HEARTBEAT_MS, 30_000)),sessionProbeMs: num(env.OKX_BOT_SESSION_PROBE_MS, 60_000),snapshotMs: num(env.OKX_BOT_SNAPSHOT_MS, 5 * 60_000),// A logged-out session needs a human. Alert on the transition, then at mostdiff --git a/workers/okx-chat-bot/health-server.js b/workers/okx-chat-bot/health-server.jsindex 198267dbd..0a290bfc1 100644--- a/workers/okx-chat-bot/health-server.js+++ b/workers/okx-chat-bot/health-server.js@@ -61,6 +61,18 @@ export function statusBody(cfg, live, daemonStats, bootAt) {agents: live.agents,workspace: live.workspace,state: { bucket: cfg.stateBucket || null, restore: live.stateRestore },+ // Who may write the bot identity. `waitingOn` names what a host that has+ // not taken the lease yet is waiting for (a rollout's old instance, or a+ // host that predates the lease).+ lease: live.lease+ ? {+ held: live.lease.held,+ holder: live.lease.holder,+ acquiredAt: live.lease.acquiredAt || null,+ waitingOn: live.lease.waitingOn,+ lostReason: live.lease.lostReason,+ }+ : null,health: live.verdict,checkedAt: live.checkedAt,};diff --git a/workers/okx-chat-bot/index.js b/workers/okx-chat-bot/index.jsindex 5c0503805..dc0c3bde5 100644--- a/workers/okx-chat-bot/index.js+++ b/workers/okx-chat-bot/index.js@@ -29,12 +29,23 @@ import { electProvider, loginCodex } from './provider.js';import { classify, loginInstructions } from './session.js';import { restoreState, snapshotState } from './state.js';import { createSupervisor } from './supervisor.js';+import { createLease, sqlLeaseStore } from './lease.js';import { buildWorkspace } from './workspace.js';import { log } from './log.js';const WORKER = 'okx-chat-bot';const BOOT_AT = new Date().toISOString();+/**+ * The single-writer lease (lease.js), or null on a host with no shared state.+ * Held from before the state restore until after the final snapshot, so no two+ * hosts ever restore, run or snapshot the bot's identity at the same time.+ */+let lease = null;++/** May this host write the shared state right now? */+const mayWrite = () => !lease || lease.fresh();+/** Live health, refreshed by the session probe and read by the HTTP handlers. */const live = {verdict: { status: 'unknown', ready: false, reason: 'booting', detail: 'probe has not run yet', needsHumanLogin: false },@@ -53,6 +64,8 @@ const live = {// `providerProbe`; this is the whole picture, so a human reading /readyz can// see which credential to fund rather than only that one of them was refused.providerLanes: [],+ // The single-writer lease's state (lease.js), null on a host without one.+ lease: null,};async function heartbeat(cfg, supervisor) {@@ -77,6 +90,9 @@ async function heartbeat(cfg, supervisor) {providerChain: live.providerLanes.map((l) => ({ lane: l.lane, code: l.code })),daemonRestarts: supervisor.stats().restarts,checkedAt: live.checkedAt,+ // Marks this host as lease-aware. A successor treats a beat WITHOUT this+ // field as a host that predates the lease and waits it out (lease.js).+ leaseHolder: lease?.state.holder ?? null,};try {await sql`@@ -242,6 +258,58 @@ async function main() {});}+ // The health server comes up first, before the lease wait. Cloud Run's+ // startup probe has to pass for a rollout to move traffic and SIGTERM the old+ // instance, and the old instance releasing the lease is exactly what this one+ // is waiting for: holding the probe behind the lease would deadlock the deploy.+ const supervisor = createSupervisor(cfg, p);+ const server = startHealthServer(cfg, live, supervisor, BOOT_AT);++ // A SIGTERM that lands while waiting has nothing to save: this host never+ // restored, ran or snapshotted anything.+ const waitAbort = new AbortController();+ const onEarlySignal = (signal) => {+ log.info('shutdown before taking the lease', { signal });+ waitAbort.abort();+ server?.close();+ process.exit(0);+ };+ process.once('SIGTERM', onEarlySignal);+ process.once('SIGINT', onEarlySignal);++ if (cfg.leaseRequired) {+ lease = createLease({+ store: sqlLeaseStore(sql),+ host: cfg.host,+ ttlMs: cfg.leaseTtlMs,+ renewMs: cfg.leaseRenewMs,+ pollMs: cfg.leasePollMs,+ legacyStaleMs: cfg.legacyBeatStaleMs,+ onLost: (reason) => void fence(cfg, supervisor, reason),+ });+ live.lease = lease.state;+ live.verdict = {+ status: 'unknown',+ ready: false,+ reason: 'lease_wait',+ detail: 'waiting for the single-writer lease before restoring the bot identity',+ needsHumanLogin: false,+ };+ const got = await lease.acquire({+ signal: waitAbort.signal,+ onWait: (reason) => {+ live.verdict = { ...live.verdict, detail: `waiting for the single-writer lease: ${reason}` };+ },+ });+ if (!got) return;+ lease.start();+ live.verdict = { status: 'unknown', ready: false, reason: 'booting', detail: 'lease held, restoring state', needsHumanLogin: false };+ } else {+ log.warn('no state bucket: running without the single-writer lease (fine for a local run, never for Cloud Run)');+ }+ process.off('SIGTERM', onEarlySignal);+ process.off('SIGINT', onEarlySignal);+await mkdir(p.agentTask, { recursive: true });live.stateRestore = await restoreState(cfg);await mkdir(p.logs, { recursive: true });@@ -261,9 +329,7 @@ async function main() {const codexLogin = await loginCodex(cfg);if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);- const supervisor = createSupervisor(cfg, p);supervisor.start();- const server = startHealthServer(cfg, live, supervisor, BOOT_AT);const probe = () => probeSession(cfg, supervisor).catch((err) => log.error('probe failed', { err: err?.message }));const probeTimer = setInterval(probe, cfg.sessionProbeMs);@@ -275,10 +341,10 @@ async function main() {() => heartbeat(cfg, supervisor).catch((err) => log.warn('heartbeat failed', { err: err?.message })),cfg.heartbeatMs,);- const snapshotTimer = setInterval(- () => snapshotState(cfg, { reason: 'timer' }).catch(() => {}),- cfg.snapshotMs,- );+ const snapshotTimer = setInterval(() => {+ if (!mayWrite()) return log.warn('snapshot skipped: the single-writer lease is not fresh', {});+ snapshotState(cfg, { reason: 'timer' }).catch(() => {});+ }, cfg.snapshotMs);const providerProbeTimer = setInterval(() => electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message })),cfg.providerProbeMs,@@ -303,9 +369,13 @@ async function main() {clearInterval(providerProbeTimer);server?.close();// Order matters: stop the daemon FIRST so its sqlite files are quiesced,- // then snapshot. A live-copy snapshot can tear; this one cannot.+ // then snapshot. A live-copy snapshot can tear; this one cannot. The+ // snapshot is written only while the lease is still ours, and the lease is+ // released only after it, so the successor restores exactly this state.await supervisor.stop();- await snapshotState(cfg, { reason: signal });+ if (mayWrite()) await snapshotState(cfg, { reason: signal });+ else log.error('final snapshot skipped: the single-writer lease is not held', { signal });+ await lease?.release();log.info('bye', {});process.exit(0);};@@ -314,6 +384,27 @@ async function main() {process.on('unhandledRejection', (err) => log.error('unhandledRejection', { err: err?.message }));}+/**+ * The lease is gone (another host took it, or it could not be renewed before it+ * would expire). Another host may already be writing the identity, so this one+ * stops its daemon and exits WITHOUT a snapshot: writing one now is exactly the+ * clobber the lease exists to prevent. Cloud Run restarts the container, and the+ * fresh process waits for the lease like any other.+ */+let fencing = false;+async function fence(cfg, supervisor, reason) {+ if (fencing) return;+ fencing = true;+ live.verdict = { status: 'down', ready: false, reason: 'lease_lost', detail: `single-writer lease lost: ${reason}`, needsHumanLogin: false };+ await sendOpsAlert(+ 'OKX chat bot fenced itself',+ `agent #${cfg.agentId} on ${cfg.host} lost the single-writer lease (${reason}) and stopped its daemon without snapshotting. It restarts and waits for the lease; no action is needed unless this repeats.`,+ { severity: 'warn' },+ ).catch(() => {});+ await supervisor.stop(5_000).catch(() => {});+ process.exit(75);+}+main().catch((err) => {log.error('fatal', { err: err?.message, stack: err?.stack });process.exit(1);diff --git a/workers/okx-chat-bot/lease.js b/workers/okx-chat-bot/lease.jsnew file mode 100644index 000000000..2b9a0d9af--- /dev/null+++ b/workers/okx-chat-bot/lease.js@@ -0,0 +1,264 @@+// okx-chat-bot: the single-writer lease.+//+// The bot's identity (the wallet keyring plus the XMTP client database) is one+// state object with exactly one legal writer. `--max-instances=1` was the only+// thing enforcing that, and it does not: Cloud Run applies max instances PER+// REVISION, and a rollout deliberately runs the new revision's instance next to+// the old one until the new one is Ready and traffic has moved. For that window+// two daemons would hold the same XMTP installation, and the old instance's+// SIGTERM snapshot would then overwrite whatever the new one had written. A torn+// identity costs a human email OTP to recover.+//+// So a host must hold this lease before it restores state, starts the daemon or+// writes a snapshot, and must keep renewing it to keep doing any of those. The+// lease is a row in the `bot_heartbeat` table the worker already writes (worker+// = LEASE_KEY), so it needs no new table, no migration and no new credential.+// Every transition is one conditional statement, so two hosts racing for it+// cannot both win.+//+// A rollout therefore sequences itself:+//+// 1. The new instance boots, serves /healthz (so Cloud Run's startup probe+// passes and the rollout can proceed) and waits, reporting `lease_wait`.+// 2. Cloud Run moves traffic and SIGTERMs the old instance, which stops its+// daemon, writes its final snapshot while it still holds the lease, and+// releases it.+// 3. The new instance takes the lease, restores THAT snapshot, and only then+// starts its daemon.+//+// A host that dies without releasing is waited out for LEASE_TTL. A host that+// cannot renew (its database is unreachable) fences itself before the TTL can+// expire under it: it stops the daemon and exits WITHOUT snapshotting, because+// by then another host may legitimately be writing.+//+// Hosts built before this lease existed never take it. They are recognised by+// their heartbeat, which carries no `leaseHolder`: while such a host is still+// beating, nobody may take the lease. That is what makes the first rollout onto+// this code as safe as every later one.++import { randomUUID } from 'node:crypto';+import { log } from './log.js';++export const LEASE_KEY = 'okx-chat-bot:lease';+const HEARTBEAT_WORKER = 'okx-chat-bot';++/**+ * The lease's storage: one `bot_heartbeat` row, driven by conditional upserts.+ * Every clock comparison uses the database's now(), so two hosts with skewed+ * clocks still agree on whether the lease has expired.+ *+ * @param {Function} sql tagged-template client (api/_lib/db.js)+ * @param {string} [key]+ */+export function sqlLeaseStore(sql, key = LEASE_KEY) {+ return {+ /** Take the lease if it is free, expired, released, or already ours. */+ async acquire(holder, ttlSec, meta = {}) {+ const row = JSON.stringify({ ...meta, holder });+ const won = await sql`+ INSERT INTO bot_heartbeat (worker, mode, last_beat_at, meta)+ VALUES (${key}, 'lease', now(), ${row}::jsonb)+ ON CONFLICT (worker) DO UPDATE+ SET last_beat_at = now(), meta = excluded.meta+ WHERE bot_heartbeat.meta->>'holder' = ${holder}+ OR bot_heartbeat.meta->>'released' = 'true'+ OR bot_heartbeat.last_beat_at < now() - (${ttlSec}::int * interval '1 second')+ RETURNING worker+ `;+ if (won.length) return { acquired: true };+ const [cur] = await sql`+ SELECT meta, extract(epoch FROM now() - last_beat_at)::float8 AS age_s+ FROM bot_heartbeat WHERE worker = ${key}+ `;+ return {+ acquired: false,+ holder: cur?.meta?.holder ?? null,+ host: cur?.meta?.host ?? null,+ ageSec: cur ? Number(cur.age_s) : null,+ };+ },+ /** Extend the lease. False means it is no longer ours. */+ async renew(holder) {+ const rows = await sql`+ UPDATE bot_heartbeat SET last_beat_at = now()+ WHERE worker = ${key} AND meta->>'holder' = ${holder}+ AND coalesce(meta->>'released', '') <> 'true'+ RETURNING worker+ `;+ return rows.length > 0;+ },+ /** Hand the lease to whoever is waiting, without waiting out the TTL. */+ async release(holder) {+ await sql`+ UPDATE bot_heartbeat+ SET meta = meta || '{"released": true}'::jsonb, last_beat_at = now()+ WHERE worker = ${key} AND meta->>'holder' = ${holder}+ `;+ },+ /** The serving host's own heartbeat, for recognising a pre-lease host. */+ async hostBeat() {+ const [row] = await sql`+ SELECT meta->>'host' AS host, meta->>'leaseHolder' AS lease_holder,+ extract(epoch FROM now() - last_beat_at)::float8 AS age_s+ FROM bot_heartbeat WHERE worker = ${HEARTBEAT_WORKER}+ `;+ return row ? { host: row.host ?? null, leaseAware: !!row.lease_holder, ageSec: Number(row.age_s) } : null;+ },+ };+}++/**+ * Does a host that predates the lease still look alive?+ *+ * Returns the reason to keep waiting, or null when the heartbeat is no obstacle:+ * absent, stale, written by a lease-aware host (the lease itself then governs),+ * or written by this same host label (a previous process of this revision, which+ * is dead: a revision never runs two instances).+ *+ * @param {{ host: string|null, leaseAware: boolean, ageSec: number }|null} beat+ * @param {{ selfHost: string, staleSec: number }} opts+ * @returns {string|null}+ */+export function legacyWriterBlocks(beat, { selfHost, staleSec }) {+ if (!beat) return null;+ if (beat.leaseAware) return null;+ if (beat.host && beat.host === selfHost) return null;+ if (!Number.isFinite(beat.ageSec) || beat.ageSec >= staleSec) return null;+ return `${beat.host || 'an unnamed host'} predates the single-writer lease and beat ${Math.round(beat.ageSec)}s ago`;+}++/**+ * @param {object} opts+ * @param {ReturnType<typeof sqlLeaseStore>} opts.store+ * @param {string} opts.host this host's label (config.resolveHost)+ * @param {number} opts.ttlMs how long an unrenewed lease stays taken+ * @param {number} opts.renewMs renewal cadence; must be well inside ttlMs+ * @param {number} opts.pollMs how often a waiting host asks again+ * @param {number} opts.legacyStaleMs how old a pre-lease host's beat must be to ignore it+ * @param {(reason: string) => void} [opts.onLost] called once when the lease is gone+ * @param {() => number} [opts.now]+ * @param {(ms: number) => Promise<void>} [opts.sleep]+ * @param {string} [opts.holder]+ */+export function createLease({+ store,+ host,+ ttlMs,+ renewMs,+ pollMs,+ legacyStaleMs,+ onLost = () => {},+ now = Date.now,+ sleep = (ms) => new Promise((r) => setTimeout(r, ms)),+ holder = `${host}#${randomUUID()}`,+}) {+ if (!(renewMs * 2 < ttlMs)) throw new Error(`lease renewMs (${renewMs}) must be under half of ttlMs (${ttlMs})`);+ const ttlSec = Math.ceil(ttlMs / 1000);+ const state = { holder, held: false, acquiredAt: 0, lastRenewAt: 0, waitingOn: null, lostReason: null };+ let timer = null;+ let renewing = false;+ let lostFired = false;++ function lose(reason) {+ if (lostFired) return;+ lostFired = true;+ state.held = false;+ state.lostReason = reason;+ if (timer) clearInterval(timer);+ timer = null;+ log.error('single-writer lease lost', { holder, reason });+ onLost(reason);+ }++ async function renewOnce() {+ if (!state.held || renewing) return;+ renewing = true;+ try {+ if (await store.renew(holder)) state.lastRenewAt = now();+ else lose('another host holds the lease');+ } catch (err) {+ // A database blip is not a lost lease, but an unrenewable one is about to+ // expire under us. Fence before it can: the renewal margin is exactly the+ // time left before another host may legitimately take it.+ const silentMs = now() - state.lastRenewAt;+ log.warn('lease renew failed', { holder, err: err?.message, silentMs });+ if (silentMs >= ttlMs - renewMs) lose(`could not renew for ${Math.round(silentMs / 1000)}s`);+ } finally {+ renewing = false;+ }+ }++ return {+ state,++ /**+ * Block until this host holds the lease. Resolves false only when aborted.+ * @param {{ signal?: AbortSignal, onWait?: (reason: string) => void }} [opts]+ */+ async acquire({ signal, onWait = () => {} } = {}) {+ let lastLogged = null;+ while (!signal?.aborted) {+ let reason = null;+ try {+ reason = legacyWriterBlocks(await store.hostBeat(), { selfHost: host, staleSec: legacyStaleMs / 1000 });+ if (!reason) {+ const r = await store.acquire(holder, ttlSec, { host, since: new Date(now()).toISOString() });+ if (r.acquired) {+ state.held = true;+ state.acquiredAt = now();+ state.lastRenewAt = now();+ state.waitingOn = null;+ log.info('single-writer lease acquired', { holder });+ return true;+ }+ reason = `${r.host || r.holder || 'another host'} holds the lease (renewed ${Math.round(r.ageSec ?? 0)}s ago)`;+ }+ } catch (err) {+ reason = `lease store unreachable: ${err?.message || err}`;+ }+ state.waitingOn = reason;+ onWait(reason);+ if (reason !== lastLogged) {+ log.warn('waiting for the single-writer lease', { holder, reason });+ lastLogged = reason;+ }+ await sleep(pollMs);+ }+ return false;+ },++ /** Start renewing. Call once, right after acquire() resolves true. */+ start() {+ if (timer || !state.held) return;+ timer = setInterval(() => void renewOnce(), renewMs);+ if (typeof timer.unref === 'function') timer.unref();+ },++ /** Exposed for tests and for a caller that wants one renewal now. */+ renewOnce,++ /**+ * True while this host may write shared state. Stricter than `held`: a+ * lease whose last renewal is older than the fencing margin is treated as+ * already gone, so no snapshot is ever written on a lease that may have+ * expired in the database.+ */+ fresh() {+ return state.held && now() - state.lastRenewAt < ttlMs - renewMs;+ },++ /** Stop renewing and hand the lease over. Best effort: a failure just leaves it to expire. */+ async release() {+ if (timer) clearInterval(timer);+ timer = null;+ if (!state.held) return;+ state.held = false;+ try {+ await store.release(holder);+ log.info('single-writer lease released', { holder });+ } catch (err) {+ log.warn('lease release failed: it will expire on its TTL', { holder, err: err?.message });+ }+ },+ };+}diff --git a/workers/okx-chat-bot/session.js b/workers/okx-chat-bot/session.jsindex 5fc154732..917b42c46 100644--- a/workers/okx-chat-bot/session.js+++ b/workers/okx-chat-bot/session.js@@ -46,6 +46,12 @@ export function providerInstructions(detail = '', lanes = []) {`The provider refused this host's credential: ${detail}`,...chain,'',+ '0. Payment-free: the anthropic-gateway lane on three.ws\'s own proxy',+ ' (/api/llm/anthropic/agents/<agent>, the free model chain). If it is in the',+ ' chain above and refused, `npm run okx:bot:gateway -- --verify` says why;',+ ' `-- --apply --rotate` re-mints its key; the API must be deployed with the',+ ' agent-scoped SDK path (`npm run deploy:gcp:full`). If it is missing from',+ ' the chain, redeploy the worker: `npm run okx:bot:deploy`.','1. Preferred (no secret): clear the project billing hold, then',' gcloud run services update okx-chat-bot --region us-central1 \\'," --update-env-vars=CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=aerial-vehicle-466722-p5,CLOUD_ML_REGION=global",@@ -56,8 +62,9 @@ export function providerInstructions(detail = '', lanes = []) {' stripped by the next deploy).','3. Or reactivate billing on the OpenAI account behind the openai-api-key secret',' and pin OKX_BOT_AI_PROVIDER=codex.',- '4. Or point the host at any funded Anthropic-wire-format gateway (config only,',- ' pre-approved). OpenRouter serves one at /api/v1/messages:',+ '4. Or point the gateway lane at a funded third-party Anthropic-wire-format',+ ' gateway instead (config only, pre-approved). OpenRouter serves one at',+ ' /api/v1/messages:',' gcloud run services update okx-chat-bot --region us-central1 \\',' --update-env-vars=OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6 \\',' --update-secrets=OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest',