Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(okx-chat-bot): elect a funded AI lane instead of stalling on a denied Vertex grant
nirholasad723e87fa69
diff --git a/tests/okx-chat-bot.test.js b/tests/okx-chat-bot.test.jsindex 1fb2bcbff..5782c28d1 100644--- a/tests/okx-chat-bot.test.js+++ b/tests/okx-chat-bot.test.js@@ -16,8 +16,9 @@ import { join } from 'node:path';import { fileURLToPath } from 'node:url';import { afterAll, describe, it, expect, vi } from 'vitest';import { classify, loginInstructions, providerInstructions } from '../workers/okx-chat-bot/session.js';-import { classifyProbeStatus, loginCodex } from '../workers/okx-chat-bot/provider.js';-import { resolveProvider, resolveHost, loadConfig, paths } from '../workers/okx-chat-bot/config.js';+import { classifyProbeStatus, electProvider, loginCodex, probeLane } from '../workers/okx-chat-bot/provider.js';+import { applyLane, providerLanes, resolveProviderChain, resolveHost, loadConfig, paths } from '../workers/okx-chat-bot/config.js';+import { cliEnv } from '../workers/okx-chat-bot/cli.js';import { createHealthHandler } from '../workers/okx-chat-bot/health-server.js';import { createSupervisor } from '../workers/okx-chat-bot/supervisor.js';import { STATE_ROOTS, STATE_EXCLUDES } from '../workers/okx-chat-bot/state.js';@@ -121,6 +122,8 @@ describe('okx-chat-bot provider selection', () => {// A host with no interactive CLI login, so these assertions read the env and// nothing else. mkdtemp gives a home that provably holds no credential file.const bare = mkdtempSync(join(tmpdir(), 'okx-bot-nohome-'));+ // The lane the host leads with: what the daemon is spawned on at boot.+ const resolveProvider = (env, home) => resolveProviderChain(env, home).head;it('prefers Claude when an Anthropic credential is present', () => {const r = resolveProvider({ ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y' }, bare);@@ -252,6 +255,145 @@ describe('okx-chat-bot provider credential probe', () => {});});+// A chain with one rung is not a chain. The host held exactly one AI lane until+// 2026-09-09, so when the GCP project's Vertex access started answering+// "Lightning dunning decision is deny" the bot received every buyer message and+// could author no reply at all, for days, while other credentials sat unused on+// the same service. These tests hold the line that whichever lane is funded is+// the lane that serves.+describe('okx-chat-bot provider chain', () => {+ const bare = mkdtempSync(join(tmpdir(), 'okx-bot-chain-'));+ const GATEWAY = {+ OKX_BOT_ANTHROPIC_BASE_URL: 'https://openrouter.ai/api',+ OKX_BOT_ANTHROPIC_AUTH_TOKEN: 'sk-or-test',+ OKX_BOT_ANTHROPIC_MODEL: 'anthropic/claude-sonnet-4.6',+ };++ afterAll(() => rmSync(bare, { recursive: true, force: true }));++ it('orders GCP credits ahead of every key, and a metered gateway last', () => {+ const lanes = providerLanes(+ {+ CLAUDE_CODE_USE_VERTEX: '1',+ ANTHROPIC_VERTEX_PROJECT_ID: 'aerial-vehicle-466722-p5',+ ANTHROPIC_API_KEY: 'sk-ant-x',+ OPENAI_API_KEY: 'sk-y',+ ...GATEWAY,+ },+ bare,+ );+ expect(lanes.map((l) => l.id)).toEqual(['vertex', 'anthropic-key', 'anthropic-gateway', 'openai-key']);+ });++ // A gateway bills a third-party account per token, so it must be a decision an+ // operator made rather than something the host picks up from a stray key.+ it('never invents a gateway lane from a half-set config', () => {+ expect(providerLanes({ OKX_BOT_ANTHROPIC_BASE_URL: 'https://openrouter.ai/api' }, bare)).toEqual([]);+ expect(providerLanes({ OKX_BOT_ANTHROPIC_AUTH_TOKEN: 'sk-or-test' }, bare)).toEqual([]);+ });++ // The container boots with CLAUDE_CODE_USE_VERTEX=1 baked in. Electing the+ // gateway without unsetting it spawns a CLI that still reaches for the dead+ // transport, which is the same silence with a different cause.+ it('unsets the Vertex transport when a gateway lane is elected', () => {+ const [lane] = providerLanes(GATEWAY, bare);+ const cfg = applyLane({ home: '/state' }, lane);+ const env = cliEnv(cfg);+ expect(env.ANTHROPIC_BASE_URL).toBe('https://openrouter.ai/api');+ expect(env.ANTHROPIC_MODEL).toBe('anthropic/claude-sonnet-4.6');+ expect('CLAUDE_CODE_USE_VERTEX' in env).toBe(false);+ });++ it('keeps a pin honest: it narrows the chain, it does not empty it', () => {+ const env = { ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y', ...GATEWAY, OKX_BOT_AI_PROVIDER: 'claude' };+ expect(resolveProviderChain(env, bare).lanes.map((l) => l.id)).toEqual(['anthropic-key', 'anthropic-gateway']);+ });++ it('probes each lane through its own env, not the ambient one', async () => {+ const seen = [];+ const fetchMock = vi.fn(async (url, init) => {+ seen.push({ url, auth: init.headers.authorization });+ return new Response('{}', { status: 200 });+ });+ vi.stubGlobal('fetch', fetchMock);+ const [lane] = providerLanes(GATEWAY, bare);+ const v = await probeLane(lane, { ANTHROPIC_API_KEY: 'sk-ant-ambient' });+ vi.unstubAllGlobals();+ expect(v).toMatchObject({ code: 'ok', lane: 'anthropic-gateway', transport: 'gateway' });+ expect(seen[0].url).toBe('https://openrouter.ai/api/v1/messages');+ expect(seen[0].auth).toBe('Bearer sk-or-test');+ });++ // Measured live on 2026-09-09: base URL `https://openrouter.ai/api/v1` makes the+ // CLI request `/api/v1/v1/messages`, which answers 404. classifyProbeStatus()+ // reads a 404 from a first-party provider as proof the credential works, and+ // that reading would have elected a lane that 404s every buyer reply.+ it('refuses to elect a gateway that only answered a complaint about the request', async () => {+ const [lane] = providerLanes(GATEWAY, bare);+ vi.stubGlobal('fetch', async () => new Response('No endpoints found', { status: 404 }));+ const v = await probeLane(lane, {});+ vi.unstubAllGlobals();+ expect(v.code).toBe('unauthorized');+ expect(v.detail).toContain('OKX_BOT_ANTHROPIC_BASE_URL');+ });++ it('elects the first lane that answers and stops paying for the rest', async () => {+ const chain = providerLanes(+ { CLAUDE_CODE_USE_VERTEX: '1', GOOGLE_CLOUD_PROJECT: 'p', ANTHROPIC_API_KEY: 'sk-ant-x', ...GATEWAY },+ bare,+ );+ const calls = [];+ vi.stubGlobal('fetch', async (url) => {+ calls.push(String(url));+ // Vertex is the dead lane; the Anthropic key answers.+ return String(url).includes('aiplatform')+ ? new Response('Lightning dunning decision is deny', { status: 403 })+ : new Response('{}', { status: 200 });+ });+ const r = await electProvider(chain, { ANTHROPIC_API_KEY: 'sk-ant-x' });+ vi.unstubAllGlobals();+ expect(r.elected.id).toBe('anthropic-key');+ expect(r.lanes.map((l) => l.code)).toEqual(['unauthorized', 'ok']);+ expect(calls.some((u) => u.includes('openrouter'))).toBe(false);+ });++ it('elects nothing when every lane is refused, and reports them all', async () => {+ const chain = providerLanes({ ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y' }, bare);+ vi.stubGlobal('fetch', async () => new Response('{"error":{"message":"no credit"}}', { status: 402 }));+ const r = await electProvider(chain, { ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y' });+ vi.unstubAllGlobals();+ expect(r.elected).toBe(null);+ expect(r.verdict.code).toBe('unauthorized');+ expect(r.lanes.map((l) => l.lane)).toEqual(['anthropic-key', 'openai-key']);+ });++ // An unprovable grant is not a refusal. The developer host runs on exactly+ // that, so calling it dead would take the stopgap offline for a lane that+ // demonstrably authors replies.+ it('serves on an unprovable lane rather than calling it refused', async () => {+ const home = mkdtempSync(join(tmpdir(), 'okx-bot-login-'));+ mkdirSync(join(home, '.claude'), { recursive: true });+ writeFileSync(join(home, '.claude', '.credentials.json'), '{}');+ const chain = providerLanes({ ANTHROPIC_API_KEY: 'sk-ant-x' }, home);+ vi.stubGlobal('fetch', async () => new Response('nope', { status: 401 }));+ const r = await electProvider(chain, { ANTHROPIC_API_KEY: 'sk-ant-x' });+ vi.unstubAllGlobals();+ rmSync(home, { recursive: true, force: true });+ expect(r.elected.id).toBe('anthropic-login');+ expect(r.verdict.code).toBe('unprobed');+ });++ it('puts every lane verdict on the status body a human funds from', () => {+ const lines = providerInstructions('dunning deny', [+ { lane: 'vertex', transport: 'vertex', code: 'unauthorized', detail: 'Lightning dunning decision is deny' },+ { lane: 'openai-key', transport: 'api-key', code: 'unauthorized', detail: 'billing_not_active' },+ ]).join('\n');+ expect(lines).toContain('vertex (vertex): unauthorized');+ expect(lines).toContain('openai-key (api-key): unauthorized - billing_not_active');+ expect(lines).toContain('OKX_BOT_ANTHROPIC_BASE_URL');+ });+});+describe('okx-chat-bot host reporting', () => {it('calls Cloud Run the durable host and names the revision', () => {const h = resolveHost({ K_SERVICE: 'okx-chat-bot', K_REVISION: 'okx-chat-bot-00001-abc' });@@ -516,6 +658,54 @@ describe('okx-chat-bot daemon supervision (smoke)', () => {await rm(home, { recursive: true, force: true });}});++ // A newly elected AI lane only reaches the spawned subsession through the+ // daemon's environment, which is fixed at spawn time, so the lane change is+ // not real until the daemon is replaced. The replacement must leave exactly+ // one daemon: the SIGTERM fires the exit handler's own restart as well, and+ // two daemons race for one lock and one XMTP identity.+ it('replaces the daemon exactly once when a lane change demands it', async () => {+ const home = await mkdtemp(join(tmpdir(), 'okx-bot-restart-'));+ const cfg = loadConfig({+ OKX_BOT_HOME: home,+ // node with a junk arg exits on its own terms; what is under test is that+ // a restart yields exactly one live child, not zero and not two.+ OKX_BOT_DAEMON_BIN: process.execPath,+ OKX_BOT_RESTART_BASE_MS: '20',+ OKX_BOT_RESTART_MAX_MS: '40',+ });+ const supervisor = createSupervisor(cfg, paths(cfg));+ try {+ supervisor.start();+ const first = supervisor.stats().pid;+ expect(first).toBeTruthy();+ await supervisor.restart('test', 1_000);+ const second = supervisor.stats().pid;+ expect(second).toBeTruthy();+ expect(second).not.toBe(first);+ } finally {+ await supervisor.stop(1_000);+ await rm(home, { recursive: true, force: true });+ }+ });++ // A shutdown that lands mid-restart must not be undone by the restart's own+ // respawn: the final snapshot runs after stop(), and a daemon brought back+ // behind it would be copied live.+ it('does not resurrect the daemon when a shutdown lands mid-restart', async () => {+ const home = await mkdtemp(join(tmpdir(), 'okx-bot-restart-race-'));+ const cfg = loadConfig({ OKX_BOT_HOME: home, OKX_BOT_DAEMON_BIN: process.execPath });+ const supervisor = createSupervisor(cfg, paths(cfg));+ try {+ supervisor.start();+ const restarting = supervisor.restart('test', 1_000);+ await supervisor.stop(1_000);+ await restarting;+ expect(supervisor.stats().pid).toBe(null);+ } finally {+ await rm(home, { recursive: true, force: true });+ }+ });});describe('okx_chat_bot subsystem health', () => {diff --git a/workers/okx-chat-bot/README.md b/workers/okx-chat-bot/README.mdindex 34e767787..31ce96459 100644--- a/workers/okx-chat-bot/README.md+++ b/workers/okx-chat-bot/README.md@@ -46,9 +46,9 @@ testable without a daemon, a wallet, or a network.| File | Role ||------|------|| `index.js` | Entrypoint. Boot order, session probe, heartbeat, ops alerts, graceful shutdown. |-| `config.js` | Env-driven config (`loadConfig`, `paths`) and AI-provider selection (`resolveProvider`). |+| `config.js` | Env-driven config (`loadConfig`, `paths`) and the AI lane chain (`providerLanes`, `resolveProviderChain`, `applyLane`). || `cli.js` | Timeout-bounded, non-throwing wrappers around the `okx-a2a` and `onchainos` binaries. |-| `provider.js` | Asks the AI provider whether it will actually serve this host, and hands codex its key. |+| `provider.js` | Probes each AI lane (`probeLane`), elects the first that serves (`electProvider`), and hands codex its key. || `session.js` | Pure health `classify()` plus `loginInstructions()`, the exact commands a human runs. || `health-server.js` | The HTTP surface: strict `/readyz`, always-200 liveness, and the `remedy` payload. || `state.js` | Tar the wallet/XMTP identity to GCS and restore it on boot (`snapshotState`, `restoreState`). |@@ -102,12 +102,15 @@ locally with no code change. Defaults are the production posture.| `OKX_BOT_STATE_BUCKET` | unset | GCS bucket for the state snapshot. Unset means ephemeral mode. || `OKX_BOT_STATE_OBJECT` | `okx-chat-bot/state.tar.gz` | Object name within that bucket. || `OKX_BOT_REPO_ROOT` | `/app` | Where the briefing and skills are read from. |-| `OKX_BOT_AI_PROVIDER` | auto | Pin the provider (`claude`, `codex`, `hermes`, `openclaw`). |+| `OKX_BOT_AI_PROVIDER` | auto | Pin the provider (`claude`, `codex`, `hermes`, `openclaw`). Narrows the chain to that CLI's lanes rather than emptying it. || `CLAUDE_CODE_USE_VERTEX` | unset | `1` routes the Claude subsession to Vertex AI, authenticated by the runtime service account. The production posture: no secret exists to leak, rotate, or forget. || `ANTHROPIC_VERTEX_PROJECT_ID` | unset | The GCP project Vertex bills. Required alongside the flag above. || `CLOUD_ML_REGION` | `global` | Vertex region for both the subsession and the credential probe. || `OKX_A2A_AI_PERMISSION_PRESET` | unset | `bypass` on a headless host. Without it the subsession stalls on a tool-approval prompt nobody is there to answer, which the buyer experiences as an unresponsive bot. |-| `OKX_BOT_PROVIDER_PROBE_MS` | `900000` | How often the provider's own API is asked whether the credential still works. |+| `OKX_BOT_ANTHROPIC_BASE_URL` | unset | Base URL of an Anthropic-wire-format gateway. The CLI appends `/v1/messages`, so OpenRouter's is `https://openrouter.ai/api`, **not** `.../api/v1`. |+| `OKX_BOT_ANTHROPIC_AUTH_TOKEN` | unset | That gateway's credential. Both it and the base URL are required, or no gateway lane exists. |+| `OKX_BOT_ANTHROPIC_MODEL` | unset | Model id in the gateway's catalog, e.g. `anthropic/claude-sonnet-4.6`. |+| `OKX_BOT_PROVIDER_PROBE_MS` | `900000` | How often every lane's own API is asked whether it will still serve. Also how quickly a recovered lane is picked up. || `OKX_BOT_HOST_LABEL` | auto | Name this host on every beat. Cloud Run names itself from `K_SERVICE`. || `OKX_BOT_HOST_DURABLE` | unset | Set to `1` to claim a non-Cloud-Run host stays up on its own. || `OKX_BOT_DAEMON_BIN` | `okx-a2a` | The XMTP daemon binary the supervisor owns. |@@ -118,15 +121,40 @@ locally with no code change. Defaults are the production posture.| `OKX_BOT_RESTART_BASE_MS` | `2000` | Daemon restart backoff floor. || `OKX_BOT_RESTART_MAX_MS` | `60000` | Daemon restart backoff ceiling. |-**Provider selection is by credential, not by preference.** A provider CLI with-no key spawns, fails to authenticate, and produces exactly the symptom this-worker exists to kill: silence on the buyer's side. So an explicit-`OKX_BOT_AI_PROVIDER` wins; otherwise Vertex (`CLAUDE_CODE_USE_VERTEX=1` plus a-project) selects `claude`, then `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`-selects `claude`, then `OPENAI_API_KEY` selects `codex`. A developer host carries-no key at all: its claude CLI was logged in by a human, so an existing-`$OKX_BOT_HOME/.claude/.credentials.json` also counts as a credential. With no-credential at all the worker boots, logs an error, and reports+### The AI chain: whichever lane is funded is the lane that serves++The host used to hold exactly **one** AI lane, which is a chain with a single+rung. On 2026-09-04 the rung snapped: this GCP project's Vertex access started+answering `PERMISSION_DENIED: Lightning dunning decision is deny`, and the bot+went on receiving every buyer message and authoring no reply at all, for days,+while other credentials sat unused on the same service.++So [`providerLanes()`](config.js) builds an ordered chain and+[`electProvider()`](provider.js) probes it, best lane first, and elects the first+one that actually answers. The election re-runs every+`OKX_BOT_PROVIDER_PROBE_MS`, so a lane that recovers (or one the owner funds) is+picked up **with no deploy**: the config is re-asserted, codex is re-logged-in if+it is the new lane, and the daemon is restarted so the subsession it spawns+carries the new credentials.++| # | Lane | Selected by | Why here |+|---|---|---|---|+| 1 | `vertex` | `CLAUDE_CODE_USE_VERTEX=1` + a project | GCP credits, authenticated by the runtime service account. Nothing to mint, rotate or forget, and the spend lands on the pool the platform prefers over any paid third-party API. |+| 2 | `anthropic-key` | `ANTHROPIC_API_KEY` | A first-party key. |+| 3 | `anthropic-gateway` | `OKX_BOT_ANTHROPIC_BASE_URL` + `..._AUTH_TOKEN` | Any service that speaks the Anthropic wire format (OpenRouter serves one). Opt-in, and last, because it bills a third-party account per token. |+| 4 | `anthropic-login` | `$OKX_BOT_HOME/.claude/.credentials.json` | A developer host, whose `claude` CLI a human logged in. |+| 5 | `openai-key` | `OPENAI_API_KEY` | The codex CLI. |++**Every lane spawns a genuinely agentic CLI, and that is not negotiable.** The+adapter does not read a reply out of the CLI's stdout: the spawned subsession+sends the reply itself through `okx-a2a` and drives the task lifecycle (accept /+negotiate / deliver). A one-shot completion call would not just answer worse, it+would break the lifecycle. The gateway lane exists precisely because it keeps the+same `claude` CLI and only moves its endpoint.++A provider CLI with no key spawns, fails to authenticate, and produces exactly+the symptom this worker exists to kill: silence on the buyer's side. So with an+empty chain the worker boots, logs an error, and reports`ai_provider_uncredentialed` rather than pretending to be healthy.**Vertex wins over every key on purpose.** The credential is the runtime service@@ -144,13 +172,27 @@ reads like an IAM problem) and the `openai-api-key` secret's account answers`billing_not_active`. A presence check calls either one green, the host reportsready, a buyer's message lands, and no reply is ever authored.-So [provider.js](provider.js) asks the provider itself: one tiny request at boot+So [provider.js](provider.js) asks each lane's own API: one tiny request at bootand every `OKX_BOT_PROVIDER_PROBE_MS`, classified into `ok`, `unauthorized`,`unreachable`, or `unprobed`. Only `unauthorized` fails readiness. `unreachable`deliberately does not: a provider outage is transient and self-heals, and paginga human for something no human can fix is how alerts stop being read. Aninteractive OAuth grant is `unprobed`, because the CLI refreshes that grant-itself and no endpoint here can prove it without reimplementing the refresh.+itself and no endpoint here can prove it without reimplementing the refresh, and+`unprobed` is the one non-`ok` verdict a host may still be elected on.++Every lane's verdict, not just the elected one's, is on `/readyz` under+`provider.chain` and in the `remedy`. A human deciding which credential to fund+needs the whole picture, not the news that one of them was refused.++**A gateway lane is elected on a 2xx and nothing else.** `classifyProbeStatus()`+reads a 400 or a 404 as proof the credential works, because api.anthropic.com had+to authenticate the caller before it could object to the request. That reasoning+does not survive a gateway: a base URL one path segment too deep, or a model id+from a different catalog, answers exactly those statuses and then fails every+real reply. Measured 2026-09-09, `https://openrouter.ai/api/v1` makes the CLI+request `/api/v1/v1/messages` and answers 404 forever, which is why the base URL+in [cloudbuild.yaml](cloudbuild.yaml) ends at `/api`.**One thing about Vertex that could not be tested here.** The billing hold deniesevery Vertex call on this project, so the credential probe was verified against@@ -297,15 +339,25 @@ private bucket and nowhere else.Skipping the seed is not a failure; it just leaves the OTP the first boot wouldotherwise have asked for.-### Switching off Vertex+### Adding a laneNever patch an AI key onto the service by hand. `--set-secrets` in the deploy stepreplaces the whole secret set, so a hand-added key survives exactly until the next-deploy and then vanishes without a single error line. To use a key instead of-Vertex, edit [cloudbuild.yaml](cloudbuild.yaml): drop `CLAUDE_CODE_USE_VERTEX`-from `--set-env-vars` (it wins over every key) and add-`ANTHROPIC_API_KEY=anthropic-api-key:latest` to `--set-secrets`, after creating-that secret and granting `three-ws@` `secretAccessor` on it.+deploy and then vanishes without a single error line. A lane belongs in+[cloudbuild.yaml](cloudbuild.yaml), where the next deploy carries it too:++- **A first-party Anthropic key**: create the secret, grant `three-ws@`+ `secretAccessor` on it, and add `ANTHROPIC_API_KEY=anthropic-api-key:latest` to+ `--set-secrets`. It ranks below Vertex, so nothing else has to change; drop+ `CLAUDE_CODE_USE_VERTEX` only if you want the key to lead.+- **A gateway**: set `OKX_BOT_ANTHROPIC_BASE_URL` and `OKX_BOT_ANTHROPIC_MODEL` in+ `--set-env-vars` and `OKX_BOT_ANTHROPIC_AUTH_TOKEN=<secret>:latest` in+ `--set-secrets`. The deploy already carries the OpenRouter one.++Because the chain elects on a live probe rather than on presence, a lane with no+funds behind it is inert (it probes `unauthorized` and is never elected) and+costs nothing to leave configured. That is the point: funding it is then a+billing action, not a deploy.Shutdown order matters and is handled on SIGTERM: the daemon is stopped **before**the final snapshot, so the sqlite files are quiesced rather than copied mid-write.diff --git a/workers/okx-chat-bot/cli.js b/workers/okx-chat-bot/cli.jsindex b4c655f3f..fc972927d 100644--- a/workers/okx-chat-bot/cli.js+++ b/workers/okx-chat-bot/cli.js@@ -20,7 +20,7 @@ const DEFAULT_TIMEOUT_MS = 60_000;* @param {ReturnType<import('./config.js').loadConfig>} cfg*/export function cliEnv(cfg) {- return {+ const env = {...process.env,HOME: cfg.home,// The onchainos installer drops its binary in $HOME/.local/bin, but the@@ -28,6 +28,14 @@ export function cliEnv(cfg) {// replaces $HOME. Keep both on PATH so a local run works either way.PATH: `${cfg.home}/.local/bin:${process.env.PATH || ''}`,};+ // The elected AI lane's overlay. A `null` value UNSETS the variable, which is+ // the only way a gateway lane can stop the spawned CLI from still reaching for+ // the Vertex transport this container was booted with.+ for (const [k, v] of Object.entries(cfg.activeLane?.env || {})) {+ if (v === null || v === undefined) delete env[k];+ else env[k] = String(v);+ }+ return env;}/**diff --git a/workers/okx-chat-bot/cloudbuild.yaml b/workers/okx-chat-bot/cloudbuild.yamlindex 768de0e1d..7f9352187 100644--- a/workers/okx-chat-bot/cloudbuild.yaml+++ b/workers/okx-chat-bot/cloudbuild.yaml@@ -110,15 +110,24 @@ steps:# there to answer, and the buyer experiences that as an unresponsive bot.# CLAUDE_CODE_USE_VERTEX points the subsession at Vertex AI, authenticated# by three-ws@ through ADC (roles/aiplatform.user, already granted).- - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global+ #+ # OKX_BOT_ANTHROPIC_* is the last rung of the AI chain (providerLanes in+ # config.js): an Anthropic-wire-format gateway the same `claude` CLI can be+ # pointed at, so a lane change never costs the task lifecycle. It is here+ # rather than in a runbook because the whole point of the chain is that a+ # funded lane is used WITHOUT a deploy: while the OpenRouter account has no+ # credit this lane simply probes 402 and is never elected, and the minute it+ # is funded the next 15-minute probe elects it on its own. The base URL is+ # the prefix the CLI appends /v1/messages to, so it ends at /api, NOT /api/v1+ # (measured 2026-09-09: one segment too deep answers 404 on every reply).+ - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global,OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6# An API key, if one is ever preferred over Vertex, belongs HERE and not# patched on afterwards: --set-secrets replaces the whole secret set, so a# hand-added key is silently stripped by the next deploy. Add# ANTHROPIC_API_KEY=anthropic-api-key:latest (or, for codex,- # OPENAI_API_KEY=openai-api-key:latest plus OKX_BOT_AI_PROVIDER=codex above)- # once the secret exists; resolveProvider in config.js prefers Vertex over- # both, so drop CLAUDE_CODE_USE_VERTEX when you mean to use the key.- - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest+ # OPENAI_API_KEY=openai-api-key:latest) once the secret exists; the chain+ # ranks Vertex above both, and pinning is only needed to force one lane.+ - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest,OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest# Liveness only. Readiness is deliberately NOT wired to Cloud Run: a# logged-out session fails /readyz and no restart can fix it, so probing it# here would put the service in a crash loop instead of paging a human.diff --git a/workers/okx-chat-bot/config.js b/workers/okx-chat-bot/config.jsindex 1a68d8779..4025b817c 100644--- a/workers/okx-chat-bot/config.js+++ b/workers/okx-chat-bot/config.js@@ -14,83 +14,193 @@ const num = (v, fallback) => {};/**- * Pick the AI provider the okx-a2a adapter should spawn for chat replies, and- * say which transport authenticates it.+ * Every AI lane this host could authenticate through, best first.*- * The adapter does NOT read a reply out of the CLI's stdout, the spawned AI- * subsession sends the reply itself through the `okx-a2a` CLI and drives the- * task lifecycle (accept / negotiate / deliver). So the provider must be a- * genuinely agentic CLI with tool access, not a one-shot completion call.+ * There used to be exactly one. That is a chain with a single rung, and on+ * 2026-09-04 the rung snapped: the GCP project's Vertex access started answering+ * `PERMISSION_DENIED: Lightning dunning decision is deny` (a billing hold that+ * reads like an IAM fault) and the host sat on that one dead lane, unable to+ * author a single reply, while other credentials the project holds went+ * unconsidered. Whichever lane is funded should be the lane that serves, and the+ * host should move to it by itself.*- * Selection is by credential, because a provider CLI with no key spawns, fails- * to authenticate, and produces the exact symptom this worker exists to kill:- * silence on the buyer's side.+ * Order is a policy, not a preference:*- * OKX_BOT_AI_PROVIDER pin explicitly (claude | codex | hermes | openclaw)- * CLAUDE_CODE_USE_VERTEX=1 → claude on Vertex AI, authenticated by the- * runtime service account through ADC. No secret- * exists to leak, rotate, or forget, and the spend- * lands on the GCP credit pool the platform already- * prefers, so this transport wins over every key.- * ANTHROPIC_API_KEY /- * CLAUDE_CODE_OAUTH_TOKEN → claude on api.anthropic.com- * OPENAI_API_KEY → codex+ * vertex GCP credits, authenticated by the runtime service account.+ * No secret exists to leak, rotate or forget, and the spend+ * lands on the pool the platform already prefers.+ * anthropic-key a first-party ANTHROPIC_API_KEY.+ * anthropic-gateway any Anthropic-wire-format gateway (OpenRouter serves one+ * at /api/v1/messages). Opt-in: it only exists when an+ * operator sets its base URL and token, because it bills a+ * third-party account per token.+ * anthropic-oauth an interactive `claude` login on a developer host.+ * openai-key the codex CLI on OPENAI_API_KEY.*- * A headless host is credentialed by env alone, but a developer host running the- * stopgap has no key at all: its claude CLI was logged in interactively and keeps- * the grant in ~/.claude/.credentials.json. Ignoring that file reports- * `ai_provider_uncredentialed` on a host whose adapter demonstrably authors- * replies, which is a false red, and a false red trains people to ignore the- * signal that this worker exists to raise.- *- * `credentialed` only answers "is a credential configured". Whether that- * credential still WORKS is a different question, and on this project both- * answers have differed: the GCP project's Vertex access and the OpenAI key are- * each present and each refuse to serve. probeProvider() in provider.js asks the- * provider itself, and its verdict is what readiness is judged on.+ * Every lane carries the env its CLI needs, so electing one is applying that+ * overlay and respawning the daemon. A `null` value means "unset this", which is+ * what lets the gateway lane switch Vertex off for the subsession it spawns.** @param {NodeJS.ProcessEnv} env- * @param {string} [home] where the CLI keeps an interactive login- * @returns {{ provider: string, transport: string, reason: string, credentialed: boolean }}+ * @param {string} home where a CLI keeps an interactive login+ * @returns {Array<{ id: string, provider: string, transport: string, reason: string, env: Record<string,string|null> }>}*/-export function resolveProvider(env = process.env, home = env.OKX_BOT_HOME || homedir()) {- const pinned = (env.OKX_BOT_AI_PROVIDER || '').trim().toLowerCase();+export function providerLanes(env = process.env, home = env.OKX_BOT_HOME || homedir()) {+ const lanes = [];const onVertex =(env.CLAUDE_CODE_USE_VERTEX === '1' || env.CLAUDE_CODE_USE_VERTEX === 'true') &&!!(env.ANTHROPIC_VERTEX_PROJECT_ID || env.GOOGLE_CLOUD_PROJECT);- const hasClaudeKey = !!(env.ANTHROPIC_API_KEY || env.CLAUDE_CODE_OAUTH_TOKEN);- const hasClaudeLogin = existsSync(join(home, '.claude', '.credentials.json'));- const claude = onVertex- ? { transport: 'vertex', reason: 'Vertex AI, authenticated by the runtime service account (no key to rotate)' }- : hasClaudeKey- ? { transport: 'api-key', reason: 'ANTHROPIC_API_KEY present' }- : hasClaudeLogin- ? { transport: 'oauth-login', reason: 'claude CLI holds an interactive login' }- : { transport: 'none', reason: 'no Anthropic credential' };- const codex = env.OPENAI_API_KEY- ? { transport: 'api-key', reason: 'OPENAI_API_KEY present' }- : { transport: 'none', reason: 'no OpenAI credential' };-- if (pinned) {- const t = pinned === 'claude' ? claude : pinned === 'codex' ? codex : { transport: 'external', reason: 'provider supplies its own auth' };- return {- provider: pinned,- transport: t.transport,- reason: `pinned by OKX_BOT_AI_PROVIDER (${t.reason})`,- credentialed: t.transport !== 'none',- };+ if (onVertex) {+ lanes.push({+ id: 'vertex',+ provider: 'claude',+ transport: 'vertex',+ reason: 'Vertex AI, authenticated by the runtime service account (no key to rotate)',+ credentialed: true,+ env: {},+ });+ }+ if (env.ANTHROPIC_API_KEY) {+ lanes.push({+ id: 'anthropic-key',+ provider: 'claude',+ transport: 'api-key',+ reason: 'ANTHROPIC_API_KEY present',+ credentialed: true,+ env: { CLAUDE_CODE_USE_VERTEX: null, ANTHROPIC_BASE_URL: null },+ });+ }+ const gateway = gatewayLane(env);+ if (gateway) lanes.push(gateway);+ if (env.CLAUDE_CODE_OAUTH_TOKEN) {+ lanes.push({+ id: 'anthropic-oauth-token',+ provider: 'claude',+ transport: 'api-key',+ reason: 'CLAUDE_CODE_OAUTH_TOKEN present',+ credentialed: true,+ env: { CLAUDE_CODE_USE_VERTEX: null, ANTHROPIC_BASE_URL: null },+ });}- if (claude.transport !== 'none') {- return { provider: 'claude', transport: claude.transport, reason: claude.reason, credentialed: true };+ if (existsSync(join(home, '.claude', '.credentials.json'))) {+ lanes.push({+ id: 'anthropic-login',+ provider: 'claude',+ transport: 'oauth-login',+ reason: 'claude CLI holds an interactive login',+ credentialed: true,+ env: {},+ });}- if (codex.transport !== 'none') {- return { provider: 'codex', transport: codex.transport, reason: `${codex.reason} (no Anthropic credential)`, credentialed: true };+ if (env.OPENAI_API_KEY) {+ lanes.push({+ id: 'openai-key',+ provider: 'codex',+ transport: 'api-key',+ reason: 'OPENAI_API_KEY present',+ credentialed: true,+ env: {},+ });+ }+ return lanes;+}++/**+ * The Anthropic-compatible gateway lane, or null when none is configured.+ *+ * `claude` speaks one wire format and reads its endpoint from ANTHROPIC_BASE_URL,+ * so any service that serves `/v1/messages` is a usable lane for the same agentic+ * CLI the adapter already spawns. That keeps the task lifecycle intact, which a+ * one-shot completion responder would break.+ *+ * Deliberately opt-in: unlike Vertex it bills a third-party account per token, so+ * it must be a decision an operator made, never something the host picks up from+ * an unrelated key that happened to be on the service.+ */+function gatewayLane(env) {+ const base = (env.OKX_BOT_ANTHROPIC_BASE_URL || '').trim().replace(/\/+$/, '');+ const token = (env.OKX_BOT_ANTHROPIC_AUTH_TOKEN || '').trim();+ if (!base || !token) return null;+ const model = (env.OKX_BOT_ANTHROPIC_MODEL || '').trim();+ let label = base;+ try {+ label = new URL(base).host;+ } catch {+ /* a malformed base URL still names itself in the reason */}return {+ id: 'anthropic-gateway',provider: 'claude',- transport: 'none',- reason: 'no AI-provider credential found, chat replies will fail until one is set',- credentialed: false,+ transport: 'gateway',+ reason: `Anthropic-compatible gateway at ${label}${model ? ` (${model})` : ''}`,+ credentialed: true,+ env: {+ CLAUDE_CODE_USE_VERTEX: null,+ ANTHROPIC_BASE_URL: base,+ ANTHROPIC_AUTH_TOKEN: token,+ ANTHROPIC_API_KEY: null,+ ...(model ? { ANTHROPIC_MODEL: model, ANTHROPIC_SMALL_FAST_MODEL: model } : {}),+ },+ };+}++/**+ * Narrow the chain to what an explicit pin allows.+ *+ * A pin names a provider CLI, not a lane, so `OKX_BOT_AI_PROVIDER=claude` keeps+ * every Anthropic lane and drops codex. A pin naming a provider with no lane at+ * all still returns an honest, uncredentialed single entry rather than silently+ * electing something the operator did not ask for.+ */+export function pinnedLanes(lanes, pinned) {+ if (!pinned) return lanes;+ const kept = lanes.filter((l) => l.provider === pinned);+ if (kept.length) return kept.map((l) => ({ ...l, reason: `pinned by OKX_BOT_AI_PROVIDER (${l.reason})` }));+ const external = pinned !== 'claude' && pinned !== 'codex';+ return [+ {+ id: `${pinned}-pinned`,+ provider: pinned,+ transport: external ? 'external' : 'none',+ reason: external+ ? 'pinned by OKX_BOT_AI_PROVIDER (provider supplies its own auth)'+ : `pinned by OKX_BOT_AI_PROVIDER (no ${pinned === 'claude' ? 'Anthropic' : 'OpenAI'} credential)`,+ env: {},+ credentialed: external,+ },+ ];+}++/**+ * Resolve the whole chain: the lanes to try, in order, and which one leads.+ *+ * `credentialed` on a lane only answers "is a credential configured". Whether it+ * still WORKS is a different question, and on this project the two answers have+ * differed in both directions: the GCP project's Vertex access and the OpenAI key+ * are each present and each refuse to serve. electProvider() in provider.js asks+ * each lane's own API, and its verdict is what readiness is judged on.+ *+ * The adapter does NOT read a reply out of the CLI's stdout: the spawned AI+ * subsession sends the reply itself through the `okx-a2a` CLI and drives the task+ * lifecycle (accept / negotiate / deliver). So every lane must spawn a genuinely+ * agentic CLI with tool access, never a one-shot completion call.+ *+ * @returns {{ lanes: Array<object>, head: object }}+ */+export function resolveProviderChain(env = process.env, home = env.OKX_BOT_HOME || homedir()) {+ const pinned = (env.OKX_BOT_AI_PROVIDER || '').trim().toLowerCase();+ const lanes = pinnedLanes(providerLanes(env, home), pinned);+ if (lanes.length) return { lanes, head: lanes[0] };+ return {+ lanes: [],+ head: {+ id: 'none',+ provider: 'claude',+ transport: 'none',+ reason: 'no AI-provider credential found, chat replies will fail until one is set',+ env: {},+ credentialed: false,+ },};}@@ -104,9 +214,29 @@ export function resolveHost(env = process.env) {return { label: `local:${hostname()}`, durable: false };}+/**+ * Point the config at one lane of the chain.+ *+ * Every consumer (the heartbeat, the health body, cliEnv) reads the flat+ * `provider*` fields, so electing a lane is one mutation here rather than a+ * plumbing change in five files. Called at boot with the head of the chain and+ * again whenever a probe elects a different one.+ *+ * @param {ReturnType<loadConfig>} cfg+ * @param {{ id: string, provider: string, transport: string, reason: string, env: object }} lane+ */+export function applyLane(cfg, lane) {+ cfg.activeLane = lane;+ cfg.provider = lane.provider;+ cfg.providerTransport = lane.transport;+ cfg.providerReason = lane.reason;+ cfg.providerCredentialed = lane.credentialed === true;+ return cfg;+}+export function loadConfig(env = process.env) {const home = env.OKX_BOT_HOME || homedir();- const provider = resolveProvider(env, home);+ const chain = resolveProviderChain(env, home);const host = resolveHost(env);return {// HOME for both CLIs. Everything durable lands under it:@@ -128,10 +258,15 @@ export function loadConfig(env = process.env) {stateBucket: (env.OKX_BOT_STATE_BUCKET || '').trim(),stateObject: env.OKX_BOT_STATE_OBJECT || 'okx-chat-bot/state.tar.gz',- provider: provider.provider,- providerTransport: provider.transport,- providerReason: provider.reason,- providerCredentialed: provider.credentialed,+ // The lanes this host may authenticate through, best first, and the one it+ // is currently using. A single-rung chain is what left the host sitting on+ // a dead Vertex credential for days; see providerLanes().+ chain: chain.lanes,+ activeLane: chain.head,+ provider: chain.head.provider,+ providerTransport: chain.head.transport,+ providerReason: chain.head.reason,+ providerCredentialed: chain.head.credentialed === true,// How often the provider's own API is asked whether the credential still// works. A configured credential is not a working one: on this project thediff --git a/workers/okx-chat-bot/health-server.js b/workers/okx-chat-bot/health-server.jsindex e53e7bbbd..198267dbd 100644--- a/workers/okx-chat-bot/health-server.js+++ b/workers/okx-chat-bot/health-server.js@@ -44,6 +44,17 @@ export function statusBody(cfg, live, daemonStats, bootAt) {verdict: live.providerProbe?.code ?? null,verdictDetail: live.providerProbe?.detail ?? null,verdictAt: live.providerProbe?.checkedAt ?? null,+ // Which lane of the chain is serving, and what every other lane+ // answered. A human deciding which credential to fund needs the whole+ // picture, not just the fact that the elected one was refused.+ lane: cfg.activeLane?.id ?? null,+ chain: (live.providerLanes || []).map((l) => ({+ lane: l.lane,+ transport: l.transport,+ verdict: l.code,+ detail: l.detail,+ checkedAt: l.checkedAt,+ })),},daemon: { status: live.daemon, ...daemonStats },session: live.wallet ? { loggedIn: live.wallet.loggedIn, email: live.wallet.email } : null,@@ -54,7 +65,7 @@ export function statusBody(cfg, live, daemonStats, bootAt) {checkedAt: live.checkedAt,};if (live.verdict.needsHumanLogin) body.remedy = loginInstructions(live.login?.loginUrl, live.login?.authSessionId);- else if (live.verdict.reason === 'ai_provider_unauthorized') body.remedy = providerInstructions(live.providerProbe?.detail);+ else if (live.verdict.reason === 'ai_provider_unauthorized') body.remedy = providerInstructions(live.providerProbe?.detail, live.providerLanes);return body;}diff --git a/workers/okx-chat-bot/index.js b/workers/okx-chat-bot/index.jsindex 148cf1bda..5c0503805 100644--- a/workers/okx-chat-bot/index.js+++ b/workers/okx-chat-bot/index.js@@ -22,10 +22,10 @@import { mkdir } from 'node:fs/promises';import { sendOpsAlert } from '../../api/_lib/alerts.js';import { sql } from '../../api/_lib/db.js';-import { loadConfig, paths } from './config.js';+import { applyLane, loadConfig, paths } from './config.js';import { agentRefresh, beginLogin, daemonStatus, exec, walletStatus } from './cli.js';import { startHealthServer } from './health-server.js';-import { loginCodex, probeProvider } from './provider.js';+import { electProvider, loginCodex } from './provider.js';import { classify, loginInstructions } from './session.js';import { restoreState, snapshotState } from './state.js';import { createSupervisor } from './supervisor.js';@@ -49,6 +49,10 @@ const live = {// which is why classify() treats a missing verdict as "no opinion" rather than// as a failure: a host must not read red for a probe that has not run yet.providerProbe: null,+ // What EVERY lane in the chain last answered. The elected lane's verdict is+ // `providerProbe`; this is the whole picture, so a human reading /readyz can+ // see which credential to fund rather than only that one of them was refused.+ providerLanes: [],};async function heartbeat(cfg, supervisor) {@@ -68,7 +72,9 @@ async function heartbeat(cfg, supervisor) {provider: cfg.provider,providerTransport: cfg.providerTransport,providerCredentialed: cfg.providerCredentialed,+ providerLane: cfg.activeLane?.id ?? null,providerVerdict: live.providerProbe?.code ?? null,+ providerChain: live.providerLanes.map((l) => ({ lane: l.lane, code: l.code })),daemonRestarts: supervisor.stats().restarts,checkedAt: live.checkedAt,};@@ -113,20 +119,46 @@ async function maybeAlert(cfg, verdict) {}).catch((err) => log.warn('ops alert failed', { err: err?.message }));}-// Ask the provider whether it still serves this host. Kept off the session-// probe's path: it is a network call to a third party, and a slow one must not-// delay the verdict about whether XMTP is delivering. A transient `unreachable`-// answer is remembered as-is, because classify() deliberately does not fail-// readiness on it.-async function probeProviderCredential(cfg) {+// Ask every AI lane whether it still serves this host, and move to the best one.+//+// Kept off the session probe's path: these are network calls to third parties,+// and a slow one must not delay the verdict about whether XMTP is delivering. A+// transient `unreachable` answer is remembered as-is, because classify()+// deliberately does not fail readiness on it.+//+// Election is what makes a dead credential survivable. The host used to hold one+// lane, so the Vertex billing hold on 2026-09-04 left it unable to author a+// single reply even though other credentials were configured. Now the lane that+// answers is the lane that serves, and a lane that recovers is picked up on the+// next tick with no deploy.+async function electProviderLane(cfg, supervisor) {const before = live.providerProbe?.code ?? null;- live.providerProbe = await probeProvider(cfg);- if (live.providerProbe.code !== before) {+ const beforeLane = cfg.activeLane?.id ?? null;+ const { elected, verdict, lanes } = await electProvider(cfg.chain);+ live.providerLanes = lanes;+ if (verdict) live.providerProbe = verdict;++ if (elected && elected.id !== beforeLane) {+ applyLane(cfg, elected);+ log.info('provider lane elected', { lane: elected.id, transport: elected.transport, from: beforeLane });+ // The adapter reads the provider out of ~/.okx-agent-task/config.toml, and+ // the spawned CLI reads its credentials out of the daemon's environment.+ // Both are set at spawn time, so a lane change is only real once the daemon+ // has been replaced with one that carries it.+ await exec(cfg, 'okx-a2a', ['config', 'provider', '--provider', cfg.provider], { timeoutMs: 30_000 });+ const codexLogin = await loginCodex(cfg);+ if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);+ await supervisor.restart(`ai lane ${beforeLane || 'none'} -> ${elected.id}`);+ }++ if (live.providerProbe && live.providerProbe.code !== before) {log[live.providerProbe.code === 'unauthorized' ? 'error' : 'info']('provider credential', {provider: cfg.provider,transport: cfg.providerTransport,+ lane: live.providerProbe.lane,code: live.providerProbe.code,detail: live.providerProbe.detail,+ lanes: lanes.map((l) => `${l.lane}=${l.code}`).join(' '),});}}@@ -200,6 +232,7 @@ async function main() {hostDurable: cfg.hostDurable,provider: cfg.provider,providerReason: cfg.providerReason,+ chain: cfg.chain.map((l) => l.id).join(' > ') || 'none',stateBucket: cfg.stateBucket || null,});if (!cfg.providerCredentialed) {@@ -247,12 +280,12 @@ async function main() {cfg.snapshotMs,);const providerProbeTimer = setInterval(- () => probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message })),+ () => electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message })),cfg.providerProbeMs,);- // Run the first credential probe immediately, but off the boot path: a slow- // provider must delay the verdict, never the daemon that receives the chat.- probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message }));+ // Run the first election immediately, but off the boot path: a slow provider+ // must delay the verdict, never the daemon that receives the chat.+ electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message }));// A first verdict early enough to be useful. It can legitimately land before// the daemon has claimed its lock, which classify() reports as// `daemon_starting` rather than `daemon_down`, so an ordinary boot no longerdiff --git a/workers/okx-chat-bot/provider.js b/workers/okx-chat-bot/provider.jsindex 429da54bc..5cc70298c 100644--- a/workers/okx-chat-bot/provider.js+++ b/workers/okx-chat-bot/provider.js@@ -131,28 +131,132 @@ async function probeOpenAiKey(env) {);}+async function probeAnthropicBearer(env, key) {+ return postJson(+ 'https://api.anthropic.com/v1/messages',+ { authorization: `Bearer ${key}`, 'anthropic-version': '2023-06-01' },+ { model: ANTHROPIC_PROBE_MODEL, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },+ );+}+/**- * Ask the configured provider whether it will serve this host.+ * Probe an Anthropic-wire-format gateway at its own base URL.*- * @param {ReturnType<import('./config.js').loadConfig>} cfg- * @param {NodeJS.ProcessEnv} [env]- * @returns {Promise<{ code: 'ok'|'unauthorized'|'unreachable'|'unprobed', detail: string, transport: string, checkedAt: number }>}+ * `${base}/v1/messages` is the same path the CLI itself builds from+ * ANTHROPIC_BASE_URL, so the probe fails exactly where a real reply would. That+ * matters more than it sounds: OpenRouter's gateway lives at+ * `https://openrouter.ai/api/v1/messages`, so its base URL is+ * `https://openrouter.ai/api` and the intuitive `.../api/v1` is one path segment+ * too deep. Configured that way the endpoint answers 404 forever.+ *+ * Gateways disagree about which header carries the credential (OpenRouter reads+ * `Authorization`, Anthropic reads `x-api-key`), so both are sent: whichever the+ * gateway reads, the other is ignored. The model id is the lane's own, because a+ * gateway that authenticates fine still refuses an id from a different catalog.+ *+ * And unlike a first-party provider, a gateway's 400/404 is NOT proof the+ * credential works. classifyProbeStatus() reads those as "the provider had to+ * authenticate me before it could object to my request", which is true of+ * api.anthropic.com and useless here: a wrong base URL or an unroutable model id+ * produces exactly that status and then fails every real reply. So a gateway+ * lane is elected on a 2xx and nothing else.*/-export async function probeProvider(cfg, env = process.env) {- const wrap = (r) => ({ ...r, transport: cfg.providerTransport, checkedAt: Date.now() });+async function probeGateway(lane) {+ const base = String(lane.env.ANTHROPIC_BASE_URL || '').replace(/\/+$/, '');+ const model = lane.env.ANTHROPIC_MODEL || ANTHROPIC_PROBE_MODEL;+ const key = lane.env.ANTHROPIC_AUTH_TOKEN || '';+ const url = `${base}/v1/messages`;+ const verdict = await postJson(+ url,+ { authorization: `Bearer ${key}`, 'x-api-key': key, 'anthropic-version': '2023-06-01' },+ { model, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },+ );+ if (verdict.code === 'ok' && /returned (400|404)/.test(verdict.detail)) {+ return {+ code: 'unauthorized',+ detail: `the gateway ${url} rejected the probe request (model ${model}): ${verdict.detail}. ` ++ 'Check OKX_BOT_ANTHROPIC_BASE_URL (it must be the prefix the CLI appends /v1/messages to) and OKX_BOT_ANTHROPIC_MODEL.',+ };+ }+ return verdict;+}- if (!cfg.providerCredentialed) {- return wrap({ code: 'unauthorized', detail: 'no AI-provider credential is configured' });+/**+ * Ask ONE lane whether it will serve this host.+ *+ * The lane's env overlay is layered over the process env first, so a lane is+ * probed exactly as the spawned CLI would experience it. Probing the ambient env+ * instead is how a gateway lane would report on Vertex's credential and elect+ * itself on someone else's verdict.+ *+ * @param {{ id: string, provider: string, transport: string, env: Record<string,string|null> }} lane+ * @param {NodeJS.ProcessEnv} [ambient]+ */+export async function probeLane(lane, ambient = process.env) {+ const env = { ...ambient };+ for (const [k, v] of Object.entries(lane.env || {})) {+ if (v === null || v === undefined) delete env[k];+ else env[k] = String(v);+ }+ const wrap = (r) => ({ ...r, lane: lane.id, transport: lane.transport, checkedAt: Date.now() });++ // Dispatch on the LANE's transport, never on what the ambient env happens to+ // hold. A developer host running on an interactive grant usually also has an+ // unrelated key in its environment, and probing that key would answer for a+ // credential this lane does not use.+ if (lane.transport === 'none') return wrap({ code: 'unauthorized', detail: 'no AI-provider credential is configured' });+ if (lane.transport === 'vertex') return wrap(await probeVertex(env));+ if (lane.transport === 'gateway') return wrap(await probeGateway(lane));+ if (lane.transport === 'api-key' && lane.provider === 'claude') {+ return env.ANTHROPIC_API_KEY+ ? wrap(await probeAnthropicKey(env))+ : wrap(await probeAnthropicBearer(env, env.CLAUDE_CODE_OAUTH_TOKEN || ''));+ }+ if (lane.transport === 'api-key' && lane.provider === 'codex' && env.OPENAI_API_KEY) {+ return wrap(await probeOpenAiKey(env));+ }++ // An interactive OAuth grant is refreshed by the CLI itself; there is no+ // endpoint this worker can call that proves the grant without reimplementing+ // that refresh. Claiming a verdict here would be inventing one, so the probe+ // abstains and the presence check stands.+ return wrap({ code: 'unprobed', detail: `no credential probe exists for the ${lane.transport} transport` });+}++// Which verdict a host would rather be on when no lane outright works. `ok` is+// the only one that serves a buyer; after that an unprovable grant beats a+// provider having a bad minute, which beats a credential that was refused+// outright. Election walks this order so the reported verdict is the best truth+// available rather than whichever lane happened to be listed first.+const VERDICT_RANK = { ok: 0, unprobed: 1, unreachable: 2, unauthorized: 3 };++/**+ * Probe the chain in order and elect the first lane that can actually serve.+ *+ * Stops at the first `ok`: the lanes are ordered by policy (GCP credits first,+ * a metered third-party gateway last), so the first working one is also the one+ * that should be used, and probing the rest would spend money to learn nothing.+ *+ * @param {Array<object>} chain+ * @param {NodeJS.ProcessEnv} [env]+ * @returns {Promise<{ elected: object|null, verdict: object, lanes: Array<object> }>}+ */+export async function electProvider(chain, env = process.env) {+ const lanes = [];+ for (const lane of chain) {+ const verdict = await probeLane(lane, env);+ lanes.push(verdict);+ if (verdict.code === 'ok') return { elected: lane, verdict, lanes };+ }+ if (!lanes.length) return { elected: null, verdict: null, lanes };+ let best = 0;+ for (let i = 1; i < lanes.length; i++) {+ if (VERDICT_RANK[lanes[i].code] < VERDICT_RANK[lanes[best].code]) best = i;}- if (cfg.providerTransport === 'vertex') return wrap(await probeVertex(env));- if (cfg.provider === 'claude' && env.ANTHROPIC_API_KEY) return wrap(await probeAnthropicKey(env));- if (cfg.provider === 'codex' && env.OPENAI_API_KEY) return wrap(await probeOpenAiKey(env));-- // An interactive OAuth grant and a CLAUDE_CODE_OAUTH_TOKEN are refreshed by- // the CLI itself; there is no endpoint this worker can call that proves the- // grant without reimplementing that refresh. Claiming a verdict here would be- // inventing one, so the probe abstains and the presence check stands.- return wrap({ code: 'unprobed', detail: `no credential probe exists for the ${cfg.providerTransport} transport` });+ // `unprobed` is the one non-ok verdict a host may still serve on: it means the+ // credential could not be asked, not that it was refused.+ const elected = lanes[best].code === 'unprobed' ? chain[best] : null;+ return { elected, verdict: lanes[best], lanes };}/**diff --git a/workers/okx-chat-bot/session.js b/workers/okx-chat-bot/session.jsindex 7da220a1f..5fc154732 100644--- a/workers/okx-chat-bot/session.js+++ b/workers/okx-chat-bot/session.js@@ -38,9 +38,13 @@ export function loginInstructions(loginUrl, authSessionId) {* is the only option with no secret to mint, rotate or forget, and it bills the* GCP credit pool the platform already prefers.*/-export function providerInstructions(detail = '') {+export function providerInstructions(detail = '', lanes = []) {+ const chain = lanes.length+ ? ['', 'Every lane in the chain was asked; none can serve:', ...lanes.map((l) => ` ${l.lane} (${l.transport}): ${l.code} - ${l.detail}`)]+ : [];return [`The provider refused this host's credential: ${detail}`,+ ...chain,'','1. Preferred (no secret): clear the project billing hold, then',' gcloud run services update okx-chat-bot --region us-central1 \\',@@ -52,6 +56,11 @@ export function providerInstructions(detail = '') {' stripped by the next deploy).','3. Or reactivate billing on the OpenAI account behind the openai-api-key secret',' and pin OKX_BOT_AI_PROVIDER=codex.',+ '4. Or point the host at any funded Anthropic-wire-format gateway (config only,',+ ' pre-approved). OpenRouter serves one at /api/v1/messages:',+ ' gcloud run services update okx-chat-bot --region us-central1 \\',+ ' --update-env-vars=OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6 \\',+ ' --update-secrets=OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest',];}diff --git a/workers/okx-chat-bot/supervisor.js b/workers/okx-chat-bot/supervisor.jsindex 0a1f01dc5..9e5e51f20 100644--- a/workers/okx-chat-bot/supervisor.js+++ b/workers/okx-chat-bot/supervisor.js@@ -18,6 +18,12 @@ import { log } from './log.js';export function createSupervisor(cfg, p) {let child = null;let stopping = false;+ // Set once, by stop(). A restart flips `stopping` on and off around its own+ // SIGTERM, so it needs a flag that a real shutdown cannot be confused with:+ // respawning a daemon after the shutdown path has begun would leave the final+ // snapshot copying a live sqlite file, which is exactly what the stop-then-+ // snapshot ordering exists to prevent.+ let terminated = false;let restarts = 0;let backoffMs = cfg.restartBaseMs;let timer = null;@@ -73,27 +79,55 @@ export function createSupervisor(cfg, p) {child.on('exit', (code, signal) => daemonDied({ code, signal }));}+ async function stopChild(timeoutMs) {+ if (timer) clearTimeout(timer);+ const proc = child;+ if (!proc) return;+ await new Promise((resolve) => {+ const done = setTimeout(() => {+ proc.kill('SIGKILL');+ resolve(undefined);+ }, timeoutMs);+ proc.once('exit', () => {+ clearTimeout(done);+ resolve(undefined);+ });+ proc.kill('SIGTERM');+ });+ }+return {start: spawnDaemon,/** Stop the child and wait for it, so state is quiesced before a snapshot. */async stop(timeoutMs = 10_000) {stopping = true;- if (timer) clearTimeout(timer);- const proc = child;- if (!proc) return;- await new Promise((resolve) => {- const done = setTimeout(() => {- proc.kill('SIGKILL');- resolve(undefined);- }, timeoutMs);- proc.once('exit', () => {- clearTimeout(done);- resolve(undefined);- });- proc.kill('SIGTERM');- });+ terminated = true;+ await stopChild(timeoutMs);log.info('daemon stopped', { restarts });},++ /**+ * Replace the running daemon with one spawned from the current environment.+ *+ * The adapter hands the spawned AI subsession the daemon's own environment,+ * fixed at spawn time, so a newly elected AI lane cannot reach the CLI any+ * other way. Deliberately a full stop-then-start rather than a signal: the+ * daemon holds a lock and a sqlite file, and only a clean exit quiesces both.+ */+ async restart(reason, timeoutMs = 10_000) {+ if (stopping) return;+ log.info('daemon restart', { reason, restarts });+ // Suppress the exit handler's own restart for the duration: without this+ // the SIGTERM below schedules a respawn AND the explicit spawn runs, and+ // two daemons race for one lock and one XMTP identity.+ stopping = true;+ await stopChild(timeoutMs);+ child = null;+ if (terminated) return;+ stopping = false;+ backoffMs = cfg.restartBaseMs;+ spawnDaemon();+ },stats() {return { restarts, pid: child?.pid ?? null, uptimeMs: child ? Date.now() - startedAt : 0 };},