Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(okx-chat-bot): elect a funded AI lane instead of stalling on a denied Vertex grant

nirholasad723e87fa69
diff --git a/tests/okx-chat-bot.test.js b/tests/okx-chat-bot.test.js
index 1fb2bcbff..5782c28d1 100644
--- a/tests/okx-chat-bot.test.js
+++ b/tests/okx-chat-bot.test.js
@@ -16,8 +16,9 @@ import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { afterAll, describe, it, expect, vi } from 'vitest';
import { classify, loginInstructions, providerInstructions } from '../workers/okx-chat-bot/session.js';
-import { classifyProbeStatus, loginCodex } from '../workers/okx-chat-bot/provider.js';
-import { resolveProvider, resolveHost, loadConfig, paths } from '../workers/okx-chat-bot/config.js';
+import { classifyProbeStatus, electProvider, loginCodex, probeLane } from '../workers/okx-chat-bot/provider.js';
+import { applyLane, providerLanes, resolveProviderChain, resolveHost, loadConfig, paths } from '../workers/okx-chat-bot/config.js';
+import { cliEnv } from '../workers/okx-chat-bot/cli.js';
import { createHealthHandler } from '../workers/okx-chat-bot/health-server.js';
import { createSupervisor } from '../workers/okx-chat-bot/supervisor.js';
import { STATE_ROOTS, STATE_EXCLUDES } from '../workers/okx-chat-bot/state.js';
@@ -121,6 +122,8 @@ describe('okx-chat-bot provider selection', () => {
// A host with no interactive CLI login, so these assertions read the env and
// nothing else. mkdtemp gives a home that provably holds no credential file.
const bare = mkdtempSync(join(tmpdir(), 'okx-bot-nohome-'));
+ // The lane the host leads with: what the daemon is spawned on at boot.
+ const resolveProvider = (env, home) => resolveProviderChain(env, home).head;
it('prefers Claude when an Anthropic credential is present', () => {
const r = resolveProvider({ ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y' }, bare);
@@ -252,6 +255,145 @@ describe('okx-chat-bot provider credential probe', () => {
});
});
+// A chain with one rung is not a chain. The host held exactly one AI lane until
+// 2026-09-09, so when the GCP project's Vertex access started answering
+// "Lightning dunning decision is deny" the bot received every buyer message and
+// could author no reply at all, for days, while other credentials sat unused on
+// the same service. These tests hold the line that whichever lane is funded is
+// the lane that serves.
+describe('okx-chat-bot provider chain', () => {
+ const bare = mkdtempSync(join(tmpdir(), 'okx-bot-chain-'));
+ const GATEWAY = {
+ OKX_BOT_ANTHROPIC_BASE_URL: 'https://openrouter.ai/api',
+ OKX_BOT_ANTHROPIC_AUTH_TOKEN: 'sk-or-test',
+ OKX_BOT_ANTHROPIC_MODEL: 'anthropic/claude-sonnet-4.6',
+ };
+
+ afterAll(() => rmSync(bare, { recursive: true, force: true }));
+
+ it('orders GCP credits ahead of every key, and a metered gateway last', () => {
+ const lanes = providerLanes(
+ {
+ CLAUDE_CODE_USE_VERTEX: '1',
+ ANTHROPIC_VERTEX_PROJECT_ID: 'aerial-vehicle-466722-p5',
+ ANTHROPIC_API_KEY: 'sk-ant-x',
+ OPENAI_API_KEY: 'sk-y',
+ ...GATEWAY,
+ },
+ bare,
+ );
+ expect(lanes.map((l) => l.id)).toEqual(['vertex', 'anthropic-key', 'anthropic-gateway', 'openai-key']);
+ });
+
+ // A gateway bills a third-party account per token, so it must be a decision an
+ // operator made rather than something the host picks up from a stray key.
+ it('never invents a gateway lane from a half-set config', () => {
+ expect(providerLanes({ OKX_BOT_ANTHROPIC_BASE_URL: 'https://openrouter.ai/api' }, bare)).toEqual([]);
+ expect(providerLanes({ OKX_BOT_ANTHROPIC_AUTH_TOKEN: 'sk-or-test' }, bare)).toEqual([]);
+ });
+
+ // The container boots with CLAUDE_CODE_USE_VERTEX=1 baked in. Electing the
+ // gateway without unsetting it spawns a CLI that still reaches for the dead
+ // transport, which is the same silence with a different cause.
+ it('unsets the Vertex transport when a gateway lane is elected', () => {
+ const [lane] = providerLanes(GATEWAY, bare);
+ const cfg = applyLane({ home: '/state' }, lane);
+ const env = cliEnv(cfg);
+ expect(env.ANTHROPIC_BASE_URL).toBe('https://openrouter.ai/api');
+ expect(env.ANTHROPIC_MODEL).toBe('anthropic/claude-sonnet-4.6');
+ expect('CLAUDE_CODE_USE_VERTEX' in env).toBe(false);
+ });
+
+ it('keeps a pin honest: it narrows the chain, it does not empty it', () => {
+ const env = { ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y', ...GATEWAY, OKX_BOT_AI_PROVIDER: 'claude' };
+ expect(resolveProviderChain(env, bare).lanes.map((l) => l.id)).toEqual(['anthropic-key', 'anthropic-gateway']);
+ });
+
+ it('probes each lane through its own env, not the ambient one', async () => {
+ const seen = [];
+ const fetchMock = vi.fn(async (url, init) => {
+ seen.push({ url, auth: init.headers.authorization });
+ return new Response('{}', { status: 200 });
+ });
+ vi.stubGlobal('fetch', fetchMock);
+ const [lane] = providerLanes(GATEWAY, bare);
+ const v = await probeLane(lane, { ANTHROPIC_API_KEY: 'sk-ant-ambient' });
+ vi.unstubAllGlobals();
+ expect(v).toMatchObject({ code: 'ok', lane: 'anthropic-gateway', transport: 'gateway' });
+ expect(seen[0].url).toBe('https://openrouter.ai/api/v1/messages');
+ expect(seen[0].auth).toBe('Bearer sk-or-test');
+ });
+
+ // Measured live on 2026-09-09: base URL `https://openrouter.ai/api/v1` makes the
+ // CLI request `/api/v1/v1/messages`, which answers 404. classifyProbeStatus()
+ // reads a 404 from a first-party provider as proof the credential works, and
+ // that reading would have elected a lane that 404s every buyer reply.
+ it('refuses to elect a gateway that only answered a complaint about the request', async () => {
+ const [lane] = providerLanes(GATEWAY, bare);
+ vi.stubGlobal('fetch', async () => new Response('No endpoints found', { status: 404 }));
+ const v = await probeLane(lane, {});
+ vi.unstubAllGlobals();
+ expect(v.code).toBe('unauthorized');
+ expect(v.detail).toContain('OKX_BOT_ANTHROPIC_BASE_URL');
+ });
+
+ it('elects the first lane that answers and stops paying for the rest', async () => {
+ const chain = providerLanes(
+ { CLAUDE_CODE_USE_VERTEX: '1', GOOGLE_CLOUD_PROJECT: 'p', ANTHROPIC_API_KEY: 'sk-ant-x', ...GATEWAY },
+ bare,
+ );
+ const calls = [];
+ vi.stubGlobal('fetch', async (url) => {
+ calls.push(String(url));
+ // Vertex is the dead lane; the Anthropic key answers.
+ return String(url).includes('aiplatform')
+ ? new Response('Lightning dunning decision is deny', { status: 403 })
+ : new Response('{}', { status: 200 });
+ });
+ const r = await electProvider(chain, { ANTHROPIC_API_KEY: 'sk-ant-x' });
+ vi.unstubAllGlobals();
+ expect(r.elected.id).toBe('anthropic-key');
+ expect(r.lanes.map((l) => l.code)).toEqual(['unauthorized', 'ok']);
+ expect(calls.some((u) => u.includes('openrouter'))).toBe(false);
+ });
+
+ it('elects nothing when every lane is refused, and reports them all', async () => {
+ const chain = providerLanes({ ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y' }, bare);
+ vi.stubGlobal('fetch', async () => new Response('{"error":{"message":"no credit"}}', { status: 402 }));
+ const r = await electProvider(chain, { ANTHROPIC_API_KEY: 'sk-ant-x', OPENAI_API_KEY: 'sk-y' });
+ vi.unstubAllGlobals();
+ expect(r.elected).toBe(null);
+ expect(r.verdict.code).toBe('unauthorized');
+ expect(r.lanes.map((l) => l.lane)).toEqual(['anthropic-key', 'openai-key']);
+ });
+
+ // An unprovable grant is not a refusal. The developer host runs on exactly
+ // that, so calling it dead would take the stopgap offline for a lane that
+ // demonstrably authors replies.
+ it('serves on an unprovable lane rather than calling it refused', async () => {
+ const home = mkdtempSync(join(tmpdir(), 'okx-bot-login-'));
+ mkdirSync(join(home, '.claude'), { recursive: true });
+ writeFileSync(join(home, '.claude', '.credentials.json'), '{}');
+ const chain = providerLanes({ ANTHROPIC_API_KEY: 'sk-ant-x' }, home);
+ vi.stubGlobal('fetch', async () => new Response('nope', { status: 401 }));
+ const r = await electProvider(chain, { ANTHROPIC_API_KEY: 'sk-ant-x' });
+ vi.unstubAllGlobals();
+ rmSync(home, { recursive: true, force: true });
+ expect(r.elected.id).toBe('anthropic-login');
+ expect(r.verdict.code).toBe('unprobed');
+ });
+
+ it('puts every lane verdict on the status body a human funds from', () => {
+ const lines = providerInstructions('dunning deny', [
+ { lane: 'vertex', transport: 'vertex', code: 'unauthorized', detail: 'Lightning dunning decision is deny' },
+ { lane: 'openai-key', transport: 'api-key', code: 'unauthorized', detail: 'billing_not_active' },
+ ]).join('\n');
+ expect(lines).toContain('vertex (vertex): unauthorized');
+ expect(lines).toContain('openai-key (api-key): unauthorized - billing_not_active');
+ expect(lines).toContain('OKX_BOT_ANTHROPIC_BASE_URL');
+ });
+});
+
describe('okx-chat-bot host reporting', () => {
it('calls Cloud Run the durable host and names the revision', () => {
const h = resolveHost({ K_SERVICE: 'okx-chat-bot', K_REVISION: 'okx-chat-bot-00001-abc' });
@@ -516,6 +658,54 @@ describe('okx-chat-bot daemon supervision (smoke)', () => {
await rm(home, { recursive: true, force: true });
}
});
+
+ // A newly elected AI lane only reaches the spawned subsession through the
+ // daemon's environment, which is fixed at spawn time, so the lane change is
+ // not real until the daemon is replaced. The replacement must leave exactly
+ // one daemon: the SIGTERM fires the exit handler's own restart as well, and
+ // two daemons race for one lock and one XMTP identity.
+ it('replaces the daemon exactly once when a lane change demands it', async () => {
+ const home = await mkdtemp(join(tmpdir(), 'okx-bot-restart-'));
+ const cfg = loadConfig({
+ OKX_BOT_HOME: home,
+ // node with a junk arg exits on its own terms; what is under test is that
+ // a restart yields exactly one live child, not zero and not two.
+ OKX_BOT_DAEMON_BIN: process.execPath,
+ OKX_BOT_RESTART_BASE_MS: '20',
+ OKX_BOT_RESTART_MAX_MS: '40',
+ });
+ const supervisor = createSupervisor(cfg, paths(cfg));
+ try {
+ supervisor.start();
+ const first = supervisor.stats().pid;
+ expect(first).toBeTruthy();
+ await supervisor.restart('test', 1_000);
+ const second = supervisor.stats().pid;
+ expect(second).toBeTruthy();
+ expect(second).not.toBe(first);
+ } finally {
+ await supervisor.stop(1_000);
+ await rm(home, { recursive: true, force: true });
+ }
+ });
+
+ // A shutdown that lands mid-restart must not be undone by the restart's own
+ // respawn: the final snapshot runs after stop(), and a daemon brought back
+ // behind it would be copied live.
+ it('does not resurrect the daemon when a shutdown lands mid-restart', async () => {
+ const home = await mkdtemp(join(tmpdir(), 'okx-bot-restart-race-'));
+ const cfg = loadConfig({ OKX_BOT_HOME: home, OKX_BOT_DAEMON_BIN: process.execPath });
+ const supervisor = createSupervisor(cfg, paths(cfg));
+ try {
+ supervisor.start();
+ const restarting = supervisor.restart('test', 1_000);
+ await supervisor.stop(1_000);
+ await restarting;
+ expect(supervisor.stats().pid).toBe(null);
+ } finally {
+ await rm(home, { recursive: true, force: true });
+ }
+ });
});
describe('okx_chat_bot subsystem health', () => {
diff --git a/workers/okx-chat-bot/README.md b/workers/okx-chat-bot/README.md
index 34e767787..31ce96459 100644
--- a/workers/okx-chat-bot/README.md
+++ b/workers/okx-chat-bot/README.md
@@ -46,9 +46,9 @@ testable without a daemon, a wallet, or a network.
| File | Role |
|------|------|
| `index.js` | Entrypoint. Boot order, session probe, heartbeat, ops alerts, graceful shutdown. |
-| `config.js` | Env-driven config (`loadConfig`, `paths`) and AI-provider selection (`resolveProvider`). |
+| `config.js` | Env-driven config (`loadConfig`, `paths`) and the AI lane chain (`providerLanes`, `resolveProviderChain`, `applyLane`). |
| `cli.js` | Timeout-bounded, non-throwing wrappers around the `okx-a2a` and `onchainos` binaries. |
-| `provider.js` | Asks the AI provider whether it will actually serve this host, and hands codex its key. |
+| `provider.js` | Probes each AI lane (`probeLane`), elects the first that serves (`electProvider`), and hands codex its key. |
| `session.js` | Pure health `classify()` plus `loginInstructions()`, the exact commands a human runs. |
| `health-server.js` | The HTTP surface: strict `/readyz`, always-200 liveness, and the `remedy` payload. |
| `state.js` | Tar the wallet/XMTP identity to GCS and restore it on boot (`snapshotState`, `restoreState`). |
@@ -102,12 +102,15 @@ locally with no code change. Defaults are the production posture.
| `OKX_BOT_STATE_BUCKET` | unset | GCS bucket for the state snapshot. Unset means ephemeral mode. |
| `OKX_BOT_STATE_OBJECT` | `okx-chat-bot/state.tar.gz` | Object name within that bucket. |
| `OKX_BOT_REPO_ROOT` | `/app` | Where the briefing and skills are read from. |
-| `OKX_BOT_AI_PROVIDER` | auto | Pin the provider (`claude`, `codex`, `hermes`, `openclaw`). |
+| `OKX_BOT_AI_PROVIDER` | auto | Pin the provider (`claude`, `codex`, `hermes`, `openclaw`). Narrows the chain to that CLI's lanes rather than emptying it. |
| `CLAUDE_CODE_USE_VERTEX` | unset | `1` routes the Claude subsession to Vertex AI, authenticated by the runtime service account. The production posture: no secret exists to leak, rotate, or forget. |
| `ANTHROPIC_VERTEX_PROJECT_ID` | unset | The GCP project Vertex bills. Required alongside the flag above. |
| `CLOUD_ML_REGION` | `global` | Vertex region for both the subsession and the credential probe. |
| `OKX_A2A_AI_PERMISSION_PRESET` | unset | `bypass` on a headless host. Without it the subsession stalls on a tool-approval prompt nobody is there to answer, which the buyer experiences as an unresponsive bot. |
-| `OKX_BOT_PROVIDER_PROBE_MS` | `900000` | How often the provider's own API is asked whether the credential still works. |
+| `OKX_BOT_ANTHROPIC_BASE_URL` | unset | Base URL of an Anthropic-wire-format gateway. The CLI appends `/v1/messages`, so OpenRouter's is `https://openrouter.ai/api`, **not** `.../api/v1`. |
+| `OKX_BOT_ANTHROPIC_AUTH_TOKEN` | unset | That gateway's credential. Both it and the base URL are required, or no gateway lane exists. |
+| `OKX_BOT_ANTHROPIC_MODEL` | unset | Model id in the gateway's catalog, e.g. `anthropic/claude-sonnet-4.6`. |
+| `OKX_BOT_PROVIDER_PROBE_MS` | `900000` | How often every lane's own API is asked whether it will still serve. Also how quickly a recovered lane is picked up. |
| `OKX_BOT_HOST_LABEL` | auto | Name this host on every beat. Cloud Run names itself from `K_SERVICE`. |
| `OKX_BOT_HOST_DURABLE` | unset | Set to `1` to claim a non-Cloud-Run host stays up on its own. |
| `OKX_BOT_DAEMON_BIN` | `okx-a2a` | The XMTP daemon binary the supervisor owns. |
@@ -118,15 +121,40 @@ locally with no code change. Defaults are the production posture.
| `OKX_BOT_RESTART_BASE_MS` | `2000` | Daemon restart backoff floor. |
| `OKX_BOT_RESTART_MAX_MS` | `60000` | Daemon restart backoff ceiling. |
-**Provider selection is by credential, not by preference.** A provider CLI with
-no key spawns, fails to authenticate, and produces exactly the symptom this
-worker exists to kill: silence on the buyer's side. So an explicit
-`OKX_BOT_AI_PROVIDER` wins; otherwise Vertex (`CLAUDE_CODE_USE_VERTEX=1` plus a
-project) selects `claude`, then `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`
-selects `claude`, then `OPENAI_API_KEY` selects `codex`. A developer host carries
-no key at all: its claude CLI was logged in by a human, so an existing
-`$OKX_BOT_HOME/.claude/.credentials.json` also counts as a credential. With no
-credential at all the worker boots, logs an error, and reports
+### The AI chain: whichever lane is funded is the lane that serves
+
+The host used to hold exactly **one** AI lane, which is a chain with a single
+rung. On 2026-09-04 the rung snapped: this GCP project's Vertex access started
+answering `PERMISSION_DENIED: Lightning dunning decision is deny`, and the bot
+went on receiving every buyer message and authoring no reply at all, for days,
+while other credentials sat unused on the same service.
+
+So [`providerLanes()`](config.js) builds an ordered chain and
+[`electProvider()`](provider.js) probes it, best lane first, and elects the first
+one that actually answers. The election re-runs every
+`OKX_BOT_PROVIDER_PROBE_MS`, so a lane that recovers (or one the owner funds) is
+picked up **with no deploy**: the config is re-asserted, codex is re-logged-in if
+it is the new lane, and the daemon is restarted so the subsession it spawns
+carries the new credentials.
+
+| # | Lane | Selected by | Why here |
+|---|---|---|---|
+| 1 | `vertex` | `CLAUDE_CODE_USE_VERTEX=1` + a project | GCP credits, authenticated by the runtime service account. Nothing to mint, rotate or forget, and the spend lands on the pool the platform prefers over any paid third-party API. |
+| 2 | `anthropic-key` | `ANTHROPIC_API_KEY` | A first-party key. |
+| 3 | `anthropic-gateway` | `OKX_BOT_ANTHROPIC_BASE_URL` + `..._AUTH_TOKEN` | Any service that speaks the Anthropic wire format (OpenRouter serves one). Opt-in, and last, because it bills a third-party account per token. |
+| 4 | `anthropic-login` | `$OKX_BOT_HOME/.claude/.credentials.json` | A developer host, whose `claude` CLI a human logged in. |
+| 5 | `openai-key` | `OPENAI_API_KEY` | The codex CLI. |
+
+**Every lane spawns a genuinely agentic CLI, and that is not negotiable.** The
+adapter does not read a reply out of the CLI's stdout: the spawned subsession
+sends the reply itself through `okx-a2a` and drives the task lifecycle (accept /
+negotiate / deliver). A one-shot completion call would not just answer worse, it
+would break the lifecycle. The gateway lane exists precisely because it keeps the
+same `claude` CLI and only moves its endpoint.
+
+A provider CLI with no key spawns, fails to authenticate, and produces exactly
+the symptom this worker exists to kill: silence on the buyer's side. So with an
+empty chain the worker boots, logs an error, and reports
`ai_provider_uncredentialed` rather than pretending to be healthy.
**Vertex wins over every key on purpose.** The credential is the runtime service
@@ -144,13 +172,27 @@ reads like an IAM problem) and the `openai-api-key` secret's account answers
`billing_not_active`. A presence check calls either one green, the host reports
ready, a buyer's message lands, and no reply is ever authored.
-So [provider.js](provider.js) asks the provider itself: one tiny request at boot
+So [provider.js](provider.js) asks each lane's own API: one tiny request at boot
and every `OKX_BOT_PROVIDER_PROBE_MS`, classified into `ok`, `unauthorized`,
`unreachable`, or `unprobed`. Only `unauthorized` fails readiness. `unreachable`
deliberately does not: a provider outage is transient and self-heals, and paging
a human for something no human can fix is how alerts stop being read. An
interactive OAuth grant is `unprobed`, because the CLI refreshes that grant
-itself and no endpoint here can prove it without reimplementing the refresh.
+itself and no endpoint here can prove it without reimplementing the refresh, and
+`unprobed` is the one non-`ok` verdict a host may still be elected on.
+
+Every lane's verdict, not just the elected one's, is on `/readyz` under
+`provider.chain` and in the `remedy`. A human deciding which credential to fund
+needs the whole picture, not the news that one of them was refused.
+
+**A gateway lane is elected on a 2xx and nothing else.** `classifyProbeStatus()`
+reads a 400 or a 404 as proof the credential works, because api.anthropic.com had
+to authenticate the caller before it could object to the request. That reasoning
+does not survive a gateway: a base URL one path segment too deep, or a model id
+from a different catalog, answers exactly those statuses and then fails every
+real reply. Measured 2026-09-09, `https://openrouter.ai/api/v1` makes the CLI
+request `/api/v1/v1/messages` and answers 404 forever, which is why the base URL
+in [cloudbuild.yaml](cloudbuild.yaml) ends at `/api`.
**One thing about Vertex that could not be tested here.** The billing hold denies
every Vertex call on this project, so the credential probe was verified against
@@ -297,15 +339,25 @@ private bucket and nowhere else.
Skipping the seed is not a failure; it just leaves the OTP the first boot would
otherwise have asked for.
-### Switching off Vertex
+### Adding a lane
Never patch an AI key onto the service by hand. `--set-secrets` in the deploy step
replaces the whole secret set, so a hand-added key survives exactly until the next
-deploy and then vanishes without a single error line. To use a key instead of
-Vertex, edit [cloudbuild.yaml](cloudbuild.yaml): drop `CLAUDE_CODE_USE_VERTEX`
-from `--set-env-vars` (it wins over every key) and add
-`ANTHROPIC_API_KEY=anthropic-api-key:latest` to `--set-secrets`, after creating
-that secret and granting `three-ws@` `secretAccessor` on it.
+deploy and then vanishes without a single error line. A lane belongs in
+[cloudbuild.yaml](cloudbuild.yaml), where the next deploy carries it too:
+
+- **A first-party Anthropic key**: create the secret, grant `three-ws@`
+ `secretAccessor` on it, and add `ANTHROPIC_API_KEY=anthropic-api-key:latest` to
+ `--set-secrets`. It ranks below Vertex, so nothing else has to change; drop
+ `CLAUDE_CODE_USE_VERTEX` only if you want the key to lead.
+- **A gateway**: set `OKX_BOT_ANTHROPIC_BASE_URL` and `OKX_BOT_ANTHROPIC_MODEL` in
+ `--set-env-vars` and `OKX_BOT_ANTHROPIC_AUTH_TOKEN=<secret>:latest` in
+ `--set-secrets`. The deploy already carries the OpenRouter one.
+
+Because the chain elects on a live probe rather than on presence, a lane with no
+funds behind it is inert (it probes `unauthorized` and is never elected) and
+costs nothing to leave configured. That is the point: funding it is then a
+billing action, not a deploy.
Shutdown order matters and is handled on SIGTERM: the daemon is stopped **before**
the final snapshot, so the sqlite files are quiesced rather than copied mid-write.
diff --git a/workers/okx-chat-bot/cli.js b/workers/okx-chat-bot/cli.js
index b4c655f3f..fc972927d 100644
--- a/workers/okx-chat-bot/cli.js
+++ b/workers/okx-chat-bot/cli.js
@@ -20,7 +20,7 @@ const DEFAULT_TIMEOUT_MS = 60_000;
* @param {ReturnType<import('./config.js').loadConfig>} cfg
*/
export function cliEnv(cfg) {
- return {
+ const env = {
...process.env,
HOME: cfg.home,
// The onchainos installer drops its binary in $HOME/.local/bin, but the
@@ -28,6 +28,14 @@ export function cliEnv(cfg) {
// replaces $HOME. Keep both on PATH so a local run works either way.
PATH: `${cfg.home}/.local/bin:${process.env.PATH || ''}`,
};
+ // The elected AI lane's overlay. A `null` value UNSETS the variable, which is
+ // the only way a gateway lane can stop the spawned CLI from still reaching for
+ // the Vertex transport this container was booted with.
+ for (const [k, v] of Object.entries(cfg.activeLane?.env || {})) {
+ if (v === null || v === undefined) delete env[k];
+ else env[k] = String(v);
+ }
+ return env;
}
/**
diff --git a/workers/okx-chat-bot/cloudbuild.yaml b/workers/okx-chat-bot/cloudbuild.yaml
index 768de0e1d..7f9352187 100644
--- a/workers/okx-chat-bot/cloudbuild.yaml
+++ b/workers/okx-chat-bot/cloudbuild.yaml
@@ -110,15 +110,24 @@ steps:
# there to answer, and the buyer experiences that as an unresponsive bot.
# CLAUDE_CODE_USE_VERTEX points the subsession at Vertex AI, authenticated
# by three-ws@ through ADC (roles/aiplatform.user, already granted).
- - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global
+ #
+ # OKX_BOT_ANTHROPIC_* is the last rung of the AI chain (providerLanes in
+ # config.js): an Anthropic-wire-format gateway the same `claude` CLI can be
+ # pointed at, so a lane change never costs the task lifecycle. It is here
+ # rather than in a runbook because the whole point of the chain is that a
+ # funded lane is used WITHOUT a deploy: while the OpenRouter account has no
+ # credit this lane simply probes 402 and is never elected, and the minute it
+ # is funded the next 15-minute probe elects it on its own. The base URL is
+ # the prefix the CLI appends /v1/messages to, so it ends at /api, NOT /api/v1
+ # (measured 2026-09-09: one segment too deep answers 404 on every reply).
+ - --set-env-vars=NODE_ENV=production,OKX_BOT_HOME=/state,OKX_BOT_REPO_ROOT=/app,OKX_BOT_AGENT_ID=2632,OKX_BOT_STATE_BUCKET=${_STATE_BUCKET},OKX_A2A_AI_PERMISSION_PRESET=bypass,CLAUDE_CODE_USE_VERTEX=1,ANTHROPIC_VERTEX_PROJECT_ID=$PROJECT_ID,CLOUD_ML_REGION=global,OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6
# An API key, if one is ever preferred over Vertex, belongs HERE and not
# patched on afterwards: --set-secrets replaces the whole secret set, so a
# hand-added key is silently stripped by the next deploy. Add
# ANTHROPIC_API_KEY=anthropic-api-key:latest (or, for codex,
- # OPENAI_API_KEY=openai-api-key:latest plus OKX_BOT_AI_PROVIDER=codex above)
- # once the secret exists; resolveProvider in config.js prefers Vertex over
- # both, so drop CLAUDE_CODE_USE_VERTEX when you mean to use the key.
- - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest
+ # OPENAI_API_KEY=openai-api-key:latest) once the secret exists; the chain
+ # ranks Vertex above both, and pinning is only needed to force one lane.
+ - --set-secrets=DATABASE_URL=okx-chat-bot-database-url:latest,OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest
# Liveness only. Readiness is deliberately NOT wired to Cloud Run: a
# logged-out session fails /readyz and no restart can fix it, so probing it
# here would put the service in a crash loop instead of paging a human.
diff --git a/workers/okx-chat-bot/config.js b/workers/okx-chat-bot/config.js
index 1a68d8779..4025b817c 100644
--- a/workers/okx-chat-bot/config.js
+++ b/workers/okx-chat-bot/config.js
@@ -14,83 +14,193 @@ const num = (v, fallback) => {
};
/**
- * Pick the AI provider the okx-a2a adapter should spawn for chat replies, and
- * say which transport authenticates it.
+ * Every AI lane this host could authenticate through, best first.
*
- * The adapter does NOT read a reply out of the CLI's stdout, the spawned AI
- * subsession sends the reply itself through the `okx-a2a` CLI and drives the
- * task lifecycle (accept / negotiate / deliver). So the provider must be a
- * genuinely agentic CLI with tool access, not a one-shot completion call.
+ * There used to be exactly one. That is a chain with a single rung, and on
+ * 2026-09-04 the rung snapped: the GCP project's Vertex access started answering
+ * `PERMISSION_DENIED: Lightning dunning decision is deny` (a billing hold that
+ * reads like an IAM fault) and the host sat on that one dead lane, unable to
+ * author a single reply, while other credentials the project holds went
+ * unconsidered. Whichever lane is funded should be the lane that serves, and the
+ * host should move to it by itself.
*
- * Selection is by credential, because a provider CLI with no key spawns, fails
- * to authenticate, and produces the exact symptom this worker exists to kill:
- * silence on the buyer's side.
+ * Order is a policy, not a preference:
*
- * OKX_BOT_AI_PROVIDER pin explicitly (claude | codex | hermes | openclaw)
- * CLAUDE_CODE_USE_VERTEX=1 → claude on Vertex AI, authenticated by the
- * runtime service account through ADC. No secret
- * exists to leak, rotate, or forget, and the spend
- * lands on the GCP credit pool the platform already
- * prefers, so this transport wins over every key.
- * ANTHROPIC_API_KEY /
- * CLAUDE_CODE_OAUTH_TOKEN → claude on api.anthropic.com
- * OPENAI_API_KEY → codex
+ * vertex GCP credits, authenticated by the runtime service account.
+ * No secret exists to leak, rotate or forget, and the spend
+ * lands on the pool the platform already prefers.
+ * anthropic-key a first-party ANTHROPIC_API_KEY.
+ * anthropic-gateway any Anthropic-wire-format gateway (OpenRouter serves one
+ * at /api/v1/messages). Opt-in: it only exists when an
+ * operator sets its base URL and token, because it bills a
+ * third-party account per token.
+ * anthropic-oauth an interactive `claude` login on a developer host.
+ * openai-key the codex CLI on OPENAI_API_KEY.
*
- * A headless host is credentialed by env alone, but a developer host running the
- * stopgap has no key at all: its claude CLI was logged in interactively and keeps
- * the grant in ~/.claude/.credentials.json. Ignoring that file reports
- * `ai_provider_uncredentialed` on a host whose adapter demonstrably authors
- * replies, which is a false red, and a false red trains people to ignore the
- * signal that this worker exists to raise.
- *
- * `credentialed` only answers "is a credential configured". Whether that
- * credential still WORKS is a different question, and on this project both
- * answers have differed: the GCP project's Vertex access and the OpenAI key are
- * each present and each refuse to serve. probeProvider() in provider.js asks the
- * provider itself, and its verdict is what readiness is judged on.
+ * Every lane carries the env its CLI needs, so electing one is applying that
+ * overlay and respawning the daemon. A `null` value means "unset this", which is
+ * what lets the gateway lane switch Vertex off for the subsession it spawns.
*
* @param {NodeJS.ProcessEnv} env
- * @param {string} [home] where the CLI keeps an interactive login
- * @returns {{ provider: string, transport: string, reason: string, credentialed: boolean }}
+ * @param {string} home where a CLI keeps an interactive login
+ * @returns {Array<{ id: string, provider: string, transport: string, reason: string, env: Record<string,string|null> }>}
*/
-export function resolveProvider(env = process.env, home = env.OKX_BOT_HOME || homedir()) {
- const pinned = (env.OKX_BOT_AI_PROVIDER || '').trim().toLowerCase();
+export function providerLanes(env = process.env, home = env.OKX_BOT_HOME || homedir()) {
+ const lanes = [];
const onVertex =
(env.CLAUDE_CODE_USE_VERTEX === '1' || env.CLAUDE_CODE_USE_VERTEX === 'true') &&
!!(env.ANTHROPIC_VERTEX_PROJECT_ID || env.GOOGLE_CLOUD_PROJECT);
- const hasClaudeKey = !!(env.ANTHROPIC_API_KEY || env.CLAUDE_CODE_OAUTH_TOKEN);
- const hasClaudeLogin = existsSync(join(home, '.claude', '.credentials.json'));
- const claude = onVertex
- ? { transport: 'vertex', reason: 'Vertex AI, authenticated by the runtime service account (no key to rotate)' }
- : hasClaudeKey
- ? { transport: 'api-key', reason: 'ANTHROPIC_API_KEY present' }
- : hasClaudeLogin
- ? { transport: 'oauth-login', reason: 'claude CLI holds an interactive login' }
- : { transport: 'none', reason: 'no Anthropic credential' };
- const codex = env.OPENAI_API_KEY
- ? { transport: 'api-key', reason: 'OPENAI_API_KEY present' }
- : { transport: 'none', reason: 'no OpenAI credential' };
-
- if (pinned) {
- const t = pinned === 'claude' ? claude : pinned === 'codex' ? codex : { transport: 'external', reason: 'provider supplies its own auth' };
- return {
- provider: pinned,
- transport: t.transport,
- reason: `pinned by OKX_BOT_AI_PROVIDER (${t.reason})`,
- credentialed: t.transport !== 'none',
- };
+ if (onVertex) {
+ lanes.push({
+ id: 'vertex',
+ provider: 'claude',
+ transport: 'vertex',
+ reason: 'Vertex AI, authenticated by the runtime service account (no key to rotate)',
+ credentialed: true,
+ env: {},
+ });
+ }
+ if (env.ANTHROPIC_API_KEY) {
+ lanes.push({
+ id: 'anthropic-key',
+ provider: 'claude',
+ transport: 'api-key',
+ reason: 'ANTHROPIC_API_KEY present',
+ credentialed: true,
+ env: { CLAUDE_CODE_USE_VERTEX: null, ANTHROPIC_BASE_URL: null },
+ });
+ }
+ const gateway = gatewayLane(env);
+ if (gateway) lanes.push(gateway);
+ if (env.CLAUDE_CODE_OAUTH_TOKEN) {
+ lanes.push({
+ id: 'anthropic-oauth-token',
+ provider: 'claude',
+ transport: 'api-key',
+ reason: 'CLAUDE_CODE_OAUTH_TOKEN present',
+ credentialed: true,
+ env: { CLAUDE_CODE_USE_VERTEX: null, ANTHROPIC_BASE_URL: null },
+ });
}
- if (claude.transport !== 'none') {
- return { provider: 'claude', transport: claude.transport, reason: claude.reason, credentialed: true };
+ if (existsSync(join(home, '.claude', '.credentials.json'))) {
+ lanes.push({
+ id: 'anthropic-login',
+ provider: 'claude',
+ transport: 'oauth-login',
+ reason: 'claude CLI holds an interactive login',
+ credentialed: true,
+ env: {},
+ });
}
- if (codex.transport !== 'none') {
- return { provider: 'codex', transport: codex.transport, reason: `${codex.reason} (no Anthropic credential)`, credentialed: true };
+ if (env.OPENAI_API_KEY) {
+ lanes.push({
+ id: 'openai-key',
+ provider: 'codex',
+ transport: 'api-key',
+ reason: 'OPENAI_API_KEY present',
+ credentialed: true,
+ env: {},
+ });
+ }
+ return lanes;
+}
+
+/**
+ * The Anthropic-compatible gateway lane, or null when none is configured.
+ *
+ * `claude` speaks one wire format and reads its endpoint from ANTHROPIC_BASE_URL,
+ * so any service that serves `/v1/messages` is a usable lane for the same agentic
+ * CLI the adapter already spawns. That keeps the task lifecycle intact, which a
+ * one-shot completion responder would break.
+ *
+ * Deliberately opt-in: unlike Vertex it bills a third-party account per token, so
+ * it must be a decision an operator made, never something the host picks up from
+ * an unrelated key that happened to be on the service.
+ */
+function gatewayLane(env) {
+ const base = (env.OKX_BOT_ANTHROPIC_BASE_URL || '').trim().replace(/\/+$/, '');
+ const token = (env.OKX_BOT_ANTHROPIC_AUTH_TOKEN || '').trim();
+ if (!base || !token) return null;
+ const model = (env.OKX_BOT_ANTHROPIC_MODEL || '').trim();
+ let label = base;
+ try {
+ label = new URL(base).host;
+ } catch {
+ /* a malformed base URL still names itself in the reason */
}
return {
+ id: 'anthropic-gateway',
provider: 'claude',
- transport: 'none',
- reason: 'no AI-provider credential found, chat replies will fail until one is set',
- credentialed: false,
+ transport: 'gateway',
+ reason: `Anthropic-compatible gateway at ${label}${model ? ` (${model})` : ''}`,
+ credentialed: true,
+ env: {
+ CLAUDE_CODE_USE_VERTEX: null,
+ ANTHROPIC_BASE_URL: base,
+ ANTHROPIC_AUTH_TOKEN: token,
+ ANTHROPIC_API_KEY: null,
+ ...(model ? { ANTHROPIC_MODEL: model, ANTHROPIC_SMALL_FAST_MODEL: model } : {}),
+ },
+ };
+}
+
+/**
+ * Narrow the chain to what an explicit pin allows.
+ *
+ * A pin names a provider CLI, not a lane, so `OKX_BOT_AI_PROVIDER=claude` keeps
+ * every Anthropic lane and drops codex. A pin naming a provider with no lane at
+ * all still returns an honest, uncredentialed single entry rather than silently
+ * electing something the operator did not ask for.
+ */
+export function pinnedLanes(lanes, pinned) {
+ if (!pinned) return lanes;
+ const kept = lanes.filter((l) => l.provider === pinned);
+ if (kept.length) return kept.map((l) => ({ ...l, reason: `pinned by OKX_BOT_AI_PROVIDER (${l.reason})` }));
+ const external = pinned !== 'claude' && pinned !== 'codex';
+ return [
+ {
+ id: `${pinned}-pinned`,
+ provider: pinned,
+ transport: external ? 'external' : 'none',
+ reason: external
+ ? 'pinned by OKX_BOT_AI_PROVIDER (provider supplies its own auth)'
+ : `pinned by OKX_BOT_AI_PROVIDER (no ${pinned === 'claude' ? 'Anthropic' : 'OpenAI'} credential)`,
+ env: {},
+ credentialed: external,
+ },
+ ];
+}
+
+/**
+ * Resolve the whole chain: the lanes to try, in order, and which one leads.
+ *
+ * `credentialed` on a lane only answers "is a credential configured". Whether it
+ * still WORKS is a different question, and on this project the two answers have
+ * differed in both directions: the GCP project's Vertex access and the OpenAI key
+ * are each present and each refuse to serve. electProvider() in provider.js asks
+ * each lane's own API, and its verdict is what readiness is judged on.
+ *
+ * The adapter does NOT read a reply out of the CLI's stdout: the spawned AI
+ * subsession sends the reply itself through the `okx-a2a` CLI and drives the task
+ * lifecycle (accept / negotiate / deliver). So every lane must spawn a genuinely
+ * agentic CLI with tool access, never a one-shot completion call.
+ *
+ * @returns {{ lanes: Array<object>, head: object }}
+ */
+export function resolveProviderChain(env = process.env, home = env.OKX_BOT_HOME || homedir()) {
+ const pinned = (env.OKX_BOT_AI_PROVIDER || '').trim().toLowerCase();
+ const lanes = pinnedLanes(providerLanes(env, home), pinned);
+ if (lanes.length) return { lanes, head: lanes[0] };
+ return {
+ lanes: [],
+ head: {
+ id: 'none',
+ provider: 'claude',
+ transport: 'none',
+ reason: 'no AI-provider credential found, chat replies will fail until one is set',
+ env: {},
+ credentialed: false,
+ },
};
}
@@ -104,9 +214,29 @@ export function resolveHost(env = process.env) {
return { label: `local:${hostname()}`, durable: false };
}
+/**
+ * Point the config at one lane of the chain.
+ *
+ * Every consumer (the heartbeat, the health body, cliEnv) reads the flat
+ * `provider*` fields, so electing a lane is one mutation here rather than a
+ * plumbing change in five files. Called at boot with the head of the chain and
+ * again whenever a probe elects a different one.
+ *
+ * @param {ReturnType<loadConfig>} cfg
+ * @param {{ id: string, provider: string, transport: string, reason: string, env: object }} lane
+ */
+export function applyLane(cfg, lane) {
+ cfg.activeLane = lane;
+ cfg.provider = lane.provider;
+ cfg.providerTransport = lane.transport;
+ cfg.providerReason = lane.reason;
+ cfg.providerCredentialed = lane.credentialed === true;
+ return cfg;
+}
+
export function loadConfig(env = process.env) {
const home = env.OKX_BOT_HOME || homedir();
- const provider = resolveProvider(env, home);
+ const chain = resolveProviderChain(env, home);
const host = resolveHost(env);
return {
// HOME for both CLIs. Everything durable lands under it:
@@ -128,10 +258,15 @@ export function loadConfig(env = process.env) {
stateBucket: (env.OKX_BOT_STATE_BUCKET || '').trim(),
stateObject: env.OKX_BOT_STATE_OBJECT || 'okx-chat-bot/state.tar.gz',
- provider: provider.provider,
- providerTransport: provider.transport,
- providerReason: provider.reason,
- providerCredentialed: provider.credentialed,
+ // The lanes this host may authenticate through, best first, and the one it
+ // is currently using. A single-rung chain is what left the host sitting on
+ // a dead Vertex credential for days; see providerLanes().
+ chain: chain.lanes,
+ activeLane: chain.head,
+ provider: chain.head.provider,
+ providerTransport: chain.head.transport,
+ providerReason: chain.head.reason,
+ providerCredentialed: chain.head.credentialed === true,
// How often the provider's own API is asked whether the credential still
// works. A configured credential is not a working one: on this project the
diff --git a/workers/okx-chat-bot/health-server.js b/workers/okx-chat-bot/health-server.js
index e53e7bbbd..198267dbd 100644
--- a/workers/okx-chat-bot/health-server.js
+++ b/workers/okx-chat-bot/health-server.js
@@ -44,6 +44,17 @@ export function statusBody(cfg, live, daemonStats, bootAt) {
verdict: live.providerProbe?.code ?? null,
verdictDetail: live.providerProbe?.detail ?? null,
verdictAt: live.providerProbe?.checkedAt ?? null,
+ // Which lane of the chain is serving, and what every other lane
+ // answered. A human deciding which credential to fund needs the whole
+ // picture, not just the fact that the elected one was refused.
+ lane: cfg.activeLane?.id ?? null,
+ chain: (live.providerLanes || []).map((l) => ({
+ lane: l.lane,
+ transport: l.transport,
+ verdict: l.code,
+ detail: l.detail,
+ checkedAt: l.checkedAt,
+ })),
},
daemon: { status: live.daemon, ...daemonStats },
session: live.wallet ? { loggedIn: live.wallet.loggedIn, email: live.wallet.email } : null,
@@ -54,7 +65,7 @@ export function statusBody(cfg, live, daemonStats, bootAt) {
checkedAt: live.checkedAt,
};
if (live.verdict.needsHumanLogin) body.remedy = loginInstructions(live.login?.loginUrl, live.login?.authSessionId);
- else if (live.verdict.reason === 'ai_provider_unauthorized') body.remedy = providerInstructions(live.providerProbe?.detail);
+ else if (live.verdict.reason === 'ai_provider_unauthorized') body.remedy = providerInstructions(live.providerProbe?.detail, live.providerLanes);
return body;
}
diff --git a/workers/okx-chat-bot/index.js b/workers/okx-chat-bot/index.js
index 148cf1bda..5c0503805 100644
--- a/workers/okx-chat-bot/index.js
+++ b/workers/okx-chat-bot/index.js
@@ -22,10 +22,10 @@
import { mkdir } from 'node:fs/promises';
import { sendOpsAlert } from '../../api/_lib/alerts.js';
import { sql } from '../../api/_lib/db.js';
-import { loadConfig, paths } from './config.js';
+import { applyLane, loadConfig, paths } from './config.js';
import { agentRefresh, beginLogin, daemonStatus, exec, walletStatus } from './cli.js';
import { startHealthServer } from './health-server.js';
-import { loginCodex, probeProvider } from './provider.js';
+import { electProvider, loginCodex } from './provider.js';
import { classify, loginInstructions } from './session.js';
import { restoreState, snapshotState } from './state.js';
import { createSupervisor } from './supervisor.js';
@@ -49,6 +49,10 @@ const live = {
// which is why classify() treats a missing verdict as "no opinion" rather than
// as a failure: a host must not read red for a probe that has not run yet.
providerProbe: null,
+ // What EVERY lane in the chain last answered. The elected lane's verdict is
+ // `providerProbe`; this is the whole picture, so a human reading /readyz can
+ // see which credential to fund rather than only that one of them was refused.
+ providerLanes: [],
};
async function heartbeat(cfg, supervisor) {
@@ -68,7 +72,9 @@ async function heartbeat(cfg, supervisor) {
provider: cfg.provider,
providerTransport: cfg.providerTransport,
providerCredentialed: cfg.providerCredentialed,
+ providerLane: cfg.activeLane?.id ?? null,
providerVerdict: live.providerProbe?.code ?? null,
+ providerChain: live.providerLanes.map((l) => ({ lane: l.lane, code: l.code })),
daemonRestarts: supervisor.stats().restarts,
checkedAt: live.checkedAt,
};
@@ -113,20 +119,46 @@ async function maybeAlert(cfg, verdict) {
}).catch((err) => log.warn('ops alert failed', { err: err?.message }));
}
-// Ask the provider whether it still serves this host. Kept off the session
-// probe's path: it is a network call to a third party, and a slow one must not
-// delay the verdict about whether XMTP is delivering. A transient `unreachable`
-// answer is remembered as-is, because classify() deliberately does not fail
-// readiness on it.
-async function probeProviderCredential(cfg) {
+// Ask every AI lane whether it still serves this host, and move to the best one.
+//
+// Kept off the session probe's path: these are network calls to third parties,
+// and a slow one must not delay the verdict about whether XMTP is delivering. A
+// transient `unreachable` answer is remembered as-is, because classify()
+// deliberately does not fail readiness on it.
+//
+// Election is what makes a dead credential survivable. The host used to hold one
+// lane, so the Vertex billing hold on 2026-09-04 left it unable to author a
+// single reply even though other credentials were configured. Now the lane that
+// answers is the lane that serves, and a lane that recovers is picked up on the
+// next tick with no deploy.
+async function electProviderLane(cfg, supervisor) {
const before = live.providerProbe?.code ?? null;
- live.providerProbe = await probeProvider(cfg);
- if (live.providerProbe.code !== before) {
+ const beforeLane = cfg.activeLane?.id ?? null;
+ const { elected, verdict, lanes } = await electProvider(cfg.chain);
+ live.providerLanes = lanes;
+ if (verdict) live.providerProbe = verdict;
+
+ if (elected && elected.id !== beforeLane) {
+ applyLane(cfg, elected);
+ log.info('provider lane elected', { lane: elected.id, transport: elected.transport, from: beforeLane });
+ // The adapter reads the provider out of ~/.okx-agent-task/config.toml, and
+ // the spawned CLI reads its credentials out of the daemon's environment.
+ // Both are set at spawn time, so a lane change is only real once the daemon
+ // has been replaced with one that carries it.
+ await exec(cfg, 'okx-a2a', ['config', 'provider', '--provider', cfg.provider], { timeoutMs: 30_000 });
+ const codexLogin = await loginCodex(cfg);
+ if (codexLogin.ran) log[codexLogin.ok ? 'info' : 'error']('codex login', codexLogin);
+ await supervisor.restart(`ai lane ${beforeLane || 'none'} -> ${elected.id}`);
+ }
+
+ if (live.providerProbe && live.providerProbe.code !== before) {
log[live.providerProbe.code === 'unauthorized' ? 'error' : 'info']('provider credential', {
provider: cfg.provider,
transport: cfg.providerTransport,
+ lane: live.providerProbe.lane,
code: live.providerProbe.code,
detail: live.providerProbe.detail,
+ lanes: lanes.map((l) => `${l.lane}=${l.code}`).join(' '),
});
}
}
@@ -200,6 +232,7 @@ async function main() {
hostDurable: cfg.hostDurable,
provider: cfg.provider,
providerReason: cfg.providerReason,
+ chain: cfg.chain.map((l) => l.id).join(' > ') || 'none',
stateBucket: cfg.stateBucket || null,
});
if (!cfg.providerCredentialed) {
@@ -247,12 +280,12 @@ async function main() {
cfg.snapshotMs,
);
const providerProbeTimer = setInterval(
- () => probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message })),
+ () => electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message })),
cfg.providerProbeMs,
);
- // Run the first credential probe immediately, but off the boot path: a slow
- // provider must delay the verdict, never the daemon that receives the chat.
- probeProviderCredential(cfg).catch((err) => log.warn('provider probe failed', { err: err?.message }));
+ // Run the first election immediately, but off the boot path: a slow provider
+ // must delay the verdict, never the daemon that receives the chat.
+ electProviderLane(cfg, supervisor).catch((err) => log.warn('provider probe failed', { err: err?.message }));
// A first verdict early enough to be useful. It can legitimately land before
// the daemon has claimed its lock, which classify() reports as
// `daemon_starting` rather than `daemon_down`, so an ordinary boot no longer
diff --git a/workers/okx-chat-bot/provider.js b/workers/okx-chat-bot/provider.js
index 429da54bc..5cc70298c 100644
--- a/workers/okx-chat-bot/provider.js
+++ b/workers/okx-chat-bot/provider.js
@@ -131,28 +131,132 @@ async function probeOpenAiKey(env) {
);
}
+async function probeAnthropicBearer(env, key) {
+ return postJson(
+ 'https://api.anthropic.com/v1/messages',
+ { authorization: `Bearer ${key}`, 'anthropic-version': '2023-06-01' },
+ { model: ANTHROPIC_PROBE_MODEL, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },
+ );
+}
+
/**
- * Ask the configured provider whether it will serve this host.
+ * Probe an Anthropic-wire-format gateway at its own base URL.
*
- * @param {ReturnType<import('./config.js').loadConfig>} cfg
- * @param {NodeJS.ProcessEnv} [env]
- * @returns {Promise<{ code: 'ok'|'unauthorized'|'unreachable'|'unprobed', detail: string, transport: string, checkedAt: number }>}
+ * `${base}/v1/messages` is the same path the CLI itself builds from
+ * ANTHROPIC_BASE_URL, so the probe fails exactly where a real reply would. That
+ * matters more than it sounds: OpenRouter's gateway lives at
+ * `https://openrouter.ai/api/v1/messages`, so its base URL is
+ * `https://openrouter.ai/api` and the intuitive `.../api/v1` is one path segment
+ * too deep. Configured that way the endpoint answers 404 forever.
+ *
+ * Gateways disagree about which header carries the credential (OpenRouter reads
+ * `Authorization`, Anthropic reads `x-api-key`), so both are sent: whichever the
+ * gateway reads, the other is ignored. The model id is the lane's own, because a
+ * gateway that authenticates fine still refuses an id from a different catalog.
+ *
+ * And unlike a first-party provider, a gateway's 400/404 is NOT proof the
+ * credential works. classifyProbeStatus() reads those as "the provider had to
+ * authenticate me before it could object to my request", which is true of
+ * api.anthropic.com and useless here: a wrong base URL or an unroutable model id
+ * produces exactly that status and then fails every real reply. So a gateway
+ * lane is elected on a 2xx and nothing else.
*/
-export async function probeProvider(cfg, env = process.env) {
- const wrap = (r) => ({ ...r, transport: cfg.providerTransport, checkedAt: Date.now() });
+async function probeGateway(lane) {
+ const base = String(lane.env.ANTHROPIC_BASE_URL || '').replace(/\/+$/, '');
+ const model = lane.env.ANTHROPIC_MODEL || ANTHROPIC_PROBE_MODEL;
+ const key = lane.env.ANTHROPIC_AUTH_TOKEN || '';
+ const url = `${base}/v1/messages`;
+ const verdict = await postJson(
+ url,
+ { authorization: `Bearer ${key}`, 'x-api-key': key, 'anthropic-version': '2023-06-01' },
+ { model, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] },
+ );
+ if (verdict.code === 'ok' && /returned (400|404)/.test(verdict.detail)) {
+ return {
+ code: 'unauthorized',
+ detail: `the gateway ${url} rejected the probe request (model ${model}): ${verdict.detail}. ` +
+ 'Check OKX_BOT_ANTHROPIC_BASE_URL (it must be the prefix the CLI appends /v1/messages to) and OKX_BOT_ANTHROPIC_MODEL.',
+ };
+ }
+ return verdict;
+}
- if (!cfg.providerCredentialed) {
- return wrap({ code: 'unauthorized', detail: 'no AI-provider credential is configured' });
+/**
+ * Ask ONE lane whether it will serve this host.
+ *
+ * The lane's env overlay is layered over the process env first, so a lane is
+ * probed exactly as the spawned CLI would experience it. Probing the ambient env
+ * instead is how a gateway lane would report on Vertex's credential and elect
+ * itself on someone else's verdict.
+ *
+ * @param {{ id: string, provider: string, transport: string, env: Record<string,string|null> }} lane
+ * @param {NodeJS.ProcessEnv} [ambient]
+ */
+export async function probeLane(lane, ambient = process.env) {
+ const env = { ...ambient };
+ for (const [k, v] of Object.entries(lane.env || {})) {
+ if (v === null || v === undefined) delete env[k];
+ else env[k] = String(v);
+ }
+ const wrap = (r) => ({ ...r, lane: lane.id, transport: lane.transport, checkedAt: Date.now() });
+
+ // Dispatch on the LANE's transport, never on what the ambient env happens to
+ // hold. A developer host running on an interactive grant usually also has an
+ // unrelated key in its environment, and probing that key would answer for a
+ // credential this lane does not use.
+ if (lane.transport === 'none') return wrap({ code: 'unauthorized', detail: 'no AI-provider credential is configured' });
+ if (lane.transport === 'vertex') return wrap(await probeVertex(env));
+ if (lane.transport === 'gateway') return wrap(await probeGateway(lane));
+ if (lane.transport === 'api-key' && lane.provider === 'claude') {
+ return env.ANTHROPIC_API_KEY
+ ? wrap(await probeAnthropicKey(env))
+ : wrap(await probeAnthropicBearer(env, env.CLAUDE_CODE_OAUTH_TOKEN || ''));
+ }
+ if (lane.transport === 'api-key' && lane.provider === 'codex' && env.OPENAI_API_KEY) {
+ return wrap(await probeOpenAiKey(env));
+ }
+
+ // An interactive OAuth grant is refreshed by the CLI itself; there is no
+ // endpoint this worker can call that proves the grant without reimplementing
+ // that refresh. Claiming a verdict here would be inventing one, so the probe
+ // abstains and the presence check stands.
+ return wrap({ code: 'unprobed', detail: `no credential probe exists for the ${lane.transport} transport` });
+}
+
+// Which verdict a host would rather be on when no lane outright works. `ok` is
+// the only one that serves a buyer; after that an unprovable grant beats a
+// provider having a bad minute, which beats a credential that was refused
+// outright. Election walks this order so the reported verdict is the best truth
+// available rather than whichever lane happened to be listed first.
+const VERDICT_RANK = { ok: 0, unprobed: 1, unreachable: 2, unauthorized: 3 };
+
+/**
+ * Probe the chain in order and elect the first lane that can actually serve.
+ *
+ * Stops at the first `ok`: the lanes are ordered by policy (GCP credits first,
+ * a metered third-party gateway last), so the first working one is also the one
+ * that should be used, and probing the rest would spend money to learn nothing.
+ *
+ * @param {Array<object>} chain
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {Promise<{ elected: object|null, verdict: object, lanes: Array<object> }>}
+ */
+export async function electProvider(chain, env = process.env) {
+ const lanes = [];
+ for (const lane of chain) {
+ const verdict = await probeLane(lane, env);
+ lanes.push(verdict);
+ if (verdict.code === 'ok') return { elected: lane, verdict, lanes };
+ }
+ if (!lanes.length) return { elected: null, verdict: null, lanes };
+ let best = 0;
+ for (let i = 1; i < lanes.length; i++) {
+ if (VERDICT_RANK[lanes[i].code] < VERDICT_RANK[lanes[best].code]) best = i;
}
- if (cfg.providerTransport === 'vertex') return wrap(await probeVertex(env));
- if (cfg.provider === 'claude' && env.ANTHROPIC_API_KEY) return wrap(await probeAnthropicKey(env));
- if (cfg.provider === 'codex' && env.OPENAI_API_KEY) return wrap(await probeOpenAiKey(env));
-
- // An interactive OAuth grant and a CLAUDE_CODE_OAUTH_TOKEN are refreshed by
- // the CLI itself; there is no endpoint this worker can call that proves the
- // grant without reimplementing that refresh. Claiming a verdict here would be
- // inventing one, so the probe abstains and the presence check stands.
- return wrap({ code: 'unprobed', detail: `no credential probe exists for the ${cfg.providerTransport} transport` });
+ // `unprobed` is the one non-ok verdict a host may still serve on: it means the
+ // credential could not be asked, not that it was refused.
+ const elected = lanes[best].code === 'unprobed' ? chain[best] : null;
+ return { elected, verdict: lanes[best], lanes };
}
/**
diff --git a/workers/okx-chat-bot/session.js b/workers/okx-chat-bot/session.js
index 7da220a1f..5fc154732 100644
--- a/workers/okx-chat-bot/session.js
+++ b/workers/okx-chat-bot/session.js
@@ -38,9 +38,13 @@ export function loginInstructions(loginUrl, authSessionId) {
* is the only option with no secret to mint, rotate or forget, and it bills the
* GCP credit pool the platform already prefers.
*/
-export function providerInstructions(detail = '') {
+export function providerInstructions(detail = '', lanes = []) {
+ const chain = lanes.length
+ ? ['', 'Every lane in the chain was asked; none can serve:', ...lanes.map((l) => ` ${l.lane} (${l.transport}): ${l.code} - ${l.detail}`)]
+ : [];
return [
`The provider refused this host's credential: ${detail}`,
+ ...chain,
'',
'1. Preferred (no secret): clear the project billing hold, then',
' gcloud run services update okx-chat-bot --region us-central1 \\',
@@ -52,6 +56,11 @@ export function providerInstructions(detail = '') {
' stripped by the next deploy).',
'3. Or reactivate billing on the OpenAI account behind the openai-api-key secret',
' and pin OKX_BOT_AI_PROVIDER=codex.',
+ '4. Or point the host at any funded Anthropic-wire-format gateway (config only,',
+ ' pre-approved). OpenRouter serves one at /api/v1/messages:',
+ ' gcloud run services update okx-chat-bot --region us-central1 \\',
+ ' --update-env-vars=OKX_BOT_ANTHROPIC_BASE_URL=https://openrouter.ai/api,OKX_BOT_ANTHROPIC_MODEL=anthropic/claude-sonnet-4.6 \\',
+ ' --update-secrets=OKX_BOT_ANTHROPIC_AUTH_TOKEN=OPENROUTER_API_KEY:latest',
];
}
diff --git a/workers/okx-chat-bot/supervisor.js b/workers/okx-chat-bot/supervisor.js
index 0a1f01dc5..9e5e51f20 100644
--- a/workers/okx-chat-bot/supervisor.js
+++ b/workers/okx-chat-bot/supervisor.js
@@ -18,6 +18,12 @@ import { log } from './log.js';
export function createSupervisor(cfg, p) {
let child = null;
let stopping = false;
+ // Set once, by stop(). A restart flips `stopping` on and off around its own
+ // SIGTERM, so it needs a flag that a real shutdown cannot be confused with:
+ // respawning a daemon after the shutdown path has begun would leave the final
+ // snapshot copying a live sqlite file, which is exactly what the stop-then-
+ // snapshot ordering exists to prevent.
+ let terminated = false;
let restarts = 0;
let backoffMs = cfg.restartBaseMs;
let timer = null;
@@ -73,27 +79,55 @@ export function createSupervisor(cfg, p) {
child.on('exit', (code, signal) => daemonDied({ code, signal }));
}
+ async function stopChild(timeoutMs) {
+ if (timer) clearTimeout(timer);
+ const proc = child;
+ if (!proc) return;
+ await new Promise((resolve) => {
+ const done = setTimeout(() => {
+ proc.kill('SIGKILL');
+ resolve(undefined);
+ }, timeoutMs);
+ proc.once('exit', () => {
+ clearTimeout(done);
+ resolve(undefined);
+ });
+ proc.kill('SIGTERM');
+ });
+ }
+
return {
start: spawnDaemon,
/** Stop the child and wait for it, so state is quiesced before a snapshot. */
async stop(timeoutMs = 10_000) {
stopping = true;
- if (timer) clearTimeout(timer);
- const proc = child;
- if (!proc) return;
- await new Promise((resolve) => {
- const done = setTimeout(() => {
- proc.kill('SIGKILL');
- resolve(undefined);
- }, timeoutMs);
- proc.once('exit', () => {
- clearTimeout(done);
- resolve(undefined);
- });
- proc.kill('SIGTERM');
- });
+ terminated = true;
+ await stopChild(timeoutMs);
log.info('daemon stopped', { restarts });
},
+
+ /**
+ * Replace the running daemon with one spawned from the current environment.
+ *
+ * The adapter hands the spawned AI subsession the daemon's own environment,
+ * fixed at spawn time, so a newly elected AI lane cannot reach the CLI any
+ * other way. Deliberately a full stop-then-start rather than a signal: the
+ * daemon holds a lock and a sqlite file, and only a clean exit quiesces both.
+ */
+ async restart(reason, timeoutMs = 10_000) {
+ if (stopping) return;
+ log.info('daemon restart', { reason, restarts });
+ // Suppress the exit handler's own restart for the duration: without this
+ // the SIGTERM below schedules a respawn AND the explicit spawn runs, and
+ // two daemons race for one lock and one XMTP identity.
+ stopping = true;
+ await stopChild(timeoutMs);
+ child = null;
+ if (terminated) return;
+ stopping = false;
+ backoffMs = cfg.restartBaseMs;
+ spawnDaemon();
+ },
stats() {
return { restarts, pid: child?.pid ?? null, uptimeMs: child ? Date.now() - startedAt : 0 };
},