Skip to content

Business operating system for Claude Code — 57 skills, 21 agents, smart daemon. Unified inbox (WhatsApp/Email/Slack/Telegram), autonomous PR merge, full-AWS monitoring, revenue (Stripe+RevenueCat), e-commerce (Shopify), marketing (Klaviyo/Meta/GA4), voice (Bland/ElevenLabs), APM (Datadog/NewRelic/OTEL), YOLO mode.

feat(rotation): never use Bedrock while OAuth available — model-var scrub, lease purge, live-session watchdog, post-swap continuation (#582)

Aurora Capitalbb88f87539e2
diff --git a/claude-ops/scripts/account-rotation/__tests__/bedrock-watchdog.test.mjs b/claude-ops/scripts/account-rotation/__tests__/bedrock-watchdog.test.mjs
new file mode 100644
index 00000000..1f4f5148
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/__tests__/bedrock-watchdog.test.mjs
@@ -0,0 +1,57 @@
+/**
+ * bedrock-watchdog tests (node --test).
+ *
+ * Run: node --test claude-ops/scripts/account-rotation/__tests__/bedrock-watchdog.test.mjs
+ *
+ * These are sandbox-safety + classification tests. The /proc and ss scanners are
+ * environment-dependent; the contract we test is that they NEVER throw and NEVER
+ * false-alarm, and that verifyBedrockSwaps classifies a dead (re-exec'd) PID as
+ * cleared.
+ */
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+
+import { scanBedrockSessions, scanBedrockNetwork, verifyBedrockSwaps } from '../bedrock-watchdog.mjs';
+
+test('scanBedrockSessions: never throws, returns an array', () => {
+ const r = scanBedrockSessions();
+ assert.ok(Array.isArray(r));
+ // Each entry, if any, has the documented shape.
+ for (const s of r) {
+ assert.equal(typeof s.pid, 'number');
+ assert.ok('sessionId' in s);
+ assert.equal(typeof s.cmdline, 'string');
+ }
+});
+
+test('scanBedrockNetwork: graceful fallback, never false-alarms', () => {
+ const r = scanBedrockNetwork([999999999]);
+ assert.equal(typeof r.available, 'boolean');
+ assert.ok(r.pids instanceof Set);
+ // For a bogus pid we must never positively report bedrock traffic.
+ assert.equal(r.pids.has(999999999), false);
+});
+
+test('scanBedrockNetwork: empty pid filter still returns a valid shape', () => {
+ const r = scanBedrockNetwork([]);
+ assert.equal(typeof r.available, 'boolean');
+ assert.ok(r.pids instanceof Set);
+});
+
+test('verifyBedrockSwaps: a dead/re-exec PID counts as cleared (not stillBedrock)', () => {
+ const logs = [];
+ // PID 999999998 is overwhelmingly unlikely to exist → /proc read fails →
+ // treated as "process gone" → cleared.
+ const { stillBedrock, cleared } = verifyBedrockSwaps([999999998], (m) => logs.push(m));
+ assert.deepEqual(stillBedrock, []);
+ assert.deepEqual(cleared, [999999998]);
+});
+
+test('verifyBedrockSwaps: empty input is a no-op', () => {
+ const logs = [];
+ const { stillBedrock, cleared } = verifyBedrockSwaps([], (m) => logs.push(m));
+ assert.deepEqual(stillBedrock, []);
+ assert.deepEqual(cleared, []);
+ assert.equal(logs.length, 0);
+});
diff --git a/claude-ops/scripts/account-rotation/__tests__/inject-continuation.test.mjs b/claude-ops/scripts/account-rotation/__tests__/inject-continuation.test.mjs
new file mode 100644
index 00000000..45654202
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/__tests__/inject-continuation.test.mjs
@@ -0,0 +1,28 @@
+/**
+ * injectContinuation tests (node --test).
+ *
+ * Run: node --test claude-ops/scripts/account-rotation/__tests__/inject-continuation.test.mjs
+ *
+ * F4 is DEFAULT-OFF (CLAUDE_ROTATION_INJECT_CONTINUATION must be '1' to enable).
+ * We verify the off path is a true no-op: it never logs and never invokes claude.
+ * (The on-path spawns a real `claude --resume` and is intentionally not exercised
+ * in unit tests — it is experimental and gated.)
+ */
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+
+import { injectContinuation } from '../bg-respawn.mjs';
+
+test('injectContinuation: default-off is a no-op (no log, no throw)', () => {
+ // The module reads CLAUDE_ROTATION_INJECT_CONTINUATION at import time; in the
+ // test process the flag is unset, so this must short-circuit before any work.
+ assert.equal(process.env.CLAUDE_ROTATION_INJECT_CONTINUATION, undefined);
+ let logged = false;
+ assert.doesNotThrow(() =>
+ injectContinuation('unit-test-sid', () => {
+ logged = true;
+ }),
+ );
+ assert.equal(logged, false, 'must not log or do work when flag is off');
+});
diff --git a/claude-ops/scripts/account-rotation/__tests__/provider-env.test.mjs b/claude-ops/scripts/account-rotation/__tests__/provider-env.test.mjs
new file mode 100644
index 00000000..5111d6c5
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/__tests__/provider-env.test.mjs
@@ -0,0 +1,124 @@
+/**
+ * provider-env tests (node --test).
+ *
+ * Run: node --test claude-ops/scripts/account-rotation/__tests__/provider-env.test.mjs
+ *
+ * Covers: bedrockModelToOAuth mapping (us./eu./apac./global./plain/versioned/
+ * unknown), scrubBedrockEnv removing every key, applyOAuthEnv (token added +
+ * Bedrock vars removed), applyBedrockEnv (Bedrock vars set + OAuth dropped).
+ */
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+
+import {
+ BEDROCK_ENV_KEYS,
+ scrubBedrockEnv,
+ bedrockModelToOAuth,
+ applyOAuthEnv,
+ applyBedrockEnv,
+} from '../provider-env.mjs';
+
+test('bedrockModelToOAuth: us. region-profile prefix', () => {
+ assert.equal(bedrockModelToOAuth('us.anthropic.claude-fable-5'), 'claude-fable-5');
+});
+
+test('bedrockModelToOAuth: eu. region-profile prefix', () => {
+ assert.equal(bedrockModelToOAuth('eu.anthropic.claude-sonnet-4-6'), 'claude-sonnet-4-6');
+});
+
+test('bedrockModelToOAuth: apac. + global. prefixes', () => {
+ assert.equal(bedrockModelToOAuth('apac.anthropic.claude-haiku-4-5'), 'claude-haiku-4-5');
+ assert.equal(bedrockModelToOAuth('global.anthropic.claude-opus-4-1'), 'claude-opus-4-1');
+});
+
+test('bedrockModelToOAuth: plain anthropic. prefix (no region)', () => {
+ assert.equal(bedrockModelToOAuth('anthropic.claude-fable-5'), 'claude-fable-5');
+});
+
+test('bedrockModelToOAuth: versioned -vN:N suffix stripped', () => {
+ assert.equal(bedrockModelToOAuth('anthropic.claude-fable-5-v1:0'), 'claude-fable-5');
+ assert.equal(bedrockModelToOAuth('us.anthropic.claude-opus-4-1-20250805-v1:0'), 'claude-opus-4-1-20250805');
+});
+
+test('bedrockModelToOAuth: already-plain claude id passes through', () => {
+ assert.equal(bedrockModelToOAuth('claude-sonnet-4-6'), 'claude-sonnet-4-6');
+});
+
+test('bedrockModelToOAuth: unknown / non-claude returns null', () => {
+ assert.equal(bedrockModelToOAuth('us.amazon.nova-pro-v1:0'), null);
+ assert.equal(bedrockModelToOAuth('gpt-4o'), null);
+ assert.equal(bedrockModelToOAuth(''), null);
+ assert.equal(bedrockModelToOAuth(null), null);
+ assert.equal(bedrockModelToOAuth(undefined), null);
+});
+
+test('scrubBedrockEnv: removes every BEDROCK_ENV_KEY, preserves the rest', () => {
+ const env = {
+ CLAUDE_CODE_USE_BEDROCK: '1',
+ AWS_BEDROCK_REGION: 'us-east-1',
+ AWS_REGION: 'us-east-1',
+ AWS_DEFAULT_REGION: 'us-east-1',
+ AWS_PROFILE: 'default',
+ AWS_DEFAULT_PROFILE: 'default',
+ AWS_ACCESS_KEY_ID: 'fake-access-key-placeholder',
+ AWS_SECRET_ACCESS_KEY: 'fake-secret-placeholder',
+ AWS_SESSION_TOKEN: 'fake-session-placeholder',
+ ANTHROPIC_MODEL: 'anthropic.claude-fable-5',
+ ANTHROPIC_SMALL_FAST_MODEL: 'anthropic.claude-haiku',
+ ANTHROPIC_DEFAULT_SONNET_MODEL: 's',
+ ANTHROPIC_DEFAULT_HAIKU_MODEL: 'h',
+ ANTHROPIC_DEFAULT_OPUS_MODEL: 'o',
+ ANTHROPIC_DEFAULT_FABLE_MODEL: 'f',
+ PATH: '/usr/bin',
+ CLAUDE_CODE_OAUTH_TOKEN: 'keep-me',
+ };
+ const returned = scrubBedrockEnv(env);
+ assert.equal(returned, env, 'mutates and returns the same object');
+ for (const k of BEDROCK_ENV_KEYS) {
+ assert.equal(k in env, false, `${k} should be deleted`);
+ }
+ assert.equal(env.PATH, '/usr/bin', 'unrelated vars preserved');
+ assert.equal(env.CLAUDE_CODE_OAUTH_TOKEN, 'keep-me', 'OAuth token preserved');
+});
+
+test('scrubBedrockEnv: tolerates null/non-object', () => {
+ assert.equal(scrubBedrockEnv(null), null);
+ assert.equal(scrubBedrockEnv(undefined), undefined);
+});
+
+test('applyOAuthEnv: adds token, removes USE_BEDROCK + ANTHROPIC_MODEL + AWS_*', () => {
+ const env = {
+ CLAUDE_CODE_USE_BEDROCK: '1',
+ AWS_REGION: 'us-east-1',
+ AWS_ACCESS_KEY_ID: 'fake-access-key-placeholder',
+ ANTHROPIC_MODEL: 'anthropic.claude-fable-5',
+ PATH: '/usr/bin',
+ };
+ applyOAuthEnv(env, 'oauth-abc123');
+ assert.equal(env.CLAUDE_CODE_OAUTH_TOKEN, 'oauth-abc123');
+ assert.equal('CLAUDE_CODE_USE_BEDROCK' in env, false);
+ assert.equal('AWS_REGION' in env, false);
+ assert.equal('AWS_ACCESS_KEY_ID' in env, false);
+ assert.equal('ANTHROPIC_MODEL' in env, false);
+ assert.equal(env.PATH, '/usr/bin');
+});
+
+test('applyBedrockEnv: sets Bedrock vars, drops OAuth token, default region+model', () => {
+ const env = { CLAUDE_CODE_OAUTH_TOKEN: 'old', PATH: '/usr/bin' };
+ applyBedrockEnv(env);
+ assert.equal(env.CLAUDE_CODE_USE_BEDROCK, '1');
+ assert.equal(env.AWS_BEDROCK_REGION, 'us-east-1');
+ assert.equal(env.AWS_REGION, 'us-east-1');
+ assert.equal(env.ANTHROPIC_MODEL, 'anthropic.claude-fable-5');
+ assert.equal('CLAUDE_CODE_OAUTH_TOKEN' in env, false);
+ assert.equal(env.PATH, '/usr/bin');
+});
+
+test('applyBedrockEnv: honors region + model override', () => {
+ const env = {};
+ applyBedrockEnv(env, 'eu-west-1', { primary: 'eu.anthropic.claude-sonnet-4-6' });
+ assert.equal(env.AWS_REGION, 'eu-west-1');
+ assert.equal(env.AWS_BEDROCK_REGION, 'eu-west-1');
+ assert.equal(env.ANTHROPIC_MODEL, 'eu.anthropic.claude-sonnet-4-6');
+});
diff --git a/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs b/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs
new file mode 100644
index 00000000..79222b83
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs
@@ -0,0 +1,254 @@
+// bedrock-watchdog.mjs — force real Bedrock sessions back to OAuth, no defer.
+//
+// The deferred-respawn sweep (bg-respawn.sweepDeferredRespawns) defers busy and
+// /loop sessions for up to BUSY_FORCE_AFTER_MS (~90 min) before force-respawning.
+// For OAuth→OAuth token refreshes that's fine. For a session bleeding METERED
+// Bedrock spend it's a 90-minute leak. Sam directive (2026-06-14): "Bedrock
+// should never be in use if any /rotate OAuth account has tokens available."
+//
+// This watchdog detects sessions actually running inference on Bedrock (by
+// reading /proc/<pid>/environ for CLAUDE_CODE_USE_BEDROCK=1) and FORCE-swaps
+// them to OAuth immediately — no 90-min defer — whenever any account has a
+// usable token. It is sandbox-safe: /proc reads that fail are skipped, never
+// fatal, and the network corroboration layer (scanBedrockNetwork) degrades to
+// a graceful "unknown" rather than ever false-alarming.
+
+import { readFileSync, readdirSync, existsSync } from 'fs';
+import { execFileSync } from 'child_process';
+
+import { listLiveBgSessions, doRespawn } from './bg-respawn.mjs';
+import { pickAccountForSession, recordSessionLease } from './session-router.mjs';
+
+const PROC = '/proc';
+
+// Infra/transient session arg-markers that inherit the Bedrock env but do NOT
+// run real inference — never worth a force-swap.
+const SKIP_ARG_MARKERS = ['--bg-pty-host', '--bg-spare'];
+const SKIP_CMD_SUBSTRINGS = ['auth login']; // `claude auth login` etc.
+
+/** Read /proc/<pid>/environ into a {KEY:VALUE} map. Returns null if unreadable. */
+function readProcEnviron(pid) {
+ try {
+ const raw = readFileSync(`${PROC}/${pid}/environ`, 'utf8');
+ const env = {};
+ for (const pair of raw.split('\0')) {
+ if (!pair) continue;
+ const eq = pair.indexOf('=');
+ if (eq === -1) continue;
+ env[pair.slice(0, eq)] = pair.slice(eq + 1);
+ }
+ return env;
+ } catch {
+ return null;
+ }
+}
+
+/** Read /proc/<pid>/cmdline as a single space-joined string. '' if unreadable. */
+function readProcCmdline(pid) {
+ try {
+ return readFileSync(`${PROC}/${pid}/cmdline`, 'utf8').split('\0').filter(Boolean).join(' ');
+ } catch {
+ return '';
+ }
+}
+
+/**
+ * Scan every ec2-user `claude` process whose /proc environ has
+ * CLAUDE_CODE_USE_BEDROCK=1, skipping infra (--bg-pty-host, --bg-spare) and
+ * transient (auth login) processes.
+ * @returns {Array<{pid:number, sessionId:string|null, cmdline:string}>}
+ */
+export function scanBedrockSessions() {
+ const out = [];
+ let pids = [];
+ try {
+ pids = readdirSync(PROC).filter((f) => /^\d+$/.test(f));
+ } catch {
+ return out; // /proc unreadable (non-Linux / sandbox) — nothing to do.
+ }
+ for (const pidStr of pids) {
+ const pid = Number(pidStr);
+ if (pid === process.pid || pid === process.ppid) continue;
+ const cmdline = readProcCmdline(pid);
+ if (!cmdline || !/\bclaude\b/.test(cmdline)) continue;
+ if (SKIP_ARG_MARKERS.some((m) => cmdline.includes(m))) continue;
+ if (SKIP_CMD_SUBSTRINGS.some((m) => cmdline.includes(m))) continue;
+
+ const env = readProcEnviron(pid);
+ if (!env) continue;
+ if (env.CLAUDE_CODE_USE_BEDROCK !== '1') continue;
+
+ out.push({
+ pid,
+ sessionId: env.CLAUDE_CODE_SESSION_ID || null,
+ cmdline,
+ });
+ }
+ return out;
+}
+
+/**
+ * Best-effort: which of the given PIDs have a LIVE established connection to a
+ * Bedrock-runtime endpoint (port 443 to an AWS range, reverse-resolving to
+ * bedrock-runtime). Used only to CORROBORATE env detection and to VALIDATE that
+ * traffic stopped after a swap — NEVER as the sole trigger.
+ *
+ * Sandbox-safe: `ss` can be socket-blind in a sandboxed shell on FRA (see memory
+ * sandboxed-ss-port-checks-blind). On any failure / empty output this returns
+ * { available:false, pids:new Set() } so callers treat it as "unknown" and never
+ * false-alarm.
+ *
+ * @param {number[]} [pids] optional filter; when omitted, all matching are returned
+ * @returns {{available:boolean, pids:Set<number>}}
+ */
+export function scanBedrockNetwork(pids) {
+ const want = pids && pids.length ? new Set(pids.map(Number)) : null;
+ let raw = '';
+ try {
+ // -t tcp, -n numeric, -p processes, state established.
+ raw = execFileSync('ss', ['-tnp', 'state', 'established'], {
+ encoding: 'utf8',
+ timeout: 4000,
+ stdio: ['ignore', 'pipe', 'ignore'],
+ });
+ } catch {
+ return { available: false, pids: new Set() };
+ }
+ if (!raw || !raw.trim()) {
+ // ss ran but returned nothing — could be genuinely no connections OR
+ // socket-blind. Either way we can't assert traffic, so signal "unknown".
+ return { available: false, pids: new Set() };
+ }
+
+ // Bedrock-runtime resolves into AWS ranges; we can't reverse-resolve every IP
+ // cheaply, so match on the well-known bedrock-runtime hostname when present in
+ // the line, else fall back to :443 peers owned by a claude pid. This is
+ // corroboration only — a miss never blocks a swap.
+ const found = new Set();
+ for (const line of raw.split('\n')) {
+ if (!line.includes(':443')) continue;
+ const isBedrock = /bedrock-runtime/i.test(line);
+ const m = line.match(/pid=(\d+)/);
+ if (!m) continue;
+ const pid = Number(m[1]);
+ if (want && !want.has(pid)) continue;
+ if (isBedrock) found.add(pid);
+ }
+ // If we couldn't positively identify any bedrock-runtime peer by name, report
+ // unavailable rather than guessing from bare :443 (avoids false-alarm).
+ if (found.size === 0) return { available: false, pids: new Set() };
+ return { available: true, pids: found };
+}
+
+/**
+ * Is this session protected by a hot-swap hold marker?
+ * (touch /tmp/claude-hotswap-hold-<pid> — see memory hotswap-hold-protection)
+ */
+function hasHotSwapHold(pid) {
+ return existsSync(`/tmp/claude-hotswap-hold-${pid}`);
+}
+
+/**
+ * Force every real Bedrock session back to OAuth when a usable token exists.
+ * No defer for Bedrock — metered bleed overrides loop/busy preservation.
+ *
+ * @param {object} config rotation config (config.accounts)
+ * @param {object} state rotation state (utilization cache)
+ * @param {(msg:string)=>void} log
+ * @returns {{swapped:number, scanned:number, swappedPids:number[]}}
+ */
+export function sweepBedrockSessions(config, state, log) {
+ const sessions = scanBedrockSessions();
+ if (sessions.length === 0) return { swapped: 0, scanned: 0, swappedPids: [] };
+
+ // Map PID → live bg-session record (id needed for `claude respawn <id>`).
+ const live = listLiveBgSessions();
+ const byPid = new Map(live.map((s) => [Number(s.pid), s]));
+
+ let swapped = 0;
+ const swappedPids = [];
+
+ for (const sess of sessions) {
+ // Is an OAuth account actually available? pickAccountForSession returns
+ // 'bedrock' only when ZERO accounts have a usable token (true last resort).
+ const sid = sess.sessionId || (byPid.get(sess.pid) && byPid.get(sess.pid).id) || `pid-${sess.pid}`;
+ const target = pickAccountForSession(sid, config, state);
+ if (!target || target === 'bedrock') {
+ // No OAuth headroom anywhere — Bedrock is the legitimate fallback. Leave it.
+ continue;
+ }
+
+ const bgSession = byPid.get(sess.pid);
+ if (!bgSession) {
+ // Bedrock claude PID with no resolvable bg-session record — can't respawn
+ // it by id (could be a foreground/--resume host). Log loudly; skip.
+ log(
+ `[bedrock-watchdog] pid ${sess.pid} on Bedrock but no bg-session id resolved — cannot force-swap (OAuth target=${target})`,
+ );
+ continue;
+ }
+
+ if (hasHotSwapHold(sess.pid)) {
+ // Honor the hold, but warn loudly — a held session is still bleeding Bedrock.
+ log(
+ `[bedrock-watchdog] ⚠️ session ${bgSession.id} (pid ${sess.pid}) is hot-swap HELD but STILL ON METERED BEDROCK (OAuth target=${target} available) — leaving per hold marker, but money is bleeding`,
+ );
+ continue;
+ }
+
+ // FORCE swap now. Do NOT defer busy/loop sessions for Bedrock.
+ try {
+ recordSessionLease(bgSession.id, target, bgSession.pid);
+ log(
+ `[bedrock-watchdog] FORCE-swapping ${bgSession.id} (pid ${sess.pid}, status ${bgSession.status}) Bedrock→OAuth ${target} (no defer — metered)`,
+ );
+ doRespawn(bgSession, log);
+ swapped++;
+ swappedPids.push(sess.pid);
+ } catch (e) {
+ log(`[bedrock-watchdog] force-swap of ${bgSession.id} failed: ${e.message?.slice(0, 100)}`);
+ }
+ }
+
+ return { swapped, scanned: sessions.length, swappedPids };
+}
+
+/**
+ * Post-swap verification: confirm the (old) swapped PIDs are no longer on
+ * Bedrock. Reads /proc environ (authoritative) and corroborates with the
+ * network scan when it is available. Logs results; returns a summary.
+ *
+ * @param {number[]} swappedPids PIDs we force-swapped on the previous sweep
+ * @param {(msg:string)=>void} log
+ * @returns {{stillBedrock:number[], cleared:number[]}}
+ */
+export function verifyBedrockSwaps(swappedPids, log) {
+ const stillBedrock = [];
+ const cleared = [];
+ for (const pid of swappedPids) {
+ const env = readProcEnviron(pid);
+ if (env === null) {
+ // PID gone (respawn re-execs into a NEW pid) — the old Bedrock process is
+ // dead, which is exactly what we want.
+ cleared.push(pid);
+ continue;
+ }
+ if (env.CLAUDE_CODE_USE_BEDROCK === '1') {
+ stillBedrock.push(pid);
+ } else {
+ cleared.push(pid);
+ }
+ }
+ if (stillBedrock.length) {
+ const net = scanBedrockNetwork(stillBedrock);
+ const detail = net.available
+ ? ` (live bedrock-runtime traffic on: ${[...net.pids].join(',') || 'none'})`
+ : ' (network corroboration unavailable)';
+ log(
+ `[bedrock-watchdog] post-swap VERIFY: ${stillBedrock.length} pid(s) still report USE_BEDROCK=1: ${stillBedrock.join(',')}${detail}`,
+ );
+ } else if (swappedPids.length) {
+ log(`[bedrock-watchdog] post-swap VERIFY: all ${swappedPids.length} swapped pid(s) off Bedrock`);
+ }
+ return { stillBedrock, cleared };
+}
diff --git a/claude-ops/scripts/account-rotation/bg-respawn.mjs b/claude-ops/scripts/account-rotation/bg-respawn.mjs
index 03b74de3..f60631f9 100644
--- a/claude-ops/scripts/account-rotation/bg-respawn.mjs
+++ b/claude-ops/scripts/account-rotation/bg-respawn.mjs
@@ -19,43 +19,59 @@
//
// Used by rotate.mjs (post-rotation) and daemon.mjs (periodic deferred sweep).
-import { readFileSync, readdirSync, writeFileSync, unlinkSync, statSync, existsSync, mkdirSync, renameSync } from 'fs';
+import {
+ readFileSync,
+ readdirSync,
+ writeFileSync,
+ unlinkSync,
+ statSync,
+ existsSync,
+ openSync,
+ readSync,
+ closeSync,
+ mkdirSync,
+ rmSync,
+} from 'fs';
import { execFileSync, execSync } from 'child_process';
-import { join } from 'path';
+import { join, dirname } from 'path';
import { homedir } from 'os';
+import { fileURLToPath } from 'url';
+import {
+ extractAccessToken,
+ recordSessionLease,
+ pickAccountForSession,
+ readVaultToken,
+} from './session-router.mjs';
+import { applyOAuthEnv, applyBedrockEnv, scrubBedrockEnv } from './provider-env.mjs';
-const SESSIONS_DIR = join(homedir(), '.claude', 'sessions');
-// Markers live in a user-private dir (0700), not the world-writable OS temp dir,
-// to avoid symlink/predictable-name attacks on the shared /tmp.
-const MARKER_DIR = join(homedir(), '.claude', 'rotation-markers');
-function ensureMarkerDir() {
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CONFIG_PATH = join(__dirname, 'config.json');
+const STATE_PATH = join(__dirname, 'state.json');
+
+function readConfig() {
try {
- mkdirSync(MARKER_DIR, { recursive: true, mode: 0o700 });
- } catch {}
+ return JSON.parse(readFileSync(CONFIG_PATH, 'utf8'));
+ } catch {
+ return { accounts: [] };
+ }
}
-const LEGACY_TMP_DIR = '/tmp';
-const DEFERRED_PREFIX = 'claude-respawn-deferred-';
-const RESPAWNED_MARKER = (id) => join(MARKER_DIR, `claude-rotation-respawned-${id}`);
-const DEFERRED_MARKER = (id) => join(MARKER_DIR, `${DEFERRED_PREFIX}${id}`);
-
-/** Move deferred markers left under /tmp by pre-upgrade daemons into MARKER_DIR. */
-function migrateLegacyDeferredMarkers() {
- ensureMarkerDir();
- let legacy = [];
+
+function readState() {
try {
- legacy = readdirSync(LEGACY_TMP_DIR).filter((f) => f.startsWith(DEFERRED_PREFIX));
+ return JSON.parse(readFileSync(STATE_PATH, 'utf8'));
} catch {
- return;
- }
- for (const f of legacy) {
- const from = join(LEGACY_TMP_DIR, f);
- const to = join(MARKER_DIR, f);
- try {
- if (existsSync(to)) unlinkSync(from);
- else renameSync(from, to);
- } catch {}
+ return { activeAccount: null, accounts: {} };
}
}
+
+function accountKey(a) {
+ return a.label || a.email;
+}
+
+
+const SESSIONS_DIR = join(homedir(), '.claude', 'sessions');
+const RESPAWNED_MARKER = (id) => `/tmp/claude-rotation-respawned-${id}`;
+const DEFERRED_MARKER = (id) => `/tmp/claude-respawn-deferred-${id}`;
const RESPAWN_THROTTLE_MS = 10 * 60_000; // never respawn the same session twice in 10 min
const BUSY_FORCE_AFTER_MS = 90 * 60_000; // busy this long after rotation → respawn anyway
@@ -102,6 +118,49 @@ function markerFresh(path, maxAgeMs) {
}
}
+// /loop sessions are idle BY DESIGN between iterations, and their loop timer
+// (CronCreate/ScheduleWakeup) is session-only — a respawn wipes it and the
+// loop never fires again (observed 2026-06-12: logmonitor 41e6eaf0 killed
+// every ~10min by the idle-respawn pass, 15m loop never survived to fire).
+// Treat them like busy: defer, force-respawn only after BUSY_FORCE_AFTER_MS.
+export function isLoopSession(id) {
+ let st;
+ try {
+ st = JSON.parse(
+ readFileSync(join(homedir(), '.claude', 'jobs', String(id), 'state.json'), 'utf8'),
+ );
+ } catch {
+ return false;
+ }
+ if (/^\/loop\b/.test(st.detail || '')) return true;
+ // `detail` is the live narration line and gets overwritten — the durable
+ // signal is a /loop invocation (or autonomous-loop sentinel) in the
+ // transcript. Scan the tail only (loops re-arm near the end).
+ const transcript =
+ st.linkScanPath ||
+ (st.sessionId &&
+ join(
+ homedir(),
+ '.claude',
+ 'projects',
+ String(st.cwd || st.originCwd || '').replace(/\//g, '-'),
+ `${st.sessionId}.jsonl`,
+ ));
+ if (!transcript || !existsSync(transcript)) return false;
+ try {
+ const size = statSync(transcript).size;
+ const TAIL = 1024 * 1024; // 1MB tail
+ const fd = openSync(transcript, 'r');
+ const buf = Buffer.alloc(Math.min(TAIL, size));
+ readSync(fd, buf, 0, buf.length, Math.max(0, size - buf.length));
+ closeSync(fd);
+ const tail = buf.toString('utf8');
+ return tail.includes('<<autonomous-loop') || tail.includes('<command-name>/loop</command-name>');
+ } catch {
+ return false;
+ }
+}
+
/** Enumerate live daemon-hosted bg sessions from ~/.claude/sessions/*.json. */
export function listLiveBgSessions() {
const out = [];
@@ -129,7 +188,61 @@ export function listLiveBgSessions() {
return out;
}
-function doRespawn(session, log) {
+// ── Post-swap continuation injection (F4) ───────────────────────────────────
+// `claude respawn <id>` re-execs a session with the transcript preserved but the
+// agent does NOT auto-resume its task — it just idles waiting for input. A
+// continuation turn nudges it to pick up where it left off.
+//
+// HAZARD: a `claude --resume <sid>` turn that overlaps the respawn (or another
+// --resume on the same sid) corrupts session state. We could NOT confirm a
+// fully corruption-safe mechanism in this environment, so per spec this is
+// DEFAULT-OFF behind CLAUDE_ROTATION_INJECT_CONTINUATION=1. When enabled we
+// guard three ways: (a) only after doRespawn has resolved the NEW pid (respawn
+// settled), (b) a per-sid lock dir so two daemons/ticks can't double-inject,
+// (c) a short settle delay. Even so, treat as experimental until a maintainer
+// validates it against a live fleet.
+const INJECT_CONTINUATION_ENABLED = process.env.CLAUDE_ROTATION_INJECT_CONTINUATION === '1';
+const CONTINUATION_PROMPT =
+ 'Your session was hot-swapped from Bedrock to OAuth to stop metered spend. ' +
+ 'Continue your previous work from where you left off.';
+
+/**
+ * Inject a one-shot continuation turn into a freshly-respawned session.
+ * No-op unless CLAUDE_ROTATION_INJECT_CONTINUATION=1. Per-sid lock prevents
+ * concurrent --resume (which corrupts state). Best-effort; never throws.
+ * @param {string} sid session/job id
+ * @param {(m:string)=>void} log
+ */
+export function injectContinuation(sid, log) {
+ if (!INJECT_CONTINUATION_ENABLED) return;
+ const lockDir = `/tmp/claude-continuation-lock-${sid}`;
+ try {
+ // Atomic mkdir lock — fails if another inject is already in flight for this sid.
+ mkdirSync(lockDir);
+ } catch {
+ log(`[bg-respawn] continuation inject for ${sid} skipped — lock held`);
+ return;
+ }
+ try {
+ // Brief settle so the re-exec'd session is fully attached before --resume.
+ try {
+ execSync('sleep 2');
+ } catch {}
+ execFileSync(claudeBin(), ['--resume', String(sid), CONTINUATION_PROMPT], {
+ timeout: 30_000,
+ stdio: 'ignore',
+ });
+ log(`[bg-respawn] injected continuation turn into ${sid}`);
+ } catch (e) {
+ log(`[bg-respawn] continuation inject for ${sid} failed: ${e.message?.slice(0, 80)}`);
+ } finally {
+ try {
+ rmSync(lockDir, { recursive: true, force: true });
+ } catch {}
+ }
+}
+
+export function doRespawn(session, log) {
const stateFile = join(homedir(), '.claude', 'jobs', String(session.id), 'state.json');
if (!existsSync(stateFile)) {
log(
@@ -138,8 +251,7 @@ function doRespawn(session, log) {
try {
process.kill(session.pid, 'SIGTERM');
try {
- ensureMarkerDir();
- writeFileSync(RESPAWNED_MARKER(session.id), String(Date.now()), { mode: 0o600 });
+ writeFileSync(RESPAWNED_MARKER(session.id), String(Date.now()));
} catch {}
try {
unlinkSync(DEFERRED_MARKER(session.id));
@@ -152,15 +264,90 @@ function doRespawn(session, log) {
}
try {
- execFileSync(claudeBin(), ['respawn', String(session.id)], { timeout: 30_000, stdio: 'pipe' });
+ const childEnv = { ...process.env };
+ let routingKey = null;
try {
- ensureMarkerDir();
- writeFileSync(RESPAWNED_MARKER(session.id), String(Date.now()), { mode: 0o600 });
+ const config = readConfig();
+ const state = readState();
+ // Re-resolve routing before respawn so stale bedrock leases and unleased
+ // sessions pick up OAuth after rotation instead of inheriting Bedrock env.
+ routingKey = pickAccountForSession(session.id, config, state);
+
+ let tokenJson = null;
+ if (routingKey && routingKey !== 'bedrock') {
+ const account = config.accounts.find((a) => accountKey(a) === routingKey);
+ if (account) tokenJson = readVaultToken(account);
+ }
+ let token = tokenJson ? extractAccessToken(tokenJson) : null;
+ if (!token && routingKey !== 'bedrock' && state.activeAccount) {
+ const active = config.accounts.find((a) => accountKey(a) === state.activeAccount);
+ if (active) {
+ tokenJson = readVaultToken(active);
+ token = tokenJson ? extractAccessToken(tokenJson) : null;
+ }
+ }
+ if (token) {
+ // Scrub ALL Bedrock vars (USE_BEDROCK, AWS_*, hardcoded ANTHROPIC_MODEL)
+ // before setting the OAuth token. Leaving ANTHROPIC_MODEL=anthropic.claude-fable-5
+ // or AWS_* in env makes the OAuth session emit invalid model ids / keep paying
+ // for Bedrock. Model resets to the subscription default catalog.
+ applyOAuthEnv(childEnv, token);
+ log(`[bg-respawn] Injecting CLAUDE_CODE_OAUTH_TOKEN for session ${session.id} (Bedrock vars scrubbed)`);
+ } else if (routingKey === 'bedrock') {
+ applyBedrockEnv(childEnv);
+ log(`[bg-respawn] Injecting Bedrock fallback env for session ${session.id}`);
+ } else {
+ scrubBedrockEnv(childEnv);
+ delete childEnv.CLAUDE_CODE_OAUTH_TOKEN;
+ log(`[bg-respawn] Scrubbed Bedrock vars for session ${session.id} (no per-session token)`);
+ }
+ } catch (err) {
+ log(`[bg-respawn] Failed to retrieve session token: ${err.message}`);
+ }
+
+ execFileSync(claudeBin(), ['respawn', String(session.id)], {
+ timeout: 30_000,
+ stdio: 'pipe',
+ env: childEnv,
+ });
+ try {
+ writeFileSync(RESPAWNED_MARKER(session.id), String(Date.now()));
} catch {}
try {
unlinkSync(DEFERRED_MARKER(session.id));
} catch {}
log(`[bg-respawn] respawned bg session ${session.id} (pid ${session.pid}, was ${session.status})`);
+
+ // Synchronously resolve new PID and persist the lease only after respawn succeeds
+ try {
+ if (routingKey) {
+ let newPid = null;
+ for (let i = 0; i < 20; i++) {
+ try {
+ execSync('sleep 0.5');
+ } catch {}
+ const live = listLiveBgSessions();
+ const found = live.find(ls => String(ls.id) === String(session.id));
+ if (found && found.pid !== session.pid) {
+ newPid = found.pid;
+ break;
+ }
+ }
+ if (newPid) {
+ log(`[bg-respawn] Updated lease for session ${session.id} with new PID ${newPid}`);
+ recordSessionLease(session.id, routingKey, newPid);
+ // F4: nudge the re-exec'd session to resume its task (default-off).
+ // Only here, after the new pid is resolved (respawn has settled), and
+ // only for sessions with saved state (spares returned earlier).
+ injectContinuation(session.id, log);
+ } else {
+ log(`[bg-respawn] Warning: could not resolve new PID for session ${session.id}`);
+ }
+ }
+ } catch (err) {
+ log(`[bg-respawn] Error updating lease with new PID: ${err.message}`);
+ }
+
return true;
} catch (e) {
log(`[bg-respawn] respawn ${session.id} failed: ${e.message?.slice(0, 80)}`);
@@ -185,14 +372,14 @@ export function respawnBgSessions(log = () => {}) {
log(`[bg-respawn] ${s.id} respawned <10min ago — skipping`);
continue;
}
- if (s.status === 'busy') {
+ if (s.status === 'busy' || isLoopSession(s.id)) {
try {
- ensureMarkerDir();
- if (!existsSync(DEFERRED_MARKER(s.id)))
- writeFileSync(DEFERRED_MARKER(s.id), String(Date.now()), { mode: 0o600 });
+ if (!existsSync(DEFERRED_MARKER(s.id))) writeFileSync(DEFERRED_MARKER(s.id), String(Date.now()));
} catch {}
deferred++;
- log(`[bg-respawn] ${s.id} busy — deferred (daemon sweep will retry, force after 90min)`);
+ log(
+ `[bg-respawn] ${s.id} ${s.status === 'busy' ? 'busy' : 'has an armed /loop'} — deferred (daemon sweep will retry, force after 90min)`,
+ );
continue;
}
if (doRespawn(s, log)) respawned++;
@@ -206,10 +393,9 @@ export function respawnBgSessions(log = () => {}) {
* BUSY_FORCE_AFTER_MS, and clears markers for sessions that exited.
*/
export function sweepDeferredRespawns(log = () => {}) {
- migrateLegacyDeferredMarkers();
let markers = [];
try {
- markers = readdirSync(MARKER_DIR).filter((f) => f.startsWith(DEFERRED_PREFIX));
+ markers = readdirSync('/tmp').filter((f) => f.startsWith('claude-respawn-deferred-'));
} catch {
return 0;
}
@@ -217,8 +403,8 @@ export function sweepDeferredRespawns(log = () => {}) {
const live = new Map(listLiveBgSessions().map((s) => [String(s.id), s]));
let handled = 0;
for (const m of markers) {
- const id = m.replace(DEFERRED_PREFIX, '');
- const markerPath = join(MARKER_DIR, m);
+ const id = m.replace('claude-respawn-deferred-', '');
+ const markerPath = `/tmp/${m}`;
const s = live.get(id);
if (!s) {
// session exited or state file gone — nothing to refresh
@@ -234,10 +420,11 @@ export function sweepDeferredRespawns(log = () => {}) {
return 0;
}
})();
- if (s.status !== 'busy' || deferredAgo > BUSY_FORCE_AFTER_MS) {
- if (s.status === 'busy') {
+ const protectedNow = s.status === 'busy' || isLoopSession(id);
+ if (!protectedNow || deferredAgo > BUSY_FORCE_AFTER_MS) {
+ if (protectedNow) {
log(
- `[bg-respawn] ${id} still busy after ${Math.round(deferredAgo / 60000)}min — force-respawning before token expiry`,
+ `[bg-respawn] ${id} still ${s.status === 'busy' ? 'busy' : 'looping'} after ${Math.round(deferredAgo / 60000)}min — force-respawning before token expiry`,
);
}
if (doRespawn(s, log)) handled++;
diff --git a/claude-ops/scripts/account-rotation/claude-settings-mode.mjs b/claude-ops/scripts/account-rotation/claude-settings-mode.mjs
index 754ae502..78763127 100644
--- a/claude-ops/scripts/account-rotation/claude-settings-mode.mjs
+++ b/claude-ops/scripts/account-rotation/claude-settings-mode.mjs
@@ -30,9 +30,55 @@ function writeSettingsAtomic(s) {
renameSync(tmp, p);
}
-/** Pinned fallbacks when list-inference-profiles fails or returns nothing. */
-const BEDROCK_PRIMARY_FALLBACK = 'us.anthropic.claude-sonnet-4-6';
-const BEDROCK_SMALL_FALLBACK = 'us.anthropic.claude-haiku-4-5-20251001-v1:0';
+const AWS_PROFILE_CANDIDATES = ['default', 'ec2-user-cli', 'healify', 'workshop'];
+
+function awsProfileProbeEnv(profile, region = 'us-east-1') {
+ const env = {
+ ...process.env,
+ AWS_PROFILE: profile,
+ AWS_DEFAULT_PROFILE: profile,
+ AWS_REGION: region,
+ AWS_DEFAULT_REGION: region,
+ };
+ env.AWS_ACCESS_KEY_ID = '';
+ env.AWS_SECRET_ACCESS_KEY = '';
+ env.AWS_SESSION_TOKEN = '';
+ return env;
+}
+
+export function resolveWorkingAwsEnv(region = 'us-east-1') {
+ try {
+ execFileSync('aws', ['sts', 'get-caller-identity', '--output', 'json'], {
+ encoding: 'utf8',
+ stdio: 'pipe',
+ timeout: 5000,
+ env: { ...process.env, AWS_REGION: region, AWS_DEFAULT_REGION: region },
+ });
+ return { env: { ...process.env, AWS_REGION: region, AWS_DEFAULT_REGION: region }, profile: process.env.AWS_PROFILE || '' };
+ } catch {}
+ for (const profile of AWS_PROFILE_CANDIDATES) {
+ const env = awsProfileProbeEnv(profile, region);
+ try {
+ execFileSync('aws', ['sts', 'get-caller-identity', '--output', 'json'], {
+ encoding: 'utf8',
+ stdio: 'pipe',
+ timeout: 5000,
+ env,
+ });
+ return { env, profile };
+ } catch {}
+ }
+ return { env: { ...process.env, AWS_REGION: region, AWS_DEFAULT_REGION: region }, profile: process.env.AWS_PROFILE || '' };
+}
+
+function getFallbacksForPrefix(prefix) {
+ return {
+ primary: `${prefix}anthropic.claude-sonnet-4-6`,
+ small: `${prefix}anthropic.claude-haiku-4-5-20251001-v1:0`,
+ opus: `${prefix}anthropic.claude-opus-4-1-20250805-v1:0`,
+ fable: `${prefix}anthropic.claude-fable-5`,
+ };
+}
/** Map AWS region to preferred cross-region inference profile prefix. */
export function preferredInferencePrefix(awsRegion) {
@@ -47,6 +93,7 @@ export function preferredInferencePrefix(awsRegion) {
function listAllInferenceProfilesSync(region) {
const summaries = [];
let startingToken;
+ const aws = resolveWorkingAwsEnv(region);
for (;;) {
const args = [
'bedrock',
@@ -63,6 +110,7 @@ function listAllInferenceProfilesSync(region) {
encoding: 'utf8',
maxBuffer: 32 * 1024 * 1024,
timeout: 60_000,
+ env: aws.env,
});
const data = JSON.parse(out);
summaries.push(...(data.inferenceProfileSummaries || []));
@@ -72,6 +120,29 @@ function listAllInferenceProfilesSync(region) {
return summaries;
}
+export function parseModelRank(inferenceProfileId) {
+ // Strip trailing version flags like -v1:0, :v1:0, or v1
+ const cleanId = inferenceProfileId.replace(/[-:]v\d+(:\d+)?$/i, '').replace(/v\d+$/i, '');
+ const matches = cleanId.match(/\d+/g);
+ if (!matches) return [0, 0, 0];
+ const nums = matches.map(n => parseInt(n, 10));
+ const major = nums[0];
+ let minor = 0;
+ let date = 0;
+ if (nums.length > 1) {
+ const last = nums[nums.length - 1];
+ if (last >= 20000000 && last <= 21000000) {
+ date = last;
+ if (nums.length === 3) {
+ minor = nums[1];
+ }
+ } else {
+ minor = nums[1];
+ }
+ }
+ return [major, minor, date];
+}
+
function rankCompareDesc(ra, rb) {
const n = Math.max(ra.length, rb.length);
for (let i = 0; i < n; i++) {
@@ -82,40 +153,6 @@ function rankCompareDesc(ra, rb) {
return 0;
}
-function sonnetRank(inferenceProfileId) {
- const m = inferenceProfileId.match(/\.anthropic\.claude-sonnet-(.+)$/);
- if (!m) return [-1];
- const tail = m[1].replace(/-v\d+:\d+$/i, '');
- // Order matters: `4-20250514` must not match as major-minor (second group YYYYMMDD).
- let mm = tail.match(/^(\d+)-(\d+)-(\d{8})$/);
- if (mm) {
- return [parseInt(mm[1], 10), parseInt(mm[2], 10), parseInt(mm[3], 10)];
- }
- mm = tail.match(/^(\d+)-(\d{8})$/);
- if (mm) return [parseInt(mm[1], 10), 0, parseInt(mm[2], 10)];
- mm = tail.match(/^(\d+)-(\d+)$/);
- if (mm) {
- const second = mm[2];
- if (/^\d{8}$/.test(second)) {
- return [parseInt(mm[1], 10), 0, parseInt(second, 10)];
- }
- return [parseInt(mm[1], 10), parseInt(second, 10), 0];
- }
- return [0, 0, 0];
-}
-
-function haikuRank(inferenceProfileId) {
- let m = inferenceProfileId.match(/\.anthropic\.claude-haiku-(\d+)-(\d+)-(\d{8})/);
- if (m) {
- return [parseInt(m[1], 10), parseInt(m[2], 10), parseInt(m[3], 10)];
- }
- m = inferenceProfileId.match(/\.anthropic\.claude-3-5-haiku-(\d{8})/);
- if (m) return [3, 5, parseInt(m[1], 10)];
- m = inferenceProfileId.match(/\.anthropic\.claude-3-haiku-(\d{8})/);
- if (m) return [3, 0, parseInt(m[1], 10)];
- return [-1, 0, 0];
-}
-
function isActiveSystem(p) {
return p?.type === 'SYSTEM_DEFINED' && p?.status === 'ACTIVE' && p?.inferenceProfileId;
}
@@ -129,56 +166,49 @@ function candidatesWithPrefix(profiles, prefix, predicate) {
return active;
}
-function pickLatestSonnet(profiles, prefix) {
- const c = candidatesWithPrefix(profiles, prefix, (id) => id.includes('.anthropic.claude-sonnet-'));
- if (!c.length) return null;
- c.sort((x, y) => rankCompareDesc(sonnetRank(x.inferenceProfileId), sonnetRank(y.inferenceProfileId)));
- return c[0].inferenceProfileId;
-}
-
-function pickLatestHaiku(profiles, prefix) {
- const c = candidatesWithPrefix(profiles, prefix, (id) => {
- if (id.includes('.anthropic.claude-haiku-')) return haikuRank(id)[0] >= 0;
- if (id.includes('.anthropic.claude-3-') && id.includes('haiku')) return true;
- return false;
- });
- if (!c.length) return null;
- c.sort((x, y) => rankCompareDesc(haikuRank(x.inferenceProfileId), haikuRank(y.inferenceProfileId)));
+function pickLatestModel(profiles, prefix, pattern, fallback) {
+ const c = candidatesWithPrefix(profiles, prefix, (id) => pattern.test(id));
+ if (!c.length) return fallback;
+ c.sort((x, y) =>
+ rankCompareDesc(
+ parseModelRank(x.inferenceProfileId),
+ parseModelRank(y.inferenceProfileId),
+ ),
+ );
return c[0].inferenceProfileId;
}
/**
- * Resolve latest Bedrock inference profile IDs for Claude Code (Sonnet + Haiku).
+ * Resolve latest Bedrock inference profile IDs for Claude Code (Sonnet, Haiku, Opus, Fable).
* @param {string} region - AWS region for `list-inference-profiles` (e.g. us-east-1)
- * @returns {{ primary: string, small: string, source: 'api' | 'fallback', detail?: string }}
+ * @returns {{ primary: string, small: string, opus: string, fable: string, source: 'api' | 'fallback', detail?: string }}
*/
export function resolveBedrockClaudeModelIds(region = 'us-east-1') {
+ const prefix = preferredInferencePrefix(region);
+ const fallbacks = getFallbacksForPrefix(prefix);
if (process.env.BEDROCK_SKIP_RESOLVE === '1') {
return {
- primary: BEDROCK_PRIMARY_FALLBACK,
- small: BEDROCK_SMALL_FALLBACK,
+ primary: fallbacks.primary,
+ small: fallbacks.small,
+ opus: fallbacks.opus,
+ fable: fallbacks.fable,
source: 'fallback',
detail: 'BEDROCK_SKIP_RESOLVE',
};
}
try {
const profiles = listAllInferenceProfilesSync(region);
- const prefix = preferredInferencePrefix(region);
- const primary = pickLatestSonnet(profiles, prefix);
- const small = pickLatestHaiku(profiles, prefix);
- if (primary && small) {
- return { primary, small, source: 'api' };
- }
- return {
- primary: primary || BEDROCK_PRIMARY_FALLBACK,
- small: small || BEDROCK_SMALL_FALLBACK,
- source: 'fallback',
- detail: primary ? 'missing-haiku' : small ? 'missing-sonnet' : 'missing-both',
- };
+ const primary = pickLatestModel(profiles, prefix, /\.anthropic\.claude-sonnet-/, fallbacks.primary);
+ const small = pickLatestModel(profiles, prefix, /(\.anthropic\.claude-.*haiku|\.anthropic\.claude-3-.*haiku)/, fallbacks.small);
+ const opus = pickLatestModel(profiles, prefix, /\.anthropic\.claude-opus-/, fallbacks.opus);
+ const fable = pickLatestModel(profiles, prefix, /\.anthropic\.claude-fable-/, fallbacks.fable);
+ return { primary, small, opus, fable, source: 'api' };
} catch (e) {
return {
- primary: BEDROCK_PRIMARY_FALLBACK,
- small: BEDROCK_SMALL_FALLBACK,
+ primary: fallbacks.primary,
+ small: fallbacks.small,
+ opus: fallbacks.opus,
+ fable: fallbacks.fable,
source: 'fallback',
detail: (e.message || String(e)).slice(0, 200),
};
@@ -187,14 +217,27 @@ export function resolveBedrockClaudeModelIds(region = 'us-east-1') {
/** Match use-bedrock.sh: Bedrock env + strip top-level model lists (OAuth catalog ids break Bedrock). */
export function persistBedrockClaudeSettings(region = 'us-east-1') {
- const { primary, small } = resolveBedrockClaudeModelIds(region);
+ const { primary, small, opus, fable } = resolveBedrockClaudeModelIds(region);
+ const aws = resolveWorkingAwsEnv(region);
const s = readSettings();
const env = s.env && typeof s.env === 'object' ? { ...s.env } : {};
env.CLAUDE_CODE_USE_BEDROCK = '1';
env.AWS_BEDROCK_REGION = region;
env.AWS_REGION = region;
+ env.AWS_DEFAULT_REGION = region;
+ if (aws.profile) {
+ env.AWS_PROFILE = aws.profile;
+ env.AWS_DEFAULT_PROFILE = aws.profile;
+ }
+ env.AWS_ACCESS_KEY_ID = '';
+ env.AWS_SECRET_ACCESS_KEY = '';
+ env.AWS_SESSION_TOKEN = '';
env.ANTHROPIC_MODEL = primary;
env.ANTHROPIC_SMALL_FAST_MODEL = small;
+ env.ANTHROPIC_DEFAULT_SONNET_MODEL = primary;
+ env.ANTHROPIC_DEFAULT_HAIKU_MODEL = small;
+ env.ANTHROPIC_DEFAULT_OPUS_MODEL = opus;
+ env.ANTHROPIC_DEFAULT_FABLE_MODEL = fable;
delete env.ANTHROPIC_API_KEY;
delete env.ANTHROPIC_AUTH_TOKEN;
delete env.ANTHROPIC_BASE_URL;
@@ -217,6 +260,10 @@ export function clearHardcodedModelsForOAuthClaudeSettings() {
'ANTHROPIC_BASE_URL',
'ANTHROPIC_MODEL',
'ANTHROPIC_SMALL_FAST_MODEL',
+ 'ANTHROPIC_DEFAULT_SONNET_MODEL',
+ 'ANTHROPIC_DEFAULT_HAIKU_MODEL',
+ 'ANTHROPIC_DEFAULT_OPUS_MODEL',
+ 'ANTHROPIC_DEFAULT_FABLE_MODEL',
]) {
delete env[k];
}
diff --git a/claude-ops/scripts/account-rotation/daemon.mjs b/claude-ops/scripts/account-rotation/daemon.mjs
index d97cd083..80adb7f8 100644
--- a/claude-ops/scripts/account-rotation/daemon.mjs
+++ b/claude-ops/scripts/account-rotation/daemon.mjs
@@ -24,7 +24,7 @@
*/
import { readFileSync, writeFileSync, existsSync, unlinkSync, appendFileSync, statSync } from 'fs';
-import { persistBedrockClaudeSettings, clearHardcodedModelsForOAuthClaudeSettings } from './claude-settings-mode.mjs';
+import { persistBedrockClaudeSettings, clearHardcodedModelsForOAuthClaudeSettings, resolveWorkingAwsEnv } from './claude-settings-mode.mjs';
import {
destinationUtilHardBlock,
DAEMON_SAFE_5H_PCT,
@@ -39,7 +39,9 @@ import { fileURLToPath } from 'url';
import { execSync, execFileSync, spawn, spawnSync } from 'child_process';
import { tmpdir } from 'os';
import { applyAccountLeases, writeLease } from './account-leases.mjs';
-import { sweepDeferredRespawns } from './bg-respawn.mjs';
+import { sweepDeferredRespawns, listLiveBgSessions, doRespawn, isLoopSession } from './bg-respawn.mjs';
+import { pickAccountForSession, recordSessionLease, readLeases, writeLeases } from './session-router.mjs';
+import { sweepBedrockSessions, verifyBedrockSwaps } from './bedrock-watchdog.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const CONFIG_PATH = join(__dirname, 'config.json');
@@ -49,7 +51,7 @@ const PID_FILE = join(__dirname, '.daemon.pid');
const ROTATE_SCRIPT = join(__dirname, 'rotate.mjs');
// Keychain account name — must match rotate.mjs convention.
-const ACTIVE_KEYCHAIN_ACCOUNT = 'unknown';
+const ACTIVE_KEYCHAIN_ACCOUNT = process.env.USER || 'unknown';
const VAULT_KEYCHAIN_ACCOUNT = process.env.CLAUDE_ROTATOR_KEYCHAIN_ACCOUNT || process.env.USER || 'claude-ops';
const RECOVERY_STATUS_PATH = join(__dirname, '.bedrock-recovery-status.json');
@@ -285,12 +287,9 @@ function readStoredToken(account) {
}
}
try {
- // spawnSync (no shell) — svc/account can't break out into a shell command.
- // `security -g` prints "password: ..." on stderr, so read both streams.
- const r = spawnSync('security', ['find-generic-password', '-s', svc, '-a', VAULT_KEYCHAIN_ACCOUNT, '-g'], {
+ const out = execSync(`security find-generic-password -s "${svc}" -a "${VAULT_KEYCHAIN_ACCOUNT}" -g 2>&1`, {
timeout: 5000,
- });
- const out = `${r.stdout?.toString() || ''}${r.stderr?.toString() || ''}`;
+ }).toString();
const m = out.match(/^password: "?(.*?)"?$/m);
return m ? m[1].replace(/\\"/g, '"') : null;
} catch {
@@ -298,6 +297,23 @@ function readStoredToken(account) {
}
}
+function deleteStoredToken(account) {
+ const svc = `Claude-Rotation-${accountKey(account)}`;
+ if (IS_LINUX) {
+ try {
+ const store = JSON.parse(readFileSync(LINUX_CRED_PATH, 'utf8'));
+ delete store[svc];
+ writeFileSync(LINUX_CRED_PATH, JSON.stringify(store, null, 2), { mode: 0o600 });
+ } catch {}
+ return;
+ }
+ try {
+ execFileSync('security', ['delete-generic-password', '-s', svc, '-a', VAULT_KEYCHAIN_ACCOUNT], {
+ stdio: 'ignore',
+ });
+ } catch {}
+}
+
function readActiveKeychainToken() {
if (IS_LINUX) {
try {
@@ -657,6 +673,19 @@ async function shouldRotate(config, state) {
timeout: 5000,
},
);
+ // macOS: Claude Code ≥2.1 reads the ACTIVE credential from the FILE
+ // ~/.claude/.credentials.json, NOT the keychain. Mirror the refreshed
+ // active token into the file (merge to preserve other fields) so new
+ // sessions pick it up. Without this the keychain write above is
+ // invisible to claude. (root-caused 2026-06-12)
+ try {
+ let fileStore = {};
+ try { fileStore = JSON.parse(readFileSync(LINUX_CRED_PATH, 'utf8')); } catch {}
+ const incoming = JSON.parse(tokenToWrite);
+ fileStore.claudeAiOauth = incoming.claudeAiOauth || incoming;
+ if (incoming.mcpOAuth) fileStore.mcpOAuth = incoming.mcpOAuth;
+ writeFileSync(LINUX_CRED_PATH, JSON.stringify(fileStore, null, 2), { mode: 0o600 });
+ } catch (_fileErr) { /* best-effort */ }
}
log('[active-refresh] Active keychain updated with mcpOAuth preserved — sessions will auto-recover');
}
@@ -693,6 +722,11 @@ async function queryLiveUtilization(account) {
},
signal: AbortSignal.timeout(4000),
});
+ if (res.status === 401) {
+ log(`[daemon-query] Account ${account.email} returned 401 (Unauthorized) — invalidating token`);
+ deleteStoredToken(account);
+ return { ok: false, authFailed: true };
+ }
if (res.status === 429) return { ok: false, rateLimited: true };
if (!res.ok) return { ok: false };
const data = await res.json();
@@ -770,6 +804,10 @@ async function findValidRotationTarget(config, state) {
log(`[pre-rotate] ${key}: usage API 429 — skipping this candidate this pass`);
continue;
}
+ if (live?.authFailed) {
+ log(`[pre-rotate] ${key}: auth failed (401) — skipping this candidate this pass`);
+ continue;
+ }
if (isLiveUtilOk(live)) {
const max = liveUtilMax(live);
const tooHot5h = live.pct5h >= DAEMON_SAFE_5H_PCT;
@@ -833,8 +871,13 @@ async function findValidRotationTarget(config, state) {
const expiry = parseTokenExpiry(tokenJson);
const isExpired = expiry > 0 && now > expiry - 5 * 60_000;
if (isExpired) {
+ log(`[pre-rotate-relaxed] ${key}: token expired/expiring — attempting refresh`);
const refreshed = await refreshSingleToken(account);
- if (!refreshed) continue;
+ if (!refreshed) {
+ log(`[pre-rotate-relaxed] ${key}: refresh FAILED — skipping`);
+ continue;
+ }
+ log(`[pre-rotate-relaxed] ${key}: refresh succeeded`);
}
const live = await queryLiveUtilization(account);
@@ -907,7 +950,7 @@ function stopClaudeDaemon() {
try {
const result = execSync('claude daemon stop --any', {
encoding: 'utf8',
- env: { ...process.env, PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin' },
+ env: { ...process.env, PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin' }
});
log(`[daemon-stop] Daemon stop success: ${result.trim()}`);
} catch (err) {
@@ -927,13 +970,16 @@ function activateBedrockFallbackFromDaemon(reason) {
}
const region = process.env.AWS_BEDROCK_REGION || 'us-east-1';
try {
+ const aws = resolveWorkingAwsEnv(region);
execFileSync('aws', ['sts', 'get-caller-identity', '--output', 'json'], {
stdio: 'pipe',
timeout: 5000,
+ env: aws.env,
});
execFileSync('aws', ['bedrock', 'list-inference-profiles', '--region', region, '--max-results', '1'], {
stdio: 'pipe',
timeout: 6000,
+ env: aws.env,
});
} catch (e) {
log(`[bedrock-fallback] AWS unreachable: ${(e.message || '').slice(0, 80)}`);
@@ -1059,7 +1105,7 @@ async function doRotation(reason) {
// into running sessions — that interrupts active work. Sessions pick up the new
// token on next 401 (auto-retry) or when the user voluntarily runs /login.
// execFileSync (not execSync) — no shell, no injection risk on targetKey.
- const result = execFileSync(process.execPath, [ROTATE_SCRIPT, '--no-browser', '--to', targetKey], {
+ const result = execFileSync('node', [ROTATE_SCRIPT, '--no-browser', '--to', targetKey], {
cwd: __dirname,
timeout: 120_000, // 2min — keychain swap is fast; no per-session injection
env: { ...process.env, NODE_NO_WARNINGS: '1' },
@@ -1181,8 +1227,11 @@ async function refreshSingleToken(account) {
{ timeout: 5000 },
);
if (kcResult.error || kcResult.status !== 0) {
- const msg = kcResult.error?.message || kcResult.stderr?.toString() || `security exit ${kcResult.status}`;
- throw new Error(`Keychain write failed: ${msg}`);
+ const detail =
+ kcResult.stderr?.toString()?.trim() ||
+ kcResult.error?.message ||
+ `exit ${kcResult.status}`;
+ throw new Error(`Keychain write failed: ${detail}`);
}
}
log(
@@ -1304,6 +1353,26 @@ async function maybeRecoverOAuthFromBedrock(config, state, sentinelPath, ctx) {
} catch (e) {
log(`[bedrock-recovery] settings clear failed: ${e.message?.slice(0, 80)}`);
}
+ // Purge stale `bedrock`-keyed session leases. Without this, bg-respawn.mjs
+ // keeps reading lease.accountKey === 'bedrock' and re-injecting
+ // CLAUDE_CODE_USE_BEDROCK on every respawn forever — sessions stay stranded
+ // on metered Bedrock long after OAuth headroom returns. (2026-06-14: 9 sessions
+ // still pinned to bedrock 17h post-recovery; Sam: "Bedrock should never be in
+ // use if any OAuth account has tokens available.") Drop the leases so the next
+ // respawn falls through to OAuth token injection.
+ try {
+ const leases = readLeases();
+ let purged = 0;
+ for (const [sid, entry] of Object.entries(leases)) {
+ if (entry?.accountKey === 'bedrock') { delete leases[sid]; purged++; }
+ }
+ if (purged > 0) {
+ writeLeases(leases);
+ log(`[bedrock-recovery] purged ${purged} stale bedrock lease(s) → respawns will use OAuth`);
+ }
+ } catch (e) {
+ log(`[bedrock-recovery] bedrock-lease purge failed: ${e.message?.slice(0, 80)}`);
+ }
try {
stopClaudeDaemon();
} catch (e) {
@@ -1379,7 +1448,7 @@ async function runFullProbe(config, state) {
try {
for (const a of config.accounts) {
const key = accountKey(a);
- if (key === state.activeAccount) continue; // active already covered by shouldRotate path
+ const isActive = key === state.activeAccount;
probed += 1;
const tokenJson = readStoredToken(a);
if (!tokenJson || tokenExpired(tokenJson)) {
@@ -1398,6 +1467,33 @@ async function runFullProbe(config, state) {
state.accounts[key] = state.accounts[key] || {};
state.accounts[key].lastUtilization = { pct, reset, ts: Date.now() };
updated += 1;
+ // Active-account self-trigger: shouldRotate only live-probes the active
+ // account when a statusline session has already written .rate-limits.json.
+ // Headless/subagent-only workloads never write it, so an active account
+ // could sit at a 100% weekly cap while every subagent dispatch failed
+ // (bit us 2026-06-12). When the probe sees the active account over the
+ // rotation threshold and no fresh trigger file exists, file one in the
+ // statusline format — the normal shouldRotate path (with its anti-thrash
+ // guards) then performs the rotation on the next tick.
+ if (isActive && pct >= UTILIZATION_ROTATE_THRESHOLD) {
+ try {
+ if (!readRealUtilization()) {
+ writeFileSync(
+ RATE_LIMITS_FILE,
+ JSON.stringify({
+ ts: Math.floor(Date.now() / 1000),
+ account_email: a.email,
+ five_hour: { pct: pct5h, reset: live.reset5h ?? null },
+ seven_day: { pct: pct7d, reset: live.reset7d ?? null },
+ source: 'daemon-active-probe',
+ }),
+ );
+ log(
+ `[active-probe] active ${key} over threshold (5h=${pct5h.toFixed(0)}% 7d=${pct7d.toFixed(0)}% >= ${UTILIZATION_ROTATE_THRESHOLD}%) — filed rotation trigger`,
+ );
+ }
+ } catch {}
+ }
}
// Compute and persist the next-rotation candidate for statusline display.
// Pick the lowest-util non-active account with a non-expired vault token.
@@ -1437,6 +1533,87 @@ async function runFullProbe(config, state) {
}
}
+async function checkSessionLeaseRotations(config, state) {
+ const sessions = listLiveBgSessions();
+ if (sessions.length === 0) return;
+
+ const leases = readLeases();
+
+ for (const s of sessions) {
+ const lease = leases[s.id];
+
+ // Case 1: Session has no lease assigned yet
+ if (!lease) {
+ const newKey = pickAccountForSession(s.id, config, state);
+ if (newKey) {
+ log(`[session-router] Session ${s.id} has no lease. Assigning to ${newKey}`);
+ recordSessionLease(s.id, newKey, s.pid);
+ if (s.status !== 'busy' && !isLoopSession(s.id)) {
+ doRespawn(s, log);
+ } else {
+ try {
+ const marker = `/tmp/claude-respawn-deferred-${s.id}`;
+ if (!existsSync(marker)) writeFileSync(marker, String(Date.now()));
+ log(`[session-router] Session ${s.id} is ${s.status === 'busy' ? 'busy' : 'a /loop session'} — deferred respawn (sweep handles it).`);
+ } catch {}
+ }
+ return; // Rotate/respawn at most one session per tick to stagger
+ }
+ continue;
+ }
+
+ // Case 2: Session has a lease, check if its account is exhausted or if it is on bedrock
+ const currentKey = lease.accountKey;
+ if (currentKey === 'bedrock') {
+ // Check if a Max account has cooled down and is now viable
+ const newKey = pickAccountForSession(s.id, config, state);
+ if (newKey && newKey !== 'bedrock') {
+ log(`[session-router] Max account available. Migrating session ${s.id} off Bedrock to ${newKey}`);
+ recordSessionLease(s.id, newKey, s.pid);
+ if (s.status !== 'busy' && !isLoopSession(s.id)) {
+ doRespawn(s, log);
+ } else {
+ try {
+ const marker = `/tmp/claude-respawn-deferred-${s.id}`;
+ if (!existsSync(marker)) writeFileSync(marker, String(Date.now()));
+ } catch {}
+ }
+ return; // Rotate at most one session per tick to stagger
+ }
+ continue;
+ }
+
+ const acct = config.accounts.find(a => accountKey(a) === currentKey);
+ if (!acct) continue;
+
+ const cached = state.accounts?.[currentKey]?.lastUtilization;
+ const util = cached?.pct ?? 0;
+ const maxUtil = acct.maxUtilPercent || config.rateLimits.destinationMaxUtilPercent;
+
+ // We no longer guess utilization based on active lease count.
+ // Instead, we rely strictly on the live metrics fetched from Claude API by the usage probe.
+ if (util >= maxUtil) {
+ log(`[session-router] Session ${s.id} leased account ${currentKey} ACTUAL utilization is exhausted (${util}% >= ${maxUtil}%). Rotating.`);
+ const newKey = pickAccountForSession(s.id, config, state);
+ if (newKey && newKey !== currentKey) {
+ log(`[session-router] Swapping lease for session ${s.id}: ${currentKey} -> ${newKey}`);
+ recordSessionLease(s.id, newKey, s.pid);
+
+ if (s.status !== 'busy' && !isLoopSession(s.id)) {
+ doRespawn(s, log);
+ } else {
+ try {
+ const marker = `/tmp/claude-respawn-deferred-${s.id}`;
+ if (!existsSync(marker)) writeFileSync(marker, String(Date.now()));
+ log(`[session-router] Session ${s.id} is busy or looping. Deferred rotation to idle state.`);
+ } catch {}
+ }
+ return; // Rotate at most one session per tick to stagger
+ }
+ }
+ }
+}
+
// ── Main loop ─────────────────────────────────────────────────────────────────
async function mainLoop() {
@@ -1446,10 +1623,13 @@ async function mainLoop() {
let lastRotatedAt = 0; // track when we last rotated
let lastStatusLog = 0; // periodic status logging
let lastLeaseBeat = 0; // cross-machine lease heartbeat (S3)
- let lastRefreshCheck = 0; // dynamic refresh check
+ const lastRefreshCheck = 0; // dynamic refresh check
let lastDriftCheck = 0; // cheap vault-based drift detection
let lastFullProbe = 0; // periodic fleet-wide live-util snapshot refresh
let lastRespawnSweep = 0; // deferred bg-session respawn retry (busy at rotation time)
+ let lastBedrockSweep = 0; // force-swap real Bedrock sessions → OAuth (no defer)
+ let pendingBedrockVerify = []; // PIDs swapped last sweep, verified next sweep
+ const BEDROCK_SWEEP_INTERVAL = 45_000; // ~45s — metered bleed must stop fast
const FULL_PROBE_INTERVAL = 5 * 60_000;
const bedrockRecCtx = {
lastCheck: 0,
@@ -1503,6 +1683,30 @@ async function mainLoop() {
}
}
+ // Bedrock force-swap watchdog (~45s): any session ACTUALLY running on
+ // metered Bedrock (CLAUDE_CODE_USE_BEDROCK=1 in its /proc environ) is
+ // force-swapped to OAuth immediately whenever a usable token exists — NO
+ // 90-min defer for busy/loop sessions (metered bleed overrides loop
+ // preservation). Next sweep verifies the previously-swapped PIDs are off
+ // Bedrock (env + best-effort network corroboration). (Sam 2026-06-14)
+ if (Date.now() - lastBedrockSweep > BEDROCK_SWEEP_INTERVAL) {
+ lastBedrockSweep = Date.now();
+ try {
+ if (pendingBedrockVerify.length) {
+ verifyBedrockSwaps(pendingBedrockVerify, (m) => log(m));
+ pendingBedrockVerify = [];
+ }
+ const { swapped, scanned, swappedPids } = sweepBedrockSessions(config, state, (m) => log(m));
+ if (swapped > 0) {
+ log(`[bedrock-watchdog] force-swapped ${swapped}/${scanned} Bedrock session(s) → OAuth`);
+ lastRotatedAt = Date.now();
+ }
+ pendingBedrockVerify = swappedPids;
+ } catch (e) {
+ log(`[bedrock-watchdog] sweep error: ${e.message?.substring(0, 100)}`);
+ }
+ }
+
if (Date.now() - lastDriftCheck > 120_000) {
lastDriftCheck = Date.now();
const stateLastRotation2 = state.lastRotation ? new Date(state.lastRotation).getTime() : 0;
@@ -1532,12 +1736,12 @@ async function mainLoop() {
const sentinelPath = join(process.env.HOME || '', '.claude', '.bedrock-fallback.json');
// Effective-Bedrock detection: settings.json can be flipped to Bedrock
- // WITHOUT writing the sentinel (use-bedrock.sh, release-session routing,
+ // WITHOUT writing the sentinel (use-bedrock.sh, app-releaser routing,
// manual edits). In that case the metered Bedrock provider stays active
// but the recovery probe below never runs, stranding the box on paid
// tokens even when OAuth accounts have headroom. Treat USE_BEDROCK=1 in
// settings.json as an equivalent recovery trigger so we always probe
- // back to free OAuth.
+ // back to free OAuth. (2026-06-13: stuck on Bedrock 19h post-recovery.)
let settingsBedrock = false;
try {
const sp = join(process.env.HOME || '', '.claude', 'settings.json');
@@ -1571,6 +1775,8 @@ async function mainLoop() {
await doRotation(reason);
lastRotatedAt = Date.now();
await sleep(RATE_LIMIT_COOLDOWN);
+ } else {
+ await checkSessionLeaseRotations(config, state);
}
} catch (err) {
log(`Daemon error: ${err.message}`);
diff --git a/claude-ops/scripts/account-rotation/provider-env.mjs b/claude-ops/scripts/account-rotation/provider-env.mjs
new file mode 100644
index 00000000..06dec4b1
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/provider-env.mjs
@@ -0,0 +1,105 @@
+// provider-env.mjs — shared provider-env helpers for the rotation daemon.
+//
+// Single source of truth for switching a child env between Bedrock (metered,
+// AWS-backed) and OAuth (Claude Max, free-until-cap). The model-var scrub is the
+// correctness fix: when we flip a session FROM Bedrock TO OAuth we must NOT leave
+// hardcoded Bedrock model ids (e.g. ANTHROPIC_MODEL=anthropic.claude-fable-5) or
+// AWS_* credentials in the env — those ids are invalid against the OAuth API and
+// the AWS vars keep the session pointed at metered Bedrock inference.
+//
+// Sam directive (2026-06-14): "Bedrock should never be in use if any /rotate
+// OAuth account has tokens available." On a successful OAuth swap the model
+// resets to the user's default subscription catalog (accepted: "auto reset to
+// default model").
+
+/**
+ * Every env var that pins a session to Bedrock / hardcodes a Bedrock model id.
+ * Mirrors claude-settings-mode.mjs `clearHardcodedModelsForOAuthClaudeSettings`
+ * (plus the AWS credential/profile vars that route inference to Bedrock).
+ * Keep this list in sync with that function.
+ */
+export const BEDROCK_ENV_KEYS = [
+ 'CLAUDE_CODE_USE_BEDROCK',
+ 'AWS_BEDROCK_REGION',
+ 'AWS_REGION',
+ 'AWS_DEFAULT_REGION',
+ 'AWS_PROFILE',
+ 'AWS_DEFAULT_PROFILE',
+ 'AWS_ACCESS_KEY_ID',
+ 'AWS_SECRET_ACCESS_KEY',
+ 'AWS_SESSION_TOKEN',
+ 'ANTHROPIC_MODEL',
+ 'ANTHROPIC_SMALL_FAST_MODEL',
+ 'ANTHROPIC_DEFAULT_SONNET_MODEL',
+ 'ANTHROPIC_DEFAULT_HAIKU_MODEL',
+ 'ANTHROPIC_DEFAULT_OPUS_MODEL',
+ 'ANTHROPIC_DEFAULT_FABLE_MODEL',
+];
+
+/**
+ * Remove every Bedrock-pinning var from `env` in place. Returns the same object.
+ * Use when switching a session TO OAuth.
+ * @param {Record<string,string>} env
+ * @returns {Record<string,string>}
+ */
+export function scrubBedrockEnv(env) {
+ if (!env || typeof env !== 'object') return env;
+ for (const k of BEDROCK_ENV_KEYS) {
+ delete env[k];
+ }
+ return env;
+}
+
+/**
+ * Map a Bedrock inference-profile / model id to its plain OAuth model id.
+ * us.anthropic.claude-fable-5 -> claude-fable-5
+ * anthropic.claude-fable-5-v1:0 -> claude-fable-5
+ * eu.anthropic.claude-sonnet-4-6 -> claude-sonnet-4-6
+ * global.anthropic.claude-opus-4-1-v1:0 -> claude-opus-4-1
+ * Strips the region-profile prefix (us./eu./apac./global.), the `anthropic.`
+ * vendor prefix, and any trailing `-vN:N` / `:vN:N` version flag.
+ * Returns null when the result is not a `claude-*` id (unknown/unmappable).
+ * @param {string} id
+ * @returns {string|null}
+ */
+export function bedrockModelToOAuth(id) {
+ if (!id || typeof id !== 'string') return null;
+ let out = id.trim();
+ // Strip region-profile prefix (us. / eu. / apac. / global.).
+ out = out.replace(/^(us|eu|apac|global)\./i, '');
+ // Strip the vendor prefix.
+ out = out.replace(/^anthropic\./i, '');
+ // Strip a trailing version flag: -v1:0, :v1:0, -v1, etc.
+ out = out.replace(/[-:]v\d+(:\d+)?$/i, '');
+ return /^claude-/.test(out) ? out : null;
+}
+
+/**
+ * Switch `env` to OAuth: scrub all Bedrock vars and set the OAuth token.
+ * The model intentionally resets to the user's default subscription catalog.
+ * @param {Record<string,string>} env
+ * @param {string} oauthToken the bare access token string
+ * @returns {Record<string,string>}
+ */
+export function applyOAuthEnv(env, oauthToken) {
+ scrubBedrockEnv(env);
+ env.CLAUDE_CODE_OAUTH_TOKEN = oauthToken;
+ return env;
+}
+
+/**
+ * Switch `env` to Bedrock fallback: set the Bedrock vars and drop any OAuth token.
+ * Mirrors the bedrock branch in bg-respawn.doRespawn / session-router.spawnWithAccount.
+ * @param {Record<string,string>} env
+ * @param {string} [region='us-east-1']
+ * @param {{primary?:string}} [models] optional model overrides; defaults to fable.
+ * @returns {Record<string,string>}
+ */
+export function applyBedrockEnv(env, region = 'us-east-1', models = {}) {
+ env.CLAUDE_CODE_USE_BEDROCK = '1';
+ env.AWS_BEDROCK_REGION = region;
+ env.AWS_REGION = region;
+ env.ANTHROPIC_MODEL = models.primary || 'anthropic.claude-fable-5';
+ delete env.CLAUDE_CODE_OAUTH_TOKEN;
+ return env;
+}
diff --git a/claude-ops/scripts/account-rotation/session-router.mjs b/claude-ops/scripts/account-rotation/session-router.mjs
index ef6c531a..038da13f 100644
--- a/claude-ops/scripts/account-rotation/session-router.mjs
+++ b/claude-ops/scripts/account-rotation/session-router.mjs
@@ -35,51 +35,55 @@
* time. It never touches the global "Claude Code-credentials" keychain entry.
*/
-import { readFileSync, writeFileSync, existsSync } from 'fs';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { spawnSync, spawn } from 'child_process';
+import { readFileSync, writeFileSync, existsSync } from "fs";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+import { spawnSync, spawn } from "child_process";
+import { applyOAuthEnv, applyBedrockEnv } from "./provider-env.mjs";
const __dirname = dirname(fileURLToPath(import.meta.url));
-const LEASES_PATH = join(__dirname, 'session-leases.json');
-const IS_LINUX = process.platform === 'linux';
+const LEASES_PATH = join(__dirname, "session-leases.json");
+const IS_LINUX = process.platform === "linux";
const LEASE_TTL_MS = 30 * 60_000; // 30 min — sessions that die without cleanup are GC'd
// ── Lease store ───────────────────────────────────────────────────────────────
-function readLeases() {
+export function readLeases() {
try {
- return JSON.parse(readFileSync(LEASES_PATH, 'utf8'));
+ if (!existsSync(LEASES_PATH)) return {};
+ return JSON.parse(readFileSync(LEASES_PATH, "utf8"));
} catch {
return {};
}
}
-function writeLeases(leases) {
+export function writeLeases(leases) {
try {
writeFileSync(LEASES_PATH, JSON.stringify(leases, null, 2));
} catch {}
}
/** Prune leases for dead/expired sessions. */
-function pruneLeases(leases) {
+export function pruneLeases(leases) {
const now = Date.now();
let pruned = false;
for (const [sid, entry] of Object.entries(leases)) {
+ let alive = false;
+ if (entry.pid) {
+ try {
+ process.kill(entry.pid, 0);
+ alive = true;
+ } catch {}
+ }
+
+ // PID-less lease fallback OR grace period for dead PIDs during respawn
const age = now - (entry.ts || 0);
- if (age > LEASE_TTL_MS) {
- // Also check if the pid is still alive
- let alive = false;
- if (entry.pid) {
- try {
- process.kill(entry.pid, 0);
- alive = true;
- } catch {}
- }
- if (!alive) {
- delete leases[sid];
- pruned = true;
- }
+ if (!alive && age <= (entry.pid ? 60000 : LEASE_TTL_MS)) {
+ alive = true;
+ }
+ if (!alive) {
+ delete leases[sid];
+ pruned = true;
}
}
return pruned;
@@ -94,27 +98,37 @@ function accountKey(a) {
// ── Vault token read — platform-aware ─────────────────────────────────────────
const KEYCHAIN_ACCOUNT =
- process.env.CLAUDE_ROTATOR_KEYCHAIN_ACCOUNT || process.env.USER || process.env.LOGNAME || 'claude-ops';
-const LINUX_CRED_PATH = join(process.env.HOME || '', '.claude', '.credentials.json');
-
-function readVaultToken(account) {
+ process.env.CLAUDE_ROTATOR_KEYCHAIN_ACCOUNT ||
+ process.env.USER ||
+ "claude-ops";
+const LINUX_CRED_PATH = join(
+ process.env.HOME || "",
+ ".claude",
+ ".credentials.json",
+);
+
+export function readVaultToken(account) {
const svc = `Claude-Rotation-${accountKey(account)}`;
if (IS_LINUX) {
try {
- const store = JSON.parse(readFileSync(LINUX_CRED_PATH, 'utf8'));
+ const store = JSON.parse(readFileSync(LINUX_CRED_PATH, "utf8"));
const val = store[svc];
if (!val) return null;
- return typeof val === 'string' ? val : JSON.stringify(val);
+ return typeof val === "string" ? val : JSON.stringify(val);
} catch {
return null;
}
}
try {
- const r = spawnSync('security', ['find-generic-password', '-s', svc, '-a', KEYCHAIN_ACCOUNT, '-g'], {
- timeout: 5000,
- encoding: 'utf8',
- });
- const out = (r.stdout || '') + (r.stderr || '');
+ const r = spawnSync(
+ "security",
+ ["find-generic-password", "-s", svc, "-a", KEYCHAIN_ACCOUNT, "-g"],
+ {
+ timeout: 5000,
+ encoding: "utf8",
+ },
+ );
+ const out = (r.stdout || "") + (r.stderr || "");
const m = out.match(/^password: "?(.*?)"?$/m);
return m ? m[1].replace(/\\"/g, '"') : null;
} catch {
@@ -132,7 +146,7 @@ function tokenExpired(tokenJson) {
}
}
-function extractAccessToken(tokenJson) {
+export function extractAccessToken(tokenJson) {
try {
return JSON.parse(tokenJson)?.claudeAiOauth?.accessToken || null;
} catch {
@@ -159,40 +173,92 @@ function extractAccessToken(tokenJson) {
* @returns {string|null} accountKey, or null to use global keychain
*/
export function pickAccountForSession(sessionId, config, state) {
- if (!process.env.CLAUDE_SESSION_ROUTING) return null; // feature-flagged off
-
+ // Always route if we're called
+ if (sessionId === 'app-releaser' || String(sessionId).includes('releaser')) return 'bedrock';
const leases = readLeases();
- pruneLeases(leases);
+ const pruned = pruneLeases(leases);
+ if (pruned) {
+ writeLeases(leases);
+ }
- // Accounts currently leased to live sessions (other than this one being assigned)
- const leasedKeys = new Set(
- Object.entries(leases)
- .filter(([sid]) => sid !== sessionId)
- .map(([, entry]) => entry.accountKey),
- );
+ // Count active leases per account key
+ const leaseCounts = {};
+ for (const a of config.accounts) {
+ leaseCounts[accountKey(a)] = 0;
+ }
+ for (const [sid, entry] of Object.entries(leases)) {
+ if (sid !== sessionId && leaseCounts[entry.accountKey] !== undefined) {
+ let alive = false;
+ if (entry.pid) {
+ try {
+ process.kill(entry.pid, 0);
+ alive = true;
+ } catch {}
+ }
+ if (alive) {
+ leaseCounts[entry.accountKey]++;
+ }
+ }
+ }
const now = Date.now();
let bestAccount = null;
+ let bestLeaseCount = Infinity;
let bestUtil = Infinity;
+ const SORT_WEIGHT_PCT = 2; // Anti-dogpiling weight per active lease for selection sorting only
+ const MAX_CONCURRENT_PER_ACCOUNT = 2; // Strict limit to prevent Anthropic "session limit"
for (const a of config.accounts) {
if (a.disabled === true) continue;
const key = accountKey(a);
- if (leasedKeys.has(key)) continue; // already owned by another live session
const tokenJson = readVaultToken(a);
if (!tokenJson || tokenExpired(tokenJson)) continue;
- // Prefer accounts with lower cached utilization
+ const leasesCount = leaseCounts[key] || 0;
+
+ // STRICT CONCURRENCY CAP
+ if (leasesCount >= MAX_CONCURRENT_PER_ACCOUNT) continue;
+
const cached = state.accounts?.[key]?.lastUtilization;
const util = cached?.pct ?? 50; // unknown = assume 50%
- if (util < bestUtil) {
- bestUtil = util;
- bestAccount = a;
+ const maxUtil = a.maxUtilPercent || config.rateLimits.destinationMaxUtilPercent;
+
+ // Only lease if ACTUAL utilization is strictly below the account's cap (e.g. 98% or 50% account cap)
+ if (util < maxUtil) {
+ // Add a tiny weight to the actual utilization to stagger concurrent spawns (anti-dogpiling)
+ const sortedUtil = util + (leasesCount * SORT_WEIGHT_PCT);
+ if (leasesCount < bestLeaseCount || (leasesCount === bestLeaseCount && sortedUtil < bestUtil)) {
+ bestLeaseCount = leasesCount;
+ bestUtil = sortedUtil;
+ bestAccount = a;
+ }
}
}
- return bestAccount ? accountKey(bestAccount) : null;
+ if (bestAccount) return accountKey(bestAccount);
+
+ // Bedrock is METERED/paid. Sam directive (2026-06-14): "Bedrock should never be
+ // in use if any /rotate OAuth account has tokens available." The primary loop
+ // above can fall through to Bedrock when every account is over its util cap or
+ // at the concurrency cap — but a usable (non-expired) OAuth token is still
+ // strictly preferable to paying for Bedrock. Second pass: pick the OAuth account
+ // with the lowest known utilization that still has a valid token, ignoring the
+ // soft util/concurrency caps. Only return 'bedrock' if ZERO accounts have a
+ // usable token (true last resort).
+ let fallbackAccount = null;
+ let fallbackUtil = Infinity;
+ for (const a of config.accounts) {
+ if (a.disabled === true) continue;
+ const tokenJson = readVaultToken(a);
+ if (!tokenJson || tokenExpired(tokenJson)) continue;
+ const util = state.accounts?.[accountKey(a)]?.lastUtilization?.pct ?? 50;
+ if (util < fallbackUtil) {
+ fallbackUtil = util;
+ fallbackAccount = a;
+ }
+ }
+ return fallbackAccount ? accountKey(fallbackAccount) : 'bedrock';
}
/**
@@ -221,11 +287,12 @@ export function releaseSessionLease(sessionId) {
* Returns null if no per-session lease exists (caller should use global keychain).
*/
export function getTokenForSession(sessionId, config) {
- if (!process.env.CLAUDE_SESSION_ROUTING) return null;
const leases = readLeases();
const lease = leases[sessionId];
if (!lease) return null;
- const account = config.accounts.find((a) => accountKey(a) === lease.accountKey);
+ const account = config.accounts.find(
+ (a) => accountKey(a) === lease.accountKey,
+ );
if (!account) return null;
return readVaultToken(account);
}
@@ -249,35 +316,42 @@ export function getTokenForSession(sessionId, config) {
* @returns {{ proc: ChildProcess, sessionId: string, accountKey: string|null }}
*/
export function spawnWithAccount(args, config, state, opts = {}) {
- const sessionId = opts.sessionId || `session-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+ const sessionId =
+ opts.sessionId ||
+ `session-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const childEnv = { ...(opts.env || process.env) };
let assignedKey = null;
- if (process.env.CLAUDE_SESSION_ROUTING) {
+ if (true) {
const key = pickAccountForSession(sessionId, config, state);
- if (key) {
+ if (key === 'bedrock') {
+ applyBedrockEnv(childEnv);
+ assignedKey = 'bedrock';
+ } else if (key) {
const account = config.accounts.find((a) => accountKey(a) === key);
if (account) {
const tokenJson = readVaultToken(account);
const accessToken = tokenJson ? extractAccessToken(tokenJson) : null;
if (accessToken) {
- childEnv.CLAUDE_CODE_OAUTH_TOKEN = accessToken;
+ // Scrub Bedrock vars (incl. hardcoded ANTHROPIC_MODEL + AWS_*) before
+ // setting the OAuth token, so the session can't keep paying for Bedrock.
+ applyOAuthEnv(childEnv, accessToken);
assignedKey = key;
}
}
}
}
- const proc = spawn('claude', args, {
+ const proc = spawn("claude", args, {
env: childEnv,
detached: opts.detached ?? false,
- stdio: opts.stdio ?? 'inherit',
+ stdio: opts.stdio ?? "inherit",
});
if (assignedKey) {
recordSessionLease(sessionId, assignedKey, proc.pid);
- proc.once('exit', () => releaseSessionLease(sessionId));
+ proc.once("exit", () => releaseSessionLease(sessionId));
}
return { proc, sessionId, accountKey: assignedKey };
@@ -289,6 +363,10 @@ export function spawnWithAccount(args, config, state, opts = {}) {
*/
export function listSessionLeases() {
const leases = readLeases();
+ const pruned = pruneLeases(leases);
+ if (pruned) {
+ writeLeases(leases);
+ }
const now = Date.now();
const result = [];
for (const [sid, entry] of Object.entries(leases)) {