Skip to content

Business operating system for Claude Code — 57 skills, 21 agents, smart daemon. Unified inbox (WhatsApp/Email/Slack/Telegram), autonomous PR merge, full-AWS monitoring, revenue (Stripe+RevenueCat), e-commerce (Shopify), marketing (Klaviyo/Meta/GA4), voice (Bland/ElevenLabs), APM (Datadog/NewRelic/OTEL), YOLO mode.

feat(crs): opt-in 429-cooldown + 401-refresher reconcilers (#705)

Aurora Capitala98fbbef4deb
diff --git a/claude-ops/scripts/account-rotation/config.example.json b/claude-ops/scripts/account-rotation/config.example.json
index d2035388..c6ae9a5e 100644
--- a/claude-ops/scripts/account-rotation/config.example.json
+++ b/claude-ops/scripts/account-rotation/config.example.json
@@ -48,6 +48,23 @@
"on7d": 85,
"floor": 3,
"freshMinutes": 15,
- "_thresholds_note": "Legacy conservative thresholds (off/on/floor) apply only when policy=conservative. Default max-out policy keeps accounts schedulable until genuine rate-limit or overload; utilization % is telemetry only. freshMinutes = max age of claudeUsage trusted for secondary signals."
+ "_thresholds_note": "Legacy conservative thresholds (off/on/floor) apply only when policy=conservative. Default max-out policy keeps accounts schedulable until genuine rate-limit or overload; utilization % is telemetry only. freshMinutes = max age of claudeUsage trusted for secondary signals.",
+ "logDir": null,
+ "logTzOffset": "+00:00",
+ "_logDir_note": "Optional: crs-429-cooldown.mjs reads claude-relay-error-<date>.log from here (absolute path, no ~ expansion) for its optional lead signal. Defaults to <plugin-data-dir>/account-rotation/crs-logs if unset — point it at your CRS deployment's actual log directory. logTzOffset must match the timezone your CRS instance timestamps logs in.",
+ "errorLogWindowMs": 300000,
+ "errorLogCooldownMs": 300000,
+ "_errorLog_note": "errorLogWindowMs: how far back crs-429-cooldown.mjs scans the log for a 429 hit. errorLogCooldownMs: how long a log-only hit holds an account before rateLimitStatus itself would (default 5m each).",
+ "stateDir": null,
+ "_stateDir_note": "Optional absolute path (no ~ expansion), shared base directory for both crs-401-refresher.mjs and crs-429-cooldown.mjs's own state/manual-disable files (crs-401-state.json, crs-429-state.json, crs-manual-disable.json — never one shared file). Defaults to <plugin-data-dir>/account-rotation if unset.",
+ "cooldownStatePath": null,
+ "_cooldownStatePath_note": "Optional: path to a separate 429/rate-limit reconciler's state file, read-only, so crs-401-refresher.mjs's re-enable sweep doesn't fight a genuine rate-limit window. Leave null if you don't run one — treated as 'no known windows', never an error. NEVER point this at the same file crs-401-refresher.mjs writes (crs-401-state.json) — each reconciler must own its own state file.",
+ "manualDisableEnabled": true,
+ "cooldownEnabled": false,
+ "_cooldownEnabled_note": "Off by default (or $CRS_COOLDOWN_ENABLED=1). Turns on crs-429-cooldown.mjs, a fast (1min-cadence) reconciler that holds an account out of rotation only on a real rateLimitStatus.isRateLimited=true from CRS (plus the optional error-log lead signal under logDir/logTzOffset above) — never on utilization. Own state file at <stateDir>/crs-429-state.json.",
+ "tokenRefreshEnabled": false,
+ "_tokenRefreshEnabled_note": "Off by default (or $CRS_TOKEN_REFRESH_ENABLED=1). Turns on crs-401-refresher.mjs, which proactively refreshes each CRS account's OAuth access token via POST /admin/claude-accounts/:id/refresh before it expires, reviving sidelined accounts on success and flagging a dead refresh_token as needs-reauth on failure. Own state file at <stateDir>/crs-401-state.json. Before reviving an account it checks cooldownStatePath (read-only) so it never re-enables one crs-429-cooldown.mjs is genuinely holding for a live rate limit.",
+ "tokenRefreshWindowMs": 1800000,
+ "_tokenRefreshWindowMs_note": "Refresh an account's token once it's within this many ms of expiry (default 30m)."
}
}
diff --git a/claude-ops/scripts/account-rotation/crs-401-refresher.mjs b/claude-ops/scripts/account-rotation/crs-401-refresher.mjs
new file mode 100644
index 00000000..35d59976
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/crs-401-refresher.mjs
@@ -0,0 +1,361 @@
+#!/usr/bin/env node
+// crs-401-refresher.mjs — CRS pool OAuth-token auto-heal (fixes a live-authentication_error
+// gap that crs-priority-daemon.mjs does not cover).
+//
+// OPT-IN. Does nothing (exits 0) unless crs.tokenRefreshEnabled is true (or
+// $CRS_TOKEN_REFRESH_ENABLED=1) — installing this plugin never silently starts a new
+// daemon.
+//
+// Gap this fills: crs-priority-daemon.mjs sidelines accounts with a stale/expiring
+// OAuth token (it will not schedule them), but nothing proactively refreshes the CRS
+// pool's OWN copy of each account's OAuth access token. Once an access token expires
+// with nobody refreshing it, every completion routed to that account returns Anthropic
+// `authentication_error` (HTTP 401) and any in-flight session gets no response — a pool
+// account can go from "healthy" to "silently 401ing" between priority-daemon ticks.
+//
+// This closes that gap headlessly (no browser) using CRS's own server-side
+// refresh_token grant endpoint: POST /admin/claude-accounts/:id/refresh
+//
+// Each tick:
+// 1. Admin-login to CRS.
+// 2. List claude accounts.
+// 3. For any account whose access token is expired or expires within
+// REFRESH_WINDOW_MS, OR that is currently sidelined (schedulable=false), call the
+// refresh endpoint.
+// 4. On success -> ensure schedulable=true (revive) and clear needs-reauth.
+// On failure (dead refresh_token) -> record needs-reauth in this reconciler's OWN
+// state file; leave the account for a human/magic-link re-auth path. We do NOT
+// hard-disable a healthy account here — crs-priority-daemon.mjs owns scheduling;
+// this reconciler only revives on a successful refresh, or sidelines a CONFIRMED
+// dead token (never a transient failure on a still-fresh token).
+//
+// State is owned exclusively by THIS reconciler (own file, atomic write, single-flight
+// lock via crs-reconciler-state.mjs) — see that module's header for why a shared state
+// file between reconcilers is unsafe.
+//
+// CONFIG (config.json "crs" block; every key overridable by env — see
+// config.example.json for the full annotated schema):
+// tokenRefreshEnabled default false — must be true (or $CRS_TOKEN_REFRESH_ENABLED=1)
+// baseUrl, adminUser, adminPasswordEnv, containerName — shared with the other CRS
+// reconcilers
+// tokenRefreshWindowMs default 1800000 (30m) — refresh if expiring within this window
+// stateDir default: <plugin-data-dir>/account-rotation. This
+// reconciler's own state file lives at
+// <stateDir>/crs-401-state.json (never shared with any
+// other reconciler's state file).
+// cooldownStatePath default: unset. Optional READ-ONLY path to a separate
+// rate-limit reconciler's state file (e.g.
+// crs-429-cooldown.mjs's <stateDir>/crs-429-state.json) so
+// this reconciler's re-enable sweep doesn't fight a
+// genuine, still-active rate-limit hold. Never written
+// here — only read.
+//
+// CLI: --dry-run=no writes · --status=print current token/reauth status · --only=<name|id>
+
+import { execFileSync } from 'child_process';
+import { readFileSync, existsSync, mkdirSync, appendFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { homedir } from 'os';
+import { loadJsonState, saveJsonStateAtomic, withOwnStateLock } from './crs-reconciler-state.mjs';
+import { crsBaseUrl, loadRotationConfig, resolveCrsAdminPassword } from './crs-pool-config.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const args = new Set(process.argv.slice(2));
+const DRY = args.has('--dry-run') || process.env.CRS_DRY === '1';
+const STATUS = args.has('--status');
+const ONLY = (process.argv.find((a) => a.startsWith('--only=')) || '').slice('--only='.length);
+
+const cfg = loadRotationConfig();
+const C = cfg.crs || {};
+const num = (v, d) => (v === undefined || v === null || v === '' || Number.isNaN(+v) ? d : +v);
+
+function truthy(v) {
+ return ['1', 'true', 'yes', 'on'].includes(String(v ?? '').trim().toLowerCase());
+}
+
+const ENABLED = truthy(process.env.CRS_TOKEN_REFRESH_ENABLED) || C.tokenRefreshEnabled === true;
+
+const BASE = crsBaseUrl(cfg);
+const ADMIN_USER = process.env.CRS_ADMIN_USER || C.adminUser || 'cradmin';
+const REFRESH_WINDOW_MS = num(process.env.CRS_REFRESH_WINDOW_MS ?? C.tokenRefreshWindowMs, 30 * 60_000);
+
+function expandHome(p) {
+ return p ? p.replace(/^~(?=$|\/)/, homedir()) : p;
+}
+
+const PLUGIN_DATA_DIR =
+ process.env.CLAUDE_PLUGIN_DATA_DIR || join(homedir(), '.claude', 'plugins', 'data', 'ops-ops-marketplace');
+const STATE_DIR = expandHome(process.env.CRS_STATE_DIR || C.stateDir) || join(PLUGIN_DATA_DIR, 'account-rotation');
+const STATE_PATH = expandHome(process.env.CRS_401_STATE_PATH) || join(STATE_DIR, 'crs-401-state.json');
+// Read-only: another reconciler's (e.g. crs-429-cooldown.mjs) own state file, so a
+// successful token refresh doesn't re-enable an account that's genuinely rate-limited.
+const COOLDOWN_STATE_PATH = expandHome(process.env.CRS_COOLDOWN_STATE_PATH || C.cooldownStatePath) || null;
+// This script's OWN operational text log — distinct from crs.logDir (which is the CRS
+// relay's error-log directory that crs-429-cooldown.mjs reads from). Env-only override
+// to avoid any config-key overlap with that unrelated concept.
+const LOG_DIR = expandHome(process.env.CRS_401_LOG_DIR) || join(homedir(), '.claude', 'logs');
+const LOG_PATH = join(LOG_DIR, 'crs-401-refresher.log');
+
+const ts = () => new Date().toISOString();
+function log(msg) {
+ const line = `${ts().slice(11, 19)} [crs-401] ${msg}`;
+ console.log(line);
+ try {
+ mkdirSync(LOG_DIR, { recursive: true });
+ appendFileSync(LOG_PATH, line + '\n');
+ } catch {}
+}
+
+async function jfetch(path, opts = {}) {
+ const r = await fetch(BASE + path, { signal: AbortSignal.timeout(30_000), ...opts });
+ const txt = await r.text();
+ let body;
+ try {
+ body = JSON.parse(txt);
+ } catch {
+ body = txt;
+ }
+ return { status: r.status, body };
+}
+
+let adminToken = null;
+async function ensureAdmin() {
+ if (adminToken) return adminToken;
+ const pw = resolveCrsAdminPassword(cfg, { adminUser: ADMIN_USER });
+ if (!pw) throw new Error('CRS admin password unavailable (set $CRS_ADMIN_PASSWORD or see crs-pool-config.mjs)');
+ const r = await jfetch('/web/auth/login', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ username: ADMIN_USER, password: pw }),
+ });
+ if (r.status !== 200 || !r.body?.token) throw new Error(`admin login failed (HTTP ${r.status})`);
+ adminToken = 'Bearer ' + r.body.token;
+ return adminToken;
+}
+
+async function listClaudeAccounts(auth) {
+ const r = await jfetch('/admin/claude-accounts', { headers: { Authorization: auth } });
+ const list = Array.isArray(r.body) ? r.body : r.body?.data || r.body?.accounts || [];
+ return list.filter((a) => a.platform === 'claude' && a.isActive !== false);
+}
+
+function tokenExpiryMs(a) {
+ const v = a.tokenExpiresAt ?? a.expiresAt ?? 0;
+ const n = Number(v);
+ return Number.isFinite(n) ? n : 0;
+}
+
+function futureMs(v) {
+ if (!v) return 0;
+ const n = Number(v);
+ if (Number.isFinite(n) && n > 0) return n < 10_000_000_000 ? n * 1000 : n;
+ const t = Date.parse(v);
+ return Number.isFinite(t) ? t : 0;
+}
+
+function authoritativeRateLimited(a, nowMs) {
+ const status = a.rateLimitStatus || {};
+ const end = futureMs(status.rateLimitEndAt || a.rateLimitEndAt || a.weeklyRateLimitEndAt);
+ const reason = status.reason || a.rateLimitReason;
+ return Boolean(
+ (status.isRateLimited === true || a.rateLimitStatus?.isRateLimited === 'true') &&
+ end > nowMs &&
+ String(reason || '').includes('authoritative_reset'),
+ );
+}
+
+function hardHeld(a, nowMs) {
+ const status = String(a.status || '');
+ if (['blocked', 'auth_repair', 'error'].includes(status)) return true;
+ return (
+ futureMs(a.rateLimitEndAt || a.rateLimitStatus?.rateLimitEndAt || a.rateLimitStatus?.resetAt || a.weeklyRateLimitEndAt) >
+ nowMs
+ );
+}
+
+// Read-only view of another reconciler's cooldown state (e.g. crs-429-cooldown.mjs).
+// Never written here — only consulted so a successful token refresh doesn't flip a
+// genuinely rate-limited account back to schedulable=true out from under it.
+function externallyHeld(accountId, nowMs) {
+ if (!COOLDOWN_STATE_PATH || !existsSync(COOLDOWN_STATE_PATH)) return false;
+ try {
+ const other = JSON.parse(readFileSync(COOLDOWN_STATE_PATH, 'utf8'));
+ const entry = other?.[accountId];
+ return Boolean(entry?.cooldownUntil > nowMs);
+ } catch {
+ return false;
+ }
+}
+
+async function refreshAccount(auth, a) {
+ const MAX_RETRIES = 3;
+ let lastErr = null;
+ let lastStatus = null;
+ let lastRetryAfter = null;
+ for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+ const r = await fetch(BASE + `/admin/claude-accounts/${a.id}/refresh`, {
+ method: 'POST',
+ signal: AbortSignal.timeout(30_000),
+ headers: { Authorization: auth, 'Content-Type': 'application/json' },
+ body: '{}',
+ });
+ const retryAfterHdr = r.headers.get('retry-after');
+ lastStatus = r.status;
+ lastRetryAfter = retryAfterHdr;
+ if (r.status === 429 && attempt < MAX_RETRIES) {
+ const delayMs = retryAfterHdr
+ ? Math.min(15 * 60_000, Math.max(2_000, parseInt(retryAfterHdr, 10) * 1000 + 2000))
+ : 30_000;
+ log(` ${a.name}: refresh 429 (attempt ${attempt}/${MAX_RETRIES}) — waiting ${Math.round(delayMs / 1000)}s (Retry-After=${retryAfterHdr ?? 'n/a'})...`);
+ await new Promise((r2) => setTimeout(r2, delayMs));
+ continue;
+ }
+ const txt = await r.text();
+ let body;
+ try {
+ body = JSON.parse(txt);
+ } catch {
+ body = txt;
+ }
+ const ok = r.status === 200 && (body?.success === true || body?.data?.success === true || body?.data?.accessToken);
+ const err = body?.message || body?.error?.message || body?.error || (typeof body === 'string' ? body.slice(0, 120) : '');
+ if (r.status === 200 || attempt === MAX_RETRIES) {
+ return { ok, status: r.status, err, retryAfter: lastRetryAfter };
+ }
+ lastErr = err;
+ await new Promise((r2) => setTimeout(r2, 5_000));
+ }
+ return { ok: false, status: lastStatus ?? 0, err: lastErr ?? 'max retries', retryAfter: lastRetryAfter };
+}
+
+async function setSchedulable(auth, a, value) {
+ const r = await jfetch(`/admin/claude-accounts/${a.id}/toggle-schedulable`, {
+ method: 'PUT',
+ headers: { Authorization: auth, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ schedulable: value }),
+ });
+ return r.status === 200;
+}
+
+async function tick() {
+ const nowMs = Date.now();
+ const state = loadJsonState(STATE_PATH, log);
+ const auth = await ensureAdmin();
+ const accounts = await listClaudeAccounts(auth);
+
+ if (STATUS) {
+ console.log(`CRS token status @ ${BASE} — ${accounts.length} claude accounts`);
+ for (const a of accounts) {
+ const exp = tokenExpiryMs(a);
+ const mins = exp ? Math.round((exp - nowMs) / 60_000) : null;
+ const flag = state[a.id]?.needsReauth ? ' NEEDS-REAUTH' : '';
+ const limited = authoritativeRateLimited(a, nowMs) ? ' AUTH-RATE-LIMITED' : '';
+ console.log(
+ ` ${a.name.padEnd(26)} sched=${String(a.schedulable !== false).padEnd(5)} exp_in=${mins === null ? '-' : mins + 'm'}${flag}${limited}`,
+ );
+ }
+ return;
+ }
+
+ const targets = accounts.filter((a) => {
+ if (hardHeld(a, nowMs)) return false;
+ if (ONLY) return a.name === ONLY || a.id === ONLY;
+ const exp = tokenExpiryMs(a);
+ const expiringSoon = !exp || exp <= nowMs + REFRESH_WINDOW_MS;
+ const sidelined = a.schedulable === false;
+ const authLimited = authoritativeRateLimited(a, nowMs);
+ return expiringSoon || sidelined || authLimited;
+ });
+
+ if (!targets.length) {
+ log(`tick: 0 refresh needed (${accounts.length} accounts all fresh > ${REFRESH_WINDOW_MS / 60000}m)`);
+ if (!DRY) saveJsonStateAtomic(STATE_PATH, state);
+ return;
+ }
+
+ let refreshed = 0;
+ let revived = 0;
+ let dead = 0;
+ for (const a of targets) {
+ const exp = tokenExpiryMs(a);
+ const mins = exp ? Math.round((exp - nowMs) / 60_000) : null;
+ const authLimited = authoritativeRateLimited(a, nowMs);
+ if (DRY) {
+ log(
+ `[dry] would ${authLimited ? 'sideline auth-rate-limited' : 'refresh'} ${a.name} (exp_in=${mins === null ? '-' : mins + 'm'}, sched=${a.schedulable !== false})`,
+ );
+ continue;
+ }
+ if (authLimited) {
+ if (a.schedulable !== false) {
+ const off = await setSchedulable(auth, a, false);
+ log(`${a.name}: authoritative rate limit until ${a.rateLimitStatus?.rateLimitEndAt || a.rateLimitEndAt} — ${off ? 'SIDELINED' : 'sideline FAILED'}`);
+ } else {
+ log(`${a.name}: authoritative rate limit already sidelined`);
+ }
+ continue;
+ }
+ const res = await refreshAccount(auth, a);
+ if (res.ok) {
+ refreshed++;
+ delete state[a.id];
+ if (a.schedulable === false && externallyHeld(a.id, Date.now())) {
+ log(`${a.name}: refreshed but leaving sidelined — held by another reconciler's cooldown (rate-limited)`);
+ } else if (a.schedulable === false && !hardHeld(a, Date.now())) {
+ const on = await setSchedulable(auth, a, true);
+ if (on) {
+ revived++;
+ log(`${a.name}: refreshed + revived (schedulable=true)`);
+ } else {
+ log(`${a.name}: refreshed but re-enable failed`);
+ }
+ } else {
+ log(`${a.name}: refreshed (exp_in was ${mins === null ? '-' : mins + 'm'})`);
+ }
+ } else {
+ dead++;
+ state[a.id] = {
+ name: a.name,
+ needsReauth: true,
+ lastFailAt: nowMs,
+ reason: `refresh HTTP ${res.status}: ${res.err}`.slice(0, 160),
+ };
+ // A schedulable account with a dead refresh_token AND an expired/near-expired
+ // access token actively 401s live traffic. Sideline it to protect the pool; a
+ // human/magic-link re-auth revives it, and the next tick re-enables on a
+ // successful refresh. We only sideline confirmed-dead + expiring tokens, never
+ // a transient failure on a still-fresh token.
+ const expired = !exp || exp <= nowMs + REFRESH_WINDOW_MS;
+ if (expired && a.schedulable !== false) {
+ const off = await setSchedulable(auth, a, false);
+ log(`${a.name}: refresh FAILED (HTTP ${res.status}: ${res.err}) — dead token, ${off ? 'SIDELINED' : 'sideline FAILED'}, needs full re-auth`);
+ } else {
+ log(`${a.name}: refresh FAILED (HTTP ${res.status}: ${res.err}) — needs full re-auth`);
+ }
+ }
+ }
+
+ const needsReauth = Object.values(state)
+ .filter((s) => s?.needsReauth)
+ .map((s) => s.name);
+ log(
+ `tick: refreshed=${refreshed} revived=${revived} dead=${dead}` +
+ (needsReauth.length ? ` | needs-reauth: ${needsReauth.join(', ')}` : ''),
+ );
+ if (!DRY) saveJsonStateAtomic(STATE_PATH, state);
+}
+
+async function main() {
+ if (!ENABLED && !STATUS) {
+ log('crs.tokenRefreshEnabled is not true (and $CRS_TOKEN_REFRESH_ENABLED!=1) — skipping tick (opt-in required)');
+ return;
+ }
+ const outcome = await withOwnStateLock(STATE_PATH, tick, log);
+ if (outcome.skipped) log('skipped tick: previous run still holds the state lock');
+}
+
+main().catch((e) => {
+ log(`ERROR: ${e.message}`);
+ process.exit(1);
+});
diff --git a/claude-ops/scripts/account-rotation/crs-429-cooldown.mjs b/claude-ops/scripts/account-rotation/crs-429-cooldown.mjs
new file mode 100644
index 00000000..0827d883
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/crs-429-cooldown.mjs
@@ -0,0 +1,327 @@
+#!/usr/bin/env node
+// crs-429-cooldown.mjs — per-account cooldown driven by real Anthropic API 429s.
+//
+// OPT-IN. Does nothing (exits 0) unless crs.cooldownEnabled is true (or
+// $CRS_COOLDOWN_ENABLED=1) — installing this plugin never silently starts a new
+// daemon. Enable it alongside crs-priority-daemon.mjs when you run a self-hosted
+// claude-relay-service (CRS) pool and want a fast, real-signal-only cooldown watcher
+// that runs on a tighter cadence than the priority daemon.
+//
+// Two signals, in priority order — both are things Anthropic (or CRS observing
+// Anthropic) actually told us, never a speculative/utilization-based guess:
+// 1. rateLimitStatus.isRateLimited = true on the account record (CRS derived
+// this from a real upstream response on an actual inference call).
+// 2. (optional) tail of today's CRS relay error log for 429 lines mentioning the
+// account name — a faster lead signal than #1 while CRS's own record catches
+// up. Reads from crs.logDir (or $CRS_LOG_DIR); a no-op on a fresh install
+// since that directory won't exist yet (this reconciler never assumes a
+// real CRS deployment's log layout).
+//
+// State is owned exclusively by THIS reconciler (own file, atomic write, single-
+// flight lock via crs-reconciler-state.mjs) so that:
+// - a process restart does not release a held account early;
+// - crs-priority-daemon.mjs (which owns utilization-based scheduling) can run on
+// its own cadence without clobbering this reconciler's cooldown record, and
+// vice versa. See crs-reconciler-state.mjs for why that separation exists.
+//
+// Cron/timer cadence: run this more often than crs-priority-daemon.mjs (e.g. every
+// 1 minute vs every 5) since it exists to react to a live 429 fast.
+//
+// CONFIG (config.json "crs" block; every key overridable by env — see
+// config.example.json for the full annotated schema):
+// cooldownEnabled default false — must be true (or $CRS_COOLDOWN_ENABLED=1)
+// baseUrl, adminUser, adminPasswordEnv, containerName — shared with the other
+// CRS reconcilers
+// stateDir default: <plugin-data-dir>/account-rotation. This
+// reconciler's own state file lives at
+// <stateDir>/crs-429-state.json (never shared with any
+// other reconciler's state file).
+// manualDisableEnabled default true — set false to skip the manual-disable
+// allow-list check entirely
+// manualDisablePath default: <stateDir>/crs-manual-disable.json (optional
+// allow-list of account ids/names to always leave alone)
+// logDir default: <plugin-data-dir>/account-rotation/crs-logs.
+// Shared with crs-401-refresher.mjs — this reconciler
+// reads claude-relay-error-<date>.log from here for its
+// optional lead signal. Unset -> that signal is disabled
+// (rateLimitStatus alone is still a real Anthropic signal
+// and is sufficient on its own).
+// logTzOffset default: "+00:00" — offset appended to naive log
+// timestamps before Date.parse; must match your CRS
+// host's log timezone if it isn't UTC.
+// errorLogWindowMs default: 300000 (5m) — how far back to scan the log
+// errorLogCooldownMs default: 300000 (5m) — cooldown granted on a log hit
+//
+// CLI: (none)=one tick · --dry-run=log decisions, no writes · --status=print
+// current held/live accounts and exit.
+
+import { readFileSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { homedir } from 'os';
+import { loadJsonState, saveJsonStateAtomic, withOwnStateLock } from './crs-reconciler-state.mjs';
+import { crsBaseUrl, loadRotationConfig, resolveCrsAdminPassword } from './crs-pool-config.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const args = new Set(process.argv.slice(2));
+const DRY = args.has('--dry-run') || process.env.CRS_DRY === '1';
+const STATUS = args.has('--status');
+
+const ts = () => new Date().toISOString().slice(11, 19);
+const log = (m) => console.log(`${ts()} [crs-429] ${m}`);
+
+const cfg = loadRotationConfig();
+const C = cfg.crs || {};
+
+function truthy(v) {
+ return ['1', 'true', 'yes', 'on'].includes(String(v ?? '').trim().toLowerCase());
+}
+
+const ENABLED = truthy(process.env.CRS_COOLDOWN_ENABLED) || C.cooldownEnabled === true;
+
+const BASE = crsBaseUrl(cfg);
+const ADMIN_USER = process.env.CRS_ADMIN_USER || C.adminUser || 'cradmin';
+const num = (v, d) => (v === undefined || v === null || v === '' || Number.isNaN(+v) ? d : +v);
+
+function expandHome(p) {
+ return p ? p.replace(/^~(?=$|\/)/, homedir()) : p;
+}
+
+const PLUGIN_DATA_DIR =
+ process.env.CLAUDE_PLUGIN_DATA_DIR || join(homedir(), '.claude', 'plugins', 'data', 'ops-ops-marketplace');
+const DEFAULT_STATE_DIR = join(PLUGIN_DATA_DIR, 'account-rotation');
+const STATE_DIR = expandHome(process.env.CRS_STATE_DIR || C.stateDir) || DEFAULT_STATE_DIR;
+
+const STATE_PATH = expandHome(process.env.CRS_COOLDOWN_STATE_PATH) || join(STATE_DIR, 'crs-429-state.json');
+const MANUAL_DISABLE_ENABLED = C.manualDisableEnabled !== false;
+const MANUAL_DISABLE_PATH =
+ expandHome(process.env.CRS_MANUAL_DISABLE_PATH || C.manualDisablePath) || join(STATE_DIR, 'crs-manual-disable.json');
+// Unset by default: the error-log lead signal is opt-in extra sensitivity, not a
+// required capability. rateLimitStatus alone (signal #1) is a real Anthropic
+// signal and is sufficient on its own. Shared logDir/logTzOffset config with
+// crs-401-refresher.mjs.
+// Defaults to a real (likely-absent-on-a-fresh-install) path rather than a null
+// "disabled" sentinel: scanErrorLog() below no-ops safely via existsSync() when
+// the directory/file isn't there, so this stays a safe no-op until you actually
+// point your CRS deployment's error-log directory at it.
+const DEFAULT_LOG_DIR = join(DEFAULT_STATE_DIR, 'crs-logs');
+const ERROR_LOG_DIR = expandHome(process.env.CRS_LOG_DIR || C.logDir) || DEFAULT_LOG_DIR;
+const ERROR_LOG_WINDOW_MS = num(process.env.CRS_ERROR_LOG_WINDOW_MS ?? C.errorLogWindowMs, 5 * 60_000);
+const ERROR_LOG_COOLDOWN_MS = num(process.env.CRS_ERROR_LOG_COOLDOWN_MS ?? C.errorLogCooldownMs, 5 * 60_000);
+const ERROR_LOG_TZ = process.env.CRS_LOG_TZ_OFFSET || C.logTzOffset || '+00:00';
+
+const jfetch = async (path, opts = {}) => {
+ const r = await fetch(BASE + path, opts);
+ const txt = await r.text();
+ let body;
+ try {
+ body = JSON.parse(txt);
+ } catch {
+ body = txt;
+ }
+ return { status: r.status, body };
+};
+
+async function login() {
+ const password = resolveCrsAdminPassword(cfg, { adminUser: ADMIN_USER });
+ if (!password) throw new Error('CRS admin password unavailable (set $CRS_ADMIN_PASSWORD or see crs-pool-config.mjs)');
+ const r = await jfetch('/web/auth/login', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ username: ADMIN_USER, password }),
+ });
+ if (r.status !== 200 || !r.body?.token) throw new Error('login failed HTTP ' + r.status);
+ return 'Bearer ' + r.body.token;
+}
+
+async function getAccounts(auth) {
+ const r = await jfetch('/admin/claude-accounts', { headers: { Authorization: auth } });
+ if (r.status !== 200) throw new Error('GET accounts failed HTTP ' + r.status);
+ const list = Array.isArray(r.body) ? r.body : r.body.data || r.body.accounts || [];
+ return list.filter((a) => a.platform === 'claude' && a.isActive !== false);
+}
+
+const toggle = async (auth, id, schedulable, metadata = {}) =>
+ jfetch('/admin/claude-accounts/' + id + '/toggle-schedulable', {
+ method: 'PUT',
+ headers: { Authorization: auth, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ schedulable, ...metadata }),
+ });
+
+const parseTs = (s) => {
+ if (!s) return null;
+ const t = Date.parse(s);
+ return Number.isFinite(t) ? t : null;
+};
+
+function loadManualDisable() {
+ if (!MANUAL_DISABLE_ENABLED) return new Set();
+ try {
+ if (!existsSync(MANUAL_DISABLE_PATH)) return new Set();
+ const o = JSON.parse(readFileSync(MANUAL_DISABLE_PATH, 'utf8'));
+ return new Set(Array.isArray(o) ? o : Object.keys(o));
+ } catch {
+ return new Set();
+ }
+}
+
+// === optional signal #2: tail today's CRS relay error log for 429 lines =====
+function scanErrorLog(nowMs) {
+ if (!ERROR_LOG_DIR) return new Map();
+ const today = new Date(nowMs).toISOString().slice(0, 10);
+ const logPath = join(ERROR_LOG_DIR, `claude-relay-error-${today}.log`);
+ if (!existsSync(logPath)) return new Map();
+ let lines;
+ try {
+ lines = readFileSync(logPath, 'utf8').split('\n').filter(Boolean);
+ } catch {
+ return new Map();
+ }
+ const recent = new Map(); // account name -> latest hit ts (ms)
+ const windowStart = nowMs - ERROR_LOG_WINDOW_MS;
+ for (const line of lines) {
+ try {
+ const e = JSON.parse(line);
+ const m = e.msg || '';
+ if (!m.includes('error status: 429')) continue;
+ const rawTs = String(e.ts || '').replace(' ', 'T');
+ const t = Date.parse(ERROR_LOG_TZ && !/[Zz]|[+-]\d\d:?\d\d$/.test(rawTs) ? rawTs + ERROR_LOG_TZ : rawTs);
+ if (!Number.isFinite(t) || t < windowStart) continue;
+ const acc = m.match(/Account:\s*([^\s"]+)/);
+ if (!acc) continue;
+ const cur = recent.get(acc[1]);
+ if (!cur || t > cur) recent.set(acc[1], t);
+ } catch {}
+ }
+ return recent;
+}
+
+async function tick() {
+ const auth = await login();
+ const accts = await getAccounts(auth);
+ const state = loadJsonState(STATE_PATH, log);
+ const nowMs = Date.now();
+ const errorLogHits = scanErrorLog(nowMs);
+ const manualDisable = loadManualDisable();
+
+ if (STATUS) {
+ const held = Object.entries(state).filter(([, s]) => s.cooldownUntil > nowMs);
+ console.log(`CRS 429-cooldown @ ${BASE} — ${accts.length} claude accounts, ${held.length} held`);
+ for (const [id, s] of held) {
+ console.log(` ${s.name || id}: held until ${new Date(s.cooldownUntil).toISOString()} (${s.reason})`);
+ }
+ return;
+ }
+
+ const changes = [];
+ for (const a of accts) {
+ if (manualDisable.has(a.id) || manualDisable.has(a.name)) continue;
+ const cur = a.schedulable !== false;
+ const rl = a.rateLimitStatus?.isRateLimited === true;
+ const rlMin = Math.max(a.rateLimitStatus?.minutesRemaining || 0, 0);
+ const rlEnd = parseTs(a.rateLimitStatus?.rateLimitEndAt);
+ const persistedRedisHoldEnd = parseTs(a.rateLimitEndAt || a.rateLimitStatus?.rateLimitEndAt);
+ const rateLimitReason = a.rateLimitReason || a.rateLimitStatus?.reason || null;
+ const transient429 = rateLimitReason === 'transient_default_cooldown';
+ const errLogHit = errorLogHits.get(a.name);
+
+ // Trust only real upstream signals here (never claudeUsage.*.utilization —
+ // that cache is fine for crs-priority-daemon.mjs's global health checks but
+ // lags live state by minutes and must not drive a per-account hold).
+ let cooldownUntil = 0;
+ let reason = null;
+ if (rl && !transient429) {
+ const t = rlEnd && rlEnd > nowMs ? rlEnd : nowMs + Math.max(rlMin, 1) * 60_000;
+ cooldownUntil = Math.max(cooldownUntil, t);
+ reason = `api-rl ${Math.max(rlMin, 1)}m remaining${rateLimitReason ? ` (${rateLimitReason})` : ''}`;
+ }
+ if (errLogHit && rl && rateLimitReason && !transient429) {
+ const t = errLogHit + ERROR_LOG_COOLDOWN_MS;
+ if (t > cooldownUntil) {
+ cooldownUntil = t;
+ reason = `429 in error log @ ${new Date(errLogHit).toISOString()}`;
+ }
+ }
+
+ const prevHeld = state[a.id]?.cooldownUntil || 0;
+
+ if (cooldownUntil > nowMs) {
+ state[a.id] = { name: a.name, cooldownUntil, reason };
+ const holdMissingInRedis = !persistedRedisHoldEnd || persistedRedisHoldEnd < cooldownUntil - 1000;
+ if (cur || holdMissingInRedis) {
+ if (DRY) {
+ changes.push({ name: a.name, action: cur ? '[dry]OFF' : '[dry]HOLD', reason, http: '-' });
+ } else {
+ const r = await toggle(auth, a.id, false, {
+ rateLimitEndAt: new Date(cooldownUntil).toISOString(),
+ rateLimitReason: reason || 'api-rl',
+ });
+ changes.push({
+ name: a.name,
+ action: cur ? 'OFF' : 'HOLD',
+ reason: `cooldown until ${new Date(cooldownUntil).toISOString().slice(11, 19)} (${reason})`,
+ http: r.status,
+ });
+ }
+ }
+ } else if (prevHeld > nowMs) {
+ // Persisted hold still wins — releasing before the recorded upstream
+ // reset just re-hammers the same account.
+ state[a.id] = { name: a.name, cooldownUntil: prevHeld, reason: state[a.id]?.reason || 'persisted-api-rl-bench' };
+ const holdMissingInRedis = !persistedRedisHoldEnd || persistedRedisHoldEnd < prevHeld - 1000;
+ if ((cur || holdMissingInRedis) && !DRY) {
+ const r = await toggle(auth, a.id, false, {
+ rateLimitEndAt: new Date(prevHeld).toISOString(),
+ rateLimitReason: state[a.id]?.reason || 'persisted-api-rl-bench',
+ });
+ changes.push({
+ name: a.name,
+ action: cur ? 'OFF' : 'HOLD',
+ reason: `persisted cooldown until ${new Date(prevHeld).toISOString().slice(11, 19)}`,
+ http: r.status,
+ });
+ } else if ((cur || holdMissingInRedis) && DRY) {
+ changes.push({ name: a.name, action: '[dry]HOLD', reason: 'persisted cooldown', http: '-' });
+ }
+ } else if (prevHeld && prevHeld <= nowMs) {
+ delete state[a.id];
+ if (!cur) {
+ if (DRY) {
+ changes.push({ name: a.name, action: '[dry]ON', reason: 'cooldown expired', http: '-' });
+ } else {
+ const r = await toggle(auth, a.id, true);
+ changes.push({ name: a.name, action: 'ON', reason: 'cooldown expired', http: r.status });
+ }
+ }
+ }
+ }
+
+ if (!DRY) saveJsonStateAtomic(STATE_PATH, state);
+ const held = Object.values(state).filter((s) => s.cooldownUntil > nowMs);
+ const live = accts.length - held.length;
+ log(
+ `tick: ${changes.length} change(s). live=${live}/${accts.length} | held=[${held
+ .map((h) => `${h.name}@${new Date(h.cooldownUntil).toISOString().slice(11, 19)}`)
+ .join(',')}]`,
+ );
+ for (const c of changes) log(` ${c.name}: ${c.action} (${c.reason || ''}) [HTTP ${c.http}]`);
+}
+
+async function main() {
+ if (!ENABLED && !STATUS) {
+ log('crs.cooldownEnabled is not true (and $CRS_COOLDOWN_ENABLED!=1) — skipping tick (opt-in required)');
+ return;
+ }
+ const outcome = await withOwnStateLock(STATE_PATH, tick, log);
+ if (outcome.skipped) log('skipped tick: previous run still holds the state lock');
+}
+
+main().catch((e) => {
+ const message = e?.message || String(e);
+ if (message === 'fetch failed' || message.includes('ECONNREFUSED') || message.includes('ECONNRESET') || message.includes('ETIMEDOUT')) {
+ log('WARN transient CRS fetch failure; skipping this tick');
+ process.exit(0);
+ }
+ log('ERROR: ' + message);
+ process.exit(1);
+});
diff --git a/claude-ops/scripts/account-rotation/crs-pool-config.mjs b/claude-ops/scripts/account-rotation/crs-pool-config.mjs
index f386f506..6d02db0b 100644
--- a/claude-ops/scripts/account-rotation/crs-pool-config.mjs
+++ b/claude-ops/scripts/account-rotation/crs-pool-config.mjs
@@ -7,6 +7,7 @@
*/
import { readFileSync, existsSync } from 'fs';
+import { execFileSync } from 'child_process';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { homedir } from 'os';
@@ -100,6 +101,66 @@ export function crsPolicy(config = loadRotationConfig()) {
return 'conservative';
}
+/** Candidate paths for the shared ops credential-store CLI (never hardcoded to one account/box). */
+export function credentialStoreCandidates() {
+ const pluginRoot = process.env.CLAUDE_PLUGIN_ROOT || join(__dirname, '..', '..');
+ return [
+ process.env.OPS_CREDENTIAL_STORE,
+ join(homedir(), '.claude', 'plugins', 'cache', 'ops-marketplace', 'ops', 'current', 'lib', 'credential-store.sh'),
+ join(homedir(), '.claude', 'plugins', 'marketplaces', 'ops-marketplace', 'claude-ops', 'lib', 'credential-store.sh'),
+ join(pluginRoot, 'lib', 'credential-store.sh'),
+ ].filter(Boolean);
+}
+
+export function resolveCredentialStorePath() {
+ const candidates = credentialStoreCandidates();
+ for (const p of candidates) {
+ if (existsSync(p)) return p;
+ }
+ return candidates[candidates.length - 1];
+}
+
+/**
+ * Resolve the CRS admin password for HTTP basic auth against a self-hosted
+ * claude-relay-service pool. Resolution order (first hit wins), none of it
+ * ever hardcoded to one deployment:
+ * 1. $CRS_ADMIN_PASSWORD
+ * 2. the env var named by config.crs.adminPasswordEnv (or $CRS_ADMIN_PASSWORD_ENV)
+ * 3. `docker inspect <container>` — reads ADMIN_PASSWORD out of a co-located
+ * Docker Compose CRS deploy's own container env
+ * 4. the shared ops credential-store CLI, key `CRS-Admin-<adminUser>`
+ * Returns null (never throws) if nothing resolves — callers decide whether that
+ * means "skip this tick" or "hard error".
+ */
+export function resolveCrsAdminPassword(config = loadRotationConfig(), opts = {}) {
+ if (process.env.CRS_ADMIN_PASSWORD) return process.env.CRS_ADMIN_PASSWORD;
+ const envName = config.crs?.adminPasswordEnv || process.env.CRS_ADMIN_PASSWORD_ENV;
+ if (envName && process.env[envName]) return process.env[envName];
+
+ const adminUser = opts.adminUser || process.env.CRS_ADMIN_USER || config.crs?.adminUser || 'cradmin';
+ const container = opts.container || process.env.CRS_CONTAINER || config.crs?.containerName || 'crs-claude-relay-1';
+ const execOpts = { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] };
+
+ try {
+ const out = execFileSync(
+ 'docker',
+ ['inspect', container, '--format', '{{range .Config.Env}}{{println .}}{{end}}'],
+ execOpts,
+ );
+ const m = out.match(/^ADMIN_PASSWORD=(.+)$/m);
+ if (m?.[1]?.trim()) return m[1].trim();
+ } catch {}
+
+ try {
+ const cred = resolveCredentialStorePath();
+ const acct = process.env.CLAUDE_ROTATOR_KEYCHAIN_ACCOUNT || process.env.USER || 'claude-ops';
+ const pw = execFileSync('bash', [cred, 'get', `CRS-Admin-${adminUser}`, acct], execOpts).trim();
+ if (pw) return pw;
+ } catch {}
+
+ return null;
+}
+
export function vaultLookupKeysForEmail(email, accounts = []) {
const keys = new Set();
if (email) keys.add(email);
diff --git a/claude-ops/scripts/account-rotation/crs-reconciler-state.mjs b/claude-ops/scripts/account-rotation/crs-reconciler-state.mjs
new file mode 100644
index 00000000..1ed92480
--- /dev/null
+++ b/claude-ops/scripts/account-rotation/crs-reconciler-state.mjs
@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+/**
+ * crs-reconciler-state.mjs — shared own-state-file helper for CRS pool reconcilers.
+ *
+ * Each reconciler that watches a claude-relay-service (CRS) account pool (cooldown
+ * tracking, token-refresh tracking, priority scoring, ...) MUST persist to its OWN
+ * state file — never a file shared with another reconciler. Sharing one file was the
+ * root cause of a documented production incident: reconciler A's non-atomic write
+ * clobbered reconciler B's cooldown data mid-tick, silently erasing real rate-limit
+ * holds and triggering a pool-wide re-enable/429 oscillation.
+ *
+ * This module gives every reconciler two small, dependency-free guarantees:
+ * 1. Atomic write — write to a temp file in the same directory, then rename() over
+ * the real path. rename() is atomic on the same filesystem, so a reader (or a
+ * concurrent tick) never observes a half-written / truncated JSON file.
+ * 2. Single-flight lock — a short-lived advisory lock (mkdir-based; no native deps,
+ * works the same on Linux and macOS) around the read-modify-write section, so two
+ * overlapping ticks of the SAME reconciler (a slow tick plus a cron/timer overlap)
+ * can't interleave writes. This is a lightweight stand-in for flock(1), not a
+ * distributed lock — sufficient for a single-host cron/timer cadence. A stale lock
+ * (older than LOCK_STALE_MS, e.g. left behind by a crashed process) is reclaimed
+ * rather than wedging the reconciler forever.
+ *
+ * Nothing here talks to CRS or knows about account schedules — it is pure state I/O so
+ * every reconciler can `import` the same tiny, well-tested primitive instead of each
+ * re-implementing tmp+rename by hand.
+ */
+
+import { readFileSync, writeFileSync, existsSync, renameSync, mkdirSync, rmdirSync, statSync } from 'fs';
+import { dirname } from 'path';
+
+const LOCK_STALE_MS = 30_000;
+
+export function loadJsonState(path, log = console.error) {
+ if (!existsSync(path)) return {};
+ try {
+ return JSON.parse(readFileSync(path, 'utf8'));
+ } catch (e) {
+ log(`[crs-reconciler-state] WARN corrupt state at ${path}, resetting: ${e.message}`);
+ return {};
+ }
+}
+
+/** Atomic write: tmp file in the same directory + rename(). */
+export function saveJsonStateAtomic(path, data) {
+ mkdirSync(dirname(path), { recursive: true });
+ const tmp = `${path}.tmp.${process.pid}`;
+ writeFileSync(tmp, JSON.stringify(data, null, 2));
+ renameSync(tmp, path);
+}
+
+/**
+ * Run `fn` (sync or async) holding a single-flight advisory lock scoped to `path`.
+ * If the lock is already held by a live tick, this returns { skipped: true } instead
+ * of blocking — the caller should log and exit 0 (cron will just try again next tick).
+ *
+ * @returns {Promise<{ skipped: boolean, result?: any }>}
+ */
+export async function withOwnStateLock(path, fn, log = console.error) {
+ const lockDir = `${path}.lock`;
+ mkdirSync(dirname(path), { recursive: true });
+ const acquired = tryAcquireLock(lockDir, log);
+ if (!acquired) return { skipped: true };
+ try {
+ const result = await fn();
+ return { skipped: false, result };
+ } finally {
+ try {
+ rmdirSync(lockDir);
+ } catch {}
+ }
+}
+
+function tryAcquireLock(lockDir, log) {
+ try {
+ mkdirSync(lockDir);
+ return true;
+ } catch (e) {
+ if (e.code !== 'EEXIST') throw e;
+ }
+ // Lock dir already exists — check whether it's stale (owner crashed mid-tick).
+ let ageMs;
+ try {
+ ageMs = Date.now() - statSync(lockDir).mtimeMs;
+ } catch {
+ // Lock dir vanished between the mkdir failure and this stat (owner just
+ // released it) — try once more to acquire it fresh.
+ try {
+ mkdirSync(lockDir);
+ return true;
+ } catch {
+ return false;
+ }
+ }
+ if (ageMs < LOCK_STALE_MS) {
+ log(`[crs-reconciler-state] lock held (age ${Math.round(ageMs / 1000)}s) at ${lockDir} — skipping this tick`);
+ return false;
+ }
+ try {
+ rmdirSync(lockDir);
+ mkdirSync(lockDir);
+ log(`[crs-reconciler-state] reclaimed stale lock (age ${Math.round(ageMs / 1000)}s) at ${lockDir}`);
+ return true;
+ } catch {
+ return false;
+ }
+}