Business operating system for Claude Code — 57 skills, 21 agents, smart daemon. Unified inbox (WhatsApp/Email/Slack/Telegram), autonomous PR merge, full-AWS monitoring, revenue (Stripe+RevenueCat), e-commerce (Shopify), marketing (Klaviyo/Meta/GA4), voice (Bland/ElevenLabs), APM (Datadog/NewRelic/OTEL), YOLO mode.
Shell agent-teamsai-agentai-automationanthropicautonomous-agentawsbusiness-operationsclaudeclaude-codeclaude-code-plugindevopsecommerceinbox-zeroklaviyomarketingmcpmorning-briefingpr-automationshopifyterminal
feat(crs): opt-in 429-cooldown + 401-refresher reconcilers (#705)
Aurora Capitala98fbbef4deb
diff --git a/claude-ops/scripts/account-rotation/config.example.json b/claude-ops/scripts/account-rotation/config.example.jsonindex d2035388..c6ae9a5e 100644--- a/claude-ops/scripts/account-rotation/config.example.json+++ b/claude-ops/scripts/account-rotation/config.example.json@@ -48,6 +48,23 @@"on7d": 85,"floor": 3,"freshMinutes": 15,- "_thresholds_note": "Legacy conservative thresholds (off/on/floor) apply only when policy=conservative. Default max-out policy keeps accounts schedulable until genuine rate-limit or overload; utilization % is telemetry only. freshMinutes = max age of claudeUsage trusted for secondary signals."+ "_thresholds_note": "Legacy conservative thresholds (off/on/floor) apply only when policy=conservative. Default max-out policy keeps accounts schedulable until genuine rate-limit or overload; utilization % is telemetry only. freshMinutes = max age of claudeUsage trusted for secondary signals.",+ "logDir": null,+ "logTzOffset": "+00:00",+ "_logDir_note": "Optional: crs-429-cooldown.mjs reads claude-relay-error-<date>.log from here (absolute path, no ~ expansion) for its optional lead signal. Defaults to <plugin-data-dir>/account-rotation/crs-logs if unset — point it at your CRS deployment's actual log directory. logTzOffset must match the timezone your CRS instance timestamps logs in.",+ "errorLogWindowMs": 300000,+ "errorLogCooldownMs": 300000,+ "_errorLog_note": "errorLogWindowMs: how far back crs-429-cooldown.mjs scans the log for a 429 hit. errorLogCooldownMs: how long a log-only hit holds an account before rateLimitStatus itself would (default 5m each).",+ "stateDir": null,+ "_stateDir_note": "Optional absolute path (no ~ expansion), shared base directory for both crs-401-refresher.mjs and crs-429-cooldown.mjs's own state/manual-disable files (crs-401-state.json, crs-429-state.json, crs-manual-disable.json — never one shared file). Defaults to <plugin-data-dir>/account-rotation if unset.",+ "cooldownStatePath": null,+ "_cooldownStatePath_note": "Optional: path to a separate 429/rate-limit reconciler's state file, read-only, so crs-401-refresher.mjs's re-enable sweep doesn't fight a genuine rate-limit window. Leave null if you don't run one — treated as 'no known windows', never an error. NEVER point this at the same file crs-401-refresher.mjs writes (crs-401-state.json) — each reconciler must own its own state file.",+ "manualDisableEnabled": true,+ "cooldownEnabled": false,+ "_cooldownEnabled_note": "Off by default (or $CRS_COOLDOWN_ENABLED=1). Turns on crs-429-cooldown.mjs, a fast (1min-cadence) reconciler that holds an account out of rotation only on a real rateLimitStatus.isRateLimited=true from CRS (plus the optional error-log lead signal under logDir/logTzOffset above) — never on utilization. Own state file at <stateDir>/crs-429-state.json.",+ "tokenRefreshEnabled": false,+ "_tokenRefreshEnabled_note": "Off by default (or $CRS_TOKEN_REFRESH_ENABLED=1). Turns on crs-401-refresher.mjs, which proactively refreshes each CRS account's OAuth access token via POST /admin/claude-accounts/:id/refresh before it expires, reviving sidelined accounts on success and flagging a dead refresh_token as needs-reauth on failure. Own state file at <stateDir>/crs-401-state.json. Before reviving an account it checks cooldownStatePath (read-only) so it never re-enables one crs-429-cooldown.mjs is genuinely holding for a live rate limit.",+ "tokenRefreshWindowMs": 1800000,+ "_tokenRefreshWindowMs_note": "Refresh an account's token once it's within this many ms of expiry (default 30m)."}}diff --git a/claude-ops/scripts/account-rotation/crs-401-refresher.mjs b/claude-ops/scripts/account-rotation/crs-401-refresher.mjsnew file mode 100644index 00000000..35d59976--- /dev/null+++ b/claude-ops/scripts/account-rotation/crs-401-refresher.mjs@@ -0,0 +1,361 @@+#!/usr/bin/env node+// crs-401-refresher.mjs — CRS pool OAuth-token auto-heal (fixes a live-authentication_error+// gap that crs-priority-daemon.mjs does not cover).+//+// OPT-IN. Does nothing (exits 0) unless crs.tokenRefreshEnabled is true (or+// $CRS_TOKEN_REFRESH_ENABLED=1) — installing this plugin never silently starts a new+// daemon.+//+// Gap this fills: crs-priority-daemon.mjs sidelines accounts with a stale/expiring+// OAuth token (it will not schedule them), but nothing proactively refreshes the CRS+// pool's OWN copy of each account's OAuth access token. Once an access token expires+// with nobody refreshing it, every completion routed to that account returns Anthropic+// `authentication_error` (HTTP 401) and any in-flight session gets no response — a pool+// account can go from "healthy" to "silently 401ing" between priority-daemon ticks.+//+// This closes that gap headlessly (no browser) using CRS's own server-side+// refresh_token grant endpoint: POST /admin/claude-accounts/:id/refresh+//+// Each tick:+// 1. Admin-login to CRS.+// 2. List claude accounts.+// 3. For any account whose access token is expired or expires within+// REFRESH_WINDOW_MS, OR that is currently sidelined (schedulable=false), call the+// refresh endpoint.+// 4. On success -> ensure schedulable=true (revive) and clear needs-reauth.+// On failure (dead refresh_token) -> record needs-reauth in this reconciler's OWN+// state file; leave the account for a human/magic-link re-auth path. We do NOT+// hard-disable a healthy account here — crs-priority-daemon.mjs owns scheduling;+// this reconciler only revives on a successful refresh, or sidelines a CONFIRMED+// dead token (never a transient failure on a still-fresh token).+//+// State is owned exclusively by THIS reconciler (own file, atomic write, single-flight+// lock via crs-reconciler-state.mjs) — see that module's header for why a shared state+// file between reconcilers is unsafe.+//+// CONFIG (config.json "crs" block; every key overridable by env — see+// config.example.json for the full annotated schema):+// tokenRefreshEnabled default false — must be true (or $CRS_TOKEN_REFRESH_ENABLED=1)+// baseUrl, adminUser, adminPasswordEnv, containerName — shared with the other CRS+// reconcilers+// tokenRefreshWindowMs default 1800000 (30m) — refresh if expiring within this window+// stateDir default: <plugin-data-dir>/account-rotation. This+// reconciler's own state file lives at+// <stateDir>/crs-401-state.json (never shared with any+// other reconciler's state file).+// cooldownStatePath default: unset. Optional READ-ONLY path to a separate+// rate-limit reconciler's state file (e.g.+// crs-429-cooldown.mjs's <stateDir>/crs-429-state.json) so+// this reconciler's re-enable sweep doesn't fight a+// genuine, still-active rate-limit hold. Never written+// here — only read.+//+// CLI: --dry-run=no writes · --status=print current token/reauth status · --only=<name|id>++import { execFileSync } from 'child_process';+import { readFileSync, existsSync, mkdirSync, appendFileSync } from 'fs';+import { join, dirname } from 'path';+import { fileURLToPath } from 'url';+import { homedir } from 'os';+import { loadJsonState, saveJsonStateAtomic, withOwnStateLock } from './crs-reconciler-state.mjs';+import { crsBaseUrl, loadRotationConfig, resolveCrsAdminPassword } from './crs-pool-config.mjs';++const __dirname = dirname(fileURLToPath(import.meta.url));+const args = new Set(process.argv.slice(2));+const DRY = args.has('--dry-run') || process.env.CRS_DRY === '1';+const STATUS = args.has('--status');+const ONLY = (process.argv.find((a) => a.startsWith('--only=')) || '').slice('--only='.length);++const cfg = loadRotationConfig();+const C = cfg.crs || {};+const num = (v, d) => (v === undefined || v === null || v === '' || Number.isNaN(+v) ? d : +v);++function truthy(v) {+ return ['1', 'true', 'yes', 'on'].includes(String(v ?? '').trim().toLowerCase());+}++const ENABLED = truthy(process.env.CRS_TOKEN_REFRESH_ENABLED) || C.tokenRefreshEnabled === true;++const BASE = crsBaseUrl(cfg);+const ADMIN_USER = process.env.CRS_ADMIN_USER || C.adminUser || 'cradmin';+const REFRESH_WINDOW_MS = num(process.env.CRS_REFRESH_WINDOW_MS ?? C.tokenRefreshWindowMs, 30 * 60_000);++function expandHome(p) {+ return p ? p.replace(/^~(?=$|\/)/, homedir()) : p;+}++const PLUGIN_DATA_DIR =+ process.env.CLAUDE_PLUGIN_DATA_DIR || join(homedir(), '.claude', 'plugins', 'data', 'ops-ops-marketplace');+const STATE_DIR = expandHome(process.env.CRS_STATE_DIR || C.stateDir) || join(PLUGIN_DATA_DIR, 'account-rotation');+const STATE_PATH = expandHome(process.env.CRS_401_STATE_PATH) || join(STATE_DIR, 'crs-401-state.json');+// Read-only: another reconciler's (e.g. crs-429-cooldown.mjs) own state file, so a+// successful token refresh doesn't re-enable an account that's genuinely rate-limited.+const COOLDOWN_STATE_PATH = expandHome(process.env.CRS_COOLDOWN_STATE_PATH || C.cooldownStatePath) || null;+// This script's OWN operational text log — distinct from crs.logDir (which is the CRS+// relay's error-log directory that crs-429-cooldown.mjs reads from). Env-only override+// to avoid any config-key overlap with that unrelated concept.+const LOG_DIR = expandHome(process.env.CRS_401_LOG_DIR) || join(homedir(), '.claude', 'logs');+const LOG_PATH = join(LOG_DIR, 'crs-401-refresher.log');++const ts = () => new Date().toISOString();+function log(msg) {+ const line = `${ts().slice(11, 19)} [crs-401] ${msg}`;+ console.log(line);+ try {+ mkdirSync(LOG_DIR, { recursive: true });+ appendFileSync(LOG_PATH, line + '\n');+ } catch {}+}++async function jfetch(path, opts = {}) {+ const r = await fetch(BASE + path, { signal: AbortSignal.timeout(30_000), ...opts });+ const txt = await r.text();+ let body;+ try {+ body = JSON.parse(txt);+ } catch {+ body = txt;+ }+ return { status: r.status, body };+}++let adminToken = null;+async function ensureAdmin() {+ if (adminToken) return adminToken;+ const pw = resolveCrsAdminPassword(cfg, { adminUser: ADMIN_USER });+ if (!pw) throw new Error('CRS admin password unavailable (set $CRS_ADMIN_PASSWORD or see crs-pool-config.mjs)');+ const r = await jfetch('/web/auth/login', {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ username: ADMIN_USER, password: pw }),+ });+ if (r.status !== 200 || !r.body?.token) throw new Error(`admin login failed (HTTP ${r.status})`);+ adminToken = 'Bearer ' + r.body.token;+ return adminToken;+}++async function listClaudeAccounts(auth) {+ const r = await jfetch('/admin/claude-accounts', { headers: { Authorization: auth } });+ const list = Array.isArray(r.body) ? r.body : r.body?.data || r.body?.accounts || [];+ return list.filter((a) => a.platform === 'claude' && a.isActive !== false);+}++function tokenExpiryMs(a) {+ const v = a.tokenExpiresAt ?? a.expiresAt ?? 0;+ const n = Number(v);+ return Number.isFinite(n) ? n : 0;+}++function futureMs(v) {+ if (!v) return 0;+ const n = Number(v);+ if (Number.isFinite(n) && n > 0) return n < 10_000_000_000 ? n * 1000 : n;+ const t = Date.parse(v);+ return Number.isFinite(t) ? t : 0;+}++function authoritativeRateLimited(a, nowMs) {+ const status = a.rateLimitStatus || {};+ const end = futureMs(status.rateLimitEndAt || a.rateLimitEndAt || a.weeklyRateLimitEndAt);+ const reason = status.reason || a.rateLimitReason;+ return Boolean(+ (status.isRateLimited === true || a.rateLimitStatus?.isRateLimited === 'true') &&+ end > nowMs &&+ String(reason || '').includes('authoritative_reset'),+ );+}++function hardHeld(a, nowMs) {+ const status = String(a.status || '');+ if (['blocked', 'auth_repair', 'error'].includes(status)) return true;+ return (+ futureMs(a.rateLimitEndAt || a.rateLimitStatus?.rateLimitEndAt || a.rateLimitStatus?.resetAt || a.weeklyRateLimitEndAt) >+ nowMs+ );+}++// Read-only view of another reconciler's cooldown state (e.g. crs-429-cooldown.mjs).+// Never written here — only consulted so a successful token refresh doesn't flip a+// genuinely rate-limited account back to schedulable=true out from under it.+function externallyHeld(accountId, nowMs) {+ if (!COOLDOWN_STATE_PATH || !existsSync(COOLDOWN_STATE_PATH)) return false;+ try {+ const other = JSON.parse(readFileSync(COOLDOWN_STATE_PATH, 'utf8'));+ const entry = other?.[accountId];+ return Boolean(entry?.cooldownUntil > nowMs);+ } catch {+ return false;+ }+}++async function refreshAccount(auth, a) {+ const MAX_RETRIES = 3;+ let lastErr = null;+ let lastStatus = null;+ let lastRetryAfter = null;+ for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {+ const r = await fetch(BASE + `/admin/claude-accounts/${a.id}/refresh`, {+ method: 'POST',+ signal: AbortSignal.timeout(30_000),+ headers: { Authorization: auth, 'Content-Type': 'application/json' },+ body: '{}',+ });+ const retryAfterHdr = r.headers.get('retry-after');+ lastStatus = r.status;+ lastRetryAfter = retryAfterHdr;+ if (r.status === 429 && attempt < MAX_RETRIES) {+ const delayMs = retryAfterHdr+ ? Math.min(15 * 60_000, Math.max(2_000, parseInt(retryAfterHdr, 10) * 1000 + 2000))+ : 30_000;+ log(` ${a.name}: refresh 429 (attempt ${attempt}/${MAX_RETRIES}) — waiting ${Math.round(delayMs / 1000)}s (Retry-After=${retryAfterHdr ?? 'n/a'})...`);+ await new Promise((r2) => setTimeout(r2, delayMs));+ continue;+ }+ const txt = await r.text();+ let body;+ try {+ body = JSON.parse(txt);+ } catch {+ body = txt;+ }+ const ok = r.status === 200 && (body?.success === true || body?.data?.success === true || body?.data?.accessToken);+ const err = body?.message || body?.error?.message || body?.error || (typeof body === 'string' ? body.slice(0, 120) : '');+ if (r.status === 200 || attempt === MAX_RETRIES) {+ return { ok, status: r.status, err, retryAfter: lastRetryAfter };+ }+ lastErr = err;+ await new Promise((r2) => setTimeout(r2, 5_000));+ }+ return { ok: false, status: lastStatus ?? 0, err: lastErr ?? 'max retries', retryAfter: lastRetryAfter };+}++async function setSchedulable(auth, a, value) {+ const r = await jfetch(`/admin/claude-accounts/${a.id}/toggle-schedulable`, {+ method: 'PUT',+ headers: { Authorization: auth, 'Content-Type': 'application/json' },+ body: JSON.stringify({ schedulable: value }),+ });+ return r.status === 200;+}++async function tick() {+ const nowMs = Date.now();+ const state = loadJsonState(STATE_PATH, log);+ const auth = await ensureAdmin();+ const accounts = await listClaudeAccounts(auth);++ if (STATUS) {+ console.log(`CRS token status @ ${BASE} — ${accounts.length} claude accounts`);+ for (const a of accounts) {+ const exp = tokenExpiryMs(a);+ const mins = exp ? Math.round((exp - nowMs) / 60_000) : null;+ const flag = state[a.id]?.needsReauth ? ' NEEDS-REAUTH' : '';+ const limited = authoritativeRateLimited(a, nowMs) ? ' AUTH-RATE-LIMITED' : '';+ console.log(+ ` ${a.name.padEnd(26)} sched=${String(a.schedulable !== false).padEnd(5)} exp_in=${mins === null ? '-' : mins + 'm'}${flag}${limited}`,+ );+ }+ return;+ }++ const targets = accounts.filter((a) => {+ if (hardHeld(a, nowMs)) return false;+ if (ONLY) return a.name === ONLY || a.id === ONLY;+ const exp = tokenExpiryMs(a);+ const expiringSoon = !exp || exp <= nowMs + REFRESH_WINDOW_MS;+ const sidelined = a.schedulable === false;+ const authLimited = authoritativeRateLimited(a, nowMs);+ return expiringSoon || sidelined || authLimited;+ });++ if (!targets.length) {+ log(`tick: 0 refresh needed (${accounts.length} accounts all fresh > ${REFRESH_WINDOW_MS / 60000}m)`);+ if (!DRY) saveJsonStateAtomic(STATE_PATH, state);+ return;+ }++ let refreshed = 0;+ let revived = 0;+ let dead = 0;+ for (const a of targets) {+ const exp = tokenExpiryMs(a);+ const mins = exp ? Math.round((exp - nowMs) / 60_000) : null;+ const authLimited = authoritativeRateLimited(a, nowMs);+ if (DRY) {+ log(+ `[dry] would ${authLimited ? 'sideline auth-rate-limited' : 'refresh'} ${a.name} (exp_in=${mins === null ? '-' : mins + 'm'}, sched=${a.schedulable !== false})`,+ );+ continue;+ }+ if (authLimited) {+ if (a.schedulable !== false) {+ const off = await setSchedulable(auth, a, false);+ log(`${a.name}: authoritative rate limit until ${a.rateLimitStatus?.rateLimitEndAt || a.rateLimitEndAt} — ${off ? 'SIDELINED' : 'sideline FAILED'}`);+ } else {+ log(`${a.name}: authoritative rate limit already sidelined`);+ }+ continue;+ }+ const res = await refreshAccount(auth, a);+ if (res.ok) {+ refreshed++;+ delete state[a.id];+ if (a.schedulable === false && externallyHeld(a.id, Date.now())) {+ log(`${a.name}: refreshed but leaving sidelined — held by another reconciler's cooldown (rate-limited)`);+ } else if (a.schedulable === false && !hardHeld(a, Date.now())) {+ const on = await setSchedulable(auth, a, true);+ if (on) {+ revived++;+ log(`${a.name}: refreshed + revived (schedulable=true)`);+ } else {+ log(`${a.name}: refreshed but re-enable failed`);+ }+ } else {+ log(`${a.name}: refreshed (exp_in was ${mins === null ? '-' : mins + 'm'})`);+ }+ } else {+ dead++;+ state[a.id] = {+ name: a.name,+ needsReauth: true,+ lastFailAt: nowMs,+ reason: `refresh HTTP ${res.status}: ${res.err}`.slice(0, 160),+ };+ // A schedulable account with a dead refresh_token AND an expired/near-expired+ // access token actively 401s live traffic. Sideline it to protect the pool; a+ // human/magic-link re-auth revives it, and the next tick re-enables on a+ // successful refresh. We only sideline confirmed-dead + expiring tokens, never+ // a transient failure on a still-fresh token.+ const expired = !exp || exp <= nowMs + REFRESH_WINDOW_MS;+ if (expired && a.schedulable !== false) {+ const off = await setSchedulable(auth, a, false);+ log(`${a.name}: refresh FAILED (HTTP ${res.status}: ${res.err}) — dead token, ${off ? 'SIDELINED' : 'sideline FAILED'}, needs full re-auth`);+ } else {+ log(`${a.name}: refresh FAILED (HTTP ${res.status}: ${res.err}) — needs full re-auth`);+ }+ }+ }++ const needsReauth = Object.values(state)+ .filter((s) => s?.needsReauth)+ .map((s) => s.name);+ log(+ `tick: refreshed=${refreshed} revived=${revived} dead=${dead}` ++ (needsReauth.length ? ` | needs-reauth: ${needsReauth.join(', ')}` : ''),+ );+ if (!DRY) saveJsonStateAtomic(STATE_PATH, state);+}++async function main() {+ if (!ENABLED && !STATUS) {+ log('crs.tokenRefreshEnabled is not true (and $CRS_TOKEN_REFRESH_ENABLED!=1) — skipping tick (opt-in required)');+ return;+ }+ const outcome = await withOwnStateLock(STATE_PATH, tick, log);+ if (outcome.skipped) log('skipped tick: previous run still holds the state lock');+}++main().catch((e) => {+ log(`ERROR: ${e.message}`);+ process.exit(1);+});diff --git a/claude-ops/scripts/account-rotation/crs-429-cooldown.mjs b/claude-ops/scripts/account-rotation/crs-429-cooldown.mjsnew file mode 100644index 00000000..0827d883--- /dev/null+++ b/claude-ops/scripts/account-rotation/crs-429-cooldown.mjs@@ -0,0 +1,327 @@+#!/usr/bin/env node+// crs-429-cooldown.mjs — per-account cooldown driven by real Anthropic API 429s.+//+// OPT-IN. Does nothing (exits 0) unless crs.cooldownEnabled is true (or+// $CRS_COOLDOWN_ENABLED=1) — installing this plugin never silently starts a new+// daemon. Enable it alongside crs-priority-daemon.mjs when you run a self-hosted+// claude-relay-service (CRS) pool and want a fast, real-signal-only cooldown watcher+// that runs on a tighter cadence than the priority daemon.+//+// Two signals, in priority order — both are things Anthropic (or CRS observing+// Anthropic) actually told us, never a speculative/utilization-based guess:+// 1. rateLimitStatus.isRateLimited = true on the account record (CRS derived+// this from a real upstream response on an actual inference call).+// 2. (optional) tail of today's CRS relay error log for 429 lines mentioning the+// account name — a faster lead signal than #1 while CRS's own record catches+// up. Reads from crs.logDir (or $CRS_LOG_DIR); a no-op on a fresh install+// since that directory won't exist yet (this reconciler never assumes a+// real CRS deployment's log layout).+//+// State is owned exclusively by THIS reconciler (own file, atomic write, single-+// flight lock via crs-reconciler-state.mjs) so that:+// - a process restart does not release a held account early;+// - crs-priority-daemon.mjs (which owns utilization-based scheduling) can run on+// its own cadence without clobbering this reconciler's cooldown record, and+// vice versa. See crs-reconciler-state.mjs for why that separation exists.+//+// Cron/timer cadence: run this more often than crs-priority-daemon.mjs (e.g. every+// 1 minute vs every 5) since it exists to react to a live 429 fast.+//+// CONFIG (config.json "crs" block; every key overridable by env — see+// config.example.json for the full annotated schema):+// cooldownEnabled default false — must be true (or $CRS_COOLDOWN_ENABLED=1)+// baseUrl, adminUser, adminPasswordEnv, containerName — shared with the other+// CRS reconcilers+// stateDir default: <plugin-data-dir>/account-rotation. This+// reconciler's own state file lives at+// <stateDir>/crs-429-state.json (never shared with any+// other reconciler's state file).+// manualDisableEnabled default true — set false to skip the manual-disable+// allow-list check entirely+// manualDisablePath default: <stateDir>/crs-manual-disable.json (optional+// allow-list of account ids/names to always leave alone)+// logDir default: <plugin-data-dir>/account-rotation/crs-logs.+// Shared with crs-401-refresher.mjs — this reconciler+// reads claude-relay-error-<date>.log from here for its+// optional lead signal. Unset -> that signal is disabled+// (rateLimitStatus alone is still a real Anthropic signal+// and is sufficient on its own).+// logTzOffset default: "+00:00" — offset appended to naive log+// timestamps before Date.parse; must match your CRS+// host's log timezone if it isn't UTC.+// errorLogWindowMs default: 300000 (5m) — how far back to scan the log+// errorLogCooldownMs default: 300000 (5m) — cooldown granted on a log hit+//+// CLI: (none)=one tick · --dry-run=log decisions, no writes · --status=print+// current held/live accounts and exit.++import { readFileSync, existsSync } from 'fs';+import { join, dirname } from 'path';+import { fileURLToPath } from 'url';+import { homedir } from 'os';+import { loadJsonState, saveJsonStateAtomic, withOwnStateLock } from './crs-reconciler-state.mjs';+import { crsBaseUrl, loadRotationConfig, resolveCrsAdminPassword } from './crs-pool-config.mjs';++const __dirname = dirname(fileURLToPath(import.meta.url));+const args = new Set(process.argv.slice(2));+const DRY = args.has('--dry-run') || process.env.CRS_DRY === '1';+const STATUS = args.has('--status');++const ts = () => new Date().toISOString().slice(11, 19);+const log = (m) => console.log(`${ts()} [crs-429] ${m}`);++const cfg = loadRotationConfig();+const C = cfg.crs || {};++function truthy(v) {+ return ['1', 'true', 'yes', 'on'].includes(String(v ?? '').trim().toLowerCase());+}++const ENABLED = truthy(process.env.CRS_COOLDOWN_ENABLED) || C.cooldownEnabled === true;++const BASE = crsBaseUrl(cfg);+const ADMIN_USER = process.env.CRS_ADMIN_USER || C.adminUser || 'cradmin';+const num = (v, d) => (v === undefined || v === null || v === '' || Number.isNaN(+v) ? d : +v);++function expandHome(p) {+ return p ? p.replace(/^~(?=$|\/)/, homedir()) : p;+}++const PLUGIN_DATA_DIR =+ process.env.CLAUDE_PLUGIN_DATA_DIR || join(homedir(), '.claude', 'plugins', 'data', 'ops-ops-marketplace');+const DEFAULT_STATE_DIR = join(PLUGIN_DATA_DIR, 'account-rotation');+const STATE_DIR = expandHome(process.env.CRS_STATE_DIR || C.stateDir) || DEFAULT_STATE_DIR;++const STATE_PATH = expandHome(process.env.CRS_COOLDOWN_STATE_PATH) || join(STATE_DIR, 'crs-429-state.json');+const MANUAL_DISABLE_ENABLED = C.manualDisableEnabled !== false;+const MANUAL_DISABLE_PATH =+ expandHome(process.env.CRS_MANUAL_DISABLE_PATH || C.manualDisablePath) || join(STATE_DIR, 'crs-manual-disable.json');+// Unset by default: the error-log lead signal is opt-in extra sensitivity, not a+// required capability. rateLimitStatus alone (signal #1) is a real Anthropic+// signal and is sufficient on its own. Shared logDir/logTzOffset config with+// crs-401-refresher.mjs.+// Defaults to a real (likely-absent-on-a-fresh-install) path rather than a null+// "disabled" sentinel: scanErrorLog() below no-ops safely via existsSync() when+// the directory/file isn't there, so this stays a safe no-op until you actually+// point your CRS deployment's error-log directory at it.+const DEFAULT_LOG_DIR = join(DEFAULT_STATE_DIR, 'crs-logs');+const ERROR_LOG_DIR = expandHome(process.env.CRS_LOG_DIR || C.logDir) || DEFAULT_LOG_DIR;+const ERROR_LOG_WINDOW_MS = num(process.env.CRS_ERROR_LOG_WINDOW_MS ?? C.errorLogWindowMs, 5 * 60_000);+const ERROR_LOG_COOLDOWN_MS = num(process.env.CRS_ERROR_LOG_COOLDOWN_MS ?? C.errorLogCooldownMs, 5 * 60_000);+const ERROR_LOG_TZ = process.env.CRS_LOG_TZ_OFFSET || C.logTzOffset || '+00:00';++const jfetch = async (path, opts = {}) => {+ const r = await fetch(BASE + path, opts);+ const txt = await r.text();+ let body;+ try {+ body = JSON.parse(txt);+ } catch {+ body = txt;+ }+ return { status: r.status, body };+};++async function login() {+ const password = resolveCrsAdminPassword(cfg, { adminUser: ADMIN_USER });+ if (!password) throw new Error('CRS admin password unavailable (set $CRS_ADMIN_PASSWORD or see crs-pool-config.mjs)');+ const r = await jfetch('/web/auth/login', {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ username: ADMIN_USER, password }),+ });+ if (r.status !== 200 || !r.body?.token) throw new Error('login failed HTTP ' + r.status);+ return 'Bearer ' + r.body.token;+}++async function getAccounts(auth) {+ const r = await jfetch('/admin/claude-accounts', { headers: { Authorization: auth } });+ if (r.status !== 200) throw new Error('GET accounts failed HTTP ' + r.status);+ const list = Array.isArray(r.body) ? r.body : r.body.data || r.body.accounts || [];+ return list.filter((a) => a.platform === 'claude' && a.isActive !== false);+}++const toggle = async (auth, id, schedulable, metadata = {}) =>+ jfetch('/admin/claude-accounts/' + id + '/toggle-schedulable', {+ method: 'PUT',+ headers: { Authorization: auth, 'Content-Type': 'application/json' },+ body: JSON.stringify({ schedulable, ...metadata }),+ });++const parseTs = (s) => {+ if (!s) return null;+ const t = Date.parse(s);+ return Number.isFinite(t) ? t : null;+};++function loadManualDisable() {+ if (!MANUAL_DISABLE_ENABLED) return new Set();+ try {+ if (!existsSync(MANUAL_DISABLE_PATH)) return new Set();+ const o = JSON.parse(readFileSync(MANUAL_DISABLE_PATH, 'utf8'));+ return new Set(Array.isArray(o) ? o : Object.keys(o));+ } catch {+ return new Set();+ }+}++// === optional signal #2: tail today's CRS relay error log for 429 lines =====+function scanErrorLog(nowMs) {+ if (!ERROR_LOG_DIR) return new Map();+ const today = new Date(nowMs).toISOString().slice(0, 10);+ const logPath = join(ERROR_LOG_DIR, `claude-relay-error-${today}.log`);+ if (!existsSync(logPath)) return new Map();+ let lines;+ try {+ lines = readFileSync(logPath, 'utf8').split('\n').filter(Boolean);+ } catch {+ return new Map();+ }+ const recent = new Map(); // account name -> latest hit ts (ms)+ const windowStart = nowMs - ERROR_LOG_WINDOW_MS;+ for (const line of lines) {+ try {+ const e = JSON.parse(line);+ const m = e.msg || '';+ if (!m.includes('error status: 429')) continue;+ const rawTs = String(e.ts || '').replace(' ', 'T');+ const t = Date.parse(ERROR_LOG_TZ && !/[Zz]|[+-]\d\d:?\d\d$/.test(rawTs) ? rawTs + ERROR_LOG_TZ : rawTs);+ if (!Number.isFinite(t) || t < windowStart) continue;+ const acc = m.match(/Account:\s*([^\s"]+)/);+ if (!acc) continue;+ const cur = recent.get(acc[1]);+ if (!cur || t > cur) recent.set(acc[1], t);+ } catch {}+ }+ return recent;+}++async function tick() {+ const auth = await login();+ const accts = await getAccounts(auth);+ const state = loadJsonState(STATE_PATH, log);+ const nowMs = Date.now();+ const errorLogHits = scanErrorLog(nowMs);+ const manualDisable = loadManualDisable();++ if (STATUS) {+ const held = Object.entries(state).filter(([, s]) => s.cooldownUntil > nowMs);+ console.log(`CRS 429-cooldown @ ${BASE} — ${accts.length} claude accounts, ${held.length} held`);+ for (const [id, s] of held) {+ console.log(` ${s.name || id}: held until ${new Date(s.cooldownUntil).toISOString()} (${s.reason})`);+ }+ return;+ }++ const changes = [];+ for (const a of accts) {+ if (manualDisable.has(a.id) || manualDisable.has(a.name)) continue;+ const cur = a.schedulable !== false;+ const rl = a.rateLimitStatus?.isRateLimited === true;+ const rlMin = Math.max(a.rateLimitStatus?.minutesRemaining || 0, 0);+ const rlEnd = parseTs(a.rateLimitStatus?.rateLimitEndAt);+ const persistedRedisHoldEnd = parseTs(a.rateLimitEndAt || a.rateLimitStatus?.rateLimitEndAt);+ const rateLimitReason = a.rateLimitReason || a.rateLimitStatus?.reason || null;+ const transient429 = rateLimitReason === 'transient_default_cooldown';+ const errLogHit = errorLogHits.get(a.name);++ // Trust only real upstream signals here (never claudeUsage.*.utilization —+ // that cache is fine for crs-priority-daemon.mjs's global health checks but+ // lags live state by minutes and must not drive a per-account hold).+ let cooldownUntil = 0;+ let reason = null;+ if (rl && !transient429) {+ const t = rlEnd && rlEnd > nowMs ? rlEnd : nowMs + Math.max(rlMin, 1) * 60_000;+ cooldownUntil = Math.max(cooldownUntil, t);+ reason = `api-rl ${Math.max(rlMin, 1)}m remaining${rateLimitReason ? ` (${rateLimitReason})` : ''}`;+ }+ if (errLogHit && rl && rateLimitReason && !transient429) {+ const t = errLogHit + ERROR_LOG_COOLDOWN_MS;+ if (t > cooldownUntil) {+ cooldownUntil = t;+ reason = `429 in error log @ ${new Date(errLogHit).toISOString()}`;+ }+ }++ const prevHeld = state[a.id]?.cooldownUntil || 0;++ if (cooldownUntil > nowMs) {+ state[a.id] = { name: a.name, cooldownUntil, reason };+ const holdMissingInRedis = !persistedRedisHoldEnd || persistedRedisHoldEnd < cooldownUntil - 1000;+ if (cur || holdMissingInRedis) {+ if (DRY) {+ changes.push({ name: a.name, action: cur ? '[dry]OFF' : '[dry]HOLD', reason, http: '-' });+ } else {+ const r = await toggle(auth, a.id, false, {+ rateLimitEndAt: new Date(cooldownUntil).toISOString(),+ rateLimitReason: reason || 'api-rl',+ });+ changes.push({+ name: a.name,+ action: cur ? 'OFF' : 'HOLD',+ reason: `cooldown until ${new Date(cooldownUntil).toISOString().slice(11, 19)} (${reason})`,+ http: r.status,+ });+ }+ }+ } else if (prevHeld > nowMs) {+ // Persisted hold still wins — releasing before the recorded upstream+ // reset just re-hammers the same account.+ state[a.id] = { name: a.name, cooldownUntil: prevHeld, reason: state[a.id]?.reason || 'persisted-api-rl-bench' };+ const holdMissingInRedis = !persistedRedisHoldEnd || persistedRedisHoldEnd < prevHeld - 1000;+ if ((cur || holdMissingInRedis) && !DRY) {+ const r = await toggle(auth, a.id, false, {+ rateLimitEndAt: new Date(prevHeld).toISOString(),+ rateLimitReason: state[a.id]?.reason || 'persisted-api-rl-bench',+ });+ changes.push({+ name: a.name,+ action: cur ? 'OFF' : 'HOLD',+ reason: `persisted cooldown until ${new Date(prevHeld).toISOString().slice(11, 19)}`,+ http: r.status,+ });+ } else if ((cur || holdMissingInRedis) && DRY) {+ changes.push({ name: a.name, action: '[dry]HOLD', reason: 'persisted cooldown', http: '-' });+ }+ } else if (prevHeld && prevHeld <= nowMs) {+ delete state[a.id];+ if (!cur) {+ if (DRY) {+ changes.push({ name: a.name, action: '[dry]ON', reason: 'cooldown expired', http: '-' });+ } else {+ const r = await toggle(auth, a.id, true);+ changes.push({ name: a.name, action: 'ON', reason: 'cooldown expired', http: r.status });+ }+ }+ }+ }++ if (!DRY) saveJsonStateAtomic(STATE_PATH, state);+ const held = Object.values(state).filter((s) => s.cooldownUntil > nowMs);+ const live = accts.length - held.length;+ log(+ `tick: ${changes.length} change(s). live=${live}/${accts.length} | held=[${held+ .map((h) => `${h.name}@${new Date(h.cooldownUntil).toISOString().slice(11, 19)}`)+ .join(',')}]`,+ );+ for (const c of changes) log(` ${c.name}: ${c.action} (${c.reason || ''}) [HTTP ${c.http}]`);+}++async function main() {+ if (!ENABLED && !STATUS) {+ log('crs.cooldownEnabled is not true (and $CRS_COOLDOWN_ENABLED!=1) — skipping tick (opt-in required)');+ return;+ }+ const outcome = await withOwnStateLock(STATE_PATH, tick, log);+ if (outcome.skipped) log('skipped tick: previous run still holds the state lock');+}++main().catch((e) => {+ const message = e?.message || String(e);+ if (message === 'fetch failed' || message.includes('ECONNREFUSED') || message.includes('ECONNRESET') || message.includes('ETIMEDOUT')) {+ log('WARN transient CRS fetch failure; skipping this tick');+ process.exit(0);+ }+ log('ERROR: ' + message);+ process.exit(1);+});diff --git a/claude-ops/scripts/account-rotation/crs-pool-config.mjs b/claude-ops/scripts/account-rotation/crs-pool-config.mjsindex f386f506..6d02db0b 100644--- a/claude-ops/scripts/account-rotation/crs-pool-config.mjs+++ b/claude-ops/scripts/account-rotation/crs-pool-config.mjs@@ -7,6 +7,7 @@*/import { readFileSync, existsSync } from 'fs';+import { execFileSync } from 'child_process';import { join, dirname } from 'path';import { fileURLToPath } from 'url';import { homedir } from 'os';@@ -100,6 +101,66 @@ export function crsPolicy(config = loadRotationConfig()) {return 'conservative';}+/** Candidate paths for the shared ops credential-store CLI (never hardcoded to one account/box). */+export function credentialStoreCandidates() {+ const pluginRoot = process.env.CLAUDE_PLUGIN_ROOT || join(__dirname, '..', '..');+ return [+ process.env.OPS_CREDENTIAL_STORE,+ join(homedir(), '.claude', 'plugins', 'cache', 'ops-marketplace', 'ops', 'current', 'lib', 'credential-store.sh'),+ join(homedir(), '.claude', 'plugins', 'marketplaces', 'ops-marketplace', 'claude-ops', 'lib', 'credential-store.sh'),+ join(pluginRoot, 'lib', 'credential-store.sh'),+ ].filter(Boolean);+}++export function resolveCredentialStorePath() {+ const candidates = credentialStoreCandidates();+ for (const p of candidates) {+ if (existsSync(p)) return p;+ }+ return candidates[candidates.length - 1];+}++/**+ * Resolve the CRS admin password for HTTP basic auth against a self-hosted+ * claude-relay-service pool. Resolution order (first hit wins), none of it+ * ever hardcoded to one deployment:+ * 1. $CRS_ADMIN_PASSWORD+ * 2. the env var named by config.crs.adminPasswordEnv (or $CRS_ADMIN_PASSWORD_ENV)+ * 3. `docker inspect <container>` — reads ADMIN_PASSWORD out of a co-located+ * Docker Compose CRS deploy's own container env+ * 4. the shared ops credential-store CLI, key `CRS-Admin-<adminUser>`+ * Returns null (never throws) if nothing resolves — callers decide whether that+ * means "skip this tick" or "hard error".+ */+export function resolveCrsAdminPassword(config = loadRotationConfig(), opts = {}) {+ if (process.env.CRS_ADMIN_PASSWORD) return process.env.CRS_ADMIN_PASSWORD;+ const envName = config.crs?.adminPasswordEnv || process.env.CRS_ADMIN_PASSWORD_ENV;+ if (envName && process.env[envName]) return process.env[envName];++ const adminUser = opts.adminUser || process.env.CRS_ADMIN_USER || config.crs?.adminUser || 'cradmin';+ const container = opts.container || process.env.CRS_CONTAINER || config.crs?.containerName || 'crs-claude-relay-1';+ const execOpts = { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] };++ try {+ const out = execFileSync(+ 'docker',+ ['inspect', container, '--format', '{{range .Config.Env}}{{println .}}{{end}}'],+ execOpts,+ );+ const m = out.match(/^ADMIN_PASSWORD=(.+)$/m);+ if (m?.[1]?.trim()) return m[1].trim();+ } catch {}++ try {+ const cred = resolveCredentialStorePath();+ const acct = process.env.CLAUDE_ROTATOR_KEYCHAIN_ACCOUNT || process.env.USER || 'claude-ops';+ const pw = execFileSync('bash', [cred, 'get', `CRS-Admin-${adminUser}`, acct], execOpts).trim();+ if (pw) return pw;+ } catch {}++ return null;+}+export function vaultLookupKeysForEmail(email, accounts = []) {const keys = new Set();if (email) keys.add(email);diff --git a/claude-ops/scripts/account-rotation/crs-reconciler-state.mjs b/claude-ops/scripts/account-rotation/crs-reconciler-state.mjsnew file mode 100644index 00000000..1ed92480--- /dev/null+++ b/claude-ops/scripts/account-rotation/crs-reconciler-state.mjs@@ -0,0 +1,107 @@+#!/usr/bin/env node+/**+ * crs-reconciler-state.mjs — shared own-state-file helper for CRS pool reconcilers.+ *+ * Each reconciler that watches a claude-relay-service (CRS) account pool (cooldown+ * tracking, token-refresh tracking, priority scoring, ...) MUST persist to its OWN+ * state file — never a file shared with another reconciler. Sharing one file was the+ * root cause of a documented production incident: reconciler A's non-atomic write+ * clobbered reconciler B's cooldown data mid-tick, silently erasing real rate-limit+ * holds and triggering a pool-wide re-enable/429 oscillation.+ *+ * This module gives every reconciler two small, dependency-free guarantees:+ * 1. Atomic write — write to a temp file in the same directory, then rename() over+ * the real path. rename() is atomic on the same filesystem, so a reader (or a+ * concurrent tick) never observes a half-written / truncated JSON file.+ * 2. Single-flight lock — a short-lived advisory lock (mkdir-based; no native deps,+ * works the same on Linux and macOS) around the read-modify-write section, so two+ * overlapping ticks of the SAME reconciler (a slow tick plus a cron/timer overlap)+ * can't interleave writes. This is a lightweight stand-in for flock(1), not a+ * distributed lock — sufficient for a single-host cron/timer cadence. A stale lock+ * (older than LOCK_STALE_MS, e.g. left behind by a crashed process) is reclaimed+ * rather than wedging the reconciler forever.+ *+ * Nothing here talks to CRS or knows about account schedules — it is pure state I/O so+ * every reconciler can `import` the same tiny, well-tested primitive instead of each+ * re-implementing tmp+rename by hand.+ */++import { readFileSync, writeFileSync, existsSync, renameSync, mkdirSync, rmdirSync, statSync } from 'fs';+import { dirname } from 'path';++const LOCK_STALE_MS = 30_000;++export function loadJsonState(path, log = console.error) {+ if (!existsSync(path)) return {};+ try {+ return JSON.parse(readFileSync(path, 'utf8'));+ } catch (e) {+ log(`[crs-reconciler-state] WARN corrupt state at ${path}, resetting: ${e.message}`);+ return {};+ }+}++/** Atomic write: tmp file in the same directory + rename(). */+export function saveJsonStateAtomic(path, data) {+ mkdirSync(dirname(path), { recursive: true });+ const tmp = `${path}.tmp.${process.pid}`;+ writeFileSync(tmp, JSON.stringify(data, null, 2));+ renameSync(tmp, path);+}++/**+ * Run `fn` (sync or async) holding a single-flight advisory lock scoped to `path`.+ * If the lock is already held by a live tick, this returns { skipped: true } instead+ * of blocking — the caller should log and exit 0 (cron will just try again next tick).+ *+ * @returns {Promise<{ skipped: boolean, result?: any }>}+ */+export async function withOwnStateLock(path, fn, log = console.error) {+ const lockDir = `${path}.lock`;+ mkdirSync(dirname(path), { recursive: true });+ const acquired = tryAcquireLock(lockDir, log);+ if (!acquired) return { skipped: true };+ try {+ const result = await fn();+ return { skipped: false, result };+ } finally {+ try {+ rmdirSync(lockDir);+ } catch {}+ }+}++function tryAcquireLock(lockDir, log) {+ try {+ mkdirSync(lockDir);+ return true;+ } catch (e) {+ if (e.code !== 'EEXIST') throw e;+ }+ // Lock dir already exists — check whether it's stale (owner crashed mid-tick).+ let ageMs;+ try {+ ageMs = Date.now() - statSync(lockDir).mtimeMs;+ } catch {+ // Lock dir vanished between the mkdir failure and this stat (owner just+ // released it) — try once more to acquire it fresh.+ try {+ mkdirSync(lockDir);+ return true;+ } catch {+ return false;+ }+ }+ if (ageMs < LOCK_STALE_MS) {+ log(`[crs-reconciler-state] lock held (age ${Math.round(ageMs / 1000)}s) at ${lockDir} — skipping this tick`);+ return false;+ }+ try {+ rmdirSync(lockDir);+ mkdirSync(lockDir);+ log(`[crs-reconciler-state] reclaimed stale lock (age ${Math.round(ageMs / 1000)}s) at ${lockDir}`);+ return true;+ } catch {+ return false;+ }+}