Business operating system for Claude Code — 57 skills, 21 agents, smart daemon. Unified inbox (WhatsApp/Email/Slack/Telegram), autonomous PR merge, full-AWS monitoring, revenue (Stripe+RevenueCat), e-commerce (Shopify), marketing (Klaviyo/Meta/GA4), voice (Bland/ElevenLabs), APM (Datadog/NewRelic/OTEL), YOLO mode.
Shell agent-teamsai-agentai-automationanthropicautonomous-agentawsbusiness-operationsclaudeclaude-codeclaude-code-plugindevopsecommerceinbox-zeroklaviyomarketingmcpmorning-briefingpr-automationshopifyterminal
fix(bedrock-watchdog): detect runtime-injected + network-only Bedrock leaks (#592)
Aurora Capitalc124e24423bd
diff --git a/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs b/claude-ops/scripts/account-rotation/bedrock-watchdog.mjsindex 864dfcd3..2aa5725e 100644--- a/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs+++ b/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs@@ -21,9 +21,10 @@ import { pickAccountForSession, recordSessionLease } from './session-router.mjs'const PROC = '/proc';-// Infra/transient session arg-markers that inherit the Bedrock env but do NOT-// run real inference — never worth a force-swap.-const SKIP_ARG_MARKERS = ['--bg-pty-host', '--bg-spare'];+// Transient processes that never run billable session inference.+// NOTE: `--bg-spare`/`--bg-pty-host` are intentionally NOT blanket-skipped here+// (a claimed spare keeps its `--bg-spare` marker forever); the scan skips them+// only when no Bedrock signal is present. See scanBedrockSessions.const SKIP_CMD_SUBSTRINGS = ['auth login']; // `claude auth login` etc./** Read /proc/<pid>/environ into a {KEY:VALUE} map. Returns null if unreadable. */@@ -53,10 +54,68 @@ function readProcCmdline(pid) {}/**- * Scan every ec2-user `claude` process whose /proc environ has- * CLAUDE_CODE_USE_BEDROCK=1, skipping infra (--bg-pty-host, --bg-spare) and- * transient (auth login) processes.- * @returns {Array<{pid:number, sessionId:string|null, cmdline:string}>}+ * Build a one-shot parent→children map of every /proc pid. Used to probe a+ * node session's child subshells for the RUNTIME Bedrock signal (see below).+ * O(P) — built once per sweep, not per session.+ */+function buildChildMap() {+ const kids = new Map(); // ppid -> [pid,...]+ let pids = [];+ try {+ pids = readdirSync(PROC).filter((f) => /^\d+$/.test(f));+ } catch {+ return kids;+ }+ for (const p of pids) {+ try {+ const stat = readFileSync(`${PROC}/${p}/stat`, 'utf8');+ // Format: `pid (comm) state ppid ...`; comm may contain spaces/parens,+ // so anchor on the LAST ')' then take field[1] (ppid) after state.+ const rp = stat.lastIndexOf(')');+ if (rp === -1) continue;+ const fields = stat.slice(rp + 2).split(' ');+ const ppid = Number(fields[1]);+ if (!Number.isFinite(ppid)) continue;+ if (!kids.has(ppid)) kids.set(ppid, []);+ kids.get(ppid).push(Number(p));+ } catch {}+ }+ return kids;+}++/**+ * RUNTIME Bedrock detection. /proc/<pid>/environ is the EXEC-TIME snapshot only;+ * a node session that read CLAUDE_CODE_USE_BEDROCK=1 from settings.json at+ * startup (a past fallback window) carries it in its live process.env but NOT in+ * /proc/environ — invisible to the exec-env scan. That runtime value IS what+ * inference uses (the real metered bill) AND is inherited by every child+ * subshell the session spawns. So we probe the session's direct children: if any+ * child's /proc/environ has USE_BEDROCK=1, the parent's runtime env has it too.+ * Returns the first leaking child pid, or null.+ */+function runtimeBedrockChild(pid, childMap) {+ for (const kid of childMap.get(pid) || []) {+ const env = readProcEnviron(kid);+ if (env && env.CLAUDE_CODE_USE_BEDROCK === '1') return kid;+ }+ return null;+}++/**+ * Scan every ec2-user `claude` process on Bedrock — by EXEC env+ * (CLAUDE_CODE_USE_BEDROCK=1 in /proc/environ) OR by RUNTIME env (a child+ * subshell carries it; see runtimeBedrockChild). Transient `auth login`+ * processes are always skipped.+ *+ * The `--bg-spare` / `--bg-pty-host` arg-markers are NOT a blanket skip: a+ * claimed spare KEEPS `--bg-spare` in its cmdline even after it becomes a live,+ * inference-running session (the marker is never rewritten), so blanket-skipping+ * it permanently hides a real leaker (root cause of the fcfcc869 opus leak). We+ * only skip those markers when there is NO Bedrock signal at all — i.e. a+ * passive spare that merely inherited the env but isn't running inference. The+ * moment a runtime (or exec) Bedrock signal is present, it is a real leak and+ * gets reported regardless of the spare/pty marker.+ * @returns {Array<{pid:number, sessionId:string|null, cmdline:string, via:'exec'|'runtime', childPid:number|null}>}*/export function scanBedrockSessions() {const out = [];@@ -66,78 +125,185 @@ export function scanBedrockSessions() {} catch {return out; // /proc unreadable (non-Linux / sandbox) — nothing to do.}+ const childMap = buildChildMap();for (const pidStr of pids) {const pid = Number(pidStr);if (pid === process.pid || pid === process.ppid) continue;const cmdline = readProcCmdline(pid);- if (!cmdline || !/\bclaude\b/.test(cmdline)) continue;- if (SKIP_ARG_MARKERS.some((m) => cmdline.includes(m))) continue;+ // A swap CANDIDATE is a Claude SESSION node process (daemon-hosted bg+ // sessions run `.../share/claude/versions/<ver> …`, foreground runs the+ // claude bin directly). Shell/bun children (gstack browse, `bash -c`, mac+ // helpers) inherit the Bedrock env but are NOT swappable sessions — they+ // serve only as the runtime-env PROBE (runtimeBedrockChild), never as+ // candidates. Matching the broad `.claude/` path substring here emitted+ // those children as bogus, un-respawnable hits.+ if (!cmdline) continue;+ const isSessionProc = /share\/claude\/versions\//.test(cmdline) || /(^|\/)claude(\s|$)/.test(cmdline);+ if (!isSessionProc) continue;if (SKIP_CMD_SUBSTRINGS.some((m) => cmdline.includes(m))) continue;const env = readProcEnviron(pid);- if (!env) continue;- if (env.CLAUDE_CODE_USE_BEDROCK !== '1') continue;+ if (!env) continue; // unreadable environ → can't resolve a session id anyway+ const execBedrock = env.CLAUDE_CODE_USE_BEDROCK === '1';+ const childPid = execBedrock ? null : runtimeBedrockChild(pid, childMap);+ const hasBedrock = execBedrock || childPid !== null;++ // Passive inheritor (spare/pty-host with NO actual Bedrock inference) — skip.+ // A real leaker (runtime or exec signal) is reported even if it's a spare.+ if (!hasBedrock) continue;out.push({pid,sessionId: env.CLAUDE_CODE_SESSION_ID || null,cmdline,+ via: execBedrock ? 'exec' : 'runtime',+ childPid,});}return out;}+// Regions where `us.anthropic.*` inference profiles + nearby Bedrock endpoints+// land. Resolved IPs are unioned; DNS rotates so we re-resolve on a short TTL.+const BEDROCK_REGIONS = ['us-east-1', 'us-west-2', 'eu-central-1', 'eu-west-1', 'eu-west-2'];+let _bedrockIpCache = { ips: new Set(), at: 0 };+const BEDROCK_IP_TTL_MS = 5 * 60_000;+/**- * Best-effort: which of the given PIDs have a LIVE established connection to a- * Bedrock-runtime endpoint (port 443 to an AWS range, reverse-resolving to- * bedrock-runtime). Used only to CORROBORATE env detection and to VALIDATE that- * traffic stopped after a swap — NEVER as the sole trigger.- *- * Sandbox-safe: `ss` can be socket-blind in a sandboxed shell on FRA (see memory- * sandboxed-ss-port-checks-blind). On any failure / empty output this returns- * { available:false, pids:new Set() } so callers treat it as "unknown" and never+ * Resolve the current bedrock-runtime endpoint IPs across BEDROCK_REGIONS.+ * Cached for BEDROCK_IP_TTL_MS (DNS rotates the A records). On total failure+ * returns whatever is cached (possibly empty) — callers degrade to no-op, never* false-alarm.+ * @returns {Set<string>}+ */+function resolveBedrockIps() {+ const now = Date.now();+ if (_bedrockIpCache.ips.size && now - _bedrockIpCache.at < BEDROCK_IP_TTL_MS) {+ return _bedrockIpCache.ips;+ }+ const ips = new Set();+ for (const r of BEDROCK_REGIONS) {+ try {+ const out = execFileSync('getent', ['ahosts', `bedrock-runtime.${r}.amazonaws.com`], {+ encoding: 'utf8',+ timeout: 3000,+ stdio: ['ignore', 'pipe', 'ignore'],+ });+ for (const line of out.split('\n')) {+ const ip = line.trim().split(/\s+/)[0];+ if (/^\d+\.\d+\.\d+\.\d+$/.test(ip)) ips.add(ip);+ }+ } catch {}+ }+ if (ips.size) _bedrockIpCache = { ips, at: now };+ return ips.size ? ips : _bedrockIpCache.ips;+}++/**+ * Which PIDs hold a LIVE established TCP connection to a bedrock-runtime IP.+ * THIS IS A VALID STANDALONE TRIGGER (2026-06-15): an established TLS socket to+ * a bedrock-runtime endpoint is ground truth that the owning process is billing+ * Bedrock right now — there is no false-positive (you cannot hold that socket+ * without using Bedrock). It catches the case the env/child probes miss: a busy+ * inference session with runtime CLAUDE_CODE_USE_BEDROCK=1 but NO live child+ * subshell at scan time (root cause of the 7f1abe98 leak the env-scan reported+ * clean — proven by a live `ss` network trace). The old hostname-regex matcher+ * was dead: `ss -tn` prints numeric peers, so /bedrock-runtime/ never matched.+ *+ * Matching is by peer IP against the resolved bedrock-runtime IP set. `ss` can+ * be socket-blind in a sandboxed shell (memory sandboxed-ss-port-checks-blind);+ * on any failure / empty output / unresolvable IPs this returns+ * { available:false, pids:new Set() } so callers treat it as "unknown" — a miss+ * never blocks anything, it just degrades to the env/child path.*- * @param {number[]} [pids] optional filter; when omitted, all matching are returned+ * @param {number[]} [pids] optional filter; when omitted, all matching pids returned* @returns {{available:boolean, pids:Set<number>}}*/export function scanBedrockNetwork(pids) {const want = pids && pids.length ? new Set(pids.map(Number)) : null;+ const bedrockIps = resolveBedrockIps();+ if (!bedrockIps.size) return { available: false, pids: new Set() }; // can't resolve → unknown+ // `ss -tnp` only attributes pids for the caller's OWN sockets, and can be+ // socket-blind under sandboxing. Prefer `sudo -n ss` (passwordless sudo is+ // provisioned for ec2-user) for full pid visibility across every session;+ // fall back to plain `ss` (works unprivileged for the daemon's own user when+ // not sandboxed). Either way, no-pid output → available:false (degrade).let raw = '';- try {- // -t tcp, -n numeric, -p processes, state established.- raw = execFileSync('ss', ['-tnp', 'state', 'established'], {- encoding: 'utf8',- timeout: 4000,- stdio: ['ignore', 'pipe', 'ignore'],- });- } catch {- return { available: false, pids: new Set() };- }- if (!raw || !raw.trim()) {- // ss ran but returned nothing — could be genuinely no connections OR- // socket-blind. Either way we can't assert traffic, so signal "unknown".- return { available: false, pids: new Set() };+ for (const argv of [+ ['sudo', ['-n', 'ss', '-tnp', 'state', 'established']],+ ['ss', ['-tnp', 'state', 'established']],+ ]) {+ try {+ raw = execFileSync(argv[0], argv[1], {+ encoding: 'utf8',+ timeout: 4000,+ stdio: ['ignore', 'pipe', 'ignore'],+ });+ if (raw && raw.trim()) break;+ } catch {+ raw = '';+ }}+ if (!raw || !raw.trim()) return { available: false, pids: new Set() };- // Bedrock-runtime resolves into AWS ranges; we can't reverse-resolve every IP- // cheaply, so match on the well-known bedrock-runtime hostname when present in- // the line, else fall back to :443 peers owned by a claude pid. This is- // corroboration only — a miss never blocks a swap.const found = new Set();+ let sawAnyPid = false;for (const line of raw.split('\n')) {if (!line.includes(':443')) continue;- const isBedrock = /bedrock-runtime/i.test(line);+ // Peer address is the 2nd host:port column. Match the LAST IP:443 on the line+ // (local addr could also be :443 in rare cases; peer is rightmost).+ const peers = [...line.matchAll(/(\d+\.\d+\.\d+\.\d+):443\b/g)];+ if (!peers.length) continue;+ const peerIp = peers[peers.length - 1][1];+ if (!bedrockIps.has(peerIp)) continue;const m = line.match(/pid=(\d+)/);- if (!m) continue;- const pid = Number(m[1]);- if (want && !want.has(pid)) continue;- if (isBedrock) found.add(pid);- }- // If we couldn't positively identify any bedrock-runtime peer by name, report- // unavailable rather than guessing from bare :443 (avoids false-alarm).- if (found.size === 0) return { available: false, pids: new Set() };- return { available: true, pids: found };+ if (m) {+ sawAnyPid = true;+ const pid = Number(m[1]);+ if (want && !want.has(pid)) continue;+ found.add(pid);+ }+ }+ // If we matched bedrock peers but `ss` showed no pid= (no CAP_NET_ADMIN /+ // socket-blind), we can't attribute them — report unavailable rather than+ // returning an empty positive that looks like "clean".+ if (found.size === 0 && !sawAnyPid) return { available: false, pids: new Set() };+ return { available: found.size > 0, pids: found };+}++/**+ * Walk a PID up its parent chain to the nearest live bg-session PID. Lets a+ * bedrock connection owned by a child `claude`/shell process be attributed to+ * the session that spawned it. Returns the bg-session record or null.+ */+function ancestorBgSession(pid, byPid, ppidMap) {+ let cur = pid;+ for (let hops = 0; cur && cur > 1 && hops < 30; hops++) {+ if (byPid.has(cur)) return byPid.get(cur);+ cur = ppidMap.get(cur) || 0;+ }+ return null;+}++/** Build a child→parent PID map from /proc (one pass). */+function buildPpidMap() {+ const m = new Map();+ let pids = [];+ try {+ pids = readdirSync(PROC).filter((f) => /^\d+$/.test(f));+ } catch {+ return m;+ }+ for (const p of pids) {+ try {+ const stat = readFileSync(`${PROC}/${p}/stat`, 'utf8');+ const rp = stat.lastIndexOf(')');+ if (rp === -1) continue;+ const ppid = Number(stat.slice(rp + 2).split(' ')[1]);+ if (Number.isFinite(ppid)) m.set(Number(p), ppid);+ } catch {}+ }+ return m;}/**@@ -158,13 +324,43 @@ function hasHotSwapHold(pid) {* @returns {{swapped:number, scanned:number, swappedPids:number[]}}*/export function sweepBedrockSessions(config, state, log) {- const sessions = scanBedrockSessions();- if (sessions.length === 0) return { swapped: 0, scanned: 0, swappedPids: [] };+ // (a) exec-env + runtime-child detection over claude session node procs.+ const envSessions = scanBedrockSessions();// Map PID → live bg-session record (id needed for `claude respawn <id>`).const live = listLiveBgSessions();const byPid = new Map(live.map((s) => [Number(s.pid), s]));+ // (b) NETWORK detection — a valid STANDALONE trigger. An established TCP+ // socket to a bedrock-runtime IP is ground truth that the owning process is+ // billing Bedrock now. Catches busy inference sessions whose runtime env has+ // CLAUDE_CODE_USE_BEDROCK=1 but which have NO live child subshell at scan time+ // (the env/child probe reports them clean — proven by the 7f1abe98 leak the+ // env-scan missed but a live `ss` trace caught). The connection may be owned by+ // a transient child `claude`/shell; attribute it to its ancestor bg-session.+ const netCandidates = [];+ try {+ const net = scanBedrockNetwork();+ if (net.available && net.pids.size) {+ const ppidMap = buildPpidMap();+ const seen = new Set();+ for (const connPid of net.pids) {+ const bg = ancestorBgSession(connPid, byPid, ppidMap);+ if (bg && !seen.has(Number(bg.pid))) {+ seen.add(Number(bg.pid));+ netCandidates.push({ pid: Number(bg.pid), sessionId: null, via: 'network', childPid: connPid });+ }+ }+ }+ } catch {}++ // Union env/child + network candidates, keyed by bg-session node pid.+ const candidatesByPid = new Map();+ for (const s of envSessions) candidatesByPid.set(s.pid, s);+ for (const s of netCandidates) if (!candidatesByPid.has(s.pid)) candidatesByPid.set(s.pid, s);+ const sessions = [...candidatesByPid.values()];+ if (sessions.length === 0) return { swapped: 0, scanned: 0, swappedPids: [] };+let swapped = 0;const swappedPids = [];@@ -188,19 +384,29 @@ export function sweepBedrockSessions(config, state, log) {continue;}+ // Hot-swap hold does NOT protect a METERED Bedrock leak (owner directive+ // 2026-06-15: "it should not be a no-op … actually swap out any bedrock /+ // metered usage directly if OAuth is at all available"). The hold exists to+ // preserve OAuth→OAuth refreshes mid-flight; for a session bleeding AWS spend+ // the math inverts — losing one in-flight tool call is strictly cheaper than+ // unbounded metered billing. Warn, then swap anyway.if (hasHotSwapHold(sess.pid)) {- // Honor the hold, but warn loudly — a held session is still bleeding Bedrock.log(- `[bedrock-watchdog] ⚠️ session ${bgSession.id} (pid ${sess.pid}) is hot-swap HELD but STILL ON METERED BEDROCK (OAuth target=${target} available) — leaving per hold marker, but money is bleeding`,+ `[bedrock-watchdog] ⚠️ session ${bgSession.id} (pid ${sess.pid}) hot-swap HELD but on METERED BEDROCK — OVERRIDING hold and force-swapping (money > in-flight call; OAuth target=${target})`,);- continue;}// FORCE swap now. Do NOT defer busy/loop sessions for Bedrock.try {recordSessionLease(bgSession.id, target, bgSession.pid);+ const viaDetail =+ sess.via === 'network'+ ? `network(bedrock-conn pid ${sess.childPid})`+ : sess.via === 'runtime'+ ? `runtime(child ${sess.childPid})`+ : 'exec-env';log(- `[bedrock-watchdog] FORCE-swapping ${bgSession.id} (pid ${sess.pid}, status ${bgSession.status}) Bedrock→OAuth ${target} (no defer — metered)`,+ `[bedrock-watchdog] FORCE-swapping ${bgSession.id} (pid ${sess.pid}, status ${bgSession.status}, via ${viaDetail}) Bedrock→OAuth ${target} (no defer — metered)`,);doRespawn(bgSession, log);swapped++;