Skip to content

Business operating system for Claude Code — 57 skills, 21 agents, smart daemon. Unified inbox (WhatsApp/Email/Slack/Telegram), autonomous PR merge, full-AWS monitoring, revenue (Stripe+RevenueCat), e-commerce (Shopify), marketing (Klaviyo/Meta/GA4), voice (Bland/ElevenLabs), APM (Datadog/NewRelic/OTEL), YOLO mode.

fix(bedrock-watchdog): detect runtime-injected + network-only Bedrock leaks (#592)

Aurora Capitalc124e24423bd
diff --git a/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs b/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs
index 864dfcd3..2aa5725e 100644
--- a/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs
+++ b/claude-ops/scripts/account-rotation/bedrock-watchdog.mjs
@@ -21,9 +21,10 @@ import { pickAccountForSession, recordSessionLease } from './session-router.mjs'
const PROC = '/proc';
-// Infra/transient session arg-markers that inherit the Bedrock env but do NOT
-// run real inference — never worth a force-swap.
-const SKIP_ARG_MARKERS = ['--bg-pty-host', '--bg-spare'];
+// Transient processes that never run billable session inference.
+// NOTE: `--bg-spare`/`--bg-pty-host` are intentionally NOT blanket-skipped here
+// (a claimed spare keeps its `--bg-spare` marker forever); the scan skips them
+// only when no Bedrock signal is present. See scanBedrockSessions.
const SKIP_CMD_SUBSTRINGS = ['auth login']; // `claude auth login` etc.
/** Read /proc/<pid>/environ into a {KEY:VALUE} map. Returns null if unreadable. */
@@ -53,10 +54,68 @@ function readProcCmdline(pid) {
}
/**
- * Scan every ec2-user `claude` process whose /proc environ has
- * CLAUDE_CODE_USE_BEDROCK=1, skipping infra (--bg-pty-host, --bg-spare) and
- * transient (auth login) processes.
- * @returns {Array<{pid:number, sessionId:string|null, cmdline:string}>}
+ * Build a one-shot parent→children map of every /proc pid. Used to probe a
+ * node session's child subshells for the RUNTIME Bedrock signal (see below).
+ * O(P) — built once per sweep, not per session.
+ */
+function buildChildMap() {
+ const kids = new Map(); // ppid -> [pid,...]
+ let pids = [];
+ try {
+ pids = readdirSync(PROC).filter((f) => /^\d+$/.test(f));
+ } catch {
+ return kids;
+ }
+ for (const p of pids) {
+ try {
+ const stat = readFileSync(`${PROC}/${p}/stat`, 'utf8');
+ // Format: `pid (comm) state ppid ...`; comm may contain spaces/parens,
+ // so anchor on the LAST ')' then take field[1] (ppid) after state.
+ const rp = stat.lastIndexOf(')');
+ if (rp === -1) continue;
+ const fields = stat.slice(rp + 2).split(' ');
+ const ppid = Number(fields[1]);
+ if (!Number.isFinite(ppid)) continue;
+ if (!kids.has(ppid)) kids.set(ppid, []);
+ kids.get(ppid).push(Number(p));
+ } catch {}
+ }
+ return kids;
+}
+
+/**
+ * RUNTIME Bedrock detection. /proc/<pid>/environ is the EXEC-TIME snapshot only;
+ * a node session that read CLAUDE_CODE_USE_BEDROCK=1 from settings.json at
+ * startup (a past fallback window) carries it in its live process.env but NOT in
+ * /proc/environ — invisible to the exec-env scan. That runtime value IS what
+ * inference uses (the real metered bill) AND is inherited by every child
+ * subshell the session spawns. So we probe the session's direct children: if any
+ * child's /proc/environ has USE_BEDROCK=1, the parent's runtime env has it too.
+ * Returns the first leaking child pid, or null.
+ */
+function runtimeBedrockChild(pid, childMap) {
+ for (const kid of childMap.get(pid) || []) {
+ const env = readProcEnviron(kid);
+ if (env && env.CLAUDE_CODE_USE_BEDROCK === '1') return kid;
+ }
+ return null;
+}
+
+/**
+ * Scan every ec2-user `claude` process on Bedrock — by EXEC env
+ * (CLAUDE_CODE_USE_BEDROCK=1 in /proc/environ) OR by RUNTIME env (a child
+ * subshell carries it; see runtimeBedrockChild). Transient `auth login`
+ * processes are always skipped.
+ *
+ * The `--bg-spare` / `--bg-pty-host` arg-markers are NOT a blanket skip: a
+ * claimed spare KEEPS `--bg-spare` in its cmdline even after it becomes a live,
+ * inference-running session (the marker is never rewritten), so blanket-skipping
+ * it permanently hides a real leaker (root cause of the fcfcc869 opus leak). We
+ * only skip those markers when there is NO Bedrock signal at all — i.e. a
+ * passive spare that merely inherited the env but isn't running inference. The
+ * moment a runtime (or exec) Bedrock signal is present, it is a real leak and
+ * gets reported regardless of the spare/pty marker.
+ * @returns {Array<{pid:number, sessionId:string|null, cmdline:string, via:'exec'|'runtime', childPid:number|null}>}
*/
export function scanBedrockSessions() {
const out = [];
@@ -66,78 +125,185 @@ export function scanBedrockSessions() {
} catch {
return out; // /proc unreadable (non-Linux / sandbox) — nothing to do.
}
+ const childMap = buildChildMap();
for (const pidStr of pids) {
const pid = Number(pidStr);
if (pid === process.pid || pid === process.ppid) continue;
const cmdline = readProcCmdline(pid);
- if (!cmdline || !/\bclaude\b/.test(cmdline)) continue;
- if (SKIP_ARG_MARKERS.some((m) => cmdline.includes(m))) continue;
+ // A swap CANDIDATE is a Claude SESSION node process (daemon-hosted bg
+ // sessions run `.../share/claude/versions/<ver> …`, foreground runs the
+ // claude bin directly). Shell/bun children (gstack browse, `bash -c`, mac
+ // helpers) inherit the Bedrock env but are NOT swappable sessions — they
+ // serve only as the runtime-env PROBE (runtimeBedrockChild), never as
+ // candidates. Matching the broad `.claude/` path substring here emitted
+ // those children as bogus, un-respawnable hits.
+ if (!cmdline) continue;
+ const isSessionProc = /share\/claude\/versions\//.test(cmdline) || /(^|\/)claude(\s|$)/.test(cmdline);
+ if (!isSessionProc) continue;
if (SKIP_CMD_SUBSTRINGS.some((m) => cmdline.includes(m))) continue;
const env = readProcEnviron(pid);
- if (!env) continue;
- if (env.CLAUDE_CODE_USE_BEDROCK !== '1') continue;
+ if (!env) continue; // unreadable environ → can't resolve a session id anyway
+ const execBedrock = env.CLAUDE_CODE_USE_BEDROCK === '1';
+ const childPid = execBedrock ? null : runtimeBedrockChild(pid, childMap);
+ const hasBedrock = execBedrock || childPid !== null;
+
+ // Passive inheritor (spare/pty-host with NO actual Bedrock inference) — skip.
+ // A real leaker (runtime or exec signal) is reported even if it's a spare.
+ if (!hasBedrock) continue;
out.push({
pid,
sessionId: env.CLAUDE_CODE_SESSION_ID || null,
cmdline,
+ via: execBedrock ? 'exec' : 'runtime',
+ childPid,
});
}
return out;
}
+// Regions where `us.anthropic.*` inference profiles + nearby Bedrock endpoints
+// land. Resolved IPs are unioned; DNS rotates so we re-resolve on a short TTL.
+const BEDROCK_REGIONS = ['us-east-1', 'us-west-2', 'eu-central-1', 'eu-west-1', 'eu-west-2'];
+let _bedrockIpCache = { ips: new Set(), at: 0 };
+const BEDROCK_IP_TTL_MS = 5 * 60_000;
+
/**
- * Best-effort: which of the given PIDs have a LIVE established connection to a
- * Bedrock-runtime endpoint (port 443 to an AWS range, reverse-resolving to
- * bedrock-runtime). Used only to CORROBORATE env detection and to VALIDATE that
- * traffic stopped after a swap — NEVER as the sole trigger.
- *
- * Sandbox-safe: `ss` can be socket-blind in a sandboxed shell on FRA (see memory
- * sandboxed-ss-port-checks-blind). On any failure / empty output this returns
- * { available:false, pids:new Set() } so callers treat it as "unknown" and never
+ * Resolve the current bedrock-runtime endpoint IPs across BEDROCK_REGIONS.
+ * Cached for BEDROCK_IP_TTL_MS (DNS rotates the A records). On total failure
+ * returns whatever is cached (possibly empty) — callers degrade to no-op, never
* false-alarm.
+ * @returns {Set<string>}
+ */
+function resolveBedrockIps() {
+ const now = Date.now();
+ if (_bedrockIpCache.ips.size && now - _bedrockIpCache.at < BEDROCK_IP_TTL_MS) {
+ return _bedrockIpCache.ips;
+ }
+ const ips = new Set();
+ for (const r of BEDROCK_REGIONS) {
+ try {
+ const out = execFileSync('getent', ['ahosts', `bedrock-runtime.${r}.amazonaws.com`], {
+ encoding: 'utf8',
+ timeout: 3000,
+ stdio: ['ignore', 'pipe', 'ignore'],
+ });
+ for (const line of out.split('\n')) {
+ const ip = line.trim().split(/\s+/)[0];
+ if (/^\d+\.\d+\.\d+\.\d+$/.test(ip)) ips.add(ip);
+ }
+ } catch {}
+ }
+ if (ips.size) _bedrockIpCache = { ips, at: now };
+ return ips.size ? ips : _bedrockIpCache.ips;
+}
+
+/**
+ * Which PIDs hold a LIVE established TCP connection to a bedrock-runtime IP.
+ * THIS IS A VALID STANDALONE TRIGGER (2026-06-15): an established TLS socket to
+ * a bedrock-runtime endpoint is ground truth that the owning process is billing
+ * Bedrock right now — there is no false-positive (you cannot hold that socket
+ * without using Bedrock). It catches the case the env/child probes miss: a busy
+ * inference session with runtime CLAUDE_CODE_USE_BEDROCK=1 but NO live child
+ * subshell at scan time (root cause of the 7f1abe98 leak the env-scan reported
+ * clean — proven by a live `ss` network trace). The old hostname-regex matcher
+ * was dead: `ss -tn` prints numeric peers, so /bedrock-runtime/ never matched.
+ *
+ * Matching is by peer IP against the resolved bedrock-runtime IP set. `ss` can
+ * be socket-blind in a sandboxed shell (memory sandboxed-ss-port-checks-blind);
+ * on any failure / empty output / unresolvable IPs this returns
+ * { available:false, pids:new Set() } so callers treat it as "unknown" — a miss
+ * never blocks anything, it just degrades to the env/child path.
*
- * @param {number[]} [pids] optional filter; when omitted, all matching are returned
+ * @param {number[]} [pids] optional filter; when omitted, all matching pids returned
* @returns {{available:boolean, pids:Set<number>}}
*/
export function scanBedrockNetwork(pids) {
const want = pids && pids.length ? new Set(pids.map(Number)) : null;
+ const bedrockIps = resolveBedrockIps();
+ if (!bedrockIps.size) return { available: false, pids: new Set() }; // can't resolve → unknown
+ // `ss -tnp` only attributes pids for the caller's OWN sockets, and can be
+ // socket-blind under sandboxing. Prefer `sudo -n ss` (passwordless sudo is
+ // provisioned for ec2-user) for full pid visibility across every session;
+ // fall back to plain `ss` (works unprivileged for the daemon's own user when
+ // not sandboxed). Either way, no-pid output → available:false (degrade).
let raw = '';
- try {
- // -t tcp, -n numeric, -p processes, state established.
- raw = execFileSync('ss', ['-tnp', 'state', 'established'], {
- encoding: 'utf8',
- timeout: 4000,
- stdio: ['ignore', 'pipe', 'ignore'],
- });
- } catch {
- return { available: false, pids: new Set() };
- }
- if (!raw || !raw.trim()) {
- // ss ran but returned nothing — could be genuinely no connections OR
- // socket-blind. Either way we can't assert traffic, so signal "unknown".
- return { available: false, pids: new Set() };
+ for (const argv of [
+ ['sudo', ['-n', 'ss', '-tnp', 'state', 'established']],
+ ['ss', ['-tnp', 'state', 'established']],
+ ]) {
+ try {
+ raw = execFileSync(argv[0], argv[1], {
+ encoding: 'utf8',
+ timeout: 4000,
+ stdio: ['ignore', 'pipe', 'ignore'],
+ });
+ if (raw && raw.trim()) break;
+ } catch {
+ raw = '';
+ }
}
+ if (!raw || !raw.trim()) return { available: false, pids: new Set() };
- // Bedrock-runtime resolves into AWS ranges; we can't reverse-resolve every IP
- // cheaply, so match on the well-known bedrock-runtime hostname when present in
- // the line, else fall back to :443 peers owned by a claude pid. This is
- // corroboration only — a miss never blocks a swap.
const found = new Set();
+ let sawAnyPid = false;
for (const line of raw.split('\n')) {
if (!line.includes(':443')) continue;
- const isBedrock = /bedrock-runtime/i.test(line);
+ // Peer address is the 2nd host:port column. Match the LAST IP:443 on the line
+ // (local addr could also be :443 in rare cases; peer is rightmost).
+ const peers = [...line.matchAll(/(\d+\.\d+\.\d+\.\d+):443\b/g)];
+ if (!peers.length) continue;
+ const peerIp = peers[peers.length - 1][1];
+ if (!bedrockIps.has(peerIp)) continue;
const m = line.match(/pid=(\d+)/);
- if (!m) continue;
- const pid = Number(m[1]);
- if (want && !want.has(pid)) continue;
- if (isBedrock) found.add(pid);
- }
- // If we couldn't positively identify any bedrock-runtime peer by name, report
- // unavailable rather than guessing from bare :443 (avoids false-alarm).
- if (found.size === 0) return { available: false, pids: new Set() };
- return { available: true, pids: found };
+ if (m) {
+ sawAnyPid = true;
+ const pid = Number(m[1]);
+ if (want && !want.has(pid)) continue;
+ found.add(pid);
+ }
+ }
+ // If we matched bedrock peers but `ss` showed no pid= (no CAP_NET_ADMIN /
+ // socket-blind), we can't attribute them — report unavailable rather than
+ // returning an empty positive that looks like "clean".
+ if (found.size === 0 && !sawAnyPid) return { available: false, pids: new Set() };
+ return { available: found.size > 0, pids: found };
+}
+
+/**
+ * Walk a PID up its parent chain to the nearest live bg-session PID. Lets a
+ * bedrock connection owned by a child `claude`/shell process be attributed to
+ * the session that spawned it. Returns the bg-session record or null.
+ */
+function ancestorBgSession(pid, byPid, ppidMap) {
+ let cur = pid;
+ for (let hops = 0; cur && cur > 1 && hops < 30; hops++) {
+ if (byPid.has(cur)) return byPid.get(cur);
+ cur = ppidMap.get(cur) || 0;
+ }
+ return null;
+}
+
+/** Build a child→parent PID map from /proc (one pass). */
+function buildPpidMap() {
+ const m = new Map();
+ let pids = [];
+ try {
+ pids = readdirSync(PROC).filter((f) => /^\d+$/.test(f));
+ } catch {
+ return m;
+ }
+ for (const p of pids) {
+ try {
+ const stat = readFileSync(`${PROC}/${p}/stat`, 'utf8');
+ const rp = stat.lastIndexOf(')');
+ if (rp === -1) continue;
+ const ppid = Number(stat.slice(rp + 2).split(' ')[1]);
+ if (Number.isFinite(ppid)) m.set(Number(p), ppid);
+ } catch {}
+ }
+ return m;
}
/**
@@ -158,13 +324,43 @@ function hasHotSwapHold(pid) {
* @returns {{swapped:number, scanned:number, swappedPids:number[]}}
*/
export function sweepBedrockSessions(config, state, log) {
- const sessions = scanBedrockSessions();
- if (sessions.length === 0) return { swapped: 0, scanned: 0, swappedPids: [] };
+ // (a) exec-env + runtime-child detection over claude session node procs.
+ const envSessions = scanBedrockSessions();
// Map PID → live bg-session record (id needed for `claude respawn <id>`).
const live = listLiveBgSessions();
const byPid = new Map(live.map((s) => [Number(s.pid), s]));
+ // (b) NETWORK detection — a valid STANDALONE trigger. An established TCP
+ // socket to a bedrock-runtime IP is ground truth that the owning process is
+ // billing Bedrock now. Catches busy inference sessions whose runtime env has
+ // CLAUDE_CODE_USE_BEDROCK=1 but which have NO live child subshell at scan time
+ // (the env/child probe reports them clean — proven by the 7f1abe98 leak the
+ // env-scan missed but a live `ss` trace caught). The connection may be owned by
+ // a transient child `claude`/shell; attribute it to its ancestor bg-session.
+ const netCandidates = [];
+ try {
+ const net = scanBedrockNetwork();
+ if (net.available && net.pids.size) {
+ const ppidMap = buildPpidMap();
+ const seen = new Set();
+ for (const connPid of net.pids) {
+ const bg = ancestorBgSession(connPid, byPid, ppidMap);
+ if (bg && !seen.has(Number(bg.pid))) {
+ seen.add(Number(bg.pid));
+ netCandidates.push({ pid: Number(bg.pid), sessionId: null, via: 'network', childPid: connPid });
+ }
+ }
+ }
+ } catch {}
+
+ // Union env/child + network candidates, keyed by bg-session node pid.
+ const candidatesByPid = new Map();
+ for (const s of envSessions) candidatesByPid.set(s.pid, s);
+ for (const s of netCandidates) if (!candidatesByPid.has(s.pid)) candidatesByPid.set(s.pid, s);
+ const sessions = [...candidatesByPid.values()];
+ if (sessions.length === 0) return { swapped: 0, scanned: 0, swappedPids: [] };
+
let swapped = 0;
const swappedPids = [];
@@ -188,19 +384,29 @@ export function sweepBedrockSessions(config, state, log) {
continue;
}
+ // Hot-swap hold does NOT protect a METERED Bedrock leak (owner directive
+ // 2026-06-15: "it should not be a no-op … actually swap out any bedrock /
+ // metered usage directly if OAuth is at all available"). The hold exists to
+ // preserve OAuth→OAuth refreshes mid-flight; for a session bleeding AWS spend
+ // the math inverts — losing one in-flight tool call is strictly cheaper than
+ // unbounded metered billing. Warn, then swap anyway.
if (hasHotSwapHold(sess.pid)) {
- // Honor the hold, but warn loudly — a held session is still bleeding Bedrock.
log(
- `[bedrock-watchdog] ⚠️ session ${bgSession.id} (pid ${sess.pid}) is hot-swap HELD but STILL ON METERED BEDROCK (OAuth target=${target} available) — leaving per hold marker, but money is bleeding`,
+ `[bedrock-watchdog] ⚠️ session ${bgSession.id} (pid ${sess.pid}) hot-swap HELD but on METERED BEDROCK — OVERRIDING hold and force-swapping (money > in-flight call; OAuth target=${target})`,
);
- continue;
}
// FORCE swap now. Do NOT defer busy/loop sessions for Bedrock.
try {
recordSessionLease(bgSession.id, target, bgSession.pid);
+ const viaDetail =
+ sess.via === 'network'
+ ? `network(bedrock-conn pid ${sess.childPid})`
+ : sess.via === 'runtime'
+ ? `runtime(child ${sess.childPid})`
+ : 'exec-env';
log(
- `[bedrock-watchdog] FORCE-swapping ${bgSession.id} (pid ${sess.pid}, status ${bgSession.status}) Bedrock→OAuth ${target} (no defer — metered)`,
+ `[bedrock-watchdog] FORCE-swapping ${bgSession.id} (pid ${sess.pid}, status ${bgSession.status}, via ${viaDetail}) Bedrock→OAuth ${target} (no defer — metered)`,
);
doRespawn(bgSession, log);
swapped++;