Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Out-of-band hard-deadline watchdog (#1633).
*
* THE PROBLEM. A `gbrain sync` that spins (e.g. synchronous catastrophic-regex
* in pack link-inference) STARVES the main event loop. When the loop never
* yields, the SIGTERM handler (process-cleanup.ts) can't run, a `--timeout`
* `setTimeout` can't fire, and the abort-flag checks between import iterations
* can't run either. The process becomes unkillable-by-SIGTERM and, under cron,
* orphans pile up for 24h+ (the reported incident). The ONLY thing that kills a
* loop-starved process is an OS signal delivered from OUTSIDE that loop.
*
* THE MECHANISM. A Bun `worker_threads` Worker runs on a real, independent OS
* thread with its own event loop. Its timer fires even while the main thread is
* in an unyielding synchronous loop. At the deadline it sends SIGTERM to its own
* process (a clean-shutdown chance if the loop happens to be responsive); at
* deadline+grace it sends SIGKILL (uncatchable — guaranteed death even when
* starved). Signaling SELF (`process.kill(process.pid, ...)`) has no PID-reuse
* footgun: the current process's PID is never reused while it's alive. (The
* rejected alternative — a detached child that signals the PARENT pid — CAN hit
* PID reuse and kill an innocent process.)
*
* `eval: true` keeps the worker body an inline string so it bakes into the
* `bun build --compile` binary with no separate-file embedding to worry about.
* Empirically validated on Bun 1.3.13 (a Worker timer fired + SIGKILLed the
* process while main was in `while(true){}`).
*
* ┌─ main thread (may be starved) ──────────────┐ ┌─ watchdog worker (OS thread) ─┐
* │ sync work / ReDoS spin / connect hang │ │ t=deadline -> SIGTERM │
* │ ...never yields... │ │ t=deadline+grace-> SIGKILL │
* │ on clean finish: handle.dispose() ──────────┼──▶│ worker.terminate() │
* └──────────────────────────────────────────────┘ └────────────────────────────────┘
*
* Reusable beyond sync (autopilot / cycle are follow-up adopters): the API is
* just (deadline, grace, label).
*/
import { Worker } from 'node:worker_threads';
// Zero-import teardown-budget leaf — safe edge, no cycle (it imports nothing).
import { MAX_TIMER_DELAY_MS } from './background-work.ts';
import { lastCommittedProgressAt, lastForwardProgressAt, onCommittedProgress, onForwardProgress } from './forward-progress.ts';
export type WatchdogAction = 'wait' | 'sigterm' | 'sigkill';
/**
* Pure decision function — the watchdog's whole state machine, extracted so it's
* unit-testable without spawning threads or real timers.
* elapsed < deadline -> 'wait'
* deadline <= elapsed < +grace -> 'sigterm' (clean-shutdown chance)
* elapsed >= deadline + grace -> 'sigkill' (guaranteed)
*/
export function watchdogDecision(elapsedMs: number, deadlineMs: number, graceMs: number): WatchdogAction {
if (elapsedMs >= deadlineMs + graceMs) return 'sigkill';
if (elapsedMs >= deadlineMs) return 'sigterm';
return 'wait';
}
export type ProgressDeadlineAction = 'wait' | 'extend' | 'stop';
/**
* Pure decision for a progress-aware deadline (mirrored in WORKER_SRC — keep
* them in lockstep). A run that is still making forward progress is never
* stopped: once the deadline passes, the deadline extends for as long as the
* last progress note is younger than `windowMs`, and stops the run only after
* a full window with no progress. `windowMs <= 0` keeps the plain wall-clock
* deadline. GBRA-133 PR 3a: `progressed` false (a run that must commit to
* count, and has committed nothing) gets one window from its start, never an
* extension from a later note: past the deadline and that window, it stops.
* elapsed < deadline -> 'wait'
* deadline passed, progress within window -> 'extend'
* deadline passed, no progress for window -> 'stop' (SIGTERM, then SIGKILL after grace)
*/
export function progressDeadlineDecision(
elapsedMs: number, deadlineMs: number, sinceProgressMs: number, windowMs: number, progressed = true,
): ProgressDeadlineAction {
if (elapsedMs < deadlineMs) return 'wait';
if (windowMs > 0 && (progressed ? sinceProgressMs : elapsedMs) < windowMs) return 'extend';
return 'stop';
}
export interface ProcessWatchdogOpts {
/** Wall-clock ms after which SIGTERM is sent. Must be > 0 or the watchdog is a no-op. */
deadlineMs: number;
/** ms after the deadline before SIGKILL. Default 30_000. */
graceMs?: number;
/** Prefix for stderr log lines, e.g. 'sync-watchdog'. Default 'watchdog'. */
label?: string;
/** Periodic "still alive, kill in Ns" heartbeat interval ms. 0 = off (default). */
heartbeatMs?: number;
/** Injectable warn sink (tests). Default writes to process.stderr. */
onWarn?: (msg: string) => void;
/**
* Progress-aware mode: past the deadline, keep the run alive while
* `noteForwardProgress()` (src/core/forward-progress.ts) fired within this
* many ms; stop only after a full window without progress. 0 = off (plain
* wall-clock deadline). The window starts at install, so a run that has not
* reported progress yet still gets one window.
*/
progressWindowMs?: number;
/**
* Written to stdout (fd 1) when the watchdog stops the run, so the agent
* operating the command sees why it stopped and how to resume, not only a
* stderr line and an exit status.
*/
stopNotice?: string;
/**
* GBRA-133 PR 3a: which notes count as progress. `committed` listens to
* `noteCommittedProgress()` only (pages written, a cursor advanced), and a
* run that has committed nothing gets one window from its start and no
* extension, and stops past its deadline and that window; `stallNotice` is then written instead of
* `stopNotice`. Default `forward`.
*/
progressSignal?: 'forward' | 'committed';
stallNotice?: string;
}
export interface WatchdogHandle {
/** Tear down the watchdog (clean completion). Idempotent. */
dispose(): void;
/** True when an out-of-band worker is actually running (false on no-op / fallback). */
readonly active: boolean;
}
const DEFAULT_GRACE_MS = 30_000;
/**
* setTimeout delay ceiling (2^31−1): Node/Bun overflow-clamp larger delays to
* ~1ms, which for the deadline+grace SUM timer would mean SIGKILLing a healthy
* process almost immediately (#4284 hardening). Aliased from the shared
* teardown-budget constant so the ceiling cannot drift between modules.
*/
export const MAX_WATCHDOG_TIMER_MS = MAX_TIMER_DELAY_MS;
/**
* Pure clamp for the two worker timers (#4284). The worker arms
* setTimeout(deadlineMs) AND setTimeout(deadlineMs + graceMs); BOTH delays
* must stay ≤ 2^31−1 or the overflowing one fires at ~1ms — for the sum
* timer that is an instant SIGKILL. Grace is clamped so the SUM fits, and a
* NON-FINITE grace is coerced to 0 (never NaN: setTimeout(deadline + NaN)
* fires at ~1ms — the exact failure this clamp exists to prevent). NaN
* deadline flows through as NaN so the caller's Number.isFinite inert-check
* still catches it; an INFINITE deadline clamps to the ceiling and ARMS
* (~24.8 days) rather than going inert — deliberate, matching the "oversized
* means longer bound" rule. Exported pure so unit tests verify the arithmetic
* without ever arming a real max-deadline worker in-suite (whose buggy
* overflow would SIGTERM the test runner itself).
*/
export function clampWatchdogTimers(deadlineMs: number, graceMs: number): { deadlineMs: number; graceMs: number } {
const d = Math.min(MAX_WATCHDOG_TIMER_MS, Math.floor(deadlineMs));
const safeGrace = Number.isFinite(graceMs) ? Math.max(0, Math.floor(graceMs)) : 0;
const g = Math.min(MAX_WATCHDOG_TIMER_MS - (Number.isFinite(d) ? Math.max(0, d) : 0), safeGrace);
return { deadlineMs: d, graceMs: g };
}
function defaultWarn(msg: string): void {
try { process.stderr.write(msg + '\n'); } catch { /* stderr may be broken */ }
}
const INERT: WatchdogHandle = { dispose() {}, get active() { return false; } };
/**
* Worker body (runs on its own OS thread). Inline string so `eval: true` bakes
* it into the compiled binary. Uses only built-ins available in a Bun worker.
* Log lines go straight to fd 2: since Bun 1.4 a worker's `process.stderr` is
* forwarded through the parent's event loop, which a starved parent never runs.
*
* `label` is validated by the caller to a safe charset before it reaches here,
* so it can't break the string literal or inject log lines.
*/
const WORKER_SRC = `
const { parentPort, workerData } = require('node:worker_threads');
const { writeSync } = require('node:fs');
const { deadlineMs, graceMs, label, heartbeatMs, progressWindowMs, stopNotice, stallNotice, requireProgress } = workerData;
const t0 = Date.now();
let lastProgress = t0;
let progressed = !requireProgress;
let extended = false;
function w(m) { try { writeSync(2, '[' + label + '] ' + m + '\\n'); } catch (e) {} }
if (progressWindowMs > 0 && parentPort) parentPort.on('message', () => { lastProgress = Date.now(); progressed = true; });
if (heartbeatMs > 0) {
const hb = setInterval(() => {
const elapsed = Math.round((Date.now() - t0) / 1000);
if (extended) {
w('parent alive ' + elapsed + 's elapsed, past the deadline and still progressing (last progress ' + Math.round((Date.now() - lastProgress) / 1000) + 's ago); stops after ' + Math.round(progressWindowMs / 1000) + 's without progress');
return;
}
if (progressWindowMs > 0) {
w('parent alive ' + elapsed + 's elapsed, deadline in ~' + Math.round((deadlineMs - (Date.now() - t0)) / 1000) + 's (extends while progressing)');
return;
}
const killIn = Math.round((deadlineMs + graceMs - (Date.now() - t0)) / 1000);
w('parent alive ' + elapsed + 's elapsed, hard-kill in ~' + killIn + 's');
}, heartbeatMs);
if (typeof hb.unref === 'function') hb.unref();
}
function stop() {
if (!progressed) {
w('deadline reached (' + Math.round(deadlineMs/1000) + 's) and the run committed nothing — a stall, not progress; sending SIGTERM for graceful shutdown');
if (stallNotice) { try { writeSync(1, stallNotice + '\\n'); } catch (e) {} }
try { process.kill(process.pid, 'SIGTERM'); } catch (e) {}
setTimeout(() => {
w('grace expired — sending SIGKILL (event loop was starved; this is the orphan-pileup backstop)');
try { process.kill(process.pid, 'SIGKILL'); } catch (e) {}
}, graceMs);
return;
}
if (progressWindowMs > 0) {
w('deadline reached (' + Math.round(deadlineMs/1000) + 's) and no progress for ' + Math.round((Date.now() - lastProgress) / 1000) + 's — sending SIGTERM for graceful shutdown');
} else {
w('deadline reached (' + Math.round(deadlineMs/1000) + 's) — sending SIGTERM for graceful shutdown');
}
if (stopNotice) { try { writeSync(1, stopNotice + '\\n'); } catch (e) {} }
try { process.kill(process.pid, 'SIGTERM'); } catch (e) {}
setTimeout(() => {
w('grace expired — sending SIGKILL (event loop was starved; this is the orphan-pileup backstop)');
try { process.kill(process.pid, 'SIGKILL'); } catch (e) {}
}, graceMs);
}
function check() {
const now = Date.now();
const since = now - lastProgress;
// mirrors progressDeadlineDecision
if (now - t0 < deadlineMs) { setTimeout(check, deadlineMs - (now - t0)); return; }
if (progressWindowMs > 0 && since < progressWindowMs) {
if (!extended) w(progressed
? 'deadline reached (' + Math.round(deadlineMs/1000) + 's) but the run is still progressing (last progress ' + Math.round(since / 1000) + 's ago) — extending; it stops after ' + Math.round(progressWindowMs / 1000) + 's without progress'
: 'deadline reached (' + Math.round(deadlineMs/1000) + 's) before the run committed anything; it stops as a stall unless it commits within ' + Math.round(progressWindowMs / 1000) + 's of its start');
extended = true;
setTimeout(check, progressWindowMs - since);
return;
}
stop();
}
setTimeout(check, deadlineMs);
`;
/**
* Install the out-of-band hard-deadline watchdog. Returns a handle whose
* `dispose()` MUST be called on clean completion (a `finally`) so the worker is
* torn down. If the deadline is non-positive, returns an inert no-op handle.
*
* Fallback: if the Worker can't be constructed (unexpected on Bun), degrades to
* an in-process timer with a loud warning. The in-process timer canNOT fire
* under event-loop starvation — it only covers the responsive case — so the
* warning tells the operator the hard guarantee is degraded.
*/
export function installProcessWatchdog(opts: ProcessWatchdogOpts): WatchdogHandle {
const warn = opts.onWarn ?? defaultWarn;
const { deadlineMs, graceMs } = clampWatchdogTimers(opts.deadlineMs, opts.graceMs ?? DEFAULT_GRACE_MS);
// Sanitize label to a safe charset (defends the inline worker string + log lines).
const label = (opts.label ?? 'watchdog').replace(/[^A-Za-z0-9_.:-]/g, '').slice(0, 40) || 'watchdog';
const heartbeatMs = Math.max(0, Math.floor(opts.heartbeatMs ?? 0));
const progressWindowMs = Math.min(MAX_WATCHDOG_TIMER_MS, Math.max(0, Math.floor(Number.isFinite(opts.progressWindowMs) ? opts.progressWindowMs! : 0)));
const stopNotice = opts.stopNotice ?? '';
const committed = opts.progressSignal === 'committed';
const stallNotice = opts.stallNotice ?? stopNotice;
if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) return INERT;
try {
const worker = new Worker(WORKER_SRC, {
eval: true,
workerData: { deadlineMs, graceMs, label, heartbeatMs, progressWindowMs, stopNotice, stallNotice, requireProgress: committed && progressWindowMs > 0 },
});
// Don't let the watchdog keep the process alive past clean completion.
(worker as unknown as { unref?: () => void }).unref?.();
// A worker-side error must never crash the host; log and move on.
worker.on('error', (err) => warn(`[${label}] watchdog worker error: ${err instanceof Error ? err.message : String(err)}`));
// Progress notes are forwarded at most once per throttle interval: a
// per-file postMessage on a 50k-file import is needless cross-thread
// traffic when the worker only compares against a window of minutes.
const throttleMs = Math.max(10, Math.min(1000, Math.floor(progressWindowMs / 4)));
let lastPost = 0;
const unsubscribe = progressWindowMs > 0 ? (committed ? onCommittedProgress : onForwardProgress)(() => {
const now = Date.now();
if (now - lastPost < throttleMs) return;
lastPost = now;
try { worker.postMessage(1); } catch { /* worker gone */ }
}) : () => {};
let disposed = false;
return {
dispose() {
if (disposed) return;
disposed = true;
unsubscribe();
void worker.terminate();
},
get active() { return !disposed; },
};
} catch (err) {
// Fallback: in-process timer. Starvation-vulnerable — say so loudly.
warn(
`[${label}] could not start out-of-band watchdog (${err instanceof Error ? err.message : String(err)}); ` +
`falling back to an in-process timer that will NOT fire if the event loop is starved.`,
);
let killed = false;
const t0 = Date.now();
let kill: ReturnType<typeof setTimeout> | undefined;
let term: ReturnType<typeof setTimeout>;
const check = () => {
const now = Date.now();
const last = committed ? lastCommittedProgressAt() : lastForwardProgressAt();
const progressed = !committed || progressWindowMs <= 0 || last > t0;
const since = now - Math.max(t0, last);
const action = progressDeadlineDecision(now - t0, deadlineMs, since, progressWindowMs, progressed);
if (action !== 'stop') {
term = setTimeout(check, action === 'wait' ? deadlineMs - (now - t0) : progressWindowMs - since);
(term as unknown as { unref?: () => void }).unref?.();
return;
}
const notice = progressed ? stopNotice : stallNotice;
if (notice) { try { process.stdout.write(notice + '\n'); } catch { /* */ } }
try { process.kill(process.pid, 'SIGTERM'); } catch { /* */ }
kill = setTimeout(() => { killed = true; try { process.kill(process.pid, 'SIGKILL'); } catch { /* */ } }, graceMs);
(kill as unknown as { unref?: () => void }).unref?.();
};
term = setTimeout(check, deadlineMs);
(term as unknown as { unref?: () => void }).unref?.();
let disposed = false;
return {
dispose() {
if (disposed || killed) return;
disposed = true;
clearTimeout(term);
if (kill) clearTimeout(kill);
},
get active() { return !disposed; },
};
}
}
/* ————————————————————————————————————————————————————————————————————————
* Loop-STALL watchdog (#4281).
*
* The hard-deadline watchdog above bounds TOTAL wall-clock time — right for a
* bounded batch (`sync`), wrong for a long-lived server whose legitimate
* lifetime is unbounded. What a server needs bounded is main-loop
* RESPONSIVENESS: `gbrain serve --http` wedged in a synchronous spin (ReDoS,
* runaway parse) stops answering requests AND stops being able to run its own
* SIGTERM cleanup, holding the PGLite write lock hostage exactly like the
* #1633 sync incident.
*
* Mechanism: the main thread "pets" a worker_threads Worker via postMessage on
* a short interval. The worker (independent OS thread, own event loop) tracks
* lag = now − lastPet. lag ≥ stall → SIGTERM once (latched: the graceful path
* gets exactly one chance PER STALL); lag ≥ stall+grace → SIGKILL (uncatchable
* — the starved-loop backstop). A loop that recovers after the SIGTERM resumes
* petting, lag collapses, the SIGKILL never fires, and the latch RESETS — a
* later, separate stall starts the escalation over from SIGTERM. The
* escalation only completes while the loop is STILL starved.
*
* Suspend forgiveness: across a system sleep (laptop lid) BOTH threads stop,
* so on wake the worker would see a huge lag on a perfectly healthy process.
* The worker detects that its own check cadence gapped (it was suspended too)
* and forgives — a fresh full stall window is required post-wake.
* ———————————————————————————————————————————————————————————————————————— */
/** Env knob for `gbrain serve --http` (opt-in; ms; 0/unset = off).
* Large PGLite brains routinely hold the loop synchronously for tens of
* seconds (WASM checkpoint, vacuum, big JSON parse) — prefer a value well
* above the floor there (e.g. 60000+) so legitimate pauses never SIGTERM a
* healthy server. */
export const SERVE_STALL_WATCHDOG_ENV = 'GBRAIN_SERVE_STALL_WATCHDOG_MS';
/**
* Floor for the env-configured stall threshold. A legitimate serve pauses the
* loop for whole seconds under heavy synchronous work (large JSON parse, WASM
* checkpoint); a sub-15s threshold would let a well-meant operator value turn
* the watchdog into a hair-trigger that SIGTERMs healthy servers.
*/
export const SERVE_STALL_FLOOR_MS = 15_000;
/** Default lag beyond the stall threshold before SIGKILL (plan #4281: 30s). */
export const STALL_DEFAULT_GRACE_MS = 30_000;
/**
* Pure decision function for the stall watchdog — mirrors the worker's inline
* logic exactly (same reason watchdogDecision exists: unit-testable without
* threads or timers). `latched` = "SIGTERM already sent for the CURRENT
* stall" (see nextStallLatch — recovery resets it).
* lag >= stall+grace -> 'sigkill' (regardless of latch)
* lag >= stall and !latched -> 'sigterm' (caller latches)
* otherwise -> 'wait' (includes the recovered-loop case:
* latched but lag back below stall — a
* petting process is never killed)
*/
export function stallDecision(lagMs: number, stallMs: number, graceMs: number, latched: boolean): WatchdogAction {
if (lagMs >= stallMs + graceMs) return 'sigkill';
if (lagMs >= stallMs && !latched) return 'sigterm';
return 'wait';
}
/**
* Pure latch-transition companion to stallDecision (mirrored in the worker —
* keep them in lockstep). The latch arms when SIGTERM fires and RESETS when
* lag recovers below the stall threshold: a process that recovered (pets
* resumed) and later re-stalls is a NEW stall, so it gets a fresh graceful
* SIGTERM chance before the SIGKILL escalation instead of skipping straight
* to SIGKILL on the second stall of its lifetime.
*/
export function nextStallLatch(action: WatchdogAction, lagMs: number, stallMs: number, latched: boolean): boolean {
if (action === 'sigterm') return true;
if (lagMs < stallMs) return false;
return latched;
}
/**
* Pure suspend-detector, mirrored in the worker. If the gap between two
* consecutive worker-side checks exceeds max(5× the check interval, 2s), the
* WATCHDOG THREAD itself was stalled (system sleep / suspend / heavy VM
* pause) — lag measured across that gap is not main-loop starvation and must
* be forgiven, or every laptop-lid wake SIGKILLs a healthy serve.
*/
export function stallCheckSawSuspend(checkGapMs: number, checkIntervalMs: number): boolean {
return checkGapMs > Math.max(5 * checkIntervalMs, 2000);
}
/**
* Resolve GBRAIN_SERVE_STALL_WATCHDOG_MS to a stall threshold in ms.
* Returns 0 for "off". Lenient (warn + off) rather than throw, matching the
* boot-timeout env posture in serve.ts: a typo'd escape hatch must never
* become a process-killer of its own.
* unset / blank -> 0, silent (the knob is opt-in)
* '0' -> 0, silent (documented explicit off)
* garbage (NaN / negative / non-finite) -> warn + 0
* (0, floor) -> warn + clamp UP to SERVE_STALL_FLOOR_MS
* >= floor -> floor'd integer
*/
export function resolveServeStallWatchdogMs(
raw: string | undefined,
warn: (msg: string) => void = defaultWarn,
): number {
if (raw === undefined || raw.trim() === '') return 0;
const n = Number(raw);
if (!Number.isFinite(n) || n < 0) {
warn(
`[serve-stall-watchdog] ignoring invalid ${SERVE_STALL_WATCHDOG_ENV}=${JSON.stringify(raw)} — ` +
`watchdog stays OFF (set a positive integer of milliseconds; floor ${SERVE_STALL_FLOOR_MS}ms, 0 disables)`,
);
return 0;
}
if (n === 0) return 0;
if (n < SERVE_STALL_FLOOR_MS) {
warn(
`[serve-stall-watchdog] ${SERVE_STALL_WATCHDOG_ENV}=${raw} is below the ${SERVE_STALL_FLOOR_MS}ms floor — ` +
`clamping up to ${SERVE_STALL_FLOOR_MS}ms (sub-floor thresholds hair-trigger on legitimate synchronous work)`,
);
return SERVE_STALL_FLOOR_MS;
}
return Math.floor(n);
}
export interface LoopStallWatchdogOpts {
/** Main-loop lag (ms since last pet) at which SIGTERM fires. Must be > 0 or the handle is inert. */
stallMs: number;
/** Additional lag beyond stallMs before SIGKILL. Default STALL_DEFAULT_GRACE_MS (30s). */
graceMs?: number;
/** Prefix for stderr log lines. Default 'stall-watchdog'. */
label?: string;
/** Pet cadence on the main thread. Default min(1s, stallMs/4); clamped ≤ stallMs/3. */
petIntervalMs?: number;
/** Worker-side lag-check cadence. Same default/clamp as petIntervalMs. */
checkIntervalMs?: number;
/** Injectable warn sink (tests). Default writes to process.stderr. */
onWarn?: (msg: string) => void;
}
/**
* Worker body for the stall watchdog (own OS thread; inline string so
* `eval: true` bakes into the compiled binary, same as WORKER_SRC). Logic
* mirrors stallDecision + nextStallLatch + stallCheckSawSuspend — keep them
* in lockstep.
* The check interval is deliberately NOT unref'd: it (with the parentPort
* listener) keeps the worker alive; the MAIN thread unrefs the worker itself
* so the watchdog never holds the host process open.
*/
const STALL_WORKER_SRC = `
const { parentPort, workerData } = require('node:worker_threads');
const { writeSync } = require('node:fs');
const { stallMs, graceMs, label, checkIntervalMs } = workerData;
let lastPet = Date.now();
let lastCheck = Date.now();
let latched = false;
function w(m) { try { writeSync(2, '[' + label + '] ' + m + '\\n'); } catch (e) {} }
if (parentPort) parentPort.on('message', () => { lastPet = Date.now(); });
setInterval(() => {
const now = Date.now();
const checkGap = now - lastCheck;
lastCheck = now;
if (checkGap > Math.max(5 * checkIntervalMs, 2000)) {
// mirrors stallCheckSawSuspend: the watchdog thread itself just stalled
// (system sleep / suspend) — forgive; require a fresh stall window.
lastPet = now;
return;
}
const lag = now - lastPet;
if (lag >= stallMs + graceMs) {
w('main loop unresponsive for ' + Math.round(lag / 1000) + 's (stall ' + Math.round(stallMs / 1000) + 's + grace ' + Math.round(graceMs / 1000) + 's) — sending SIGKILL (loop starved through the grace window; orphan-lock backstop)');
try { process.kill(process.pid, 'SIGKILL'); } catch (e) {}
} else if (lag >= stallMs && !latched) {
latched = true;
w('main loop unresponsive for ' + Math.round(lag / 1000) + 's (>= ' + Math.round(stallMs / 1000) + 's stall threshold) — sending SIGTERM for graceful shutdown');
try { process.kill(process.pid, 'SIGTERM'); } catch (e) {}
} else if (latched && lag < stallMs) {
// mirrors nextStallLatch: the loop recovered (pets resumed), so a later
// re-stall is a NEW stall — re-arm the graceful SIGTERM rather than
// skipping straight to SIGKILL on the second stall of this lifetime.
latched = false;
w('main loop recovered — re-arming graceful SIGTERM for any future stall');
}
}, checkIntervalMs);
`;
/**
* Install the loop-stall watchdog. Returns a handle whose `dispose()` MUST be
* called on clean completion (a `finally`) so the worker + pet interval are
* torn down. Inert no-op handle when stallMs is non-positive / non-finite.
*
* No in-process fallback exists for this watchdog: an in-process timer cannot
* observe its own loop's starvation (it fires late or never — exactly the
* failure being guarded). If the Worker can't be constructed we warn loudly
* and return an inert handle.
*/
export function installLoopStallWatchdog(opts: LoopStallWatchdogOpts): WatchdogHandle {
const warn = opts.onWarn ?? defaultWarn;
const stallMs = Number.isFinite(opts.stallMs) ? Math.floor(opts.stallMs) : Number.NaN;
if (!Number.isFinite(stallMs) || stallMs <= 0) return INERT;
// Non-finite grace coerces to the DEFAULT (not 0): grace only feeds lag
// comparisons here (no setTimeout arming, so no #4284 overflow class), and
// a NaN comparison would silently disable the SIGKILL escalation forever.
const rawGrace = opts.graceMs ?? STALL_DEFAULT_GRACE_MS;
const graceMs = Number.isFinite(rawGrace) ? Math.max(0, Math.floor(rawGrace)) : STALL_DEFAULT_GRACE_MS;
// Sanitize label to a safe charset (defends the inline worker string + log lines).
const label = (opts.label ?? 'stall-watchdog').replace(/[^A-Za-z0-9_.:-]/g, '').slice(0, 40) || 'stall-watchdog';
// Cadences must sit well below the stall threshold or every check would see
// a false "lag": clamp both to ≤ stallMs/3 (floor 10ms).
const maxCadence = Math.max(10, Math.floor(stallMs / 3));
const petIntervalMs = Math.min(maxCadence, Math.max(10, Math.floor(opts.petIntervalMs ?? Math.min(1000, stallMs / 4))));
const checkIntervalMs = Math.min(maxCadence, Math.max(10, Math.floor(opts.checkIntervalMs ?? Math.min(1000, stallMs / 4))));
try {
const worker = new Worker(STALL_WORKER_SRC, {
eval: true,
workerData: { stallMs, graceMs, label, checkIntervalMs },
});
// Don't let the watchdog keep the process alive past clean completion.
(worker as unknown as { unref?: () => void }).unref?.();
// A worker-side error must never crash the host; log and move on.
worker.on('error', (err) => warn(`[${label}] stall-watchdog worker error: ${err instanceof Error ? err.message : String(err)}`));
// The pet: proof-of-life from the main event loop. postMessage after
// terminate is a no-op in worker_threads, but guard anyway.
const pet = setInterval(() => { try { worker.postMessage(1); } catch { /* worker gone */ } }, petIntervalMs);
(pet as unknown as { unref?: () => void }).unref?.();
let disposed = false;
return {
dispose() {
if (disposed) return;
disposed = true;
clearInterval(pet);
void worker.terminate();
},
get active() { return !disposed; },
};
} catch (err) {
warn(
`[${label}] could not start loop-stall watchdog (${err instanceof Error ? err.message : String(err)}); ` +
`running WITHOUT a stall backstop — an in-process fallback cannot observe its own starved loop.`,
);
return INERT;
}
}