Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Pure-function coverage for the watchdog state machine (#1633). No threads, no
* real timers — the spawn-based integration lives in
* test/process-watchdog-harness.test.ts (Bun-pinned, real processes).
*/
import { describe, test, expect } from 'bun:test';
import {
watchdogDecision,
progressDeadlineDecision,
installProcessWatchdog,
clampWatchdogTimers,
MAX_WATCHDOG_TIMER_MS,
stallDecision,
nextStallLatch,
stallCheckSawSuspend,
resolveServeStallWatchdogMs,
installLoopStallWatchdog,
SERVE_STALL_FLOOR_MS,
STALL_DEFAULT_GRACE_MS,
} from '../src/core/process-watchdog.ts';
describe('watchdogDecision', () => {
const deadline = 1000;
const grace = 300;
test('waits before the deadline', () => {
expect(watchdogDecision(0, deadline, grace)).toBe('wait');
expect(watchdogDecision(999, deadline, grace)).toBe('wait');
});
test('SIGTERM at the deadline boundary (inclusive)', () => {
expect(watchdogDecision(1000, deadline, grace)).toBe('sigterm');
expect(watchdogDecision(1299, deadline, grace)).toBe('sigterm');
});
test('SIGKILL at deadline+grace boundary (inclusive)', () => {
expect(watchdogDecision(1300, deadline, grace)).toBe('sigkill');
expect(watchdogDecision(5000, deadline, grace)).toBe('sigkill');
});
test('zero grace goes straight to SIGKILL at the deadline', () => {
expect(watchdogDecision(999, deadline, 0)).toBe('wait');
expect(watchdogDecision(1000, deadline, 0)).toBe('sigkill');
});
});
describe('installProcessWatchdog (handle contract)', () => {
test('non-positive deadline returns an inert no-op handle', () => {
const warns: string[] = [];
const h0 = installProcessWatchdog({ deadlineMs: 0, onWarn: (m) => warns.push(m) });
expect(h0.active).toBe(false);
h0.dispose(); // idempotent, no throw
const hNeg = installProcessWatchdog({ deadlineMs: -5, onWarn: (m) => warns.push(m) });
expect(hNeg.active).toBe(false);
});
test('active handle disposes idempotently without killing the test process', () => {
// Long deadline so it never fires during the test; dispose tears it down.
const h = installProcessWatchdog({ deadlineMs: 60_000, graceMs: 60_000, label: 'unit-wd' });
expect(h.active).toBe(true);
h.dispose();
expect(h.active).toBe(false);
h.dispose(); // second dispose is a no-op
expect(h.active).toBe(false);
});
test('label is sanitized to a safe charset', () => {
// A nasty label must not throw at construction (it is stripped before the
// inline worker string). We dispose immediately so nothing fires.
const h = installProcessWatchdog({ deadlineMs: 60_000, label: "evil'; \n process.exit(1) //" });
expect(h.active).toBe(true);
h.dispose();
});
});
describe('clampWatchdogTimers (#4284 joint-overflow clamp)', () => {
// Pure-function tests ONLY: setTimeout overflow-fires above 2^31−1, so a
// buggy clamp armed on a REAL worker would SIGTERM this test process at
// ~1ms (process.kill(process.pid)). Never arm a max-deadline worker here.
test('normal values pass through floored', () => {
expect(clampWatchdogTimers(5000.9, 600.2)).toEqual({ deadlineMs: 5000, graceMs: 600 });
});
test('a max deadline forces grace to zero so the SUM timer cannot overflow', () => {
const { deadlineMs, graceMs } = clampWatchdogTimers(MAX_WATCHDOG_TIMER_MS, 30_000);
expect(deadlineMs).toBe(MAX_WATCHDOG_TIMER_MS);
expect(graceMs).toBe(0);
expect(deadlineMs + graceMs).toBeLessThanOrEqual(MAX_WATCHDOG_TIMER_MS);
});
test('an oversized deadline clamps to the ceiling; the sum still fits', () => {
const { deadlineMs, graceMs } = clampWatchdogTimers(Number.MAX_SAFE_INTEGER, 30_000);
expect(deadlineMs).toBe(MAX_WATCHDOG_TIMER_MS);
expect(deadlineMs + graceMs).toBeLessThanOrEqual(MAX_WATCHDOG_TIMER_MS);
});
test('a near-ceiling deadline trims grace to exactly fit the sum', () => {
const { deadlineMs, graceMs } = clampWatchdogTimers(MAX_WATCHDOG_TIMER_MS - 10_000, 30_000);
expect(deadlineMs).toBe(MAX_WATCHDOG_TIMER_MS - 10_000);
expect(graceMs).toBe(10_000);
});
test('negative grace clamps to zero; NaN deadline flows through for the inert-check', () => {
expect(clampWatchdogTimers(5000, -1).graceMs).toBe(0);
// installProcessWatchdog's Number.isFinite inert-check must still see NaN.
expect(Number.isFinite(clampWatchdogTimers(Number.NaN, 100).deadlineMs)).toBe(false);
});
test('NaN grace is coerced to 0, never armed as a ~1ms overflow SIGKILL', () => {
// setTimeout(deadline + NaN) = setTimeout(NaN) fires at ~1ms — the exact
// failure class this clamp exists to prevent (multi-specialist finding).
const { deadlineMs, graceMs } = clampWatchdogTimers(5000, Number.NaN);
expect(deadlineMs).toBe(5000);
expect(graceMs).toBe(0);
});
test('Infinity deadline clamps to the ceiling and arms (documented contract)', () => {
// Deliberate: "oversized means longer bound" — an infinite deadline
// becomes ~24.8 days armed, not inert (see the clamp docstring).
const { deadlineMs, graceMs } = clampWatchdogTimers(Number.POSITIVE_INFINITY, 30_000);
expect(deadlineMs).toBe(MAX_WATCHDOG_TIMER_MS);
expect(deadlineMs + graceMs).toBeLessThanOrEqual(MAX_WATCHDOG_TIMER_MS);
});
test('installProcessWatchdog stays inert on NaN and non-positive deadlines post-clamp', () => {
const h = installProcessWatchdog({ deadlineMs: Number.NaN });
expect(h.active).toBe(false);
h.dispose();
});
});
describe('stallDecision (#4281 loop-stall state machine)', () => {
const stall = 300;
const grace = 200;
test('waits below the stall threshold regardless of latch state', () => {
expect(stallDecision(0, stall, grace, false)).toBe('wait');
expect(stallDecision(299, stall, grace, false)).toBe('wait');
expect(stallDecision(299, stall, grace, true)).toBe('wait');
});
test('SIGTERM at the stall boundary (inclusive) when unlatched', () => {
expect(stallDecision(300, stall, grace, false)).toBe('sigterm');
expect(stallDecision(499, stall, grace, false)).toBe('sigterm');
});
test('latch suppresses a repeat SIGTERM inside the grace window', () => {
expect(stallDecision(300, stall, grace, true)).toBe('wait');
expect(stallDecision(499, stall, grace, true)).toBe('wait');
});
test('SIGKILL at stall+grace (inclusive) regardless of latch', () => {
expect(stallDecision(500, stall, grace, false)).toBe('sigkill');
expect(stallDecision(500, stall, grace, true)).toBe('sigkill');
expect(stallDecision(60_000, stall, grace, true)).toBe('sigkill');
});
test('zero grace escalates straight to SIGKILL at the stall threshold', () => {
expect(stallDecision(299, stall, 0, false)).toBe('wait');
expect(stallDecision(300, stall, 0, false)).toBe('sigkill');
});
test('recovered lag after a latch returns wait — a petting loop is never killed', () => {
// The SIGTERM already went out (latched); if the loop recovered and pets
// resumed, lag drops below stall and the SIGKILL escalation MUST not fire.
expect(stallDecision(10, stall, grace, true)).toBe('wait');
});
});
describe('nextStallLatch (wave-D review: latch resets on recovery)', () => {
const stall = 300;
const grace = 200;
test('arms on sigterm, holds through the same stall, resets on recovery', () => {
let latched = false;
// First stall crosses the threshold → SIGTERM, latch arms.
let action = stallDecision(300, stall, grace, latched);
expect(action).toBe('sigterm');
latched = nextStallLatch(action, 300, stall, latched);
expect(latched).toBe(true);
// Still the SAME stall (in-grace lag): no SIGTERM spam, latch holds.
action = stallDecision(450, stall, grace, latched);
expect(action).toBe('wait');
latched = nextStallLatch(action, 450, stall, latched);
expect(latched).toBe(true);
// Pets resume, lag collapses below stall: latch RESETS.
action = stallDecision(10, stall, grace, latched);
expect(action).toBe('wait');
latched = nextStallLatch(action, 10, stall, latched);
expect(latched).toBe(false);
});
test('a recovered-then-restalled loop gets a fresh SIGTERM before SIGKILL', () => {
// Lifetime latch (pre-fix): the second stall's graceful window was
// silently skipped — the process waited mute until the SIGKILL line.
let latched = true; // SIGTERM already sent for a PREVIOUS stall
latched = nextStallLatch('wait', 5, stall, latched); // recovery tick
expect(latched).toBe(false);
// Re-stall: lag past stall, still inside grace → SIGTERM fires AGAIN.
expect(stallDecision(350, stall, grace, latched)).toBe('sigterm');
});
test('the latch never resets mid-stall (one SIGTERM per stall window)', () => {
let latched = true;
latched = nextStallLatch('wait', 450, stall, latched); // lag still >= stall
expect(latched).toBe(true);
expect(stallDecision(450, stall, grace, latched)).toBe('wait');
});
test('sigkill ticks do not disturb the latch', () => {
expect(nextStallLatch('sigkill', 600, stall, true)).toBe(true);
});
});
describe('stallCheckSawSuspend (#4281 suspend forgiveness)', () => {
test('a normal check cadence is not a suspend', () => {
expect(stallCheckSawSuspend(25, 25)).toBe(false);
expect(stallCheckSawSuspend(80, 25)).toBe(false);
});
test('a check gap past max(5x interval, 2s) is a suspend (laptop-lid wake must not SIGKILL)', () => {
expect(stallCheckSawSuspend(3000, 25)).toBe(true); // 2s floor governs
expect(stallCheckSawSuspend(10_000, 1000)).toBe(true); // 5x interval governs
});
test('below the 5x-interval line with a large interval is still normal jitter', () => {
// interval 1000 -> threshold max(5000, 2000) = 5000; 4000 is jitter, not suspend
expect(stallCheckSawSuspend(4000, 1000)).toBe(false);
});
});
describe('resolveServeStallWatchdogMs (#4281 env knob)', () => {
const collect = () => {
const warns: string[] = [];
return { warns, warn: (m: string) => warns.push(m) };
};
test('unset / blank means OFF, silently (opt-in knob)', () => {
const { warns, warn } = collect();
expect(resolveServeStallWatchdogMs(undefined, warn)).toBe(0);
expect(resolveServeStallWatchdogMs('', warn)).toBe(0);
expect(resolveServeStallWatchdogMs(' ', warn)).toBe(0);
expect(warns).toEqual([]);
});
test('explicit 0 is a documented OFF, no warning', () => {
const { warns, warn } = collect();
expect(resolveServeStallWatchdogMs('0', warn)).toBe(0);
expect(warns).toEqual([]);
});
test('garbage values warn and stay OFF (a typo must not arm a killer)', () => {
for (const raw of ['abc', '15s', '-1', 'Infinity', 'NaN']) {
const { warns, warn } = collect();
expect(resolveServeStallWatchdogMs(raw, warn)).toBe(0);
expect(warns.length).toBe(1);
expect(warns[0]).toContain('GBRAIN_SERVE_STALL_WATCHDOG_MS');
}
});
test('below the 15s floor clamps UP with a warning (never a hair-trigger)', () => {
const { warns, warn } = collect();
expect(resolveServeStallWatchdogMs('5000', warn)).toBe(SERVE_STALL_FLOOR_MS);
expect(warns.length).toBe(1);
expect(warns[0]).toContain('floor');
});
test('at/above the floor passes through, floored to an integer', () => {
const { warns, warn } = collect();
expect(resolveServeStallWatchdogMs('15000', warn)).toBe(15_000);
expect(resolveServeStallWatchdogMs('61000.9', warn)).toBe(61_000);
expect(warns).toEqual([]);
});
test('the floor constant is 15s and the default grace is 30s (plan #4281 contract)', () => {
expect(SERVE_STALL_FLOOR_MS).toBe(15_000);
expect(STALL_DEFAULT_GRACE_MS).toBe(30_000);
});
});
describe('installLoopStallWatchdog (handle contract)', () => {
test('non-positive / NaN stallMs returns an inert no-op handle', () => {
for (const stallMs of [0, -5, Number.NaN]) {
const h = installLoopStallWatchdog({ stallMs });
expect(h.active).toBe(false);
h.dispose(); // idempotent, no throw
}
});
test('active handle disposes idempotently without killing the test process', () => {
// Long stall so nothing can fire during the test; dispose tears down the
// worker and the pet interval.
const h = installLoopStallWatchdog({ stallMs: 60_000, graceMs: 60_000, label: 'unit-stall' });
expect(h.active).toBe(true);
h.dispose();
expect(h.active).toBe(false);
h.dispose(); // second dispose is a no-op
expect(h.active).toBe(false);
});
test('label is sanitized to a safe charset', () => {
const h = installLoopStallWatchdog({ stallMs: 60_000, label: "evil'; \n process.exit(1) //" });
expect(h.active).toBe(true);
h.dispose();
});
});
describe('progressDeadlineDecision (large-brain ceiling: never stop a progressing run)', () => {
test('waits before the deadline whatever the progress age', () => {
expect(progressDeadlineDecision(999, 1000, 10_000, 500)).toBe('wait');
});
test('past the deadline, recent progress extends', () => {
expect(progressDeadlineDecision(5000, 1000, 499, 500)).toBe('extend');
});
test('past the deadline, a full window without progress stops (inclusive)', () => {
expect(progressDeadlineDecision(5000, 1000, 500, 500)).toBe('stop');
});
test('a zero window keeps the strict wall-clock deadline', () => {
expect(progressDeadlineDecision(1000, 1000, 0, 0)).toBe('stop');
});
});