Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/** `gbrain jobs supervisor` (dispatched by runJobs in src/commands/jobs.ts). */
import { hasFlag, parseFlag, parseJobIsolationFlag, parseMaxRssFlag, parseNiceFlag, type JobsCommandContext } from './shared.ts';
import { applyNiceness, formatNice, getEffectiveNiceness } from '../../core/minions/niceness.ts';
import { resolveChildCliInvocation } from '../../core/minions/job-isolation.ts';
import { WORKER_EXIT_CONFIGURATION } from '../../core/minions/worker-exit-codes.ts';
import { writeStdoutFinal } from '../../core/cli-force-exit.ts';
import { opError } from '../../core/ops/contract.ts';
import { usageError, writeCliRefusal } from '../../cli/cli-error.ts';
import type { BrainEngine } from '../../core/engine.ts';
export async function runJobsSupervisor({ args, engine, engineOrNull }: JobsCommandContext): Promise<void> {
// Dispatcher for supervisor subcommands:
// gbrain jobs supervisor → foreground start (back-compat)
// gbrain jobs supervisor start [--detach] → foreground or detached start
// gbrain jobs supervisor status → JSON liveness + queue stats
// gbrain jobs supervisor stop → SIGTERM + drain wait
const { MinionSupervisor, DEFAULT_PID_FILE } = await import('../../core/minions/supervisor.ts');
const { writeSupervisorEvent } = await import('../../core/minions/handlers/supervisor-audit.ts');
const supCmd = args[1];
const isStatusCmd = supCmd === 'status';
const isStopCmd = supCmd === 'stop';
const isStartCmd = supCmd === 'start' || supCmd === undefined || supCmd === '--detach' ||
(typeof supCmd === 'string' && supCmd.startsWith('--'));
const jsonMode = hasFlag(args, '--json');
const pidFile = parseFlag(args, '--pid-file') ?? DEFAULT_PID_FILE;
// ----- status subcommand -----
if (isStatusCmd) {
await showSupervisorStatus({ args, engine, engineOrNull }, pidFile, jsonMode);
return;
}
// ----- stop subcommand -----
if (isStopCmd) {
await stopSupervisor(engine, args, pidFile, jsonMode);
return;
}
// ----- start subcommand (default) -----
// D2: refusals keep their stderr line; under --json the envelope is the stdout document
// (written directly: the `jobs` record has no record-wide json contract, see runJobsSupervisor's detach branch).
if (!isStartCmd) {
process.exit(writeCliRefusal(usageError(`Unknown supervisor subcommand: ${supCmd}. Expected: start, status, stop.`,
'Example: gbrain jobs supervisor start --detach --json'), 'jobs', { json: jsonMode }));
}
const config = (await import('../../core/config.ts')).loadConfig();
if (config?.engine === 'pglite') {
process.exit(writeCliRefusal(opError('config_error', 'Supervisor requires Postgres. PGLite uses an exclusive file lock that blocks other processes.',
'On PGLite, run background maintenance through `gbrain serve` (or run jobs inline with `gbrain jobs submit --follow`).'),
'jobs', { json: jsonMode, human: 'Error: Supervisor requires Postgres. PGLite uses an exclusive file lock that blocks other processes.' }));
}
const { resolveGbrainCliPath } = await import('../autopilot.ts');
const concurrency = parseInt(parseFlag(args, '--concurrency') ?? '2', 10);
const queueName = parseFlag(args, '--queue') ?? 'default';
const maxCrashes = parseInt(parseFlag(args, '--max-crashes') ?? '10', 10);
// --health-interval (supervisor): validate same as `jobs work` so NaN /
// negative / sub-1000ms typos fail-fast instead of silently disabling
// the supervisor's own health probe.
const supHealthRaw = parseFlag(args, '--health-interval');
let healthInterval = 60_000;
if (supHealthRaw !== undefined) {
const parsed = parseInt(supHealthRaw, 10);
if (!Number.isFinite(parsed) || parsed < 0) {
process.exit(writeCliRefusal(usageError(`--health-interval must be a non-negative integer (ms), got "${supHealthRaw}"`,
'Example: gbrain jobs supervisor start --health-interval 60000'), 'jobs',
{ json: jsonMode, human: `Error: --health-interval must be a non-negative integer (ms), got "${supHealthRaw}"` }));
}
if (parsed > 0 && parsed < 1000) {
const msg = `--health-interval ${parsed} is suspiciously low (likely a unit-confusion typo). ` +
`The flag takes milliseconds; for 60-second probes pass 60000. Use 0 to disable.`;
process.exit(writeCliRefusal(usageError(msg, 'Example: gbrain jobs supervisor start --health-interval 60000'), 'jobs', { json: jsonMode, human: `Error: ${msg}` }));
}
healthInterval = parsed;
}
const allowShellJobs = hasFlag(args, '--allow-shell-jobs') ||
process.env.GBRAIN_ALLOW_SHELL_JOBS === '1'; // same literal the shell handler checks
const detach = hasFlag(args, '--detach');
// Supervisor's --max-rss: explicit wins; absent → cgroup-aware auto-size
// (issue #1678). The supervisor is the main production path, so the
// watchdog is on by default — but at a realistic, RAM-relative cap
// instead of the old flat 2048MB footgun.
const { resolveDefaultMaxRssMb: resolveSupMaxRss } =
await import('../../core/minions/rss-default.ts');
const maxRssMb = parseMaxRssFlag(args) ?? resolveSupMaxRss();
// --nice N (issue #1815): validated here (fail-fast on bad input even for
// --detach), but APPLIED only in the foreground-start path below — applying
// before the --detach branch would renice the throwaway parent that forks
// and exits, not the long-lived re-exec'd child (Codex #1).
const supNice = parseNiceFlag(args);
const explicitCliPath = parseFlag(args, '--cli-path');
const workerInvocation = explicitCliPath
? { cmd: explicitCliPath, argsPrefix: [] }
: resolveChildCliInvocation({}, process.execPath, process.argv[1], resolveGbrainCliPath);
if (!workerInvocation) {
writeCliRefusal(opError('config_error', 'Could not resolve this worker installation. Repair the CLI or pass --cli-path.',
'Pass --cli-path <path to the gbrain CLI>, or reinstall gbrain.'), 'jobs', { json: jsonMode });
process.exit(WORKER_EXIT_CONFIGURATION);
}
// --detach: fork a background supervisor, print PID payload, exit 0.
// D2: the payload is the one stdout document (writeStdoutFinal works without
// the guard; the `jobs` record declares no record-wide json because other
// jobs subcommands stream). status names the outcome; stderr_log is the
// detached child's log.
// #4418: the child gets a DURABLE stderr sink (audit-dir log, null-device
// fallback) instead of inheriting the invoker's stderr — an inherited
// capture pipe closing killed the worker (SIGPIPE 141) and then the
// supervisor itself on their next stderr write. See detached-stderr.ts.
if (detach) {
const { spawnDetachedSupervisor } = await import('../../core/minions/detached-stderr.ts');
const started = spawnDetachedSupervisor(
process.execPath,
process.argv[1],
process.argv.slice(2).filter(a => a !== '--detach'),
);
if (started.pid === undefined) {
process.exit(writeCliRefusal(opError('unavailable', 'The detached supervisor process could not be started.',
`Run \`gbrain jobs supervisor start\` in the foreground to see the error${started.stderrPath ? `, or read ${started.stderrPath}` : ''}.`),
'jobs', { json: true }));
}
const payload = {
event: 'started',
status: 'started',
supervisor_pid: started.pid,
pid_file: pidFile,
detached: true,
...(started.stderrPath ? { stderr_log: started.stderrPath } : {}),
};
await writeStdoutFinal(`${JSON.stringify(payload)}\n`);
process.exit(0);
}
// Foreground start. Renice THIS process (the long-lived supervisor) now,
// after the --detach fork-and-exit branch (Codex #1). The worker inherits
// it via the spawn env; the supervisor also passes `--nice` down so the
// worker re-applies it (see buildWorkerArgs).
const supervisorPid = process.pid;
let supNiceResult: ReturnType<typeof applyNiceness> | undefined;
if (supNice !== undefined) {
supNiceResult = applyNiceness(supNice);
if (!supNiceResult.applied) {
console.error(
`[gbrain jobs] could not set supervisor niceness to ${supNice}: ${supNiceResult.error ?? 'unknown'}. ` +
`Negative nice needs privilege; running at niceness ${supNiceResult.effective ?? 'unchanged'}.`,
);
}
}
const supervisor = new MinionSupervisor(engine, {
concurrency,
queue: queueName,
pidFile,
maxCrashes,
healthInterval,
cliPath: workerInvocation.cmd,
cliArgsPrefix: workerInvocation.argsPrefix,
allowShellJobs,
json: jsonMode,
maxRssMb,
jobIsolation: parseJobIsolationFlag(args),
...(supNice !== undefined ? { nice_requested: supNice } : {}),
...(supNiceResult?.effective != null ? { nice_effective: supNiceResult.effective } : {}),
...(supNiceResult?.error ? { nice_error: supNiceResult.error } : {}),
onEvent: (emission) => writeSupervisorEvent(emission, supervisorPid),
});
await supervisor.start();
}
/** `gbrain jobs supervisor status`: liveness + queue stats; always exits the process. */
async function showSupervisorStatus(
{ args, engine, engineOrNull }: Pick<JobsCommandContext, 'args' | 'engine' | 'engineOrNull'>,
pidFile: string,
jsonMode: boolean,
): Promise<void> {
const { readSupervisorEvents, summarizeCrashes } = await import('../../core/minions/handlers/supervisor-audit.ts');
const { readSupervisorPid } = await import('../../core/minions/supervisor-pid.ts');
const { readWorkers, workerRegistryReadFailure } = await import('../../core/minions/worker-registry.ts');
const pidStatus = readSupervisorPid(pidFile);
const supervisorPid = pidStatus.pid;
const pidfileRunning = pidStatus.running;
const events = readSupervisorEvents({ sinceMs: 24 * 60 * 60 * 1000 });
const lastStart = events.filter(e => e.event === 'started').pop()?.ts ?? null;
// issue #2227 fix #1/#3: the pidfile is HOME-derived, so a supervisor
// started under a different $HOME (keeper=/root vs ops=/data) reads as
// "not running" here even when it is healthy — the false signal that
// makes an operator spawn a duplicate. Fall back to the queue-scoped DB
// singleton lock (#1849), the HOME-independent authority. PID-reuse-safe:
// isLockHolderLive keys on lock freshness, never process.kill.
const supQueue = parseFlag(args, '--queue') ?? 'default';
let detectedViaDbLock = false;
let dbLockHolder: { holder_pid: number; holder_host: string } | null = null;
if (!pidfileRunning && engineOrNull) {
try {
const { inspectLock, isLockHolderLive } = await import('../../core/db-lock.ts');
const { supervisorLockId, SUPERVISOR_LOCK_TTL_MIN } = await import('../../core/minions/supervisor.ts');
const snap = await inspectLock(engine, supervisorLockId(supQueue));
if (snap && isLockHolderLive(snap, SUPERVISOR_LOCK_TTL_MIN)) {
detectedViaDbLock = true;
dbLockHolder = { holder_pid: snap.holder_pid, holder_host: snap.holder_host };
}
} catch {
// Pre-migration brains / transient DB errors: fall back to pidfile-only.
}
}
const running = pidfileRunning || detectedViaDbLock;
// Surface the supervisor's recorded config from the latest `started`
// event (concurrency + effective --max-rss) so split-$HOME deployments
// see what the live-but-pidfile-invisible supervisor is running.
const startedEvt = events.filter(e => e.event === 'started').pop() ?? null;
// Shared classifier — same code path runs in `gbrain doctor` so the
// two surfaces cannot drift on what counts as a crash. Supersedes
// v0.35.4.0's binary `classifyWorkerExit({code})` on this surface;
// see doctor.ts for the layering rationale.
const summary = summarizeCrashes(events);
const maxCrashesEvent = events.filter(e => e.event === 'max_crashes_exceeded').pop() ?? null;
// Niceness (issue #1815): measure live workers + the supervisor itself.
// workers is null when the registry is unreadable: liveness unknown, not zero.
let workers: Array<{ pid: number; queue: string; nice_requested: number | null; nice: number | null }> | null = null;
let workerRegistryError: { errno: string; path: string | null } | null = null;
try {
workers = readWorkers().map(w => ({
pid: w.pid,
queue: w.queue,
nice_requested: w.nice_requested,
nice: w.nice_now,
}));
} catch (e) {
workerRegistryError = workerRegistryReadFailure(e);
}
const supervisorNice = pidfileRunning && supervisorPid !== null
? getEffectiveNiceness(supervisorPid)
: null;
const { readOwnerProcessingStatus } = await import('../../core/minions/processing-state.ts');
const { resolve: resolvePidPath } = await import('node:path');
const processing = pidfileRunning && supervisorPid !== null
? readOwnerProcessingStatus('supervisor', resolvePidPath(pidFile), supervisorPid)
: null;
const status = {
running,
processing_ready: processing?.processing_ready ?? false,
processing_state: processing?.processing_state ?? (running ? 'unknown' : 'stopped'),
processing_stage: processing?.processing_stage ?? (running ? 'unknown' : 'stopped'),
retry_at: processing?.retry_at ?? null,
reason_code: processing?.reason_code ?? null,
blocked_since: processing?.blocked_since ?? null,
worker_identity: processing?.worker_identity ?? null,
child_identity: processing?.child_identity ?? null,
detected_via: detectedViaDbLock ? 'db_lock' : (pidfileRunning ? 'pidfile' : null),
supervisor_pid: supervisorPid ?? dbLockHolder?.holder_pid ?? null,
db_lock_holder: dbLockHolder,
pid_file: pidFile,
queue: supQueue,
last_start: lastStart,
concurrency: typeof startedEvt?.concurrency === 'number' ? startedEvt.concurrency : null,
max_rss_mb: typeof startedEvt?.max_rss_mb === 'number' ? startedEvt.max_rss_mb : null,
crashes_24h: summary.total,
clean_exits_24h: summary.clean_exits,
crashes_by_cause: summary.by_cause,
max_crashes_exceeded: !!maxCrashesEvent,
nice: supervisorNice,
workers,
...(workerRegistryError ? { worker_registry_unreadable: true, worker_registry_errno: workerRegistryError.errno, worker_registry_path: workerRegistryError.path } : {}),
};
if (jsonMode) {
console.log(JSON.stringify(status, null, 2));
} else {
const via = detectedViaDbLock ? ' (detected via DB lock; pidfile not found at the configured path)' : '';
console.log(`Supervisor: ${running ? 'running' : 'not running'}${via}`);
console.log(` Processing: ${status.processing_state}`);
console.log(` Stage: ${status.processing_stage}${status.retry_at ? `; next retry ${status.retry_at}` : ''}`);
if (status.processing_state === 'configuration_blocked') {
console.log(` Repair: ${status.reason_code ?? 'local configuration failure'}; repair the installation and restart the supervisor. See docs/guides/minions-fix.md.`);
}
if (status.supervisor_pid) console.log(` PID: ${status.supervisor_pid}${detectedViaDbLock ? ` @ ${dbLockHolder?.holder_host}` : ''}`);
console.log(` PID file: ${pidFile}`);
if (detectedViaDbLock && status.concurrency !== null) console.log(` Concurrency: ${status.concurrency}${status.max_rss_mb !== null ? ` (max-rss ${status.max_rss_mb}MB)` : ''}`);
if (lastStart) console.log(` Last start: ${lastStart}`);
console.log(` Crashes (24h): ${summary.total} (runtime=${summary.by_cause.runtime_error} oom=${summary.by_cause.oom_or_external_kill} unknown=${summary.by_cause.unknown} legacy=${summary.by_cause.legacy})`);
console.log(` Clean exits (24h): ${summary.clean_exits}`);
if (supervisorNice !== null) console.log(` Nice (supervisor): ${formatNice(supervisorNice)}`);
if (workerRegistryError) {
console.log(` Workers: unknown (worker registry unreadable: ${workerRegistryError.errno}${workerRegistryError.path ? ` at ${workerRegistryError.path}` : ''})`);
}
for (const w of workers ?? []) {
const req = w.nice_requested !== null && w.nice !== null && w.nice_requested !== w.nice
? ` (requested ${formatNice(w.nice_requested)})` : '';
console.log(` Worker pid ${w.pid} [${w.queue}]: nice ${w.nice !== null ? formatNice(w.nice) : '?'}${req}`);
}
if (maxCrashesEvent) console.log(` ⚠ Max crashes exceeded at ${maxCrashesEvent.ts}`);
}
process.exit(running ? 0 : 1);
}
/** #5062: supervisor's own 35s worker drain + 5s SIGKILL confirmation + slack. */
const STOP_WAIT_MS = 45_000;
/**
* `gbrain jobs supervisor stop`: SIGTERM, wait for the exit, then verify the
* stop before calling it drained; always exits the process. #5062: "the pid
* is gone" is not enough. Success also needs the supervisor's own `stopped`
* audit row (after its `shutting_down`) saying the worker drained, every
* worker it spawned gone, and its queue lock row released. Anything less is
* reported with the failing check and exits 1. W9F item 7: the audit rows are
* the run's own (from its `started` row, searched back across weekly files);
* a worker is live only when its pid is alive with the start time recorded at
* spawn; a run with no `started` row on record is `unverified`, never
* `drained`; and a PID file whose pid now belongs to a different process
* (start time mismatch) is reported `stale_pid_file` without signaling it.
*/
async function stopSupervisor(engine: BrainEngine, args: string[], pidFile: string, jsonMode: boolean): Promise<void> {
const { existsSync, readFileSync } = await import('fs');
const { hostname } = await import('os');
const { readSupervisorRun } = await import('../../core/minions/handlers/supervisor-audit.ts');
const { supervisorLockId } = await import('../../core/minions/supervisor.ts');
const { inspectLock } = await import('../../core/db-lock.ts');
const { processStartTime } = await import('../../core/pglite-lock.ts');
const report = (payload: Record<string, unknown>, human: string, ok: boolean): never => {
if (jsonMode) console.log(JSON.stringify(payload));
else if (ok) console.log(human);
else console.error(human);
process.exit(ok ? 0 : 1);
};
const isAlive = (pid: number): boolean => {
try { process.kill(pid, 0); return true; } catch (e) { return (e as NodeJS.ErrnoException).code === 'EPERM'; }
};
// W9F item 7: a live pid whose kernel start time differs from the recorded
// one is a different (recycled) process. Unknown start times compare equal.
const sameProcess = (pid: number, recordedStart: unknown): boolean => {
if (!isAlive(pid)) return false;
const current = processStartTime(pid);
return typeof recordedStart !== 'string' || current === null || current === recordedStart;
};
if (!existsSync(pidFile)) {
report({ stopped: false, reason: 'pid_file_missing', pid_file: pidFile }, `No PID file at ${pidFile}; supervisor not running.`, false);
}
let supervisorPid: number;
let recordedStart: string | null;
try {
const [pidLine, startLine] = readFileSync(pidFile, 'utf8').trim().split('\n');
supervisorPid = parseInt(pidLine, 10);
if (isNaN(supervisorPid) || supervisorPid <= 0) throw new Error('invalid pid');
recordedStart = /^\d+$/.test(startLine?.trim() ?? '') ? startLine!.trim() : null;
} catch (err) {
return report({ stopped: false, reason: 'pid_file_corrupt', error: String(err) }, `PID file corrupt: ${err}`, false);
}
const before = readSupervisorRun(supervisorPid, { supervisorStart: recordedStart });
const supervisorStart = recordedStart ?? before.started?.supervisor_start;
if (isAlive(supervisorPid) && !sameProcess(supervisorPid, supervisorStart)) {
report({ stopped: false, reason: 'stale_pid_file', supervisor_pid: supervisorPid, pid_file: pidFile },
`PID file ${pidFile} names pid ${supervisorPid}, which now belongs to a different process; the supervisor is not running and nothing was signaled.`, true);
}
const queue = (typeof before.started?.queue === 'string' ? before.started.queue : undefined) ?? parseFlag(args, '--queue') ?? 'default';
try { process.kill(supervisorPid, 'SIGTERM'); }
catch (err: unknown) {
const code = (err as NodeJS.ErrnoException)?.code;
const payload = {
stopped: false,
reason: code === 'ESRCH' ? 'process_gone' : 'kill_failed',
supervisor_pid: supervisorPid,
};
if (jsonMode) console.log(JSON.stringify(payload));
else console.error(`Cannot signal PID ${supervisorPid}: ${err}`);
process.exit(code === 'ESRCH' ? 0 : 1);
}
const deadline = Date.now() + STOP_WAIT_MS;
while (Date.now() < deadline && isAlive(supervisorPid)) {
await new Promise(r => setTimeout(r, 250));
}
if (isAlive(supervisorPid)) {
report({ stopped: false, supervisor_pid: supervisorPid, reason: 'timeout', waited_ms: STOP_WAIT_MS },
`Supervisor ${supervisorPid} did not exit within ${STOP_WAIT_MS / 1000}s.`, false);
}
const run = readSupervisorRun(supervisorPid, { supervisorStart: typeof supervisorStart === 'string' ? supervisorStart : null });
if (!run.started) {
report({ stopped: true, supervisor_pid: supervisorPid, queue, reason: 'unverified', drained: false, audit_files: run.files },
`Supervisor ${supervisorPid} exited, but its audit trail has no record of the run (missing, pruned or corrupt; searched ${run.files.length ? run.files.join(', ') : 'no existing audit files'}), so the drain cannot be verified. ` +
'Inspect in-flight jobs with `gbrain jobs list --status active`.', false);
}
const events = run.events;
const shuttingIdx = events.map(e => e.event).lastIndexOf('shutting_down');
const stoppedEvt = shuttingIdx >= 0 ? events.slice(shuttingIdx + 1).find(e => e.event === 'stopped') : undefined;
// The live set: spawns with no later exit. An exit row carries the child's
// pid (null: no process was created); a row written before exits carried
// one (no `pid` key) pairs with the oldest open spawn.
const open: Array<{ pid: number; start: unknown }> = [];
let legacyPairing = false;
for (const e of events) {
if (e.event === 'worker_spawned' && typeof e.pid === 'number') open.push({ pid: e.pid, start: e.pid_start });
if (e.event !== 'worker_exited' || e.pid === null) continue;
if (!('pid' in e)) legacyPairing = true;
const idx = typeof e.pid === 'number' ? open.findIndex(w => w.pid === e.pid) : 0;
if (idx >= 0) open.splice(idx, 1);
}
const liveWorkers = open.filter(w => sameProcess(w.pid, w.start)).map(w => w.pid);
let lockReleased: boolean | null;
try {
const snap = await inspectLock(engine, supervisorLockId(queue));
lockReleased = !(snap && snap.holder_pid === supervisorPid && snap.holder_host === hostname());
} catch {
lockReleased = null;
}
const checks = {
process_exited: true,
stopped_event: stoppedEvt !== undefined,
worker_drained: stoppedEvt?.drained === true,
workers_exited: liveWorkers.length === 0,
lock_released: lockReleased,
};
const reason = !checks.stopped_event ? 'no_stopped_event'
: !checks.workers_exited ? 'worker_still_running'
: lockReleased === null ? 'lock_unverified'
: !lockReleased ? 'lock_still_held'
: !checks.worker_drained ? 'forced'
: 'drained';
const drained = reason === 'drained';
const human = drained
? `Supervisor ${supervisorPid} stopped; worker drained.`
: `Supervisor ${supervisorPid} exited but the stop is not verified as drained (${reason}): ` +
`${JSON.stringify(checks)}${liveWorkers.length > 0 ? `; live worker pid(s): ${liveWorkers.join(', ')}` : ''}. ` +
'Inspect in-flight jobs with `gbrain jobs list --status active`.';
if (legacyPairing) console.error('Warning: audit rows written before worker exits carried a pid were paired with spawns by order.');
report({ stopped: true, supervisor_pid: supervisorPid, queue, reason, drained, checks,
...(liveWorkers.length > 0 ? { live_worker_pids: liveWorkers } : {}),
...(legacyPairing ? { warnings: ['legacy_exit_pairing'] } : {}) }, human, drained);
}