Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { authorizeJobExecution, withSubmissionAuthority } from './submission-authority.ts';
/**
* `gbrain jobs run-child` core (issue #5 — per-job process isolation).
*
* The child side of the isolation boundary. The PARENT worker owns claim,
* lock renewal, completeJob/failJob and all attempt accounting; this process
* only executes the handler and reports ONE outcome file (see
* job-isolation.ts for the protocol). Child-owns-engine: every ctx callback
* below is token-fenced, so if the job is reclaimed while we run, our writes
* degrade to no-ops.
*
* Deliberately runs NONE of the worker machinery: no health probe, no stall
* detection, no lock timer — the parent is the sole liveness owner. What it
* does install:
*
* - SIGTERM handler → fires shutdownSignal ONLY (inline signal-separation
* parity): cooperative handlers keep ctx.signal live, finish inside the
* drain window, and write their outcome before the parent escalates to a
* group SIGKILL. Handlers that watch shutdownSignal (shell) run their own
* SIGTERM→grace→SIGKILL cleanup.
* - Parent-liveness watchdog: polls `process.kill(parentPid, 0)` every 15s
* (a ppid check is DEAD CODE under tini — the child's ppid is tini,
* which outlives the worker). On parent death: abort both signals, and
* hard-exit after a 30s grace so an orphaned LLM-bound handler doesn't
* burn spend to completion. Lock expiry + the stall sweeper requeue the
* job on the parent's side of the world.
*
* The CLI layer (jobs.ts case 'run-child') owns engine.disconnect() and
* process.exit() — this module returns an exit code (engine-ownership
* invariant, same as MinionWorker).
*/
import type { BrainEngine } from '../engine.ts';
import { MinionQueue } from './queue.ts';
import type { MinionHandler } from './types.ts';
import { buildJobContext } from './job-context.ts';
import { runWithJobSpend } from './spend-authorization.ts';
import { withChatPhase } from '../ai/chat-usage.ts';
import {
JOB_CHILD_EXIT_NOT_CLAIMED,
JOB_CHILD_EXIT_RESULT_WRITE_FAILED,
WORKER_EXIT_CONFIGURATION,
} from './worker-exit-codes.ts';
import { childConfigurationError, encodeHandlerError, unrefTimer, writeChildOutcomeFile } from './job-isolation.ts';
import { isLocalConfigurationError } from './configuration-error.ts';
import { jobChildBuildVerdict } from './child-readiness.ts';
export function writeChildBootstrapError(resultPath: string, error: unknown): number {
if (!isLocalConfigurationError(error)) return 1;
try { writeChildOutcomeFile(resultPath, encodeHandlerError(error)); }
catch { return JOB_CHILD_EXIT_RESULT_WRITE_FAILED; }
return WORKER_EXIT_CONFIGURATION;
}
export interface RunChildOpts {
jobId: number;
lockToken: string;
resultPath: string;
/** Worker pid for the liveness watchdog; 0/absent disables the watchdog. */
parentPid: number;
/** #7245: the worker's build (`CHILD_PARENT_BUILD_ENV`); absent from a worker older than #7245. */
parentBuild?: string;
}
export interface RunChildInjectables {
/** Hermetic tests inject a handler map instead of registerBuiltinHandlers. */
resolveHandler: (name: string) => MinionHandler | undefined;
/** Watchdog cadence override (default 15s). */
parentPollMs?: number;
/** Orphan hard-exit grace override (default 30s). */
orphanGraceMs?: number;
/** Exit hook for tests (default process.exit for the orphan path only). */
hardExit?: (code: number) => void;
}
export async function runChildJobEntry(
engine: BrainEngine,
opts: RunChildOpts,
injectables: RunChildInjectables,
): Promise<number> {
const queue = new MinionQueue(engine);
// Ground truth is the DB row, re-read here (one SELECT) rather than a
// serialized payload: the parent claimed it, but reclaim/cancel can race
// our startup. A mismatch means we must not run the handler at all.
const job = await queue.getJob(opts.jobId);
if (!job || job.status !== 'active' || job.lock_token !== opts.lockToken) {
process.stderr.write(
`[run-child] job ${opts.jobId} is not claimed by this token ` +
`(status=${job?.status ?? 'missing'}) — exiting without running the handler\n`,
);
return JOB_CHILD_EXIT_NOT_CLAIMED;
}
const handler = injectables.resolveHandler(job.name);
if (!handler) {
// Parity with inline mode, which dead-letters a missing handler
// immediately (failJob 'dead') — 'unrecoverable' reconstructs to
// UnrecoverableError parent-side, so the parent dead-letters on attempt 1
// instead of retrying a deterministic condition (adversarial-review P3).
try {
writeChildOutcomeFile(opts.resultPath, {
outcome: 'error',
errorKind: 'unrecoverable',
message: `No handler for job type '${job.name}' in the isolation child`,
});
return 0;
} catch {
return JOB_CHILD_EXIT_RESULT_WRITE_FAILED;
}
}
const abort = new AbortController();
const shutdown = new AbortController();
// SIGTERM = worker shutdown: fire ONLY shutdownSignal, preserving the
// inline signal-separation contract (worker.ts aborts only shutdownAbort on
// SIGTERM; per-job ctx.signal stays live so cooperative handlers finish +
// report inside the drain window instead of aborting mid-deploy —
// adversarial-review P2). The parent's group SIGKILL at drain end is the
// backstop for handlers that keep running.
const onSigterm = (): void => {
if (!shutdown.signal.aborted) shutdown.abort(new Error('worker-shutdown'));
};
// Parent death (orphan) aborts BOTH: nobody will SIGKILL us, the lock will
// expire and the job will be requeued — stop the handler outright.
const onOrphaned = (): void => {
onSigterm();
if (!abort.signal.aborted) abort.abort(new Error('worker-shutdown'));
};
process.on('SIGTERM', onSigterm);
// Parent-liveness watchdog (unref'd — never keeps the child alive).
const pollMs = injectables.parentPollMs ?? 15_000;
const graceMs = injectables.orphanGraceMs ?? 30_000;
const hardExit = injectables.hardExit ?? ((code: number) => process.exit(code));
let watchdog: ReturnType<typeof setInterval> | null = null;
if (opts.parentPid > 0) {
watchdog = setInterval(() => {
try {
process.kill(opts.parentPid, 0);
} catch {
process.stderr.write(
`[run-child] parent worker (pid ${opts.parentPid}) is gone — aborting handler; ` +
`hard exit in ${Math.round(graceMs / 1000)}s\n`,
);
if (watchdog != null) clearInterval(watchdog);
onOrphaned();
const t = setTimeout(() => hardExit(1), graceMs);
unrefTimer(t);
}
}, pollMs);
unrefTimer(watchdog);
}
try {
const context = buildJobContext(
engine,
queue,
job,
opts.lockToken,
abort.signal,
shutdown.signal,
);
// #7245: a build that differs from the worker's is recorded in the job's
// history and warned once; a missing protocol or feature refuses the job.
const verdict = jobChildBuildVerdict(opts.parentBuild, opts.parentPid);
if (verdict) {
process.stderr.write(`[run-child] ${verdict.note}\n`);
try { await context.log(verdict.note); }
catch (error) { process.stderr.write(`[run-child] could not record the build skew on job ${job.id}: ${error instanceof Error ? error.message : String(error)}\n`); }
if (verdict.refused) return writeChildBootstrapError(opts.resultPath, childConfigurationError('child_protocol_incompatible'));
}
let outcome;
try {
// #4218: same phase attribution as the in-process worker path — the
// isolated child runs its own gateway, so the wrap must live here too.
const authority = await authorizeJobExecution(engine, job);
const result = await withSubmissionAuthority(authority, () => withChatPhase(`job:${job.name}`, () => runWithJobSpend(engine, job, context, handler)), abort.signal);
// completeJob's {value: x} wrap decision must run BEFORE JSON
// serialization: a JSON round-trip changes typeof for Date /
// toJSON-bearing results (object → string), which would flip the wrap
// parent-side (adversarial-review P3, result-shape parity). Wrap here;
// the parent's own wrap is then a no-op (object/undefined passthrough).
const wrapped = result != null
? (typeof result === 'object' ? (result as Record<string, unknown>) : { value: result })
: undefined;
outcome = { outcome: 'success' as const, result: wrapped };
} catch (err) {
outcome = encodeHandlerError(err);
}
try {
writeChildOutcomeFile(opts.resultPath, outcome);
} catch (writeErr) {
const msg = writeErr instanceof Error ? writeErr.message : String(writeErr);
process.stderr.write(`[run-child] failed to write outcome file: ${msg}\n`);
return JOB_CHILD_EXIT_RESULT_WRITE_FAILED;
}
return 0;
} finally {
if (watchdog != null) clearInterval(watchdog);
process.off('SIGTERM', onSigterm);
}
}