Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.9 KB
Raw Blame
/** `gbrain jobs work` (dispatched by runJobs in src/commands/jobs.ts). */
import { hasFlag, parseFlag, parseJobIsolationFlag, parseMaxRssFlag, parseNiceFlag, resolveWorkerConcurrency, type JobsCommandContext } from './shared.ts';
import { applyNiceness, formatNice } from '../../core/minions/niceness.ts';
import { resolveChildCliInvocation } from '../../core/minions/job-isolation.ts';
import { checkWorkerStartup, reportWorkerConfiguration, reportWorkerReady, reportWorkerStarting } from '../jobs-readiness.ts';
import { LocalConfigurationError, isLocalConfigurationError } from '../../core/minions/configuration-error.ts';
import { WORKER_EXIT_CONFIGURATION, WORKER_EXIT_DRAIN_FORCED, WORKER_EXIT_RSS_WATCHDOG } from '../../core/minions/worker-exit-codes.ts';
import { registerSignalOwner } from '../../core/process-cleanup.ts';
import { MinionWorker, type UnhealthyReason } from '../../core/minions/worker.ts';
import type { MinionQueue } from '../../core/minions/queue.ts';
import type { BrainEngine } from '../../core/engine.ts';
import { loadConfig } from '../../core/config.ts';
import { startPersistenceConsumer } from '../../core/persistence/service.ts';
import { describeLanePolicy, resolveLanePolicy } from '../../core/minions/lanes.ts';
import { drainWakesUntilIdle } from '../../core/wakeshift/drain.ts';

export async function maybeRunWorkerStartupRecovery(
  queue: MinionQueue,
  env: NodeJS.ProcessEnv = process.env,
  readinessVerified = false,
): Promise<void> {
  if (env.GBRAIN_SUPERVISED === '1' && !readinessVerified) return;
  let deadline: ReturnType<typeof setTimeout> | undefined;
  try {
    const recovered = await Promise.race([
      queue.reconcileOrphanedPrivateQueues({
        reason: 'worker startup recovery: orphaned dream-inline private queue',
      }),
      new Promise<never>((_, reject) => {
        deadline = setTimeout(() => reject(new Error('private-queue startup recovery did not settle within 30 seconds; recovery is unconfirmed')), 30_000);
      }),
    ]);
    if (recovered.cancelled_jobs > 0) {
      console.error(
        `[gbrain jobs] private-queue startup recovery: cancelled ${recovered.cancelled_jobs} ` +
        `job(s) across ${recovered.cancelled_queues} orphaned queue(s)`,
      );
    }
  } catch (e) {
    if (isLocalConfigurationError(e)) throw e;
    console.error(`[gbrain jobs] private-queue startup recovery failed: ${e instanceof Error ? e.message : String(e)}`);
  } finally {
    if (deadline) clearTimeout(deadline);
  }
}

export async function runJobsWork({ args, engine, queue }: JobsCommandContext): Promise<void> {
  // Lazy: jobs.ts imports this module statically, so a static import back would be a cycle.
  const { registerBuiltinHandlers } = await import('../jobs.ts');
  // --allow-shell-jobs (supervisor pass-through, see buildWorkerArgs): the
  // startup cwd-.env quarantine drops GBRAIN_ALLOW_SHELL_JOBS when a .env
  // in this worker's cwd assigns it, so the flag re-asserts the operator's
  // opt-in AFTER preflight. Read sites keep checking the env var.
  if (hasFlag(args, '--allow-shell-jobs')) process.env.GBRAIN_ALLOW_SHELL_JOBS = '1';

  const queueName = parseFlag(args, '--queue') ?? 'default';
  if (engine.kind === 'pglite') {
    await drainPgliteQueue(engine, queue, queueName, registerBuiltinHandlers);
    // Wakeshift (plan D8): deliver the wakes this drain's jobs owe before the process exits; a no-op while off.
    await drainWakesUntilIdle(engine).catch(e => console.error(`[gbrain jobs] wake drain failed: ${e instanceof Error ? e.message : String(e)}`));
    return;
  }
  const concurrency = resolveWorkerConcurrency(args);
  // --max-rss: explicit value wins (including 0 to disable the watchdog).
  // Absent → cgroup-aware auto-size (issue #1678): the flat 2048MB default
  // killed legit embed work (~10GB) on every cycle and produced a silent
  // ~400×/24h respawn loop. See src/core/minions/rss-default.ts.
  const maxRssExplicit = parseMaxRssFlag(args);
  const { resolveDefaultMaxRssMb, describeDefaultMaxRss } =
    await import('../../core/minions/rss-default.ts');
  const maxRssMb = maxRssExplicit ?? resolveDefaultMaxRssMb();

  // --health-interval: self-health-check period in ms. 0 disables. Default: 60_000 (60s).
  // Provides DB liveness probes + stall detection for bare workers.
  // Automatically skipped when running under a supervisor (GBRAIN_SUPERVISED=1).
  // Validated aggressively (parity with --max-rss): reject NaN/negative/non-integer
  // values, and reject suspicious sub-1000ms values that are likely a unit-confusion
  // typo (e.g. "--health-interval 60" thinking the unit is seconds).
  const healthRaw = parseFlag(args, '--health-interval');
  let healthCheckInterval = 60_000;
  if (healthRaw !== undefined) {
    const parsed = parseInt(healthRaw, 10);
    if (!Number.isFinite(parsed) || parsed < 0) {
      console.error(`Error: --health-interval must be a non-negative integer (ms), got "${healthRaw}"`);
      process.exit(1);
    }
    if (parsed > 0 && parsed < 1000) {
      console.error(
        `Error: --health-interval ${parsed} is suspiciously low (likely a unit-confusion typo). ` +
        `The flag takes milliseconds; for 60-second probes pass 60000. Use 0 to disable.`,
      );
      process.exit(1);
    }
    healthCheckInterval = parsed;
  }

  // --nice N (issue #1815): renice this worker process so background work
  // yields CPU to foreground tasks without sacrificing concurrency. Applied
  // at the CLI layer (worker.ts stays embeddable). Niceness inherits to the
  // worker's spawned children (shell jobs / subagents) automatically.
  const niceVal = parseNiceFlag(args);
  let niceResult: ReturnType<typeof applyNiceness> | undefined;
  if (niceVal !== undefined) {
    niceResult = applyNiceness(niceVal);
    if (!niceResult.applied) {
      console.error(
        `[gbrain jobs] could not set niceness to ${niceVal}: ${niceResult.error ?? 'unknown'}. ` +
        `Negative nice needs privilege; running at niceness ${niceResult.effective ?? 'unchanged'}.`,
      );
    }
  }

  // issue #5: per-job process isolation. Resolve + validate the child CLI
  // invocation ONCE at startup and refuse to start on failure — a bad
  // path discovered per-job would release every claim as infra failures
  // (never dead-lettering, but never progressing either).
  const jobIsolation = parseJobIsolationFlag(args);
  let childCliInvocation: { cmd: string; argsPrefix: string[] } | null = null;
  let childTiniPath = '';
  if (jobIsolation === 'process') {
    const { resolveGbrainCliPath } = await import('../autopilot.ts');
    const inv = resolveChildCliInvocation(
      process.env,
      process.execPath,
      process.argv[1],
      () => resolveGbrainCliPath(),
    );
    if (!inv) {
      reportWorkerConfiguration(new LocalConfigurationError('child_executable_invalid', 'No child CLI could be resolved.'));
      process.exit(WORKER_EXIT_CONFIGURATION);
    }
    // Canonicalize BEFORE validating: existsSync on a relative name checks
    // cwd while spawn() resolves via PATH — the validated file and the
    // executed binary could differ (security review). Resolving to an
    // absolute path makes the fail-fast check and the spawn agree.
    const { existsSync: childCliExists } = await import('node:fs');
    const { resolve: resolveCliPath } = await import('node:path');
    inv.cmd = resolveCliPath(inv.cmd);
    if (!childCliExists(inv.cmd)) {
      reportWorkerConfiguration(new LocalConfigurationError('child_executable_invalid', 'The selected child CLI does not exist.'));
      process.exit(WORKER_EXIT_CONFIGURATION);
    }
    childCliInvocation = inv;
    const { detectTini } = await import('../../core/minions/spawn-helpers.ts');
    childTiniPath = detectTini();
    if (maxRssMb > 0) {
      console.error(
        `[gbrain jobs] note: with process isolation on, --max-rss ${maxRssMb} MB applies to the worker and to each ` +
        'job child\'s process tree separately; a child past it is stopped and its attempt fails. ' +
        'Size host memory for (concurrency + 1) x the cap.',
      );
    }
  }

  let childIdentity: Awaited<ReturnType<typeof checkWorkerStartup>>;
  try {
    childIdentity = await checkWorkerStartup(engine, childCliInvocation, childTiniPath);
  } catch (error) {
    if (isLocalConfigurationError(error)) {
      reportWorkerConfiguration(error);
      process.exit(WORKER_EXIT_CONFIGURATION);
    }
    console.error('[health] Worker readiness is temporarily unavailable; no jobs were admitted. The supervisor will retry.');
    process.exit(1);
  }

  try {
    await queue.ensureSchema();
    await maybeRunWorkerStartupRecovery(queue, process.env, true);
  } catch (error) {
    if (isLocalConfigurationError(error)) {
      reportWorkerConfiguration(error);
      process.exit(WORKER_EXIT_CONFIGURATION);
    }
    console.error('[gbrain jobs] Worker schema readiness failed; no jobs were admitted. Inspect doctor before retrying.');
    process.exit(1);
  }

  // issue #6: the direct-pool kill switch collapses lock renewal, health
  // probes, and handler workload onto ONE shared pool — silently. Make
  // the collapse loud at startup so a later 'pool_starved' incident has
  // an obvious prior warning instead of a mystery.
  {
    const { getConnectionRouting } = await import('../../core/minions/db-probe.ts');
    const cm = getConnectionRouting(engine);
    if (cm?.isDualPoolActive && !cm.isDualPoolActive()) {
      const killSwitched = cm.describeMode?.().kill_switch_active === true;
      console.error(
        `[gbrain jobs] single-pool mode: lock renewal, health probes and handler workload share ` +
        `one connection pool${killSwitched ? ' (direct-lane kill switch is active)' : ''}. ` +
        `Under heavy handler load this pool can starve the lock heartbeat. For Supabase brains, ` +
        `ensure the direct (5432) host is reachable or set GBRAIN_DIRECT_DATABASE_URL.`,
      );
    }
  }

  const { resolveTerminalHook, terminalHookNotifier } = await import('../../core/minions/terminal-hook.ts');
  const terminalHook = resolveTerminalHook(loadConfig());
  const lanes = await resolveLanePolicy(engine);
  const worker = new MinionWorker(engine, {
    queue: queueName, concurrency, maxRssMb, healthCheckInterval,
    jobIsolation, childCliInvocation, childTiniPath,
    longJobNames: lanes.longJobNames, shortJobReservedSlots: lanes.shortJobReservedSlots,
    ...(terminalHook ? { onJobSettled: terminalHookNotifier(engine, terminalHook).onJobSettled } : {}),
  });
  try {
    await registerBuiltinHandlers(worker, engine);
  } catch (error) {
    if (!isLocalConfigurationError(error)) throw error;
    reportWorkerConfiguration(error);
    process.exit(WORKER_EXIT_CONFIGURATION);
  }

  // Subscribe to self-health failures emitted by the worker. Library code
  // (worker.ts) never calls process.exit directly so it stays embeddable;
  // this CLI layer is the right place to terminate the process and let
  // the external PM (systemd, Docker, cron watchdog) restart cleanly.
  worker.on('unhealthy', exitOnUnhealthy);

  const isSupervisedChild = process.env.GBRAIN_SUPERVISED === '1';
  let watchdogNote = '';
  if (maxRssMb > 0) {
    if (maxRssExplicit !== undefined) {
      watchdogNote = `, watchdog: ${maxRssMb}MB (explicit)`;
    } else {
      const d = describeDefaultMaxRss();
      watchdogNote = `, watchdog: ${maxRssMb}MB (auto-sized from ${Math.round(d.basisMb / 1024)}GB ${d.source} RAM)`;
    }
  }
  // issue #1801 (fix #2): the DB-liveness probe runs under supervision too;
  // only stall detection is supervised-off. Report accordingly.
  const healthNote = healthCheckInterval > 0
    ? (isSupervisedChild
        ? `, db-probe: ${Math.round(healthCheckInterval / 1000)}s`
        : `, health-check: ${Math.round(healthCheckInterval / 1000)}s`)
    : '';
  const niceNote = niceResult ? `, nice: ${formatNice(niceResult.effective ?? niceVal!)}` : '';
  const isolationNote = jobIsolation === 'process'
    ? `, isolation: process (child cli: ${childCliInvocation?.cmd}${childTiniPath ? ', tini' : ''})`
    : '';
  const hookNote = terminalHook ? `, terminal hook: ${terminalHook.argv[0]}` : '';
  console.log(`Minion worker started (queue: ${queueName}, concurrency: ${concurrency}, ${describeLanePolicy(concurrency, lanes.shortJobReservedSlots)}${watchdogNote}${healthNote}${niceNote}${isolationNote}${hookNote})`);
  console.log(`Registered handlers: ${worker.registeredNames.join(', ')}`);

  // Register in the live worker registry (issue #1815) so jobs stats / doctor
  // can report this worker's effective niceness. Cleanup runs on BOTH the
  // finally below AND process.on('exit') — the unhealthy handler's
  // process.exit(1) bypasses the awaited finally (Codex #10).
  const { registerWorker } = await import('../../core/minions/worker-registry.ts');
  const unregisterWorker = registerWorker({
    pid: process.pid,
    queue: queueName,
    nice_requested: niceVal ?? null,
    nice_effective: niceResult ? niceResult.effective : null,
    started_at: Date.now(),
  });
  process.on('exit', () => unregisterWorker());

  // #5062: own SIGTERM/SIGHUP/broken-pipe so the generic CLI cleanup pass
  // cannot exit(143) ahead of the drain and orphan in-flight jobs.
  const releaseSignalOwner = registerSignalOwner('jobs-work', () => worker.requestShutdown());

  // GBRA-133 PR 2a.4: the worker is a resident; its notifier syncs this host's sources on commit (sync.on_commit) and
  // drains wanted markers. One owner per host (the notifier lease); it never delays the first claim.
  const { startResidentNotifier } = await import('../../core/notifier/resident.ts');
  const notifier = startResidentNotifier(engine, { resident: 'jobs-worker', log: line => console.error(line) });
  // GBRA-133 2a.4: a queued sync job wakes the idle claim loop (LISTEN on a direct session, a short poll otherwise).
  const { startSyncQueueWake } = await import('../../core/minions/sync-queue-wake.ts');
  const syncQueueWake = await startSyncQueueWake(engine, worker.idleWaiter, { queue: queueName });
  startWorkerPersistenceConsumer(engine);

  try {
    reportWorkerStarting('worker_startup');
    worker.once('ready', () => reportWorkerReady(childIdentity));
    await worker.start();
  } finally {
    await notifier?.stop().catch(() => undefined);
    unregisterWorker();
    releaseSignalOwner();
    await syncQueueWake.stop();
    if (worker.configurationError) {
      const releases = worker.configurationReleaseResults;
      const unconfirmed = releases.filter(result => result.outcome === 'unconfirmed').length;
      console.error(`[health] Configuration shutdown settled ${releases.length} claim(s); ${unconfirmed} release(s) unconfirmed.`);
      if (unconfirmed > 0) {
        console.error('[health] release-unconfirmed: execution or database release could not be confirmed. Lease expiry may consume stall budget or allow duplicate side effects; inspect affected jobs before retrying.');
      }
    }
    // Release the DB connection pool immediately on shutdown so
    // PgBouncer slots are freed rather than waiting for TCP keepalive
    // (~minutes). Disconnect failure is best-effort but logged loudly:
    // a silent shutdown disconnect error is exactly the bug class the
    // v0.26.9 D14 direction (isUndefinedColumnError, oauth-provider)
    // was created to surface. The CLI is the engine owner here, not
    // the worker — keeping disconnect at this layer preserves the
    // "engine ownership stays with the creator" invariant that broke
    // tests in earlier waves of this branch.
    try { await engine.disconnect(); }
    catch (e) { console.error('[gbrain jobs work] engine disconnect failed during shutdown:', e); }

    if (worker.configurationError) {
      process.exit(WORKER_EXIT_CONFIGURATION);
    }

    // If the RSS watchdog (not a normal SIGTERM) drained the worker, exit
    // with the distinct WORKER_EXIT_RSS_WATCHDOG code so the supervisor
    // classifies the drain as `rss_watchdog` (cause-keyed backoff + loud
    // alert) instead of a silent `clean_exit`. The worker exposes the
    // intent; the CLI owns process.exit (same ownership boundary as the
    // engine-disconnect above). Explicit process.exit also guarantees the
    // code even if a lingering handle would otherwise keep the process
    // alive past natural exit (issue #1678, Codex #7).
    if (worker.rssWatchdogTriggered) {
      process.exit(WORKER_EXIT_RSS_WATCHDOG);
    }
    // #5062: claims were handed back while their handlers still ran; exit
    // now (a running handler would otherwise keep the process alive) with a
    // code that says the drain did not complete.
    if (worker.drainForced) {
      process.exit(WORKER_EXIT_DRAIN_FORCED);
    }
  }
}

/**
 * #7002: a resident worker joins the persistence-consumer election at boot (Postgres only; this function runs after the
 * PGLite branch returned). Under the existing election it starts waiter-only beside a live owner and promotes itself
 * when that owner lapses, so queued effects drain after a serve stops even while no job runs. persistence.single_consumer
 * keeps its meaning (service.ts is unchanged). engine.disconnect() stops it through registerBeforeDisconnect.
 */
export function startWorkerPersistenceConsumer(engine: BrainEngine, log: (line: string) => void = (line) => console.error(line)): boolean {
  try {
    startPersistenceConsumer(engine, loadConfig() ?? { engine: engine.kind });
    return true;
  } catch (error) {
    log(`[gbrain jobs] persistence consumer did not start: ${error instanceof Error ? error.message : String(error)}; queued effects wait for the next resident owner`);
    return false;
  }
}

/**
 * PGLite has no background worker (the brain is single-writer, so a daemon
 * would lock every other command out). There `gbrain jobs work` drains (agent-
 * first operator wave E5): it runs the queue's waiting jobs in the foreground,
 * registered as a live worker, and exits once none it can run are waiting or
 * active. Jobs scheduled for a later retry stay queued and are named.
 */
export async function drainPgliteQueue(
  engine: BrainEngine,
  queue: MinionQueue,
  queueName: string,
  registerHandlers: (worker: MinionWorker, engine: BrainEngine) => Promise<void>,
): Promise<{ delayed: number }> {
  await queue.ensureSchema();
  const worker = new MinionWorker(engine, { queue: queueName, pollInterval: 200, healthCheckInterval: 0 });
  await registerHandlers(worker, engine);
  const names = worker.registeredNames;
  const count = async (statuses: string[]): Promise<number> => {
    const [row] = await engine.executeRaw<{ n: number }>(
      'SELECT count(*)::int AS n FROM minion_jobs WHERE queue = $1 AND status = ANY($2::text[]) AND name = ANY($3::text[])',
      [queueName, statuses, names]);
    return Number(row?.n ?? 0);
  };
  const { registerWorker } = await import('../../core/minions/worker-registry.ts');
  const unregister = registerWorker({ pid: process.pid, queue: queueName, nice_requested: null, nice_effective: null, started_at: Date.now() });
  console.log(`Draining queue '${queueName}' in the foreground (PGLite has no background worker): ${await count(['waiting', 'active'])} runnable job(s).`);
  const running = worker.start();
  const poll = setInterval(() => {
    count(['waiting', 'active']).then(n => { if (n === 0) worker.stop(); }, () => worker.stop());
  }, 500);
  try {
    await running;
  } finally {
    clearInterval(poll);
    unregister();
  }
  const delayed = await count(['delayed']);
  console.log(`Queue '${queueName}' drained.${delayed > 0 ? ` ${delayed} job(s) are scheduled for a later retry; run \`gbrain jobs work${queueName === 'default' ? '' : ` --queue ${queueName}`}\` again after their backoff.` : ''}`);
  return { delayed };
}

/**
 * `jobs work` listener for the worker's self-health failures: report the
 * failing layer and exit so the process manager restarts the worker.
 */
function exitOnUnhealthy(info: UnhealthyReason): void {
  if (info.reason === 'client_misconfigured') {
    reportWorkerConfiguration(info.error);
    setTimeout(() => {
      console.error('[health] release-unconfirmed: shutdown exceeded its deadline; lease expiry may consume stall budget.');
      process.exit(WORKER_EXIT_CONFIGURATION);
    }, 31_000);
    return;
  }
  if (info.reason === 'db_dead') {
    // issue #6: name the failing LAYER, not just "DB unreachable" —
    // that message sent operators chasing database capacity while the
    // real fault was client-side pool exhaustion. Exiting is still
    // correct recovery either way (it frees every client-held slot).
    if (info.verdict === 'pool_starved') {
      console.error(
        `[health] FATAL: connection-pool path saturated after ${info.consecutiveFailures} probes — ` +
        `the database server itself is reachable. (${info.message}) ` +
        `Likely causes: long-running handler queries holding pool slots, or too-small GBRAIN_POOL_SIZE ` +
        `for this workload. Consider --job-isolation process for long-running handlers ` +
        `(handler connections then die with each job's child process). ` +
        `Exiting for process-manager restart (frees all client-held slots).`,
      );
    } else if (info.verdict === 'server_unreachable') {
      console.error(
        `[health] FATAL: database server unreachable after ${info.consecutiveFailures} probes ` +
        `(both pooler and direct lanes failed). (${info.message}) ` +
        `Exiting for process-manager restart.`,
      );
    } else {
      console.error(
        `[health] FATAL: DB probe failed ${info.consecutiveFailures} consecutive times (${info.message}). ` +
        `Exiting for process-manager restart.`,
      );
    }
  } else if (info.reason === 'child_spawn_failing') {
    console.error(
      `[health] FATAL: ${info.consecutiveFailures} consecutive job-child spawn/bootstrap ` +
      `failures (${info.message}). The child CLI is deterministically broken — fix the ` +
      `worker's child CLI configuration (or GBRAIN_JOB_CHILD_CLI). Exiting for ` +
      `process-manager restart.`,
    );
  } else {
    console.error(
      `[health] FATAL: Worker stalled — ${info.waitingCount} waiting job(s) for ` +
      `registered handlers, ${info.idleMinutes}m idle. Exiting for process-manager restart.`,
    );
  }
  process.exit(1);
}