Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
33.8 KB
Raw Blame
/**
 * `gbrain autopilot` daemon: boot, the managed worker, shutdown and the tick
 * loop. runAutopilot (src/commands/autopilot.ts) dispatches here when no
 * install/uninstall/status/pause mode flag is present.
 */
import type { BrainEngine } from '../core/engine.ts';
import { ChildWorkerSupervisor } from '../core/minions/child-worker-supervisor.ts';
import { MIGRATE_PAUSE_MARKER_PREFIX, autopilotLockPath, autopilotPaused, autopilotPausedMarkerPath, markerHolderAlive } from '../core/autopilot-paths.ts';
import { OwnerProcessingState } from '../core/minions/processing-state.ts';
import { mkdirSync, readFileSync, unlinkSync, utimesSync, writeFileSync } from 'fs';
import { gbrainPath as gbrainHomePath, loadConfig, loadConfigFileOnly } from '../core/config.ts';
import { join } from 'path';
import { loadPreferences } from '../core/preferences.ts';
import { registerCleanup } from '../core/process-cleanup.ts';
import { dispatchAutopilotTick, sleepWatchingHeads } from './autopilot-dispatch.ts';
import { readTickHealth } from './autopilot-health.ts';
import type { BrainHealth } from '../core/types.ts';
import { writeAutopilotHeartbeat } from '../core/autopilot-heartbeat.ts';
import { runNightlyQualityProbeStep, runParserProbeStep } from './autopilot-probes.ts';
import { resolveChildCliInvocation } from '../core/minions/job-isolation.ts';
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
import {
  attemptAutopilotSelfUpgrade,
  computeAutopilotIdle,
  autopilotEngineIdentity,
  chatBootWarning,
  classifyReconnectError,
  decideLockAcquisition,
  guardAutopilotEngine,
  logError,
  parseArg,
  reconcileSelfUpgradeAtBoot,
  resolveGbrainCliPath,
  shouldSpawnAutopilotWorker,
} from './autopilot.ts';
import { restartServeAfterSelfUpgrade } from './upgrade-restart.ts';

/**
 * Mutable daemon state shared by the tick loop, the shutdown path and the tick
 * steps. Each field keeps the name of the closure variable it replaced.
 */
export interface AutopilotDaemonState {
  stopping: boolean;
  /** #1872: the in-flight inline runCycle, drained by closeEngine on shutdown. */
  inflightInlineCycle: Promise<unknown> | null;
  consecutiveErrors: number;
  /**
   * #2608: once-per-process no-chat-provider warning. A keyless daemon used
   * to run every cycle "green" while all LLM phases silently no-op'd
   * (chronicle reported no_events, propose_takes skipped, …) — the operator
   * had no signal that shell-profile keys never reached launchd/systemd.
   */
  noChatProviderWarned: boolean;
  /**
   * v0.37.7.0 #1162 — counter for consecutive reconnect failures.
   * Reset on every successful health probe or reconnect. Threshold
   * controlled by GBRAIN_AUTOPILOT_MAX_RECONNECT_FAILS env (default 30).
   */
  autopilotReconnectFails: number;
  /** Consecutive --no-worker ticks with no live worker signal (see NO_WORKER_WARN_TICKS). */
  noWorkerConsecutiveIdle: number;
  /**
   * v0.36+ T8: track time since last full cycle for the 60-min floor.
   * Initialized to "long ago" (0) so the first tick on a healthy brain still
   * runs the full cycle (phase-coupling exercise) before settling into
   * targeted-submit mode.
   */
  lastFullCycleAt: number;
  /** Log the pause/resume transition once each, not every poll. */
  pausedAnnounced: boolean;
  /** GBRA-133 2a.4: per source, the checkout HEAD this daemon last dispatched (or decided) a freshness sync for. */
  freshnessHeads: Map<string, string>;
}

export async function runAutopilotDaemon(engine: BrainEngine, args: string[]): Promise<void> {
  const repoPath = parseArg(args, '--repo') || await engine.getConfig('sync.repo_path');
  // Same NaN guard as the status path: a typo'd interval would otherwise
  // reach setTimeout(NaN) → 0ms and busy-loop the daemon against the DB.
  const rawBaseInterval = parseInt(parseArg(args, '--interval') || '300', 10);
  const baseInterval = Number.isFinite(rawBaseInterval) && rawBaseInterval > 0 ? rawBaseInterval : 300;
  const jsonMode = args.includes('--json');
  const forceInline = args.includes('--inline');
  const noWorker = !shouldSpawnAutopilotWorker(args);

  if (!repoPath) {
    console.error('No repo path. Use --repo or run gbrain sync --repo first.');
    process.exit(1);
  }

  // Lock file to prevent concurrent instances (#14).
  // v0.37.7.0 #1226: route through gbrainPath() so the lockfile lives
  // under GBRAIN_HOME when set, not the hardcoded ~/.gbrain. Pre-fix,
  // two brains sharing GBRAIN_HOME=different-paths still wrote to the
  // same global lockfile and one would silently respawn the other
  // forever.
  const lockPath = autopilotLockPath();
  acquireAutopilotLock(lockPath);

  console.log(`Autopilot starting. Repo: ${repoPath}, interval: ${baseInterval}s`);

  // #2608: LLM phases (chronicle extract, dream synthesis, enrich) gate on
  // isAvailable('chat') and silently no-op when no chat provider resolves —
  // the classic symptom of a daemon shell that never sourced the API keys
  // (see writeWrapperScript below). One loud boot-time line makes that
  // failure mode visible in the daemon log instead of manifesting as
  // "autopilot runs green but nothing gets extracted".
  // console.log, NOT console.error: launchd/systemd route stderr to
  // autopilot.err, which install output and showStatus never reference —
  // stdout is the autopilot.log sink on all four install targets.
  // Bare isAvailable('chat') probes the GLOBAL chat model on purpose — it
  // mirrors the phases named above; facts extraction gates model-aware
  // (core/facts/extract.ts) and doctor owns that diagnosis.
  try {
    const { isAvailable } = await import('../core/ai/gateway.ts');
    const warn = chatBootWarning(isAvailable('chat'), gbrainHomePath());
    if (warn) console.log(warn);
  } catch { /* diagnostic only — never blocks the loop */ }

  // Mode resolution: Minions dispatch when the user has opted in AND the
  // worker daemon can actually run (Postgres only; PGLite's exclusive file
  // lock blocks a separate worker process).
  const mode = loadPreferences().minion_mode ?? 'pain_triggered';
  const cfg = loadConfig();
  const engineType = cfg?.engine ?? 'pglite';
  const useMinionsDispatch = mode !== 'off' && engineType === 'postgres' && !forceInline;
  const spawnManagedWorker = useMinionsDispatch && !noWorker;

  // Engine identity at boot, re-checked every tick. A cross-engine migration
  // flips config.json at the END of its copy; this long-lived process would
  // otherwise keep syncing into the ABANDONED source engine indefinitely —
  // the health probe keeps succeeding (the old engine stays alive as the
  // preserved backup) and reconnect() deliberately restores the config
  // captured at connect() (#2034), never the new file. Same silent-divergence
  // class as the dead-daemon incident, moved to after the flip.
  const engineIdentityAtBoot = autopilotEngineIdentity(loadConfigFileOnly());

  // v0.42 self-upgrade: if a prior tick swapped the binary and exited for
  // relaunch, we're now the relaunched process — reconcile the breadcrumb so a
  // crash-on-launch is recorded known-bad and a success is confirmed.
  reconcileSelfUpgradeAtBoot();

  const state: AutopilotDaemonState = {
    stopping: false,
    inflightInlineCycle: null,
    consecutiveErrors: 0,
    noChatProviderWarned: false,
    autopilotReconnectFails: 0,
    noWorkerConsecutiveIdle: 0,
    lastFullCycleAt: 0,
    pausedAnnounced: false,
    freshnessHeads: new Map(),
  };
  let childSupervisor: ChildWorkerSupervisor | null = null;
  const processingState = spawnManagedWorker ? new OwnerProcessingState('autopilot', 'default') : null;
  const configurationBlocked = () => processingState?.blocked ?? false;
  if (processingState) engine = guardAutopilotEngine(engine, processingState);

  // #1872: graceful engine shutdown. On PGLite the cycle steps run INLINE in
  // this process, so a hard `process.exit` mid-write (systemctl stop →
  // SIGTERM) kills WASM Postgres with the WAL dirty and can corrupt the
  // brain. Two exit paths must both close the engine:
  //   - autopilot's own shutdown() below (owns SIGINT + internal stops like
  //     max_crashes / cycle-failure-cap), and
  //   - process-cleanup's SIGTERM handler (installed inside cli.ts's
  //     import.meta.main seam before main() dispatches; it runs the cleanup
  //     registry with a 3s deadline and then exits) —
  //     which is why closeEngine is ALSO registered there.
  // closeEngine aborts the in-flight inline cycle (runCycle checks the
  // signal between phases and threads it into phase sub-work), gives it a
  // short bounded window to wind down, then disconnects. PGLite's
  // disconnect() drains the pending query and checkpoints before closing;
  // a second call is a no-op (disconnect snapshots + nulls the handle), so
  // both paths firing is safe.
  const shutdownAbort = new AbortController();
  // GBRA-133 PR 2a.4: the daemon is a resident; its notifier syncs this host's sources on commit (sync.on_commit) and
  // drains wanted markers, between the wall-clock cycles. Stopped before the engine closes.
  const { startResidentNotifier } = await import('../core/notifier/resident.ts');
  const notifier = startResidentNotifier(engine, { resident: 'autopilot', log: line => console.log(line) });
  const closeEngine = async () => {
    shutdownAbort.abort(new Error('autopilot shutdown'));
    await notifier?.stop().catch(() => undefined);
    if (state.inflightInlineCycle) {
      // ponytail: 2s cap keeps us inside process-cleanup's 3s deadline; a
      // between-phase abort resolves instantly, a mid-phase one may not.
      await Promise.race([
        state.inflightInlineCycle.catch(() => { /* cycle errors already logged by the loop */ }),
        new Promise((r) => setTimeout(r, 2_000)),
      ]);
    }
    try { await engine.disconnect(); } catch { /* best-effort */ }
  };
  const deregisterEngineClose = registerCleanup('autopilot-engine-close', closeEngine);

  if (spawnManagedWorker) {
    // Lazy: `shutdown` is declared below; the worker only calls it after the loop starts.
    childSupervisor = await startAutopilotWorker(processingState, state, (sig) => shutdown(sig));
  } else if (!useMinionsDispatch) {
    const why = mode === 'off'
      ? 'minion_mode=off'
      : (engineType !== 'postgres' ? 'engine=pglite' : 'flag=--inline');
    console.log(`[autopilot] running steps inline (${why})`);
  } else {
    console.log('[autopilot] --no-worker set: dispatch loop only (worker managed externally)');
  }

  // Async shutdown with 35s drain window for the worker child. The worker
  // has its own SIGTERM handler (minions/worker.ts:79-85) that drains
  // in-flight jobs for up to 30s before exit. We give it 35s here to
  // account for signal-delivery latency, then SIGKILL as a last resort.
  //
  // No `process.on('exit')` handler — its callback runs synchronously and
  // cannot await the worker's drain.
  const shutdown = async (sig: string) => {
    if (configurationBlocked() && sig !== 'SIGTERM' && sig !== 'SIGINT') return;
    if (state.stopping) return;
    state.stopping = true;
    console.log(`Autopilot stopping (${sig}).`);
    if (childSupervisor) {
      childSupervisor.killChild('SIGTERM');
      await childSupervisor.awaitChildExit(35_000);
      if (childSupervisor.childAlive) {
        childSupervisor.killChild('SIGKILL');
      }
    }
    if (configurationBlocked() && sig !== 'SIGTERM' && sig !== 'SIGINT') {
      state.stopping = false;
      return;
    }
    // #1872: abort the in-flight inline cycle and close the engine BEFORE
    // process.exit — a hard exit mid-write corrupts PGLite's WASM Postgres.
    await closeEngine();
    deregisterEngineClose();
    processingState?.close();
    try { unlinkSync(lockPath); } catch { /* already gone */ }
    process.exit(sig === 'max_crashes' || sig === 'cycle-failure-cap' ? 1 : 0);
  };
  process.on('SIGTERM', () => { void shutdown('SIGTERM'); });
  process.on('SIGINT',  () => { void shutdown('SIGINT'); });

  const AUTOPILOT_MAX_RECONNECT_FAILS = Math.max(
    1,
    Number(process.env.GBRAIN_AUTOPILOT_MAX_RECONNECT_FAILS) || 30,
  );

  while (!state.stopping) {
    const cycleStart = Date.now();

    // Refresh the lock mtime so another cron-fired autopilot doesn't
    // declare the instance stale after 10 minutes (Codex C).
    try { utimesSync(lockPath, new Date(), new Date()); } catch { /* best-effort */ }
    writeAutopilotHeartbeat({ phase: 'cycle', interval_s: null, base_interval_s: baseInterval, next_tick_at: null });

    if (processingState && !processingState.snapshot.processing_ready) {
      await new Promise(r => setTimeout(r, 250));
      continue;
    }

    await warnNoChatProviderOnce(state);

    // Post-migration convergence: if the file-plane engine identity changed
    // since boot, this process is connected to the wrong engine. Exit through
    // the clean shutdown path (engine close matters for PGLite WAL) so the
    // supervisor relaunches on the new config; the same relaunch contract the
    // self-upgrade swap relies on. Cron and one-shot targets simply pick up
    // the new config on their next run.
    // A torn or failed read (concurrent config write, transient EACCES) must
    // not restart the daemon: skip the comparison unless the file read
    // actually produced a config — a genuine migration flip never yields null.
    let identityNow: string | null = null;
    try {
      const fileCfg = loadConfigFileOnly();
      identityNow = fileCfg ? autopilotEngineIdentity(fileCfg) : null;
    } catch { /* torn read mid-write; check again next tick */ }
    if (identityNow !== null && identityNow !== engineIdentityAtBoot) {
      console.log('[autopilot] engine config changed on disk (migration?) — exiting for relaunch on the new engine.');
      await shutdown('engine-config-changed');
      return;
    }

    // Cooperative pause (see autopilotPausedMarkerPath). Checked AFTER the
    // heartbeat so a paused daemon still reads as alive, and BEFORE any DB
    // work so a cross-engine migration is not racing our writes into an
    // engine that is about to stop being the configured one.
    if (autopilotPaused()) {
      // Self-heal an orphan: a migrate-owned marker whose recorded pid is dead
      // was leaked by a killed migration (SIGKILL, power loss — anything its
      // own cleanup could not catch). Nothing else ever deletes it, and an
      // orphan parks this daemon forever. An operator's manual hold (no
      // migrate signature) is never touched, and a live migrate's marker
      // reads alive and is honored.
      let orphaned = false;
      try {
        const body = readFileSync(autopilotPausedMarkerPath(), 'utf-8');
        orphaned = body.startsWith(MIGRATE_PAUSE_MARKER_PREFIX) && markerHolderAlive(body) === 'dead';
      } catch { /* vanished or unreadable: fall through to the normal pause */ }
      if (orphaned) {
        console.log('[autopilot] clearing an orphaned pause marker (its migrate process is dead); resuming.');
        try { unlinkSync(autopilotPausedMarkerPath()); } catch { /* already gone */ }
      }
      if (autopilotPaused()) {
        if (!state.pausedAnnounced) {
          console.log('[autopilot] paused (autopilot-paused marker present) — skipping cycles until it clears.');
          state.pausedAnnounced = true;
        }
        // Poll faster than a normal tick so a migration's quiesce window is short.
        await new Promise((r) => setTimeout(r, Math.min(baseInterval, 30) * 1000));
        continue;
      }
    }
    if (state.pausedAnnounced) {
      console.log('[autopilot] resumed — pause marker cleared.');
      state.pausedAnnounced = false;
    }

    // DB health check (reconnect if needed); see probeDatabaseOrReconnect.
    const dbProbe = await probeDatabaseOrReconnect(engine, state, configurationBlocked, AUTOPILOT_MAX_RECONNECT_FAILS);
    if (dbProbe === 'skip') continue;
    if (dbProbe === 'fatal') break;

    // v0.42 self-upgrade silent channel (opt-in self_upgrade.mode=auto). Runs
    // each tick; cache TTL throttles the actual GitHub fetch. On apply it swaps
    // + exits for supervisor relaunch (never returns). No-op unless mode=auto.
    if (configurationBlocked()) continue;
    await attemptAutopilotSelfUpgrade(engine, engineType, lockPath, () => !configurationBlocked());
    if (configurationBlocked()) continue;
    await restartServeAfterSelfUpgrade(() => computeAutopilotIdle(engine, engineType));
    if (configurationBlocked()) continue;

    // --no-worker peer-liveness probe (v0.19.1). Runs every cycle, cheap
    // (single SELECT). See the NO_WORKER_WARN_TICKS comment for caveats.
    if (noWorker && useMinionsDispatch) await probeNoWorkerPeer(engine, state, baseInterval);

    if (!(await (await import('../core/persistence/automatic-sync-policy.ts')).automaticGitPullDisabled(engine))) {
      const reconciled = await (await import('./autopilot-git-reconcile.ts')).autopilotReconcile(engine);
      for (const line of reconciled.lines) (reconciled.ok ? console.log : console.error)(`[autopilot] git-reconcile: ${line}`);
      if (configurationBlocked()) continue;
    }

    const tick = useMinionsDispatch
      ? await dispatchAutopilotTick(engine, state, { repoPath, baseInterval, jsonMode })
      : { cycleOk: await runInlineCycle(engine, state, repoPath, shutdownAbort, jsonMode) };

    if (configurationBlocked()) continue;
    // 4. Adaptive interval; a dispatch tick hands over the health it routed on (#7236), the inline path reads it now.
    const interval = await adaptiveInterval(engine, baseInterval, cycleStart, jsonMode, tick.health);

    if (configurationBlocked()) continue;
    if (tick.cycleOk) {
      state.consecutiveErrors = 0;
    } else {
      state.consecutiveErrors++;
      if (state.consecutiveErrors >= 5) {
        console.error('5 consecutive cycle failures. Stopping autopilot.');
        await shutdown('cycle-failure-cap');
        if (!state.stopping) continue;
        break;
      }
    }

    await runNightlyQualityProbeStep(engine, cfg);
    await runParserProbeStep(engine, cfg);

    // Wait for next cycle; #7003: --status grades staleness from this daemon's own next tick.
    writeAutopilotHeartbeat({ phase: 'sleep', interval_s: interval, base_interval_s: baseInterval, next_tick_at: new Date(Date.now() + interval * 1000).toISOString() });
    if (useMinionsDispatch) await sleepWatchingHeads(engine, state, { sleepMs: interval * 1000, baseInterval, jsonMode }, { notifierActive: () => notifier !== null });
    else await new Promise(r => setTimeout(r, interval * 1000));
  }
}

/** Take the single-instance lock file (#14): exit when a live holder owns it, take over a stale one. Best-effort. */
function acquireAutopilotLock(lockPath: string): void {
  try {
    mkdirSync(gbrainHomePath(), { recursive: true });
    const decision = decideLockAcquisition(lockPath, process.pid);
    if (decision.action === 'exit') {
      // #4300: say WHY we refused, loudly, so a bricked daemon is diagnosable
      // from launchd/systemd logs without strace-ing the lock probe.
      const detail =
        decision.holderState === 'alive-autopilot'
          ? 'a live gbrain autopilot process'
          : decision.holderState === 'alive-unknown'
            ? 'a live process whose command line could not be inspected (fresh lock — will become stealable once stale)'
            : 'a live non-gbrain process holding a fresh lock (will become stealable once stale)';
      console.error(
        `[autopilot] refusing to start: lock ${lockPath} is held by pid ${decision.holderPid} — ${detail}. Exiting.`,
      );
      process.exit(0);
    }
    if (decision.action === 'takeover') {
      console.log(`Stale autopilot lock found (${decision.reason}). Taking over.`);
    }
    writeFileSync(lockPath, String(process.pid));
  } catch { /* best-effort */ }
}

/** Spawn the managed `gbrain jobs work` child under ChildWorkerSupervisor (Minions dispatch mode). */
async function startAutopilotWorker(
  processingState: OwnerProcessingState | null,
  state: AutopilotDaemonState,
  shutdown: (sig: string) => Promise<void>,
): Promise<ChildWorkerSupervisor> {
  const invocation = resolveChildCliInvocation({}, process.execPath, process.argv[1], resolveGbrainCliPath);
  if (!invocation) throw new Error('Could not resolve the worker CLI. Repair the current GBrain installation.');
  // Cgroup-aware auto-sized RSS watchdog cap (issue #1678). The old flat
  // 2048MB killed legit embed work (~10GB) on every cycle → silent
  // ~400×/24h respawn loop. resolveDefaultMaxRssMb clamps 0.5×min(cgroup,
  // RAM) to [4096,16384]. Bare `gbrain jobs work` resolves the same default;
  // we pass it explicitly so the spawn log + child agree.
  const { resolveDefaultMaxRssMb } = await import('../core/minions/rss-default.ts');
  const autopilotMaxRssMb = resolveDefaultMaxRssMb();
  const childSupervisor = new ChildWorkerSupervisor({
    processingState: processingState ?? undefined,
    onConfigurationBlocked: (status) => {
      console.error(`[autopilot] processing configuration-blocked (${status?.reason_code ?? 'unknown'}); repair the worker/child installation and explicitly restart autopilot.`);
    },
    cliPath: invocation.cmd,
    args: [...invocation.argsPrefix, 'jobs', 'work', '--max-rss', String(autopilotMaxRssMb)],
    env: { ...process.env, GBRAIN_SUPERVISED: undefined } as Record<string, string | undefined>,
    maxCrashes: 5,
    isStopping: () => state.stopping,
    onMaxCrashesExceeded: (count, max) => {
      console.error(`[autopilot] ${count}/${max} consecutive worker crashes, giving up.`);
      void shutdown('max_crashes');
    },
    onEvent: (event) => {
      // Route ChildWorkerSupervisor events to autopilot's stderr log.
      // Matches the prior console output shape so operators reading
      // existing logs see the same lines.
      if (event.kind === 'worker_startup_timeout') {
        console.error(`[autopilot] worker readiness was not confirmed within ${event.timeoutMs}ms; stopping this worker and retrying with bounded backoff.`);
      } else if (event.kind === 'worker_spawned') {
        console.log(
          `[autopilot] Minions worker spawned (pid: ${event.pid}, watchdog: ${autopilotMaxRssMb}MB${event.tini ? ', tini: active' : ''})`,
        );
      } else if (event.kind === 'worker_spawn_failed') {
        console.error(
          `[autopilot] worker spawn failed (${event.phase}): ${event.error}${event.errnoCode ? ` (code=${event.errnoCode})` : ''}`,
        );
      } else if (event.kind === 'worker_exited') {
        console.error(
          `[autopilot] worker exited code=${event.code} signal=${event.signal} after ${event.runDurationMs}ms, crashCount=${event.crashCount}, cause=${event.likelyCause}`,
        );
      } else if (event.kind === 'backoff') {
        if (event.reason === 'budget_exceeded') {
          console.error(
            `[autopilot] clean-restart budget exceeded; backing off ${event.ms}ms before next spawn`,
          );
        } else if (event.reason === 'crash') {
          console.error(
            `[autopilot] crash backoff ${event.ms}ms (crashCount=${event.crashCount})`,
          );
        }
        // reason='clean_exit' with ms:0 is the steady-state watchdog drain;
        // logging every iteration would be noisy. Keep silent (the
        // worker_exited line already covers the user-visible signal).
      } else if (event.kind === 'health_warn') {
        console.error(
          `[autopilot] health_warn: ${event.reason} count=${event.count} window=${event.windowMs}ms`,
        );
      }
    },
  });
  // Fire-and-forget; runs alongside the dispatch loop. shutdown() drives
  // the child-supervisor's isStopping accessor + drain.
  void childSupervisor.run();
  return childSupervisor;
}

/** Once per process: warn when no chat provider is available to the daemon. */
async function warnNoChatProviderOnce(state: AutopilotDaemonState): Promise<void> {
  // #2608: loud once-per-process signal when no chat provider is servable.
  // Without this a keyless daemon looks healthy forever while every LLM
  // phase quietly skips.
  if (!state.noChatProviderWarned) {
    state.noChatProviderWarned = true;
    try {
      const { isAvailable } = await import('../core/ai/gateway.ts');
      if (!isAvailable('chat')) {
        console.error(
          `[autopilot] WARN: no chat provider is available to this daemon — LLM-dependent ` +
          `phases (chronicle event extraction, propose_takes, synthesize, …) will skip. ` +
          `Shell-profile exports often do not reach launchd/systemd: put KEY=value lines in ` +
          `${join(gbrainHomePath(), 'env')} (sourced by the wrapper), then re-run ` +
          '`gbrain autopilot --install` to reload the daemon.',
        );
      }
    } catch { /* gateway unconfigured — the cycle surfaces its own errors */ }
  }
}

/**
 * One tick's DB liveness probe: 'ok' when the engine answers (directly or
 * after a reconnect), 'skip' when processing is configuration-blocked, and
 * 'fatal' after it set state.stopping on an unrecoverable error or the
 * consecutive-failure cap.
 */
async function probeDatabaseOrReconnect(
  engine: BrainEngine,
  state: AutopilotDaemonState,
  configurationBlocked: () => boolean,
  AUTOPILOT_MAX_RECONNECT_FAILS: number,
): Promise<'ok' | 'skip' | 'fatal'> {
  // DB health check (reconnect if needed).
  //
  // v0.37.7.0 #1162: classify reconnect failures. Pre-fix, the
  // catch logged the error and looped forever — when `database_url`
  // was unset/malformed the loop spammed `config.database_url
  // undefined` until launchd was killed manually. Now:
  //   - Recoverable transient (network blip, pool saturated, 503) →
  //     log + retry next tick. Up to GBRAIN_AUTOPILOT_MAX_RECONNECT_FAILS
  //     consecutive failures before exit (default 30 = ~5min at
  //     10s ticks).
  //   - Unrecoverable (database_url unset, malformed URL, auth
  //     failure) → exit immediately with a clear stderr line.
  //     ThrottleInterval=60 in the launchd plist (v0.37.7.0) ensures
  //     launchd's KeepAlive backoff actually backs off instead of
  //     thrashing.
  try {
    await engine.getConfig('version');
    state.autopilotReconnectFails = 0; // reset on success
  } catch (probeErr) {
    if (configurationBlocked()) return 'skip';
    try {
      // #2034: use reconnect() — it restores the config captured at connect()
      // and avoids the null-connection window. The previous
      // `disconnect()` + bare `connect()` lost the config (throwing
      // `database_url undefined` on every retry → FATAL restart-loop on any
      // transient DB blip) AND tore down the pool postgres.js can otherwise
      // self-heal.
      await engine.reconnect({ error: probeErr });
      state.autopilotReconnectFails = 0;
    } catch (e) {
      if (configurationBlocked()) return 'skip';
      logError('reconnect', e);
      state.autopilotReconnectFails++;
      const klass = classifyReconnectError(e);
      if (klass === 'crash') {
        // A gbrain BUG, not an operator misconfiguration. Say so plainly
        // instead of blaming the config, and keep retrying: a code defect must
        // not permanently disable the daemon. The consecutive-failure cap below
        // still bounds it.
        console.error(
          `[autopilot] BUG: internal error during reconnect (${(e as Error).message ?? 'unknown'}). ` +
          `This is a gbrain defect, not a configuration problem — please report it. ` +
          `Retrying (${state.autopilotReconnectFails}/${AUTOPILOT_MAX_RECONNECT_FAILS}).`,
        );
      } else if (klass === 'unrecoverable') {
        console.error(
          `[autopilot] FATAL: unrecoverable DB error (${(e as Error).message ?? 'unknown'}). ` +
          `Exiting so launchd ThrottleInterval can apply backoff.`,
        );
        state.stopping = true;
        setCliExitVerdict(1);
        return 'fatal';
      }
      if (state.autopilotReconnectFails >= AUTOPILOT_MAX_RECONNECT_FAILS) {
        console.error(
          `[autopilot] FATAL: ${state.autopilotReconnectFails} consecutive reconnect failures. ` +
          `Last error: ${(e as Error).message ?? 'unknown'}. Exiting.`,
        );
        state.stopping = true;
        setCliExitVerdict(1);
        return 'fatal';
      }
    }
  }
  return 'ok';
}

// Peer-worker liveness for --no-worker mode. The probe is a proxy, not
// ground truth: SELECT count(*) of active jobs with a recent lock_until
// refresh. A queue with only waiting jobs and a healthy idle worker
// reads as "no worker" (false positive); a worker that died 110s ago
// while holding a lock reads as "alive" until lock_until expires.
// Good enough for V1 — a ground-truth minion_workers heartbeat table
// is tracked as v0.19.1 follow-up B7. When the probe sees no signal
// for NO_WORKER_WARN_TICKS consecutive cycles, log a loud warning so
// the operator can spot "I set --no-worker but forgot to start one"
// before the queue piles up.
const NO_WORKER_WARN_TICKS = 3;

/** --no-worker peer-liveness probe (v0.19.1); see NO_WORKER_WARN_TICKS. */
async function probeNoWorkerPeer(engine: BrainEngine, state: AutopilotDaemonState, baseInterval: number): Promise<void> {
    try {
      const rows = await (engine as any).executeRaw?.(
        `SELECT count(*)::int AS n FROM minion_jobs
             WHERE status = 'active'
               AND lock_until IS NOT NULL
               AND lock_until > now() - interval '2 minutes'`,
      );
      const liveWorkerSignal = Number((rows as Array<{ n: number }>)?.[0]?.n ?? 0);
      if (liveWorkerSignal === 0) {
        state.noWorkerConsecutiveIdle++;
        if (state.noWorkerConsecutiveIdle === NO_WORKER_WARN_TICKS) {
          // Fire loud on the Nth consecutive idle tick; don't repeat on every
          // subsequent cycle (the operator already saw it), re-arm once a
          // live worker is seen again.
          console.error(
            `[autopilot] WARNING: --no-worker set and no worker has claimed a job in ~${NO_WORKER_WARN_TICKS * baseInterval}s. ` +
            `Jobs will pile up in 'waiting' until a worker starts. ` +
            `Probe is a proxy (lock_until refresh) and can false-positive on idle queues — see B7 for ground-truth follow-up.`,
          );
        }
      } else {
        if (state.noWorkerConsecutiveIdle >= NO_WORKER_WARN_TICKS) {
          console.log('[autopilot] --no-worker probe: live worker signal detected; warning re-armed.');
        }
        state.noWorkerConsecutiveIdle = 0;
      }
    } catch (e) {
      // Probe failures never block the main dispatch loop. Log once per
      // failure class; ignore repeated errors (common shape: DB reconnect
      // blip between ticks).
      logError('no-worker-probe', e);
    }
}

/** Inline fallback cycle (minion_mode=off, PGLite or --inline); returns cycleOk. */
async function runInlineCycle(
  engine: BrainEngine,
  state: AutopilotDaemonState,
  repoPath: string,
  shutdownAbort: AbortController,
  jsonMode: boolean,
): Promise<boolean> {
  let cycleOk = true;
    // Inline fallback — delegate to runCycle so lint + backlinks +
    // orphan sweep run too (previously this path only did sync +
    // extract + embed, which didn't match the Minions-dispatch
    // path's phase set). Now both converge on the same primitive.
    try {
      const { runCycle } = await import('../core/cycle.ts');
      // #1872: track the promise so closeEngine can drain it on shutdown,
      // and pass the abort signal so the cycle winds down between phases.
      const cyclePromise = runCycle(engine, {
        brainDir: repoPath,
        // Autopilot daemon path: pulls by default (matches
        // pre-v0.17 autopilot behavior). CLI dream defaults false
        // for cron safety; that choice is scoped to dream only.
        pull: true,
        signal: shutdownAbort.signal,
        yieldBetweenPhases: async () => {
          await new Promise(r => setImmediate(r));
        },
      });
      state.inflightInlineCycle = cyclePromise;
      const report = await cyclePromise.finally(() => { state.inflightInlineCycle = null; });
      // Only 'failed' (every attempted phase failed) trips the autopilot
      // circuit breaker. 'partial' means at least one phase warned or
      // failed while others ran — that's a soft signal, not a fatal
      // condition. Treating 'partial' as failure here caused respawn
      // storms under KeepAlive=true on brains where a single phase
      // (typically `orphans`) emits a 'warn' every cycle in steady state.
      if (report.status === 'failed') {
        cycleOk = false;
      }
      if (jsonMode) {
        process.stderr.write(JSON.stringify({ event: 'cycle-inline', status: report.status, duration_ms: report.duration_ms, totals: report.totals }) + '\n');
      } else {
        const t = report.totals;
        console.log(`[cycle-inline ${report.status}] lint=${t.lint_fixes} backlinks=${t.backlinks_added} synced=${t.pages_synced} extracted=${t.pages_extracted} embedded=${t.pages_embedded} orphans=${t.orphans_found}`);
      }
    } catch (e) { logError('cycle-inline', e); cycleOk = false; }
  return cycleOk;
}

/**
 * 4. Adaptive interval; returns the next interval in seconds. Routes on `tickHealth` when the tick already read it
 * (#7236), else reads it once; an unknown score (degraded read) keeps the base interval.
 */
export async function adaptiveInterval(engine: BrainEngine, baseInterval: number, cycleStart: number, jsonMode: boolean, tickHealth?: BrainHealth | null): Promise<number> {
  let interval = baseInterval;
  try {
    const health = tickHealth === undefined ? await readTickHealth(engine, jsonMode) : tickHealth;
    const score = health?.brain_score ?? null;
    interval = score === null ? baseInterval
             : score >= 90 ? baseInterval * 2
             : score < 70 ? Math.max(Math.floor(baseInterval / 2), 60)
             : baseInterval;

    const elapsed = ((Date.now() - cycleStart) / 1000).toFixed(0);
    const line = `[cycle] score=${score ?? 'unknown'} elapsed=${elapsed}s next=${interval}s`;
    if (jsonMode) {
      process.stderr.write(JSON.stringify({ event: 'cycle', brain_score: score, elapsed_s: Number(elapsed), next_s: interval }) + '\n');
    } else {
      console.log(line);
    }
  } catch (e) { logError('health', e); }
  return interval;
}