Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.6 KB
Raw Blame
/**
 * `gbrain autopilot pause|resume` (fix wave 3, Lane D): the operator pause is
 * its own marker. The daemon and workers honor it, status shows it,
 * `autopilot --install` (which `gbrain upgrade` runs) keeps it, and resume
 * clears only it — never a migration or restore hold.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { withEnv, emptyHome } from './helpers/with-env.ts';
import { autopilotOperatorPauseMarkerPath, autopilotPaused, autopilotPausedMarkerPath } from '../src/core/autopilot-paths.ts';
import { runAutopilotPauseCommand } from '../src/commands/autopilot-pause.ts';
import { resolveAutopilotPositionals, runAutopilot, runAutopilotStatus } from '../src/commands/autopilot.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { MinionWorker } from '../src/core/minions/worker.ts';
import { waitFor } from './helpers/wait-for.ts';

function out(run: () => unknown): string {
  const lines: string[] = [];
  const log = console.log;
  console.log = (...parts: unknown[]) => { lines.push(parts.map(String).join(' ')); };
  try { run(); } finally { console.log = log; }
  return lines.join('\n');
}

describe('autopilot operator pause', () => {
  test('pause and resume are subcommands', () => {
    expect(resolveAutopilotPositionals(['pause', '--reason', 'upgrade window'])).toEqual(['--pause', '--reason', 'upgrade window']);
    expect(resolveAutopilotPositionals(['resume'])).toEqual(['--resume']);
  });

  test('pause writes its own marker, pauses the daemon and workers, and status shows it', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, () => {
      expect(autopilotPaused()).toBe(false);
      const text = out(() => runAutopilotPauseCommand(['--pause', '--reason', 'upgrade window']));
      expect(text).toContain('Autopilot paused on this host');
      expect(text).toContain('gbrain autopilot resume');
      expect(existsSync(autopilotOperatorPauseMarkerPath())).toBe(true);
      expect(existsSync(autopilotPausedMarkerPath())).toBe(false);
      expect(autopilotPaused()).toBe(true);
      expect(readFileSync(autopilotOperatorPauseMarkerPath(), 'utf8')).toContain('upgrade window');
      const status = JSON.parse(out(() => runAutopilotStatus(['--status', '--json'])));
      expect(status.paused_reason).toContain('operator pause');
      expect(status.paused_reason).toContain('upgrade window');
      // Idempotent: pausing again keeps the pause.
      expect(out(() => runAutopilotPauseCommand(['--pause']))).toContain('already paused');
      expect(out(() => runAutopilotPauseCommand(['--resume']))).toContain('Operator pause cleared');
      expect(autopilotPaused()).toBe(false);
      expect(out(() => runAutopilotPauseCommand(['--resume']))).toContain('No operator pause was set');
    });
  });

  test('resume never clears a migration hold', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, () => {
      const hold = autopilotPausedMarkerPath();
      mkdirSync(dirname(hold), { recursive: true });
      writeFileSync(hold, `paused by gbrain migrate (pid ${process.pid})\n`);
      out(() => runAutopilotPauseCommand(['--pause']));
      const text = out(() => runAutopilotPauseCommand(['--resume']));
      expect(text).toContain('Another hold is still present and was not touched');
      expect(existsSync(hold)).toBe(true);
      expect(autopilotPaused()).toBe(true);
    });
  });

  test('autopilot --install (what gbrain upgrade runs) keeps the operator pause and clears only a leaked hold', async () => {
    const home = mkdtempSync(join(tmpdir(), 'gbrain-autopilot-home-'));
    const repo = mkdtempSync(join(tmpdir(), 'gbrain-autopilot-repo-'));
    const bin = mkdtempSync(join(tmpdir(), 'gbrain-autopilot-bin-'));
    writeFileSync(join(bin, 'gbrain'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
    await withEnv({ GBRAIN_HOME: emptyHome(), HOME: home, PATH: `${bin}:${process.env.PATH}` }, async () => {
      out(() => runAutopilotPauseCommand(['--pause', '--reason', 'upgrading hosts']));
      writeFileSync(autopilotPausedMarkerPath(), 'leaked by a dead migration\n');
      const engine = { kind: 'postgres', getConfig: async () => null } as never;
      const log = console.log;
      console.log = () => {};
      try { await runAutopilot(engine, ['--install', '--yes', '--target', 'ephemeral-container', '--repo', repo, '--no-inject']); } finally { console.log = log; }
      expect(existsSync(autopilotPausedMarkerPath())).toBe(false);
      expect(existsSync(autopilotOperatorPauseMarkerPath())).toBe(true);
      expect(autopilotPaused()).toBe(true);
    });
  });

  test('the daemon loop skips cycles while the operator pause holds and resumes when it clears', async () => {
    const home = mkdtempSync(join(tmpdir(), 'gbrain-operator-pause-daemon-'));
    const env = { PATH: process.env.PATH, HOME: home, GBRAIN_HOME: home, GBRAIN_DISABLE_UPDATE_CHECK: '1', GBRAIN_MODEL_DISCOVERY: 'off' };
    const cli = (...args: string[]) => Bun.spawn([process.execPath, join(import.meta.dir, '..', 'src', 'cli.ts'), ...args],
      { cwd: home, env, stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' });
    try {
      expect(await cli('init', '--pglite', '--no-embedding').exited).toBe(0);
      mkdirSync(join(home, 'repo'));
      expect(await cli('autopilot', 'pause', '--reason', 'upgrade window').exited).toBe(0);
      const daemon = cli('autopilot', '--repo', join(home, 'repo'), '--interval', '1', '--inline');
      let stdout = '';
      const reader = (async () => { for await (const chunk of daemon.stdout) stdout += new TextDecoder().decode(chunk); })();
      try {
        await waitFor(() => stdout.includes('[autopilot] paused'), { timeoutMs: 30_000, label: 'daemon reports the pause' });
        expect(stdout).not.toContain('[autopilot] resumed');
        expect(await cli('autopilot', 'resume').exited).toBe(0);
        await waitFor(() => stdout.includes('[autopilot] resumed'), { timeoutMs: 30_000, label: 'daemon resumes' });
      } finally {
        daemon.kill('SIGTERM');
        await daemon.exited; await reader;
      }
    } finally { rmSync(home, { recursive: true, force: true }); }
  }, 120_000);
});

describe('job workers honor the operator pause', () => {
  let engine: PGLiteEngine;
  beforeAll(async () => { engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); });
  afterAll(async () => { await engine.disconnect(); });
  beforeEach(async () => { await engine.executeRaw('DELETE FROM minion_jobs'); });

  test('a job worker claims nothing while the operator pause holds and runs the job after resume', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, async () => {
      const worker = new MinionWorker(engine, { pollInterval: 25, lockDuration: 30_000 });
      worker.register('operator-pause-probe', async () => 'ran');
      let running: Promise<void> | undefined;
      try {
        out(() => runAutopilotPauseCommand(['--pause', '--reason', 'upgrade window']));
        const queue = new MinionQueue(engine);
        const job = await queue.add('operator-pause-probe', {}, {}, { allowProtectedSubmit: true });
        const lines: string[] = [];
        const log = console.log;
        console.log = (...parts: unknown[]) => { lines.push(parts.map(String).join(' ')); };
        try {
          running = worker.start();
          await waitFor(() => lines.some(line => line.includes('pause marker present')), { timeoutMs: 15_000, label: 'worker parks on the pause' });
        } finally { console.log = log; }
        expect((await queue.getJob(job.id))).toMatchObject({ status: 'waiting', attempts_started: 0 });
        out(() => runAutopilotPauseCommand(['--resume']));
        await waitFor(async () => (await queue.getJob(job.id))?.status === 'completed', { timeoutMs: 15_000, label: 'job completes after resume' });
      } finally {
        worker.stop(); await running;
      }
    });
  }, 60_000);

  test('a job claimed as the operator pause lands is released back to the queue un-run', async () => {
    await withEnv({ GBRAIN_HOME: emptyHome() }, async () => {
      const worker = new MinionWorker(engine, { pollInterval: 25, lockDuration: 30_000 });
      let runs = 0;
      worker.register('operator-pause-probe', async () => { runs++; return 'ran'; });
      const queue = (worker as unknown as { queue: MinionQueue }).queue;
      const claim = queue.claim.bind(queue);
      queue.claim = (async (...args: Parameters<MinionQueue['claim']>) => {
        const claimed = await claim(...args);
        if (claimed && runs === 0) out(() => runAutopilotPauseCommand(['--pause', '--reason', 'upgrade window']));
        return claimed;
      }) as MinionQueue['claim'];
      let running: Promise<void> | undefined;
      try {
        const job = await new MinionQueue(engine).add('operator-pause-probe', {}, {}, { allowProtectedSubmit: true });
        const lines: string[] = [];
        const log = console.log;
        console.log = (...parts: unknown[]) => { lines.push(parts.map(String).join(' ')); };
        try {
          running = worker.start();
          await waitFor(() => lines.some(line => line.includes('pause marker appeared after claim')), { timeoutMs: 15_000, label: 'worker releases the claimed job' });
        } finally { console.log = log; }
        expect(runs).toBe(0);
        expect(await queue.getJob(job.id)).toMatchObject({ status: 'delayed', lock_token: null });
      } finally {
        worker.stop(); await running;
      }
    });
  }, 60_000);
});