Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.2 KB
Raw Blame
/**
 * GST-2869: `gbrain sync` stopped by SIGTERM or SIGHUP in the middle of a managed pass recorded nothing: process-cleanup's
 * generic handler released the locks and exited 143, the pass row stayed open and status could only guess `pass_interrupted`.
 * The CLI now owns those signals while a managed pass is live: the pass ends its row `resumable` / `terminated`, the drain
 * stops `terminated`, status reads `pass_terminated`, the run exits 128 + n, and a terminated pass that committed nothing
 * still builds the abandoned streak in every reader. A worker's sync job, or a run without the CLI opt-in, keeps today's path.
 */
import { afterEach, describe, expect, spyOn, test } from 'bun:test';
import { execFileSync } from 'node:child_process';
import { mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { performSync, runSync } from '../src/commands/sync.ts';
import { currentExitCode, setCliExitVerdict } from '../src/core/cli-force-exit.ts';
import * as processCleanup from '../src/core/process-cleanup.ts';
import { _resetForTests, CLEANUP_DEADLINE_MS, hasSignalOwner, installSignalHandlers, registerSignalOwner } from '../src/core/process-cleanup.ts';
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { readBlockedManagedSyncs } from '../src/core/persistence/sync-blocked.ts';
import { abandonedPasses, passStatusNext, passStatusState } from '../src/core/persistence/sync-pass-status.ts';
import * as passRowModule from '../src/core/persistence/sync-pass-row.ts';
import * as watermarkModule from '../src/core/persistence/applied-watermark.ts';
import { MANAGED_SYNC_PASS_OP, readPassRow, transientEnding, type PassRow } from '../src/core/persistence/sync-pass-row.ts';
import { abortStop, SyncTerminated } from '../src/core/persistence/sync-stop-class.ts';
import { readSyncStatus } from '../src/core/persistence/sync-status.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { managedBrain } from './helpers/managed-brain.ts';
import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
import { waitFor } from './helpers/wait-for.ts';

const git = (root: string, args: string[]) => execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]', '-c', 'commit.gpgsign=false', ...args],
  { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
const now = Date.parse('2026-11-17T12:00:00Z');
const row = (outcome: string, stop: string | null, streak?: number): PassRow => ({ updated_at: new Date(now - 10_000).toISOString(),
  value: { source_id: 's', incarnation: 'i', lease_token: 't', obligations: { baseline_commit: null, mismatch: {}, not_converged_streak: 0, faults: {}, ...(streak ? { abandoned_streak: streak } : {}) },
    telemetry: { pass_id: 'p', target: 'h', started_at: '', ended_at: null, kind: 'full', round: 0, outcome, stop_reason: stop, candidates: 0, settled: 0, remaining: 0, counts: {}, duration_ms: null } } } as never);

afterEach(() => { installFaultHook(undefined); _resetForTests(); setCliExitVerdict(0); });

describe('GST-2869: a terminated pass', () => {
  test('reads pass_terminated with the plain sync next, and counts toward the abandoned streak unless it banked work', () => {
    const terminated = row('resumable', 'terminated');
    expect(transientEnding(terminated.value.telemetry)).toBe(true);
    expect(passStatusState(terminated, null, null, now, true)).toEqual(['resumable', 'pass_terminated']);
    const next = passStatusNext('s', 'pass_terminated', terminated, ['--source', 's'], null, true);
    expect(next.argv).toEqual(['gbrain', 'sync', '--source', 's']);
    expect(next.why).toContain('termination signal');
    expect(abandonedPasses(terminated, now, true)).toBe(1);
    expect(abandonedPasses(terminated, now, true, true)).toBe(0);
    expect(passStatusState(row('resumable', 'terminated', 1), null, null, now, true)).toEqual(['blocked', 'pass_abandoned']);
    // A deadline or SIGINT stop keeps resetting the streak, as before.
    expect(transientEnding(row('resumable', 'timeout').value.telemetry)).toBe(false);
    expect(passStatusState(row('resumable', 'timeout'), null, null, now, true)).toEqual(['resumable', 'pass_unfinished']);
  });

  test('the first abort decides the stop reason: a deadline that fired first stays a timeout', () => {
    const deadline = new AbortController(), signal = new AbortController();
    const both = AbortSignal.any([deadline.signal, signal.signal]);
    deadline.abort(new Error('deadline'));
    signal.abort(new SyncTerminated('SIGTERM'));
    expect(abortStop(both)).toBe('timeout');
    const first = new AbortController(), later = new AbortController();
    const other = AbortSignal.any([first.signal, later.signal]);
    first.abort(new SyncTerminated('SIGHUP'));
    later.abort(new Error('deadline'));
    expect(abortStop(other)).toBe('terminated');
    expect(abortStop(undefined)).toBe('timeout');
  });

  test('a signal owner can claim only some signals; the rest take the generic path', async () => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    try {
      installSignalHandlers();
      const seen: string[] = [];
      const release = registerSignalOwner('t', signal => { seen.push(signal); }, ['SIGTERM', 'SIGHUP']);
      expect(hasSignalOwner()).toBe(true);
      process.emit('SIGTERM');
      process.emit('SIGHUP');
      expect(seen).toEqual(['SIGTERM', 'SIGHUP']);
      process.emit('SIGPIPE' as never);
      expect(seen).toEqual(['SIGTERM', 'SIGHUP']);
      // The generic path ran: its cleanup pass then exits 141.
      await Bun.sleep(50);
      expect(exit).toHaveBeenCalledWith(141);
      release();
      expect(hasSignalOwner()).toBe(false);
    } finally { exit.mockRestore(); }
  });
});

const SYNC = ['--source', 'default', '--no-pull', '--no-embed', '--json'];
const page = (i: number, tag: string) => `---\ntype: concept\ntitle: p${i}\n---\n\nNote ${i}${tag}.\n`;
const seed = ({ root }: { root: string }) => {
  git(root, ['init', '-q']);
  for (let i = 0; i < 40; i++) { mkdirSync(join(root, `d${i % 4}`), { recursive: true }); writeFileSync(join(root, `d${i % 4}/p${i}.md`), page(i, '')); }
  git(root, ['add', '-A']); git(root, ['commit', '-qm', 'init']);
};
const editAll = (root: string, tag: string) => {
  for (let i = 0; i < 40; i++) writeFileSync(join(root, `d${i % 4}/p${i}.md`), page(i, `, ${tag}`));
  git(root, ['commit', '-qam', tag]);
};
const quiet = async <T>(run: () => Promise<T>): Promise<T> => {
  const out = spyOn(process.stdout, 'write').mockImplementation((() => true) as never);
  try { return await run(); } finally { out.mockRestore(); }
};
/** Emits `signal` the first time the next pass reaches `at`, as a supervisor's kill lands mid-pass. */
const signalAtRound = (signal: NodeJS.Signals, at = 'sync:round_start') => {
  let sent = false;
  installFaultHook(point => { if (point === at && !sent) { sent = true; process.emit(signal as never); } });
};
const passRow = async (engine: BrainEngine) => {
  const [{ incarnation }] = await engine.executeRaw<{ incarnation: string }>("SELECT incarnation::text AS incarnation FROM sources WHERE id='default'");
  return (await readPassRow(engine, 'default', incarnation))!.value;
};

for (const backend of testBackends()) {
  const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;

  test(`GST-2869 [${backend}]: SIGTERM mid-pass records terminated, exits 143, and the next sync resumes`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      expect(currentExitCode()).toBe(0);
      editAll(root, 'one');
      signalAtRound('SIGTERM');
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      installFaultHook(undefined);
      expect(currentExitCode()).toBe(143);
      expect(exit).not.toHaveBeenCalled();
      expect(kill).not.toHaveBeenCalled();
      expect(hasSignalOwner()).toBe(false);
      const ended = await passRow(engine);
      expect([ended.telemetry.outcome, ended.telemetry.stop_reason]).toEqual(['resumable', 'terminated']);
      const status = await readSyncStatus(engine, 'default', { detail: false });
      expect([status.state, status.reason]).toEqual(['resumable', 'pass_terminated']);
      expect(status.next?.argv?.slice(0, 4)).toEqual(['gbrain', 'sync', '--source', 'default']);

      setCliExitVerdict(0);
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      expect(currentExitCode()).toBe(0);
      expect((await passRow(engine)).telemetry.outcome).toBe('synced');
    } finally { exit.mockRestore(); kill.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  test(`GST-2869 [${backend}]: SIGTERM inside a batch records terminated, not the batch's timeout`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      editAll(root, 'batch');
      signalAtRound('SIGTERM', 'sync:batch_entries');
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      installFaultHook(undefined);
      expect(currentExitCode()).toBe(143);
      const ended = await passRow(engine);
      expect([ended.telemetry.outcome, ended.telemetry.stop_reason]).toEqual(['resumable', 'terminated']);
    } finally { exit.mockRestore(); kill.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  test(`GST-2869 [${backend}]: two terminated passes that committed nothing carry the streak through the pin and read pass_abandoned`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      editAll(root, 'two');
      for (const signal of ['SIGTERM', 'SIGHUP'] as const) {
        setCliExitVerdict(0);
        signalAtRound(signal);
        await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
        installFaultHook(undefined);
        expect(currentExitCode()).toBe(signal === 'SIGHUP' ? 129 : 143);
      }
      expect((await passRow(engine)).obligations.abandoned_streak).toBe(1);
      const status = await readSyncStatus(engine, 'default', { detail: false });
      expect([status.state, status.reason]).toEqual(['blocked', 'pass_abandoned']);
      expect((await readBlockedManagedSyncs(engine, ['default'])).filter(entry => entry.source_id === 'default'))
        .toEqual([expect.objectContaining({ code: 'pass_abandoned' })]);
    } finally { exit.mockRestore(); kill.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  test(`GST-2869 [${backend}]: under another owner (a worker) or without the CLI opt-in, sync claims no signal`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      const seen: string[] = [];
      const releaseWorker = registerSignalOwner('jobs-work', signal => { seen.push(signal); });
      editAll(root, 'three');
      signalAtRound('SIGTERM');
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      installFaultHook(undefined);
      releaseWorker();
      expect(seen).toEqual(['SIGTERM']);
      expect(currentExitCode()).toBe(0);
      expect((await passRow(engine)).telemetry.outcome).toBe('synced');
      let ownedDuringPass: boolean | undefined;
      editAll(root, 'four');
      installFaultHook(point => { if (point === 'sync:round_start') ownedDuringPass ??= hasSignalOwner(); });
      await quiet(() => runSync(engine, SYNC));
      expect(ownedDuringPass).toBe(false);
      expect((await passRow(engine)).telemetry.outcome).toBe('synced');
    } finally { exit.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  // Value: protects=a second SIGTERM while the pass saves exits now through the generic path (release + re-raise);
  //   fails_when=the interrupt.signal.aborted guard in the sync signal owner is dropped; why_new=no test sends a second signal;
  //   seam=none; no_claim=the real process death (process.kill mocked; pglite unit only)
  test(`GST-2869 [${backend}]: a second SIGTERM while the pass saves re-raises it through the generic path`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      editAll(root, 'twice');
      let sent = false;
      installFaultHook(point => { if (point === 'sync:round_start' && !sent) { sent = true; process.emit('SIGTERM' as never); process.emit('SIGTERM' as never); } });
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      installFaultHook(undefined);
      expect(kill).toHaveBeenCalledTimes(1);
      expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM');
      expect(hasSignalOwner()).toBe(false);
      expect(currentExitCode()).toBe(143);
    } finally { exit.mockRestore(); kill.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  // Value: protects=a SIGTERM before any managed pass is live takes the generic path and records no terminated pass;
  //   fails_when=the !managedPassLive() guard is dropped, so a pre-pass signal aborts the run instead of exiting;
  //   why_new=existing arms signal only mid-pass; seam=none; no_claim=the real process death (process.kill mocked; pglite unit only)
  test(`GST-2869 [${backend}]: a SIGTERM with no live managed pass takes the generic path and records nothing`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      editAll(root, 'early');
      signalAtRound('SIGTERM', 'sync:lease_acquired');
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      installFaultHook(undefined);
      expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM');
      expect(currentExitCode()).toBe(0);
      const ended = await passRow(engine);
      expect([ended.telemetry.outcome, ended.telemetry.stop_reason]).toEqual(['synced', null]);
    } finally { exit.mockRestore(); kill.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  // Value: protects=a save that outlasts the cleanup window still exits 128 + n through triggerCleanupAndExit (143 SIGTERM, 129 SIGHUP);
  //   fails_when=the fallback timer is dropped, uses another delay, or exits with a fixed or wrong code; why_new=no arm reaches the fallback;
  //   seam=none; no_claim=a real 3 s wall-clock wait (the timer callback is captured and fired by hand; cleanup exit spied)
  test(`GST-2869 [${backend}]: a save that outlasts the cleanup window exits 128 + n through the generic cleanup`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    const trigger = spyOn(processCleanup, 'triggerCleanupAndExit').mockImplementation((async () => undefined) as never);
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      for (const [signal, code] of [['SIGTERM', 143], ['SIGHUP', 129]] as const) {
        setCliExitVerdict(0);
        editAll(root, `late-${signal}`);
        const fallbacks: Array<() => void> = [];
        let sent = false;
        installFaultHook(point => {
          if (point !== 'sync:round_start' || sent) return;
          sent = true;
          const realSetTimeout = globalThis.setTimeout;
          globalThis.setTimeout = ((fn: () => void, ms?: number, ...rest: unknown[]) => {
            if (ms === CLEANUP_DEADLINE_MS) fallbacks.push(fn);
            return realSetTimeout(fn, ms, ...rest);
          }) as typeof setTimeout;
          try { process.emit(signal as never); } finally { globalThis.setTimeout = realSetTimeout; }
        });
        await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
        installFaultHook(undefined);
        expect(currentExitCode()).toBe(code);
        expect(trigger).not.toHaveBeenCalled();
        expect(fallbacks).toHaveLength(1);
        fallbacks[0]();
        expect(trigger).toHaveBeenCalledWith(code);
        trigger.mockClear();
      }
    } finally { exit.mockRestore(); kill.mockRestore(); trigger.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  // Value: protects=a terminated run keeps the cleanup fallback armed until its result is out, so a stalled read after the save still exits;
  //   fails_when=the fallback is cleared when the pass returns, before the watermark read and the JSON result; why_new=other arms finish fast;
  //   seam=none; no_claim=a real stalled database (the watermark read is held by a spy)
  test(`GST-2869 [${backend}]: a terminated run keeps the fallback armed until its result is out`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    let release: (value: null) => void = () => undefined;
    const stalled = new Promise<null>(resolve => { release = resolve; });
    const watermark = spyOn(watermarkModule, 'readAppliedWatermark');
    const realSetTimeout = globalThis.setTimeout, realClearTimeout = globalThis.clearTimeout;
    const fallbacks: unknown[] = [], cleared = new Set<unknown>();
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      editAll(root, 'stalled-tail');
      let sent = false;
      installFaultHook(point => {
        if (point !== 'sync:round_start' || sent) return;
        sent = true;
        globalThis.setTimeout = ((fn: () => void, ms?: number, ...rest: unknown[]) => {
          const handle = realSetTimeout(fn, ms, ...rest);
          if (ms === CLEANUP_DEADLINE_MS) fallbacks.push(handle);
          return handle;
        }) as typeof setTimeout;
        try { process.emit('SIGTERM' as never); } finally { globalThis.setTimeout = realSetTimeout; }
      });
      globalThis.clearTimeout = ((handle?: Parameters<typeof clearTimeout>[0]) => { cleared.add(handle); realClearTimeout(handle); }) as typeof clearTimeout;
      watermark.mockClear();
      watermark.mockImplementation((() => stalled) as never);
      const running = quiet(() => runSync(engine, SYNC, { ownSignals: true }));
      await waitFor(() => watermark.mock.calls.length > 0, { timeoutMs: 60_000 });
      expect(fallbacks).toHaveLength(1);
      expect(cleared.has(fallbacks[0])).toBe(false);
      release(null);
      await running;
      installFaultHook(undefined);
      expect(cleared.has(fallbacks[0])).toBe(true);
      expect(currentExitCode()).toBe(143);
      expect((await passRow(engine)).telemetry.stop_reason).toBe('terminated');
    } finally { globalThis.clearTimeout = realClearTimeout; watermark.mockRestore(); exit.mockRestore(); kill.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  // Value: protects=gbrain sync --all stopped by SIGTERM records the pass terminated and exits 143;
  //   fails_when=runSyncAll stops wrapping dispatch in the signal owner or drops the terminated exit verdict;
  //   why_new=existing arms run single-source only; seam=none; no_claim=the skipped extraction and backup extras (not observed; pglite unit only)
  test(`GST-2869 [${backend}]: sync --all stopped by SIGTERM records terminated and exits 143`, () => managedBrain(async ({ engine, root }) => {
    const exit = spyOn(process, 'exit').mockImplementation((() => undefined) as never);
    const kill = spyOn(process, 'kill').mockImplementation((() => true) as never);
    const ALL = ['--all', '--no-pull', '--no-embed', '--json'];
    try {
      installSignalHandlers();
      await quiet(() => runSync(engine, ALL, { ownSignals: true }));
      expect(currentExitCode()).toBe(0);
      editAll(root, 'all');
      signalAtRound('SIGTERM');
      await quiet(() => runSync(engine, ALL, { ownSignals: true }));
      installFaultHook(undefined);
      expect(currentExitCode()).toBe(143);
      expect(exit).not.toHaveBeenCalled();
      expect(kill).not.toHaveBeenCalled();
      expect(hasSignalOwner()).toBe(false);
      const ended = await passRow(engine);
      expect([ended.telemetry.outcome, ended.telemetry.stop_reason]).toEqual(['resumable', 'terminated']);
    } finally { exit.mockRestore(); kill.mockRestore(); }
  }, { databaseUrl, setup: seed }), 120_000);

  // Value: protects=an abort the sync took as a termination, landing while the pass row is pinned, saves stop_reason terminated;
  //   fails_when=the pin-site save goes back to a fixed 'timeout'; why_new=existing arms abort after the pin; seam=none;
  //   no_claim=a real signal at the pin (managedPassLive is false there); a writePassRow spy aborts right after the pin write
  test(`GST-2869 [${backend}]: a termination abort while the pass is pinned records terminated, not timeout`, () => managedBrain(async ({ engine, root }) => {
    await quiet(() => performSync(engine, { repoPath: root, sourceId: 'default', noPull: true, noEmbed: true }));
    editAll(root, 'pin');
    const controller = new AbortController();
    const realWrite = passRowModule.writePassRow;
    const write = spyOn(passRowModule, 'writePassRow').mockImplementation(async (tx, value, opts = {}) => {
      const written = await realWrite(tx, value, opts);
      if (opts.pin && !opts.clock && value.telemetry.outcome === 'open' && !controller.signal.aborted) controller.abort(new SyncTerminated('SIGTERM'));
      return written;
    });
    try {
      await quiet(() => performSync(engine, { repoPath: root, sourceId: 'default', noPull: true, noEmbed: true, drain: true, signal: controller.signal }));
    } finally { write.mockRestore(); }
    expect(controller.signal.aborted).toBe(true);
    const ended = await passRow(engine);
    expect([ended.telemetry.outcome, ended.telemetry.stop_reason, ended.telemetry.round]).toEqual(['resumable', 'terminated', 0]);
  }, { databaseUrl, setup: seed }), 120_000);
}

/**
 * Measured on a generated 24,000-hold PGLite mirror: a real SIGTERM reached the sync handler 19-33 s after the signal while
 * the pass's progress timers kept firing, and master's generic handler waits as long. Once the handler runs, the row is
 * saved within milliseconds, which the pglite arms above pin. This names the wait so a fix has a test to turn green.
 */
test.todo('GST-2869 follow-up [pglite]: a real SIGTERM reaches the sync handler within the 3 s window while a managed pass writes pages', () => {});