Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.4 KB
Raw Blame
/**
 * Engine graduation versus every other client of the brain: older released
 * binaries, a `gbrain serve` respawned mid-copy, a resident serve that must
 * hand the source over, and stale CLI or MCP configurations after cutover.
 *
 * Protects: §3.3 and §6.4 of the plan plus §13 CEO/DX (tombstone, marker
 * hand-off, split-brain detection, database fence, one-step stale-client fix).
 * - An older release pointed at the tombstoned path fails (EEXIST/ENOTDIR)
 *   and never creates a fresh empty brain there. This half runs before the
 *   orchestrator lands, against a tombstone written to the Tombstone contract.
 * - In the rename-to-tombstone window an older release can create a stray
 *   brain; the next `--resume` refuses with `graduation_split_brain` and the
 *   target stays non-authoritative.
 * - A write by an older release in a lock gap before the tombstone reaches
 *   the target on resume (source digests re-checked).
 * - An older release configured with the target URL cannot write while the
 *   target is fenced (copying, and between verify and cutover).
 * - A respawned stdio or HTTP serve exits with `graduation_in_progress`,
 *   writes nothing and takes no lock; a resident serve hands off within 30 s.
 * - A stale CLI config and a stale MCP config each recover by running the
 *   refusal's fix once.
 * Serve processes and stale clients run in graduation-clients-serve.test.ts
 * (shared steps: test/helpers/graduation-clients-cases.ts).
 * Not covered elsewhere: these are the only multi-process client tests.
 * Older binaries are built once per tag (test/helpers/graduation-e2e.ts
 * `olderReleaseBinary`, cached under GBRAIN_OLDER_RELEASE_DIR).
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { readFileSync, statSync } from 'node:fs';
import { join } from 'node:path';
import {
  codeOf, custodyPaths, DATABASE_URL, digestChanges, fixOf, gbrain, graduationTest, olderReleaseBinary,
  previousReleaseTags, release, startGbrain, stateDigest, waitForEvent, type GbrainResult,
} from '../helpers/graduation-e2e.ts';
import { closeCases, fresh, staleHome } from '../helpers/graduation-clients-cases.ts';
import { configuredEngine, expectGraduated, planAndRun, targetRowState, withTarget } from '../helpers/graduation-scenarios.ts';

let older: { tag: string; binary: string }[] = [];
let olderError: string | null = null;

beforeAll(() => {
  try { older = previousReleaseTags(2).map(tag => ({ tag, binary: olderReleaseBinary(tag) })); }
  catch (error) { olderError = String(error); }
}, 900_000);

afterAll(closeCases);

/** The Tombstone contract written by hand (O_EXCL, 0600, fsync file + parent), for the pre-integration half. */

async function olderRun(binary: string, home: string, argv: string[], stdin?: string): Promise<GbrainResult> {
  return gbrain(argv, { home, binary, stdin, timeoutMs: 120_000 });
}

describe.skipIf(!DATABASE_URL)('graduation: older released binaries', () => {
  test('older releases refuse a tombstoned path and never create a data dir there', async () => {
    expect(olderError).toBeNull();
    expect(older.length).toBe(2);
    const c = await fresh('older-tombstone');
    const { argv, env } = await planAndRun(c);
    expect((await gbrain(argv, { home: c.fx.home, env, timeoutMs: 900_000 })).code).toBe(0);
    const home = staleHome(c);
    const tombstoneBytes = readFileSync(c.fx.dataDir);
    const retained = custodyPaths(c.fx.dataDir).graduated;
    expect(retained.length).toBe(1);
    const retainedBefore = await stateDigest(retained[0]);
    for (const { tag, binary } of older) {
      for (const [argv, stdin] of [[['query', 'acme'], undefined], [['put', 'notes/older-write', '--source', 'default'], 'older binary write\n']] as const) {
        const r = await olderRun(binary, home, [...argv], stdin);
        expect({ tag, argv: argv[0], failed: r.code !== 0 }).toEqual({ tag, argv: argv[0], failed: true });
        expect(`${r.stdout}${r.stderr}`).toMatch(/EEXIST|ENOTDIR|engine_graduated/);
        expect(statSync(c.fx.dataDir).isFile()).toBe(true);
        expect(readFileSync(c.fx.dataDir).equals(tombstoneBytes)).toBe(true);
      }
    }
    expect(digestChanges(retainedBefore, await stateDigest(retained[0]))).toEqual([]);
  }, 900_000);

  graduationTest('an older release in the rename-to-tombstone window: a stray brain is reported as split brain and the target stays withheld', async () => {
    const c = await fresh('older-window');
    const { argv, env } = await planAndRun(c);
    const child = startGbrain(argv, { home: c.fx.home, env, hooks: { events: c.events, pause: ['moved_aside'] } });
    await waitForEvent(c.events, e => e.event === 'paused', child);
    child.kill('SIGKILL');
    await child.exited;
    const home = staleHome(c);
    const stray = await olderRun(older[0].binary, home, ['put', 'notes/stray', '--source', 'default'], 'stray brain write\n');
    const createdStray = custodyPaths(c.fx.dataDir).dataDirIsDirectory;
    const resumed = await gbrain(['migrate', '--resume', '--json'], { home: c.fx.home, timeoutMs: 600_000 });
    if (createdStray) {
      expect(resumed.code).toBe(1);
      expect(codeOf(resumed.json)).toBe('graduation_split_brain');
      expect(fixOf(resumed.json)?.next).toBe('ask_user');
      expect(await targetRowState(c.target.url)).not.toBe('authoritative');
      const status = await gbrain(['migrate', '--status', '--json'], { home: c.fx.home });
      const text = JSON.stringify(status.json);
      expect(text).toContain(c.fx.dataDir);
      expect(text).toContain('.graduated-');
    } else {
      expect(stray.code).not.toBe(0);
      expect(resumed.code).toBe(0);
      await expectGraduated(c, 'window without a stray brain');
    }
  }, 900_000);

  graduationTest('an older release after the tombstone fails EEXIST; resume completes', async () => {
    const c = await fresh('older-after-tombstone');
    const { argv, env } = await planAndRun(c);
    const child = startGbrain(argv, { home: c.fx.home, env, hooks: { events: c.events, pause: ['tombstoned'] } });
    await waitForEvent(c.events, e => e.event === 'paused', child);
    child.kill('SIGKILL');
    await child.exited;
    for (const { binary } of older) {
      const r = await olderRun(binary, staleHome(c), ['query', 'acme']);
      expect(r.code).not.toBe(0);
      expect(statSync(c.fx.dataDir).isFile()).toBe(true);
    }
    expect((await gbrain(['migrate', '--resume', '--json'], { home: c.fx.home, timeoutMs: 600_000 })).code).toBe(0);
    await expectGraduated(c, 'after tombstone kill');
  }, 900_000);

  for (const boundary of ['verified', 'source_closed'] as const) {
    graduationTest(`an older-release write to the source in the lock gap after ${boundary} reaches the target on resume`, async () => {
      const c = await fresh(`older-gap-${boundary}`);
      const { argv, env } = await planAndRun(c);
      const child = startGbrain(argv, { home: c.fx.home, env, hooks: { events: c.events, pause: [boundary] } });
      await waitForEvent(c.events, e => e.event === 'paused', child);
      child.kill('SIGKILL');
      await child.exited;
      const aware = await gbrain(['put', 'notes/lock-gap-aware', '--source', 'default', '--json'], { home: c.fx.home, stdin: 'a graduation-aware binary must not write here\n', timeoutMs: 120_000 });
      if (boundary === 'source_closed') {
        expect({ code: aware.code, error: codeOf(aware.json) }).toEqual({ code: 1, error: 'graduation_interrupted' });
      }
      const write = await olderRun(older[0].binary, c.fx.home, ['put', 'notes/lock-gap-write', '--source', 'default'], 'written by an older release in the lock gap\n');
      expect({ code: write.code, stderr: write.code === 0 ? '' : write.stderr.slice(-1500) }).toEqual({ code: 0, stderr: '' });
      const resumed = await gbrain(['migrate', '--resume', '--json'], { home: c.fx.home, timeoutMs: 900_000 });
      expect({ code: resumed.code, stderr: resumed.code === 0 ? '' : resumed.stderr.slice(-3000) }).toEqual({ code: 0, stderr: '' });
      await withTarget(c.target.url, async t => {
        const [row] = await t.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM pages WHERE source_id='default' AND slug='notes/lock-gap-write'`);
        expect(Number(row.n)).toBe(1);
      });
      expect(configuredEngine(c.fx).engine).toBe('postgres');
    }, 900_000);
  }

  graduationTest('an older release configured with the target URL cannot write while the target is fenced', async () => {
    const c = await fresh('older-vs-fence');
    const { argv, env } = await planAndRun(c, { extra: ['--batch-size', '2'] });
    const child = startGbrain(argv, { home: c.fx.home, env, hooks: { events: c.events, pause: ['batch_copied@pages', 'verified'] } });
    for (const ordinal of [1, 2]) {
      const paused = await waitForEvent(c.events, e => e.event === 'paused' && e.ordinal === ordinal, child);
      const before = await stateDigest(join(c.fx.dir, 'nonexistent'), c.target.url);
      const probeHome = join(c.fx.dir, `older-target-${ordinal}`);
      const r = await olderRun(older[0].binary, probeHome, ['put', 'notes/fenced', '--source', 'default'], 'must not land\n');
      expect({ boundary: paused.boundary, failed: r.code !== 0 }).toEqual({ boundary: paused.boundary, failed: true });
      expect(digestChanges(before, await stateDigest(join(c.fx.dir, 'nonexistent'), c.target.url))).toEqual([]);
      release(c.events, ordinal);
    }
    const done = await child.exited;
    expect(done.code).toBe(0);
    await expectGraduated(c, 'after fenced probes');
  }, 900_000);
});