Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.6 KB
Raw Blame
/**
 * Refactor wave 1, W4 sync (A17 + Eng contract "Sync"): controlled-order
 * tests for the SyncRun phases.
 *
 * Each test pins one interleaving between an awaited phase step and a
 * concurrent actor (the stall watchdog, a caller --timeout/SIGINT abort, the
 * SIGTERM cleanup pass, a commit landing mid-run), then asserts the DURABLE
 * state after a restart — a fresh engine on the same PGLite data directory —
 * not just the returned status: the bookmark (`sync.last_commit`), the
 * op-checkpoint rows, and that a resumed run converges without re-importing
 * banked files.
 *
 * Interleavings are forced with latches, never sleeps: `getPage(slug,
 * {includeDeleted: true})` is the first engine call of every per-file import
 * (src/core/import-file.ts), so holding it holds one file's import "in
 * flight"; `executeRawDirect` carries every checkpoint write
 * (src/core/op-checkpoint.ts). The stall watchdog runs on real timers with a
 * 1s budget and is observed through its stderr line.
 *
 * Serial: spawns git, mutates GBRAIN_SYNC_* env, drives the process-cleanup
 * SIGTERM handler, and shares one on-disk PGLite brain.
 */
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, spyOn, test } from 'bun:test';
import { execSync } from 'child_process';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import type { BrainEngine } from '../src/core/engine.ts';
import { loadOpCheckpoint, syncFingerprint } from '../src/core/op-checkpoint.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { _resetForTests as resetProcessCleanup, installSignalHandlers } from '../src/core/process-cleanup.ts';
import { performSync } from '../src/commands/sync.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';

const ENV_KEYS = [
  'GBRAIN_SYNC_STALL_ABORT_SECONDS',
  'GBRAIN_SYNC_CHECKPOINT_EVERY',
  'GBRAIN_SYNC_CHECKPOINT_SECONDS',
] as const;

let brainDir: string;
let engine: PGLiteEngine;
let repoPath: string;

async function openEngine(init: boolean): Promise<PGLiteEngine> {
  const e = new PGLiteEngine();
  await e.connect({ engine: 'pglite', database_path: join(brainDir, 'brain.pglite') });
  if (init) await e.initSchema();
  return e;
}

/** Restart: drop the engine and reopen the same (already initialized) data directory. */
async function restart(): Promise<void> {
  await engine.disconnect();
  engine = await openEngine(false);
}

function git(args: string): string {
  return execSync(`git ${args}`, { cwd: repoPath, stdio: 'pipe' }).toString().trim();
}

function commitPages(names: string[], msg: string): string {
  mkdirSync(join(repoPath, 'notes'), { recursive: true });
  for (const n of names) writeFileSync(join(repoPath, 'notes', `${n}.md`), `---\ntype: concept\ntitle: ${n}\n---\n\nBody for ${n}.\n`);
  git('add -A');
  git(`commit -q -m ${JSON.stringify(msg)}`);
  return git('rev-parse HEAD');
}

async function bookmark(): Promise<string | null> {
  return engine.getConfig('sync.last_commit');
}

async function banked(lastCommit: string): Promise<string[]> {
  return (await loadOpCheckpoint(engine, { op: 'sync', fingerprint: syncFingerprint({ lastCommit }) })).sort();
}

async function pinned(lastCommit: string): Promise<string[]> {
  return loadOpCheckpoint(engine, { op: 'sync-target', fingerprint: syncFingerprint({ lastCommit }) });
}

type Gate = { reached: Promise<string>; release: () => void };

/**
 * Hold the Nth per-file import at its first engine call (one import may read
 * its page more than once; only the first read of each slug counts). Returns
 * the gate and a restore handle; `imported` records every slug whose import
 * started, in order.
 */
function latchImport(target: BrainEngine, nth: number): { gate: Gate; imported: string[]; restore: () => void } {
  const orig = target.getPage;
  const imported: string[] = [];
  let release!: () => void;
  let reach!: (slug: string) => void;
  const held = new Promise<void>((r) => { release = r; });
  const reached = new Promise<string>((r) => { reach = r; });
  target.getPage = async function (this: BrainEngine, slug: string, opts?: Parameters<BrainEngine['getPage']>[1]) {
    if (opts && (opts as { includeDeleted?: boolean }).includeDeleted === true && !imported.includes(slug)) {
      imported.push(slug);
      if (imported.length === nth) { reach(slug); await held; }
    }
    return orig.call(this, slug, opts);
  } as BrainEngine['getPage'];
  return { gate: { reached, release }, imported, restore: () => { target.getPage = orig; } };
}

/** Count per-file imports that start while `fn` runs. */
async function countImports<T>(fn: () => Promise<T>): Promise<{ value: T; imports: string[] }> {
  const { gate, imported, restore } = latchImport(engine, Number.MAX_SAFE_INTEGER);
  void gate;
  try { return { value: await fn(), imports: imported }; } finally { restore(); }
}

/** Capture stderr lines (serr routes through console.error) and resolve when one matches. */
function watchStderr(pattern: RegExp): { seen: Promise<void>; lines: string[]; restore: () => void } {
  const lines: string[] = [];
  let hit!: () => void;
  const seen = new Promise<void>((r) => { hit = r; });
  const spy = spyOn(console, 'error').mockImplementation((...args: unknown[]) => {
    const line = args.map(String).join(' ');
    lines.push(line);
    if (pattern.test(line)) hit();
  });
  return { seen, lines, restore: () => spy.mockRestore() };
}

describe('SyncRun controlled-order interleavings (W4 sync, A17)', () => {
  beforeAll(async () => {
    brainDir = mkdtempSync(join(tmpdir(), 'gbrain-syncrun-order-'));
    engine = await openEngine(true);
  }, 120_000);

  afterAll(async () => {
    if (engine) await engine.disconnect();
    rmSync(brainDir, { recursive: true, force: true });
  }, 60_000);

  let c0: string;
  beforeEach(async () => {
    await resetPgliteState(engine);
    repoPath = mkdtempSync(join(tmpdir(), 'gbrain-syncrun-repo-'));
    execSync('git init -q', { cwd: repoPath });
    git('config user.email "[email protected]"');
    git('config user.name "Test"');
    commitPages(['base'], 'initial');
    const first = await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
    expect(first.status).toBe('first_sync');
    c0 = (await bookmark())!;
    process.env.GBRAIN_SYNC_CHECKPOINT_EVERY = '1';
  }, 60_000);

  afterEach(() => {
    for (const k of ENV_KEYS) delete process.env[k];
    rmSync(repoPath, { recursive: true, force: true });
  });

  test('watchdog fires while an import is in flight: the file completes, partial stall_timeout, banked work survives restart', async () => {
    process.env.GBRAIN_SYNC_STALL_ABORT_SECONDS = '1';
    const c1 = commitPages(['a', 'b', 'c'], 'three');
    const { gate, imported, restore } = latchImport(engine, 2);
    const stderr = watchStderr(/no import progress for 1s — aborting \(stall watchdog\)/);
    try {
      const running = performSync(engine, { repoPath, noPull: true, noEmbed: true });
      const inFlight = await gate.reached;
      await stderr.seen; // the watchdog fired while file 2 was still awaiting
      gate.release();
      const res = await running;
      expect(res.status).toBe('partial');
      expect(res.reason).toBe('stall_timeout');
      expect(res.filesImported).toBe(2); // the in-flight file finished (no mid-import kill)
      expect(imported.length).toBe(2); // the drain stopped before file 3
      expect(await engine.getPage(inFlight)).not.toBeNull();
    } finally { stderr.restore(); restore(); }

    await restart();
    expect(await bookmark()).toBe(c0); // never advanced on a partial
    expect((await banked(c0)).length).toBe(2);
    expect(await pinned(c0)).toEqual([c1]);

    delete process.env.GBRAIN_SYNC_STALL_ABORT_SECONDS;
    const resumed = await countImports(() => performSync(engine, { repoPath, noPull: true, noEmbed: true }));
    expect(resumed.value.status).toBe('synced');
    expect(resumed.imports.length).toBe(1); // only the unbanked file re-enters the drain
    expect(await bookmark()).toBe(c1);
    expect(await banked(c0)).toEqual([]);
    for (const s of ['notes/a', 'notes/b', 'notes/c']) expect(await engine.getPage(s)).not.toBeNull();
  }, 60_000);

  test('the in-flight import completes before the watchdog budget: progress resets and the run converges', async () => {
    process.env.GBRAIN_SYNC_STALL_ABORT_SECONDS = '1';
    const c1 = commitPages(['a', 'b', 'c'], 'three');
    const { gate, restore } = latchImport(engine, 2);
    const stderr = watchStderr(/stall watchdog/);
    try {
      const running = performSync(engine, { repoPath, noPull: true, noEmbed: true });
      await gate.reached;
      gate.release(); // completes well inside the 1s budget
      const res = await running;
      expect(res.status).toBe('synced');
      expect(stderr.lines.some((l) => /stall watchdog/.test(l))).toBe(false);
    } finally { stderr.restore(); restore(); }

    await restart();
    expect(await bookmark()).toBe(c1);
    expect(await banked(c0)).toEqual([]);
    expect(await pinned(c0)).toEqual([]);
  }, 60_000);

  test('caller abort lands while an import is awaiting: the next per-file check stops the drain with partial timeout', async () => {
    const c1 = commitPages(['a', 'b', 'c'], 'three');
    const ac = new AbortController();
    const { gate, imported, restore } = latchImport(engine, 2);
    try {
      const running = performSync(engine, { repoPath, noPull: true, noEmbed: true, signal: ac.signal });
      await gate.reached;
      ac.abort();
      gate.release();
      const res = await running;
      expect(res.status).toBe('partial');
      expect(res.reason).toBe('timeout');
      expect(res.filesImported).toBe(2);
      expect(res.bankedFiles).toBe(2);
      expect(imported.length).toBe(2);
    } finally { restore(); }

    await restart();
    expect(await bookmark()).toBe(c0);
    expect((await banked(c0)).length).toBe(2);
    expect(await pinned(c0)).toEqual([c1]);
    const resumed = await countImports(() => performSync(engine, { repoPath, noPull: true, noEmbed: true }));
    expect(resumed.value.status).toBe('synced');
    expect(resumed.imports.length).toBe(1);
    expect(await bookmark()).toBe(c1);
  }, 60_000);

  test('caller abort lands during the LAST import: the drain finishes and the post-drain check still refuses to advance', async () => {
    const c1 = commitPages(['a', 'b', 'c'], 'three');
    const ac = new AbortController();
    const { gate, imported, restore } = latchImport(engine, 3);
    try {
      const running = performSync(engine, { repoPath, noPull: true, noEmbed: true, signal: ac.signal });
      await gate.reached;
      ac.abort();
      gate.release();
      const res = await running;
      expect(res.status).toBe('partial');
      expect(res.reason).toBe('timeout');
      expect(res.filesImported).toBe(3);
      expect(imported.length).toBe(3);
    } finally { restore(); }

    await restart();
    expect(await bookmark()).toBe(c0); // D-V3-1: abort never advances last_commit
    expect((await banked(c0)).length).toBe(3);
    const resumed = await countImports(() => performSync(engine, { repoPath, noPull: true, noEmbed: true }));
    expect(resumed.value.status).toBe('synced');
    expect(resumed.imports).toEqual([]); // everything was banked; the resume only advances
    expect(await bookmark()).toBe(c1);
    expect(await banked(c0)).toEqual([]);
  }, 60_000);

  test('watchdog fires during a checkpoint write: the write lands, the banked count matches the durable rows', async () => {
    process.env.GBRAIN_SYNC_STALL_ABORT_SECONDS = '1';
    const c1 = commitPages(['a', 'b', 'c'], 'three');
    const orig = engine.executeRawDirect;
    let release!: () => void;
    let reach!: () => void;
    const held = new Promise<void>((r) => { release = r; });
    const reached = new Promise<void>((r) => { reach = r; });
    let pathWrites = 0;
    engine.executeRawDirect = async function (this: BrainEngine, sql: string, params?: unknown[]) {
      if (Array.isArray(params) && params[0] === 'sync' && Array.isArray(params[2]) && ++pathWrites === 2) {
        reach();
        await held;
      }
      return orig.call(this, sql, params);
    } as BrainEngine['executeRawDirect'];
    const stderr = watchStderr(/no import progress for 1s — aborting \(stall watchdog\)/);
    try {
      const running = performSync(engine, { repoPath, noPull: true, noEmbed: true });
      await reached; // the second file's checkpoint flush is mid-write
      await stderr.seen; // the watchdog fired during the write
      release();
      const res = await running;
      expect(res.status).toBe('partial');
      expect(res.reason).toBe('stall_timeout');
      expect(res.bankedFiles).toBe(2);
      expect(res.filesImported).toBe(2);
    } finally { stderr.restore(); engine.executeRawDirect = orig; }

    await restart();
    expect(await bookmark()).toBe(c0);
    expect((await banked(c0)).length).toBe(2); // the in-flight write is durable and counted exactly once
    expect(await pinned(c0)).toEqual([c1]);
  }, 60_000);

  test('SIGTERM cleanup banks the unflushed delta while an import is in flight', async () => {
    process.env.GBRAIN_SYNC_CHECKPOINT_EVERY = '1000'; // only the first-file flush fires on its own
    process.env.GBRAIN_SYNC_CHECKPOINT_SECONDS = '3600';
    const c1 = commitPages(['a', 'b', 'c', 'd'], 'four');
    const ac = new AbortController();
    const { gate, restore } = latchImport(engine, 4);
    const exits: number[] = [];
    const exitSpy = spyOn(process, 'exit').mockImplementation(((code?: number) => { exits.push(code ?? 0); }) as typeof process.exit);
    resetProcessCleanup();
    installSignalHandlers();
    try {
      const running = performSync(engine, { repoPath, noPull: true, noEmbed: true, signal: ac.signal });
      await gate.reached; // three files completed; only the first was flushed
      expect((await banked(c0)).length).toBe(1);
      process.emit('SIGTERM');
      for (let i = 0; i < 200 && exits.length === 0; i++) await new Promise((r) => setTimeout(r, 10));
      expect(exits).toEqual([143]);
      // The cleanup pass banked the delta with no retry and moved nothing else.
      expect((await banked(c0)).length).toBe(3);
      expect(await bookmark()).toBe(c0);
      ac.abort();
      gate.release();
      const res = await running;
      expect(res.status).toBe('partial');
    } finally {
      restore();
      exitSpy.mockRestore();
      resetProcessCleanup();
    }

    await restart();
    expect(await bookmark()).toBe(c0);
    expect(await pinned(c0)).toEqual([c1]);
    delete process.env.GBRAIN_SYNC_CHECKPOINT_EVERY;
    delete process.env.GBRAIN_SYNC_CHECKPOINT_SECONDS;
    const resumed = await countImports(() => performSync(engine, { repoPath, noPull: true, noEmbed: true }));
    expect(resumed.value.status).toBe('synced');
    expect(resumed.imports).toEqual([]); // partial() banked the last file too
    expect(await bookmark()).toBe(c1);
  }, 60_000);

  test('a commit landing mid-drain: the bookmark advances to the pin, and the next run picks up the new commit', async () => {
    const c1 = commitPages(['a', 'b'], 'two');
    const { gate, restore } = latchImport(engine, 1);
    let c2 = '';
    try {
      const running = performSync(engine, { repoPath, noPull: true, noEmbed: true });
      await gate.reached;
      c2 = commitPages(['late'], 'forward commit during the drain');
      gate.release();
      const res = await running;
      expect(res.status).toBe('synced');
      expect(res.toCommit).toBe(c1);
    } finally { restore(); }

    await restart();
    expect(await bookmark()).toBe(c1); // pinned target, not live HEAD
    expect(await engine.getPage('notes/late')).toBeNull();
    const next = await countImports(() => performSync(engine, { repoPath, noPull: true, noEmbed: true }));
    expect(next.value.status).toBe('synced');
    expect(next.imports).toEqual(['notes/late']);
    expect(await bookmark()).toBe(c2);
  }, 60_000);

  test('a gc\'d bookmark object falls back to full sync: the bookmark reaches HEAD, removed files are reconciled, and a restart is up to date', async () => {
    const opts = { repoPath, noPull: true, noEmbed: true, sourceId: 'default' } as const;
    const sourceAnchor = async () =>
      (await engine.executeRaw<{ last_commit: string | null }>(`SELECT last_commit FROM sources WHERE id = 'default'`))[0]?.last_commit ?? null;
    await performSync(engine, { ...opts, full: true });
    commitPages(['keep', 'gone'], 'two');
    expect((await performSync(engine, opts)).status).toBe('synced');
    const stored = (await sourceAnchor())!;
    // Rewrite history so the stored bookmark object no longer exists anywhere;
    // the rewritten line still commits gone.md before deleting it (a file that
    // was never committed is kept as DB-only content, #2426).
    git('checkout -q --orphan rewritten');
    git('add -A');
    git('commit -q -m rewritten');
    git('rm -q notes/gone.md');
    git('commit -q -m "remove gone"');
    git('branch -D master 2>/dev/null || git branch -D main 2>/dev/null || true');
    git('reflog expire --expire=now --all');
    git('gc -q --prune=now');
    const head = git('rev-parse HEAD');
    expect(() => git(`cat-file -t ${stored}`)).toThrow();

    const stderr = watchStderr(/object missing \(gc'd after history rewrite\)\. Running full reimport\./);
    let res;
    try { res = await performSync(engine, opts); } finally { stderr.restore(); }
    expect(stderr.lines.some((l) => /Running full reimport/.test(l))).toBe(true);
    expect(res.status).toBe('first_sync');
    expect(res.deleted).toBe(1);

    await restart();
    expect(await sourceAnchor()).toBe(head);
    expect(await engine.getPage('notes/gone')).toBeNull();
    expect(await engine.getPage('notes/keep')).not.toBeNull();
    expect((await performSync(engine, opts)).status).toBe('up_to_date');
  }, 90_000);
});