Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.7 KB
Raw Blame
/**
 * #7679 state matrix 7c of `gbrain sources refresh`: an interrupted refresh row of each kind, settled by the owner scan, a
 * plain refresh and --resume. Split from persistence-worktree-refresh.test.ts with 7d
 * (worktree-refresh-unmoved-matrix-7679.test.ts) so each Postgres file fits run-e2e.sh's per-file cap (#8025). Postgres runs
 * through test/e2e/worktree-refresh-settle-matrix-postgres.test.ts.
 */
import { afterAll, beforeAll, expect, test } from 'bun:test';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { runPersistenceAdministration } from '../src/core/persistence/administration.ts';
import { refreshWorktree, resumeWorktreeRefreshes } from '../src/core/persistence/worktree-refresh.ts';
import { withCoordinatedWrite } from '../src/core/persistence/context.ts';
import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
import { commitFile, git, page, withRefreshFixture } from './helpers/worktree-refresh-fixture.ts';

const backends = testBackends();
const engines: BrainEngine[] = [];
let closePostgres: (() => Promise<void>) | undefined;

beforeAll(async () => {
  if (backends.includes('pglite')) {
    const engine = new PGLiteEngine();
    await engine.connect({}); await engine.initSchema(); engines.push(engine);
  }
  if (backends.includes('postgres')) {
    const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
    engines.push(pg.engine); closePostgres = pg.close;
  }
}, 120_000);
afterAll(async () => {
  for (const engine of engines) { await disposePersistenceConsumer(engine); if (engine.kind === 'pglite') await engine.disconnect(); }
  await closePostgres?.();
});

const refusedWith = async (promise: Promise<unknown>, code: string) => {
  const error = await promise.then(() => null, e => e as { code?: string; suggestion?: string; message?: string; docs?: string });
  expect(error, `expected refusal ${code}`).not.toBeNull();
  expect(error!.code).toBe(code);
  return error!;
};

test('7c. #7679 state matrix: an interrupted refresh of each kind, settled by the owner scan, a plain refresh, --resume and --abandon', async () => {
  // Value: protects=each kind of interrupted refresh row leaves the row section 1.5 names when the owner scan, a plain refresh (adoption) or
  // --resume (resumeHeld) settles it; fails_when=a kind falls through to a sibling's path (an unverified HEAD adopted as the target, a syncing
  // row with an unrelated HEAD completed or refreshed over, a dead drain left standing); why_new=the #7679 tests cover syncing orphans with HEAD at
  // the target only; seam=none; no_claim=a real process death (the crash is a hook throw; the refresh lock is released as on exit).
  // #7718-#7727, #7729: --abandon rolls back only a row whose checkout never moved (a dead drain, a fence at the start commit) and admits writes
  // again; a row whose HEAD reached the target refuses refresh_in_progress and stays as it is, and an unrelated HEAD escalates to
  // recovery_required; fails_when=abandon drops its syncing or at-target guard (files left ahead of the database) or aborts over an unrelated HEAD.
  type Kind = 'draining' | 'fenced_old' | 'fenced_target' | 'fenced_unrelated' | 'merged_target' | 'merged_unrelated' | 'syncing_at_target' | 'syncing_descendant'
    | 'syncing_unrelated' | 'syncing_unrelated_synced' | 'syncing_lagging';
  type Action = 'owner_scan' | 'plain' | 'resume' | 'abandon';
  const kinds: Kind[] = ['draining', 'fenced_old', 'fenced_target', 'fenced_unrelated', 'merged_target', 'merged_unrelated', 'syncing_at_target', 'syncing_descendant',
    'syncing_unrelated', 'syncing_unrelated_synced'];
  const stateOf = (kind: Kind) => kind.split('_')[0] as 'draining' | 'fenced' | 'merged' | 'syncing';
  const crashAt = { draining: 'drained', fenced: 'fenced', merged: 'fenced', syncing: 'syncing' } as const;
  // The owner scan's row and moved count; a merged row's bookkeeping leaves it syncing, and a syncing row with an unrelated HEAD waits for --resume.
  const scanned: Record<Kind, [string, number]> = { draining: ['aborted', 1], fenced_old: ['aborted', 1], fenced_target: ['syncing', 1], fenced_unrelated: ['recovery_required', 1],
    merged_target: ['syncing', 1], merged_unrelated: ['recovery_required', 1], syncing_at_target: ['completed', 1], syncing_descendant: ['syncing', 0], syncing_unrelated: ['syncing', 0], syncing_unrelated_synced: ['syncing', 0], syncing_lagging: ['syncing', 0] };
  // syncing_lagging (HEAD at the target, members still at the start commit) is what the crash at syncing leaves; the other paths cover it in 7e/7d.
  const cells = [...kinds.flatMap(kind => (['owner_scan', 'plain', 'resume', 'abandon'] as Action[]).map(action => [kind, action] as [Kind, Action])),
    ['syncing_lagging', 'abandon'] as [Kind, Action]];
  // Test cost (#7782): a cell that leaves the checkout settled (no unfinished refresh, a HEAD the upstream contains, every member synced to it) hands
  // its fixture to the next cell instead of building a brain per cell, so each chain is at most one settled cell and then one unsettled cell.
  const endsSettled = ([kind, action]: [Kind, Action]) => action === 'abandon' ? kind === 'fenced_old' : !kind.includes('_unrelated') && kind !== 'syncing_descendant'
    && !(action === 'owner_scan' && (kind === 'fenced_target' || kind === 'merged_target'));
  const settledCells = cells.filter(endsSettled), unsettledCells = cells.filter(cell => !endsSettled(cell));
  const chains = [...unsettledCells.map((cell, i) => settledCells[i] ? [settledCells[i], cell] : [cell]), ...settledCells.slice(unsettledCells.length).map(cell => [cell])];
  expect(chains.flat()).toHaveLength(cells.length);
  for (const engine of engines) for (const chain of chains) await withRefreshFixture(engine, async f => { for (const [kind, action] of chain) {
    const label = `${kind} x ${action}`;
    const earlier = await f.refreshRows();
    if (earlier.length) {
      expect(earlier.map(r => r.state).filter(state => state !== 'aborted' && state !== 'completed'), `${label}: handoff`).toEqual([]);
      expect(git(f.root, 'merge-base', 'HEAD', 'origin/main'), `${label}: handoff`).toBe(f.head());
      expect([await f.lastCommit(f.alpha), await f.lastCommit(f.beta)], `${label}: handoff`).toEqual([f.head(), f.head()]);
    }
    const cellRows = async () => (await f.refreshRows()).filter(r => !earlier.some(e => e.id === r.id));
    const target = f.push('alpha/two.md', page('Alpha two', `Upstream for ${label}.`));
    const old = f.head();
    await expect(refreshWorktree(f.engine, f.alpha, { hooks: { boundary: point => { if (point === crashAt[stateOf(kind)]) throw new Error(`simulated crash at ${point}`); } } }))
      .rejects.toThrow('simulated crash');
    await disposePersistenceConsumer(f.engine);
    const [orphan] = await cellRows();
    // The resident consumer's owner scan may already have settled it; restore what the crash left.
    // GST-1674: an _unrelated kind is a merge that ran and was then reset under an unrelated commit, so its row records the merge as started.
    await f.engine.executeRaw(`UPDATE persistence_worktree_refreshes SET state=$2,completed_at=NULL,outcome=CASE WHEN $3::boolean THEN outcome||'{"merge_started":true}'::jsonb ELSE outcome END WHERE id=$1::uuid`,
      [orphan.id, stateOf(kind), kind.includes('_unrelated')]);
    if (kind === 'fenced_target' || kind === 'merged_target') git(f.root, 'merge', '-q', '--ff-only', target);
    if (kind === 'syncing_descendant') commitFile(f.root, 'beta/local.md', page('Local', 'Committed on top of the target.'));
    if (kind.includes('_unrelated')) { git(f.root, 'reset', '-q', '--hard', old); commitFile(f.root, 'beta/unrelated.md', page('Unrelated', 'Not the upstream commit.')); }
    if (kind === 'syncing_at_target' || kind === 'syncing_unrelated_synced') for (const member of [f.alpha, f.beta]) await f.engine.transaction(tx => withCoordinatedWrite(tx, [member],
      () => tx.executeRaw('UPDATE sources SET last_commit=$2 WHERE id=$1', [member, target]), TEST_WRITE_ATTRIBUTION));
    const head = f.head();
    const rows = async () => (await cellRows()).map(r => r.state);
    if (kind === 'draining') {
      // Ship audit (Step 7). Value: protects=writer status tells an orphaned draining refresh's writers to --resume it, never to retry (only a live
      // drain finishes on its own); fails_when=the draining branch of next stops checking liveness (master's wording); why_new=test 10 reads a live
      // drain only; seam=none; no_claim=a real process death (the crash is a hook throw; the refresh lock is released as on exit).
      const [status] = (await runPersistenceAdministration(f.engine, 'writer_status', { source_id: f.alpha }) as { worktree_refreshes: Array<Record<string, any>> }).worktree_refreshes;
      expect(status, label).toMatchObject({ refresh_id: orphan.id, state: 'draining', liveness: 'orphaned', auto_recovery: 'next_refresh',
        fix: { argv: ['gbrain', 'sources', 'refresh', f.alpha, '--resume'] }, next: `gbrain sources refresh ${f.alpha} --resume` });
    }
    if (action === 'owner_scan') {
      const [state, moved] = scanned[kind];
      expect(await resumeWorktreeRefreshes(f.engine), label).toBe(moved);
      expect(await rows(), label).toEqual([state]);
      expect(f.head(), label).toBe(head);
      continue;
    }
    if (action === 'abandon') {
      const members = [await f.lastCommit(f.alpha), await f.lastCommit(f.beta)];
      const run = refreshWorktree(f.engine, f.alpha, { abandon: true });
      if (kind === 'draining' || kind === 'fenced_old') {
        expect(await run, label).toMatchObject({ status: 'abandoned', refresh_id: orphan.id, old_head: old, target_head: target });
        expect(await rows(), label).toEqual(['aborted']);
        expect(((await cellRows())[0].outcome.refusal as { code: string }).code, label).toBe('refresh_abandoned');
      } else if (kind === 'fenced_unrelated' || kind === 'merged_unrelated') {
        const refused = await refusedWith(run, 'refresh_recovery_required');
        expect(refused.message, label).toContain(head);
        expect(await rows(), label).toEqual(['recovery_required']);
      } else {
        const refused = await refusedWith(run, 'refresh_in_progress');
        expect(refused.suggestion, label).toContain(`gbrain sources refresh ${f.alpha} --resume`);
        expect(await rows(), label).toEqual([stateOf(kind)]);
      }
      expect(f.head(), label).toBe(head);
      expect([await f.lastCommit(f.alpha), await f.lastCommit(f.beta)], label).toEqual(members);
      if (kind === 'draining') expect((await f.put(f.alpha, 'notes/after-abandon', page('After', 'admitted again'))).state ?? 'committed', label).toBe('committed');
      continue;
    }
    const run = refreshWorktree(f.engine, f.alpha, action === 'resume' ? { resume: true } : {});
    if (kind.includes('_unrelated')) {
      await refusedWith(run, 'refresh_recovery_required');
      expect(await rows(), label).toEqual(['recovery_required']);
      expect(f.head(), label).toBe(head);
      continue;
    }
    if (kind === 'syncing_descendant' && action === 'plain') {
      // The adoption completes the orphan at the descendant HEAD; the refresh itself then refuses the unpushed local commit.
      await refusedWith(run, 'refresh_diverged');
      expect(await rows(), label).toEqual(['completed']);
      expect(f.head(), label).toBe(head);
      expect(await f.lastCommit(f.alpha), label).toBe(head);
      expect(await f.lastCommit(f.beta), label).toBe(head);
      continue;
    }
    const result = await run;
    const rolledBack = kind === 'draining' || kind === 'fenced_old';
    if (action === 'resume') {
      expect(result, label).toMatchObject({ status: rolledBack ? 'abandoned' : 'completed', refresh_id: orphan.id });
      expect(await rows(), label).toEqual([rolledBack ? 'aborted' : 'completed']);
      expect(f.head(), label).toBe(head);
      continue;
    }
    if (rolledBack) {
      expect(result, label).toMatchObject({ status: 'completed', target_head: target, recovered: { refresh_id: orphan.id, state: 'aborted', status: 'abandoned' } });
      expect(await rows(), label).toEqual(['aborted', 'completed']);
    } else {
      expect(result, label).toMatchObject({ status: 'already_current', recovered: { refresh_id: orphan.id, state: 'completed', status: 'completed', previous_holder: { pid: process.pid } } });
      expect(await rows(), label).toEqual(['completed']);
    }
    const synced = kind === 'syncing_descendant' ? head : target;
    expect(f.head(), label).toBe(synced);
    expect(await f.lastCommit(f.alpha), label).toBe(synced);
    expect(await f.lastCommit(f.beta), label).toBe(synced);
  } });
}, 900_000);