Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.4 KB
Raw Blame
/**
 * #6712 PR 4b lane R (design v3.2 §3.7, probe 36): every reader that read the managed-sync cursor reads the pass row of a
 * non-company source. Fixtures on in-memory PGLite: one managed source per test (a claimed worktree over a real Git
 * checkout, so HEAD reads), its pass row written with `writePassRow`, and a pass that stops on every tick for an hour
 * simulated by writing the row once per five-minute tick and stamping `updated_at` back to that tick.
 *
 * Unhealthy (blocked, a not-converged streak, a settle-mismatch streak): the backlog lists it, movement is `not_moving`,
 * doctor `managed_sync_not_moving` and `git_held_files` warn, `gbrain sources writer movement` exits 1, the blocked
 * reader lists it, doctor `managed_sync_blocked` fails past the hour and get_brain_health's `sync_blocked` band counts it.
 * Healthy (converged at HEAD, ended `synced`): nothing pending anywhere; a commit past the converged target is pending
 * again. Also: the lease holder's progress is the pass row's write, `set-strategy` refuses during a drain lease or an
 * open episode, the checkpoint-blocker map lists what blocks an open pass's checkpoint, receipt compaction keeps an open
 * pass's receipts and the newest pass receipt, and the repair busy set adds nothing for an open pass (probe 53, busy side).
 */
import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { syncLockId, tryAcquireDbLock } from '../src/core/db-lock.ts';
import { withEnv } from './helpers/with-env.ts';
import { MANAGED_SYNC_PASS_OP, writePassRow, type ManagedSyncPassValue } from '../src/core/persistence/sync-pass-row.ts';
import { readManagedSyncBacklog } from '../src/core/persistence/sync-drain.ts';
import { readSourceMovement } from '../src/core/persistence/sync-movement.ts';
import { movementReport } from '../src/commands/sources-writer-movement-cli.ts';
import { managedSyncMovementEntry } from '../src/commands/doctor/checks/managed-sync-movement.ts';
import { managedSyncBlockedEntry } from '../src/commands/doctor/checks/managed-sync-blocked.ts';
import { gitHeldFilesCheck } from '../src/commands/doctor/checks/git-holds.ts';
import { readBlockedManagedSyncs } from '../src/core/persistence/sync-blocked.ts';
import { readBrainHealthBands } from '../src/commands/doctor/brain-health-bands.ts';
import { readDrainHolder } from '../src/core/persistence/sync-lease.ts';
import { readCheckpointBlockerMap } from '../src/core/persistence/terminal-failures.ts';
import { compactReceipt, compactWriteReceiptsDetailed } from '../src/core/persistence/journal.ts';
import { loadRepairBusySet, repairBusy } from '../src/core/persistence/repair-busy.ts';
import { runSources } from '../src/commands/sources.ts';

const home = mkdtempSync(join(tmpdir(), 'gbrain-pass-readers-'));
let engine: BrainEngine;
const MIN = 60_000;
const ago = (ms: number) => new Date(Date.now() - ms).toISOString();

interface Fixture { id: string; root: string; worktreeId: string; incarnation: string }
const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, '-c', 'user.name=alice-example', '-c', '[email protected]', '-c', 'commit.gpgsign=false', ...args], { encoding: 'utf8' }).trim();
function commit(root: string, file: string, body: string): string {
  writeFileSync(join(root, file), body);
  git(root, 'add', file);
  git(root, 'commit', '-q', '-m', `edit ${file}`);
  return git(root, 'rev-parse', 'HEAD');
}
async function managedSource(): Promise<Fixture & { head: string }> {
  const id = `pr-${randomUUID().slice(0, 8)}`, root = join(home, id);
  mkdirSync(root, { recursive: true });
  git(root, 'init', '-q');
  const head = commit(root, 'note.md', '# Note\n');
  await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, root]);
  await claimWorktree(engine, id, root);
  await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
  const [row] = await engine.executeRaw<{ worktree_id: string; incarnation: string }>(
    'SELECT b.worktree_id::text AS worktree_id, b.source_incarnation::text AS incarnation FROM persistence_source_bindings b WHERE b.source_id=$1', [id]);
  return { id, root, worktreeId: row!.worktree_id, incarnation: row!.incarnation, head };
}

type PassShape = { outcome: ManagedSyncPassValue['telemetry']['outcome']; stop_reason?: string | null; converged?: string | null; not_converged_streak?: number;
  mismatch?: ManagedSyncPassValue['obligations']['mismatch']; remaining?: number; stuck_since?: string | null; pass_id?: string; target?: string;
  episode?: ManagedSyncPassValue['obligations']['episode']; ended_at?: string };
function passValue(f: Fixture, s: PassShape): ManagedSyncPassValue {
  const started = ago(2 * MIN);
  return { source_id: f.id, incarnation: f.incarnation, lease_token: randomUUID(),
    obligations: { baseline_commit: null, mismatch: s.mismatch ?? {}, not_converged_streak: s.not_converged_streak ?? 0, faults: {},
      ...(s.stuck_since !== undefined ? { stuck_since: s.stuck_since } : {}), ...(s.episode ? { episode: s.episode } : {}),
      ...(s.converged ? { converged: { target: s.converged, at: started, request_hwm: '0', open_at_stamp: [], parser_fp: 'fp', pack: null, selection_fp: 'sel', last_full_at: started, passes_since_full: 1 } } : {}) },
    telemetry: { pass_id: s.pass_id ?? randomUUID(), target: s.target ?? s.converged ?? 'HEAD', started_at: started, ended_at: s.outcome === 'open' ? null : s.ended_at ?? ago(MIN), kind: 'full', round: 4,
      outcome: s.outcome, stop_reason: s.stop_reason ?? null, candidates: 10, settled: 10 - (s.remaining ?? 0), remaining: s.remaining ?? 0, counts: {}, duration_ms: null, timings_ms: {} } };
}
async function writePass(f: Fixture, s: PassShape, updatedAt?: string): Promise<ManagedSyncPassValue> {
  const value = passValue(f, s);
  expect(await writePassRow(engine, value, { pin: true })).toBe(true);
  if (updatedAt) await engine.executeRaw('UPDATE op_checkpoints SET updated_at=$3::timestamptz WHERE op=$1 AND fingerprint=$2', [MANAGED_SYNC_PASS_OP, `${f.id}:${f.incarnation}`, updatedAt]);
  return value;
}
/**
 * A pass that ends `ending` on every five-minute tick for an hour (12 ticks); the newest row is five minutes old. It has been
 * stopping for 61 minutes: doctor `managed_sync_blocked` and the health band fail only past the hour (strictly longer than
 * MANAGED_SYNC_BLOCKED_FAIL_MS), so a `stuck_since` of exactly an hour would sit on the boundary.
 */
async function stuckForAnHour(f: Fixture & { head: string }, ending: 'blocked' | 'not_converged' | 'settle_mismatch'): Promise<void> {
  const stuckSince = ago(61 * MIN);
  for (let tick = 1; tick <= 12; tick++) {
    await writePass(f, { outcome: ending === 'blocked' ? 'blocked' : 'resumable', stop_reason: ending === 'blocked' ? 'reconcile_read_incomplete' : 'not_converged',
      not_converged_streak: ending === 'not_converged' ? tick : 0, remaining: 3, target: f.head, stuck_since: stuckSince,
      mismatch: ending === 'settle_mismatch' ? { 'notes/flaky.md': { level: 'abc', streak: Math.min(tick, 2) } } : {} }, ago((65 - tick * 5) * MIN));
  }
}
/** The same hour, from a gbrain that writes no `stuck_since`: one row, untouched for the hour. */
const stuckRowOnly = (f: Fixture & { head: string }) => writePass(f, { outcome: 'resumable', stop_reason: 'not_converged', not_converged_streak: 12, remaining: 3, target: f.head, ended_at: ago(61 * MIN) }, ago(61 * MIN));
const converged = (f: Fixture & { head: string }) => writePass(f, { outcome: 'synced', converged: f.head, target: f.head });
const forget = (f: Fixture) => engine.executeRaw('DELETE FROM op_checkpoints WHERE op=$1 AND fingerprint=$2', [MANAGED_SYNC_PASS_OP, `${f.id}:${f.incarnation}`]);
const one = async (f: Fixture) => (await readSourceMovement(engine, { sourceIds: [f.id] }))[0]!;
const ctx = () => ({ engine }) as unknown as Parameters<typeof managedSyncMovementEntry.run>[0];
const doctorMovement = async (f: Fixture) => {
  const [check] = await managedSyncMovementEntry.run(ctx()) as unknown as Array<{ status: string; message: string; details: { sources: Array<{ source_id: string }> } }>;
  return { status: check!.status, mine: check!.details.sources.some(s => s.source_id === f.id), message: check!.message };
};

let requestSeq = 0;
async function request(f: Fixture, state: 'committed' | 'running' | 'queued', intent: Record<string, unknown>, completedAt?: string): Promise<string> {
  const requestId = randomUUID();
  await engine.executeRaw(`INSERT INTO persistence_requests(principal_kind,principal_id,request_id,operation,source_id,source_incarnation,slug,worktree_id,digest,authority,intent,intent_bytes,terminal_reservation,state,completed_at)
    VALUES('local_cli','cli:example',$1::uuid,'submit_job',$2,$3::uuid,$4,$5::uuid,'d','{}'::jsonb,$6::text::jsonb,1,16384,$7,$8::timestamptz)`,
  [requestId, f.id, f.incarnation, `notes/r${++requestSeq}`, f.worktreeId, JSON.stringify(intent), state, completedAt ?? null]);
  return requestId;
}
const compacted = async (requestId: string) => (await engine.executeRaw<{ compacted: boolean }>('SELECT compacted FROM persistence_requests WHERE request_id=$1::uuid', [requestId]))[0]!.compacted;

beforeAll(async () => {
  const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engine = lite;
  for (const [key, value] of [['persistence.sync_preparation_ms', '1000'], ['persistence.maintenance_preparation_ms', '1000'], ['persistence.preparation_ceiling_ms', '60000']]) {
    await engine.executeRaw('INSERT INTO config(key,value) VALUES($1,$2) ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value', [key, value]);
  }
}, 120_000);
afterAll(async () => { await engine.disconnect(); rmSync(home, { recursive: true, force: true }); });

/** Each §3.7 health reader, asserting it reads the source unhealthy. */
const UNHEALTHY: Array<[string, (f: Fixture, ending: string) => Promise<void>]> = [
  ['backlog (readManagedSyncBacklog)', async (f, ending) => {
    const [backlog] = await readManagedSyncBacklog(engine, [f.id]);
    expect(backlog).toMatchObject({ source_id: f.id, remaining: 3, pass: { stuck: ending } });
  }],
  ['movement (readSourceMovement)', async (f, ending) => {
    expect(await one(f)).toMatchObject({ movement_state: 'not_moving', data_moving: false, cursor: null, pass: { stuck: ending } });
  }],
  ['sources writer movement', async f => {
    const after = await one(f);
    expect(movementReport(after, after, 60_000)).toMatchObject({ state: 'not_moving', exit: 1, code: 'managed_sync_not_moving' });
  }],
  ['doctor managed_sync_not_moving', async f => {
    expect(await doctorMovement(f)).toMatchObject({ status: 'warn', mine: true });
  }],
  ['doctor git_held_files', async (f, ending) => {
    const held = await gitHeldFilesCheck(engine, [f.id]);
    expect(held.status).toBe('warn');
    expect((held.details as { passes?: Array<{ source_id: string; reason: string }> }).passes).toEqual([expect.objectContaining({ source_id: f.id, reason: ending })]);
  }],
  ['sync-blocked (readBlockedManagedSyncs)', async f => {
    const [blocked] = await readBlockedManagedSyncs(engine, [f.id]);
    expect(blocked).toMatchObject({ source_id: f.id });
    expect(Date.now() - Date.parse(blocked!.blocked_since)).toBeGreaterThanOrEqual(60 * MIN);
  }],
  ['doctor managed_sync_blocked', async f => {
    const [check] = await managedSyncBlockedEntry.run(ctx()) as unknown as Array<{ status: string; details?: { sources: Array<{ source_id: string }> } }>;
    expect(check!.status).toBe('fail');
    expect(check!.details?.sources.map(s => s.source_id)).toContain(f.id);
  }],
  ['get_brain_health sync_blocked band', async f => {
    expect((await readBrainHealthBands(engine, { sourceIds: [f.id] })).sync_blocked.band).toBe('1-9');
  }],
];

describe('probe 36: a pass that keeps stopping reads unhealthy on every reader', () => {
  for (const ending of ['blocked', 'not_converged', 'settle_mismatch'] as const) {
    for (const [reader, assertUnhealthy] of UNHEALTHY) {
      test(`${ending} on every tick for an hour: ${reader}`, () => withEnv({ GBRAIN_HOME: home }, async () => {
        const f = await managedSource();
        try {
          await stuckForAnHour(f, ending);
          await assertUnhealthy(f, ending);
        } finally { await forget(f); }
      }), 60_000);
    }
  }

  test('a not-converged row nobody touched for an hour (no stuck_since) also reads unhealthy and past the hour', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    try {
      await stuckRowOnly(f);
      expect((await one(f)).movement_state).toBe('not_moving');
      const [blocked] = await readBlockedManagedSyncs(engine, [f.id]);
      expect(Date.now() - Date.parse(blocked!.blocked_since)).toBeGreaterThanOrEqual(60 * MIN);
      expect((await readBrainHealthBands(engine, { sourceIds: [f.id] })).sync_blocked.band).toBe('1-9');
    } finally { await forget(f); }
  }), 60_000);

  test('converged at HEAD and ended synced reads healthy everywhere', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    try {
      await converged(f);
      expect(await readManagedSyncBacklog(engine, [f.id])).toEqual([]);
      const healthy = await one(f);
      expect(healthy).toMatchObject({ movement_state: 'nothing_pending', cursor: null });
      expect(movementReport(healthy, healthy, 60_000)).toMatchObject({ state: 'nothing_pending', exit: 0 });
      expect(await doctorMovement(f)).toMatchObject({ mine: false });
      expect((await gitHeldFilesCheck(engine, [f.id])).status).toBe('ok');
      expect(await readBlockedManagedSyncs(engine, [f.id])).toEqual([]);
      expect((await readBrainHealthBands(engine, { sourceIds: [f.id] })).sync_blocked.band).toBe('none');
    } finally { await forget(f); }
  }), 60_000);

  test('a commit past the converged target makes the source pending again, not stuck', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    try {
      await converged(f);
      const next = commit(f.root, 'later.md', '# Later\n');
      const [behind] = await readManagedSyncBacklog(engine, [f.id]);
      expect(behind).toMatchObject({ source_id: f.id, pass: { head: next, converged_target: f.head, stuck: null } });
      expect((await one(f)).movement_state).toBe('parked');
      expect(await readBlockedManagedSyncs(engine, [f.id])).toEqual([]);
    } finally { await forget(f); }
  }), 60_000);
});

describe('the other cursor readers', () => {
  test("the drain holder's progress is the pass row's newest write", () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
    try {
      expect(lock).not.toBeNull();
      await writePass(f, { outcome: 'open', remaining: 5, target: f.head }, ago(2 * MIN));
      const holder = await readDrainHolder(engine, f.id);
      expect(holder?.progress?.last_progress_at).not.toBeNull();
      expect(Math.abs(Date.parse(holder!.progress!.last_progress_at!) - (Date.now() - 2 * MIN))).toBeLessThan(MIN);
    } finally { await lock?.release(); await forget(f); }
  }), 60_000);

  test('set-strategy refuses during a drain lease and during an open episode, and allows it after', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    const cli = async (args: string[]) => {
      const err: string[] = [];
      const seen: { exit: number | null } = { exit: null };
      const exitSpy = spyOn(process, 'exit').mockImplementation(((code?: number) => { seen.exit = code ?? 0; throw new Error('EXIT'); }) as never);
      const logSpy = spyOn(console, 'log').mockImplementation(() => {});
      const errSpy = spyOn(console, 'error').mockImplementation((...parts: unknown[]) => { err.push(parts.map(String).join(' ')); });
      try { await runSources(engine, args); } catch (error) { if (!(error instanceof Error && error.message === 'EXIT')) throw error; }
      finally { exitSpy.mockRestore(); logSpy.mockRestore(); errSpy.mockRestore(); }
      return { exit: seen.exit, err: err.join('\n') };
    };
    const strategy = async () => (await engine.executeRaw<{ strategy: string | null }>("SELECT config->>'strategy' AS strategy FROM sources WHERE id=$1", [f.id]))[0]!.strategy;
    try {
      const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
      try {
        const leased = await cli(['set-strategy', f.id, 'code']);
        expect(leased.exit).toBe(1);
        expect(leased.err).toContain('sync_run_unfinished');
        expect(await strategy()).toBeNull();
      } finally { await lock?.release(); }
      await writePass(f, { outcome: 'resumable', target: f.head, remaining: 4,
        episode: { opened_at: ago(MIN), full_id: randomUUID(), selection: { exclude: [], includeHidden: [], strategy: null, srcSubpath: null, mode: 'committed' }, inherited: { noEmbed: false, noExtract: false, noSchemaPack: false, from: 'flag' } } });
      const episode = await cli(['set-strategy', f.id, 'code']);
      expect(episode.exit).toBe(1);
      expect(episode.err).toContain('sync_run_unfinished');
      expect(episode.err).toContain(`gbrain sync --source ${f.id} --no-pull`);
      expect(await strategy()).toBeNull();
      await converged(f);
      expect((await cli(['set-strategy', f.id, 'code'])).exit).toBeNull();
      expect(await strategy()).toBe('code');
    } finally { await forget(f); }
  }), 60_000);

  test("an open pass's admitted checkpoint lists what blocks it", () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    try {
      const pass = await writePass(f, { outcome: 'open', remaining: 0, target: f.head });
      const page = await request(f, 'running', { kind: 'managed_sync_import', passId: pass.telemetry.pass_id, path: 'note.md', slug: 'note' });
      await request(f, 'queued', { kind: 'managed_sync_checkpoint', passId: pass.telemetry.pass_id });
      const map = await readCheckpointBlockerMap(engine, [f.id]);
      expect(map.get(f.id)).toEqual([expect.objectContaining({ request_id: page, state: 'running', recovery: false, knock_on: false })]);
      await engine.executeRaw("UPDATE persistence_requests SET state='committed', completed_at=now() WHERE source_id=$1", [f.id]);
      expect((await readCheckpointBlockerMap(engine, [f.id])).has(f.id)).toBe(false);
    } finally { await forget(f); }
  }), 60_000);

  test("compaction keeps an open pass's receipts and the newest pass receipt of the worktree", () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    try {
      const pass = await writePass(f, { outcome: 'open', remaining: 2, target: f.head });
      const inPass = await request(f, 'committed', { kind: 'managed_sync_import', passId: pass.telemetry.pass_id, path: 'note.md' }, ago(3 * MIN));
      const newest = await request(f, 'committed', { kind: 'managed_sync_import', passId: randomUUID(), path: 'later.md' }, ago(2 * MIN));
      await compactWriteReceiptsDetailed(engine, 0);
      expect(await compacted(inPass)).toBe(false);
      expect(await compacted(newest)).toBe(false);
      const [row] = await engine.executeRaw<{ id: string }>('SELECT id::text AS id, principal_kind, principal_id FROM persistence_requests WHERE request_id=$1::uuid', [inPass]);
      expect(await compactReceipt(engine, row as never)).toEqual({ skipped: 'open_sync_pass' });
      await writePass(f, { outcome: 'synced', converged: f.head, pass_id: pass.telemetry.pass_id });
      await compactWriteReceiptsDetailed(engine, 0);
      expect(await compacted(inPass)).toBe(true);
      expect(await compacted(newest)).toBe(false);
    } finally { await forget(f); }
  }), 60_000);

  test('probe 53 (busy side): an open pass with no open request makes no repair candidate busy', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    try {
      await writePass(f, { outcome: 'open', remaining: 3, target: f.head });
      const busy = await loadRepairBusySet(engine, f.id, f.incarnation);
      expect(busy.unknown).toBeNull();
      expect(repairBusy(busy, { path: 'note.md', slug: 'note' })).toBe(false);
    } finally { await forget(f); }
  }), 60_000);
});