Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.3 KB
Raw Blame
import { afterAll, beforeAll, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { LATEST_VERSION, runMigrations } from '../src/core/migrate.ts';
import { PERSISTENCE_REQUEST_RECOVERY_INDEX_SQL } from '../src/core/persistence/schema.ts';
import type { OperationContext } from '../src/core/ops/contract.ts';
import { activatePersistence } from '../src/core/persistence/activation.ts';
import { claimWorktree, acquireWorktree, type WorktreeBinding } from '../src/core/persistence/ownership.ts';
import { localHostId, registerLocalWriter } from '../src/core/persistence/identity.ts';
import { submissionAuthority } from '../src/core/persistence/authority.ts';
import { admitWrite, claimNextWrite, completeWrite, clearResolvedRecovery, getWriteRequestById, prepareRecovery } from '../src/core/persistence/journal.ts';
import { publishMutation, recoverPublication } from '../src/core/persistence/coordinator.ts';
import { withCoordinatedWrite } from '../src/core/persistence/context.ts';
import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';
import { reserveEffectRecovery, recoverEffectPublication } from '../src/core/persistence/effect-recovery.ts';
import type { EffectRecovery, PersistenceEffect } from '../src/core/persistence/effect-model.ts';
import { recoveryStagingFile } from '../src/core/persistence/staging.ts';
import { sha256 } from '../src/core/persistence/digest.ts';
import { serializePageToMarkdown } from '../src/core/markdown.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { withEnv } from './helpers/with-env.ts';

interface Fixture { engine: BrainEngine; sourceId: string; root: string; path: string; binding: WorktreeBinding }
const fixtures: Fixture[][] = [];
const engines: BrainEngine[] = [];
let hostId: string;
let home: string;
let closePostgres: (() => Promise<void>) | undefined;
beforeAll(async () => {
  home = realpathSync(mkdtempSync(join(tmpdir(), 'gbrain-staging-tests-')));
  await withEnv({ GBRAIN_HOME: home }, async () => {
    hostId = localHostId();
    const engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); engines.push(engine);
    if (process.env.DATABASE_URL) {
      const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL); engines.push(pg.engine); closePostgres = pg.close;
    }
    for (const engine of engines) {
      const cases: Fixture[] = [];
      for (let i = 0; i < 12; i++) {
        const sourceId = `staging-case-${i}`;
        const root = join(home, `${engine.kind}-${i}`); mkdirSync(root);
        const path = join(root, 'page.md'); writeFileSync(path, 'original');
        await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', [sourceId, root]);
        const binding = await claimWorktree(engine, sourceId, root, hostId);
        cases.push({ engine, sourceId, root, path, binding });
      }
      await registerLocalWriter(engine, 'cli');
      await activatePersistence(engine, { confirmQuiesced: true });
      fixtures.push(cases);
    }
  });
}, 120_000);
afterAll(async () => {
  for (const engine of engines.filter(engine => engine.kind === 'pglite')) await engine.disconnect();
  await closePostgres?.();
  if (home) rmSync(home, { recursive: true, force: true });
});

async function accepted(f: Fixture) {
  const authority = await submissionAuthority({ engine: f.engine, remote: false, sourceId: f.sourceId } as OperationContext,
    'put_page', f.sourceId, f.binding.source_incarnation, 'page');
  const current = await f.engine.readPageSnapshot('page', { sourceId: f.sourceId });
  const input = { principal: authority.principal, authority, operation: 'put_page', sourceId: f.sourceId,
    sourceIncarnation: f.binding.source_incarnation, slug: 'page', worktreeId: f.binding.worktree_id,
    topologyGeneration: f.binding.topology_generation, pageId: current?.page.id ?? null,
    callerIntent: { content: 'replacement' }, intent: { content: 'replacement' } };
  const row = await admitWrite(f.engine, input);
  const claimed = (await claimNextWrite(f.engine, hostId))!; expect(claimed.id).toBe(row.id);
  const prepared = { observedRevision: current?.revision ?? null, file: { path: f.path, root: f.root, content: 'replacement' },
    apply: async (tx: BrainEngine) => {
      await tx.putPage('page', { type: 'note', title: 'Stage example', compiled_truth: 'replacement', frontmatter: {} }, { sourceId: f.sourceId });
      return { slug: 'page' };
    } };
  return { row: claimed, prepared };
}
function temporaryFiles(f: Fixture) { return readdirSync(f.root).filter(name => name.includes('.tmp.')); }
async function bytes(f: Fixture) {
  const [row] = await f.engine.executeRaw<{ recovery_bytes: string }>('SELECT recovery_bytes FROM persistence_counters WHERE key=$1', [`worktree:${f.binding.worktree_id}`]);
  return Number(row.recovery_bytes);
}

test('journaled flush failure removes owned stage and releases accounting only after rollback', async () => withEnv({ GBRAIN_HOME: home }, async () => {
  for (const cases of fixtures) {
    const f = cases[0]; const { row, prepared } = await accepted(f);
    let stage: string | undefined;
    const done = await publishMutation(f.engine, row, prepared, hostId, {
      boundary: async name => {
        if (name !== 'prepared') return;
        const saved = (await getWriteRequestById(f.engine, row.id))!;
        stage = saved.recovery!.staging!.publication!.path;
        expect(existsSync(stage)).toBe(false); expect(await bytes(f)).toBeGreaterThan(0);
      },
      stagingFlushed: () => {
        expect(readFileSync(stage!, 'utf8')).toBe('replacement');
        expect(readFileSync(f.path, 'utf8')).toBe('original');
        throw new Error('fixture flushed write failed');
      },
    });
    expect(done.state).toBe('failed'); expect(temporaryFiles(f)).toEqual([]);
    expect(readFileSync(f.path, 'utf8')).toBe('original'); expect(await bytes(f)).toBe(0);
    expect(await f.engine.readPageSnapshot('page', { sourceId: f.sourceId })).toBeNull();
  }
}));

for (const [index, suffix] of [[7, ['page.md']], [8, ['missing', 'page.md']]] as const) {
  test(`blocked file ancestor settles failed publication and releases its worktree (${index})`, async () => withEnv({ GBRAIN_HOME: home }, async () => {
    for (const cases of fixtures) {
      const f = cases[index]; const { row, prepared } = await accepted(f);
      const blocker = join(f.root, 'blocked'); writeFileSync(blocker, 'preserved blocker');
      prepared.file.path = join(blocker, ...suffix);
      let reserved = false;
      const done = await publishMutation(f.engine, row, prepared, hostId, { boundary: async name => {
        if (name !== 'prepared') return;
        reserved = true;
        expect((await getWriteRequestById(f.engine, row.id))!.recovery!.beforeHash).toBeNull();
        expect(await bytes(f)).toBeGreaterThan(0);
      } });
      expect(reserved).toBe(true);
      expect(done).toMatchObject({ state: 'failed', error_code: 'storage_error', recovery: null });
      expect(readFileSync(blocker, 'utf8')).toBe('preserved blocker');
      expect(readFileSync(f.path, 'utf8')).toBe('original');
      expect(await f.engine.readPageSnapshot('page', { sourceId: f.sourceId })).toBeNull();
      expect(await bytes(f)).toBe(0);
      const retained = await getWriteRequestById(f.engine, row.id);
      const next = await accepted(f);
      expect((await publishMutation(f.engine, next.row, next.prepared, hostId)).state).toBe('committed');
      expect(await getWriteRequestById(f.engine, row.id)).toEqual(retained);
      expect(readFileSync(blocker, 'utf8')).toBe('preserved blocker');
      expect(await bytes(f)).toBe(0);
    }
  }));
}

for (const [index, unexpected] of [[1, 'rep'], [2, 'unknown bytes']] as const) {
  test(`unexpected/partial staging is retained without releasing recovery capacity (${index})`, async () => withEnv({ GBRAIN_HOME: home }, async () => {
    for (const cases of fixtures) {
      const f = cases[index]; const { row, prepared } = await accepted(f); let stage = '';
      const done = await publishMutation(f.engine, row, prepared, hostId, { boundary: async name => {
        if (name !== 'prepared') return;
        stage = (await getWriteRequestById(f.engine, row.id))!.recovery!.staging!.publication!.path;
        writeFileSync(stage, unexpected); throw new Error('fixture stage collision');
      } });
      expect(done.state).toBe('recovering'); expect(done.blocked_reason).toBe('unexpected_staging_bytes');
      const reserved = await bytes(f); expect(reserved).toBeGreaterThan(0);
      expect((await recoverPublication(f.engine, row.id, hostId)).blocked_reason).toBe('unexpected_staging_bytes');
      expect(readFileSync(stage, 'utf8')).toBe(unexpected); expect(await bytes(f)).toBe(reserved);
      expect(readFileSync(f.path, 'utf8')).toBe('original');
      writeFileSync(stage, 'replacement'); // Explicit fixture repair to exactly known attempted bytes.
      expect((await recoverPublication(f.engine, row.id, hostId)).state).toBe('failed');
      expect(temporaryFiles(f)).toEqual([]); expect(await bytes(f)).toBe(0);
    }
  }));
}

test('terminal recovery survives index upgrade and fences its worktree until staging cleanup completes', async () => withEnv({ GBRAIN_HOME: home }, async () => {
  for (const cases of fixtures) {
    const f = cases[3]; const { row, prepared } = await accepted(f); let stage = '';
    const done = await publishMutation(f.engine, row, prepared, hostId, { boundary: async name => {
      if (name !== 'after_commit') return;
      stage = (await getWriteRequestById(f.engine, row.id))!.recovery!.staging!.publication!.path;
      writeFileSync(stage, 'unexpected committed-stage bytes'); throw new Error('fixture lost response');
    } });
    expect(done.state).toBe('committed'); expect(done.blocked_reason).toBe('unexpected_staging_bytes');
    const outcome = done.outcome; const reserved = await bytes(f); expect(reserved).toBeGreaterThan(0);
    await expect(clearResolvedRecovery(f.engine, row.id)).rejects.toMatchObject({ code: 'unexpected_staging_bytes' });
    const indexDefinition = async () => {
      const [index] = await f.engine.executeRaw<{ definition: string; valid: boolean }>(`SELECT pg_get_indexdef(indexrelid) AS definition,
        indisvalid AS valid FROM pg_index WHERE indexrelid=to_regclass('persistence_requests_recovery')`);
      expect(index?.valid).toBe(true);
      expect(index?.definition).toContain('(worktree_id, sequence)');
      expect(index?.definition).toContain('WHERE (recovery IS NOT NULL)');
    };
    await indexDefinition();
    const retained = await getWriteRequestById(f.engine, row.id);
    const counters = await f.engine.executeRaw('SELECT * FROM persistence_counters ORDER BY key');
    await f.engine.executeRaw('DROP INDEX persistence_requests_recovery');
    await f.engine.setConfig('version', '158');
    const upgraded = await runMigrations(f.engine);
    expect(upgraded.applied).toBeGreaterThanOrEqual(1); expect(upgraded.current).toBe(LATEST_VERSION);
    await indexDefinition();
    // Bootstrap composition and migration share an idempotent index definition.
    await f.engine.executeRaw(PERSISTENCE_REQUEST_RECOVERY_INDEX_SQL);
    expect(await getWriteRequestById(f.engine, row.id)).toEqual(retained);
    expect(await f.engine.executeRaw('SELECT * FROM persistence_counters ORDER BY key')).toEqual(counters);
    const authority = await submissionAuthority({ engine: f.engine, remote: false, sourceId: f.sourceId } as OperationContext,
      'put_page', f.sourceId, f.binding.source_incarnation, 'follower');
    const follower = await admitWrite(f.engine, { principal: authority.principal, authority, operation: 'put_page',
      sourceId: f.sourceId, sourceIncarnation: f.binding.source_incarnation, slug: 'follower',
      worktreeId: f.binding.worktree_id, topologyGeneration: f.binding.topology_generation,
      callerIntent: { content: 'following update' }, intent: { content: 'following update' } });
    expect(await claimNextWrite(f.engine, hostId)).toBeNull();
    expect((await getWriteRequestById(f.engine, follower.id))!.state).toBe('queued');
    const healthy = cases[6]; const independent = await accepted(healthy);
    expect((await publishMutation(healthy.engine, independent.row, independent.prepared, hostId)).state).toBe('committed');
    expect(readFileSync(healthy.path, 'utf8')).toBe('replacement');
    expect((await getWriteRequestById(f.engine, follower.id))!.state).toBe('queued');

    // A stale/direct publication attempt must repeat the fence under native
    // exclusion even when it did not enter through the current claim query.
    const [staleClaim] = await f.engine.executeRaw<typeof row>(`UPDATE persistence_requests SET state='running',
      execution_token=$2::uuid,claim_expires_at=now()+interval '30 seconds' WHERE id=$1::uuid RETURNING *`,
    [follower.id, randomUUID()]);
    let applied = false;
    const followerPath = join(f.root, 'follower.md');
    const followerPrepared = { observedRevision: null, file: { path: followerPath, root: f.root, content: 'following update' },
      apply: async (tx: BrainEngine) => {
        applied = true;
        await tx.putPage('follower', { type: 'note', title: 'Follower example', compiled_truth: 'following update', frontmatter: {} }, { sourceId: f.sourceId });
        return { slug: 'follower' };
      } };
    expect(await publishMutation(f.engine, staleClaim, followerPrepared, hostId)).toMatchObject({ state: 'queued', blocked_reason: 'recovery_required' });
    expect(applied).toBe(false); expect(existsSync(followerPath)).toBe(false);
    expect(await bytes(f)).toBe(reserved);
    expect((await getWriteRequestById(f.engine, row.id))!.outcome).toEqual(outcome);
    writeFileSync(stage, 'replacement');
    const lock = await acquireWorktree(f.binding); expect(lock).not.toBeNull();
    try {
      expect((await recoverPublication(f.engine, row.id, hostId)).recovery).not.toBeNull();
      expect(existsSync(stage)).toBe(true); expect(await bytes(f)).toBe(reserved);
    } finally { await lock!.release(); }
    const recovered = await recoverPublication(f.engine, row.id, hostId);
    expect(recovered.state).toBe('committed'); expect(recovered.outcome).toEqual(outcome);
    expect(readFileSync(f.path, 'utf8')).toBe('replacement'); expect(temporaryFiles(f)).toEqual([]);
    expect(await bytes(f)).toBe(0); expect((await getWriteRequestById(f.engine, row.id))!.recovery).toBeNull();
    const next = (await claimNextWrite(f.engine, hostId))!; expect(next.id).toBe(follower.id);
    expect((await publishMutation(f.engine, next, followerPrepared, hostId)).state).toBe('committed');
    expect(applied).toBe(true); expect(readFileSync(followerPath, 'utf8')).toBe('following update');
    expect((await f.engine.readPageSnapshot('follower', { sourceId: f.sourceId }))!.page.compiled_truth).toBe('following update');
    expect(await bytes(f)).toBe(0);
  }
}));

test('legacy recovery gains durable restoration staging before rewriting the old file', async () => withEnv({ GBRAIN_HOME: home }, async () => {
  for (const cases of fixtures) {
    const f = cases[4]; const { row } = await accepted(f);
    await prepareRecovery(f.engine, row, { version: 1, path: f.path, root: f.root, before: Buffer.from('original').toString('base64'),
      beforeHash: sha256('original'), afterHash: sha256('replacement'), mode: 0o600, ownerEpoch: String(f.binding.owner_epoch), attempt: row.execution_token! }, 4096);
    writeFileSync(f.path, 'replacement');
    const recovered = await recoverPublication(f.engine, row.id, hostId);
    expect(recovered.state).toBe('queued'); expect(recovered.recovery).toBeNull();
    expect(readFileSync(f.path, 'utf8')).toBe('original'); expect(temporaryFiles(f)).toEqual([]); expect(await bytes(f)).toBe(0);
    await f.engine.transaction(tx => completeWrite(tx, recovered, 'cancelled', {}));
  }
}));

test('withdrawal mirror staging preserves unexpected bytes and only finishes forward', async () => withEnv({ GBRAIN_HOME: home }, async () => {
  for (const cases of fixtures) {
    const f = cases[5]; const { row } = await accepted(f);
    await f.engine.transaction(tx => withCoordinatedWrite(tx, [f.sourceId], () => tx.putPage('page',
      { type: 'note', title: 'Withdrawn example', compiled_truth: 'Current sanitized content.', frontmatter: {} }, { sourceId: f.sourceId }), TEST_WRITE_ATTRIBUTION));
    const snapshot = (await f.engine.readPageSnapshot('page', { sourceId: f.sourceId }))!;
    await f.engine.transaction(tx => completeWrite(tx, row, 'committed', { revision: snapshot.revision }));
    const [effect] = await f.engine.executeRaw<PersistenceEffect>(`INSERT INTO persistence_effects
      (request_id,kind,revision,data,source_id,source_incarnation,worktree_id,state,execution_token)
      VALUES($1::uuid,'withdrawal-mirror',$2::uuid,'{"source_scan":true}',$3,$4::uuid,$5::uuid,'running',$6::uuid) RETURNING *`,
    [row.id, snapshot.revision, f.sourceId, f.binding.source_incarnation, f.binding.worktree_id, randomUUID()]);
    const after = serializePageToMarkdown(snapshot.page, snapshot.tags);
    const record: EffectRecovery = { version: 1, kind: 'withdrawal-mirror', path: f.path, root: f.root,
      beforeHash: sha256('original'), afterHash: sha256(after), after: Buffer.from(after).toString('base64'), mode: 0o600,
      ownerEpoch: String(f.binding.owner_epoch), pageId: snapshot.page.id, sourceIncarnation: snapshot.sourceIncarnation,
      slug: 'page', revision: snapshot.revision, staging: { publication: recoveryStagingFile(f.path, after) } };
    const lock = await acquireWorktree(f.binding); expect(lock).not.toBeNull();
    try {
      await reserveEffectRecovery(f.engine, effect, record, 8192, hostId);
      writeFileSync(record.staging!.publication!.path, 'Unexpected mirror fragment.');
      await expect(recoverEffectPublication(f.engine, effect, hostId)).rejects.toMatchObject({ code: 'unexpected_staging_bytes' });
      expect(await bytes(f)).toBe(8192); expect(readFileSync(f.path, 'utf8')).toBe('original');
      unlinkSync(record.staging!.publication!.path); // Remove only this fixture's deliberate unexpected file.
      await recoverEffectPublication(f.engine, effect, hostId);
      expect(readFileSync(f.path, 'utf8')).toBe(after); expect(temporaryFiles(f)).toEqual([]); expect(await bytes(f)).toBe(0);
      expect((await f.engine.readPageSnapshot('page', { sourceId: f.sourceId }))!.revision).toBe(snapshot.revision);
      expect((await getWriteRequestById(f.engine, row.id))!.state).toBe('committed');
    } finally { await lock!.release(); }
  }
}));

test('journaled long target name commits file and page and releases recovery accounting (#5861)', async () => withEnv({ GBRAIN_HOME: home }, async () => {
  for (const cases of fixtures) {
    const f = cases[9];
    f.path = join(f.root, 'm'.repeat(212) + '.md');
    writeFileSync(f.path, 'original');
    const { row, prepared } = await accepted(f);
    const done = await publishMutation(f.engine, row, prepared, hostId);
    expect(done.state).toBe('committed');
    expect(readFileSync(f.path, 'utf8')).toBe('replacement');
    expect((await f.engine.readPageSnapshot('page', { sourceId: f.sourceId }))!.page.compiled_truth).toBe('replacement');
    expect(temporaryFiles(f)).toEqual([]);
    expect(await bytes(f)).toBe(0);
    expect((await getWriteRequestById(f.engine, row.id))!.recovery).toBeNull();
  }
}));

test('populated legacy deletion journal reserves a short restoration stage before restoring (#5861)', async () => withEnv({ GBRAIN_HOME: home }, async () => {
  for (const cases of fixtures) {
    const f = cases[10];
    f.path = join(f.root, 'm'.repeat(212) + '.md');
    writeFileSync(f.path, 'original');
    const { row } = await accepted(f);
    const legacy = `${f.path}.tmp.${randomUUID()}`;
    await prepareRecovery(f.engine, row, { version: 1, path: f.path, root: f.root,
      before: Buffer.from('original').toString('base64'), beforeHash: sha256('original'), afterHash: null,
      mode: 0o600, ownerEpoch: String(f.binding.owner_epoch), attempt: row.execution_token!,
      staging: { restoration: { path: legacy, hash: sha256('original'), bytes: 8 } } }, 8192);
    unlinkSync(f.path);
    let durableStage = false;
    const recovered = await recoverPublication(f.engine, row.id, hostId, false, undefined, false, {
      fileBoundary: (name, current) => {
        if (name !== 'before_restore') return;
        expect(current.recovery!.staging!.restoration!.path).not.toBe(legacy);
        durableStage = true;
      },
    });
    expect(durableStage).toBe(true); expect(recovered.state).toBe('queued'); expect(recovered.recovery).toBeNull();
    expect(readFileSync(f.path, 'utf8')).toBe('original'); expect(temporaryFiles(f)).toEqual([]); expect(await bytes(f)).toBe(0);
    await f.engine.transaction(tx => completeWrite(tx, recovered, 'cancelled', {}));
  }
}));

test('populated legacy withdrawal journal reserves a short stage and finishes only forward (#5861)', async () => withEnv({ GBRAIN_HOME: home }, async () => {
  for (const cases of fixtures) {
    const f = cases[11];
    f.path = join(f.root, 'm'.repeat(212) + '.md'); writeFileSync(f.path, 'original');
    const { row } = await accepted(f);
    await f.engine.transaction(tx => withCoordinatedWrite(tx, [f.sourceId], () => tx.putPage('page',
      { type: 'note', title: 'Withdrawn example', compiled_truth: 'Current sanitized content.', frontmatter: {} }, { sourceId: f.sourceId }), TEST_WRITE_ATTRIBUTION));
    const snapshot = (await f.engine.readPageSnapshot('page', { sourceId: f.sourceId }))!;
    await f.engine.transaction(tx => completeWrite(tx, row, 'committed', { revision: snapshot.revision }));
    const [effect] = await f.engine.executeRaw<PersistenceEffect>(`INSERT INTO persistence_effects
      (request_id,kind,revision,data,source_id,source_incarnation,worktree_id,state,execution_token)
      VALUES($1::uuid,'withdrawal-mirror',$2::uuid,'{"source_scan":true}',$3,$4::uuid,$5::uuid,'running',$6::uuid) RETURNING *`,
    [row.id, snapshot.revision, f.sourceId, f.binding.source_incarnation, f.binding.worktree_id, randomUUID()]);
    const after = serializePageToMarkdown(snapshot.page, snapshot.tags);
    const legacy = `${f.path}.tmp.${randomUUID()}`;
    const record: EffectRecovery = { version: 1, kind: 'withdrawal-mirror', path: f.path, root: f.root,
      beforeHash: sha256('original'), afterHash: sha256(after), after: Buffer.from(after).toString('base64'), mode: 0o600,
      ownerEpoch: String(f.binding.owner_epoch), pageId: snapshot.page.id, sourceIncarnation: snapshot.sourceIncarnation,
      slug: 'page', revision: snapshot.revision, staging: { publication: { path: legacy, hash: sha256(after), bytes: Buffer.byteLength(after) } } };
    const lock = await acquireWorktree(f.binding); expect(lock).not.toBeNull();
    try {
      await reserveEffectRecovery(f.engine, effect, record, 8192, hostId);
      const [retry] = await f.engine.executeRaw<PersistenceEffect>('SELECT * FROM persistence_effects WHERE id=$1', [effect.id]);
      await recoverEffectPublication(f.engine, retry, hostId);
      expect(readFileSync(f.path, 'utf8')).toBe(after); expect(temporaryFiles(f)).toEqual([]); expect(await bytes(f)).toBe(0);
      expect((await f.engine.readPageSnapshot('page', { sourceId: f.sourceId }))!.revision).toBe(snapshot.revision);
    } finally { await lock!.release(); }
  }
}));