Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.5 KB
Raw Blame
/**
 * #8984 on real Postgres: a retry of an interrupted managed remove/purge after its source was removed and re-added
 * refuses with source_retire_superseded and keeps the new source's pages, including after batches already deleted
 * part of the old source.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import type { BrainEngine } from '../../src/core/engine.ts';
import { registerLocalWriter } from '../../src/core/persistence/identity.ts';
import { claimWorktree } from '../../src/core/persistence/ownership.ts';
import { runManagedSourceLifecycle, SOURCE_RETIRE_BATCH_PAGES } from '../../src/core/persistence/source-lifecycle.ts';
import { disposePersistenceConsumer } from '../../src/core/persistence/service.ts';
import { persistencePostgresTemplate } from '../helpers/persistence-postgres.ts';
import { requirePostgresTestDatabase } from '../helpers/test-backends.ts';
import { withEnv } from '../helpers/with-env.ts';

const source = 'retire-again';
const PAGES = SOURCE_RETIRE_BATCH_PAGES * 2 + 17;
const databaseUrl = requirePostgresTestDatabase();
let template: Awaited<ReturnType<typeof persistencePostgresTemplate>>;
beforeAll(async () => {
  template = await persistencePostgresTemplate(databaseUrl, 2, async engine => {
    for (const id of [source, 'keep-other']) await engine.executeRaw('INSERT INTO sources(id,name) VALUES($1,$1)', [id]);
    await seed(engine, source, PAGES);
    await seed(engine, 'keep-other', 20);
  });
}, 600_000);
afterAll(async () => { await template?.dispose(); });

function seed(engine: BrainEngine, id: string, n: number, from = 1) {
  return engine.executeRaw(`INSERT INTO pages(source_id,slug,type,title,compiled_truth)
    SELECT $1,'notes/p-'||g,'note','Page '||g,'Body '||g FROM generate_series($2::int,$3::int) g`, [id, from, from + n - 1]);
}
const pages = async (engine: BrainEngine, id = source) => (await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1', [id]))[0].n;
const incarnation = async (engine: BrainEngine) => (await engine.executeRaw<{ incarnation: string }>('SELECT incarnation FROM sources WHERE id=$1', [source]))[0]?.incarnation;
const change = async (engine: BrainEngine, requestId: string) => (await engine.executeRaw<{ state: string; source_incarnation: string }>(
  'SELECT state,source_incarnation FROM persistence_topology_changes WHERE request_id=$1::uuid', [requestId]))[0];
/** Raw page writes stand in for an import; the managed guard admits them only while the brain is briefly unmanaged. */
async function unmanaged(engine: BrainEngine, run: () => Promise<unknown>) {
  await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  try { await run(); } finally { await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1'); }
}

describe('retrying a managed remove after its source was re-added (postgres)', () => {
  async function fixture(run: (engine: BrainEngine, root: string) => Promise<void>) {
    const { engine, close } = await template.clone();
    const home = mkdtempSync(join(tmpdir(), 'gbrain-retire-again-'));
    try {
      await withEnv({ GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, async () => {
        await registerLocalWriter(engine, 'cli');
        for (const id of [source, 'keep-other']) {
          const root = join(home, id); mkdirSync(root);
          writeFileSync(join(root, 'example.md'), '---\ntitle: Example\ntype: note\n---\n\nCanonical\n');
          await engine.executeRaw('UPDATE sources SET local_path=$2 WHERE id=$1', [id, root]);
          await claimWorktree(engine, id, root);
        }
        await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
        await run(engine, join(home, source));
        expect(await pages(engine, 'keep-other')).toBe(20);
      });
    } finally { await disposePersistenceConsumer(engine); await close(); rmSync(home, { recursive: true, force: true }); }
  }
  async function readd(engine: BrainEngine, root: string, n: number) {
    expect(await runManagedSourceLifecycle(engine, { operation: 'add', sourceId: source, path: root })).toMatchObject({ state: 'committed' });
    await unmanaged(engine, () => seed(engine, source, n, 9000));
  }

  test('a retry after the final transaction failed and the source was removed and re-added refuses', () => fixture(async (engine, root) => {
    await engine.executeRaw(`CREATE FUNCTION retire_final_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
      RAISE EXCEPTION 'injected final retire failure'; END $$`);
    await engine.executeRaw('CREATE TRIGGER retire_final BEFORE DELETE ON sources FOR EACH ROW EXECUTE FUNCTION retire_final_fn()');
    const requestId = randomUUID();
    const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });
    const first = await incarnation(engine);
    try { await expect(remove()).rejects.toThrow('injected final retire failure'); }
    finally { await engine.executeRaw('DROP TRIGGER retire_final ON sources'); await engine.executeRaw('DROP FUNCTION retire_final_fn()'); }
    expect(await pages(engine)).toBe(0);
    expect(await change(engine, requestId)).toMatchObject({ state: 'recovering', source_incarnation: first });
    expect(await runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true })).toMatchObject({ state: 'committed' });
    expect(await change(engine, requestId)).toMatchObject({ state: 'failed', source_incarnation: first });
    await readd(engine, root, 1);
    for (let i = 0; i < 2; i++) await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });
    expect(await pages(engine)).toBe(1);
  }), 300_000);

  test('a batched retry after partial deletion refuses once the source was replaced, and settles its record', () => fixture(async (engine, root) => {
    await engine.executeRaw('CREATE TABLE retire_fail(armed boolean NOT NULL, seen integer NOT NULL)');
    await engine.executeRaw(`CREATE FUNCTION retire_fail_fn() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE n integer; BEGIN
      UPDATE retire_fail SET seen=seen+1 WHERE armed RETURNING seen INTO n;
      IF n > ${SOURCE_RETIRE_BATCH_PAGES} THEN RAISE EXCEPTION 'injected retire batch failure'; END IF; RETURN OLD; END $$`);
    await engine.executeRaw('CREATE TRIGGER retire_fail BEFORE DELETE ON pages FOR EACH ROW EXECUTE FUNCTION retire_fail_fn()');
    await engine.executeRaw('INSERT INTO retire_fail VALUES(true,0)');
    const requestId = randomUUID();
    const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });
    const first = await incarnation(engine);
    await expect(remove()).rejects.toThrow('injected retire batch failure');
    await engine.executeRaw('DELETE FROM retire_fail');
    expect(await pages(engine)).toBe(PAGES - SOURCE_RETIRE_BATCH_PAGES);
    expect(await change(engine, requestId)).toMatchObject({ state: 'recovering', source_incarnation: first });
    await unmanaged(engine, async () => {
      await engine.executeRaw('DELETE FROM pages WHERE source_id=$1', [source]);
      await engine.executeRaw('DELETE FROM persistence_source_bindings WHERE source_id=$1', [source]);
      await engine.executeRaw('DELETE FROM sources WHERE id=$1', [source]);
    });
    await readd(engine, root, 3);
    await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });
    expect(await change(engine, requestId)).toMatchObject({ state: 'failed', source_incarnation: first });
    await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });
    expect(await pages(engine)).toBe(3);
  }), 300_000);

  test('a retry under a replaced CLI registration after the source was removed and re-added refuses', () => fixture(async (engine, root) => {
    await engine.executeRaw('CREATE TABLE retire_fail(armed boolean NOT NULL, seen integer NOT NULL)');
    await engine.executeRaw(`CREATE FUNCTION retire_fail_fn() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE n integer; BEGIN
      UPDATE retire_fail SET seen=seen+1 WHERE armed RETURNING seen INTO n;
      IF n > ${SOURCE_RETIRE_BATCH_PAGES} THEN RAISE EXCEPTION 'injected retire batch failure'; END IF; RETURN OLD; END $$`);
    await engine.executeRaw('CREATE TRIGGER retire_fail BEFORE DELETE ON pages FOR EACH ROW EXECUTE FUNCTION retire_fail_fn()');
    await engine.executeRaw('INSERT INTO retire_fail VALUES(true,0)');
    const requestId = randomUUID();
    const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });
    const first = await incarnation(engine);
    await expect(remove()).rejects.toThrow('injected retire batch failure');
    await engine.executeRaw('DROP TRIGGER retire_fail ON pages');
    expect(await change(engine, requestId)).toMatchObject({ state: 'recovering', source_incarnation: first });
    await registerLocalWriter(engine, 'cli', undefined, true);
    expect(await runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true })).toMatchObject({ state: 'committed' });
    await readd(engine, root, 2);
    for (let i = 0; i < 2; i++) await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });
    expect(await pages(engine)).toBe(2);
  }), 300_000);
});