Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.2 KB
Raw Blame
/**
 * #7702 / #8905 on PGLite: a managed remove or purge deletes a source's pages in bounded batches behind the same
 * preconditions as its final transaction, never deletes for a dry run or a refused operation, and a retry with the
 * same request_id resumes after an interrupted batch.
 */
import { afterAll, beforeAll, beforeEach, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv } from './helpers/with-env.ts';
import { registerLocalWriter } from '../src/core/persistence/identity.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { runManagedSourceLifecycle, SOURCE_RETIRE_BATCH_PAGES } from '../src/core/persistence/source-lifecycle.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { journalLimitKey } from '../src/core/persistence/limits.ts';

let engine: PGLiteEngine;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
});

afterAll(async () => {
  await disposePersistenceConsumer(engine);
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
});

const PAGES = SOURCE_RETIRE_BATCH_PAGES * 2 + 17;
const source = 'retire-batched';

async function fixture(run: () => Promise<void>) {
  const home = mkdtempSync(join(tmpdir(), 'gbrain-retire-batched-'));
  try {
    await withEnv({ GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, async () => {
      await registerLocalWriter(engine, 'cli');
      for (const id of [source, 'keep-other']) {
        const root = join(home, id); mkdirSync(root);
        writeFileSync(join(root, 'example.md'), '---\ntitle: Example\ntype: note\n---\n\nCanonical\n');
        await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', [id, root]);
        await claimWorktree(engine, id, root);
        await engine.executeRaw(`INSERT INTO pages(source_id,slug,type,title,compiled_truth)
          SELECT $1,'notes/p-'||g,'note','Page '||g,'Body '||g FROM generate_series(1,$2::int) g`, [id, id === source ? PAGES : 20]);
        await engine.executeRaw(`INSERT INTO content_chunks(page_id,chunk_index,chunk_text)
          SELECT id,0,'chunk of '||slug FROM pages WHERE source_id=$1`, [id]);
      }
      await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
      await run();
      expect(await pages('keep-other')).toBe(20);
    });
  } finally { rmSync(home, { recursive: true, force: true }); }
}
const pages = async (id: string) => (await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1', [id]))[0].n;
const registered = async () => (await engine.executeRaw('SELECT id FROM sources WHERE id=$1', [source])).length;

test('remove and purge of a source larger than one batch delete every page and count them all', () => fixture(async () => {
  await runManagedSourceLifecycle(engine, { operation: 'archive', sourceId: source });
  expect(await runManagedSourceLifecycle(engine, { operation: 'purge', sourceId: source, confirmDestructive: true }))
    .toMatchObject({ state: 'committed', operation: 'purge', pages_deleted: PAGES });
  expect(await registered()).toBe(0);
  expect(await pages(source)).toBe(0);
  expect((await engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM content_chunks c LEFT JOIN pages p ON p.id=c.page_id WHERE p.id IS NULL`))[0].n).toBe(0);
}), 60_000);

test('a dry run or a refused remove/purge deletes no page', () => fixture(async () => {
  await runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, dryRun: true });
  await expect(runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source })).rejects.toMatchObject({ code: 'invalid_params' });
  await expect(runManagedSourceLifecycle(engine, { operation: 'purge', sourceId: source, confirmDestructive: true })).rejects.toMatchObject({ code: 'invalid_params' });
  await runManagedSourceLifecycle(engine, { operation: 'archive', sourceId: source });
  expect(await runManagedSourceLifecycle(engine, { operation: 'purge', sourceId: source, confirmDestructive: true, expiredOnly: true }))
    .toMatchObject({ state: 'committed', noop: true });
  await expect(runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, expectedIncarnation: randomUUID() }))
    .rejects.toMatchObject({ code: 'source_changed' });
  expect(await pages(source)).toBe(PAGES);
  expect(await registered()).toBe(1);
}), 60_000);

test('a lock wait stops the remove with write_pending instead of shrinking the batch; the retry resumes at full size', () => fixture(async () => {
  await engine.executeRaw('CREATE SEQUENCE retire_lock_seq');
  await engine.executeRaw(`CREATE FUNCTION retire_lock_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
    IF nextval('retire_lock_seq') = 1 THEN RAISE EXCEPTION 'canceling statement due to lock timeout' USING ERRCODE = 'lock_not_available'; END IF;
    RETURN NULL; END $$`);
  await engine.executeRaw('CREATE TRIGGER retire_lock AFTER DELETE ON pages FOR EACH STATEMENT EXECUTE FUNCTION retire_lock_fn()');
  try {
    const requestId = randomUUID();
    const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });
    await expect(remove()).rejects.toMatchObject({ code: 'write_pending', reason: 'lock_timeout' });
    expect(await pages(source)).toBe(PAGES);
    expect(await remove()).toMatchObject({ state: 'committed', operation: 'remove', pages_deleted: PAGES });
    expect(await pages(source)).toBe(0);
  } finally {
    for (const sql of ['DROP TRIGGER IF EXISTS retire_lock ON pages', 'DROP FUNCTION IF EXISTS retire_lock_fn()', 'DROP SEQUENCE IF EXISTS retire_lock_seq']) await engine.executeRaw(sql);
  }
}), 60_000);

test('a bounded wait at COMMIT stops the remove without halving, and only committed batches count', () => fixture(async () => {
  await engine.executeRaw('CREATE SEQUENCE retire_commit_seq');
  await engine.executeRaw(`CREATE FUNCTION retire_commit_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
    IF nextval('retire_commit_seq') = ${SOURCE_RETIRE_BATCH_PAGES + 1} THEN RAISE EXCEPTION 'canceling statement due to statement timeout' USING ERRCODE = TG_ARGV[0]; END IF;
    RETURN NULL; END $$`);
  try {
    for (const errcode of ['query_canceled', 'lock_not_available']) {
      await engine.executeRaw('SELECT setval($1, 1, false)', ['retire_commit_seq']);
      await engine.executeRaw(`CREATE CONSTRAINT TRIGGER retire_commit AFTER DELETE ON pages DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION retire_commit_fn('${errcode}')`);
      const before = await pages(source);
      const err = await runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true }).then(() => null, (e: Error & { code?: string; suggestion?: string }) => e);
      await engine.executeRaw('DROP TRIGGER retire_commit ON pages');
      expect({ errcode, code: err?.code }).toEqual({ errcode, code: 'write_pending' });
      expect(await pages(source)).toBe(before - SOURCE_RETIRE_BATCH_PAGES);
      expect(err?.suggestion).toContain(`already deleted ${SOURCE_RETIRE_BATCH_PAGES} page(s)`);
    }
  } finally {
    for (const sql of ['DROP TRIGGER IF EXISTS retire_commit ON pages', 'DROP FUNCTION IF EXISTS retire_commit_fn()', 'DROP SEQUENCE IF EXISTS retire_commit_seq']) await engine.executeRaw(sql);
  }
}), 60_000);

test('an interrupted batch keeps the committed batches and the same request_id resumes to completion', () => fixture(async () => {
  await engine.executeRaw('CREATE TABLE retire_fail(armed boolean NOT NULL, seen integer NOT NULL)');
  await engine.executeRaw(`CREATE FUNCTION retire_fail_fn() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE n integer; BEGIN
    UPDATE retire_fail SET seen=seen+1 WHERE armed RETURNING seen INTO n;
    IF n > ${SOURCE_RETIRE_BATCH_PAGES * 2} THEN RAISE EXCEPTION 'injected retire batch failure'; END IF; RETURN OLD; END $$`);
  await engine.executeRaw('CREATE TRIGGER retire_fail BEFORE DELETE ON pages FOR EACH ROW EXECUTE FUNCTION retire_fail_fn()');
  try {
    await engine.executeRaw('INSERT INTO retire_fail VALUES(true,0)');
    const requestId = randomUUID();
    const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });
    await expect(remove()).rejects.toThrow('injected retire batch failure');
    expect(await pages(source)).toBe(PAGES - SOURCE_RETIRE_BATCH_PAGES * 2);
    expect(await registered()).toBe(1);
    expect(await engine.executeRaw('SELECT state,recovery FROM persistence_topology_changes WHERE request_id=$1::uuid', [requestId])).toEqual([{ state: 'recovering', recovery: null }]);
    await engine.executeRaw('DELETE FROM retire_fail');
    const receipt = await remove();
    expect(receipt).toMatchObject({ state: 'committed', operation: 'remove', pages_deleted: PAGES - SOURCE_RETIRE_BATCH_PAGES * 2 });
    expect(await remove()).toEqual(receipt);
    expect(await pages(source)).toBe(0);
    expect(await registered()).toBe(0);
  } finally {
    for (const sql of ['DROP TRIGGER IF EXISTS retire_fail ON pages', 'DROP FUNCTION IF EXISTS retire_fail_fn()', 'DROP TABLE IF EXISTS retire_fail']) await engine.executeRaw(sql);
  }
}), 60_000);

test('a remove refuses before deleting when the receipt quota is spent, and a CLI writer revoked mid-run stops it at the next batch', () => fixture(async () => {
  const [brain] = await engine.executeRaw<{ n: string }>("SELECT lifetime_ids::text AS n FROM persistence_counters WHERE key='brain'");
  await engine.executeRaw('INSERT INTO config(key,value) VALUES($1,$2) ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value', [journalLimitKey('brainLifetimeIds'), brain?.n ?? '0']);
  await expect(runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true })).rejects.toMatchObject({ code: 'write_quota_exhausted' });
  expect(await pages(source)).toBe(PAGES);
  await engine.executeRaw('DELETE FROM config WHERE key=$1', [journalLimitKey('brainLifetimeIds')]);
  const revoke = "CREATE FUNCTION retire_revoke_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN UPDATE persistence_local_writers SET revoked_at=now() WHERE lane='cli'; RETURN NULL; END $$";
  await engine.executeRaw(revoke);
  await engine.executeRaw('CREATE TRIGGER retire_revoke AFTER DELETE ON pages FOR EACH STATEMENT EXECUTE FUNCTION retire_revoke_fn()');
  try {
    await expect(runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true }))
      .rejects.toMatchObject({ code: 'permission_denied', suggestion: expect.stringContaining(`already deleted ${SOURCE_RETIRE_BATCH_PAGES} page(s)`) });
    expect(await pages(source)).toBe(PAGES - SOURCE_RETIRE_BATCH_PAGES);
    expect(await registered()).toBe(1);
  } finally {
    for (const sql of ['DROP TRIGGER IF EXISTS retire_revoke ON pages', 'DROP FUNCTION IF EXISTS retire_revoke_fn()']) await engine.executeRaw(sql);
  }
}), 60_000);

test('a batch that hits the statement bound is retried at half the size until it fits', () => fixture(async () => {
  await engine.executeRaw(`CREATE FUNCTION retire_bound_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
    IF (SELECT count(*) FROM gone) > 100 THEN RAISE EXCEPTION 'canceling statement due to statement timeout' USING ERRCODE = 'query_canceled'; END IF; RETURN NULL; END $$`);
  await engine.executeRaw('CREATE TRIGGER retire_bound AFTER DELETE ON pages REFERENCING OLD TABLE AS gone FOR EACH STATEMENT EXECUTE FUNCTION retire_bound_fn()');
  try {
    expect(await runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true }))
      .toMatchObject({ state: 'committed', operation: 'remove', pages_deleted: PAGES });
    expect(await pages(source)).toBe(0);
  } finally {
    for (const sql of ['DROP TRIGGER IF EXISTS retire_bound ON pages', 'DROP FUNCTION IF EXISTS retire_bound_fn()']) await engine.executeRaw(sql);
  }
}), 60_000);