Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.4 KB
Raw Blame
/**
 * #6288 part 1 / #6352 (P2.3, UC2): a publish that neither resolves nor rejects. `boundPublication` never settles before
 * the publish does (nothing releases or reports a request while its transaction can still commit), marks it overdue at
 * the ceiling, aborts its signal after the grace (it reaches the transaction only on an engine that takes
 * `transaction(fn, { signal })`) and reports stuck after the settle window. Barrier cases hold a real publishMutation at
 * `before_publication` (after the token lock), `after_publication` (after the file rename) and `before_commit`, then
 * release it: the write commits exactly once. On Postgres the publish transaction carries a transaction-local
 * idle_in_transaction_session_timeout, so a publisher idle past it is ended by the server and commits nothing.
 * Synthetic content only.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { operations, type OperationContext } from '../src/core/operations.ts';
import { submissionAuthority } from '../src/core/persistence/authority.ts';
import { admitWrite, claimNextWrite, getWriteRequestById } from '../src/core/persistence/journal.ts';
import { localHostId } from '../src/core/persistence/identity.ts';
import { claimWorktree, getWorktreeBinding } from '../src/core/persistence/ownership.ts';
import { preparePageMutation } from '../src/core/persistence/page-prepare.ts';
import { publishMutation } from '../src/core/persistence/coordinator.ts';
import { publishSingleWrite } from '../src/core/persistence/group-publish.ts';
import { bindPublicationTimeouts, boundPublication, publicationIdleTimeoutMs, publicationTransaction, PUBLICATION_CEILING_DEFAULT_MS } from '../src/core/persistence/publication-deadline.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { _resetWriteThroughCacheForTest } from '../src/core/write-through.ts';
import { testBackends } from './helpers/test-backends.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv } from './helpers/with-env.ts';

const putPage = operations.find(o => o.name === 'put_page')!;
const page = (body: string) => `---\ntitle: Example\ntype: note\n---\n\n${body}\n`;
const tick = (ms: number) => new Promise(r => setTimeout(r, ms));

/** A manual scheduler for `PublicationBound.later`: time moves only through `advance`, so a stalled runner cannot fire a phase early. */
function manualClock() {
  let now = 0;
  const pending: Array<{ at: number; fn: () => void; cancelled: boolean; fired: boolean }> = [];
  return {
    later: (ms: number, fn: () => void) => { const entry = { at: now + ms, fn, cancelled: false, fired: false }; pending.push(entry); return () => { entry.cancelled = true; }; },
    async advance(ms: number) {
      now += ms;
      for (const entry of [...pending].sort((a, b) => a.at - b.at)) if (!entry.cancelled && !entry.fired && entry.at <= now) { entry.fired = true; entry.fn(); }
      await Promise.resolve();
    },
    pending: () => pending.filter(entry => !entry.cancelled && !entry.fired).length,
  };
}

describe('boundPublication', () => {
  test('never settles before the publish; overdue at the ceiling, signal aborted after the grace, stuck after the settle window', async () => {
    const events: string[] = [];
    let release!: (value: string) => void, seen: AbortSignal | undefined;
    const fake = { transaction: (fn: (tx: unknown) => Promise<unknown>, opts?: { signal?: AbortSignal }) => { seen = opts?.signal; return fn({}); } } as unknown as BrainEngine;
    let stuckSettled: Promise<void> | undefined;
    const clock = manualClock();
    const work = boundPublication(() => publicationTransaction(fake, () => new Promise<string>(resolve => { release = resolve; })),
      { ceilingMs: 20, graceMs: 20, settleMs: 20, later: clock.later, onOverdue: () => events.push('overdue'), onStuck: settled => { events.push('stuck'); stuckSettled = settled; } });
    let outcome: string | undefined;
    void work.then(value => { outcome = value; });
    await clock.advance(10);
    // A loaded runner stalling here used to let the wall-clock abort fire before the next assertion; the manual clock does not move.
    { const stalled = performance.now(); while (performance.now() - stalled < 15); }
    expect(events).toEqual([]); expect(seen?.aborted).toBe(false);
    await clock.advance(20); expect(events).toEqual(['overdue']); expect(seen?.aborted).toBe(false);
    await clock.advance(20); expect(seen?.aborted).toBe(true); expect(events).toEqual(['overdue']);
    await clock.advance(25); expect(events).toEqual(['overdue', 'stuck']);
    expect(outcome).toBeUndefined();
    release('committed');
    expect(await work).toBe('committed');
    await stuckSettled;
    expect(clock.pending()).toBe(0);
  });

  test('a publish that settles in time fires nothing; a rejection propagates; outside a publication no signal is passed', async () => {
    const events: string[] = [];
    expect(await boundPublication(async () => 'ok', { ceilingMs: 30, onOverdue: () => events.push('overdue'), onStuck: () => events.push('stuck') })).toBe('ok');
    await expect(boundPublication(async () => { throw new Error('boom'); }, { ceilingMs: 30 })).rejects.toThrow('boom');
    await tick(50);
    expect(events).toEqual([]);
    let opts: unknown = 'unset';
    await publicationTransaction({ transaction: (fn: (tx: unknown) => Promise<unknown>, o?: unknown) => { opts = o; return fn({}); } } as unknown as BrainEngine, async () => 1);
    expect(opts).toBeUndefined();
    expect(PUBLICATION_CEILING_DEFAULT_MS).toBe(300_000);
    expect(publicationIdleTimeoutMs(60_000)).toBe(60_000);
  });
});

for (const kind of testBackends()) {
  describe(`a publish held past the publication ceiling (${kind})`, () => {
    let engine: BrainEngine;
    let close: () => Promise<void>;
    let root = '';
    const home = mkdtempSync(join(tmpdir(), 'gbrain-pub-deadline-home-'));
    beforeAll(async () => {
      if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));
      else { const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engine = lite; close = () => lite.disconnect(); }
    }, 120_000);
    afterAll(async () => { await withEnv({ GBRAIN_HOME: home }, () => disposePersistenceConsumer(engine)); await close?.(); rmSync(home, { recursive: true, force: true }); });
    beforeEach(async () => {
      await withEnv({ GBRAIN_HOME: home }, () => disposePersistenceConsumer(engine));
      if (kind === 'pglite') await resetPgliteState(engine as PGLiteEngine);
      _resetWriteThroughCacheForTest();
      if (kind === 'postgres' && root) return;
      root = mkdtempSync(join(home, 'brain-'));
      await engine.setConfig('sync.repo_path', root);
      if (kind === 'postgres') {
        // Postgres binds no worktree on first write; claim it once for this file (a second claim would need a transfer).
        await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
        await engine.executeRaw("UPDATE sources SET local_path=$1 WHERE id='default'", [root]);
        await withEnv({ GBRAIN_HOME: home }, () => claimWorktree(engine, 'default', root));
        await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
      }
    });
    const context = (): OperationContext => ({ engine, config: { engine: kind, embedding_disabled: true } as OperationContext['config'],
      logger: { info() {}, warn() {}, error() {} }, dryRun: false, remote: false, sourceId: 'default' });

    /** Admits and claims one put_page of `slug` the way the resident consumer would, after a first write bound the worktree. */
    async function claimedPut(slug: string) {
      await putPage.handler(context(), { slug: `notes/bootstrap-${randomUUID().slice(0, 8)}`, content: page('Binds the worktree.'), request_id: randomUUID() });
      await disposePersistenceConsumer(engine);
      const binding = (await getWorktreeBinding(engine, 'default'))!;
      const [source] = await engine.executeRaw<{ incarnation: string }>("SELECT incarnation FROM sources WHERE id='default'");
      const authority = await submissionAuthority(context(), 'put_page', 'default', source!.incarnation, slug);
      const intent = { content: page('Written by a process that cannot lock.') };
      await admitWrite(engine, { principal: authority.principal, authority, operation: 'put_page', sourceId: 'default', sourceIncarnation: source!.incarnation,
        slug, pageId: null, worktreeId: binding.worktree_id, topologyGeneration: binding.topology_generation, requestId: randomUUID(), callerIntent: intent, intent });
      const row = (await claimNextWrite(engine, localHostId()))!;
      return { row, binding, prepared: await preparePageMutation(engine, row, context().config) };
    }
    for (const boundary of ['before_publication', 'after_publication', 'before_commit'] as const) {
      test(`a publish held at ${boundary} past the ceiling is reported overdue and stuck, never settled early; released, it commits once (on Postgres after the aborted attempt is requeued)`, async () => withEnv({ GBRAIN_HOME: home }, async () => {
        const slug = `notes/held-${boundary.replaceAll('_', '-')}`;
        const { row, prepared } = await claimedPut(slug);
        let open!: () => void;
        const barrier = new Promise<void>(resolve => { open = resolve; });
        let reached!: () => void;
        const atBarrier = new Promise<void>(resolve => { reached = resolve; });
        const events: string[] = [];
        // The ceiling leaves the publish time to reach its barrier, so the abort always lands on a body parked there.
        const work = boundPublication(() => publishMutation(engine, row, prepared, localHostId(), { boundary: async name => { if (name === boundary) { reached(); await barrier; } } }),
          { ceilingMs: 1500, graceMs: 30, settleMs: 30, onOverdue: () => events.push('overdue'), onStuck: () => events.push('stuck') });
        let settled = false;
        void work.then(() => { settled = true; }, () => { settled = true; });
        await atBarrier;
        expect(events).toEqual([]);
        await tick(1700);
        expect(events).toEqual(['overdue', 'stuck']);
        expect(settled).toBe(false);
        // PGLite has one connection, held by the open publish transaction: a read there would wait behind it.
        if (kind === 'postgres') expect((await getWriteRequestById(engine, row.id))?.state).toBe('running');
        open();
        let done = await work;
        if (kind === 'postgres') {
          // The signal discarded the transaction's connection: nothing committed, and the claim went back to the queue.
          expect(done.state).toBe('queued');
          expect(await engine.getPage(slug, { sourceId: 'default' })).toBeNull();
          // The ceiling cancel counts against persistence.max_transient_retries and backs the claim off first.
          const [charged] = await engine.executeRaw<{ n: number; waiting: boolean }>(
            'SELECT transient_interruptions AS n, next_attempt_at > now() AS waiting FROM persistence_requests WHERE id=$1::uuid', [row.id]);
          expect(charged).toEqual({ n: 1, waiting: true });
          expect(await claimNextWrite(engine, localHostId())).toBeNull();
          await engine.executeRaw('UPDATE persistence_requests SET next_attempt_at=now() WHERE id=$1::uuid', [row.id]);
          const again = (await claimNextWrite(engine, localHostId()))!;
          expect(again.id).toBe(row.id);
          done = await publishMutation(engine, again, await preparePageMutation(engine, again, context().config), localHostId());
        }
        expect(done.state).toBe('committed');
        const [count] = await engine.executeRaw<{ n: number | string }>("SELECT count(*) AS n FROM persistence_requests WHERE id=$1::uuid AND state='committed'", [row.id]);
        expect(Number(count!.n)).toBe(1);
        expect(await engine.getPage(slug, { sourceId: 'default' })).not.toBeNull();
      }), 120_000);
    }

    test('the publish transaction opens with the idle bound (Postgres) and receives the publication signal', async () => withEnv({ GBRAIN_HOME: home }, async () => {
      const { row, prepared } = await claimedPut('notes/bound-wiring');
      const first: string[] = [], signals: Array<AbortSignal | undefined> = [];
      const recording = new Proxy(engine, { get(target, key) {
        if (key === 'transaction') return <T>(fn: (tx: BrainEngine) => Promise<T>, opts?: { signal?: AbortSignal }) => {
          signals.push(opts?.signal);
          let seen = false;
          return target.transaction(tx => fn(new Proxy(tx, { get(t, k) {
            if (k === 'executeRaw') return (sql: string, ...rest: unknown[]) => { if (!seen) { seen = true; first.push(sql); } return (t.executeRaw as (...a: unknown[]) => unknown)(sql, ...rest); };
            const member = Reflect.get(t, k, t); return typeof member === 'function' ? member.bind(t) : member;
          } })));
        };
        const member = Reflect.get(target, key, target); return typeof member === 'function' ? member.bind(target) : member;
      } }) as BrainEngine;
      const done = await boundPublication(() => publishMutation(recording, row, prepared, localHostId()), { ceilingMs: 60_000 });
      expect(done.state).toBe('committed');
      const publish = first.findIndex(sql => sql.includes('idle_in_transaction_session_timeout'));
      if (kind === 'postgres') expect(publish).toBeGreaterThanOrEqual(0); else expect(publish).toBe(-1);
      expect(signals.some(signal => signal instanceof AbortSignal)).toBe(true);
    }), 120_000);

    for (const path of kind === 'postgres' ? ['publishMutation', 'publishSingleWrite'] as const : ['publishMutation'] as const)
    test(`a refused timeout bind fails the publication inside its own transaction, with no unhandled rejection, and leaves no page (${path})`, async () => withEnv({ GBRAIN_HOME: home }, async () => {
      const slug = `notes/bind-refused-${path.toLowerCase()}`;
      const { row, prepared } = await claimedPut(slug);
      const unhandled: unknown[] = [];
      const onUnhandled = (reason: unknown) => { unhandled.push(reason); };
      process.on('unhandledRejection', onUnhandled);
      try {
        const refusing = new Proxy(engine, { get(target, key) {
          if (key === 'transaction') return <T>(fn: (tx: BrainEngine) => Promise<T>) => target.transaction(tx => fn(new Proxy(tx, { get(t, k) {
            if (k === 'kind') return 'postgres';
            if (k === 'executeRaw') return (sql: string, ...rest: unknown[]) => sql.includes('idle_in_transaction_session_timeout')
              ? Promise.reject(Object.assign(new Error('SET refused by the pooler'), { code: '0A000' })) : (t.executeRaw as (...a: unknown[]) => unknown)(sql, ...rest);
            const member = Reflect.get(t, k, t); return typeof member === 'function' ? member.bind(t) : member;
          } })));
          const member = Reflect.get(target, key, target); return typeof member === 'function' ? member.bind(target) : member;
        } }) as BrainEngine;
        const outcome = await (path === 'publishMutation' ? publishMutation(refusing, row, prepared, localHostId()) : publishSingleWrite(refusing, row, prepared, localHostId())).then(done => ({ done }), error => ({ error }));
        await tick(50);
        expect(unhandled).toEqual([]);
        if ('done' in outcome) expect(outcome.done.state).not.toBe('committed');
        else expect(String((outcome.error as Error).message)).toContain('SET refused');
        expect(await engine.getPage(slug, { sourceId: 'default' })).toBeNull();
      } finally { process.off('unhandledRejection', onUnhandled); }
    }), 120_000);

    test(kind === 'postgres' ? 'a publish transaction idle past its bound is ended by the server and commits nothing' : 'PGLite: the idle bound is never applied (an ended idle transaction would wedge its only connection)', async () => {
      const before = await engine.executeRaw<{ n: number | string }>("SELECT count(*) AS n FROM config WHERE key='p23.probe'");
      const attempt = publicationTransaction(engine, async tx => {
        await bindPublicationTimeouts(tx);
        const [shown] = await tx.executeRaw<{ v: string }>("SELECT current_setting('idle_in_transaction_session_timeout') AS v");
        // PGLite would enforce the bound too, and an idle transaction it ends wedges its only connection, so it is never set there.
        expect(shown!.v === '0').toBe(kind !== 'postgres');
        if (kind === 'postgres') await tx.executeRaw("SELECT set_config('idle_in_transaction_session_timeout','300ms',true)");
        await tx.executeRaw("INSERT INTO config(key,value) VALUES('p23.probe','x')");
        await tick(1200);
        await tx.executeRaw('SELECT 1');
        return 'committed';
      });
      if (kind === 'postgres') await expect(attempt).rejects.toBeDefined();
      else expect(await attempt).toBe('committed');
      const [after] = await engine.executeRaw<{ n: number | string }>("SELECT count(*) AS n FROM config WHERE key='p23.probe'");
      expect(Number(after!.n) - Number(before[0]!.n)).toBe(kind === 'postgres' ? 0 : 1);
      if (kind !== 'postgres') await engine.executeRaw("DELETE FROM config WHERE key='p23.probe'");
    }, 30_000);
  });
}