Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.0 KB
Raw Blame
/**
 * #6288 (P2.3) with #6443's transaction signal, grouped route: a put_pages group held at `after_publication` (its files
 * already renamed, its transaction still open) past the ceiling and its grace is cancelled: Postgres discards the
 * connection, the rejection waits for the parked body, then surfaces as CONNECTION_CLOSED. Nothing from that attempt
 * commits, the group's restoration puts the original file bytes back, and the retry commits every member exactly once
 * with one set of effects, each file matching its committed row, no member left running. Synthetic content only.
 */
import { describe, expect, test } from 'bun:test';
import { mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { randomUUID } from 'node:crypto';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { hasDatabase } from './helpers.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { withEnv } from '../helpers/with-env.ts';
import type { PostgresEngine } from '../../src/core/postgres-engine.ts';
import { dispatchToolCall } from '../../src/mcp/dispatch.ts';
import { acquireWorktree, claimWorktree, getWorktreeBinding } from '../../src/core/persistence/ownership.ts';
import { localHostId, registerLocalWriter, withVerifiedLocalRegistration, type LocalRegistration } from '../../src/core/persistence/identity.ts';
import { disposePersistenceConsumer, preparePersistedMutation } from '../../src/core/persistence/service.ts';
import { claimGroupFollowers, claimNextWrite, publicationGroupKey } from '../../src/core/persistence/journal.ts';
import { executeClaimedGroup } from '../../src/core/persistence/group-publish.ts';
import { installFaultHook } from '../../src/core/persistence/fault-points.ts';
import type { WriteRequest } from '../../src/core/persistence/model.ts';
import { _resetWriteThroughCacheForTest } from '../../src/core/write-through.ts';

const d = hasDatabase() ? describe : describe.skip;
const config = { engine: 'postgres' as const, embedding_disabled: true };
const page = (slug: string, body: string) => ({ slug, content: `---\ntitle: ${slug}\ntype: note\n---\n\n${body}\n` });
const tick = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));

d('publication deadline on a grouped publish (Postgres)', () => {
  test('a group cancelled after its files were renamed commits nothing, restores the files, and its retry commits every member once', async () => {
    const fixtureDir = mkdtempSync(join(tmpdir(), 'gbrain-group-deadline-pg-'));
    const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
    const engine: PostgresEngine = pg.engine;
    const root = join(fixtureDir, 'brain'); mkdirSync(join(root, 'notes'), { recursive: true });
    _resetWriteThroughCacheForTest();
    let registration: LocalRegistration;
    const dispatch = async (name: string, params: Record<string, unknown>) => {
      const response = await withVerifiedLocalRegistration(engine, registration, () => dispatchToolCall(engine, name, params, {
        remote: true, config, sourceId: 'default',
        auth: { token: 'fixture', clientId: 'fixture-client', scopes: ['read', 'write'], sourceId: 'default', boundSlugPrefixes: ['notes'] },
        logger: { info() {}, warn() {}, error() {} },
      }));
      return JSON.parse((response.content[0] as { text: string }).text) as Record<string, any>;
    };
    const batchRows = () => engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE operation='put_page' AND intent ? 'page_batch' ORDER BY sequence");
    try {
      await withEnv({ GBRAIN_HOME: join(fixtureDir, 'home') }, async () => {
        await engine.setConfig('sync.repo_path', root);
        await claimWorktree(engine, 'default', root);
        registration = await registerLocalWriter(engine, 'stdio', { sourceIds: ['default'], operations: null, scopes: ['read', 'write'], slugPrefixes: ['notes'] });
        try {
          // h-1 replaces an existing file, so the restoration has original bytes to bring back.
          expect((await dispatch('put_page', { ...page('notes/h-1', 'Original body.'), request_id: randomUUID() })).state).toBe('committed');
          const existing = join(root, 'notes', 'h-1.md');
          const original = readFileSync(existing, 'utf8');
          const [current] = await engine.executeRaw<{ revision: string }>("SELECT knowledge_revision::text AS revision FROM pages WHERE slug='notes/h-1'");
          await disposePersistenceConsumer(engine);
          const holder = await acquireWorktree((await getWorktreeBinding(engine, 'default'))!, 5000);
          try {
            const batch = await dispatch('put_pages', { request_id: randomUUID(), wait_ms: 0, pages: [
              { ...page('notes/h-1', 'Replacement body.'), expected_revision: current!.revision }, page('notes/h-2', 'Second body.'), page('notes/h-3', 'Third body.')] });
            expect(batch.state).toBe('pending');
            await disposePersistenceConsumer(engine);
          } finally { await holder!.release(); }
          await engine.executeRaw("UPDATE persistence_requests SET state='queued',execution_token=NULL,claim_expires_at=NULL,blocked_reason=NULL WHERE state='running' AND recovery IS NULL");
          const head = (await claimNextWrite(engine, localHostId()))!;
          const followers = await claimGroupFollowers(engine, head, publicationGroupKey(head)!, 7);
          expect([head, ...followers].map(row => row.slug)).toEqual(['notes/h-1', 'notes/h-2', 'notes/h-3']);

          // Hold the first attempt after its last member's file rename (every file renamed, nothing committed).
          let release!: () => void;
          const released = new Promise<void>(resolve => { release = resolve; });
          let held = false, renamedWhileHeld: string | undefined;
          installFaultHook(async (point, detail) => {
            if (point !== 'publication:after_publication' || held) return;
            const [row] = await engine.executeRaw<{ slug: string }>('SELECT slug FROM persistence_requests WHERE request_id=$1::uuid', [detail.requestId]);
            if (row?.slug !== 'notes/h-3') return;
            held = true;
            renamedWhileHeld = readFileSync(existing, 'utf8');
            await released;
          });
          const events: string[] = [];
          let rolledBackBytes: string | undefined;
          const settled: WriteRequest[] = [];
          const run = executeClaimedGroup(engine, [head, ...followers], {
            hostId: localHostId(), prepare: async row => preparePersistedMutation(engine, row, config as never),
            settled: row => settled.push(row),
            publication: { ceilingMs: 1500, graceMs: 100, settleMs: 100, onOverdue: () => events.push('overdue'), onStuck: () => events.push('stuck') },
            hooks: { rolledBack: async () => { rolledBackBytes ??= readFileSync(existing, 'utf8'); } },
          });
          for (let i = 0; i < 100 && !held; i++) await tick(50);
          expect(held).toBe(true);
          expect(renamedWhileHeld).toContain('Replacement body.');
          await tick(2000);
          expect(events).toEqual(['overdue', 'stuck']);
          // The cancelled attempt committed nothing while its body is parked.
          expect((await batchRows()).filter(row => row.state === 'committed')).toEqual([]);
          release();
          await run;
          installFaultHook(undefined);
          // Restoration ran after the parked body settled: the original bytes were back before any retry.
          expect(rolledBackBytes).toBe(original);

          // The cancelled publication was charged to each member as a transient interruption, which schedules its next claim.
          expect((await batchRows()).map(row => row.transient_interruptions)).toEqual([1, 1, 1]);
          // Whatever the group left queued, the ordinary route publishes it once that backoff has passed.
          for (let i = 0; i < 100; i++) {
            const next = await claimNextWrite(engine, localHostId());
            if (!next) {
              if (!(await batchRows()).some(row => row.state === 'queued')) break;
              await tick(100);
              continue;
            }
            const more = publicationGroupKey(next) ? await claimGroupFollowers(engine, next, publicationGroupKey(next)!, 7) : [];
            await executeClaimedGroup(engine, [next, ...more], { hostId: localHostId(), prepare: async row => preparePersistedMutation(engine, row, config as never), settled: row => settled.push(row) });
          }
          const rows = await batchRows();
          expect(rows.map(row => [row.slug, row.state])).toEqual([['notes/h-1', 'committed'], ['notes/h-2', 'committed'], ['notes/h-3', 'committed']]);
          expect(rows.every(row => row.execution_token === null || row.state !== 'running')).toBe(true);
          expect(await engine.executeRaw("SELECT id FROM persistence_requests WHERE state='running' OR recovery IS NOT NULL")).toEqual([]);
          const [dupes] = await engine.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM (SELECT request_id FROM persistence_requests GROUP BY request_id HAVING count(*)>1) d");
          expect(Number(dupes!.n)).toBe(0);
          const effects = await engine.executeRaw<{ slug: string; kind: string; n: number }>(`SELECT r.slug,e.kind,count(*)::int AS n FROM persistence_effects e
            JOIN persistence_requests r ON r.id=e.request_id WHERE r.intent ? 'page_batch' GROUP BY r.slug,e.kind HAVING count(*)>1`);
          expect(effects).toEqual([]);
          for (const [slug, body] of [['notes/h-1', 'Replacement body.'], ['notes/h-2', 'Second body.'], ['notes/h-3', 'Third body.']] as const) {
            const file = readFileSync(join(root, `${slug}.md`), 'utf8');
            expect(file).toContain(body);
            const snapshot = (await engine.readPageSnapshot(slug, { sourceId: 'default' }))!;
            expect(snapshot.page.compiled_truth).toContain(body);
          }
        } finally { installFaultHook(undefined); await disposePersistenceConsumer(engine); }
      });
    } finally {
      _resetWriteThroughCacheForTest();
      await pg.close();
      rmSync(fixtureDir, { recursive: true, force: true });
    }
  }, 120_000);
});