Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.9 KB
Raw Blame
/**
 * LPH (d) part 2: no version 2 withdrawal-mirror record while a gbrain that cannot read one may claim it.
 *
 * 1. Protects: while the segmented-storage writer gate sees a live writer below SEGMENTED_WRITER_FLOOR in its 7-day window
 *    (an import run of v0.60.290.0 on another host, which reads a version 2 record's `after: null` as a version 1 record
 *    and wedges the worktree), a withdrawal on a segmented, file-backed page commits but its mirror writes no recovery
 *    record and stages nothing: it refuses `segmented_mirror_writer_gate` (fix.next `tell_user_to_run` `gbrain upgrade`,
 *    naming the host), retries, and parks after PARK_AFTER_FAILURES attempts like any failed mirror, so writes on the
 *    worktree are not held; the canonical file keeps the withdrawn row meanwhile. The gate is read from its TTL cache, not
 *    once per attempt. Once the older writer is outside the window, the parked mirror runs again by itself and the file
 *    loses the row. A hold left only by this host's own writes from before it upgraded, which the upgraded build has since
 *    superseded with the same kind of write (doctor `segmented_storage` reports it as information), keeps the pages switch
 *    off but does not hold the mirror; an older write from another host on top of it does, and doctor warns again.
 * 2. Fails when: the mirror stages a version 2 record while a live older writer is seen (master), the hold never parks or
 *    parks without its code, the gate is read per attempt, the parked mirror never resumes after the gate clears, or a
 *    superseded self-hold keeps withdrawn rows in files for its whole window.
 * 3. PGLite always; PostgreSQL when DATABASE_URL is set. The older writers are ingest_log rows stamped v0.60.290.0 with the
 *    segment-reader mark, as test/large-pages/segmented-switch.test.ts seeds the doctor segmented_storage cases.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { segmentedStorageCheck } from '../../src/commands/doctor/checks/segmented-storage.ts';
import { cliRenderContext, renderAction } from '../../src/core/agent-output.ts';
import { OperationError } from '../../src/core/ops/contract.ts';
import { SEGMENTED_WRITER_FLOOR } from '../../src/core/page-body/writer-floor.ts';
import { _resetSegmentedWriterGateForTests, readPageBodyPolicy, segmentedWriterGateWork } from '../../src/core/page-body/limits.ts';
import { runPersistenceEffects } from '../../src/core/persistence/effects.ts';
import { PARK_AFTER_FAILURES } from '../../src/core/persistence/effect-model.ts';
import { installFaultHook } from '../../src/core/persistence/fault-points.ts';
import { localHostId } from '../../src/core/persistence/identity.ts';
import { _resetGatedMirrorResumeForTests, assertSegmentedMirrorGate } from '../../src/core/persistence/segmented-mirror-gate.ts';
import { compareWriterVersions } from '../../src/core/persistence/writer-versions.ts';
import { VERSION } from '../../src/version.ts';
import { openTwinBrain, type TwinBrain } from '../helpers/segmented-twins.ts';
import { exported, fileOf, settle } from '../helpers/segmented-withdrawal.ts';
import { testBackends } from '../helpers/test-backends.ts';
import { waitFor } from '../helpers/wait-for.ts';
import { withEnv } from '../helpers/with-env.ts';

const SEGMENTED = 'people/sei-example';
const CLAIM = 'Twin Example vault code is 5510';
const SELF_CLAIM = 'Twin Example vault code is 5520';
const OTHER_CLAIM = 'Twin Example vault code is 5530';
const KEPT = 'Twin Example adopted a narwhalbot';
const OLDER = '0.60.290.0';
const json = (value: unknown) => typeof value === 'string' ? JSON.parse(value) : value;

for (const kind of testBackends()) {
  describe(kind + ': a segmented mirror holds while an older gbrain could claim its recovery record', () => {
    let brain: TwinBrain;
    let self = '';
    const oldHost = randomUUID();
    const otherHost = randomUUID();
    /** Every recovery record the mirror reserves passes the before_stage seam. */
    const reserved: unknown[] = [];

    const olderRun = (host: string, type: string, ref: string, version: string, ago: string) => brain.engine.executeRaw(
      `INSERT INTO ingest_log (source_id, source_type, source_ref, pages_updated, summary, writer_version, writer_host_id, writer_segment_reader, created_at)
       VALUES ($1, $2, $3, '[]'::jsonb, 'run', $4, $5::uuid, true, now() - $6::interval)`, [brain.sourceId, type, ref, version, host, ago]);
    const resetGate = () => { _resetSegmentedWriterGateForTests(); _resetGatedMirrorResumeForTests(); };
    const pass = () => withEnv(brain.env(), async () => {
      await brain.engine.executeRaw(`UPDATE persistence_effects SET next_attempt_at=now() WHERE source_id=$1 AND state='queued'`, [brain.sourceId]);
      await runPersistenceEffects(brain.engine, { engine: kind, embedding_disabled: true } as never, { hostId: localHostId(), limit: 20 });
    });
    const mirrorOf = async (requestId: string) => (await brain.engine.executeRaw<{ state: string; error_code: string | null; recovery: unknown; data: any; attempts: number }>(
      `SELECT e.state, e.error_code, e.recovery, e.data, e.attempts FROM persistence_effects e JOIN persistence_requests r ON r.id=e.request_id
        WHERE r.request_id=$1::uuid AND e.kind='withdrawal-mirror'`, [requestId]))[0]!;
    /** Purges `claim` and runs effect passes until its mirror parks or commits; returns the request id and any record it held. */
    const purgeUntilSettled = async (claim: string) => {
      const [fact] = await brain.engine.executeRaw<{ id: number }>('SELECT id FROM facts WHERE source_id=$1 AND source_markdown_slug=$2 AND fact=$3', [brain.sourceId, SEGMENTED, claim]);
      const dry = await brain.call('purge_fact', { id: Number(fact!.id), dry_run: true }, SEGMENTED);
      const requestId = randomUUID();
      expect((await brain.call('purge_fact', { id: Number(fact!.id), confirm: dry.confirm_token, expected_revision: dry.expected_revision, request_id: requestId }, SEGMENTED)).state).toBe('committed');
      const recordsSeen: unknown[] = [];
      await waitFor(async () => {
        await pass();
        const row = await mirrorOf(requestId);
        if (row.recovery) recordsSeen.push(json(row.recovery));
        return row.state === 'failed' || row.state === 'committed';
      }, { timeoutMs: 120_000, intervalMs: 100, label: 'the mirror parked or committed' });
      return { requestId, recordsSeen };
    };
    const doctor = () => withEnv(brain.env(), () => segmentedStorageCheck(brain.engine));

    beforeAll(async () => {
      brain = await openTwinBrain(kind, [], async seed => { await seed.setSwitch(true); await seed.put(SEGMENTED, 1_100_000); });
      const facts = [CLAIM, SELF_CLAIM, OTHER_CLAIM, KEPT];
      brain.setChatReply(JSON.stringify({ facts: facts.map(fact => ({ fact, kind: 'fact', entity: SEGMENTED, confidence: 1, notability: 'high' })) }));
      expect(await brain.call('extract_facts', { turn_text: 'Synthetic meeting record. ' + facts.join('. ') + '.', visibility: 'world', valid_from: '2026-10-15T00:00:00Z' }, SEGMENTED))
        .toMatchObject({ inserted: facts.length });
      expect(await brain.storageOf(SEGMENTED)).toBe('segmented');
      await settle(brain, kind);
      self = await withEnv(brain.env(), async () => localHostId());
      installFaultHook(async (point, detail) => {
        if (point !== 'effect_recovery:before_stage') return;
        const [row] = await brain.engine.executeRaw<{ recovery: unknown }>('SELECT recovery FROM persistence_effects WHERE id=$1', [detail.effectId]);
        reserved.push(json(row?.recovery));
      });
    }, 600_000);

    afterAll(() => { installFaultHook(undefined); return brain.close(); }, 120_000);

    test('the mirror writes no version 2 record, parks with segmented_mirror_writer_gate, and resumes once the older writer ages out', async () => {
      expect(compareWriterVersions(OLDER, SEGMENTED_WRITER_FLOOR)).toBeLessThan(0);
      await olderRun(oldHost, 'import', 'lp-d-older-writer', OLDER, '1 hour');
      resetGate();
      const file = fileOf(brain, SEGMENTED);
      const before = readFileSync(file);
      expect(before.toString('utf8')).toContain(CLAIM);
      const reads = segmentedWriterGateWork.reads;
      reserved.length = 0;
      const { requestId, recordsSeen } = await purgeUntilSettled(CLAIM);
      const held = await mirrorOf(requestId);
      // No record was reserved or seen, nothing was staged, and the file still holds the purged claim.
      expect(reserved).toEqual([]);
      expect(recordsSeen).toEqual([]);
      expect(held.recovery).toBeNull();
      expect(readFileSync(file).equals(before)).toBe(true);
      // Parked like any failed mirror after its attempts, with the gate's code on the target.
      expect(held.state).toBe('failed');
      expect(held.error_code).toBe('targets_parked');
      expect(json(held.data).parked).toEqual([{ slug: SEGMENTED, error_code: 'segmented_mirror_writer_gate' }]);
      expect(held.attempts).toBeGreaterThanOrEqual(PARK_AFTER_FAILURES);
      // The gate was read from its cache: at most once for all of those attempts.
      expect(segmentedWriterGateWork.reads - reads).toBeLessThanOrEqual(1);
      // Its refusal names the older host and tells the user to upgrade it.
      const refusal = await assertSegmentedMirrorGate(brain.engine, { source_id: brain.sourceId } as never, SEGMENTED, self).then(() => null, error => error);
      expect(refusal).toBeInstanceOf(OperationError);
      expect((refusal as OperationError).code).toBe('segmented_mirror_writer_gate');
      expect((refusal as OperationError).suggestion).toContain(oldHost);
      expect((refusal as OperationError).fix?.argv).toEqual(['gbrain', 'upgrade']);
      expect((refusal as OperationError).fix?.user_message).toContain(oldHost);
      expect(renderAction((refusal as OperationError).fix!, cliRenderContext()).next).toBe('tell_user_to_run');

      // The older writer ages out of the window: the parked mirror runs again by itself and the file loses the row.
      await brain.engine.executeRaw(`UPDATE ingest_log SET created_at=now() - interval '8 days' WHERE source_ref='lp-d-older-writer'`);
      resetGate();
      await waitFor(async () => { await pass(); return (await mirrorOf(requestId)).state === 'committed'; }, { timeoutMs: 120_000, intervalMs: 100, label: 'the mirror resumed and committed' });
      await settle(brain, kind);
      const after = readFileSync(file);
      expect(after.toString('utf8')).not.toContain(CLAIM);
      expect(after.toString('utf8')).toContain(KEPT);
      expect(after.equals(await exported(brain, SEGMENTED))).toBe(true);
      expect(reserved).toHaveLength(1);
      expect(reserved[0]).toMatchObject({ version: 2, after: null });
    }, 600_000);

    test('this host\'s own superseded pre-upgrade writes do not hold the mirror; an older write from another host does', async () => {
      const file = fileOf(brain, SEGMENTED);
      // This host synced with v0.60.290.0, then upgraded and synced again: the switch stays held for 7 days, doctor reports
      // the hold as information, and the mirror proceeds.
      await olderRun(self, 'sync', 'lp-d-self-before-upgrade', OLDER, '2 hours');
      await olderRun(self, 'sync', 'lp-d-self-after-upgrade', VERSION, '1 hour');
      resetGate();
      expect((await readPageBodyPolicy(brain.engine)).segmented).toBe(false);
      expect(await doctor()).toMatchObject({ status: 'ok', severity: 'info' });
      reserved.length = 0;
      const selfHold = await purgeUntilSettled(SELF_CLAIM);
      expect(await mirrorOf(selfHold.requestId)).toMatchObject({ state: 'committed', error_code: null, recovery: null });
      expect(readFileSync(file, 'utf8')).not.toContain(SELF_CLAIM);
      expect(reserved).toHaveLength(1);
      expect(reserved[0]).toMatchObject({ version: 2, after: null });
      await settle(brain, kind);

      // Another host's older write on top of it is live: doctor warns, and the mirror holds and parks naming only that host.
      await olderRun(otherHost, 'import', 'lp-d-other-host', OLDER, '30 minutes');
      resetGate();
      expect(await doctor()).toMatchObject({ status: 'warn', fix: { argv: ['gbrain', 'upgrade'] } });
      reserved.length = 0;
      const before = readFileSync(file);
      const otherHold = await purgeUntilSettled(OTHER_CLAIM);
      const held = await mirrorOf(otherHold.requestId);
      expect(reserved).toEqual([]);
      expect(otherHold.recordsSeen).toEqual([]);
      expect(held.state).toBe('failed');
      expect(json(held.data).parked).toEqual([{ slug: SEGMENTED, error_code: 'segmented_mirror_writer_gate' }]);
      expect(readFileSync(file).equals(before)).toBe(true);
      const refusal = await assertSegmentedMirrorGate(brain.engine, { source_id: brain.sourceId } as never, SEGMENTED, self).then(() => null, error => error) as OperationError;
      expect(refusal.code).toBe('segmented_mirror_writer_gate');
      expect(refusal.suggestion).toContain(otherHost);
      expect(refusal.suggestion).not.toContain(self);
    }, 600_000);
  });
}