Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.0 KB
Raw Blame
/**
 * #6188 PR3 fence census (`src/core/fence-repair/census.ts`, `census-store.ts`).
 *
 * Protects: the candidate finder sees every malformed facts or takes fence
 * once (an `invalid_fence` hold, a stored page, an unsynced working-tree
 * file), with its planned tier, bound to the bytes it judged and location
 * only; the DB scan's watermark never skips a page whose transaction
 * committed after a pass that started later than its `updated_at`; the
 * one-time backfill and the file walk resume across deadlines instead of
 * restarting, so damage beyond several deadlines is found; a later walk reads
 * only changed, untracked and stored files; census, candidate and attempt
 * rows survive the 7-day purge while their source incarnation lives; the
 * normalization trend counts a resumed sync run and a replayed write once.
 * Fails when: the watermark advances to the pass start (late commits lost),
 * a cursor restarts at the prefix, dedup counts a page twice, a finding keeps
 * cell text, or the purge drops live census state.
 * Seams: a ticking clock for deadlines; PGLite always, Postgres through
 * test/e2e/fence-census-postgres.test.ts (a second session holds a real open
 * transaction). Synthetic content only.
 */
import { afterAll, afterEach, beforeAll, describe, expect, setSystemTime, test } from 'bun:test';
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { PostgresEngine } from '../src/core/postgres-engine.ts';
import { sha256 } from '../src/core/persistence/digest.ts';
import { writeGitHold } from '../src/core/persistence/sync-holds.ts';
import { purgeStaleCheckpoints } from '../src/core/op-checkpoint.ts';
import { FENCE_REPAIR_ATTEMPT_OP } from '../src/core/fence-repair/attempts.ts';
import { listFenceCandidates, runFenceCensus, summarizeFenceCensus } from '../src/core/fence-repair/census.ts';
import {
  FENCE_CANDIDATE_OP, FENCE_SCAN_OP, FENCE_TREND_OP, readScanRecord, readTrend, recordPublicationFenceTrend, recordSyncRunTrend,
} from '../src/core/fence-repair/census-store.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';

const backends = testBackends();
const FB = '<!--- gbrain:facts:begin -->', FE = '<!--- gbrain:facts:end -->';
const FH = '| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |';
const FS = '|---|-------|------|------------|------------|------------|------------|-------------|--------|---------|';
const T = '<!--- gbrain:takes:begin -->', TE = '<!--- gbrain:takes:end -->';
const TH = '| # | claim | kind | who | weight | since | source |\n|---|---|---|---|---|---|---|';
// Unique strings that must never leave the page: a claim, a holder and a kind.
const CLAIM = 'Sentinelclaimzq8 ships quarterly', HOLDER = 'Sentinelholderzq8 Example', KIND = 'sentinelkindzq8';
const SECRETS = [CLAIM, HOLDER, KIND, 'Sentinelclaimzq8', 'Sentinelholderzq8'];
const factsRow = (n: number, claim: string) => `| ${n} | ${claim} | fact | 1.0 | private | medium | 2026-01-01 |  | call |  |`;
/** Missing end marker after the table: Tier 1 closes it (deterministic). */
const DETERMINISTIC = ['## Facts', '', FB, FH, FS, factsRow(1, 'Synthetic fact'), factsRow(2, 'Second synthetic fact'), '', ''].join('\n');
/** A display-name holder: the resolver tier. */
const RESOLVER = `${T}\n${TH}\n| 1 | ${CLAIM} | take | ${HOLDER} | 0.7 | 2026-01 | chat |\n${TE}\n`;
/** A takes kind gbrain never chooses: a manual edit. */
const MANUAL = `${T}\n${TH}\n| 1 | Synthetic take | ${KIND} | brain | 0.7 | 2026-01 | chat |\n${TE}\n`;
/** Rows with no header: the model tier. */
const LLM = ['## Facts', '', FB, factsRow(1, 'Synthetic fact'), factsRow(2, 'Another synthetic fact'), FE, ''].join('\n');
const CLEAN = ['## Facts', '', FB, FH, FS, factsRow(1, 'Clean synthetic fact'), FE, ''].join('\n');
const md = (title: string, body: string) => `---\ntitle: ${title}\n---\n${body}`;
const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
const roots: string[] = [];

/** A deterministic clock that moves `step` ms on every read, so deadlines stop scans after a fixed amount of work. */
function ticking(step = 1) {
  let t = Date.parse('2026-10-06T12:00:00Z');
  return { now: () => new Date(t += step), at: () => t };
}

for (const backend of backends) {
  describe(`${backend}: fence census`, () => {
    let engine: BrainEngine;
    let second: PostgresEngine | undefined;
    let close: () => Promise<void> = async () => {};

    beforeAll(async () => {
      if (backend === 'pglite') {
        const pglite = new PGLiteEngine();
        await pglite.connect({});
        await pglite.initSchema();
        engine = pglite;
        close = () => pglite.disconnect();
        return;
      }
      const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
      engine = pg.engine;
      second = new PostgresEngine();
      await second.connect({ database_url: pg.databaseUrl, poolSize: 2 });
      close = async () => { await second!.disconnect(); await pg.close(); };
    }, 120_000);

    afterAll(async () => { await close(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
    afterEach(() => { setSystemTime(); });

    async function source(files: Record<string, string> | null, opts: { git?: boolean } = {}) {
      const id = `cen-${randomUUID().replace(/-/g, '').slice(0, 12)}`;
      const root = files ? mkdtempSync(join(tmpdir(), 'fence-census-')) : null;
      const write = (path: string, content: string) => { mkdirSync(join(root!, path, '..'), { recursive: true }); writeFileSync(join(root!, path), content); };
      const commit = () => { git(root!, 'add', '-A'); git(root!, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'content'); return git(root!, 'rev-parse', 'HEAD'); };
      if (root) {
        roots.push(root);
        for (const [path, content] of Object.entries(files!)) write(path, content);
        if (opts.git !== false) { git(root, 'init', '-q'); commit(); }
      }
      await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', [id, root]);
      const [{ incarnation }] = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id=$1', [id]);
      const stored = (slug: string, body: string) => engine.putPage(slug, { type: 'note', title: slug, compiled_truth: body, timeline: '' }, { sourceId: id });
      const census = (extra: Partial<Parameters<typeof runFenceCensus>[1]> = {}) => runFenceCensus(engine, { sourceIds: [id], deadline: Date.now() + 120_000, ...extra });
      const keys = async () => (await listFenceCandidates(engine, [id])).map(c => c.key);
      return { id, root: root!, incarnation, write, commit, stored, census, keys };
    }

    const expectNoSecrets = (text: string) => { for (const secret of SECRETS) expect(text).not.toContain(secret); };

    test('holds, stored pages and files are found once each, tiered, bound to their bytes and location-only', async () => {
      const s = await source({ 'notes/det.md': md('Det', DETERMINISTIC), 'notes/both.md': md('Both', RESOLVER), 'notes/held.md': md('Held', MANUAL),
        'notes/clean.md': md('Clean', CLEAN), 'notes/model.md': md('Model', LLM) });
      await s.stored('notes/both', RESOLVER);
      await s.stored('notes/stored-only', DETERMINISTIC);
      await s.stored('notes/clean', CLEAN);
      const observed = new Date().toISOString();
      await engine.transaction(tx => writeGitHold(tx, { source_id: s.id, incarnation: s.incarnation, path: 'notes/held.md', source_path: 'notes/held.md', slug: 'notes/held',
        page_id: null, code: 'invalid_fence', message: 'Fence takes_kind_unsupported: in the takes fence (body), row 1, column kind.', upstream_version: 'v1',
        observed_at: observed, run_id: 'run-1', mode: 'managed',
        meta: { reason: 'takes_kind_unsupported', recovery_version: 1, fence_version: 1,
          fence: { reason: 'takes_kind_unsupported', fence: 'takes', section: 'body', rows: [1], columns: ['kind'], line: 4 } } }));

      expect(await s.census()).toEqual([{ source_id: s.id, complete: true, fresh_at: expect.any(String), backfill_done: true, pages_caught_up: true, files: 'done' }]);
      const list = await listFenceCandidates(engine, [s.id]);
      expect(list.map(c => [c.key, c.bucket, c.origins, c.tier])).toEqual([
        ['notes/both', 'page', ['page', 'file'], 'resolver'],
        ['notes/det', 'file', ['file'], 'deterministic'],
        ['notes/held', 'hold', ['hold', 'file'], 'manual'],
        ['notes/model', 'file', ['file'], 'llm'],
        ['notes/stored-only', 'page', ['page'], 'deterministic'],
      ]);
      const both = list.find(c => c.key === 'notes/both')!;
      expect(both).toMatchObject({ reasons: ['holder_unresolved'], location: { fence: 'takes', section: 'body', rows: [1], columns: ['who'] }, path: 'notes/both.md' });
      expect(both.bound.page_sha256).toMatch(/^[0-9a-f]{64}$/);
      expect(both.bound.file_sha256).toBe(sha256(md('Both', RESOLVER)));
      expect(typeof both.bound.updated_at).toBe('string');
      expect(list.find(c => c.key === 'notes/held')!).toMatchObject({ path: 'notes/held.md', bound: { upstream_version: 'v1' } });

      const [census] = await summarizeFenceCensus(engine, [s.id]);
      expect(census).toMatchObject({ source_id: s.id, total: 5, holds: { manual: 1, total: 1 }, pages: { resolver: 1, deterministic: 1, total: 2 },
        files: { deterministic: 1, llm: 1, total: 2 }, by_tier: { deterministic: 2, resolver: 1, llm: 1, manual: 1, total: 5 },
        scan: { complete: true, backfill_done: true, walking: false } });
      expect(census!.oldest_hold_at).toBeTruthy();

      // Fixing the file and the stored copy clears both parts; the stored-only page stays.
      s.write('notes/both.md', md('Both', CLEAN)); s.commit();
      await s.stored('notes/both', CLEAN);
      await s.census();
      expect(await s.keys()).toEqual(['notes/det', 'notes/held', 'notes/model', 'notes/stored-only']);

      const rows = await engine.executeRaw<{ completed_keys: unknown }>('SELECT completed_keys FROM op_checkpoints WHERE op=ANY($1::text[]) AND fingerprint LIKE $2',
        [[FENCE_CANDIDATE_OP, FENCE_SCAN_OP], `${s.id}:%`]);
      for (const blob of [JSON.stringify(rows), JSON.stringify(list), JSON.stringify(await summarizeFenceCensus(engine, [s.id]))]) expectNoSecrets(blob);
    }, 60_000);

    test('the incremental pass reads pages updated since the watermark, including a late commit older than the previous pass', async () => {
      const s = await source(null);
      for (const slug of ['notes/a', 'notes/b', 'notes/c']) await s.stored(slug, CLEAN);
      expect((await s.census())[0]).toMatchObject({ complete: true, files: 'none' });
      expect(await s.keys()).toEqual([]);

      await s.stored('notes/a', DETERMINISTIC);
      await s.census();
      expect(await s.keys()).toEqual(['notes/a']);

      // A transaction that set updated_at before the previous pass started and committed after it ended.
      const { watermark } = (await readScanRecord(engine, s.id, s.incarnation))!.pages;
      const [late] = await engine.executeRaw<{ before_now: boolean }>(`UPDATE pages SET compiled_truth=$1, updated_at=$2::timestamptz + interval '90 seconds'
        WHERE source_id=$3 AND slug='notes/b' RETURNING updated_at < now() - interval '1 second' AS before_now`, [RESOLVER, watermark, s.id]);
      expect(late!.before_now).toBe(true);
      await s.census();
      expect(await s.keys()).toEqual(['notes/a', 'notes/b']);

      await s.stored('notes/a', CLEAN);
      await engine.executeRaw('UPDATE pages SET deleted_at=now(), updated_at=now() WHERE source_id=$1 AND slug=$2', [s.id, 'notes/b']);
      await s.census();
      expect(await s.keys()).toEqual([]);
    }, 60_000);

    if (backend === 'pglite') {
      // GBRA-75 wave 10: the pass orders by the timestamp its keyset compares, not the text it returns, so small
      // batches never skip a page, even in a session time zone whose text offsets change (a DST fallback hour). It also
      // runs with the clock pinned inside the 2026 and 2027 fallback hours, where fixed instants fell behind the watermark.
      for (const clock of [null, '2026-11-01T05:30:00Z', '2027-11-07T05:30:00Z']) test(`the incremental pass in batches of one reads every page across a DST fallback in the session time zone${clock ? ` (clock pinned at ${clock})` : ''}`, async () => {
        if (clock) setSystemTime(new Date(clock));
        try {
          const s = await source(null);
          const slugs = ['notes/dst-a', 'notes/dst-b', 'notes/dst-c', 'notes/dst-d'];
          for (const slug of slugs) await s.stored(slug, CLEAN);
          await s.census();
          const [{ zone }] = await engine.executeRaw<{ zone: string }>(`SELECT current_setting('TimeZone') AS zone`);
          try {
            await engine.executeRaw(`SELECT set_config('TimeZone', 'America/New_York', false)`);
            // The next fallback (first Sunday of November) still ahead of the database clock, so every instant is past the
            // pass watermark: 01:20-04, 01:40-04, then 01:10-05 and 01:30-05, whose text sorts differently from their instants.
            const [{ ms }] = await engine.executeRaw<{ ms: number }>(`SELECT (extract(epoch FROM now()) * 1000)::float8 AS ms`);
            let first = 0;
            for (let year = new Date(Number(ms)).getUTCFullYear(); first <= Number(ms); year++) first = Date.UTC(year, 10, 1 + (7 - new Date(Date.UTC(year, 10, 1)).getUTCDay()) % 7, 5, 20);
            const instants = [0, 20, 50, 70].map(m => new Date(first + m * 60_000).toISOString());
            const local = await engine.executeRaw<{ at: string }>(`SELECT unnest($1::timestamptz[])::text AS at`, [instants]);
            expect(local.map(r => r.at.slice(11))).toEqual(['01:20:00-04', '01:40:00-04', '01:10:00-05', '01:30:00-05']);
            for (const [i, at] of instants.entries()) {
              await engine.executeRaw('UPDATE pages SET compiled_truth=$1, updated_at=$2::timestamptz WHERE source_id=$3 AND slug=$4', [DETERMINISTIC, at, s.id, slugs[i]]);
            }
            await s.census({ batchSize: 1 });
            expect(await s.keys()).toEqual(slugs);
          } finally {
            await engine.executeRaw(`SELECT set_config('TimeZone', $1, false)`, [zone]);
          }
        } finally {
          if (clock) setSystemTime();
        }
      }, 60_000);
    }

    if (backend === 'postgres') {
      test('a writer transaction open during a pass is read after it commits, with no grace period', async () => {
        const s = await source(null);
        await s.stored('notes/late', CLEAN);
        await s.census({ lateCommitGraceMs: 0 });
        let release!: () => void;
        let updated!: () => void;
        const gate = new Promise<void>(resolve => { release = resolve; });
        const wrote = new Promise<void>(resolve => { updated = resolve; });
        const writer = second!.transaction(async tx => {
          await tx.executeRaw('UPDATE pages SET compiled_truth=$1, updated_at=now() WHERE source_id=$2 AND slug=$3', [MANUAL, s.id, 'notes/late']);
          updated();
          await gate;
        });
        await wrote;
        await new Promise(resolve => setTimeout(resolve, 50));
        await s.census({ lateCommitGraceMs: 0 });
        expect(await s.keys()).toEqual([]);
        release();
        await writer;
        await s.census({ lateCommitGraceMs: 0 });
        expect(await s.keys()).toEqual(['notes/late']);
      }, 60_000);
    }

    test('the backfill resumes across deadlines and finds damage past several of them', async () => {
      const s = await source(null);
      for (let i = 0; i < 30; i++) await s.stored(`notes/p-${String(i).padStart(2, '0')}`, CLEAN);
      await s.stored('notes/z-late', LLM);
      const clock = ticking();
      const progress: number[] = [];
      let partial = 0;
      for (let run = 0; run < 60; run++) {
        const [result] = await s.census({ now: clock.now, deadline: clock.at() + 6, batchSize: 4 });
        const record = (await readScanRecord(engine, s.id, s.incarnation))!;
        if (result!.complete) break;
        partial++;
        expect((await summarizeFenceCensus(engine, [s.id]))[0]!.scan.complete).toBe(false);
        progress.push(record.pages.backfill.done ? Number.MAX_SAFE_INTEGER : record.pages.backfill.after_id);
      }
      expect(partial).toBeGreaterThanOrEqual(3);
      // Never restarted: every partial run ended further along than the last.
      for (let i = 1; i < progress.length; i++) expect(progress[i]!).toBeGreaterThanOrEqual(progress[i - 1]!);
      expect(new Set(progress).size).toBeGreaterThanOrEqual(3);
      expect(await s.keys()).toEqual(['notes/z-late']);
    }, 120_000);

    test('the file walk resumes from its cursor across several deadlines and finds damage past them', async () => {
      const files: Record<string, string> = {};
      for (let i = 0; i < 24; i++) files[`notes/a-${String(i).padStart(2, '0')}.md`] = md(`A${i}`, CLEAN);
      files['notes/z-one.md'] = md('Z1', DETERMINISTIC);
      files['notes/z-two.md'] = md('Z2', MANUAL);
      const s = await source(files);
      const clock = ticking();
      const cursors: string[] = [];
      let partial = 0;
      for (let run = 0; run < 60; run++) {
        const [result] = await s.census({ now: clock.now, deadline: clock.at() + 12 });
        if (result!.complete) break;
        partial++;
        const pass = (await readScanRecord(engine, s.id, s.incarnation))!.files?.pass;
        if (pass?.after) cursors.push(pass.after);
        expect((await summarizeFenceCensus(engine, [s.id]))[0]!.scan).toMatchObject({ complete: false });
      }
      expect(partial).toBeGreaterThanOrEqual(3);
      expect(cursors.length).toBeGreaterThanOrEqual(3);
      for (let i = 1; i < cursors.length; i++) expect(cursors[i]! > cursors[i - 1]!).toBe(true);
      expect(await s.keys()).toEqual(['notes/z-one', 'notes/z-two']);
      expect((await readScanRecord(engine, s.id, s.incarnation))!.files).toMatchObject({ pass: null, walked: { commit: git(s.root, 'rev-parse', 'HEAD') } });
    }, 120_000);

    test('after the first walk, changed, untracked and deleted files are read again', async () => {
      const s = await source({ 'notes/one.md': md('One', CLEAN), 'notes/two.md': md('Two', CLEAN), 'notes/gone.md': md('Gone', DETERMINISTIC) });
      await s.census();
      expect(await s.keys()).toEqual(['notes/gone']);
      s.write('notes/one.md', md('One', RESOLVER));
      unlinkSync(join(s.root, 'notes/gone.md'));
      const head = s.commit();
      s.write('notes/fresh.md', md('Fresh', DETERMINISTIC));
      s.write('notes/two.md', md('Two', MANUAL));
      await s.census();
      expect(await s.keys()).toEqual(['notes/fresh', 'notes/one', 'notes/two']);
      expect((await readScanRecord(engine, s.id, s.incarnation))!.files!.walked!.commit).toBe(head);

      const plain = await source({ 'notes/old.md': md('Old', CLEAN) }, { git: false });
      await plain.census();
      expect(await plain.keys()).toEqual([]);
      plain.write('notes/new.md', md('New', LLM));
      await plain.census();
      expect(await plain.keys()).toEqual(['notes/new']);
    }, 60_000);

    test('a source with nothing to scan stores nothing, so an empty brain stays empty (graduation target check)', async () => {
      const s = await source(null);
      const runs = await s.census();
      expect(runs).toMatchObject([{ source_id: s.id, complete: true, files: 'none' }]);
      expect(await engine.executeRaw('SELECT 1 FROM op_checkpoints WHERE fingerprint LIKE $1', [`${s.id}:%`])).toEqual([]);
      expect((await summarizeFenceCensus(engine, [s.id], runs))[0]!.scan).toMatchObject({ complete: true });
      expect((await summarizeFenceCensus(engine, [s.id]))[0]!.scan).toMatchObject({ complete: false });
      // Once it holds a page, its cursors are kept so later runs scan incrementally.
      await s.stored('notes/one', CLEAN);
      await s.census();
      expect((await readScanRecord(engine, s.id, s.incarnation))!.census).toMatchObject({ complete: true });
    }, 60_000);

    test('census, candidate and attempt rows survive the 7-day purge while their source incarnation lives; trend rows age out', async () => {
      const s = await source(null);
      const dead = randomUUID();
      const insert = (op: string, fingerprint: string, incarnation: string) => engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at)
        VALUES($1,$2,$3::text::jsonb,now() - interval '10 days')`, [op, fingerprint, JSON.stringify([{ source_id: s.id, incarnation, day: '2026-01-01', count: 1 }])]);
      for (const op of [FENCE_CANDIDATE_OP, FENCE_SCAN_OP, FENCE_REPAIR_ATTEMPT_OP]) {
        await insert(op, `${s.id}:${s.incarnation}:live`, s.incarnation);
        await insert(op, `${s.id}:${dead}:dead`, dead);
      }
      await insert(FENCE_TREND_OP, `${s.id}:run:old`, s.incarnation);
      await purgeStaleCheckpoints(engine);
      const left = await engine.executeRaw<{ op: string; fingerprint: string }>('SELECT op, fingerprint FROM op_checkpoints WHERE fingerprint LIKE $1 ORDER BY op, fingerprint', [`${s.id}:%`]);
      expect(left).toEqual([FENCE_CANDIDATE_OP, FENCE_REPAIR_ATTEMPT_OP, FENCE_SCAN_OP].sort().map(op => ({ op, fingerprint: `${s.id}:${s.incarnation}:live` })));
    }, 60_000);

    test('the trend counts a resumed sync run and a replayed page write once, and skips per-file sync outcomes', async () => {
      const s = await source(null);
      await recordSyncRunTrend(engine, { sourceId: s.id, runId: 'run-1', day: '2026-10-05', count: 3, byClass: { close_fence: 3 }, writers: { 'notes/': 3 } });
      await recordSyncRunTrend(engine, { sourceId: s.id, runId: 'run-1', day: '2026-10-06', count: 5, byClass: { close_fence: 4, renumber: 1 }, writers: { 'notes/': 5 } });
      const row = { id: randomUUID(), source_id: s.id, principal_kind: 'local_cli' };
      const at = new Date('2026-10-06T10:00:00Z');
      await recordPublicationFenceTrend(engine, row, { fences_normalized: { count: 1, by_class: { renumber: 1 } } }, at);
      await recordPublicationFenceTrend(engine, row, { fences_normalized: { count: 1, by_class: { renumber: 1 } } }, at);
      await recordPublicationFenceTrend(engine, { ...row, id: randomUUID() }, { fences_normalized: [{ class: 'renumber' }] }, at);
      await recordPublicationFenceTrend(engine, { ...row, id: randomUUID() }, { status: 'imported' }, at);
      expect((await readTrend(engine, [s.id], '2026-09-30')).get(s.id)).toEqual([
        { day: '2026-10-05', count: 5, by_class: { close_fence: 4, renumber: 1 }, writers: { 'notes/': 5 } },
        { day: '2026-10-06', count: 1, by_class: { renumber: 1 }, writers: { local_cli: 1 } },
      ]);
      expect((await readTrend(engine, [s.id], '2026-10-06')).get(s.id)!.map(day => day.day)).toEqual(['2026-10-06']);
    }, 60_000);
  });
}