Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
test: pair refresh-barrier guards with race tests (expiry recheck, admission lock, publication re-auth, running drain)
garrytan98a2c29587d3
diff --git a/test/persistence-worktree-refresh.test.ts b/test/persistence-worktree-refresh.test.tsindex f00b636bd4..31718516aa 100644--- a/test/persistence-worktree-refresh.test.ts+++ b/test/persistence-worktree-refresh.test.ts@@ -44,6 +44,7 @@ import { withCoordinatedWrite } from '../src/core/persistence/context.ts';import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';import { _resetCliExitVerdictForTests, currentExitCode } from '../src/core/cli-force-exit.ts';import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';+import { installFaultHook } from '../src/core/persistence/fault-points.ts';import { testBackends } from './helpers/test-backends.ts';import { withEnv } from './helpers/with-env.ts';import { commitFile, git, page, withRefreshFixture, type RefreshFixture } from './helpers/worktree-refresh-fixture.ts';@@ -1100,6 +1101,35 @@ test('13c. a committed write whose publisher still holds the worktree lock (reco} finally { await clear(); }}), 120_000);+test('13e. a write the consumer claimed and is still preparing holds the drain until it finishes', () => each(async f => {+ f.push('alpha/two.md', page('Alpha two', 'Upstream.'));+ const requestId = randomUUID();+ let reached!: () => void; const preparing = new Promise<void>(resolve => { reached = resolve; });+ let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });+ installFaultHook(async (point, detail) => { if (point === 'consumer:preparing' && detail.requestId === requestId) { reached(); await held; } });+ const points: string[] = [];+ let write: Promise<unknown> | undefined;+ let refresh: Promise<{ status: string }> | undefined;+ try {+ write = f.put(f.alpha, 'notes/preparing', page('Preparing', 'claimed before the refresh started'), requestId);+ await preparing;+ const [claimed] = await f.engine.executeRaw<{ state: string }>('SELECT state FROM persistence_requests WHERE request_id=$1::uuid', [requestId]);+ expect(claimed!.state).toBe('running');+ refresh = refreshWorktree(f.engine, f.alpha, { hooks: { boundary: async point => { points.push(point); } } });+ await new Promise(resolve => setTimeout(resolve, 1000));+ expect(points).toEqual([]);+ release();+ await write;+ expect((await refresh).status).toBe('completed');+ expect(points[0]).toBe('drained');+ } finally {+ release();+ installFaultHook(undefined);+ await write?.catch(() => undefined);+ await refresh?.catch(() => undefined);+ }+}), 120_000);+test('13d. a publisher that clears its record and releases the lock between the precheck probe and its lock attempt is not refused', () => each(async f => {// TOCTOU on 13c's fix: the precheck sees the record, then the publisher finishes before the lock probe.const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));diff --git a/test/refresh-write-barrier.test.ts b/test/refresh-write-barrier.test.tsindex 301fc1a520..7b37e0dd9a 100644--- a/test/refresh-write-barrier.test.ts+++ b/test/refresh-write-barrier.test.ts@@ -11,6 +11,7 @@* Seams: the real refresh state machine paused at its drained boundary (`hooks.boundary`); GBRAIN_REFRESH_BARRIER.*/import { afterAll, afterEach, beforeAll, beforeEach, expect, spyOn, test } from 'bun:test';+import postgres from '#postgres';import { randomUUID } from 'node:crypto';import { readFileSync, rmSync, writeFileSync } from 'node:fs';import { join } from 'node:path';@@ -34,6 +35,7 @@ import { _setRefreshFenceWaitForTests } from '../src/core/persistence/admission-import { _resetRefreshBarrierCacheForTests, deferredBacklog, expireDeferredWrites } from '../src/core/persistence/refresh-barrier.ts';import type { WriteRequest } from '../src/core/persistence/model.ts';import { quarantineSharedSkillRestore } from '../src/core/shared-skills/restore.ts';+import { installFaultHook } from '../src/core/persistence/fault-points.ts';import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';import { testBackends } from './helpers/test-backends.ts';import { withEnv } from './helpers/with-env.ts';@@ -42,6 +44,7 @@ import { git, page, withRefreshFixture, type RefreshFixture } from './helpers/woconst backends = testBackends();const engines: BrainEngine[] = [];let closePostgres: (() => Promise<void>) | undefined;+let postgresUrl: string | undefined;beforeAll(async () => {if (backends.includes('pglite')) {@@ -50,7 +53,7 @@ beforeAll(async () => {}if (backends.includes('postgres')) {const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);- engines.push(pg.engine); closePostgres = pg.close;+ engines.push(pg.engine); closePostgres = pg.close; postgresUrl = pg.databaseUrl;}}, 120_000);afterAll(async () => {@@ -93,6 +96,32 @@ async function settle(f: RefreshFixture, row: WriteRequest): Promise<WriteRequesreturn waitForWrite(f.engine, (await getWriteRequestById(f.engine, row.id))!, { engine: f.engine.kind } as never, 30_000);}+/**+ * Postgres only: on a separate connection, moves refresh `refreshId` to aborted and keeps that transaction open while `start`+ * runs, until `start` waits on that connection's lock (or settles), then commits and returns what `start` returns.+ */+async function endRefreshDuring<T>(f: RefreshFixture, refreshId: string, start: () => Promise<T>): Promise<T> {+ const sql = postgres(postgresUrl!, { max: 1, prepare: false });+ try {+ const conn = await sql.reserve();+ try {+ await conn.unsafe('BEGIN');+ const [{ pid }] = await conn.unsafe<Array<{ pid: number }>>('SELECT pg_backend_pid() AS pid');+ await conn.unsafe("UPDATE persistence_worktree_refreshes SET state='aborted',completed_at=now(),updated_at=now() WHERE id=$1::uuid", [refreshId]);+ let settled = false;+ const running = start().finally(() => { settled = true; });+ running.catch(() => undefined);+ for (let i = 0; i < 800 && !settled; i++) {+ const [waiting] = await f.engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pg_stat_activity WHERE $1::int = ANY(pg_blocking_pids(pid))', [pid]);+ if (Number(waiting!.n) > 0) break;+ await new Promise(resolve => setTimeout(resolve, 25));+ }+ await conn.unsafe('COMMIT');+ return await running;+ } finally { conn.release(); }+ } finally { await sql.end(); }+}+async function outstanding(f: RefreshFixture): Promise<number> {const [row] = await f.engine.executeRaw<{ n: number }>("SELECT outstanding_count::int AS n FROM persistence_counters WHERE key='brain'");return Number(row?.n ?? 0);@@ -199,6 +228,41 @@ test('a deferred write past the wait limit ends refresh_barrier_expired and releexpect((await refresh).status).toBe('completed');}), 180_000);+test('an expiry racing the end of its refresh leaves the write deferred, and it commits after the release (Postgres)', () => each(async f => {+ if (f.engine.kind !== 'postgres') return;+ let deferred!: WriteRequest;+ await duringRefresh(f, async () => {+ const out = await call(f, 'put_page', { slug: 'notes/expiry-race', content: page('Expiry race', 'body') });+ deferred = await requestRow(f, out.body.request_id);+ expect(deferred.state).toBe('deferred');+ await disposePersistenceConsumer(f.engine);+ await f.engine.executeRaw("INSERT INTO config(key,value) VALUES('persistence.refresh_barrier_max_wait_ms','1') ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value");+ try {+ await new Promise(resolve => setTimeout(resolve, 20));+ const [{ owner }] = await f.engine.executeRaw<{ owner: string }>('SELECT owner_host_id::text AS owner FROM persistence_worktrees WHERE id=$1::uuid', [f.worktreeId]);+ const expired = await endRefreshDuring(f, deferred.deferred_refresh_id!, () => expireDeferredWrites(f.engine, owner));+ expect(expired.map(row => row.id)).toEqual([]);+ expect((await getWriteRequestById(f.engine, deferred.id))!.state).toBe('deferred');+ } finally {+ await f.engine.executeRaw("DELETE FROM config WHERE key='persistence.refresh_barrier_max_wait_ms'");+ }+ }).catch(error => { if (!(error instanceof OperationError)) throw error; });+ expect((await settle(f, deferred)).state).toBe('committed');+}), 180_000);++test('a write admitted while its refresh ends is never deferred to the released refresh (Postgres)', () => each(async f => {+ if (f.engine.kind !== 'postgres') return;+ let row!: WriteRequest;+ await duringRefresh(f, async () => {+ const active = (await f.refreshRows()).find(r => r.state === 'draining')!;+ const out = await endRefreshDuring(f, active.id, () => call(f, 'put_page', { slug: 'notes/admission-race', content: page('Admission race', 'body') }));+ row = await requestRow(f, out.body.request_id);+ expect(row.deferred_refresh_id).toBeNull();+ expect(row.state).not.toBe('deferred');+ }).catch(error => { if (!(error instanceof OperationError)) throw error; });+ expect((await settle(f, row)).state).toBe('committed');+}), 180_000);+test('a shared-skill restore cancels a deferred write too and releases its counters (restore is PGLite-only)', () => each(async f => {if (f.engine.kind !== 'pglite') return;let deferred!: WriteRequest;@@ -354,6 +418,24 @@ test('a deferred write whose token is revoked while it waits ends permission_denexpect(Number(n)).toBe(0);}), 180_000);+test('a queued write whose token is revoked after it was claimed ends permission_denied with nothing applied', () => each(async f => {+ const minted = await mintLegacyToken(f.engine, { name: `token-${randomUUID()}`, scopes: ['read', 'write'], sourceGrant: [f.alpha], takesHolders: ['world'] });+ const requestId = randomUUID();+ installFaultHook(async (point, detail) => {+ if (point === 'consumer:preparing' && detail.requestId === requestId) await f.engine.executeRaw('UPDATE access_tokens SET revoked_at=now() WHERE id=$1', [minted.id]);+ });+ try {+ await dispatchToolCall(f.engine, 'put_page', { slug: 'notes/revoked-queued', content: page('Revoked queued', 'never applied'), request_id: requestId }, { remote: true, transport: 'http',+ sourceId: f.alpha, admissionOrigin: 'interactive', takesHoldersAllowList: ['world'],+ auth: { token: '', clientId: minted.id, principal: { kind: 'legacy_token', id: minted.id }, sourceId: f.alpha, allowedSources: [f.alpha], scopes: ['read', 'write'] } } as never);+ } finally { installFaultHook(undefined); }+ const row = await requestRow(f, requestId);+ expect(row.deferred_refresh_id).toBeNull();+ expect(await settle(f, row)).toMatchObject({ state: 'failed', error_code: 'permission_denied' });+ const [{ n }] = await f.engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1 AND slug=$2', [f.alpha, 'notes/revoked-queued']);+ expect(Number(n)).toBe(0);+}), 180_000);+test('a write admitted after the release waits behind the released deferred write on its worktree', () => each(async f => {let first!: WriteRequest;const refresh = await duringRefresh(f, async () => {