Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

test: pair refresh-barrier guards with race tests (expiry recheck, admission lock, publication re-auth, running drain)

garrytan98a2c29587d3
diff --git a/test/persistence-worktree-refresh.test.ts b/test/persistence-worktree-refresh.test.ts
index f00b636bd4..31718516aa 100644
--- a/test/persistence-worktree-refresh.test.ts
+++ b/test/persistence-worktree-refresh.test.ts
@@ -44,6 +44,7 @@ import { withCoordinatedWrite } from '../src/core/persistence/context.ts';
import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';
import { _resetCliExitVerdictForTests, currentExitCode } from '../src/core/cli-force-exit.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';
import { commitFile, git, page, withRefreshFixture, type RefreshFixture } from './helpers/worktree-refresh-fixture.ts';
@@ -1100,6 +1101,35 @@ test('13c. a committed write whose publisher still holds the worktree lock (reco
} finally { await clear(); }
}), 120_000);
+test('13e. a write the consumer claimed and is still preparing holds the drain until it finishes', () => each(async f => {
+ f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
+ const requestId = randomUUID();
+ let reached!: () => void; const preparing = new Promise<void>(resolve => { reached = resolve; });
+ let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
+ installFaultHook(async (point, detail) => { if (point === 'consumer:preparing' && detail.requestId === requestId) { reached(); await held; } });
+ const points: string[] = [];
+ let write: Promise<unknown> | undefined;
+ let refresh: Promise<{ status: string }> | undefined;
+ try {
+ write = f.put(f.alpha, 'notes/preparing', page('Preparing', 'claimed before the refresh started'), requestId);
+ await preparing;
+ const [claimed] = await f.engine.executeRaw<{ state: string }>('SELECT state FROM persistence_requests WHERE request_id=$1::uuid', [requestId]);
+ expect(claimed!.state).toBe('running');
+ refresh = refreshWorktree(f.engine, f.alpha, { hooks: { boundary: async point => { points.push(point); } } });
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ expect(points).toEqual([]);
+ release();
+ await write;
+ expect((await refresh).status).toBe('completed');
+ expect(points[0]).toBe('drained');
+ } finally {
+ release();
+ installFaultHook(undefined);
+ await write?.catch(() => undefined);
+ await refresh?.catch(() => undefined);
+ }
+}), 120_000);
+
test('13d. a publisher that clears its record and releases the lock between the precheck probe and its lock attempt is not refused', () => each(async f => {
// TOCTOU on 13c's fix: the precheck sees the record, then the publisher finishes before the lock probe.
const target = f.push('alpha/two.md', page('Alpha two', 'Upstream.'));
diff --git a/test/refresh-write-barrier.test.ts b/test/refresh-write-barrier.test.ts
index 301fc1a520..7b37e0dd9a 100644
--- a/test/refresh-write-barrier.test.ts
+++ b/test/refresh-write-barrier.test.ts
@@ -11,6 +11,7 @@
* Seams: the real refresh state machine paused at its drained boundary (`hooks.boundary`); GBRAIN_REFRESH_BARRIER.
*/
import { afterAll, afterEach, beforeAll, beforeEach, expect, spyOn, test } from 'bun:test';
+import postgres from '#postgres';
import { randomUUID } from 'node:crypto';
import { readFileSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
@@ -34,6 +35,7 @@ import { _setRefreshFenceWaitForTests } from '../src/core/persistence/admission-
import { _resetRefreshBarrierCacheForTests, deferredBacklog, expireDeferredWrites } from '../src/core/persistence/refresh-barrier.ts';
import type { WriteRequest } from '../src/core/persistence/model.ts';
import { quarantineSharedSkillRestore } from '../src/core/shared-skills/restore.ts';
+import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';
@@ -42,6 +44,7 @@ import { git, page, withRefreshFixture, type RefreshFixture } from './helpers/wo
const backends = testBackends();
const engines: BrainEngine[] = [];
let closePostgres: (() => Promise<void>) | undefined;
+let postgresUrl: string | undefined;
beforeAll(async () => {
if (backends.includes('pglite')) {
@@ -50,7 +53,7 @@ beforeAll(async () => {
}
if (backends.includes('postgres')) {
const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
- engines.push(pg.engine); closePostgres = pg.close;
+ engines.push(pg.engine); closePostgres = pg.close; postgresUrl = pg.databaseUrl;
}
}, 120_000);
afterAll(async () => {
@@ -93,6 +96,32 @@ async function settle(f: RefreshFixture, row: WriteRequest): Promise<WriteReques
return waitForWrite(f.engine, (await getWriteRequestById(f.engine, row.id))!, { engine: f.engine.kind } as never, 30_000);
}
+/**
+ * Postgres only: on a separate connection, moves refresh `refreshId` to aborted and keeps that transaction open while `start`
+ * runs, until `start` waits on that connection's lock (or settles), then commits and returns what `start` returns.
+ */
+async function endRefreshDuring<T>(f: RefreshFixture, refreshId: string, start: () => Promise<T>): Promise<T> {
+ const sql = postgres(postgresUrl!, { max: 1, prepare: false });
+ try {
+ const conn = await sql.reserve();
+ try {
+ await conn.unsafe('BEGIN');
+ const [{ pid }] = await conn.unsafe<Array<{ pid: number }>>('SELECT pg_backend_pid() AS pid');
+ await conn.unsafe("UPDATE persistence_worktree_refreshes SET state='aborted',completed_at=now(),updated_at=now() WHERE id=$1::uuid", [refreshId]);
+ let settled = false;
+ const running = start().finally(() => { settled = true; });
+ running.catch(() => undefined);
+ for (let i = 0; i < 800 && !settled; i++) {
+ const [waiting] = await f.engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pg_stat_activity WHERE $1::int = ANY(pg_blocking_pids(pid))', [pid]);
+ if (Number(waiting!.n) > 0) break;
+ await new Promise(resolve => setTimeout(resolve, 25));
+ }
+ await conn.unsafe('COMMIT');
+ return await running;
+ } finally { conn.release(); }
+ } finally { await sql.end(); }
+}
+
async function outstanding(f: RefreshFixture): Promise<number> {
const [row] = await f.engine.executeRaw<{ n: number }>("SELECT outstanding_count::int AS n FROM persistence_counters WHERE key='brain'");
return Number(row?.n ?? 0);
@@ -199,6 +228,41 @@ test('a deferred write past the wait limit ends refresh_barrier_expired and rele
expect((await refresh).status).toBe('completed');
}), 180_000);
+test('an expiry racing the end of its refresh leaves the write deferred, and it commits after the release (Postgres)', () => each(async f => {
+ if (f.engine.kind !== 'postgres') return;
+ let deferred!: WriteRequest;
+ await duringRefresh(f, async () => {
+ const out = await call(f, 'put_page', { slug: 'notes/expiry-race', content: page('Expiry race', 'body') });
+ deferred = await requestRow(f, out.body.request_id);
+ expect(deferred.state).toBe('deferred');
+ await disposePersistenceConsumer(f.engine);
+ await f.engine.executeRaw("INSERT INTO config(key,value) VALUES('persistence.refresh_barrier_max_wait_ms','1') ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value");
+ try {
+ await new Promise(resolve => setTimeout(resolve, 20));
+ const [{ owner }] = await f.engine.executeRaw<{ owner: string }>('SELECT owner_host_id::text AS owner FROM persistence_worktrees WHERE id=$1::uuid', [f.worktreeId]);
+ const expired = await endRefreshDuring(f, deferred.deferred_refresh_id!, () => expireDeferredWrites(f.engine, owner));
+ expect(expired.map(row => row.id)).toEqual([]);
+ expect((await getWriteRequestById(f.engine, deferred.id))!.state).toBe('deferred');
+ } finally {
+ await f.engine.executeRaw("DELETE FROM config WHERE key='persistence.refresh_barrier_max_wait_ms'");
+ }
+ }).catch(error => { if (!(error instanceof OperationError)) throw error; });
+ expect((await settle(f, deferred)).state).toBe('committed');
+}), 180_000);
+
+test('a write admitted while its refresh ends is never deferred to the released refresh (Postgres)', () => each(async f => {
+ if (f.engine.kind !== 'postgres') return;
+ let row!: WriteRequest;
+ await duringRefresh(f, async () => {
+ const active = (await f.refreshRows()).find(r => r.state === 'draining')!;
+ const out = await endRefreshDuring(f, active.id, () => call(f, 'put_page', { slug: 'notes/admission-race', content: page('Admission race', 'body') }));
+ row = await requestRow(f, out.body.request_id);
+ expect(row.deferred_refresh_id).toBeNull();
+ expect(row.state).not.toBe('deferred');
+ }).catch(error => { if (!(error instanceof OperationError)) throw error; });
+ expect((await settle(f, row)).state).toBe('committed');
+}), 180_000);
+
test('a shared-skill restore cancels a deferred write too and releases its counters (restore is PGLite-only)', () => each(async f => {
if (f.engine.kind !== 'pglite') return;
let deferred!: WriteRequest;
@@ -354,6 +418,24 @@ test('a deferred write whose token is revoked while it waits ends permission_den
expect(Number(n)).toBe(0);
}), 180_000);
+test('a queued write whose token is revoked after it was claimed ends permission_denied with nothing applied', () => each(async f => {
+ const minted = await mintLegacyToken(f.engine, { name: `token-${randomUUID()}`, scopes: ['read', 'write'], sourceGrant: [f.alpha], takesHolders: ['world'] });
+ const requestId = randomUUID();
+ installFaultHook(async (point, detail) => {
+ if (point === 'consumer:preparing' && detail.requestId === requestId) await f.engine.executeRaw('UPDATE access_tokens SET revoked_at=now() WHERE id=$1', [minted.id]);
+ });
+ try {
+ await dispatchToolCall(f.engine, 'put_page', { slug: 'notes/revoked-queued', content: page('Revoked queued', 'never applied'), request_id: requestId }, { remote: true, transport: 'http',
+ sourceId: f.alpha, admissionOrigin: 'interactive', takesHoldersAllowList: ['world'],
+ auth: { token: '', clientId: minted.id, principal: { kind: 'legacy_token', id: minted.id }, sourceId: f.alpha, allowedSources: [f.alpha], scopes: ['read', 'write'] } } as never);
+ } finally { installFaultHook(undefined); }
+ const row = await requestRow(f, requestId);
+ expect(row.deferred_refresh_id).toBeNull();
+ expect(await settle(f, row)).toMatchObject({ state: 'failed', error_code: 'permission_denied' });
+ const [{ n }] = await f.engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1 AND slug=$2', [f.alpha, 'notes/revoked-queued']);
+ expect(Number(n)).toBe(0);
+}), 180_000);
+
test('a write admitted after the release waits behind the released deferred write on its worktree', () => each(async f => {
let first!: WriteRequest;
const refresh = await duringRefresh(f, async () => {