Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Tests: what master does with a killed or orphaned write at the head of a source queue, and a classic receipt git_state after its commit lands (GST-2572)

garrytan2d8ab3d4d6b4
diff --git a/test/fixtures/stuck-head-worker.ts b/test/fixtures/stuck-head-worker.ts
new file mode 100644
index 0000000000..d7814e94ab
--- /dev/null
+++ b/test/fixtures/stuck-head-worker.ts
@@ -0,0 +1,54 @@
+/**
+ * Child process for test/persistence-stuck-head-9227.test.ts: one `put_page` that stops at a named fault seam and
+ * waits there, so the parent can SIGKILL the process at that point and read what the death leaves behind.
+ * Argument: a JSON config file. Prints one JSON line per event on stdout: `{event:'boundary', point}` when it reaches
+ * the seam for its own request, then nothing more (the parent kills it); without a seam, `{event:'done', state, ms}` or
+ * `{event:'failure', code, message}`.
+ * GBRAIN_HOME is the parent's, so both processes are the same host.
+ */
+import { readFileSync } from 'node:fs';
+import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
+import { PostgresEngine } from '../../src/core/postgres-engine.ts';
+import { operations } from '../../src/core/operations.ts';
+import { installFaultHook, type FaultPoint } from '../../src/core/persistence/fault-points.ts';
+
+export interface StuckHeadWorkerConfig {
+ engine: 'pglite' | 'postgres';
+ databasePath?: string;
+ databaseUrl?: string;
+ sourceId: string;
+ slug: string;
+ content: string;
+ requestId: string;
+ /** Absent: the write runs to its end and the worker reports its state and how long it took. */
+ point?: FaultPoint;
+ /** The reply deadline (default 30000, the most a write may wait). */
+ waitMs?: number;
+ /** A remote OAuth caller instead of the local CLI writer. */
+ auth?: Record<string, unknown>;
+}
+
+const emit = (event: Record<string, unknown>) => process.stdout.write(`${JSON.stringify(event)}\n`);
+
+if (import.meta.main) {
+ const config: StuckHeadWorkerConfig = JSON.parse(readFileSync(process.argv[2]!, 'utf8'));
+ const engine = config.engine === 'pglite' ? new PGLiteEngine() : new PostgresEngine();
+ try {
+ await engine.connect(config.engine === 'pglite' ? { engine: 'pglite', database_path: config.databasePath } : { database_url: config.databaseUrl, poolSize: 4 });
+ if (config.point) installFaultHook(async (point, detail) => {
+ if (point !== config.point || detail.requestId !== config.requestId) return;
+ emit({ event: 'boundary', point });
+ await new Promise(() => {});
+ });
+ const put = operations.find(op => op.name === 'put_page')!;
+ const started = performance.now();
+ const out = await put.handler({ engine, config: { engine: config.engine, embedding_disabled: true } as never, logger: { info() {}, warn() {}, error() {} },
+ dryRun: false, remote: config.auth !== undefined, sourceId: config.sourceId, ...(config.auth ? { auth: config.auth as never } : {}) },
+ { slug: config.slug, content: config.content, request_id: config.requestId, wait_ms: config.waitMs ?? 30_000 });
+ emit({ event: 'done', state: (out as { state?: string }).state ?? null, ms: Math.round(performance.now() - started) });
+ process.exit(0);
+ } catch (error) {
+ emit({ event: 'failure', code: (error as { code?: string }).code ?? null, message: error instanceof Error ? error.message : String(error) });
+ process.exit(1);
+ }
+}
diff --git a/test/persistence-receipt-git-state.test.ts b/test/persistence-receipt-git-state.test.ts
new file mode 100644
index 0000000000..7d1378f8c6
--- /dev/null
+++ b/test/persistence-receipt-git-state.test.ts
@@ -0,0 +1,87 @@
+/**
+ * A classic (not `canonical: git`) worktree-backed page write: what its receipt says about Git once the Git effect
+ * has committed the file. Pins master's answer: the receipt's `persistence.git_state` is the value its publication
+ * transaction stored (`queued`, effect-journal.ts queuePublicationEffects) and never moves, because the Git effect's
+ * completion (completeEffect) writes only its own effect row, whose outcome is `{ git: 'committed' }` without the commit
+ * oid; the request's outcome is not touched after the write commits. The live state is the receipt's `effects` entry
+ * for `git`. No commit oid is recorded anywhere for such a write. PGLite and, with DATABASE_URL, Postgres.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { randomUUID } from 'node:crypto';
+import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { operations, type OperationContext } from '../src/core/operations.ts';
+import { runPersistenceEffects } from '../src/core/persistence/effects.ts';
+import { localHostId } from '../src/core/persistence/identity.ts';
+import { claimWorktree } from '../src/core/persistence/ownership.ts';
+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+import { testBackends } from './helpers/test-backends.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const op = (name: string) => operations.find(entry => entry.name === name)!;
+const git = (cwd: string, ...args: string[]) => execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+
+for (const backend of testBackends()) describe(`[${backend}] a classic source's receipt after its Git commit lands`, () => {
+ let engine: BrainEngine;
+ let closePostgres: (() => Promise<void>) | undefined;
+ const home = realpathSync(mkdtempSync(join(tmpdir(), `gbrain-receipt-git-${backend}-`)));
+ const repo = join(home, 'repo');
+ const env = { GBRAIN_HOME: home };
+ const ctx = (): OperationContext => ({ engine, config: { engine: backend, embedding_disabled: true } as never,
+ logger: { info() {}, warn() {}, error() {} }, dryRun: false, remote: false, sourceId: 'repo-source' });
+
+ beforeAll(async () => withEnv(env, async () => {
+ if (backend === 'pglite') { engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); }
+ else { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!); engine = pg.engine; closePostgres = pg.close; }
+ mkdirSync(repo);
+ git(repo, 'init', '-q', '-b', 'main');
+ git(repo, 'config', 'user.name', 'Example');
+ git(repo, 'config', 'user.email', '[email protected]');
+ writeFileSync(join(repo, 'seed.md'), 'seed\n');
+ git(repo, 'add', 'seed.md');
+ git(repo, 'commit', '-qm', 'seed');
+ // The durability hook banner is what marks the checkout hardened, so the Git effect commits (no push: no remote).
+ mkdirSync(join(repo, '.git', 'hooks'), { recursive: true });
+ writeFileSync(join(repo, '.git', 'hooks', 'post-commit'), '#!/usr/bin/env bash\n# gbrain brain-durability post-commit hook (v0.42.44+)\nexit 0\n');
+ chmodSync(join(repo, '.git', 'hooks', 'post-commit'), 0o755);
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', ['repo-source', repo]);
+ await claimWorktree(engine, 'repo-source', repo);
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
+ }), 120_000);
+ afterAll(async () => {
+ await disposePersistenceConsumer(engine);
+ if (closePostgres) await closePostgres(); else await engine.disconnect();
+ rmSync(home, { recursive: true, force: true });
+ }, 60_000);
+
+ test('git_state stays queued with no commit oid after the commit lands; only effects[git] reads committed', async () => withEnv(env, async () => {
+ const requestId = randomUUID();
+ const written = await op('put_page').handler(ctx(), { slug: 'notes/landed', request_id: requestId, wait_ms: 20_000,
+ content: '---\ntitle: Landed\ntype: note\n---\n# Landed\n\nA page whose commit lands later.\n' }) as Record<string, any>;
+ expect(written).toMatchObject({ state: 'committed', persistence: { git_state: 'queued' } });
+ const before = git(repo, 'rev-parse', 'HEAD');
+ await disposePersistenceConsumer(engine);
+ await runPersistenceEffects(engine, { engine: backend, embedding_disabled: true }, { hostId: localHostId(), limit: 8 });
+ const head = git(repo, 'rev-parse', 'HEAD');
+ expect(head).not.toBe(before);
+ expect(git(repo, 'log', '-1', '--format=%H', '--', 'notes/landed.md')).toBe(head);
+ const [effect] = await engine.executeRaw<{ state: string; outcome: Record<string, unknown> }>(
+ `SELECT e.state,e.outcome FROM persistence_effects e JOIN persistence_requests r ON r.id=e.request_id WHERE r.request_id=$1::uuid AND e.kind='git'`, [requestId]);
+ expect(effect!.state).toBe('committed');
+ expect(effect!.outcome).toMatchObject({ git: 'committed' });
+ expect(JSON.stringify(effect!.outcome)).not.toContain(head);
+ const receipt = await op('get_write_request').handler(ctx(), { request_id: requestId }) as Record<string, any>;
+ expect(receipt.persistence).toEqual({ mode: 'filesystem', file_written: true, git_state: 'queued' });
+ expect(receipt.persistence.git_commit).toBeUndefined();
+ expect(receipt.effects).toContainEqual(expect.objectContaining({ kind: 'git', state: 'committed' }));
+ const [stored] = await engine.executeRaw<{ outcome: Record<string, any> }>('SELECT outcome FROM persistence_requests WHERE request_id=$1::uuid', [requestId]);
+ expect(stored!.outcome.persistence).toEqual({ mode: 'filesystem', file_written: true, git_state: 'queued' });
+ expect(JSON.stringify(stored!.outcome)).not.toContain(head);
+ }), 120_000);
+});
diff --git a/test/persistence-stuck-head-9227.test.ts b/test/persistence-stuck-head-9227.test.ts
new file mode 100644
index 0000000000..eb9b1beaa0
--- /dev/null
+++ b/test/persistence-stuck-head-9227.test.ts
@@ -0,0 +1,193 @@
+/**
+ * gbrain#9227: a write killed at the head of a source's queue, or one whose principal is gone, must not wedge the
+ * source. Each case kills a real child process (test/fixtures/stuck-head-worker.ts) with SIGKILL at a fault seam and
+ * reads what master does with the head it left, on PGLite and, with DATABASE_URL, on Postgres:
+ *
+ * 1. Killed while preparing: the claim is `running`/`preparing` with a 30 s lease and no recovery record.
+ * 2. Killed inside the publication transaction of a database-only page: the transaction rolls back and the claim keeps
+ * the publication lease it stamped (`publicationLeaseMs`, 365 s by default) under a dead owner.
+ * 3. Killed after the page file was written (a worktree-backed source): the recovery record survives, the file holds the
+ * new bytes and the database rolled back.
+ * 4. The submitting OAuth client is revoked while its write is stuck at the head: no other principal can see or
+ * cancel it (control.ts cancelWriteRequest is scoped to the principal).
+ *
+ * Findings this file pins: on Postgres a later write's own one-shot consumer proves a same-host owner dead (#6650) and
+ * reclaims cases 1, 2 and 4 at once; case 3 is recovered at once on both engines (the recovery scan); a revoked
+ * principal's write ends `failed`/`permission_denied` when it is next prepared. Two cases wait out the lease: an owner
+ * Postgres cannot prove dead (another boot or pid namespace), and every database-only head on PGLite, where the idle
+ * probe never counts a claim left by an earlier process, so cases 1, 2 and 4 hold their source for the rest of the
+ * lease (30 s or 365 s). Meanwhile later writes are accepted and wait (`write_pending`), and `gbrain sources writer
+ * status` reads the dead claim as a live one (`claim_running`, `resumes_on_its_own: false`). A lapsed lease is
+ * modelled by moving `claim_expires_at` into the past, the clock the reclaim reads.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { randomUUID } from 'node:crypto';
+import { execFileSync, spawn } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
+import { runPersistenceAdministration } from '../src/core/persistence/administration.ts';
+import { claimWorktree } from '../src/core/persistence/ownership.ts';
+import { cancelWriteRequest } from '../src/core/persistence/control.ts';
+import { readLocalWriter } from '../src/core/persistence/identity.ts';
+import type { FaultPoint } from '../src/core/persistence/fault-points.ts';
+import type { StuckHeadWorkerConfig } from './fixtures/stuck-head-worker.ts';
+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+import { testBackends } from './helpers/test-backends.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const page = (title: string) => `---\ntitle: ${title}\ntype: note\n---\n# ${title}\n\nBody of ${title}.\n`;
+const FOREIGN_BOOT = '00000000-0000-0000-0000-000000000000';
+type Head = { state: string; error_code: string | null; preparation_attempts: number; recovery: boolean; phase: string | null; token: string | null; lease_s: number | null };
+type WorkerEvent = { event: 'boundary' | 'done' | 'failure'; state?: string; code?: string; ms?: number; message?: string };
+
+for (const backend of testBackends()) describe(`[${backend}] a stuck head of a source's write queue (#9227)`, () => {
+ let engine: BrainEngine;
+ let reopen: () => Promise<void> = async () => {};
+ let databaseUrl = '';
+ let closePostgres: (() => Promise<void>) | undefined;
+ const home = realpathSync(mkdtempSync(join(tmpdir(), `gbrain-stuck-head-${backend}-`)));
+ const dataDir = join(home, 'data');
+ const repo = join(home, 'repo');
+ const env = { GBRAIN_HOME: home };
+ const postgres = backend === 'postgres';
+
+ beforeAll(async () => withEnv(env, async () => {
+ if (!postgres) {
+ reopen = async () => { const opened = new PGLiteEngine(); await opened.connect({ engine: 'pglite', database_path: dataDir }); engine = opened; };
+ await reopen();
+ await engine.initSchema();
+ } else {
+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
+ engine = pg.engine; databaseUrl = pg.databaseUrl; closePostgres = pg.close;
+ }
+ mkdirSync(repo);
+ execFileSync('git', ['-C', repo, 'init', '-q']);
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', ['repo-source', repo]);
+ await claimWorktree(engine, 'repo-source', repo);
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
+ await engine.executeRaw(`INSERT INTO oauth_clients(client_id,client_name,scope,source_id) VALUES('client-a','Example client','read write','default')`);
+ }), 120_000);
+ afterAll(async () => {
+ await disposePersistenceConsumer(engine);
+ if (closePostgres) await closePostgres(); else await engine.disconnect();
+ rmSync(home, { recursive: true, force: true });
+ }, 60_000);
+
+ /** Runs one put_page in a child process on this brain and host; with a seam, SIGKILLs it there. PGLite is closed meanwhile (one process at a time). */
+ async function worker(config: Partial<StuckHeadWorkerConfig> & { slug: string }): Promise<WorkerEvent> {
+ await disposePersistenceConsumer(engine);
+ if (!postgres) await engine.disconnect();
+ const requestId = config.requestId ?? randomUUID();
+ const file = join(home, `worker-${requestId}.json`);
+ writeFileSync(file, JSON.stringify({ engine: backend, databasePath: dataDir, databaseUrl, sourceId: 'default', content: page(config.slug), requestId, ...config }));
+ const child = spawn('bun', [join(import.meta.dir, 'fixtures/stuck-head-worker.ts'), file], { env: { ...process.env, HOME: home, GBRAIN_HOME: home }, stdio: ['ignore', 'pipe', 'pipe'] });
+ let out = '', err = '';
+ child.stderr.on('data', chunk => { err += chunk; });
+ const exited = new Promise<void>(resolve => child.once('exit', () => resolve()));
+ try {
+ const event = await new Promise<WorkerEvent>((resolve, reject) => {
+ child.stdout.on('data', chunk => { out += chunk; const line = out.split('\n').find(text => text.startsWith('{')); if (line) resolve(JSON.parse(line)); });
+ child.once('exit', () => setTimeout(() => reject(new Error(`worker exited without an event: ${out}${err.slice(-2000)}`)), 100));
+ });
+ if (event.event === 'boundary') child.kill('SIGKILL');
+ await exited;
+ return event;
+ } finally {
+ if (!postgres) await withEnv(env, reopen);
+ }
+ }
+ const killAt = (point: FaultPoint, config: Partial<StuckHeadWorkerConfig> & { slug: string; requestId: string }) =>
+ worker({ ...config, point }).then(event => expect(event).toEqual({ event: 'boundary', point }));
+ const head = async (requestId: string): Promise<Head> => (await engine.executeRaw<Head>(`SELECT state,error_code,preparation_attempts,recovery IS NOT NULL AS recovery,
+ claim_phase->>'phase' AS phase,execution_token::text AS token,round(EXTRACT(EPOCH FROM claim_expires_at-updated_at))::int AS lease_s
+ FROM persistence_requests WHERE request_id=$1::uuid`, [requestId]))[0]!;
+ const blocker = async (source: string, requestId: string) => {
+ const status = await withEnv(env, () => runPersistenceAdministration(engine, 'writer_status', { source_id: source })) as { blockers: Array<Record<string, any>> };
+ return status.blockers.find(row => row.request_id === requestId)!;
+ };
+ const lapse = (requestId: string) => engine.executeRaw(`UPDATE persistence_requests SET claim_expires_at=now()-interval '1 second' WHERE request_id=$1::uuid AND state='running'`, [requestId]);
+
+ /**
+ * The head after the kill, then what a later write to the same source from a new process sees. Postgres: it proves the
+ * owner dead and the head and the write both commit at once. PGLite: the write waits behind the head until its lease lapses.
+ */
+ async function laterWrites(source: string, stuck: string, slug: string): Promise<Head> {
+ if (postgres) {
+ const later = await worker({ slug, sourceId: source });
+ expect(later).toMatchObject({ event: 'done', state: 'committed' });
+ expect(later.ms!).toBeLessThan(10_000);
+ return head(stuck);
+ }
+ const waiting = await worker({ slug, sourceId: source, waitMs: 3_000 });
+ expect(waiting).toMatchObject({ event: 'failure', code: 'write_pending' });
+ const held = await head(stuck);
+ expect(held.state).toBe('running');
+ await lapse(stuck);
+ expect(await worker({ slug: `${slug}-after-lapse`, sourceId: source })).toMatchObject({ event: 'done', state: 'committed' });
+ const [pending] = await engine.executeRaw<{ state: string }>(`SELECT state FROM persistence_requests WHERE slug=$1 AND source_id=$2`, [slug, source]);
+ expect(pending?.state).toBe('committed');
+ return head(stuck);
+ }
+
+ test('killed while preparing: a 30 s claim under a dead owner; writer status calls it a live claim; the head commits', async () => {
+ const stuck = randomUUID();
+ await killAt('consumer:preparing', { slug: 'notes/killed-preparing', requestId: stuck });
+ expect(await head(stuck)).toMatchObject({ state: 'running', phase: 'preparing', recovery: false, preparation_attempts: 0, lease_s: 30 });
+ const status = await blocker('default', stuck);
+ expect(status).toMatchObject({ state: 'running', claim: { phase: 'preparing', resumes_on_its_own: false, owner_live: postgres ? true : null, next: { code: 'claim_running' } } });
+ expect(await laterWrites('default', stuck, 'notes/after-preparing')).toMatchObject({ state: 'committed', recovery: false });
+ }, 120_000);
+
+ test('killed inside the publication transaction of a database-only page: the dead owner keeps a 365 s publication lease; the head commits', async () => {
+ const stuck = randomUUID();
+ await killAt('publication:before_commit', { slug: 'notes/killed-publishing', requestId: stuck });
+ expect(await head(stuck)).toMatchObject({ state: 'running', phase: 'publishing', recovery: false, lease_s: 365 });
+ expect(await blocker('default', stuck)).toMatchObject({ claim: { phase: 'publishing', resumes_on_its_own: false, next: { code: 'claim_running' } } });
+ expect(await laterWrites('default', stuck, 'notes/after-publishing')).toMatchObject({ state: 'committed' });
+ }, 120_000);
+
+ test('killed after writing the page file: the recovery record is finished at once on both engines and the head commits', async () => {
+ expect(await worker({ slug: 'notes/prior', sourceId: 'repo-source' })).toMatchObject({ event: 'done', state: 'committed' });
+ const stuck = randomUUID();
+ await killAt('publication:after_publication', { slug: 'notes/killed-after-file', requestId: stuck, sourceId: 'repo-source' });
+ expect(await head(stuck)).toMatchObject({ state: 'running', phase: 'publishing', recovery: true });
+ expect(readFileSync(join(repo, 'notes/killed-after-file.md'), 'utf8')).toContain('Body of notes/killed-after-file.');
+ const later = await worker({ slug: 'notes/after-file', sourceId: 'repo-source' });
+ expect(later).toMatchObject({ event: 'done', state: 'committed' });
+ expect(later.ms!).toBeLessThan(10_000);
+ expect(await head(stuck)).toMatchObject({ state: 'committed', recovery: false });
+ expect(readFileSync(join(repo, 'notes/killed-after-file.md'), 'utf8')).toContain('Body of notes/killed-after-file.');
+ }, 120_000);
+
+ test('a revoked OAuth client\'s stuck write: no other principal can cancel it; it ends failed permission_denied when next prepared', async () => {
+ const stuck = randomUUID();
+ await killAt('consumer:preparing', { slug: 'notes/revoked-client', requestId: stuck,
+ auth: { token: 'synthetic-token', clientId: 'client-a', principal: { kind: 'oauth_client', id: 'client-a' }, sourceId: 'default', scopes: ['read', 'write'] } });
+ await engine.executeRaw(`UPDATE oauth_clients SET deleted_at=now() WHERE client_id='client-a'`);
+ const local = await withEnv(env, () => readLocalWriter(engine, 'cli'));
+ expect(await cancelWriteRequest(engine, { kind: 'local_cli', id: local.id }, stuck)).toBeNull();
+ expect(await head(stuck)).toMatchObject({ state: 'running', phase: 'preparing' });
+ expect(await laterWrites('default', stuck, 'notes/after-revoked')).toMatchObject({ state: 'failed', error_code: 'permission_denied' });
+ }, 120_000);
+
+ if (postgres) test('an owner Postgres cannot prove dead (another boot) holds the head until its lease lapses, then the head commits', async () => {
+ const stuck = randomUUID();
+ await killAt('consumer:preparing', { slug: 'notes/foreign-owner', requestId: stuck });
+ // Every claim the killed process held (its request and any effect it was draining) reads as another boot's.
+ const [{ pid }] = await engine.executeRaw<{ pid: string }>(`SELECT claim_phase->'owner'->>'pid' AS pid FROM persistence_requests WHERE request_id=$1::uuid`, [stuck]);
+ await engine.executeRaw(`UPDATE persistence_requests SET claim_phase=jsonb_set(claim_phase,'{owner,boot_id}',to_jsonb($2::text)) WHERE claim_phase->'owner'->>'pid'=$1`, [pid, FOREIGN_BOOT]);
+ await engine.executeRaw(`UPDATE persistence_effects SET claim_owner=jsonb_set(claim_owner,'{owner,boot_id}',to_jsonb($2::text)) WHERE claim_owner->'owner'->>'pid'=$1`, [pid, FOREIGN_BOOT]);
+ const before = await head(stuck);
+ expect(before).toMatchObject({ state: 'running', phase: 'preparing', lease_s: 30 });
+ expect(await worker({ slug: 'notes/behind-foreign-owner', waitMs: 3_000 })).toMatchObject({ event: 'failure', code: 'write_pending' });
+ expect(await head(stuck)).toMatchObject({ state: 'running', token: before.token });
+ await lapse(stuck);
+ expect(await worker({ slug: 'notes/after-foreign-owner' })).toMatchObject({ event: 'done', state: 'committed' });
+ expect(await head(stuck)).toMatchObject({ state: 'committed' });
+ }, 120_000);
+});