Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6278 R1: the `owner_unavailable` reasons matrix. One reason per owner
* condition, in a fixed order; each with its message, `why`, actor, next
* step and read-only verify; `host_mismatch` prints both host ids' first 8
* characters and, for local callers only, the observed `host.json` path, and
* is never retryable; the in-progress reasons render `next: wait`.
* `maintenancePreflight` refuses every condition with its own reason and
* never claims or transfers ownership to get past it; the HTTP envelope of
* each refusal carries no local path.
* Fails when: a condition loses its reason, falls back to the generic text,
* leaks the identity path to a remote caller, or lets an agent retry a host
* mismatch forever.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { cliRenderContext, deriveNext, toAgentError, type RenderContext } from '../src/core/agent-output.ts';
import { CODES } from '../src/core/error-registry.ts';
import { OperationError } from '../src/core/ops/contract.ts';
import { checkOwner, ownerRefusal, ownerUnavailableError, OWNER_UNAVAILABLE_REASONS, OWNER_WAIT_SECONDS, type OwnerUnavailableReason } from '../src/core/persistence/owner-refusal.ts';
import { claimWorktree, getWorktreeBinding, type WorktreeBinding } from '../src/core/persistence/ownership.ts';
import { localHostId, persistenceHome } from '../src/core/persistence/identity.ts';
import { maintenancePreflight } from '../src/core/persistence/prepared-maintenance.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { FENCE_REASONS } from '../src/core/fence-repair/reasons.ts';
import { fenceHoldStatus } from '../src/core/fence-repair/hold-fix.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';
const OWNER = 'aaaaaaaa-1111-4111-8111-111111111111', LOCAL = 'bbbbbbbb-2222-4222-8222-222222222222';
const INCARNATION = 'cccccccc-3333-4333-8333-333333333333';
const binding = (over: Partial<WorktreeBinding> = {}): WorktreeBinding => ({ worktree_id: randomUUID(), source_id: 'src', source_incarnation: INCARNATION, relative_path: '',
topology_generation: 1, owner_host_id: LOCAL, owner_epoch: 1, state: 'active', local_path: '/srv/brain', coordination_path: '/srv/brain/.gbrain-coordination', git_durability: null, ...over });
const http: RenderContext = { transport: 'http', surface: 'full', isCallable: () => false, preapproved: () => false, principal: 'client-example' };
describe('the matrix', () => {
test('checks the conditions in one order and names the first that fails', () => {
const at = (b: WorktreeBinding | null, incarnation = INCARNATION) => checkOwner(b, incarnation, LOCAL).reason;
expect(at(null)).toBe('binding_missing');
expect(at(binding({ owner_host_id: OWNER, state: 'draining' }), randomUUID())).toBe('incarnation_changed');
expect(at(binding({ owner_host_id: OWNER, state: 'draining' }))).toBe('host_mismatch');
expect(at(binding({ state: 'draining', local_path: null }))).toBe('transfer_in_progress');
expect(at(binding({ state: 'recovering', local_path: null }))).toBe('clone_in_progress');
expect(at(binding({ local_path: null, coordination_path: null }))).toBe('local_path_missing');
expect(at(binding({ coordination_path: null }))).toBe('coordination_path_missing');
const ok = checkOwner(binding(), INCARNATION, LOCAL);
expect(ok.reason).toBeNull();
if (ok.reason === null) expect(ok.binding.local_path).toBe('/srv/brain');
});
test('every reason is in the registry vocabulary, has a fence reason the hold routes as owner, and a read-only writer-status verify', () => {
const registry = CODES.owner_unavailable.reasons!;
for (const reason of OWNER_UNAVAILABLE_REASONS) {
expect(registry).toContain(reason);
const row = ownerRefusal({ sourceId: 'src', reason, binding: binding({ owner_host_id: OWNER }), incarnation: INCARNATION, hostId: LOCAL, remote: true, work: 'maintenance' });
expect(row.fix.verify?.argv).toEqual(['gbrain', 'sources', 'writer', 'status', '--source', 'src', '--json']);
expect(row.fix.argv).toEqual(['gbrain', 'sources', 'writer', 'status', '--source', 'src', '--json']);
expect(row.fix.consent).toEqual([]);
expect(row.why).toContain('never claims or transfers ownership');
if (reason !== 'binding_missing') {
expect(FENCE_REASONS[reason].actor).toBe('host_admin');
expect(fenceHoldStatus({ reason: 'invalid_fence', fence_repair: { reason, tier: null, at: '2026-10-08T00:00:00Z', next_attempt_after: null } } as never).state).toBe('owner');
}
}
});
test('in-progress work is a wait with its delay; the rest is a host administrator decision; host_mismatch is never retryable', () => {
const cli = cliRenderContext();
const next = (reason: OwnerUnavailableReason) => deriveNext(ownerRefusal({ sourceId: 'src', reason, binding: binding(), incarnation: INCARNATION, hostId: LOCAL, remote: false, work: 'maintenance' }).fix, cli);
expect(next('transfer_in_progress')).toBe('wait');
expect(next('clone_in_progress')).toBe('wait');
expect(ownerRefusal({ sourceId: 'src', reason: 'clone_in_progress', binding: binding(), incarnation: INCARNATION, hostId: LOCAL, remote: false, work: 'maintenance' }).fix.why).toContain(`Wait ${OWNER_WAIT_SECONDS} seconds`);
for (const reason of ['host_mismatch', 'binding_missing', 'incarnation_changed', 'local_path_missing', 'coordination_path_missing'] as const) expect(next(reason)).toBe('tell_user_to_run');
const retryable = Object.fromEntries(OWNER_UNAVAILABLE_REASONS.map(reason => [reason,
ownerRefusal({ sourceId: 'src', reason, binding: binding(), incarnation: INCARNATION, hostId: LOCAL, remote: false, work: 'maintenance' }).retryable]));
expect(retryable).toEqual({ host_mismatch: false, transfer_in_progress: true, clone_in_progress: true, binding_missing: false, incarnation_changed: false, coordination_path_missing: false, local_path_missing: false });
});
test('host_mismatch prints both ids\' first 8 characters, and the observed host.json path to local callers only', async () => {
await withEnv({ GBRAIN_HOME: mkdtempSync(join(tmpdir(), 'gbrain-owner-refusal-')) }, async () => {
const path = join(persistenceHome(), 'host.json');
const local = ownerUnavailableError({ sourceId: 'src', reason: 'host_mismatch', binding: binding({ owner_host_id: OWNER }), incarnation: INCARNATION, hostId: LOCAL, remote: false, work: 'fence repair' });
expect(local.message).toContain('aaaaaaaa'); expect(local.message).toContain('bbbbbbbb');
expect(local.message).not.toContain(OWNER); expect(local.message).not.toContain(LOCAL);
expect(local.why).toContain(path);
expect(local.reason).toBe('host_mismatch');
expect(local.retryable).toBe(false);
const envelope = toAgentError(local, { transport: 'cli', command: 'repair', render: cliRenderContext() });
expect(envelope).toMatchObject({ code: 'owner_unavailable', reason: 'host_mismatch', class: 'retryable', retryable: false, fix: { next: 'tell_user_to_run', actor: 'host_admin' } });
const remote = ownerUnavailableError({ sourceId: 'src', reason: 'host_mismatch', binding: binding({ owner_host_id: OWNER }), incarnation: INCARNATION, hostId: LOCAL, remote: true, work: 'fence repair' });
const rendered = JSON.stringify(toAgentError(remote, { transport: 'http', op: 'chronicle_day', render: http }));
expect(rendered).not.toContain(path); expect(rendered).not.toContain(persistenceHome()); expect(rendered).not.toContain('read its identity from');
expect(rendered).toContain('aaaaaaaa'); expect(rendered).toContain('bbbbbbbb');
expect(JSON.parse(rendered)).toMatchObject({ reason: 'host_mismatch', retryable: false });
// The in-progress reasons keep the code's retryable class.
const wait = toAgentError(ownerUnavailableError({ sourceId: 'src', reason: 'transfer_in_progress', binding: binding({ state: 'draining' }), incarnation: INCARNATION, hostId: LOCAL, remote: false, work: 'maintenance' }),
{ transport: 'cli', command: 'chronicle', render: cliRenderContext() });
expect(wait).toMatchObject({ reason: 'transfer_in_progress', retryable: true, fix: { next: 'wait' } });
});
});
});
describe('maintenancePreflight', () => {
const backends = testBackends();
const engines: BrainEngine[] = [];
let closePostgres: (() => Promise<void>) | undefined;
const dataDir = mkdtempSync(join(tmpdir(), 'gbrain-owner-preflight-'));
beforeAll(async () => {
if (backends.includes('pglite')) { const engine = new PGLiteEngine(); await engine.connect({ database_path: join(dataDir, 'pglite') }); await engine.initSchema(); engines.push(engine); }
if (backends.includes('postgres')) { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!); engines.push(pg.engine); closePostgres = pg.close; }
}, 120_000);
afterAll(async () => {
for (const engine of engines) { await disposePersistenceConsumer(engine); await engine.disconnect(); }
await closePostgres?.(); rmSync(dataDir, { recursive: true, force: true });
});
async function topology(engine: BrainEngine, sql: string, params: unknown[]) {
await engine.transaction(async tx => { await tx.executeRaw("SELECT set_config('gbrain.topology_change','on',true)"); await tx.executeRaw(sql, params); });
}
async function fixture(run: (engine: BrainEngine, sourceId: string, root: string) => Promise<void>) {
for (const engine of engines) {
const dir = mkdtempSync(join(tmpdir(), 'gbrain-owner-preflight-src-'));
const root = join(dir, 'brain'); mkdirSync(root);
const sourceId = `owner-${randomUUID().slice(0, 8)}`;
try {
await withEnv({ GBRAIN_HOME: join(dir, 'home') }, async () => {
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', [sourceId, root]);
await engine.setConfig('sync.write_through', 'true');
await claimWorktree(engine, sourceId, root);
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
await run(engine, sourceId, root);
});
} finally {
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
rmSync(dir, { recursive: true, force: true });
}
}
}
const refusal = async (engine: BrainEngine, sourceId: string): Promise<OperationError> => {
try { await maintenancePreflight(engine, sourceId); } catch (error) { if (error instanceof OperationError) return error; throw error; }
throw new Error('preflight did not refuse');
};
test('each owner condition refuses owner_unavailable with its own reason, never claims ownership, and the remote envelope carries no local path', () => fixture(async (engine, sourceId) => {
const before = (await getWorktreeBinding(engine, sourceId))!;
const worktree = before.worktree_id;
expect(await maintenancePreflight(engine, sourceId)).toMatchObject({ binding: { worktree_id: worktree } });
const stranger = randomUUID();
const seen: Record<string, OperationError> = {};
await topology(engine, 'UPDATE persistence_worktrees SET owner_host_id=$2::uuid WHERE id=$1::uuid', [worktree, stranger]);
seen.host_mismatch = await refusal(engine, sourceId);
await topology(engine, 'UPDATE persistence_worktrees SET owner_host_id=$2::uuid, state=$3 WHERE id=$1::uuid', [worktree, localHostId(), 'draining']);
seen.transfer_in_progress = await refusal(engine, sourceId);
await topology(engine, 'UPDATE persistence_worktrees SET state=$2 WHERE id=$1::uuid', [worktree, 'recovering']);
seen.clone_in_progress = await refusal(engine, sourceId);
await topology(engine, 'UPDATE persistence_worktrees SET state=$2 WHERE id=$1::uuid', [worktree, 'active']);
await topology(engine, "UPDATE persistence_host_bindings SET coordination_path='' WHERE worktree_id=$1::uuid", [worktree]);
seen.coordination_path_missing = await refusal(engine, sourceId);
await topology(engine, 'DELETE FROM persistence_host_bindings WHERE worktree_id=$1::uuid', [worktree]);
seen.local_path_missing = await refusal(engine, sourceId);
await topology(engine, 'INSERT INTO persistence_host_bindings(worktree_id,host_id,local_path,coordination_path) VALUES($1::uuid,$2::uuid,$3,$4)',
[worktree, localHostId(), before.local_path, before.coordination_path]);
await topology(engine, 'UPDATE persistence_source_bindings SET source_incarnation=$2::uuid WHERE source_id=$1', [sourceId, randomUUID()]);
seen.incarnation_changed = await refusal(engine, sourceId);
await topology(engine, 'DELETE FROM persistence_source_bindings WHERE source_id=$1', [sourceId]);
seen.binding_missing = await refusal(engine, sourceId);
for (const [reason, error] of Object.entries(seen)) {
expect([reason, error.code, error.reason]).toEqual([reason, 'owner_unavailable', reason]);
expect(error.fix?.argv).toEqual(['gbrain', 'sources', 'writer', 'status', '--source', sourceId, '--json']);
expect(error.why).toContain('never claims or transfers ownership');
const remote = JSON.stringify(toAgentError(error, { transport: 'http', op: 'chronicle_day', render: http }));
expect(remote).not.toContain(persistenceHome());
}
expect(seen.host_mismatch!.message).toContain(stranger.slice(0, 8));
expect(seen.host_mismatch!.message).toContain(localHostId().slice(0, 8));
expect(seen.host_mismatch!.why).toContain(join(persistenceHome(), 'host.json'));
expect(seen.host_mismatch!.retryable).toBe(false);
expect(seen.transfer_in_progress!.fix?.actor).toBe('provider');
expect(seen.clone_in_progress!.fix?.actor).toBe('provider');
// Nothing above claimed or transferred: the worktree still has no source binding and the stranger's host id never became this host's.
expect(await getWorktreeBinding(engine, sourceId)).toBeNull();
const [row] = await engine.executeRaw<{ owner_host_id: string; state: string }>('SELECT owner_host_id,state FROM persistence_worktrees WHERE id=$1::uuid', [worktree]);
expect(row).toMatchObject({ owner_host_id: localHostId(), state: 'active' });
}), 60_000);
});