Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* E2E: real SIGKILL mid-sync on Postgres (test-plan F7).
*
* The in-process, simulated-kill coverage of op_checkpoint_paths lives in
* test/op-checkpoint.test.ts (PGLite CRUD/union semantics). This file adds the
* two things a simulation cannot prove, against a live Postgres:
*
* 1. A REAL `bun src/cli.ts sync` child killed with SIGKILL mid-import
* leaves a durable partial checkpoint (0 < bankedFiles < total) and does
* NOT advance the per-source `sources.last_commit` bookmark.
* 2. SIGKILL is untrappable, so process-cleanup never runs and the
* source and filesystem `gbrain_cycle_locks` rows are STRANDED: holder pid dead, TTL
* still live, heartbeat recent. Lock-reclaim mechanics under test
* (src/core/db-lock.ts:tryAcquireDbLock):
* - a fresh strand is NOT stealable — the upsert's ON CONFLICT gate
* requires ttl_expires_at < NOW() AND last_refreshed_at older than
* the steal grace (GBRAIN_LOCK_STEAL_GRACE_SECONDS, default derived
* ~600s at 30-min TTL), and the same-host dead-PID fallback (#1780
* Gap 3) requires lock age >= 60s (HOLDER_TAKEOVER_GRACE_MS). So an
* immediate second sync is refused (SyncLockBusyError, exit 1).
* - a dead holder ages out: it stops refreshing, its TTL lapses, and
* last_refreshed_at drifts past the grace — at which point the
* upsert steal branch takes the row. The default windows (30-min
* TTL) are too long for a test, so we simulate the age-out by
* backdating the stranded row's timestamps and shrink the grace via
* the documented GBRAIN_LOCK_STEAL_GRACE_SECONDS env knob, driving
* the second sync through the exact production steal predicate.
* 3. The second sync resumes from the banked checkpoint (never re-imports
* banked paths), converges — every file imported exactly once (page
* count == file count, no duplicate slugs), last_commit advances to the
* pinned target, checkpoint rows clear, and the lock row is released.
*
* DETERMINISM (CEO Finding 11): the kill point is derived by POLLING the DB
* for the first banked op_checkpoint_paths row (GBRAIN_SYNC_CHECKPOINT_EVERY=1
* banks after the first file), THEN sending SIGKILL. Never sleep-then-kill.
*
* Isolated-DB etiquette: fresh init gets a dedicated test database, then this
* legacy checkpoint/TTL-lock scenario uses DB-only sync within that database.
*
* Run: DATABASE_URL=... GBRAIN_TEST_ALLOW_DATABASE_URL=1 \
* bun test --timeout=180000 test/e2e/sync-sigkill-resume-postgres.test.ts
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { mkdtempSync, writeFileSync, rmSync, mkdirSync, realpathSync } from 'fs';
import { createHash, randomUUID } from 'node:crypto';
import { join } from 'path';
import { execSync } from 'child_process';
import { tmpdir, hostname } from 'os';
import postgres from '#postgres'
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
import { loadOpCheckpoint, syncFingerprint } from '../../src/core/op-checkpoint.ts';
import { assertSafeE2eDatabaseUrl } from '../helpers/db-guard.ts';
const DATABASE_URL = process.env.DATABASE_URL;
const describeE2E = DATABASE_URL ? describe : describe.skip;
if (!DATABASE_URL) {
console.log('Skipping E2E sync SIGKILL tests (DATABASE_URL not set)');
}
const CLI = join(import.meta.dir, '..', '..', 'src', 'cli.ts');
/** Unique per-run source id: 1-32 lowercase alnum + interior hyphens. */
const SRC_ID = `sigkill-e2e-${Math.random().toString(36).slice(2, 10)}`;
const LOCK_KEY = `gbrain-sync:${SRC_ID}`;
/** New files added in commit B (the incremental range the kill lands in). */
const NEW_FILES = 30;
/** Total pages after convergence: the commit-A seed page + the 30 new files. */
const TOTAL_PAGES = NEW_FILES + 1;
let home: string;
let repoDir: string;
let filesystemLockKey: string;
let engine: PostgresEngine;
let commitA = '';
let commitB = '';
let ckptFingerprint = '';
let killedPid = 0;
let bankedAfterKill = 0;
let isolatedUrl = '';
let databaseName = '';
let createdDatabase = false;
let admin: ReturnType<typeof postgres> | undefined;
/** Live child handle so afterAll can reap a leak if an assertion throws first. */
let liveChild: ReturnType<typeof Bun.spawn> | null = null;
// ── helpers ─────────────────────────────────────────────────────────────
function baseEnv(extra: Record<string, string> = {}): Record<string, string> {
const env: Record<string, string> = {};
for (const [k, v] of Object.entries(process.env)) {
if (v !== undefined) env[k] = v;
}
// Ambient knobs must not leak into the children — each spawn sets its own.
for (const k of [
'GBRAIN_SYNC_CHECKPOINT_EVERY', 'GBRAIN_SYNC_CHECKPOINT_SECONDS',
'GBRAIN_SYNC_MAX_CHECKPOINT_FAILURES', 'GBRAIN_SYNC_YIELD_EVERY',
'GBRAIN_SYNC_STALL_ABORT_SECONDS', 'GBRAIN_LOCK_STEAL_GRACE_SECONDS',
'GBRAIN_SOURCE', 'GBRAIN_BRAIN_ID',
]) delete env[k];
env.GBRAIN_HOME = home;
env.DATABASE_URL = isolatedUrl;
delete env.GBRAIN_DATABASE_URL;
return { ...env, ...extra };
}
interface RunResult { exitCode: number; stdout: string; stderr: string; }
async function runCli(args: string[], extraEnv: Record<string, string> = {}): Promise<RunResult> {
const proc = Bun.spawn({
cmd: ['bun', CLI, ...args],
env: baseEnv(extraEnv),
cwd: home, // isolate from repo-root dotfiles / bun .env autoload
stdin: 'ignore',
stdout: 'pipe',
stderr: 'pipe',
});
const [stdout, stderr, exitCode] = await Promise.all([
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
proc.exited,
]);
return { exitCode, stdout, stderr };
}
function git(args: string, cwd: string): string {
return execSync(`git ${args}`, { cwd, stdio: 'pipe' }).toString().trim();
}
function sleep(ms: number): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}
/** A few KB of chunkable markdown per file so the import drain has real per-file cost. */
function fileBody(i: number): string {
const lines = [
'---',
`title: SIGKILL Fixture ${i}`,
'type: note',
'---',
'',
];
for (let p = 0; p < 12; p++) {
lines.push(`## Section ${p} of fixture ${i}`);
lines.push('');
lines.push(
`Fixture file ${i}, paragraph ${p}. This content exists to give the sync ` +
`import drain measurable per-file work — chunking, hashing, and several ` +
`chunk inserts against a live Postgres — so a SIGKILL lands mid-range ` +
`deterministically after the first checkpoint bank. Marker token ` +
`sigkill-fixture-${i}-${p} keeps every chunk distinct across files.`.repeat(2),
);
lines.push('');
}
return lines.join('\n');
}
async function bankedCount(): Promise<number> {
const rows = await engine.executeRaw<{ n: number | string }>(
`SELECT count(*)::int AS n FROM op_checkpoint_paths WHERE op = 'sync' AND fingerprint = $1`,
[ckptFingerprint],
);
return Number(rows[0]?.n ?? 0);
}
interface LockRow {
id: string;
holder_pid: number;
holder_host: string;
ttl_expired: boolean;
age_ms: number;
last_refreshed_at: unknown;
}
function fixtureLockIds(): string[] {
return [LOCK_KEY, filesystemLockKey].sort();
}
async function lockRows(): Promise<LockRow[]> {
return engine.executeRaw<LockRow>(
`SELECT id, holder_pid::int AS holder_pid, holder_host,
(ttl_expires_at < NOW()) AS ttl_expired,
(EXTRACT(EPOCH FROM (NOW() - acquired_at)) * 1000)::float8 AS age_ms,
last_refreshed_at
FROM gbrain_cycle_locks WHERE id = ANY($1::text[]) ORDER BY id`,
[fixtureLockIds()],
);
}
async function sourceLastCommit(): Promise<string | null> {
const rows = await engine.executeRaw<{ last_commit: string | null }>(
`SELECT last_commit FROM sources WHERE id = $1`,
[SRC_ID],
);
return rows[0]?.last_commit ?? null;
}
// ── suite ───────────────────────────────────────────────────────────────
describeE2E('E2E: real SIGKILL mid-sync on Postgres — checkpoint bank, stranded lock, exactly-once resume', () => {
beforeAll(async () => {
assertSafeE2eDatabaseUrl(DATABASE_URL!);
databaseName = `gbrain_test_sigkill_${randomUUID().replaceAll('-', '')}`;
admin = postgres(DATABASE_URL!, { max: 1, prepare: false });
await admin.unsafe(`CREATE DATABASE ${databaseName}`);
createdDatabase = true;
const url = new URL(DATABASE_URL!);
url.pathname = `/${databaseName}`;
isolatedUrl = url.toString();
home = mkdtempSync(join(tmpdir(), 'gbrain-sigkill-home-'));
repoDir = mkdtempSync(join(tmpdir(), 'gbrain-sigkill-repo-'));
// The fixture is itself the Git worktree root, with no registered parent.
// Pin both locks independently of the lock implementation's helpers.
filesystemLockKey = `gbrain-fs:${createHash('sha256').update(JSON.stringify(realpathSync(repoDir))).digest('hex')}`;
// Fixture repo, commit A: one seed file (establishes last_commit so the
// killed run takes the INCREMENTAL checkpointed path, not first_sync).
git('init', repoDir);
git('config user.email "[email protected]"', repoDir);
git('config user.name "Test"', repoDir);
mkdirSync(join(repoDir, 'notes'), { recursive: true });
writeFileSync(join(repoDir, 'notes/seed.md'), [
'---', 'title: Seed Page', 'type: note', '---', '', 'Commit-A seed content.',
].join('\n'));
execSync('git add -A && git commit -m "commit A: seed"', { cwd: repoDir, stdio: 'pipe' });
commitA = git('rev-parse HEAD', repoDir);
const init = await runCli(['init', '--non-interactive', '--no-embedding', '--url', isolatedUrl]);
if (init.exitCode !== 0) throw new Error(`init failed: ${init.stderr || init.stdout}`);
const legacy = new PostgresEngine();
await legacy.connect({ database_url: isolatedUrl });
try {
const [brain] = await legacy.executeRaw<{ enabled: boolean }>('SELECT enabled FROM persistence_brain WHERE singleton=1');
expect(brain.enabled).toBe(true);
await legacy.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
} finally {
await legacy.disconnect();
}
const add = await runCli(['sources', 'add', SRC_ID, '--path', repoDir, '--no-federated']);
if (add.exitCode !== 0) throw new Error(`sources add failed: ${add.stderr || add.stdout}`);
// Parent poll/assert connection (schema already at latest via init above).
engine = new PostgresEngine();
await engine.connect({ database_url: isolatedUrl });
}, 120_000);
afterAll(async () => {
if (liveChild && liveChild.exitCode === null && liveChild.signalCode === null) {
try { liveChild.kill('SIGKILL'); await liveChild.exited; } catch { /* best-effort */ }
}
try {
if (engine) await engine.disconnect();
} finally {
try {
if (admin && createdDatabase) await admin.unsafe(`DROP DATABASE ${databaseName} WITH (FORCE)`);
} finally {
await admin?.end();
if (home) rmSync(home, { recursive: true, force: true });
if (repoDir) rmSync(repoDir, { recursive: true, force: true });
}
}
}, 60_000);
test('baseline: first sync converges and pins last_commit at commit A', async () => {
const r = await runCli(['sync', '--source', SRC_ID, '--no-embed', '--no-pull']);
expect(r.exitCode).toBe(0);
expect(await sourceLastCommit()).toBe(commitA);
const pages = await engine.executeRaw<{ n: number | string }>(
`SELECT count(*)::int AS n FROM pages WHERE source_id = $1`, [SRC_ID],
);
expect(Number(pages[0].n)).toBe(1);
}, 120_000);
test('SIGKILL after the first banked row: partial checkpoint, bookmark frozen, lock stranded', async () => {
// Commit B: 30 new files — the incremental range the kill lands inside.
for (let i = 1; i <= NEW_FILES; i++) {
writeFileSync(join(repoDir, `notes/f${String(i).padStart(2, '0')}.md`), fileBody(i));
}
execSync('git add -A && git commit -m "commit B: 30 files"', { cwd: repoDir, stdio: 'pipe' });
commitB = git('rev-parse HEAD', repoDir);
// Checkpoint key = (op, syncFingerprint(sourceId, lastCommit)) — lastCommit
// is the commit-A bookmark, fixed across kill + resume (#1794).
ckptFingerprint = syncFingerprint({ sourceId: SRC_ID, lastCommit: commitA });
const proc = Bun.spawn({
cmd: ['bun', CLI, 'sync', '--source', SRC_ID, '--no-embed', '--no-pull'],
env: baseEnv({
GBRAIN_SYNC_CHECKPOINT_EVERY: '1', // bank after the FIRST file (F7 spec)
GBRAIN_SYNC_CHECKPOINT_SECONDS: '1', // time-based flush floor
GBRAIN_SYNC_YIELD_EVERY: '1', // yield per file: heartbeat fires, drain slows
}),
cwd: home,
stdin: 'ignore',
stdout: 'pipe',
stderr: 'pipe',
});
liveChild = proc;
killedPid = proc.pid;
// Consume pipes from the start so the child can never block on a full pipe.
const stdoutP = new Response(proc.stdout).text();
const stderrP = new Response(proc.stderr).text();
// DETERMINISM RULE: poll the DB for the first banked op_checkpoint_paths
// row, THEN SIGKILL — never sleep-then-kill.
const t0 = Date.now();
let polls = 0;
let banked = 0;
while (Date.now() - t0 < 60_000) {
if (proc.exitCode !== null || proc.signalCode !== null) break;
banked = await bankedCount();
polls++;
if (banked >= 1) break;
await sleep(5);
}
const msToFirstBank = Date.now() - t0;
if (banked < 1 || proc.exitCode !== null || proc.signalCode !== null) {
const [so, se] = await Promise.all([stdoutP, stderrP]);
throw new Error(
`child was not killable mid-import (banked=${banked}, exit=${proc.exitCode}, ` +
`signal=${proc.signalCode}, polls=${polls}, ${msToFirstBank}ms)\n` +
`stdout:\n${so}\nstderr:\n${se}`,
);
}
proc.kill('SIGKILL');
await proc.exited;
await Promise.all([stdoutP, stderrP]); // drain pipes
liveChild = null;
// Widened re-read: TS narrowed proc.signalCode to null at the guard above
// and doesn't invalidate that across proc.kill()/await.
expect(proc.signalCode as NodeJS.Signals | null).toBe('SIGKILL');
console.log(
`[f7] first banked row after ${msToFirstBank}ms (${polls} polls, banked=${banked} at kill dispatch)`,
);
// (1) Partial checkpoint: >0 banked (poll guaranteed) and < total (the
// kill landed mid-range) — and the pinned target row names commit B.
bankedAfterKill = await bankedCount();
console.log(`[f7] banked after kill: ${bankedAfterKill}/${NEW_FILES}`);
expect(bankedAfterKill).toBeGreaterThan(0);
expect(bankedAfterKill).toBeLessThan(NEW_FILES);
const target = await loadOpCheckpoint(engine, { op: 'sync-target', fingerprint: ckptFingerprint });
expect(target).toEqual([commitB]);
// (1) Bookmark frozen: last_commit did NOT advance.
expect(await sourceLastCommit()).toBe(commitA);
// (2) Both source and filesystem locks are STRANDED: held by the dead child's pid on
// this host, TTL still live (30-min default), heartbeat recorded — SIGKILL
// bypassed process-cleanup's release.
const locks = await lockRows();
expect(locks.map(lock => lock.id)).toEqual(fixtureLockIds());
for (const lock of locks) {
expect(lock.holder_pid).toBe(killedPid);
expect(lock.holder_host).toBe(hostname());
expect(lock.ttl_expired).toBe(false);
expect(lock.last_refreshed_at).not.toBeNull();
}
// The holder really is dead (proc.exited resolved above reaps the child,
// so a signal-0 probe gets ESRCH, not a zombie's success).
let probeErr: NodeJS.ErrnoException | null = null;
try { process.kill(killedPid, 0); } catch (e) { probeErr = e as NodeJS.ErrnoException; }
expect(probeErr?.code).toBe('ESRCH');
// Grace semantics, fail-closed half: a FRESH strand (TTL live, heartbeat
// recent, age < the 60s dead-PID takeover grace) must NOT be stolen — the
// immediate retry is refused with the lock-busy error. Guard the premise
// first so a pathologically slow runner fails loudly instead of flipping
// the refusal into a reclaim.
for (const lock of locks) expect(lock.age_ms).toBeLessThan(40_000);
const busy = await runCli(['sync', '--source', SRC_ID, '--no-embed', '--no-pull']);
expect(busy.exitCode).toBe(1);
expect(busy.stderr).toContain('Another sync is in progress');
expect(busy.stderr).toContain(String(killedPid));
// The refused run changed nothing.
expect(await sourceLastCommit()).toBe(commitA);
const stillStranded = await lockRows();
expect(stillStranded.map(lock => lock.id)).toEqual(fixtureLockIds());
for (const lock of stillStranded) {
expect(lock.holder_pid).toBe(killedPid);
expect(lock.holder_host).toBe(hostname());
}
}, 120_000);
test('second sync reclaims the aged-out stranded lock via TTL + steal grace and converges exactly-once', async () => {
expect(bankedAfterKill).toBeGreaterThan(0); // depends on the kill test
// Simulate both dead-holder locks aging out (a killed process never refreshes:
// its TTL lapses and last_refreshed_at drifts). Backdating the row is the
// time-warp for the 30-min TTL; the steal grace is shrunk for real via the
// documented env knob so the reclaim goes through tryAcquireDbLock's
// production ON CONFLICT predicate:
// ttl_expires_at < NOW() AND last_refreshed_at < NOW() - grace.
const aged = await engine.executeRaw<{ id: string }>(
`UPDATE gbrain_cycle_locks
SET ttl_expires_at = NOW() - INTERVAL '1 second',
last_refreshed_at = NOW() - INTERVAL '10 minutes',
acquired_at = acquired_at - INTERVAL '10 minutes'
WHERE id = ANY($1::text[]) AND holder_pid = $2 AND holder_host = $3
RETURNING id`,
[fixtureLockIds(), killedPid, hostname()],
);
expect(aged.map(lock => lock.id).sort()).toEqual(fixtureLockIds());
const r = await runCli(
['sync', '--source', SRC_ID, '--no-embed', '--no-pull'],
{ GBRAIN_LOCK_STEAL_GRACE_SECONDS: '60' },
);
expect(r.exitCode, `stdout:\n${r.stdout}\nstderr:\n${r.stderr}`).toBe(0);
// It RESUMED the banked checkpoint (did not restart): the resume banner
// reports exactly the rows banked before the kill.
const combined = r.stdout + r.stderr;
expect(combined).toContain('resuming checkpoint');
const m = combined.match(/resuming checkpoint: (\d+) file\(s\) already done/);
expect(m).not.toBeNull();
expect(Number(m![1])).toBe(bankedAfterKill);
// (3a) Converged exactly once: page count equals file count, no duplicate
// slugs, and every commit-B file landed.
const agg = await engine.executeRaw<{ total: number | string; distinct_slugs: number | string; new_files: number | string }>(
`SELECT count(*)::int AS total,
count(DISTINCT slug)::int AS distinct_slugs,
count(*) FILTER (WHERE slug LIKE 'notes/f%')::int AS new_files
FROM pages WHERE source_id = $1`,
[SRC_ID],
);
expect(Number(agg[0].total)).toBe(TOTAL_PAGES);
expect(Number(agg[0].distinct_slugs)).toBe(TOTAL_PAGES);
expect(Number(agg[0].new_files)).toBe(NEW_FILES);
// (3b) last_commit NOW advanced to the pinned target (commit B).
expect(await sourceLastCommit()).toBe(commitB);
// (3c) Checkpoint fully cleared on completion (parents cascade the paths).
expect(await bankedCount()).toBe(0);
const parents = await engine.executeRaw<{ n: number | string }>(
`SELECT count(*)::int AS n FROM op_checkpoints WHERE op IN ('sync', 'sync-target') AND fingerprint = $1`,
[ckptFingerprint],
);
expect(Number(parents[0].n)).toBe(0);
// (3d) Both reclaimed locks were released by withRefreshingLock's finally.
expect(await lockRows()).toEqual([]);
}, 120_000);
});