Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6340: a managed drain whose pass dies on a dropped database connection
* (`write ECONNABORTED host:port`, the session-mode pooler closing the socket
* mid-statement) resumes instead of exiting. Before this change the drain's
* transient classifier did not know `ECONNABORTED` (only `ECONNRESET`), so the
* error escaped `runDrain`, the CLI exited 1 and the operator relaunched by
* hand; four such deaths cost one brain fifty minutes in an afternoon. Now the
* drain reconnects, waits 5 / 15 / 45 s and re-enters the pass at the stored
* cursor; three drops in a row with no page committed between them end the
* drain `blocked` / `connection_lost` with the same resume command as `next`.
* `CONNECTION_DESTROYED` / `CONNECTION_CLOSED` stay out of the retry (#6329:
* this process's own settle of a cancelled statement, the stall path).
*/
import { describe, expect, spyOn, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { CONNECTION_RETRY_MS, CONNECTION_STRIKES, drainJsonFields, drainNext, formatDrainSummary, isConnectionDrop, runDrain } from '../src/core/persistence/sync-drain.ts';
import { isRetryableConnError } from '../src/core/retry-matcher.ts';
import { ERROR_CATALOGUE } from '../src/core/error-catalogue.ts';
import { CODES } from '../src/core/error-registry.ts';
import type { SyncResult } from '../src/commands/sync.ts';
const base: SyncResult = { status: 'synced', fromCommit: 'a', toCommit: 'b', added: 0, modified: 0, deleted: 0, renamed: 0, chunksCreated: 0, embedded: 0, pagesAffected: [] };
const RESUME = 'gbrain sync --source s --no-pull --retry-failed';
const dropped = (errno = 'ECONNABORTED') => Object.assign(new Error(`write ${errno} db.example.invalid:5432`), { code: errno });
function captureStderr(): { lines: string[]; restore(): void } {
const lines: string[] = [];
const spy = spyOn(console, 'error').mockImplementation((...args: unknown[]) => { lines.push(args.map(String).join(' ')); });
return { lines, restore: () => spy.mockRestore() };
}
describe('classifying a dropped connection', () => {
test('the socket errnos a pooler drop surfaces are retryable connection errors; the settle codes and page faults are not', () => {
for (const errno of ['ECONNABORTED', 'ECONNRESET', 'ETIMEDOUT', 'EPIPE']) {
expect(isRetryableConnError(dropped(errno))).toBe(true);
expect(isConnectionDrop(dropped(errno))).toBe(true);
expect(isConnectionDrop(new Error(`write ${errno} undefined:undefined`))).toBe(true);
}
expect(isConnectionDrop(Object.assign(new Error('write CONNECTION_DESTROYED'), { code: 'CONNECTION_DESTROYED' }))).toBe(false);
expect(isConnectionDrop(Object.assign(new Error('write CONNECTION_CLOSED host:5432'), { code: 'CONNECTION_CLOSED' }))).toBe(false);
expect(isConnectionDrop(Object.assign(new Error('A page changed after this sync cursor was enumerated.'), { code: 'revision_conflict' }))).toBe(false);
expect(isConnectionDrop(Object.assign(new Error('canceling statement due to statement timeout'), { code: '57014' }))).toBe(false);
});
test('the schedule is 5, 15 and 45 seconds and the strike count matches it', () => {
expect([...CONNECTION_RETRY_MS]).toEqual([5_000, 15_000, 45_000]);
expect(CONNECTION_STRIKES).toBe(3);
});
});
describe('runDrain over a dropping connection', () => {
test('a drop is reconnected and retried; a pass that then commits resets the strikes and the drain finishes synced', async () => {
let passes = 0;
const reconnects: unknown[] = [];
const err = captureStderr();
try {
const result = await runDrain({ announce: true, progressMs: 1, backoffMs: 2, reconnect: async error => { reconnects.push(error); },
pass: async (_signal, onProgress) => {
passes++;
// Drops on passes 1 and 2, a page on pass 3, drops on 4 and 5 (the counter restarted at the commit), done on 6.
if (passes === 1 || passes === 2 || passes === 4 || passes === 5) throw dropped(passes === 4 ? 'ETIMEDOUT' : 'ECONNABORTED');
if (passes === 3) { onProgress({ phase: 'managed_sync.page_committed', bankedFiles: 1, total: 2 }); return { ...base, status: 'partial', reason: 'writer_yield', managedCursor: { index: 1, total: 2 } }; }
onProgress({ phase: 'managed_sync.page_committed', bankedFiles: 2, total: 2 });
return { ...base, managedCursor: { index: 2, total: 2 } };
} });
expect(result.drain).toMatchObject({ outcome: 'synced', passes: 6, written: 2 });
expect(result.drain!.stop_reason).toBeUndefined();
expect(reconnects).toHaveLength(4);
expect(reconnects.every(error => isConnectionDrop(error))).toBe(true);
const drops = err.lines.filter(line => line.includes('database connection dropped'));
expect(drops).toHaveLength(4);
expect(drops[0]).toMatch(/^\[sync\] 0\/\? processed · database connection dropped \(write ECONNABORTED db\.example\.invalid:5432\); reconnecting and retrying in \d+s \(1 of 3\)$/);
expect(drops[2]).toContain('(1 of 3)');
} finally { err.restore(); }
});
test('three drops with no page between them end the drain blocked / connection_lost with the resume command as next, loopable', async () => {
let passes = 0;
const result = await runDrain({ backoffMs: 2, pass: async () => { passes++; throw dropped(); } });
expect(passes).toBe(3);
expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'connection_lost', connection: { drops: 3, last_error: 'write ECONNABORTED db.example.invalid:5432' } });
const next = drainNext(result, RESUME, 's')!;
expect(next).toMatchObject({ command: RESUME, safe_to_loop: true, retry_after_ms: 60_000, code: 'connection_lost', docs: ERROR_CATALOGUE.sync_drain_connection_lost.docs });
expect(next.why).toContain('3 times in a row');
expect(next.why).toContain('without re-freezing');
expect(ERROR_CATALOGUE.sync_drain_connection_lost).toEqual({ code: 'connection_lost', docs: 'docs/guides/write-refusals.md#drain-connection-lost' });
expect(CODES.connection_lost).toMatchObject({ class: 'retryable', retryable: true });
expect(drainJsonFields(result, RESUME, 's')).toMatchObject({ outcome: 'blocked', next: { command: RESUME, safe_to_loop: true } });
const summary = formatDrainSummary(result, RESUME, 's').join('\n');
expect(summary).toContain('Database connection dropped 3 times in a row');
expect(summary).toContain(`Next: ${RESUME} (safe to rerun in a loop)`);
});
test('a connection URL in the error text never reaches the report, and a drop after a successful pass keeps that pass\'s counts', async () => {
let passes = 0;
const leaky = Object.assign(new Error('write ECONNABORTED postgresql://user:[email protected]:5432/brain'), { code: 'ECONNABORTED' });
const pending: SyncResult = { ...base, status: 'partial', reason: 'writer_yield', added: 4, managedCursor: { index: 4, total: 9 } };
const result = await runDrain({ backoffMs: 2, pass: async () => { if (++passes === 1) return pending; throw leaky; } });
expect(result).toMatchObject({ added: 4, drain: { outcome: 'blocked', stop_reason: 'connection_lost', remaining: 5 } });
expect(JSON.stringify(result)).not.toContain('secret');
expect(result.drain!.connection!.last_error).toBe('write ECONNABORTED <url>');
});
// #6586: on Bun 1.4.2 a drifted spawnSync loop made the drain's git reads return `spawnSync git ETIMEDOUT`; the drain
// printed "database connection dropped", reconnected the database and counted a strike toward `connection_lost`.
test('a timed-out child process is not a drop: no reconnect, no "database connection dropped" line, a transient retry', async () => {
const child = spawnSync('sh', ['-c', 'sleep 5'], { timeout: 50 }).error!;
expect(child).toMatchObject({ code: 'ETIMEDOUT', syscall: 'spawnSync sh' });
const git = Object.assign(new Error('spawnSync git ETIMEDOUT'), { code: 'ETIMEDOUT', syscall: 'spawnSync git', errno: -110 });
for (const error of [child, git, new Error('reading notes/a.md: spawnSync git ETIMEDOUT'), Object.assign(new Error('spawn git ENOENT'), { code: 'ENOENT', syscall: 'spawn git' })]) {
expect(isConnectionDrop(error)).toBe(false);
}
expect(isConnectionDrop(dropped('ETIMEDOUT'))).toBe(true);
let passes = 0;
const reconnects: unknown[] = [];
const err = captureStderr();
try {
const result = await runDrain({ announce: true, progressMs: 1, backoffMs: 2, reconnect: async error => { reconnects.push(error); },
pass: async (_signal, onProgress) => {
if (++passes <= 2) throw git;
onProgress({ phase: 'managed_sync.page_committed', bankedFiles: 1, total: 1 });
return { ...base, managedCursor: { index: 1, total: 1 } };
} });
expect(result.drain).toMatchObject({ outcome: 'synced', passes: 3 });
expect(reconnects).toEqual([]);
expect(err.lines.filter(line => line.includes('database connection dropped'))).toEqual([]);
} finally { err.restore(); }
passes = 0;
await expect(runDrain({ backoffMs: 2, reconnect: async error => { reconnects.push(error); }, pass: async () => { passes++; throw git; } })).rejects.toBe(git);
expect(reconnects).toEqual([]);
expect(passes).toBeGreaterThan(1);
});
test('a settle-discard code is not a drop: the drain ends with the error as before', async () => {
const destroyed = Object.assign(new Error('write CONNECTION_DESTROYED'), { code: 'CONNECTION_DESTROYED' });
await expect(runDrain({ backoffMs: 2, pass: async () => { throw destroyed; } })).rejects.toBe(destroyed);
});
});