Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.42.20.0 (#1762 / #1745 / #1775 reliability wave) — process background-work
* registry. Single source of truth for "drain every fire-and-forget sink before
* the CLI exits / disconnects."
*
* WHY THIS EXISTS (rule-of-four): four independent fire-and-forget sinks each
* write to the DB after an op returns its response —
* - `last-retrieved.ts` UPDATE pages.last_retrieved_at (#1247/#1269/#1290)
* - `facts/queue.ts` facts:absorb Haiku job + logIngest (#1762)
* - `search/hybrid.ts` query_cache write
* - `eval-capture.ts` eval_candidates INSERT
* On PGLite, if `engine.disconnect()` nulls `_db` while one of these is in
* flight, the sink's "not connected" error path re-pumps via queueMicrotask and
* spins `db.close()` into a 100%-CPU busy-loop that pins the single-writer lock
* (the #1762 incident). The fix is to DRAIN every sink before disconnect. A
* registry (not a hand-written N-call helper) makes that structural: a future
* 5th sink that registers is auto-drained, and the drain is invoked from THREE
* exit points (op-dispatch success finally, op-dispatch error catch, CLI_ONLY
* finally) without repeating the sink list at each.
*
* register (at module import) ──┐
* facts (order 0) │
* last-retrieved (order 1) │
* search-cache (order 2) ├─► Map<name, drainer>
* eval-capture (order 3) │
* volunteer-events (order 4) │
* search-telemetry (order 5) ┘ (#4143 — was the unregistered 5th sink)
* │ CLI exit │ engine.disconnect() (#4143)
* ▼ ▼
* drainAllBackgroundWorkForCliExit drainBackgroundWorkBeforeDisconnect
* mode 'exit', abort allowed mode 'disconnect', NO abort
* └───────────► sort by (order, name) ◄───────────┘
* for each: await drain(timeoutMs, mode)
* if unfinished>0 && abort && allowAbort:
* await abort() ◄─ facts shutdown()
*
* Registration MUST live in the enqueue-owning module (so "module not imported
* ⇒ no work enqueued ⇒ nothing to drain" holds). The Map is keyed by name so a
* re-import / test mock REPLACES rather than duplicating (an array would
* double-register).
*/
/**
* Which exit point is draining (#4143). 'exit' = CLI teardown, engine still
* live afterward until the caller disconnects — sinks may flush residual
* buffers. 'disconnect' = an engine is mid-disconnect (its handle may already
* be nulled) — sinks must only await IN-FLIGHT work settling and must not
* start new writes (a lossy buffer stays lossy here by design).
*
* NOTE: this module stays a zero-import leaf ON PURPOSE — both engines import
* it statically (check-engine-dynamic-import files), so any import added here
* risks a cycle. The warn below is a local once-per-process Set, not
* utils.warnOncePerProcess, for exactly that reason.
*/
import { AsyncLocalStorage } from 'node:async_hooks';
export type BackgroundWorkDrainMode = 'exit' | 'disconnect';
/**
* GBRA-156: search-time writes (the page_retrievals bump, the eval_candidates
* insert, the semantic-cache store) start after the response inside a request
* that owns a deferral scope (the warm CLI socket), instead of mid-request,
* where they competed with the response stage for the search's connections.
* Outside a scope (a cold CLI, which drains them before exit), they start at
* once, as before. node:async_hooks is a builtin, so this module stays a leaf.
*/
interface Deferral { released: boolean; queue: Array<() => void> }
const responseDeferral = new AsyncLocalStorage<Deferral | null>();
/** Start `start` after the enclosing request's response, or now when no request defers (or its response already went). */
export function deferUntilResponse(start: () => void): void {
const scope = responseDeferral.getStore();
if (scope && !scope.released) scope.queue.push(start); else start();
}
/** Runs `fn` with a deferral scope; `release` starts every write it deferred and returns how many. */
export async function withResponseDeferral<T>(fn: () => Promise<T>): Promise<{ value: T; release: () => number }> {
const scope: Deferral = { released: false, queue: [] };
const value = await responseDeferral.run(scope, fn);
return {
value,
release: () => responseDeferral.run(null, () => {
scope.released = true;
const started = scope.queue.splice(0);
for (const start of started) { try { start(); } catch { /* a deferred sink never throws into the caller */ } }
return started.length;
}),
};
}
/**
* Shared teardown budgets (#4284). This module is the natural home: it is the
* zero-import leaf that every teardown participant (both engines,
* cli-force-exit, process-watchdog) already imports, so budget numbers defined
* here cannot drift between the components that must agree on them.
*/
/**
* setTimeout delay ceiling: Node/Bun overflow-clamp anything above 2^31−1 to
* ~1ms (TimeoutOverflowWarning), so an oversized delay must be clamped to
* mean "longer bound", never an instant spurious fire.
*/
export const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;
/** Per-sink bound for one drain pass (the `runDrainers` default below). */
export const SINK_DRAIN_TIMEOUT_MS = 2000;
/**
* #4143/#4284 — the in-loop bound for PGlite.close() inside disconnect().
* Read per call (not module load) so tests and incident responders can set
* GBRAIN_PGLITE_CLOSE_TIMEOUT_MS without subprocess gymnastics; floor 1s,
* ceiling 2^31−1. Lives here (not in the engine) so cli-force-exit's
* computed teardown deadline budgets the SAME bound the engine will actually
* honor — a hardcoded copy drifted the moment the bound became tunable.
* HONEST SCOPE lives at the close site in pglite-engine.ts: this bound
* catches a close that still YIELDS; a wedged loop needs the out-of-band
* watchdog.
*/
export function pgliteCloseTimeoutMs(): number {
return Math.min(
MAX_TIMER_DELAY_MS,
Math.max(1000, Number(process.env.GBRAIN_PGLITE_CLOSE_TIMEOUT_MS ?? '') || 5000),
);
}
export interface BackgroundWorkDrainer {
/** Stable identity; also the Map key (idempotent registration). */
name: string;
/**
* Explicit drain order — lower runs first. Facts is 0 so its abort-path DB
* `logIngest` gets the freshest live-engine window before the fast
* last-retrieved / search-cache drains. Ties break by name for determinism.
*/
order: number;
/**
* Resolve when in-flight work settles OR the bound elapses; report leftovers.
* `mode` (#4143) tells the sink whether residual buffers may flush ('exit',
* engine still live) or only in-flight work may be awaited ('disconnect').
* Pre-#4143 drainers that ignore the parameter keep their exact behavior.
*/
drain(timeoutMs: number, mode: BackgroundWorkDrainMode): Promise<{ unfinished: number }>;
/**
* Optional hard-stop for stragglers (facts-queue: `shutdown()`). AWAITED by
* the registry so the aborted job's DB write settles against a live engine
* BEFORE the caller disconnects. Only invoked when `drain` reports unfinished.
*/
abort?(): Promise<void>;
}
const drainers = new Map<string, BackgroundWorkDrainer>();
/** Register (or replace, by name) a fire-and-forget sink drainer. */
export function registerBackgroundWorkDrainer(d: BackgroundWorkDrainer): void {
drainers.set(d.name, d);
}
/**
* Number of registered sinks. Used by `finishCliTeardown` (cli-force-exit.ts)
* to COMPUTE its backstop deadline from the bounds it guards — a 5th sink
* registering automatically widens the deadline instead of silently making
* the worst-case bounded drain exceed a static number (#2084 eng-review D9).
*/
export function backgroundWorkSinkCount(): number {
return drainers.size;
}
/**
* Test seam — registers a drainer and returns an unregister handle. Preferred
* over a blunt reset: real sink modules register at import time and won't re-run
* that top-level side effect on a second import, so a global clear would
* silently drop the production drainers for the rest of the test process.
*/
export function __registerDrainerForTest(d: BackgroundWorkDrainer): () => void {
drainers.set(d.name, d);
return () => { drainers.delete(d.name); };
}
/** Test seam — snapshot of registered drainer names (sorted), for assertions. */
export function __listDrainerNamesForTest(): string[] {
return [...drainers.keys()].sort();
}
/**
* CLI-EXIT-ONLY. `abort()` is a permanent process-level state change on a sink
* (the facts queue's `shutdown()` sets `shuttingDown=true` for the process
* lifetime). NEVER call this in a long-lived process (`gbrain serve`). Drains
* every registered sink before `engine.disconnect()` so a PGLite `db.close()`
* can't race in-flight work into the re-pump busy-loop (#1762).
*
* Best-effort and non-throwing: one sink's failure never blocks the others or
* the subsequent disconnect.
*/
export async function drainAllBackgroundWorkForCliExit(opts?: { timeoutMs?: number }): Promise<void> {
await runDrainers(opts?.timeoutMs ?? SINK_DRAIN_TIMEOUT_MS, { allowAbort: true, mode: 'exit' });
}
/**
* ENGINE-DISCONNECT drain (#4143). Called by BOTH engines' `disconnect()` so
* an in-flight fire-and-forget statement settles before the underlying handle
* closes — PGLite's `close()` deadlocks PERMANENTLY (close's promise AND the
* in-flight query's promise never settle) when a statement is in flight.
* Unlike the CLI-exit drain this NEVER calls `abort()` (a permanent
* process-level state change, wrong for a long-lived `gbrain serve` that
* disconnects one engine) and passes mode 'disconnect' so sinks skip residual
* buffer flushes (new writes would fail against the already-nulled handle
* anyway — that failing fast is intended, not a bug).
*
* Best-effort and non-throwing; O(~0) when every sink's fast path is empty,
* which is the ordinary case for the hundreds of test `afterEach(disconnect)`
* hooks.
*/
export async function drainBackgroundWorkBeforeDisconnect(opts?: { timeoutMs?: number }): Promise<void> {
await runDrainers(opts?.timeoutMs ?? SINK_DRAIN_TIMEOUT_MS, { allowAbort: false, mode: 'disconnect' });
}
const warnedOnce = new Set<string>();
async function runDrainers(
timeoutMs: number,
opts: { allowAbort: boolean; mode: BackgroundWorkDrainMode },
): Promise<void> {
const ordered = [...drainers.values()].sort(
(a, b) => a.order - b.order || a.name.localeCompare(b.name),
);
for (const d of ordered) {
try {
const { unfinished } = await d.drain(timeoutMs, opts.mode);
if (unfinished > 0 && d.abort && opts.allowAbort) {
// codex #9: AWAIT — the facts:absorb job writes its absorb-log to the
// DB on settle; the abort must finish against a live engine before the
// caller disconnects.
await d.abort();
} else if (unfinished > 0 && !opts.allowAbort) {
// #4143: a silent partial drain at disconnect is the invisible-failure
// class this registry exists to kill — say it once per sink.
const key = `drain-unfinished:${d.name}`;
if (!warnedOnce.has(key)) {
warnedOnce.add(key);
console.error(`[background-work] sink '${d.name}' still had ${unfinished} unfinished item(s) after the ${timeoutMs}ms disconnect drain`);
}
}
} catch {
/* best-effort; never block disconnect on one sink's failure */
}
}
}