Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6288 part 1 / #6352 (P2.3, UC2): a publish that neither resolves nor rejects used to hold its claim, its root and its
* locks forever while the lease kept renewing. What bounds it now:
*
* - On Postgres the publish transaction itself is the real bound. `declareDurablePersistence` already sets a
* transaction-local `statement_timeout` (5 s) and `lock_timeout`; `bindPublicationTimeouts` adds a transaction-local
* `idle_in_transaction_session_timeout` of the publication ceiling, so a publisher stuck between statements with the
* transaction open is ended by the server (its connection is terminated and nothing it did can commit). Both are
* transaction-scoped, so they hold behind a transaction pooler that drops startup parameters.
* - `boundPublication` never settles before the publish does, so nothing releases or reports a request or group while
* its transaction can still commit. At the ceiling it marks the publication overdue (the claim stamp reads
* `waiting_on: publication_deadline`); a grace later it aborts the publication's signal, passed to `transaction(fn, { signal })`:
* on Postgres that discards the transaction's connection and the publish rejects with an AbortError (nothing commits);
* if the publish still has not settled shortly after, the owner reports `restart_required`. The root stays held: restarting
* the owner process is the only honest recovery, never a promised reclaim.
* - On PGLite a running transaction cannot be cancelled (the signal is checked only before BEGIN): the ceiling only marks
* the publication overdue and, past the grace, `restart_required`.
*
* The wave 9 yield between groups is outside any one `boundPublication` call, so it never counts against a group.
*/
import { AsyncLocalStorage } from 'node:async_hooks';
import type { BrainEngine, TransactionOptions } from '../engine.ts';
import { gucMilliseconds, resolveSessionTimeouts } from '../db.ts';
import { declareDurablePersistence } from './protocol.ts';
import { engineKey } from '../engine-key.ts';
export const PUBLICATION_CEILING_KEY = 'persistence.publication_ceiling_ms';
/** `persistence.publication_ceiling_ms`' default (60000 to 3600000 accepted). */
export const PUBLICATION_CEILING_DEFAULT_MS = 5 * 60_000;
/** After the ceiling: how long before the publication's signal is aborted, and how long after that before `restart_required`. */
export const PUBLICATION_GRACE_MS = 30_000;
export const PUBLICATION_SETTLE_MS = 5_000;
/** #6574: the claim lease a publication takes before it opens its transaction: past it, the owner has already reported restart_required. */
export function publicationLeaseMs(ceilingMs: number, marginMs = 30_000): number {
return ceilingMs + PUBLICATION_GRACE_MS + PUBLICATION_SETTLE_MS + marginMs;
}
const CEILING_TTL_MS = 5_000;
/** Per engine; a publication transaction reads its engine's entry. */
let ceilings = new WeakMap<object, { value: number; at: number }>();
const signals = new AsyncLocalStorage<AbortSignal>();
/** Test seam (`_setPublicationTimingForTest`): a ceiling and grace below the accepted range, so a test can overrun the ceiling. */
let testTiming: { ceilingMs: number; graceMs: number } | undefined;
/** The engine's ceiling (cached per engine for 5 s); anything missing or outside 60000..3600000 reads as the default. */
export async function readPublicationCeilingMs(engine: Pick<BrainEngine, 'getConfig'>): Promise<number> {
if (testTiming) return testTiming.ceilingMs;
const key = engineKey(engine);
const cached = ceilings.get(key);
if (cached && Date.now() - cached.at < CEILING_TTL_MS) return cached.value;
const n = Number((await engine.getConfig(PUBLICATION_CEILING_KEY).catch(() => null))?.trim());
const value = Number.isInteger(n) && n >= 60_000 && n <= 3_600_000 ? n : PUBLICATION_CEILING_DEFAULT_MS;
ceilings.set(key, { value, at: Date.now() });
return value;
}
/** The ceiling `engine` (or the engine its transaction runs on) last read, without a read; the default before the first. */
function lastReadCeilingMs(engine: object): number {
return ceilings.get(engineKey(engine))?.value ?? PUBLICATION_CEILING_DEFAULT_MS;
}
/** Test seam: the next readPublicationCeilingMs reads the config again. */
export function _resetPublicationCeilingForTest(): void {
ceilings = new WeakMap();
}
/** Test seam: every publication's ceiling and grace (undefined restores the configured ceiling and `PUBLICATION_GRACE_MS`). */
export function _setPublicationTimingForTest(timing: { ceilingMs: number; graceMs: number } | undefined): void { testTiming = timing; }
/**
* Large-pages hardening (c): whether the current publication (the `boundPublication` this async context runs in) was
* cancelled at the ceiling. transient-retry.ts counts such a publication's failure against `persistence.max_transient_retries`.
*/
export function publicationDeadlinePassed(): boolean {
const signal = signals.getStore();
return !!signal?.aborted && (signal.reason as { code?: unknown } | null)?.code === 'publication_deadline';
}
/** The idle-in-transaction bound a publish transaction gets: the ceiling, never above the pool's own session value. */
export function publicationIdleTimeoutMs(ceilingMs: number): number {
const session = gucMilliseconds(resolveSessionTimeouts().idle_in_transaction_session_timeout);
return session !== null && session > 0 ? Math.min(ceilingMs, session) : ceilingMs;
}
/**
* Sets the transaction-local idle bound. Callers await it (or send it first in their `pipelined` batch), so a refused SET
* fails this publication's transaction with its own error. Skipped on PGLite: it would enforce the bound, and a
* transaction it ends there wedges the engine's only connection.
*/
export async function bindPublicationTimeouts(tx: BrainEngine): Promise<void> {
if (tx.kind !== 'postgres') return;
await tx.executeRaw("SELECT set_config('idle_in_transaction_session_timeout',$1,true)", [`${publicationIdleTimeoutMs(lastReadCeilingMs(tx))}ms`]);
}
/**
* #6712 L6: the publication's opening statement: the idle bound above (Postgres only, for the same reason) and
* `declareDurablePersistence` in one round trip. A refused SET fails the publication's transaction as above.
*/
export async function declarePublication(tx: BrainEngine, bound: Pick<BrainEngine, 'kind'> = tx): Promise<string | null> {
return declareDurablePersistence(tx, undefined, undefined, bound.kind === 'postgres' ? `${publicationIdleTimeoutMs(lastReadCeilingMs(tx))}ms` : undefined);
}
/**
* `engine.transaction`, passing the current publication's signal on as `{ signal }`. An aborted transaction rejects
* while its body may still be parked, so a rejection waits for the body to settle first: nothing (recovery included)
* acts on the publication while the body can still rename a file. `pipelineFirst` (GBRA-156) is for a body that opens
* with `declarePublication` and does nothing that needs BEGIN answered first; a lane group's body does (its begun mark
* orders the next group's BEGIN), so the group runner passes none.
*/
export async function publicationTransaction<T>(engine: BrainEngine, fn: (tx: BrainEngine) => Promise<T>,
{ pipelineFirst }: Pick<TransactionOptions, 'pipelineFirst'> = {}): Promise<T> {
const signal = signals.getStore();
const first = pipelineFirst ? { pipelineFirst } : {};
if (!signal) return pipelineFirst ? engine.transaction(fn, first) : engine.transaction(fn);
let body: Promise<unknown> = Promise.resolve();
try {
return await engine.transaction(tx => { const run = fn(tx); body = run; return run; }, { signal, ...first });
} catch (error) {
await body.then(() => undefined, () => undefined);
const closed = (error as { cause?: { code?: unknown } })?.cause?.code;
// The deadline discarded the connection: report it as the closed connection it is, which callers already retry.
if (!signal.aborted || typeof closed !== 'string') throw error;
throw Object.assign(new Error(`publication_deadline: the publish transaction passed the ceiling and its connection was discarded (${closed})`),
{ name: 'PublicationDeadlineError', code: closed, cause: error });
}
}
export interface PublicationBound {
ceilingMs: number;
graceMs?: number;
settleMs?: number;
/** At the ceiling, with the publish still running. */
onOverdue?(): void;
/** Past the ceiling, the grace and the settle window: the owner must restart. `settled` resolves when the publish finally does. */
onStuck?(settled: Promise<void>): void;
/** Scheduler seam: run `fn` once after `ms`, returning its cancel. Default: a real timer that does not hold the process open. Tests drive a manual clock through it. */
later?(ms: number, fn: () => void): () => void;
}
function realTimer(ms: number, fn: () => void): () => void {
const timer = setTimeout(fn, ms);
timer.unref?.();
return () => clearTimeout(timer);
}
/** Runs `publish` under the ceiling; settles only with `publish`'s own outcome. */
export function boundPublication<T>(publish: () => Promise<T>, bound: PublicationBound): Promise<T> {
const controller = new AbortController();
const work = signals.run(controller.signal, publish);
const settled = work.then(() => undefined, () => undefined);
const schedule = bound.later ?? realTimer;
const cancels: Array<() => void> = [];
const later = (ms: number, fn: () => void) => { cancels.push(schedule(ms, fn)); };
const grace = bound.graceMs ?? testTiming?.graceMs ?? PUBLICATION_GRACE_MS, settle = bound.settleMs ?? PUBLICATION_SETTLE_MS;
later(bound.ceilingMs, () => bound.onOverdue?.());
later(bound.ceilingMs + grace, () => controller.abort(Object.assign(new Error('publication_deadline'), { code: 'publication_deadline' })));
later(bound.ceilingMs + grace + settle, () => bound.onStuck?.(settled));
void settled.then(() => { for (const cancel of cancels) cancel(); });
return work;
}