Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Typed retry-eligible-error predicates (v0.30.1, finding C4).
*
* Three v0.30.1 sites need to decide whether to retry on a given error:
* - db.ts:connectWithRetry (existing — auth, conn-refused, ECONNRESET)
* - migrate.ts retry wrapper (statement_timeout 57014 + conn-reset)
* - backfill-base.ts adaptive retry (statement_timeout 57014 + conn drop)
*
* Before this module these predicates lived inline at each site and drifted
* over time. One source of truth here; new call sites import the typed
* helper instead of pattern-matching the same regexes again.
*
* TWO-AXIS DESIGN NOTE (do not "fix" one side to match the other):
* this module answers "should I retry?" — and deliberately treats
* `password authentication failed` as RETRYABLE (auth race during DNS
* failover). The REASON classifier (`src/core/pg-access-classify.ts`)
* answers "what went wrong and what fixes it?" — and reports the same
* error as `auth_failed` with `transient: false` (a persistently wrong
* password is not transient). Both are correct on their own axis.
* This module's only exports consumed there are the predicates plus the
* `getCode`/`getMessage` shape helpers; no new patterns land here.
*/
const CONN_PATTERNS = [
/password authentication failed/i,
/connection refused/i,
/the database system is starting up/i,
/Connection terminated unexpectedly/i,
/ECONNRESET/i,
/connection.*closed/i,
/server closed the connection/i,
/could not connect to server/i,
// v0.41.2.1: gbrain's own GBrainError thrown by getConnection() when
// the singleton pool was nulled (engine.disconnect mid-cycle, or
// postgres.js's auto-recovery between queries). Matches the literal
// message shape from PR #1416's reported batch-loss incident.
/No database connection/i,
// v0.42.5.0 (issue #1678): postgres.js throws errors carrying
// `code: 'CONNECTION_ENDED'` (a LIBRARY code, not an 08xxx SQLSTATE) when a
// transaction-mode pooler reaps an idle socket between queries. Without an
// explicit match it was only accidentally caught by /connection.*closed/i.
// Match the message form too for wrappers that fold the code into the text.
/CONNECTION_ENDED/i,
// v0.42.x (#1794): Supavisor transaction-pooler session exhaustion. When all
// upstream connections are checked out the pooler rejects new sessions
// ("MaxClientsInSessionMode" / EMAXCONNSESSION) and Postgres raises SQLSTATE
// 53300 (too_many_connections). Both are transient under load — without a
// retry, the checkpoint write (and every other pool-contending write) is
// dropped during the exact spike #1794's resumable sync must survive.
/EMAXCONNSESSION/i,
// #6340: the socket errnos postgres.js surfaces as "write <ERRNO> host:port" when a
// session-mode pooler drops the connection mid-statement (ECONNABORTED was the one
// that ended four managed catch-up runs in one afternoon). Same class as ECONNRESET.
/ECONNABORTED/i,
/ETIMEDOUT/i,
/EPIPE/i,
/too many clients already/i,
/max.*clients?.*in session mode/i,
/remaining connection slots are reserved/i,
];
interface PgError {
code?: string;
message?: string;
cause?: unknown;
// v0.41.2.1: gbrain's GBrainError uses `problem` (typed) + `detail` so
// callers can switch on the engine-state class without string matching.
problem?: string;
}
export function getCode(err: unknown): string | undefined {
if (err && typeof err === 'object') {
const code = (err as PgError).code;
if (typeof code === 'string') return code;
}
return undefined;
}
/** Socket errnos and postgres.js library codes that mean an open connection went away: decided by the code, never the message. */
const CONNECTION_CODES: ReadonlySet<string> = new Set(['ECONNRESET', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE', 'CONNECTION_CLOSED', 'CONNECTION_ENDED', 'CONNECTION_DESTROYED']);
/** True for a socket errno or a postgres.js connection code. */
export function isConnectionCode(code: string | undefined): boolean {
return code !== undefined && CONNECTION_CODES.has(code);
}
/**
* GBRA-133 S-70: does this message name a path? Connection text is only read from a message that names none, so an
* unexpected error quoting a file such as `notes/connection closed.md` is never taken for a dropped connection.
* URLs (a connection string, an endpoint) are not paths.
*/
export function quotesPath(message: string): boolean {
return /[\\/]/.test(message.replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, ''));
}
export function getMessage(err: unknown): string {
if (err instanceof Error) return err.message;
if (err && typeof err === 'object') {
const msg = (err as PgError).message;
if (typeof msg === 'string') return msg;
}
return String(err ?? '');
}
/**
* GST-2314: a prepared statement whose cached plan went stale because a migration changed a table it reads (SQLSTATE
* 0A000 "cached plan must not change result type"). Outside a transaction the driver re-prepares it; inside one the
* transaction is already aborted, so the caller sees the error and a re-run on a fresh transaction succeeds. Every other
* 0A000 (feature_not_supported) stays a refusal. The C routine (RevalidateCachedQuery) is matched first because the
* server translates the message under a non-English lc_messages.
*/
export function isStalePlanError(err: unknown): boolean {
if (getCode(err) !== '0A000') return false;
return (err as { routine?: unknown }).routine === 'RevalidateCachedQuery' || /cached plan must not change result type/i.test(getMessage(err));
}
/**
* SQLSTATE 57014: query_canceled / statement_timeout.
* Postgres signals this when a statement exceeds `statement_timeout`.
*/
export function isStatementTimeoutError(err: unknown): boolean {
if (getCode(err) === '57014') return true;
const msg = getMessage(err);
return /statement_timeout|canceling statement due to statement timeout/i.test(msg);
}
/**
* GST-2260: runs `transaction` once more when a stale plan aborted it. The abort rolled back every statement it ran, so
* the second run is safe only for a callback that does nothing but database work and in-memory computation: never wrap
* one that writes a file, calls the network or changes state that outlives the transaction.
*/
export async function retryOnStalePlan<T>(transaction: () => Promise<T>): Promise<T> {
try {
return await transaction();
} catch (error) {
if (!isStalePlanError(error)) throw error;
return transaction();
}
}
/**
* SQLSTATE 55P03: lock_not_available.
* Postgres signals this when `lock_timeout` or `NOWAIT` would block.
*/
export function isLockTimeoutError(err: unknown): boolean {
if (getCode(err) === '55P03') return true;
const msg = getMessage(err);
return /lock_not_available|could not obtain lock/i.test(msg);
}
/**
* Connection-level errors that are typically transient: TCP resets,
* pooler restarts, server-starting-up, auth race during DNS failover.
* Distinguish from statement_timeout / lock_timeout via the dedicated
* predicates above.
*/
export function isRetryableConnError(err: unknown): boolean {
// Statement / lock timeouts are NOT connection errors. Callers that
// want to retry on those use isStatementTimeoutError / isLockTimeoutError
// explicitly so they can apply different backoff (e.g. backfill halves
// batch size on stmt timeout but reconnects on conn drop).
if (isStatementTimeoutError(err) || isLockTimeoutError(err)) return false;
const code = getCode(err);
// Postgres connection-level codes:
// 08000 connection_exception
// 08003 connection_does_not_exist
// 08006 connection_failure
// 08001 sqlclient_unable_to_establish_sqlconnection
// 08004 sqlserver_rejected_establishment_of_sqlconnection
if (code && /^08/.test(code)) return true;
// v0.42.5.0 (issue #1678): postgres.js's library-level connection-ended
// code. Not an 08xxx SQLSTATE, so the /^08/ test above misses it.
if (code === 'CONNECTION_ENDED') return true;
// issue #1720: postgres.js also throws code 'CONNECTION_CLOSED' when the
// pooler closes the socket mid-query ("write CONNECTION_CLOSED host:port").
// The message form is already caught by /connection.*closed/i below; match
// the code too for wrappers that rethrow with the code but a new message.
if (code === 'CONNECTION_CLOSED') return true;
if (isConnectionCode(code)) return true;
// v0.42.x (#1794): SQLSTATE 53300 too_many_connections — pool/pooler
// exhaustion. Starts with 53 not 08, so the /^08/ test above misses it.
// Transient: the spike clears as in-flight queries release connections.
if (code === '53300') return true;
// v0.41.2.1: typed-shape match for gbrain's own GBrainError
// (problem === 'No database connection'). Avoids brittle string match
// when the error wrapper is gbrain-internal.
if (
err && typeof err === 'object' &&
(err as PgError).problem === 'No database connection'
) {
return true;
}
const msg = getMessage(err);
return !quotesPath(msg) && CONN_PATTERNS.some(p => p.test(msg));
}
/**
* #9540: a connect to a local Unix socket that is gone, because the server shut down and removed it (`connect ENOENT
* <dir>/.s.PGSQL.<port>`). Kept out of `isRetryableConnError`: its first-connect callers would then wait out
* retries on a mistyped socket path. A long-running loop that already reached the server treats it as a lost connection.
*/
export function isVanishedSocketError(err: unknown): boolean {
return getCode(err) === 'ENOENT' && (err as { syscall?: unknown }).syscall === 'connect';
}
/**
* #9540 follow-up: a server that refuses new connections because it is shutting down (SQLSTATE 57P03) or has stopped
* listening on TCP (`connect ECONNREFUSED`). Kept out of `isRetryableConnError` for the reason
* `isVanishedSocketError` is: a long-running loop that already reached the server treats it as a lost connection.
*/
export function isServerRefusingConnections(err: unknown): boolean {
const code = getCode(err);
return code === '57P03' || (code === 'ECONNREFUSED' && (err as { syscall?: unknown }).syscall === 'connect');
}
/**
* #6355: did the database session go away under a statement that was already sent? Narrower than
* `isRetryableConnError` (no auth or startup races): the connection-class SQLSTATEs, postgres.js's own
* connection codes, the server's admin/crash shutdown codes (`pg_terminate_backend`, a failover) and the
* socket errnos. Such an error says nothing about whether the statement's transaction committed, so a
* caller that sent a write must re-read its retained request id instead of inferring rollback.
*/
export function isConnectionLoss(err: unknown): boolean {
const code = getCode(err);
if (code && (/^08/.test(code) || isConnectionCode(code) || ['57P01', '57P02', '57P03'].includes(code))) return true;
const msg = getMessage(err);
return !quotesPath(msg) && /\b(ECONNRESET|ECONNABORTED|ETIMEDOUT|EPIPE)\b|Connection terminated unexpectedly|server closed the connection|connection.*closed|terminating connection due to administrator command/i.test(msg);
}
/**
* issue #1685 (CODEX #8): is this error specifically a POOLER REAP — postgres.js's
* library-level `CONNECTION_ENDED` code (the transaction-mode pooler dropping an
* idle socket between ticks)? Narrower than `isRetryableConnError`, which also
* matches 08xxx SQLSTATEs, network blips, and auth races. Used by
* `PostgresEngine.reconnect()` to label the pool-recovery audit honestly so a
* generic reconnect isn't mis-recorded as a reap.
*/
export function isConnectionEndedError(err: unknown): boolean {
const code = getCode(err);
if (code === 'CONNECTION_ENDED') return true;
const msg = getMessage(err);
return /CONNECTION_ENDED/i.test(msg);
}
/**
* Did postgres.js abandon a connection handshake because its own
* `connect_timeout` timer fired? It reports that as code `CONNECT_TIMEOUT`
* with the message "write CONNECT_TIMEOUT <host>:<port>" (the host and port
* read "undefined:undefined" once the socket has been upgraded to TLS).
*
* Kept OUT of `isRetryableConnError` on purpose: on a process's first connect
* a handshake timeout usually means the host does not route, which
* pg-access-classify reports as `network_unreachable`, and retrying cannot
* fix that. A caller that has already reached the same URL in this process
* opts in through connectWithRetry's `retryConnectTimeout`.
*/
export function isConnectTimeoutError(err: unknown): boolean {
if (getCode(err) === 'CONNECT_TIMEOUT') return true;
return /\bCONNECT_TIMEOUT\b/.test(getMessage(err));
}
/**
* Convenience: is this error retryable for ANY reason (connection drop OR
* statement timeout)? Backfill uses this — callers that need finer-grained
* dispatch (different backoff per kind) call the dedicated predicates.
*/
export function isRetryableError(err: unknown): boolean {
return isRetryableConnError(err) || isStatementTimeoutError(err);
}