Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Retry a transaction a stale cached plan aborted, where re-running it is safe
garrytanb32ea051767b
diff --git a/src/core/minions/queue.ts b/src/core/minions/queue.tsindex 8e5302aa23..7b265eccfa 100644--- a/src/core/minions/queue.ts+++ b/src/core/minions/queue.ts@@ -46,6 +46,7 @@ import {logBatchRetry as auditLogBatchRetry,logBatchExhausted as auditLogBatchExhausted,} from '../audit/batch-retry-audit.ts';+import { retryOnStalePlan } from '../retry-matcher.ts';/** Trusted 4th argument, kept outside user-spread job options. */export interface TrustedSubmitOpts {submissionAuthority?: SubmissionAuthority;@@ -1716,7 +1717,8 @@ export class MinionQueue {* stranding the parent in waiting-children forever.*/async completeJob(id: number, lockToken: string, result?: Record<string, unknown>): Promise<MinionJob | null> {- return this.engine.transaction(async (tx) => {+ // GST-2260: database work only, so a completion a stale plan aborted runs again instead of failing a job that ran.+ return retryOnStalePlan(() => this.engine.transaction(async (tx) => {// Peek at parent_job_id before the UPDATE so we can lock the parent row// FIRST. Without this SELECT FOR UPDATE, two siblings completing// concurrently each see the other as still active (pre-commit snapshot@@ -1806,7 +1808,7 @@ export class MinionQueue {}return completed;- });+ }));}/**@@ -1831,7 +1833,8 @@ export class MinionQueue {backoffMs?: number,result?: Record<string, unknown>,): Promise<MinionJob | null> {- return this.engine.transaction(async (tx) => {+ // GST-2260: database work only, so a failure record a stale plan aborted runs again.+ return retryOnStalePlan(() => this.engine.transaction(async (tx) => {// Lock the parent row first so concurrent sibling completions/failures// serialize on the parent — same race fix as completeJob.const peek = await tx.executeRaw<{ parent_job_id: number | null }>(@@ -1943,7 +1946,7 @@ export class MinionQueue {}return failed;- });+ }));}/**diff --git a/src/core/oauth-grants.ts b/src/core/oauth-grants.tsindex 0b4dacd080..83a4a67a16 100644--- a/src/core/oauth-grants.ts+++ b/src/core/oauth-grants.ts@@ -9,6 +9,7 @@ import { generateToken, hashToken } from './utils.ts';import { hasScope, parseScopeString } from './scope.ts';import { TOKEN_TTL_MAX_SECONDS, intersectGrantedScopes } from './grants/model.ts';import { safeHexEqual } from './timing-safe.ts';+import { retryOnStalePlan } from './retry-matcher.ts';import { clientHoldsAuthorization, GrantError, recordFirstConsentInTransaction } from './grants/service.ts';export type OAuthTransaction = <T>(fn: (sql: SqlQuery) => Promise<T>) => Promise<T>;@@ -126,12 +127,14 @@ export class OAuthGrants {private now(): number { return (this.options.now ?? Date.now)(); }private async locked<T>(clientId: string, fn: (sql: SqlQuery, row: ClientRow) => Promise<T>): Promise<T> {- if (!this.options.transaction) throw new ServerError('OAuth grant transactions are unavailable');- return this.options.transaction(async sql => {+ const options = this.options;+ if (!options.transaction) throw new ServerError('OAuth grant transactions are unavailable');+ // GST-2260: every caller's callback is database work only, so a transaction a stale plan aborted runs again.+ return retryOnStalePlan(() => options.transaction!(async sql => {const [row] = await sql`SELECT * FROM oauth_clients WHERE client_id = ${clientId} FOR UPDATE`;assertActive(row);return fn(sql, row);- });+ }));}private prune(): void {diff --git a/src/core/persistence/admission-retry.ts b/src/core/persistence/admission-retry.tsindex a302c15c37..4bf57e47cf 100644--- a/src/core/persistence/admission-retry.ts+++ b/src/core/persistence/admission-retry.ts@@ -1,7 +1,7 @@import { AsyncLocalStorage } from 'node:async_hooks';import { setTimeout as delay } from 'node:timers/promises';import { markNotAdmitted, OperationError, opError } from '../ops/contract.ts';-import { getCode, isConnectionLoss } from '../retry-matcher.ts';+import { getCode, isConnectionLoss, isStalePlanError } from '../retry-matcher.ts';/** #6355: how many times an admission attempt is re-run after the session dropped under it, and the pauses before each re-run. */export const CONNECTION_LOSS_RETRY_MS = [100, 300, 900] as const;@@ -50,7 +50,7 @@ export async function retryWriteAdmission<T>(requestId: string, attempt: (remainawait recover?.(error).catch(() => undefined);continue;}- if (!['40001', '40P01', '55P03', '57014'].includes(code ?? '')) throw error;+ if (!['40001', '40P01', '55P03', '57014'].includes(code ?? '') && !isStalePlanError(error)) throw error;const remaining = deadline - performance.now();if (remaining <= 25) {const unavailable = new OperationError('storage_error', 'Write admission is temporarily blocked by database contention.',diff --git a/src/core/persistence/coordinator.ts b/src/core/persistence/coordinator.tsindex cef9d0a01c..ff3a761933 100644--- a/src/core/persistence/coordinator.ts+++ b/src/core/persistence/coordinator.ts@@ -57,6 +57,7 @@ import { segmentWriteScope } from '../page-body/ingest.ts';import { publicationBeliefOptions, withBeliefBatch } from '../belief/ledger.ts';import { markBeliefCommits } from '../belief/append.ts';import { classifyPgAccessError } from '../pg-access-classify.ts';+import { isStalePlanError } from '../retry-matcher.ts';import { commitThroughDoor, doorCommitLanded, doorMarker, replayAfterDoorCommit, sourceCanonicalGit, strictChanged } from './git-door.ts';import { assertNoPrivateCommit, declareStrictPublication } from './git-declared.ts';@@ -255,7 +256,8 @@ function conflictCode(code: string): boolean { return ['revision_required','revifunction failureState(code: string): 'conflict' | 'failed' | 'cancelled' { return code === 'admission_paused' ? 'cancelled' : conflictCode(code) ? 'conflict' : 'failed'; }export function transientDatabaseFailure(error: unknown): boolean {return ['40001','40P01','55P03','57014','53300','57P01','57P02','57P03','08000','08003','08006','08001','08004',- 'ECONNRESET','ECONNREFUSED','ETIMEDOUT','CONNECTION_CLOSED','CONNECTION_ENDED'].includes(String((error as {code?:string})?.code)) || connectionCapFailure(error);+ 'ECONNRESET','ECONNREFUSED','ETIMEDOUT','CONNECTION_CLOSED','CONNECTION_ENDED'].includes(String((error as {code?:string})?.code)) || connectionCapFailure(error)+ || isStalePlanError(error);}/*** GBRA-133: a pooler's client cap (Supavisor `EMAXCONNSESSION`, "too many clients already") is the same contention asdiff --git a/src/core/retry-matcher.ts b/src/core/retry-matcher.tsindex 05d198d509..41ee72f347 100644--- a/src/core/retry-matcher.ts+++ b/src/core/retry-matcher.ts@@ -112,6 +112,31 @@ export function isStatementTimeoutError(err: unknown): boolean {return /statement_timeout|canceling statement due to statement timeout/i.test(msg);}+/**+ * GST-2260: SQLSTATE 0A000 from RevalidateCachedQuery, "cached plan must not change result type": another session+ * changed a column of a table a prepared statement reads, so its cached result type no longer holds. The vendored+ * driver re-prepares every connection's statements after the first one, so the statement succeeds when it runs again;+ * inside a transaction that means re-running the whole transaction, which this error has already rolled back. Matched+ * on the routine, never the code alone: other 0A000 errors (an unsupported feature) are permanent.+ */+export function isStalePlanError(err: unknown): boolean {+ return getCode(err) === '0A000' && (err as { routine?: unknown } | null)?.routine === 'RevalidateCachedQuery';+}++/**+ * GST-2260: runs `transaction` once more when a stale plan aborted it. The abort rolled back every statement it ran, so+ * the second run is safe only for a callback that does nothing but database work and in-memory computation: never wrap+ * one that writes a file, calls the network or changes state that outlives the transaction.+ */+export async function retryOnStalePlan<T>(transaction: () => Promise<T>): Promise<T> {+ try {+ return await transaction();+ } catch (error) {+ if (!isStalePlanError(error)) throw error;+ return transaction();+ }+}+/*** SQLSTATE 55P03: lock_not_available.* Postgres signals this when `lock_timeout` or `NOWAIT` would block.