Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Retry a transaction a stale cached plan aborted, where re-running it is safe

garrytanb32ea051767b
diff --git a/src/core/minions/queue.ts b/src/core/minions/queue.ts
index 8e5302aa23..7b265eccfa 100644
--- a/src/core/minions/queue.ts
+++ b/src/core/minions/queue.ts
@@ -46,6 +46,7 @@ import {
logBatchRetry as auditLogBatchRetry,
logBatchExhausted as auditLogBatchExhausted,
} from '../audit/batch-retry-audit.ts';
+import { retryOnStalePlan } from '../retry-matcher.ts';
/** Trusted 4th argument, kept outside user-spread job options. */
export interface TrustedSubmitOpts {
submissionAuthority?: SubmissionAuthority;
@@ -1716,7 +1717,8 @@ export class MinionQueue {
* stranding the parent in waiting-children forever.
*/
async completeJob(id: number, lockToken: string, result?: Record<string, unknown>): Promise<MinionJob | null> {
- return this.engine.transaction(async (tx) => {
+ // GST-2260: database work only, so a completion a stale plan aborted runs again instead of failing a job that ran.
+ return retryOnStalePlan(() => this.engine.transaction(async (tx) => {
// Peek at parent_job_id before the UPDATE so we can lock the parent row
// FIRST. Without this SELECT FOR UPDATE, two siblings completing
// concurrently each see the other as still active (pre-commit snapshot
@@ -1806,7 +1808,7 @@ export class MinionQueue {
}
return completed;
- });
+ }));
}
/**
@@ -1831,7 +1833,8 @@ export class MinionQueue {
backoffMs?: number,
result?: Record<string, unknown>,
): Promise<MinionJob | null> {
- return this.engine.transaction(async (tx) => {
+ // GST-2260: database work only, so a failure record a stale plan aborted runs again.
+ return retryOnStalePlan(() => this.engine.transaction(async (tx) => {
// Lock the parent row first so concurrent sibling completions/failures
// serialize on the parent — same race fix as completeJob.
const peek = await tx.executeRaw<{ parent_job_id: number | null }>(
@@ -1943,7 +1946,7 @@ export class MinionQueue {
}
return failed;
- });
+ }));
}
/**
diff --git a/src/core/oauth-grants.ts b/src/core/oauth-grants.ts
index 0b4dacd080..83a4a67a16 100644
--- a/src/core/oauth-grants.ts
+++ b/src/core/oauth-grants.ts
@@ -9,6 +9,7 @@ import { generateToken, hashToken } from './utils.ts';
import { hasScope, parseScopeString } from './scope.ts';
import { TOKEN_TTL_MAX_SECONDS, intersectGrantedScopes } from './grants/model.ts';
import { safeHexEqual } from './timing-safe.ts';
+import { retryOnStalePlan } from './retry-matcher.ts';
import { clientHoldsAuthorization, GrantError, recordFirstConsentInTransaction } from './grants/service.ts';
export type OAuthTransaction = <T>(fn: (sql: SqlQuery) => Promise<T>) => Promise<T>;
@@ -126,12 +127,14 @@ export class OAuthGrants {
private now(): number { return (this.options.now ?? Date.now)(); }
private async locked<T>(clientId: string, fn: (sql: SqlQuery, row: ClientRow) => Promise<T>): Promise<T> {
- if (!this.options.transaction) throw new ServerError('OAuth grant transactions are unavailable');
- return this.options.transaction(async sql => {
+ const options = this.options;
+ if (!options.transaction) throw new ServerError('OAuth grant transactions are unavailable');
+ // GST-2260: every caller's callback is database work only, so a transaction a stale plan aborted runs again.
+ return retryOnStalePlan(() => options.transaction!(async sql => {
const [row] = await sql`SELECT * FROM oauth_clients WHERE client_id = ${clientId} FOR UPDATE`;
assertActive(row);
return fn(sql, row);
- });
+ }));
}
private prune(): void {
diff --git a/src/core/persistence/admission-retry.ts b/src/core/persistence/admission-retry.ts
index a302c15c37..4bf57e47cf 100644
--- a/src/core/persistence/admission-retry.ts
+++ b/src/core/persistence/admission-retry.ts
@@ -1,7 +1,7 @@
import { AsyncLocalStorage } from 'node:async_hooks';
import { setTimeout as delay } from 'node:timers/promises';
import { markNotAdmitted, OperationError, opError } from '../ops/contract.ts';
-import { getCode, isConnectionLoss } from '../retry-matcher.ts';
+import { getCode, isConnectionLoss, isStalePlanError } from '../retry-matcher.ts';
/** #6355: how many times an admission attempt is re-run after the session dropped under it, and the pauses before each re-run. */
export const CONNECTION_LOSS_RETRY_MS = [100, 300, 900] as const;
@@ -50,7 +50,7 @@ export async function retryWriteAdmission<T>(requestId: string, attempt: (remain
await recover?.(error).catch(() => undefined);
continue;
}
- if (!['40001', '40P01', '55P03', '57014'].includes(code ?? '')) throw error;
+ if (!['40001', '40P01', '55P03', '57014'].includes(code ?? '') && !isStalePlanError(error)) throw error;
const remaining = deadline - performance.now();
if (remaining <= 25) {
const unavailable = new OperationError('storage_error', 'Write admission is temporarily blocked by database contention.',
diff --git a/src/core/persistence/coordinator.ts b/src/core/persistence/coordinator.ts
index cef9d0a01c..ff3a761933 100644
--- a/src/core/persistence/coordinator.ts
+++ b/src/core/persistence/coordinator.ts
@@ -57,6 +57,7 @@ import { segmentWriteScope } from '../page-body/ingest.ts';
import { publicationBeliefOptions, withBeliefBatch } from '../belief/ledger.ts';
import { markBeliefCommits } from '../belief/append.ts';
import { classifyPgAccessError } from '../pg-access-classify.ts';
+import { isStalePlanError } from '../retry-matcher.ts';
import { commitThroughDoor, doorCommitLanded, doorMarker, replayAfterDoorCommit, sourceCanonicalGit, strictChanged } from './git-door.ts';
import { assertNoPrivateCommit, declareStrictPublication } from './git-declared.ts';
@@ -255,7 +256,8 @@ function conflictCode(code: string): boolean { return ['revision_required','revi
function failureState(code: string): 'conflict' | 'failed' | 'cancelled' { return code === 'admission_paused' ? 'cancelled' : conflictCode(code) ? 'conflict' : 'failed'; }
export function transientDatabaseFailure(error: unknown): boolean {
return ['40001','40P01','55P03','57014','53300','57P01','57P02','57P03','08000','08003','08006','08001','08004',
- 'ECONNRESET','ECONNREFUSED','ETIMEDOUT','CONNECTION_CLOSED','CONNECTION_ENDED'].includes(String((error as {code?:string})?.code)) || connectionCapFailure(error);
+ 'ECONNRESET','ECONNREFUSED','ETIMEDOUT','CONNECTION_CLOSED','CONNECTION_ENDED'].includes(String((error as {code?:string})?.code)) || connectionCapFailure(error)
+ || isStalePlanError(error);
}
/**
* GBRA-133: a pooler's client cap (Supavisor `EMAXCONNSESSION`, "too many clients already") is the same contention as
diff --git a/src/core/retry-matcher.ts b/src/core/retry-matcher.ts
index 05d198d509..41ee72f347 100644
--- a/src/core/retry-matcher.ts
+++ b/src/core/retry-matcher.ts
@@ -112,6 +112,31 @@ export function isStatementTimeoutError(err: unknown): boolean {
return /statement_timeout|canceling statement due to statement timeout/i.test(msg);
}
+/**
+ * GST-2260: SQLSTATE 0A000 from RevalidateCachedQuery, "cached plan must not change result type": another session
+ * changed a column of a table a prepared statement reads, so its cached result type no longer holds. The vendored
+ * driver re-prepares every connection's statements after the first one, so the statement succeeds when it runs again;
+ * inside a transaction that means re-running the whole transaction, which this error has already rolled back. Matched
+ * on the routine, never the code alone: other 0A000 errors (an unsupported feature) are permanent.
+ */
+export function isStalePlanError(err: unknown): boolean {
+ return getCode(err) === '0A000' && (err as { routine?: unknown } | null)?.routine === 'RevalidateCachedQuery';
+}
+
+/**
+ * GST-2260: runs `transaction` once more when a stale plan aborted it. The abort rolled back every statement it ran, so
+ * the second run is safe only for a callback that does nothing but database work and in-memory computation: never wrap
+ * one that writes a file, calls the network or changes state that outlives the transaction.
+ */
+export async function retryOnStalePlan<T>(transaction: () => Promise<T>): Promise<T> {
+ try {
+ return await transaction();
+ } catch (error) {
+ if (!isStalePlanError(error)) throw error;
+ return transaction();
+ }
+}
+
/**
* SQLSTATE 55P03: lock_not_available.
* Postgres signals this when `lock_timeout` or `NOWAIT` would block.