Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): bound the idle-in-transaction timeout in the proactive refresh cron (R3-L1)

Diego Zacarias32a4cadd2fbf
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 367afda39..c72a36360 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.655",
+ "version": "0.1.656",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.655",
+ "version": "0.1.656",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 924123efd..f9b771b36 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.655",
+ "version": "0.1.656",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/oauth/store.ts b/control-api/src/oauth/store.ts
index b65037f49..74e13565c 100644
--- a/control-api/src/oauth/store.ts
+++ b/control-api/src/oauth/store.ts
@@ -806,10 +806,12 @@ function remoteGrantKeyFromRow(row: RemoteGrantWindowDbRow): OAuthGrantKey {
* via `refreshGenericGrant`), so an unattended one has the same silent-expiry risk
* and belongs in the sweep (R1-L1). The DCR secret-expiry sweep stays remote-only
* — dynamic clients are a remote-lane concept — so this function keeps its name.
- * A snapshot SELECT with NO lock — the cron re-claims each row under
- * `FOR UPDATE SKIP LOCKED` in its own short transaction
- * ({@link claimRemoteGrantForRefresh}) so no pool connection is pinned across the
- * refresh POSTs.
+ * A snapshot SELECT with NO lock — this enumeration does not hold a connection
+ * across the whole sweep; the cron re-claims each row under `FOR UPDATE SKIP
+ * LOCKED` in its OWN short transaction ({@link claimRemoteGrantForRefresh}), and
+ * that per-row transaction DOES pin its connection across ITS refresh POST,
+ * bounded by the carrier idle-in-transaction timeout
+ * (`boundOAuthRefreshLockIdleTimeout`).
*
* Filter (§4 eligibility + window): `provider IN ('remote','generic')`, non-null
* expiry in `(now+Br, now+Bp]`, and `grant_kind='shared' OR (grant_kind='user' AND
diff --git a/control-api/src/services/oauthProactiveRefreshCron.ts b/control-api/src/services/oauthProactiveRefreshCron.ts
index 07450ada7..685a38bda 100644
--- a/control-api/src/services/oauthProactiveRefreshCron.ts
+++ b/control-api/src/services/oauthProactiveRefreshCron.ts
@@ -22,8 +22,8 @@
* (`OAUTH_PROACTIVE_REFRESH_CRON_ENABLED`).
*/
import { config } from '../config.js'
-import type { DbClient } from '../db.js'
-import { pool, withTransaction } from '../db.js'
+import type { DbClient, DbTransactionClient } from '../db.js'
+import { boundOAuthRefreshLockIdleTimeout, pool, withTransaction } from '../db.js'
import type { K8sGateway } from '../k8s.js'
import {
RecipeNotFoundError,
@@ -78,8 +78,13 @@ export interface ProactiveRefreshSweepOptions {
export interface ProactiveRefreshSweepDeps {
/** Snapshot reads (candidate enumeration + DCR list) — no lock, no txn. */
db: DbClient
- /** Runs `work` inside one short transaction (BEGIN…COMMIT/ROLLBACK). */
- runInTransaction: <T>(work: (txDb: DbClient) => Promise<T>) => Promise<T>
+ /**
+ * Runs `work` inside one short transaction (BEGIN…COMMIT/ROLLBACK). The client
+ * is transaction-scoped (branded {@link DbTransactionClient}) so the sweep can
+ * bound the carrier's idle timeout across the in-transaction refresh POST, same
+ * as the reactive path.
+ */
+ runInTransaction: <T>(work: (txDb: DbTransactionClient) => Promise<T>) => Promise<T>
encryptionKey: Buffer
fetchFn: typeof fetch
}
@@ -203,6 +208,12 @@ export async function runProactiveRefreshSweep(
// reactive path already renewed it out of the window since the snapshot.
const claimed = await claimRemoteGrantForRefresh(txDb, key, window)
if (!claimed) return 'skipped' as const
+ // The refresh POST runs INSIDE this transaction while it holds the row
+ // lock, so bound the idle-in-transaction tenancy just like the reactive
+ // path: `withTransaction` sets no such timeout, and if the AS hangs mid
+ // POST neither statement_timeout (no statement is running) nor the
+ // transport abort guarantees the lock + pool connection is released.
+ await boundOAuthRefreshLockIdleTimeout(txDb)
// requireBackground is PER-grantKind, not fixed. SEC-5 governs `user`
// grants only (unattended reuse needs explicit background consent), so
// those keep `true` — redundant with the enumeration filter, kept as
diff --git a/control-api/test/services.oauthProactiveRefreshCron.test.ts b/control-api/test/services.oauthProactiveRefreshCron.test.ts
index fa61d6222..9bbd76adf 100644
--- a/control-api/test/services.oauthProactiveRefreshCron.test.ts
+++ b/control-api/test/services.oauthProactiveRefreshCron.test.ts
@@ -9,7 +9,8 @@
* by the real-Postgres suite and the reactive `tokenHelper` tests.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import type { DbClient } from '../src/db.js'
+import type { DbClient, DbTransactionClient } from '../src/db.js'
+import { OAUTH_REFRESH_LOCK_CARRIER_IDLE_TIMEOUT_MS } from '../src/db.js'
import type { K8sGateway } from '../src/k8s.js'
import type { ExpiringDynamicClient } from '../src/oauth/dynamicClientStore.js'
import { listExpiringDynamicClients } from '../src/oauth/dynamicClientStore.js'
@@ -52,9 +53,13 @@ vi.mock('../src/oauth/tokenHelper.js', async () => {
const OPTS = { proactiveBufferMs: 300_000, reactiveBufferMs: 60_000, dcrWarnMs: 604_800_000 }
const gateway = {} as K8sGateway
-const fakeTx: DbClient = { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) }
+// Transaction-scoped spy client: its `.query` records every statement the sweep
+// issues inside the refresh transaction (e.g. the SET LOCAL idle timeout). The
+// brand is nominal-only, so the cast is the test's blessed mint point.
+const fakeTxQuery = vi.fn(async () => ({ rows: [], rowCount: 0 }))
+const fakeTx = { query: fakeTxQuery } as unknown as DbTransactionClient
const deps: ProactiveRefreshSweepDeps = {
- db: { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) },
+ db: { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) } as DbClient,
runInTransaction: work => work(fakeTx),
encryptionKey: Buffer.alloc(32),
fetchFn: vi.fn() as unknown as typeof fetch,
@@ -122,6 +127,34 @@ describe('runProactiveRefreshSweep — token candidates', () => {
)
})
+ it('bounds the carrier idle timeout inside the refresh tx, BEFORE the refresh POST', async () => {
+ // Defense-in-depth symmetry with the reactive path (reactiveTokenHelper.ts):
+ // the refresh POST runs inside the transaction that holds the row lock, so
+ // the sweep MUST emit `SET LOCAL idle_in_transaction_session_timeout` on the
+ // tx client before handing off to getAccessToken (the POST). Assert the
+ // observable statement on the tx client + its ordering, not an internal spy.
+ // Fails against the pre-fix head, which never emitted this statement.
+ mockedEnumerate.mockResolvedValue([sharedKey('srv-a', 'ctx-1')])
+ await runProactiveRefreshSweep(gateway, OPTS, deps)
+
+ const idleCallIdx = fakeTxQuery.mock.calls.findIndex(
+ ([text]) =>
+ typeof text === 'string' &&
+ text.includes("set_config('idle_in_transaction_session_timeout'")
+ )
+ expect(idleCallIdx).toBeGreaterThanOrEqual(0)
+ // SET LOCAL (third positional arg true) so it reverts on COMMIT/ROLLBACK.
+ expect(fakeTxQuery.mock.calls[idleCallIdx][0]).toContain('$1, true)')
+ expect(fakeTxQuery.mock.calls[idleCallIdx][1]).toEqual([
+ String(OAUTH_REFRESH_LOCK_CARRIER_IDLE_TIMEOUT_MS),
+ ])
+ // Emitted BEFORE the refresh POST (getAccessToken) — else the lock+connection
+ // is unbounded across a hung AS, the exact gap this fix closes.
+ expect(fakeTxQuery.mock.invocationCallOrder[idleCallIdx]).toBeLessThan(
+ mockedGetToken.mock.invocationCallOrder[0]
+ )
+ })
+
it('B1: requireBackground is per-grantKind — false for shared (context identity), true for user', async () => {
// B1 regression: passing requireBackground:true for shared keys makes
// getOAuthGrant's shared branch add `AND background = true` and return