Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): bound the idle-in-transaction timeout in the proactive refresh cron (R3-L1)
Diego Zacarias32a4cadd2fbf
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 367afda39..c72a36360 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.655",+ "version": "0.1.656","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.655",+ "version": "0.1.656","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 924123efd..f9b771b36 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.655",+ "version": "0.1.656","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/oauth/store.ts b/control-api/src/oauth/store.tsindex b65037f49..74e13565c 100644--- a/control-api/src/oauth/store.ts+++ b/control-api/src/oauth/store.ts@@ -806,10 +806,12 @@ function remoteGrantKeyFromRow(row: RemoteGrantWindowDbRow): OAuthGrantKey {* via `refreshGenericGrant`), so an unattended one has the same silent-expiry risk* and belongs in the sweep (R1-L1). The DCR secret-expiry sweep stays remote-only* — dynamic clients are a remote-lane concept — so this function keeps its name.- * A snapshot SELECT with NO lock — the cron re-claims each row under- * `FOR UPDATE SKIP LOCKED` in its own short transaction- * ({@link claimRemoteGrantForRefresh}) so no pool connection is pinned across the- * refresh POSTs.+ * A snapshot SELECT with NO lock — this enumeration does not hold a connection+ * across the whole sweep; the cron re-claims each row under `FOR UPDATE SKIP+ * LOCKED` in its OWN short transaction ({@link claimRemoteGrantForRefresh}), and+ * that per-row transaction DOES pin its connection across ITS refresh POST,+ * bounded by the carrier idle-in-transaction timeout+ * (`boundOAuthRefreshLockIdleTimeout`).** Filter (§4 eligibility + window): `provider IN ('remote','generic')`, non-null* expiry in `(now+Br, now+Bp]`, and `grant_kind='shared' OR (grant_kind='user' ANDdiff --git a/control-api/src/services/oauthProactiveRefreshCron.ts b/control-api/src/services/oauthProactiveRefreshCron.tsindex 07450ada7..685a38bda 100644--- a/control-api/src/services/oauthProactiveRefreshCron.ts+++ b/control-api/src/services/oauthProactiveRefreshCron.ts@@ -22,8 +22,8 @@* (`OAUTH_PROACTIVE_REFRESH_CRON_ENABLED`).*/import { config } from '../config.js'-import type { DbClient } from '../db.js'-import { pool, withTransaction } from '../db.js'+import type { DbClient, DbTransactionClient } from '../db.js'+import { boundOAuthRefreshLockIdleTimeout, pool, withTransaction } from '../db.js'import type { K8sGateway } from '../k8s.js'import {RecipeNotFoundError,@@ -78,8 +78,13 @@ export interface ProactiveRefreshSweepOptions {export interface ProactiveRefreshSweepDeps {/** Snapshot reads (candidate enumeration + DCR list) — no lock, no txn. */db: DbClient- /** Runs `work` inside one short transaction (BEGIN…COMMIT/ROLLBACK). */- runInTransaction: <T>(work: (txDb: DbClient) => Promise<T>) => Promise<T>+ /**+ * Runs `work` inside one short transaction (BEGIN…COMMIT/ROLLBACK). The client+ * is transaction-scoped (branded {@link DbTransactionClient}) so the sweep can+ * bound the carrier's idle timeout across the in-transaction refresh POST, same+ * as the reactive path.+ */+ runInTransaction: <T>(work: (txDb: DbTransactionClient) => Promise<T>) => Promise<T>encryptionKey: BufferfetchFn: typeof fetch}@@ -203,6 +208,12 @@ export async function runProactiveRefreshSweep(// reactive path already renewed it out of the window since the snapshot.const claimed = await claimRemoteGrantForRefresh(txDb, key, window)if (!claimed) return 'skipped' as const+ // The refresh POST runs INSIDE this transaction while it holds the row+ // lock, so bound the idle-in-transaction tenancy just like the reactive+ // path: `withTransaction` sets no such timeout, and if the AS hangs mid+ // POST neither statement_timeout (no statement is running) nor the+ // transport abort guarantees the lock + pool connection is released.+ await boundOAuthRefreshLockIdleTimeout(txDb)// requireBackground is PER-grantKind, not fixed. SEC-5 governs `user`// grants only (unattended reuse needs explicit background consent), so// those keep `true` — redundant with the enumeration filter, kept asdiff --git a/control-api/test/services.oauthProactiveRefreshCron.test.ts b/control-api/test/services.oauthProactiveRefreshCron.test.tsindex fa61d6222..9bbd76adf 100644--- a/control-api/test/services.oauthProactiveRefreshCron.test.ts+++ b/control-api/test/services.oauthProactiveRefreshCron.test.ts@@ -9,7 +9,8 @@* by the real-Postgres suite and the reactive `tokenHelper` tests.*/import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'-import type { DbClient } from '../src/db.js'+import type { DbClient, DbTransactionClient } from '../src/db.js'+import { OAUTH_REFRESH_LOCK_CARRIER_IDLE_TIMEOUT_MS } from '../src/db.js'import type { K8sGateway } from '../src/k8s.js'import type { ExpiringDynamicClient } from '../src/oauth/dynamicClientStore.js'import { listExpiringDynamicClients } from '../src/oauth/dynamicClientStore.js'@@ -52,9 +53,13 @@ vi.mock('../src/oauth/tokenHelper.js', async () => {const OPTS = { proactiveBufferMs: 300_000, reactiveBufferMs: 60_000, dcrWarnMs: 604_800_000 }const gateway = {} as K8sGateway-const fakeTx: DbClient = { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) }+// Transaction-scoped spy client: its `.query` records every statement the sweep+// issues inside the refresh transaction (e.g. the SET LOCAL idle timeout). The+// brand is nominal-only, so the cast is the test's blessed mint point.+const fakeTxQuery = vi.fn(async () => ({ rows: [], rowCount: 0 }))+const fakeTx = { query: fakeTxQuery } as unknown as DbTransactionClientconst deps: ProactiveRefreshSweepDeps = {- db: { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) },+ db: { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) } as DbClient,runInTransaction: work => work(fakeTx),encryptionKey: Buffer.alloc(32),fetchFn: vi.fn() as unknown as typeof fetch,@@ -122,6 +127,34 @@ describe('runProactiveRefreshSweep — token candidates', () => {)})+ it('bounds the carrier idle timeout inside the refresh tx, BEFORE the refresh POST', async () => {+ // Defense-in-depth symmetry with the reactive path (reactiveTokenHelper.ts):+ // the refresh POST runs inside the transaction that holds the row lock, so+ // the sweep MUST emit `SET LOCAL idle_in_transaction_session_timeout` on the+ // tx client before handing off to getAccessToken (the POST). Assert the+ // observable statement on the tx client + its ordering, not an internal spy.+ // Fails against the pre-fix head, which never emitted this statement.+ mockedEnumerate.mockResolvedValue([sharedKey('srv-a', 'ctx-1')])+ await runProactiveRefreshSweep(gateway, OPTS, deps)++ const idleCallIdx = fakeTxQuery.mock.calls.findIndex(+ ([text]) =>+ typeof text === 'string' &&+ text.includes("set_config('idle_in_transaction_session_timeout'")+ )+ expect(idleCallIdx).toBeGreaterThanOrEqual(0)+ // SET LOCAL (third positional arg true) so it reverts on COMMIT/ROLLBACK.+ expect(fakeTxQuery.mock.calls[idleCallIdx][0]).toContain('$1, true)')+ expect(fakeTxQuery.mock.calls[idleCallIdx][1]).toEqual([+ String(OAUTH_REFRESH_LOCK_CARRIER_IDLE_TIMEOUT_MS),+ ])+ // Emitted BEFORE the refresh POST (getAccessToken) — else the lock+connection+ // is unbounded across a hung AS, the exact gap this fix closes.+ expect(fakeTxQuery.mock.invocationCallOrder[idleCallIdx]).toBeLessThan(+ mockedGetToken.mock.invocationCallOrder[0]+ )+ })+it('B1: requireBackground is per-grantKind — false for shared (context identity), true for user', async () => {// B1 regression: passing requireBackground:true for shared keys makes// getOAuthGrant's shared branch add `AND background = true` and return