Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): guard token refresh against a disconnect race (R1-B1)
Diego Zacarias10472897358b
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 53fde9aec..73cd7fa1f 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.506",+ "version": "0.1.507","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.506",+ "version": "0.1.507","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex 190e8d55c..7e2fe6f5d 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.506",+ "version": "0.1.507","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/oauth/store.ts b/control-api/src/oauth/store.tsindex 39e22036e..db19fc7bf 100644--- a/control-api/src/oauth/store.ts+++ b/control-api/src/oauth/store.ts@@ -107,8 +107,13 @@ export type UpsertOAuthGrantInput = OAuthGrantKey & {}/**- * Insert or update an oauth_grants row, encrypting access + refresh tokens- * at rest with the supplied AES-256-GCM key.+ * Insert (or replace) an oauth_grants row for the ALTA/CONSENT path only —+ * `user` and `service` grants, encrypting access + refresh tokens at rest with+ * the supplied AES-256-GCM key. This is an `INSERT … ON CONFLICT DO UPDATE`, so+ * it MAY create the row. Token refresh must NOT go through here (it would+ * resurrect a concurrently-deleted grant, R1-B1) — use+ * {@link refreshOAuthGrantTokens}. Shared grants are bootstrapped via+ * {@link bootstrapSharedOAuthGrant} and never inserted here.*/export async function upsertOAuthGrant(db: DbClient,@@ -156,14 +161,114 @@ export async function upsertOAuthGrant(}if (input.grantKind === 'shared') {- // Refresh path ONLY. A shared grant is bootstrapped once via- // `bootstrapSharedOAuthGrant` (INSERT … ON CONFLICT DO NOTHING, first-wins- // tokens AND bootstrapper). Token refresh is a plain UPDATE by key so it- // never rewrites the team identity or the `bootstrapped_by_user_id` audit- // column (mini-spec 05 §3: conflating the two would let "the last bootstrap- // silently steal the identity"). The row is guaranteed to exist here because- // refresh only runs after a successful `getOAuthGrant`.- await db.query(+ // A shared grant is NEVER inserted through this path: first consent goes+ // through `bootstrapSharedOAuthGrant` (first-wins tokens AND bootstrapper),+ // and token refresh goes through `refreshOAuthGrantTokens` (UPDATE by key).+ // Reaching here means a caller mis-routed a shared grant into the+ // consent/insert path — fail loudly rather than mint one against the+ // service INSERT below.+ throw new Error('upsertOAuthGrant does not handle shared grants')+ }++ await db.query(+ `INSERT INTO oauth_grants (+ owner_kind, recipe_namespace, recipe_name, user_id, oauth_client_id, grant_kind,+ provider, access_token_encrypted, refresh_token_encrypted,+ access_token_expires_at, updated_at+ ) VALUES ($1, $2, $3, NULL, $4, 'service', $5, $6, $7, $8, NOW())+ ON CONFLICT (recipe_namespace, recipe_name, oauth_client_id) WHERE grant_kind = 'service'+ DO UPDATE SET+ provider = EXCLUDED.provider,+ access_token_encrypted = EXCLUDED.access_token_encrypted,+ refresh_token_encrypted = EXCLUDED.refresh_token_encrypted,+ access_token_expires_at = EXCLUDED.access_token_expires_at,+ updated_at = NOW()`,+ [+ ownerKind,+ input.recipeNamespace,+ input.recipeName,+ input.oauthClientId,+ input.provider,+ accessTokenEncrypted,+ refreshTokenEncrypted,+ accessTokenExpiresAt,+ ]+ )+}++export type RefreshOAuthGrantTokensInput = OAuthGrantKey & {+ provider: string+ accessToken: string+ refreshToken?: string+ /** Seconds until access token expires; null if provider didn't supply expires_in. */+ accessTokenExpiresInSec?: number+}++/**+ * Persist refreshed tokens onto an EXISTING grant row — an UPDATE keyed by the+ * grant's own coordinates, for every flavor (`user`/`service`/`shared`) — and+ * report whether a row was actually touched.+ *+ * Distinct from {@link upsertOAuthGrant} on purpose (R1-B1). The consent/alta+ * path inserts, but refresh must NEVER resurrect a row that was concurrently+ * deleted: a disconnect (DELETE) can land between the token read+ * (`getOAuthGrant`) and this write. An `INSERT … ON CONFLICT` would recreate the+ * deleted grant with fresh tokens; an UPDATE by key touches 0 rows once the row+ * is gone, and the caller surfaces that as "needs reauth" rather than silently+ * re-granting.+ *+ * The `shared` branch is the former `upsertOAuthGrant` "Refresh path ONLY"+ * UPDATE, folded here (D4) so the refresh UPDATE lives in exactly one place: a+ * plain UPDATE by key that never rewrites the team identity or the+ * `bootstrapped_by_user_id` audit column (mini-spec 05 §3).+ *+ * Returns `{ updated }` — `false` when no row matched (the grant was deleted in+ * the refresh window).+ */+export async function refreshOAuthGrantTokens(+ db: DbClient,+ encryptionKey: Buffer,+ input: RefreshOAuthGrantTokensInput+): Promise<{ updated: boolean }> {+ const accessTokenEncrypted = encryptOAuthSecret(encryptionKey, input.accessToken)+ const refreshTokenEncrypted = input.refreshToken+ ? encryptOAuthSecret(encryptionKey, input.refreshToken)+ : null+ const accessTokenExpiresAt =+ typeof input.accessTokenExpiresInSec === 'number'+ ? new Date(Date.now() + input.accessTokenExpiresInSec * 1000)+ : null++ const ownerKind = resolveOwnerKind(input)++ if (input.grantKind === 'user') {+ const result = await db.query(+ `UPDATE oauth_grants+ SET provider = $6,+ access_token_encrypted = $7,+ refresh_token_encrypted = $8,+ access_token_expires_at = $9,+ updated_at = NOW()+ WHERE owner_kind = $1 AND recipe_namespace = $2 AND recipe_name = $3+ AND user_id = $4 AND oauth_client_id = $5 AND grant_kind = 'user'+ RETURNING id`,+ [+ ownerKind,+ input.recipeNamespace,+ input.recipeName,+ input.userId,+ input.oauthClientId,+ input.provider,+ accessTokenEncrypted,+ refreshTokenEncrypted,+ accessTokenExpiresAt,+ ]+ )+ return { updated: (result.rowCount ?? 0) > 0 }+ }++ if (input.grantKind === 'shared') {+ const result = await db.query(`UPDATE oauth_grantsSET provider = $6,access_token_encrypted = $7,@@ -171,7 +276,8 @@ export async function upsertOAuthGrant(access_token_expires_at = $9,updated_at = NOW()WHERE owner_kind = $1 AND recipe_namespace = $2 AND recipe_name = $3- AND context_id = $4 AND oauth_client_id = $5 AND grant_kind = 'shared'`,+ AND context_id = $4 AND oauth_client_id = $5 AND grant_kind = 'shared'+ RETURNING id`,[ownerKind,input.recipeNamespace,@@ -184,22 +290,19 @@ export async function upsertOAuthGrant(accessTokenExpiresAt,])- return+ return { updated: (result.rowCount ?? 0) > 0 }}- await db.query(- `INSERT INTO oauth_grants (- owner_kind, recipe_namespace, recipe_name, user_id, oauth_client_id, grant_kind,- provider, access_token_encrypted, refresh_token_encrypted,- access_token_expires_at, updated_at- ) VALUES ($1, $2, $3, NULL, $4, 'service', $5, $6, $7, $8, NOW())- ON CONFLICT (recipe_namespace, recipe_name, oauth_client_id) WHERE grant_kind = 'service'- DO UPDATE SET- provider = EXCLUDED.provider,- access_token_encrypted = EXCLUDED.access_token_encrypted,- refresh_token_encrypted = EXCLUDED.refresh_token_encrypted,- access_token_expires_at = EXCLUDED.access_token_expires_at,- updated_at = NOW()`,+ const result = await db.query(+ `UPDATE oauth_grants+ SET provider = $5,+ access_token_encrypted = $6,+ refresh_token_encrypted = $7,+ access_token_expires_at = $8,+ updated_at = NOW()+ WHERE owner_kind = $1 AND recipe_namespace = $2 AND recipe_name = $3+ AND user_id IS NULL AND oauth_client_id = $4 AND grant_kind = 'service'+ RETURNING id`,[ownerKind,input.recipeNamespace,@@ -211,6 +314,7 @@ export async function upsertOAuthGrant(accessTokenExpiresAt,])+ return { updated: (result.rowCount ?? 0) > 0 }}/**diff --git a/control-api/src/oauth/tokenHelper.ts b/control-api/src/oauth/tokenHelper.tsindex e8166e5f6..60bdcc51e 100644--- a/control-api/src/oauth/tokenHelper.ts+++ b/control-api/src/oauth/tokenHelper.ts@@ -11,7 +11,7 @@ import {getOAuthProviderAdapter,isKnownOAuthProvider,} from './providers.js'-import { type OAuthGrantKey, getOAuthGrant, upsertOAuthGrant } from './store.js'+import { type OAuthGrantKey, getOAuthGrant, refreshOAuthGrantTokens } from './store.js'/*** Fetch the current access token for a grant (user or service), refreshing on@@ -153,19 +153,24 @@ export async function getAccessToken(}// Some providers omit refresh_token on refresh — keep the previous one.- // Spread `...input` so the grant key (grantKind + owner/identifiers) is carried- // through verbatim — the refresh re-upserts the SAME row. For `user`/`service`- // this is ON CONFLICT DO UPDATE (identity columns excluded); for `shared` it- // is a plain UPDATE by key (see store.ts) that never rewrites- // `bootstrapped_by_user_id` or the shared identity. Columns outside the key- // are preserved either way.- await upsertOAuthGrant(deps.db, deps.encryptionKey, {+ // Persist via `refreshOAuthGrantTokens` (UPDATE by key), NOT an upsert. The+ // grant row was read above, but a concurrent disconnect (DELETE) can land+ // between that read and this write; an `INSERT … ON CONFLICT` would RESURRECT+ // the deleted grant with fresh tokens (R1-B1), whereas an UPDATE touches 0+ // rows once the row is gone. Spread `...input` so the grant key+ // (grantKind + owner/identifiers) targets the SAME row verbatim; for `shared`+ // the UPDATE never rewrites `bootstrapped_by_user_id` or the shared identity.+ const refreshed = await refreshOAuthGrantTokens(deps.db, deps.encryptionKey, {...input,provider: decl.provider,accessToken: parsed.accessToken,refreshToken: parsed.refreshToken ?? grant.refreshToken,accessTokenExpiresInSec: parsed.expiresIn,})+ // 0 rows ⇒ the grant was deleted during the refresh window. Surface it as+ // "needs reauth" (no_grant) rather than return a token for a revoked grant —+ // and crucially do NOT recreate the row.+ if (!refreshed.updated) return { kind: 'no_grant' }const expiresAt =typeof parsed.expiresIn === 'number'diff --git a/control-api/test/oauth.helpers.test.ts b/control-api/test/oauth.helpers.test.tsindex 78f50c254..50af9bcbe 100644--- a/control-api/test/oauth.helpers.test.ts+++ b/control-api/test/oauth.helpers.test.ts@@ -331,11 +331,12 @@ describe('getAccessToken (O5.1)', () => {)expect(result).toMatchObject({ kind: 'ok', accessToken: 'AT_REFRESHED' })- // SELECT + INSERT (upsert).+ // SELECT + UPDATE-by-key (R1-B1: refresh never re-inserts, so a concurrently+ // deleted grant is not resurrected).expect(db.query).toHaveBeenCalledTimes(2)- expect((db.query.mock.calls[1] as unknown as [string, unknown[]])[0]).toContain(- 'INSERT INTO oauth_grants'- )+ const refreshSql = (db.query.mock.calls[1] as unknown as [string, unknown[]])[0]+ expect(refreshSql).toContain('UPDATE oauth_grants')+ expect(refreshSql).not.toContain('INSERT INTO oauth_grants')})it('returns no_grant when the access token is expired and there is no refresh token', async () => {@@ -550,13 +551,17 @@ describe('getAccessToken (O5.1)', () => {})expect(result).toMatchObject({ kind: 'ok', accessToken: 'AT_REFRESHED' })- const [upsertSql, upsertParams] = db.query.mock.calls[1] as unknown as [string, unknown[]]- expect(upsertSql).toContain(- "ON CONFLICT (recipe_namespace, recipe_name, oauth_client_id) WHERE grant_kind = 'service'"- )- // Service INSERT carries 8 params (owner_kind, ns, name, oauthClientId,+ const [refreshSql, refreshParams] = db.query.mock.calls[1] as unknown as [string, unknown[]]+ // R1-B1: service refresh is an UPDATE by key (user_id IS NULL, grant_kind+ // 'service'), never an INSERT/ON CONFLICT — a deleted service grant is not+ // resurrected.+ expect(refreshSql).toContain('UPDATE oauth_grants')+ expect(refreshSql).toContain('user_id IS NULL')+ expect(refreshSql).toContain("grant_kind = 'service'")+ expect(refreshSql).not.toContain('ON CONFLICT')+ // Service refresh carries 8 params (owner_kind, ns, name, oauthClientId,// provider, accessToken, refreshToken, expiresAt) — no user_id.- expect(upsertParams).toHaveLength(8)+ expect(refreshParams).toHaveLength(8)})// Suppresses unused-import warnings for helpers we kept exporteddiff --git a/control-api/test/oauth.store.sharedGrants.test.ts b/control-api/test/oauth.store.sharedGrants.test.tsindex 27ec98044..70096ef45 100644--- a/control-api/test/oauth.store.sharedGrants.test.ts+++ b/control-api/test/oauth.store.sharedGrants.test.ts@@ -7,6 +7,7 @@ import {deleteOAuthGrant,getOAuthGrant,oauthGrantExists,+ refreshOAuthGrantTokens,upsertOAuthGrant,} from '../src/oauth/store.js'@@ -77,16 +78,21 @@ describe('oauth store — shared (context-identity) grants', () => {expect(out.inserted).toBe(false)})- it('upsertOAuthGrant(shared) is a plain UPDATE by key and never touches bootstrapped_by_user_id', async () => {- const { db, calls } = fakeDb()- await upsertOAuthGrant(db, KEY, {+ it('refreshOAuthGrantTokens(shared) is a plain UPDATE by key and never touches bootstrapped_by_user_id', async () => {+ // R1-B1: the shared "Refresh path ONLY" UPDATE now lives in+ // refreshOAuthGrantTokens (folded out of upsertOAuthGrant); upsertOAuthGrant+ // rejects shared grants (asserted below).+ const { db, calls } = fakeDb([{ id: 1 }], 1)+ const out = await refreshOAuthGrantTokens(db, KEY, {...SHARED_KEY,provider: 'google',accessToken: 'A2',refreshToken: 'R2',accessTokenExpiresInSec: 3600,})+ expect(out.updated).toBe(true)expect(calls[0].text).toContain('UPDATE oauth_grants')+ expect(calls[0].text).toContain('RETURNING id')expect(calls[0].text).not.toContain('INSERT INTO oauth_grants')expect(calls[0].text).not.toContain('bootstrapped_by_user_id')expect(calls[0].text).toContain("grant_kind = 'shared'")@@ -99,6 +105,23 @@ describe('oauth store — shared (context-identity) grants', () => {])})+ it('refreshOAuthGrantTokens returns updated=false when no row matched (grant deleted mid-refresh)', async () => {+ const { db } = fakeDb([], 0)+ const out = await refreshOAuthGrantTokens(db, KEY, {+ ...SHARED_KEY,+ provider: 'google',+ accessToken: 'A2',+ })+ expect(out.updated).toBe(false)+ })++ it('upsertOAuthGrant rejects a shared grant (consent uses bootstrapSharedOAuthGrant)', async () => {+ const { db } = fakeDb()+ await expect(+ upsertOAuthGrant(db, KEY, { ...SHARED_KEY, provider: 'google', accessToken: 'A2' })+ ).rejects.toThrow('does not handle shared grants')+ })+it('getOAuthGrant(shared) filters owner_kind + context_id + grant_kind=shared and maps audit columns', async () => {const { db, calls } = fakeDb([{diff --git a/control-api/test/oauth.tokenHelper.refreshRace.test.ts b/control-api/test/oauth.tokenHelper.refreshRace.test.tsnew file mode 100644index 000000000..274e1e422--- /dev/null+++ b/control-api/test/oauth.tokenHelper.refreshRace.test.ts@@ -0,0 +1,125 @@+import { describe, expect, it, vi } from 'vitest'+import {+ type RecipeReader,+ type RecipeWithOAuthClients,+ type SecretReader,+} from '../src/oauth/callback.js'+import { deriveOAuthEncryptionKey, encryptOAuthSecret } from '../src/oauth/encryption.js'+import { getAccessToken } from '../src/oauth/tokenHelper.js'++// R1-B1 — refresh↔DELETE race. A disconnect (DELETE) can land between the token+// read (`getOAuthGrant`) and the refresh write. Refresh MUST NOT resurrect the+// deleted row: an UPDATE-by-key touches 0 rows and the broker returns no_grant.+//+// This drives the stable public API `getAccessToken` (which exists at the parent+// sha) so the T3 check exercises the behavior, not a missing symbol. At parent+// (314dff6a) the refresh persists via `INSERT … ON CONFLICT DO UPDATE`, which+// recreates the deleted grant and returns `{ kind: 'ok' }` — so both assertions+// below fail there.++const KEY = deriveOAuthEncryptionKey(+ '00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff'+)++function recipeWithSalesforce(): RecipeWithOAuthClients {+ return {+ metadata: { name: 'crm', namespace: 'sandbox-recipes' },+ spec: {+ oauthClients: [+ {+ id: 'salesforce',+ provider: 'salesforce',+ clientIdRef: { name: 'salesforce-creds', key: 'client-id' },+ clientSecretRef: { name: 'salesforce-creds', key: 'client-secret' },+ scopes: ['api', 'refresh_token'],+ },+ ],+ },+ }+}++const recipeReader: RecipeReader = { read: async () => recipeWithSalesforce() }+const secretReader: SecretReader = {+ read: async () => ({ 'client-id': 'CID', 'client-secret': 'CSEC' }),+}++const okRefreshFetch = vi.fn(+ async () =>+ ({+ ok: true,+ status: 200,+ json: async () => ({+ access_token: 'AT_REFRESHED',+ refresh_token: 'RT_NEW',+ expires_in: 3600,+ token_type: 'Bearer',+ }),+ text: async () => '',+ }) as Response+)++/**+ * A db whose SELECT returns a STALE user grant (with a refresh token), but whose+ * refresh WRITE reports 0 affected rows — the grant was DELETEd concurrently.+ * Records every SQL string so the test can assert no INSERT was issued.+ */+function dbDeletedDuringRefresh() {+ const staleRow = {+ owner_kind: 'mcpserver',+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'gdrive',+ user_id: 'user-1',+ context_id: null,+ bootstrapped_by_user_id: null,+ oauth_client_id: 'salesforce',+ grant_kind: 'user',+ provider: 'salesforce',+ access_token_encrypted: encryptOAuthSecret(KEY, 'AT_STALE'),+ refresh_token_encrypted: encryptOAuthSecret(KEY, 'RT_STALE'),+ access_token_expires_at: new Date(Date.now() - 60_000), // expired → refresh+ updated_at: new Date(),+ background: false,+ }+ const sqls: string[] = []+ const query = vi.fn(async (text: string) => {+ sqls.push(text)+ if (text.trimStart().startsWith('SELECT')) {+ return { rows: [staleRow], rowCount: 1 }+ }+ // The concurrent DELETE already removed the row: the refresh write matches+ // nothing. (At parent, an INSERT … ON CONFLICT would instead re-create it.)+ return { rows: [], rowCount: 0 }+ })+ return { db: { query }, sqls }+}++describe('getAccessToken — refresh↔DELETE race (R1-B1)', () => {+ it('returns no_grant and does NOT resurrect a grant deleted during refresh', async () => {+ const { db, sqls } = dbDeletedDuringRefresh()++ const result = await getAccessToken(+ {+ grantKind: 'user',+ ownerKind: 'mcpserver',+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'gdrive',+ userId: 'user-1',+ oauthClientId: 'salesforce',+ },+ {+ db,+ recipeReader,+ secretReader,+ fetchFn: okRefreshFetch as unknown as typeof fetch,+ encryptionKey: KEY,+ }+ )++ // The row is gone: broker must say "needs reauth", never mint against it.+ expect(result.kind).toBe('no_grant')+ // And crucially, the write path must be an UPDATE — never an INSERT that+ // would recreate the disconnected grant.+ expect(sqls.some(s => s.includes('INSERT INTO oauth_grants'))).toBe(false)+ expect(sqls.some(s => s.includes('UPDATE oauth_grants'))).toBe(true)+ })+})