Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): guard token refresh against a disconnect race (R1-B1)

Diego Zacarias10472897358b
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 53fde9aec..73cd7fa1f 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.506",
+ "version": "0.1.507",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.506",
+ "version": "0.1.507",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index 190e8d55c..7e2fe6f5d 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.506",
+ "version": "0.1.507",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/oauth/store.ts b/control-api/src/oauth/store.ts
index 39e22036e..db19fc7bf 100644
--- a/control-api/src/oauth/store.ts
+++ b/control-api/src/oauth/store.ts
@@ -107,8 +107,13 @@ export type UpsertOAuthGrantInput = OAuthGrantKey & {
}
/**
- * Insert or update an oauth_grants row, encrypting access + refresh tokens
- * at rest with the supplied AES-256-GCM key.
+ * Insert (or replace) an oauth_grants row for the ALTA/CONSENT path only —
+ * `user` and `service` grants, encrypting access + refresh tokens at rest with
+ * the supplied AES-256-GCM key. This is an `INSERT … ON CONFLICT DO UPDATE`, so
+ * it MAY create the row. Token refresh must NOT go through here (it would
+ * resurrect a concurrently-deleted grant, R1-B1) — use
+ * {@link refreshOAuthGrantTokens}. Shared grants are bootstrapped via
+ * {@link bootstrapSharedOAuthGrant} and never inserted here.
*/
export async function upsertOAuthGrant(
db: DbClient,
@@ -156,14 +161,114 @@ export async function upsertOAuthGrant(
}
if (input.grantKind === 'shared') {
- // Refresh path ONLY. A shared grant is bootstrapped once via
- // `bootstrapSharedOAuthGrant` (INSERT … ON CONFLICT DO NOTHING, first-wins
- // tokens AND bootstrapper). Token refresh is a plain UPDATE by key so it
- // never rewrites the team identity or the `bootstrapped_by_user_id` audit
- // column (mini-spec 05 §3: conflating the two would let "the last bootstrap
- // silently steal the identity"). The row is guaranteed to exist here because
- // refresh only runs after a successful `getOAuthGrant`.
- await db.query(
+ // A shared grant is NEVER inserted through this path: first consent goes
+ // through `bootstrapSharedOAuthGrant` (first-wins tokens AND bootstrapper),
+ // and token refresh goes through `refreshOAuthGrantTokens` (UPDATE by key).
+ // Reaching here means a caller mis-routed a shared grant into the
+ // consent/insert path — fail loudly rather than mint one against the
+ // service INSERT below.
+ throw new Error('upsertOAuthGrant does not handle shared grants')
+ }
+
+ await db.query(
+ `INSERT INTO oauth_grants (
+ owner_kind, recipe_namespace, recipe_name, user_id, oauth_client_id, grant_kind,
+ provider, access_token_encrypted, refresh_token_encrypted,
+ access_token_expires_at, updated_at
+ ) VALUES ($1, $2, $3, NULL, $4, 'service', $5, $6, $7, $8, NOW())
+ ON CONFLICT (recipe_namespace, recipe_name, oauth_client_id) WHERE grant_kind = 'service'
+ DO UPDATE SET
+ provider = EXCLUDED.provider,
+ access_token_encrypted = EXCLUDED.access_token_encrypted,
+ refresh_token_encrypted = EXCLUDED.refresh_token_encrypted,
+ access_token_expires_at = EXCLUDED.access_token_expires_at,
+ updated_at = NOW()`,
+ [
+ ownerKind,
+ input.recipeNamespace,
+ input.recipeName,
+ input.oauthClientId,
+ input.provider,
+ accessTokenEncrypted,
+ refreshTokenEncrypted,
+ accessTokenExpiresAt,
+ ]
+ )
+}
+
+export type RefreshOAuthGrantTokensInput = OAuthGrantKey & {
+ provider: string
+ accessToken: string
+ refreshToken?: string
+ /** Seconds until access token expires; null if provider didn't supply expires_in. */
+ accessTokenExpiresInSec?: number
+}
+
+/**
+ * Persist refreshed tokens onto an EXISTING grant row — an UPDATE keyed by the
+ * grant's own coordinates, for every flavor (`user`/`service`/`shared`) — and
+ * report whether a row was actually touched.
+ *
+ * Distinct from {@link upsertOAuthGrant} on purpose (R1-B1). The consent/alta
+ * path inserts, but refresh must NEVER resurrect a row that was concurrently
+ * deleted: a disconnect (DELETE) can land between the token read
+ * (`getOAuthGrant`) and this write. An `INSERT … ON CONFLICT` would recreate the
+ * deleted grant with fresh tokens; an UPDATE by key touches 0 rows once the row
+ * is gone, and the caller surfaces that as "needs reauth" rather than silently
+ * re-granting.
+ *
+ * The `shared` branch is the former `upsertOAuthGrant` "Refresh path ONLY"
+ * UPDATE, folded here (D4) so the refresh UPDATE lives in exactly one place: a
+ * plain UPDATE by key that never rewrites the team identity or the
+ * `bootstrapped_by_user_id` audit column (mini-spec 05 §3).
+ *
+ * Returns `{ updated }` — `false` when no row matched (the grant was deleted in
+ * the refresh window).
+ */
+export async function refreshOAuthGrantTokens(
+ db: DbClient,
+ encryptionKey: Buffer,
+ input: RefreshOAuthGrantTokensInput
+): Promise<{ updated: boolean }> {
+ const accessTokenEncrypted = encryptOAuthSecret(encryptionKey, input.accessToken)
+ const refreshTokenEncrypted = input.refreshToken
+ ? encryptOAuthSecret(encryptionKey, input.refreshToken)
+ : null
+ const accessTokenExpiresAt =
+ typeof input.accessTokenExpiresInSec === 'number'
+ ? new Date(Date.now() + input.accessTokenExpiresInSec * 1000)
+ : null
+
+ const ownerKind = resolveOwnerKind(input)
+
+ if (input.grantKind === 'user') {
+ const result = await db.query(
+ `UPDATE oauth_grants
+ SET provider = $6,
+ access_token_encrypted = $7,
+ refresh_token_encrypted = $8,
+ access_token_expires_at = $9,
+ updated_at = NOW()
+ WHERE owner_kind = $1 AND recipe_namespace = $2 AND recipe_name = $3
+ AND user_id = $4 AND oauth_client_id = $5 AND grant_kind = 'user'
+ RETURNING id`,
+ [
+ ownerKind,
+ input.recipeNamespace,
+ input.recipeName,
+ input.userId,
+ input.oauthClientId,
+ input.provider,
+ accessTokenEncrypted,
+ refreshTokenEncrypted,
+ accessTokenExpiresAt,
+ ]
+ )
+ return { updated: (result.rowCount ?? 0) > 0 }
+ }
+
+ if (input.grantKind === 'shared') {
+ const result = await db.query(
`UPDATE oauth_grants
SET provider = $6,
access_token_encrypted = $7,
@@ -171,7 +276,8 @@ export async function upsertOAuthGrant(
access_token_expires_at = $9,
updated_at = NOW()
WHERE owner_kind = $1 AND recipe_namespace = $2 AND recipe_name = $3
- AND context_id = $4 AND oauth_client_id = $5 AND grant_kind = 'shared'`,
+ AND context_id = $4 AND oauth_client_id = $5 AND grant_kind = 'shared'
+ RETURNING id`,
[
ownerKind,
input.recipeNamespace,
@@ -184,22 +290,19 @@ export async function upsertOAuthGrant(
accessTokenExpiresAt,
]
)
- return
+ return { updated: (result.rowCount ?? 0) > 0 }
}
- await db.query(
- `INSERT INTO oauth_grants (
- owner_kind, recipe_namespace, recipe_name, user_id, oauth_client_id, grant_kind,
- provider, access_token_encrypted, refresh_token_encrypted,
- access_token_expires_at, updated_at
- ) VALUES ($1, $2, $3, NULL, $4, 'service', $5, $6, $7, $8, NOW())
- ON CONFLICT (recipe_namespace, recipe_name, oauth_client_id) WHERE grant_kind = 'service'
- DO UPDATE SET
- provider = EXCLUDED.provider,
- access_token_encrypted = EXCLUDED.access_token_encrypted,
- refresh_token_encrypted = EXCLUDED.refresh_token_encrypted,
- access_token_expires_at = EXCLUDED.access_token_expires_at,
- updated_at = NOW()`,
+ const result = await db.query(
+ `UPDATE oauth_grants
+ SET provider = $5,
+ access_token_encrypted = $6,
+ refresh_token_encrypted = $7,
+ access_token_expires_at = $8,
+ updated_at = NOW()
+ WHERE owner_kind = $1 AND recipe_namespace = $2 AND recipe_name = $3
+ AND user_id IS NULL AND oauth_client_id = $4 AND grant_kind = 'service'
+ RETURNING id`,
[
ownerKind,
input.recipeNamespace,
@@ -211,6 +314,7 @@ export async function upsertOAuthGrant(
accessTokenExpiresAt,
]
)
+ return { updated: (result.rowCount ?? 0) > 0 }
}
/**
diff --git a/control-api/src/oauth/tokenHelper.ts b/control-api/src/oauth/tokenHelper.ts
index e8166e5f6..60bdcc51e 100644
--- a/control-api/src/oauth/tokenHelper.ts
+++ b/control-api/src/oauth/tokenHelper.ts
@@ -11,7 +11,7 @@ import {
getOAuthProviderAdapter,
isKnownOAuthProvider,
} from './providers.js'
-import { type OAuthGrantKey, getOAuthGrant, upsertOAuthGrant } from './store.js'
+import { type OAuthGrantKey, getOAuthGrant, refreshOAuthGrantTokens } from './store.js'
/**
* Fetch the current access token for a grant (user or service), refreshing on
@@ -153,19 +153,24 @@ export async function getAccessToken(
}
// Some providers omit refresh_token on refresh — keep the previous one.
- // Spread `...input` so the grant key (grantKind + owner/identifiers) is carried
- // through verbatim — the refresh re-upserts the SAME row. For `user`/`service`
- // this is ON CONFLICT DO UPDATE (identity columns excluded); for `shared` it
- // is a plain UPDATE by key (see store.ts) that never rewrites
- // `bootstrapped_by_user_id` or the shared identity. Columns outside the key
- // are preserved either way.
- await upsertOAuthGrant(deps.db, deps.encryptionKey, {
+ // Persist via `refreshOAuthGrantTokens` (UPDATE by key), NOT an upsert. The
+ // grant row was read above, but a concurrent disconnect (DELETE) can land
+ // between that read and this write; an `INSERT … ON CONFLICT` would RESURRECT
+ // the deleted grant with fresh tokens (R1-B1), whereas an UPDATE touches 0
+ // rows once the row is gone. Spread `...input` so the grant key
+ // (grantKind + owner/identifiers) targets the SAME row verbatim; for `shared`
+ // the UPDATE never rewrites `bootstrapped_by_user_id` or the shared identity.
+ const refreshed = await refreshOAuthGrantTokens(deps.db, deps.encryptionKey, {
...input,
provider: decl.provider,
accessToken: parsed.accessToken,
refreshToken: parsed.refreshToken ?? grant.refreshToken,
accessTokenExpiresInSec: parsed.expiresIn,
})
+ // 0 rows ⇒ the grant was deleted during the refresh window. Surface it as
+ // "needs reauth" (no_grant) rather than return a token for a revoked grant —
+ // and crucially do NOT recreate the row.
+ if (!refreshed.updated) return { kind: 'no_grant' }
const expiresAt =
typeof parsed.expiresIn === 'number'
diff --git a/control-api/test/oauth.helpers.test.ts b/control-api/test/oauth.helpers.test.ts
index 78f50c254..50af9bcbe 100644
--- a/control-api/test/oauth.helpers.test.ts
+++ b/control-api/test/oauth.helpers.test.ts
@@ -331,11 +331,12 @@ describe('getAccessToken (O5.1)', () => {
)
expect(result).toMatchObject({ kind: 'ok', accessToken: 'AT_REFRESHED' })
- // SELECT + INSERT (upsert).
+ // SELECT + UPDATE-by-key (R1-B1: refresh never re-inserts, so a concurrently
+ // deleted grant is not resurrected).
expect(db.query).toHaveBeenCalledTimes(2)
- expect((db.query.mock.calls[1] as unknown as [string, unknown[]])[0]).toContain(
- 'INSERT INTO oauth_grants'
- )
+ const refreshSql = (db.query.mock.calls[1] as unknown as [string, unknown[]])[0]
+ expect(refreshSql).toContain('UPDATE oauth_grants')
+ expect(refreshSql).not.toContain('INSERT INTO oauth_grants')
})
it('returns no_grant when the access token is expired and there is no refresh token', async () => {
@@ -550,13 +551,17 @@ describe('getAccessToken (O5.1)', () => {
}
)
expect(result).toMatchObject({ kind: 'ok', accessToken: 'AT_REFRESHED' })
- const [upsertSql, upsertParams] = db.query.mock.calls[1] as unknown as [string, unknown[]]
- expect(upsertSql).toContain(
- "ON CONFLICT (recipe_namespace, recipe_name, oauth_client_id) WHERE grant_kind = 'service'"
- )
- // Service INSERT carries 8 params (owner_kind, ns, name, oauthClientId,
+ const [refreshSql, refreshParams] = db.query.mock.calls[1] as unknown as [string, unknown[]]
+ // R1-B1: service refresh is an UPDATE by key (user_id IS NULL, grant_kind
+ // 'service'), never an INSERT/ON CONFLICT — a deleted service grant is not
+ // resurrected.
+ expect(refreshSql).toContain('UPDATE oauth_grants')
+ expect(refreshSql).toContain('user_id IS NULL')
+ expect(refreshSql).toContain("grant_kind = 'service'")
+ expect(refreshSql).not.toContain('ON CONFLICT')
+ // Service refresh carries 8 params (owner_kind, ns, name, oauthClientId,
// provider, accessToken, refreshToken, expiresAt) — no user_id.
- expect(upsertParams).toHaveLength(8)
+ expect(refreshParams).toHaveLength(8)
})
// Suppresses unused-import warnings for helpers we kept exported
diff --git a/control-api/test/oauth.store.sharedGrants.test.ts b/control-api/test/oauth.store.sharedGrants.test.ts
index 27ec98044..70096ef45 100644
--- a/control-api/test/oauth.store.sharedGrants.test.ts
+++ b/control-api/test/oauth.store.sharedGrants.test.ts
@@ -7,6 +7,7 @@ import {
deleteOAuthGrant,
getOAuthGrant,
oauthGrantExists,
+ refreshOAuthGrantTokens,
upsertOAuthGrant,
} from '../src/oauth/store.js'
@@ -77,16 +78,21 @@ describe('oauth store — shared (context-identity) grants', () => {
expect(out.inserted).toBe(false)
})
- it('upsertOAuthGrant(shared) is a plain UPDATE by key and never touches bootstrapped_by_user_id', async () => {
- const { db, calls } = fakeDb()
- await upsertOAuthGrant(db, KEY, {
+ it('refreshOAuthGrantTokens(shared) is a plain UPDATE by key and never touches bootstrapped_by_user_id', async () => {
+ // R1-B1: the shared "Refresh path ONLY" UPDATE now lives in
+ // refreshOAuthGrantTokens (folded out of upsertOAuthGrant); upsertOAuthGrant
+ // rejects shared grants (asserted below).
+ const { db, calls } = fakeDb([{ id: 1 }], 1)
+ const out = await refreshOAuthGrantTokens(db, KEY, {
...SHARED_KEY,
provider: 'google',
accessToken: 'A2',
refreshToken: 'R2',
accessTokenExpiresInSec: 3600,
})
+ expect(out.updated).toBe(true)
expect(calls[0].text).toContain('UPDATE oauth_grants')
+ expect(calls[0].text).toContain('RETURNING id')
expect(calls[0].text).not.toContain('INSERT INTO oauth_grants')
expect(calls[0].text).not.toContain('bootstrapped_by_user_id')
expect(calls[0].text).toContain("grant_kind = 'shared'")
@@ -99,6 +105,23 @@ describe('oauth store — shared (context-identity) grants', () => {
])
})
+ it('refreshOAuthGrantTokens returns updated=false when no row matched (grant deleted mid-refresh)', async () => {
+ const { db } = fakeDb([], 0)
+ const out = await refreshOAuthGrantTokens(db, KEY, {
+ ...SHARED_KEY,
+ provider: 'google',
+ accessToken: 'A2',
+ })
+ expect(out.updated).toBe(false)
+ })
+
+ it('upsertOAuthGrant rejects a shared grant (consent uses bootstrapSharedOAuthGrant)', async () => {
+ const { db } = fakeDb()
+ await expect(
+ upsertOAuthGrant(db, KEY, { ...SHARED_KEY, provider: 'google', accessToken: 'A2' })
+ ).rejects.toThrow('does not handle shared grants')
+ })
+
it('getOAuthGrant(shared) filters owner_kind + context_id + grant_kind=shared and maps audit columns', async () => {
const { db, calls } = fakeDb([
{
diff --git a/control-api/test/oauth.tokenHelper.refreshRace.test.ts b/control-api/test/oauth.tokenHelper.refreshRace.test.ts
new file mode 100644
index 000000000..274e1e422
--- /dev/null
+++ b/control-api/test/oauth.tokenHelper.refreshRace.test.ts
@@ -0,0 +1,125 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+ type RecipeReader,
+ type RecipeWithOAuthClients,
+ type SecretReader,
+} from '../src/oauth/callback.js'
+import { deriveOAuthEncryptionKey, encryptOAuthSecret } from '../src/oauth/encryption.js'
+import { getAccessToken } from '../src/oauth/tokenHelper.js'
+
+// R1-B1 — refresh↔DELETE race. A disconnect (DELETE) can land between the token
+// read (`getOAuthGrant`) and the refresh write. Refresh MUST NOT resurrect the
+// deleted row: an UPDATE-by-key touches 0 rows and the broker returns no_grant.
+//
+// This drives the stable public API `getAccessToken` (which exists at the parent
+// sha) so the T3 check exercises the behavior, not a missing symbol. At parent
+// (314dff6a) the refresh persists via `INSERT … ON CONFLICT DO UPDATE`, which
+// recreates the deleted grant and returns `{ kind: 'ok' }` — so both assertions
+// below fail there.
+
+const KEY = deriveOAuthEncryptionKey(
+ '00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff'
+)
+
+function recipeWithSalesforce(): RecipeWithOAuthClients {
+ return {
+ metadata: { name: 'crm', namespace: 'sandbox-recipes' },
+ spec: {
+ oauthClients: [
+ {
+ id: 'salesforce',
+ provider: 'salesforce',
+ clientIdRef: { name: 'salesforce-creds', key: 'client-id' },
+ clientSecretRef: { name: 'salesforce-creds', key: 'client-secret' },
+ scopes: ['api', 'refresh_token'],
+ },
+ ],
+ },
+ }
+}
+
+const recipeReader: RecipeReader = { read: async () => recipeWithSalesforce() }
+const secretReader: SecretReader = {
+ read: async () => ({ 'client-id': 'CID', 'client-secret': 'CSEC' }),
+}
+
+const okRefreshFetch = vi.fn(
+ async () =>
+ ({
+ ok: true,
+ status: 200,
+ json: async () => ({
+ access_token: 'AT_REFRESHED',
+ refresh_token: 'RT_NEW',
+ expires_in: 3600,
+ token_type: 'Bearer',
+ }),
+ text: async () => '',
+ }) as Response
+)
+
+/**
+ * A db whose SELECT returns a STALE user grant (with a refresh token), but whose
+ * refresh WRITE reports 0 affected rows — the grant was DELETEd concurrently.
+ * Records every SQL string so the test can assert no INSERT was issued.
+ */
+function dbDeletedDuringRefresh() {
+ const staleRow = {
+ owner_kind: 'mcpserver',
+ recipe_namespace: 'sandbox-recipes',
+ recipe_name: 'gdrive',
+ user_id: 'user-1',
+ context_id: null,
+ bootstrapped_by_user_id: null,
+ oauth_client_id: 'salesforce',
+ grant_kind: 'user',
+ provider: 'salesforce',
+ access_token_encrypted: encryptOAuthSecret(KEY, 'AT_STALE'),
+ refresh_token_encrypted: encryptOAuthSecret(KEY, 'RT_STALE'),
+ access_token_expires_at: new Date(Date.now() - 60_000), // expired → refresh
+ updated_at: new Date(),
+ background: false,
+ }
+ const sqls: string[] = []
+ const query = vi.fn(async (text: string) => {
+ sqls.push(text)
+ if (text.trimStart().startsWith('SELECT')) {
+ return { rows: [staleRow], rowCount: 1 }
+ }
+ // The concurrent DELETE already removed the row: the refresh write matches
+ // nothing. (At parent, an INSERT … ON CONFLICT would instead re-create it.)
+ return { rows: [], rowCount: 0 }
+ })
+ return { db: { query }, sqls }
+}
+
+describe('getAccessToken — refresh↔DELETE race (R1-B1)', () => {
+ it('returns no_grant and does NOT resurrect a grant deleted during refresh', async () => {
+ const { db, sqls } = dbDeletedDuringRefresh()
+
+ const result = await getAccessToken(
+ {
+ grantKind: 'user',
+ ownerKind: 'mcpserver',
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'gdrive',
+ userId: 'user-1',
+ oauthClientId: 'salesforce',
+ },
+ {
+ db,
+ recipeReader,
+ secretReader,
+ fetchFn: okRefreshFetch as unknown as typeof fetch,
+ encryptionKey: KEY,
+ }
+ )
+
+ // The row is gone: broker must say "needs reauth", never mint against it.
+ expect(result.kind).toBe('no_grant')
+ // And crucially, the write path must be an UPDATE — never an INSERT that
+ // would recreate the disconnected grant.
+ expect(sqls.some(s => s.includes('INSERT INTO oauth_grants'))).toBe(false)
+ expect(sqls.some(s => s.includes('UPDATE oauth_grants'))).toBe(true)
+ })
+})