Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Reactive refresh row-lock serialization (mini-spec 16, R2-M1), on a REAL
* Postgres so the BLOCKING `FOR UPDATE` and its mutual exclusion with the
* proactive `FOR UPDATE SKIP LOCKED` are exercised against the actual schema —
* a mock `DbClient` cannot model row locking.
*
* Fixtures come from the REAL producers (`bootstrapSharedOAuthGrant`, T1). The
* fake token endpoint (a `PinnedTransport`, like the proactive test) models a
* ROTATING refresh token: the first POST rotates it and returns fresh tokens; any
* later POST that presents an already-used refresh token gets `400 invalid_grant`.
* Counting POSTs is the observable (T4): the bug is two POSTs (double-spend) that
* kill the grant; the fix is exactly one.
*
* CONTROL_API_REAL_PG_ADMIN_URL=postgres://<user>:<pass>@<host>:5432 npm test -- \
* test/oauth.reactiveRefreshLock.realPostgres.integration.test.ts
*/
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomUUID } from 'node:crypto'
import { Pool } from 'pg'
import { config } from '../src/config.js'
import type { DbClient, DbTransactionClient } from '../src/db.js'
import { initDb } from '../src/db.js'
import type {
PinnedRawResponse,
PinnedTransport,
PinnedTransportInput,
} from '../src/http/pinnedFetch.js'
import type { RecipeWithOAuthClients } from '../src/oauth/callback.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import { getAccessTokenReactive } from '../src/oauth/reactiveTokenHelper.js'
import {
type OAuthGrantKey,
bootstrapSharedOAuthGrant,
claimRemoteGrantForRefresh,
getOAuthGrant,
} from '../src/oauth/store.js'
import { getAccessToken } from '../src/oauth/tokenHelper.js'
import { type McpServerResource, normalizeMcpServerOwnerDecl } from '../src/routes/mcpOauth.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
import { MockGateway } from './mockGateway.js'
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
function databaseUrl(baseUrl: string, database: string): string {
const url = new URL(baseUrl)
url.pathname = `/${database}`
return url.toString()
}
const KEY = deriveOAuthEncryptionKey(config.oauthEncryptionKey)
const NS = config.mcpServersNamespace
// Bp/Br as the proactive sweep uses them; a token in the proactive window is
// treated as stale by BOTH sides here (refreshBufferMs = Bp), which is what lets
// the proactive claim (window predicate) and the reactive refresh contend on the
// SAME row — the crux of the P↔R case.
const WINDOW = { proactiveBufferMs: 300_000, reactiveBufferMs: 60_000 }
const IN_WINDOW_SEC = 200 // 60s < 200s ≤ 300s → proactive window AND stale under Bp
const REMOTE_OAUTH_ID = 'https://control.example.com/.well-known/evenfire-mcp-client'
const REMOTE_VALIDATED_IP = '93.184.216.34'
describeRealPostgres('oauth reactive refresh — row-lock serialization (real Postgres)', () => {
const database = `control_api_reactive_lock_${randomUUID().replace(/-/g, '')}`
let adminPool: Pool
let dbPool: Pool
let db: DbClient
beforeAll(async () => {
if (!adminUrl) throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required')
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)
dbPool = new Pool({ connectionString: databaseUrl(adminUrl, database) })
// pg-pool can resolve end() before its clients' asynchronous socket closes.
// Register physical closure before initDb creates the first connection.
await initDb({ connect: () => dbPool.connect() })
db = { query: (text, values) => dbPool.query(text, values) }
})
afterAll(async () => {
try {
await endPoolAndWaitForClients(dbPool)
if (adminPool) {
await adminPool.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname = $1 AND pid <> pg_backend_pid()`,
[database]
)
await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)
}
} finally {
await adminPool?.end()
}
})
const sharedKey = (server: string, ctx: string): OAuthGrantKey => ({
grantKind: 'shared',
ownerKind: 'mcpserver',
recipeNamespace: NS,
recipeName: server,
contextId: ctx,
oauthClientId: REMOTE_OAUTH_ID,
crUid: uidOf(server),
})
async function seedShared(server: string, ctx: string, expiresInSec: number): Promise<void> {
await install(server)
await bootstrapSharedOAuthGrant(db, KEY, {
...sharedKey(server, ctx),
bootstrappedByUserId: 'user-1',
provider: 'remote',
accessToken: 'AT-0',
refreshToken: 'RT-0',
accessTokenExpiresInSec: expiresInSec,
})
}
// The owner CR lives in a gateway (which assigns its `metadata.uid`, as the
// apiserver does) and is read back through the REAL owner normalizer the
// refresh engine consumes (T1). Grants are sealed with that uid, as the consent
// callback does, and keys carry it, as the readers derive them.
const gateway = new MockGateway(NS)
const uids = new Map<string, string>()
async function install(server: string): Promise<void> {
if (uids.has(server)) return
await gateway.createResource(
'mcpservers',
{
metadata: { name: server },
spec: {
contextRef: 'ctx-1',
auth: { type: 'oauth' },
oauth: {
source: 'remote',
id: REMOTE_OAUTH_ID,
clientMode: 'public',
authorizationEndpoint: 'https://as.example.com/authorize',
tokenEndpoint: 'https://as.example.com/token',
issuer: 'https://as.example.com',
resource: 'https://as.example.com',
grantScope: 'user',
scopes: ['read'],
bearerInBody: false,
supportsRefresh: true,
},
},
},
NS
)
const cr = (await gateway.getResource('mcpservers', server, NS)) as {
metadata: { uid: string }
}
uids.set(server, cr.metadata.uid)
}
function uidOf(server: string): string {
const uid = uids.get(server)
if (!uid) throw new Error(`fixture: ${server} not installed`)
return uid
}
const ownerReader = {
read: async (name: string): Promise<RecipeWithOAuthClients | null> =>
normalizeMcpServerOwnerDecl(
(await gateway.getResource('mcpservers', name, NS)) as McpServerResource
),
}
/**
* Rotating-refresh-token AS. Each accepted POST invalidates the presented
* refresh token and mints a new pair; reusing a spent refresh token → 400
* invalid_grant (what a real AS does on rotation reuse, killing the grant).
*/
function rotatingAs() {
const validRefreshTokens = new Set<string>(['RT-0'])
let generation = 0
let postCount = 0
const transport: PinnedTransport = async (
input: PinnedTransportInput
): Promise<PinnedRawResponse> => {
postCount += 1
const presented = new URLSearchParams(input.body ?? '').get('refresh_token') ?? ''
if (!validRefreshTokens.has(presented)) {
return {
status: 400,
headers: { 'content-type': 'application/json' },
bodyText: JSON.stringify({ error: 'invalid_grant' }),
}
}
validRefreshTokens.delete(presented)
generation += 1
const next = `RT-${generation}`
validRefreshTokens.add(next)
return {
status: 200,
headers: { 'content-type': 'application/json' },
bodyText: JSON.stringify({
access_token: `AT-${generation}`,
refresh_token: next,
expires_in: 3600,
}),
}
}
return {
transport,
get postCount() {
return postCount
},
}
}
const reactiveDeps = (transport: PinnedTransport) => ({
db,
recipeReader: ownerReader,
secretReader: { read: async () => ({}) },
fetchFn: (async () => {
throw new Error('remote refresh must not use fetchFn')
}) as unknown as typeof fetch,
encryptionKey: KEY,
resolveDns: async () => [REMOTE_VALIDATED_IP],
pinnedTransport: transport,
// Bp so an in-proactive-window token is treated as stale by the reactive path
// too — exactly the overlap that produced the double-spend.
refreshBufferMs: WINDOW.proactiveBufferMs,
// Reactive slow-path tx bound to the TEST pool (the module `pool` points at the
// real configured DB, not this ephemeral one), so the row lock lands here.
runInTransaction: async <T>(work: (txDb: DbTransactionClient) => Promise<T>): Promise<T> => {
const client = await dbPool.connect()
try {
await client.query('BEGIN')
const txDb = {
query: (t: string, v?: unknown[]) => client.query(t, v),
} as unknown as DbTransactionClient
const result = await work(txDb)
await client.query('COMMIT')
return result
} catch (err) {
await client.query('ROLLBACK').catch(() => {})
throw err
} finally {
client.release()
}
},
})
async function waitForLockWaiter(): Promise<void> {
for (let i = 0; i < 200; i += 1) {
const r = await adminPool.query(
`SELECT count(*)::int AS n FROM pg_stat_activity
WHERE datname = $1 AND wait_event_type = 'Lock'`,
[database]
)
if (((r.rows[0] as { n: number }).n ?? 0) > 0) return
await new Promise(res => setTimeout(res, 25))
}
throw new Error('the reactive refresh never blocked on the row lock')
}
it('P↔R: a reactive refresh blocks on the proactive row lock, then re-reads fresh (1 POST)', async () => {
await seedShared('pr', 'ctx-1', IN_WINDOW_SEC)
const key = sharedKey('pr', 'ctx-1')
const as = rotatingAs()
// Proactive side: claim (FOR UPDATE SKIP LOCKED) + refresh on the SAME tx,
// held OPEN (uncommitted) so it owns the row lock while the reactive contends.
const c1 = await dbPool.connect()
try {
await c1.query('BEGIN')
const c1Tx = { query: (t: string, v?: unknown[]) => c1.query(t, v) }
const claimed = await claimRemoteGrantForRefresh(c1Tx, key, WINDOW)
expect(claimed).toBe(true)
const proactive = await getAccessToken(
{ ...key, requireBackground: false },
{
db: c1Tx,
recipeReader: ownerReader,
secretReader: { read: async () => ({}) },
fetchFn: (async () => {
throw new Error('remote refresh must not use fetchFn')
}) as unknown as typeof fetch,
encryptionKey: KEY,
resolveDns: async () => [REMOTE_VALIDATED_IP],
pinnedTransport: as.transport,
refreshBufferMs: WINDOW.proactiveBufferMs,
}
)
expect(proactive.kind).toBe('ok')
expect(as.postCount).toBe(1) // proactive did the single POST
// Reactive side: still sees the OLD (uncommitted) token → wants to refresh →
// blocks on the row lock. It must NOT double-spend.
const reactivePromise = getAccessTokenReactive(key, reactiveDeps(as.transport))
await waitForLockWaiter()
await c1.query('COMMIT') // releases the lock; row now carries the fresh token
const reactive = await reactivePromise
expect(reactive.kind).toBe('ok')
} finally {
c1.release()
}
// The observable: exactly ONE POST total; the grant survives with the rotated
// token; no invalid_grant was ever provoked.
expect(as.postCount).toBe(1)
const after = await getOAuthGrant(db, KEY, key)
expect(after?.accessToken).toBe('AT-1')
expect(after?.refreshToken).toBe('RT-1')
})
it('R↔R: two concurrent reactive refreshes over one grant issue exactly 1 POST', async () => {
await seedShared('rr', 'ctx-1', IN_WINDOW_SEC)
const key = sharedKey('rr', 'ctx-1')
const as = rotatingAs()
const [a, b] = await Promise.all([
getAccessTokenReactive(key, reactiveDeps(as.transport)),
getAccessTokenReactive(key, reactiveDeps(as.transport)),
])
expect(a.kind).toBe('ok')
expect(b.kind).toBe('ok')
expect(as.postCount).toBe(1) // one refreshed, the other re-read fresh under the lock
const after = await getOAuthGrant(db, KEY, key)
expect(after?.accessToken).toBe('AT-1')
expect(after?.refreshToken).toBe('RT-1')
})
})