Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

refactor(control-api): read the live CR uid after the dcr row read and CAS on cr_uid (RP-R4-LC-01)

Diego Zacarias6619778d8042
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index ace292e60..cb2c2fbf4 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.666",
+ "version": "0.1.667",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.666",
+ "version": "0.1.667",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index ebcadcb60..a96510aea 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.666",
+ "version": "0.1.667",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/oauth/dynamicClientStore.ts b/control-api/src/oauth/dynamicClientStore.ts
index a783f4fa3..6f7212777 100644
--- a/control-api/src/oauth/dynamicClientStore.ts
+++ b/control-api/src/oauth/dynamicClientStore.ts
@@ -37,6 +37,8 @@ function resolveOwnerKind(input: { ownerKind?: 'mcpserver' }): 'mcpserver' {
/** One decrypted row of `dynamic_clients`. */
export interface DynamicClientRow {
+ /** Surrogate row id; part of the identity a reclaim must still find unchanged. */
+ id: string
ownerKind: 'mcpserver'
serverNamespace: string
serverName: string
@@ -59,6 +61,12 @@ export interface DynamicClientRow {
crUid?: string
createdAt: Date
updatedAt: Date
+ /**
+ * Age of the row (`NOW() - updated_at`) as measured by the database at read
+ * time, so the pending-TTL decision never mixes the pod clock with the DB clock
+ * that stamped `updated_at`.
+ */
+ ageMs: number
}
export type UpsertDynamicClientInput = DynamicClientKey & {
@@ -173,6 +181,7 @@ export async function upsertDynamicClient(
}
interface DynamicClientDbRow {
+ id: string | number
owner_kind: 'mcpserver'
server_namespace: string
server_name: string
@@ -188,12 +197,14 @@ interface DynamicClientDbRow {
cr_uid: string | null
created_at: Date
updated_at: Date
+ age_ms: number
}
-const SELECT_COLUMNS = `owner_kind, server_namespace, server_name, issuer, client_id, client_mode,
+const SELECT_COLUMNS = `id, owner_kind, server_namespace, server_name, issuer, client_id, client_mode,
client_secret_encrypted, registration_access_token_encrypted,
registration_client_uri, client_id_issued_at, client_secret_expires_at,
- install_id, cr_uid, created_at, updated_at`
+ install_id, cr_uid, created_at, updated_at,
+ (EXTRACT(EPOCH FROM (NOW() - updated_at)) * 1000)::float8 AS age_ms`
/** Read one dynamic client, decrypting its secret and registration token. */
export async function getDynamicClient(
@@ -211,6 +222,7 @@ export async function getDynamicClient(
if (result.rows.length === 0) return null
const row = result.rows[0] as DynamicClientDbRow
return {
+ id: String(row.id),
ownerKind: row.owner_kind,
serverNamespace: row.server_namespace,
serverName: row.server_name,
@@ -230,6 +242,7 @@ export async function getDynamicClient(
crUid: row.cr_uid ?? undefined,
createdAt: row.created_at,
updatedAt: row.updated_at,
+ ageMs: Number(row.age_ms),
}
}
@@ -311,9 +324,13 @@ export type ExistingDynamicClientClass = 'in-use' | 'in-progress' | 'reclaimable
/**
* Decide who owns a conflicting `dynamic_clients` row — the one rule for "whose
- * row is this?" (D4). Pure and total; the caller supplies wall-clock `now`, the
- * pending TTL, and `liveCrUid` (the uid of a live McpServer with this name, when
- * one exists at conflict time).
+ * row is this?" (D4). Pure and total; the caller supplies the row's DB-measured
+ * age, the pending TTL, and `liveCrUid` (the uid of a live McpServer with this
+ * name, when one exists).
+ *
+ * `liveCrUid` is only meaningful if it was read AFTER the row it is classified
+ * with: a CR created and bound between an earlier K8s read and the row read would
+ * otherwise make a live install's row look orphaned.
*
* A live CR owns the name unconditionally: it covers a bound-live row AND the
* pre-check→INSERT TOCTOU race where the CR appeared after the pending INSERT.
@@ -323,16 +340,15 @@ export type ExistingDynamicClientClass = 'in-use' | 'in-progress' | 'reclaimable
* today — all reclaimable except the fresh pending one.
*/
export function classifyExistingDynamicClient(
- row: { installId: string | null; crUid: string | null; updatedAt: Date },
- ctx: { now: Date; pendingTtlMs: number; liveCrUid?: string }
+ row: { installId: string | null; crUid: string | null; ageMs: number },
+ ctx: { pendingTtlMs: number; liveCrUid?: string }
): ExistingDynamicClientClass {
if (ctx.liveCrUid !== undefined) return 'in-use'
if (row.crUid != null) return 'reclaimable'
if (row.installId != null) {
- // In-progress iff `updated_at > now − TTL` (i.e. age strictly under the TTL);
- // at exactly the TTL the row is expired and reclaimable.
- const ageMs = ctx.now.getTime() - row.updatedAt.getTime()
- return ageMs < ctx.pendingTtlMs ? 'in-progress' : 'reclaimable'
+ // In-progress iff age is strictly under the TTL; at exactly the TTL the row is
+ // expired and reclaimable.
+ return row.ageMs < ctx.pendingTtlMs ? 'in-progress' : 'reclaimable'
}
return 'reclaimable'
}
@@ -380,25 +396,42 @@ export async function insertDynamicClientPending(
}
/**
- * Take over a `reclaimable` row (orphan or legacy) for a new install, atomically.
- * Opens a transaction, `SELECT … FOR UPDATE`s the row, re-runs
- * {@link classifyExistingDynamicClient} on the LOCKED row (its state may have
- * changed since the losing INSERT), and only if still `reclaimable` overwrites the
- * credentials with the new install's `install_id` and `cr_uid = NULL`.
+ * The identity of a `dynamic_clients` row as the caller read and classified it.
+ * A reclaim only applies to the row in exactly this state.
+ */
+export type ObservedDynamicClient = Pick<DynamicClientRow, 'id' | 'installId' | 'crUid'>
+
+/**
+ * Take over a row the caller classified `reclaimable` (orphan or legacy) for a
+ * new install, atomically. Does NOT classify: the caller did, from `observed` and
+ * a live-CR read taken after it. This applies the takeover only if the row still
+ * has the observed `id`, `install_id` AND `cr_uid` — one guard, evaluated on the
+ * row-locked latest version, so:
+ * - a concurrent reclaim that won first changed `install_id` → we lose;
+ * - the owner binding its pending row (`cr_uid` NULL → uid) after our read →
+ * we lose, and a live install's client is never overwritten or revoked;
+ * - the row deleted (teardown) → 0 rows → we lose (conservative; the operator's
+ * retry goes through the INSERT).
+ * Re-classifying under the lock is deliberately absent: it would re-decide with a
+ * K8s read that predates the locked version, which is exactly the stale-uid bug.
*
- * The UPDATE is a CAS on `id` + `install_id` read under the lock, so two
- * concurrent reclaims cannot both win: the second serializes behind the first,
- * re-reads a now-fresh pending row, and classifies it `in-progress` →
- * `{ reclaimed: false }`. Returns the OLD row's RFC 7592 handle (decrypted, when
- * present) so the caller can best-effort revoke the superseded client at the AS.
+ * The guard does not compare `updated_at`: every production writer that moves it
+ * also changes `id` (INSERT), `install_id` (reclaim) or `cr_uid` (bind), so the
+ * observed age can only have grown and an expired pending row stays expired.
+ * `upsertDynamicClient` refreshes `updated_at` with identity intact and would break
+ * that premise; it has no production caller and must not gain one on this table.
+ *
+ * Returns the OLD row's RFC 7592 handle (decrypted, when present) so the caller
+ * can best-effort revoke the superseded client at the AS.
+ *
+ * The reclaim is a single statement, so the transaction adds no atomicity; the
+ * runner is kept only as the injection point.
*
- * No leading `db` param: all work runs inside the transaction this function owns.
* `runInTransaction` is a REQUIRED injected runner (pass `withTransaction` from
* db.ts) rather than a default import — this module sits on the config init path
* (config → tokenHelper → callback → here), so importing db.ts's `withTransaction`
* value at module top-level would close an init cycle back to `config` and leave
- * it undefined. The injection also lets tests bind the runner to a disposable pool,
- * matching the `runInTransaction` idiom in reactiveTokenHelper.
+ * it undefined. The injection also lets tests bind the runner to a disposable pool.
*/
export async function reclaimOrphanDynamicClient(
encryptionKey: Buffer,
@@ -406,7 +439,7 @@ export async function reclaimOrphanDynamicClient(
key: DynamicClientKey
newCredentials: UpsertDynamicClientInput
newInstallId: string
- ctx: { now: Date; pendingTtlMs: number; liveCrUid?: string }
+ observed: ObservedDynamicClient
},
runInTransaction: <T>(work: (tx: DbTransactionClient) => Promise<T>) => Promise<T>
): Promise<{ reclaimed: boolean; oldHandle?: DynamicClientRegistrationHandle }> {
@@ -414,32 +447,17 @@ export async function reclaimOrphanDynamicClient(
const creds = encodeCredentialColumns(encryptionKey, input.newCredentials)
return runInTransaction(async tx => {
- const locked = await tx.query(
- `SELECT id, install_id, cr_uid, updated_at,
- registration_client_uri, registration_access_token_encrypted
- FROM dynamic_clients
- WHERE owner_kind = $1 AND server_namespace = $2 AND server_name = $3
- FOR UPDATE`,
- [ownerKind, input.key.serverNamespace, input.key.serverName]
- )
- if (locked.rows.length === 0) return { reclaimed: false }
- const row = locked.rows[0] as {
- id: string
- install_id: string | null
- cr_uid: string | null
- updated_at: Date
- registration_client_uri: string | null
- registration_access_token_encrypted: string | null
- }
-
- const klass = classifyExistingDynamicClient(
- { installId: row.install_id, crUid: row.cr_uid, updatedAt: row.updated_at },
- input.ctx
- )
- if (klass !== 'reclaimable') return { reclaimed: false }
-
const updated = await tx.query(
- `UPDATE dynamic_clients
+ `WITH prev AS (
+ SELECT id, registration_client_uri, registration_access_token_encrypted
+ FROM dynamic_clients
+ WHERE owner_kind = $1 AND server_namespace = $2 AND server_name = $3
+ AND id = $13
+ AND install_id IS NOT DISTINCT FROM $14
+ AND cr_uid IS NOT DISTINCT FROM $15
+ FOR UPDATE
+ )
+ UPDATE dynamic_clients d
SET issuer = $4,
client_id = $5,
client_mode = $6,
@@ -451,8 +469,9 @@ export async function reclaimOrphanDynamicClient(
install_id = $12,
cr_uid = NULL,
updated_at = NOW()
- WHERE owner_kind = $1 AND server_namespace = $2 AND server_name = $3
- AND id = $13 AND install_id IS NOT DISTINCT FROM $14`,
+ FROM prev
+ WHERE d.id = prev.id
+ RETURNING prev.registration_client_uri, prev.registration_access_token_encrypted`,
[
ownerKind,
input.key.serverNamespace,
@@ -466,19 +485,24 @@ export async function reclaimOrphanDynamicClient(
creds.clientIdIssuedAt,
creds.clientSecretExpiresAt,
input.newInstallId,
- row.id,
- row.install_id,
+ input.observed.id,
+ input.observed.installId ?? null,
+ input.observed.crUid ?? null,
]
)
if ((updated.rowCount ?? 0) !== 1) return { reclaimed: false }
+ const prev = updated.rows[0] as {
+ registration_client_uri: string | null
+ registration_access_token_encrypted: string | null
+ }
const oldHandle =
- row.registration_client_uri && row.registration_access_token_encrypted
+ prev.registration_client_uri && prev.registration_access_token_encrypted
? {
- registrationClientUri: row.registration_client_uri,
+ registrationClientUri: prev.registration_client_uri,
registrationAccessToken: decryptOAuthSecret(
encryptionKey,
- row.registration_access_token_encrypted
+ prev.registration_access_token_encrypted
),
}
: undefined
diff --git a/control-api/src/routes/admin/remoteMcp.ts b/control-api/src/routes/admin/remoteMcp.ts
index d613baeb7..81d99900a 100644
--- a/control-api/src/routes/admin/remoteMcp.ts
+++ b/control-api/src/routes/admin/remoteMcp.ts
@@ -808,11 +808,11 @@ export function createAdminRemoteMcpRouter(
// confidential client is orphaned at the AS. Mirrors the revoke-on-abort
// discipline of persistFailed503 and every explicit return below.
try {
- // Someone already holds the name. A live CR owns it unconditionally; else the
- // persisted row's lifetime decides (in-progress = another saga; reclaimable =
- // orphan/legacy). Re-read the live uid HERE (a CR may have appeared since the
- // pre-check — the TOCTOU the fence closes).
- const liveCrUid = await readLiveMcpServerUid(gateway, serverName, targetNs)
+ // Someone already holds the name. `existing` is always the LAST row read,
+ // and the live-uid read happens only after it: an owner that creates its CR
+ // and binds between the two is then seen as `in-use`, and one that binds
+ // after the K8s read breaks the reclaim's cr_uid guard. Reading K8s first
+ // would classify a just-bound live row as an orphan and revoke its client.
let existing = await getDynamicClient(db, encryptionKey, dynamicClientKey)
if (!existing) {
// The row was deleted between our INSERT-conflict and this read (a teardown
@@ -838,13 +838,16 @@ export function createAdminRemoteMcpRouter(
res.status(409).json({ error: 'server_name_in_use' })
return
}
+ // A live CR owns the name unconditionally (also covers a CR that appeared
+ // since the pre-check).
+ const liveCrUid = await readLiveMcpServerUid(gateway, serverName, targetNs)
const klass = classifyExistingDynamicClient(
{
installId: existing.installId ?? null,
crUid: existing.crUid ?? null,
- updatedAt: existing.updatedAt,
+ ageMs: existing.ageMs,
},
- { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid }
+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid }
)
if (klass === 'in-use' || klass === 'in-progress') {
await revokeMintedClient()
@@ -857,19 +860,21 @@ export function createAdminRemoteMcpRouter(
})
return
}
- // reclaimable: take over the orphan/legacy row atomically (CAS in a tx).
+ // reclaimable: take over the orphan/legacy row only if it is still exactly
+ // the one we observed (id + install_id + cr_uid).
const reclaim = await reclaimOrphanDynamicClient(
encryptionKey,
{
key: dynamicClientKey,
newCredentials: dcrCredentials,
newInstallId: installId,
- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid },
+ observed: existing,
},
runInTransaction
)
if (!reclaim.reclaimed) {
- // Raced: another saga won the reclaim (or re-claimed the freed name).
+ // The row changed since we read it: another saga reclaimed it, its owner
+ // bound it, or it was deleted.
await revokeMintedClient()
res.status(409).json({ error: 'install_in_progress' })
return
diff --git a/control-api/test/fixtures/remoteOAuthDiscovery.ts b/control-api/test/fixtures/remoteOAuthDiscovery.ts
index 85cd74f09..b397c800e 100644
--- a/control-api/test/fixtures/remoteOAuthDiscovery.ts
+++ b/control-api/test/fixtures/remoteOAuthDiscovery.ts
@@ -383,8 +383,8 @@ export function makeDcrTransport(opts: {
*
* It models the install-identity statements the route now drives: the pending INSERT
* (`ON CONFLICT DO NOTHING RETURNING id`), the legacy upsert (`ON CONFLICT DO UPDATE`),
- * the bind UPDATE, the reclaim CAS UPDATE, and both DELETE shapes (by-install and
- * by-name). The AUTHORITATIVE SQL semantics — real ON CONFLICT, the FOR UPDATE + CAS
+ * the bind UPDATE, the reclaim guarded UPDATE, and both DELETE shapes (by-install and
+ * by-name). The AUTHORITATIVE SQL semantics — real ON CONFLICT, the reclaim guard
* race — are certified by the real-Postgres suite
* (`oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts`); this
* harness only lets the route-level wiring (classify → HTTP code, which 7592 DELETE
@@ -502,7 +502,9 @@ export function makeInMemoryDynamicClientsDb(): {
const key = keyOf(values[0], values[1], values[2])
const row = rows.get(key)
if (!row) return { rows: [], rowCount: 0 }
- // Reclaim CAS: WHERE id = $13 AND install_id IS NOT DISTINCT FROM $14.
+ // Reclaim: one guard on the observed identity — id = $13 AND install_id IS
+ // NOT DISTINCT FROM $14 AND cr_uid IS NOT DISTINCT FROM $15 — RETURNING the
+ // OLD row's RFC 7592 handle.
if (text.includes('IS NOT DISTINCT FROM')) {
const [
,
@@ -517,12 +519,21 @@ export function makeInMemoryDynamicClientsDb(): {
client_id_issued_at,
client_secret_expires_at,
newInstallId,
- lockedId,
- lockedInstallId,
+ observedId,
+ observedInstallId,
+ observedCrUid,
] = values
- if (row.id !== lockedId || row.install_id !== lockedInstallId) {
+ if (
+ String(row.id) !== String(observedId) ||
+ (row.install_id ?? null) !== (observedInstallId ?? null) ||
+ (row.cr_uid ?? null) !== (observedCrUid ?? null)
+ ) {
return { rows: [], rowCount: 0 }
}
+ const oldHandle = {
+ registration_client_uri: row.registration_client_uri,
+ registration_access_token_encrypted: row.registration_access_token_encrypted,
+ }
Object.assign(row, {
issuer,
client_id,
@@ -536,7 +547,7 @@ export function makeInMemoryDynamicClientsDb(): {
cr_uid: null,
updated_at: new Date(),
})
- return { rows: [], rowCount: 1 }
+ return { rows: [oldHandle], rowCount: 1 }
}
// Bind: WHERE install_id = $4 AND cr_uid IS NULL, SET cr_uid = $5.
const [, , , installId, crUid] = values
@@ -551,9 +562,11 @@ export function makeInMemoryDynamicClientsDb(): {
const [owner_kind, server_namespace, server_name] = values
const row = rows.get(keyOf(owner_kind, server_namespace, server_name))
// Return a SNAPSHOT (shallow copy), as a real SELECT does: a later UPDATE must
- // not retroactively mutate a row already read (e.g. the reclaim reads the OLD
- // handle from its FOR UPDATE snapshot AFTER issuing the overwriting UPDATE).
- return { rows: row ? [{ ...row }] : [], rowCount: row ? 1 : 0 }
+ // not retroactively mutate a row already read. `age_ms` mirrors the store's
+ // `NOW() - updated_at` column, measured at read time.
+ if (!row) return { rows: [], rowCount: 0 }
+ const ageMs = Date.now() - (row.updated_at as Date).getTime()
+ return { rows: [{ ...row, age_ms: ageMs }], rowCount: 1 }
}
return { rows: [], rowCount: 0 }
},
diff --git a/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts b/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts
index 99782e42a..9da823f32 100644
--- a/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts
+++ b/control-api/test/oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts
@@ -44,38 +44,36 @@ const NS = config.mcpServersNamespace
// classifyExistingDynamicClient is pure — cover its exhaustive table with no DB.
describe('classifyExistingDynamicClient — ownership decision table', () => {
- const now = new Date('2026-01-01T00:00:00.000Z')
- const fresh = new Date(now.getTime() - 1000)
- const expired = new Date(now.getTime() - PENDING_TTL_MS - 1000)
+ const fresh = 1000
+ const expired = PENDING_TTL_MS + 1000
it('a live CR owns the name regardless of the row state → in-use', () => {
expect(
classifyExistingDynamicClient(
- { installId: null, crUid: null, updatedAt: fresh },
- { now, pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }
+ { installId: null, crUid: null, ageMs: fresh },
+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }
)
).toBe('in-use')
expect(
classifyExistingDynamicClient(
- { installId: 'i1', crUid: 'uid-old', updatedAt: expired },
- { now, pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }
+ { installId: 'i1', crUid: 'uid-old', ageMs: expired },
+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }
)
).toBe('in-use')
// A live CR beats even a fresh pending row (the pre-check→INSERT TOCTOU race).
expect(
classifyExistingDynamicClient(
- { installId: 'i1', crUid: null, updatedAt: fresh },
- { now, pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }
+ { installId: 'i1', crUid: null, ageMs: fresh },
+ { pendingTtlMs: PENDING_TTL_MS, liveCrUid: 'uid-live' }
)
).toBe('in-use')
})
it('at exactly the TTL boundary a pending row is reclaimable (in-progress is strictly under)', () => {
- const atBoundary = new Date(now.getTime() - PENDING_TTL_MS)
expect(
classifyExistingDynamicClient(
- { installId: 'i1', crUid: null, updatedAt: atBoundary },
- { now, pendingTtlMs: PENDING_TTL_MS }
+ { installId: 'i1', crUid: null, ageMs: PENDING_TTL_MS },
+ { pendingTtlMs: PENDING_TTL_MS }
)
).toBe('reclaimable')
})
@@ -83,8 +81,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {
it('a fresh pending row with no live CR → in-progress', () => {
expect(
classifyExistingDynamicClient(
- { installId: 'i1', crUid: null, updatedAt: fresh },
- { now, pendingTtlMs: PENDING_TTL_MS }
+ { installId: 'i1', crUid: null, ageMs: fresh },
+ { pendingTtlMs: PENDING_TTL_MS }
)
).toBe('in-progress')
})
@@ -92,8 +90,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {
it('a bound orphan (cr_uid set, no live CR) → reclaimable', () => {
expect(
classifyExistingDynamicClient(
- { installId: 'i1', crUid: 'uid-gone', updatedAt: fresh },
- { now, pendingTtlMs: PENDING_TTL_MS }
+ { installId: 'i1', crUid: 'uid-gone', ageMs: fresh },
+ { pendingTtlMs: PENDING_TTL_MS }
)
).toBe('reclaimable')
})
@@ -101,8 +99,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {
it('an expired pending row → reclaimable', () => {
expect(
classifyExistingDynamicClient(
- { installId: 'i1', crUid: null, updatedAt: expired },
- { now, pendingTtlMs: PENDING_TTL_MS }
+ { installId: 'i1', crUid: null, ageMs: expired },
+ { pendingTtlMs: PENDING_TTL_MS }
)
).toBe('reclaimable')
})
@@ -110,8 +108,8 @@ describe('classifyExistingDynamicClient — ownership decision table', () => {
it('a legacy row (both null, no live CR) → reclaimable', () => {
expect(
classifyExistingDynamicClient(
- { installId: null, crUid: null, updatedAt: fresh },
- { now, pendingTtlMs: PENDING_TTL_MS }
+ { installId: null, crUid: null, ageMs: fresh },
+ { pendingTtlMs: PENDING_TTL_MS }
)
).toBe('reclaimable')
})
@@ -198,38 +196,80 @@ describeRealPostgres('dynamicClientStore install identity (real Postgres)', () =
expect(after?.registrationClientUri).toBe(`https://as.example.com/reg/${name}`)
})
- it('reclaimOrphanDynamicClient CAS: takes over a reclaimable row, returns the old handle, second reclaim loses', async () => {
+ const keyOf = (serverName: string) => ({ serverNamespace: NS, serverName })
+
+ /**
+ * Age a real row past the pending TTL. Only `updated_at` moves (the DB clock the
+ * store measures age against); every other column stays as the producer wrote it.
+ */
+ async function expirePending(serverName: string): Promise<void> {
+ await db.query(
+ `UPDATE dynamic_clients
+ SET updated_at = NOW() - ($1::bigint * INTERVAL '1 millisecond')
+ WHERE server_namespace = $2 AND server_name = $3`,
+ [PENDING_TTL_MS + 60_000, NS, serverName]
+ )
+ }
+
+ /** Every persisted column; `ageMs` is a read-time measurement, not row state. */
+ const rowState = (row: Awaited<ReturnType<typeof getDynamicClient>>) => {
+ if (!row) return row
+ const { ageMs: _ageMs, ...state } = row
+ return state
+ }
+
+ it('the row age is measured by the database: fresh pending → in-progress, aged → reclaimable', async () => {
+ const name = `srv-age-${randomUUID().slice(0, 8)}`
+ await insertDynamicClientPending(db, KEY, { ...baseCreds(name), installId: randomUUID() })
+
+ const fresh = await getDynamicClient(db, KEY, keyOf(name))
+ expect(fresh?.id).toMatch(/^\d+$/)
+ expect(fresh?.ageMs).toBeGreaterThanOrEqual(0)
+ expect(fresh?.ageMs).toBeLessThan(PENDING_TTL_MS)
+ expect(
+ classifyExistingDynamicClient(
+ { installId: fresh?.installId ?? null, crUid: null, ageMs: fresh?.ageMs ?? 0 },
+ { pendingTtlMs: PENDING_TTL_MS }
+ )
+ ).toBe('in-progress')
+
+ await expirePending(name)
+ const aged = await getDynamicClient(db, KEY, keyOf(name))
+ expect(aged?.ageMs).toBeGreaterThanOrEqual(PENDING_TTL_MS)
+ expect(
+ classifyExistingDynamicClient(
+ { installId: aged?.installId ?? null, crUid: null, ageMs: aged?.ageMs ?? 0 },
+ { pendingTtlMs: PENDING_TTL_MS }
+ )
+ ).toBe('reclaimable')
+ })
+
+ it('reclaimOrphanDynamicClient takes over the observed orphan and returns the old handle', async () => {
const name = `srv-reclaim-${randomUUID().slice(0, 8)}`
const install1 = randomUUID()
- const uidGone = randomUUID()
// Seed a BOUND ORPHAN via the real producers: pending insert, then bind to a
- // uid whose CR no longer exists (liveCrUid undefined at reclaim time).
+ // uid whose CR no longer exists.
await insertDynamicClientPending(db, KEY, {
...baseCreds(name, { clientId: 'cid-old', clientSecret: 'secret-old' }),
registrationAccessToken: 'rat-old',
registrationClientUri: 'https://as.example.com/reg/old',
installId: install1,
})
- expect(
- (
- await bindDynamicClientToResource(
- db,
- { serverNamespace: NS, serverName: name },
- install1,
- uidGone
- )
- ).bound
- ).toBe(true)
+ expect((await bindDynamicClientToResource(db, keyOf(name), install1, randomUUID())).bound).toBe(
+ true
+ )
+ const observed = await getDynamicClient(db, KEY, keyOf(name))
+ if (!observed) throw new Error('seeded row missing')
const install2 = randomUUID()
const reclaim = await reclaimOrphanDynamicClient(
KEY,
{
- key: { serverNamespace: NS, serverName: name },
+ key: keyOf(name),
newCredentials: baseCreds(name, { clientId: 'cid-new', clientSecret: 'secret-new' }),
newInstallId: install2,
- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid: undefined },
+ observed,
},
runInTransaction
)
@@ -240,54 +280,131 @@ describeRealPostgres('dynamicClientStore install identity (real Postgres)', () =
registrationAccessToken: 'rat-old',
})
- const owned = await getDynamicClient(db, KEY, { serverNamespace: NS, serverName: name })
+ const owned = await getDynamicClient(db, KEY, keyOf(name))
+ expect(owned?.id).toBe(observed.id)
expect(owned?.installId).toBe(install2)
expect(owned?.crUid).toBeUndefined()
expect(owned?.clientId).toBe('cid-new')
expect(owned?.clientSecret).toBe('secret-new')
+ expect(owned?.registrationClientUri).toBe(`https://as.example.com/reg/${name}`)
+ })
+
+ // A pending row observed past its TTL (a crashed-looking saga) whose owner then
+ // BINDS it to its freshly created CR before the reclaim lands. Only `cr_uid`
+ // changes on bind, so the reclaim must refuse on the cr_uid guard — otherwise a
+ // live install's credentials are overwritten and its client revoked at the AS.
+ it('a row bound by its owner after it was observed is never reclaimed', async () => {
+ const name = `srv-bindrace-${randomUUID().slice(0, 8)}`
+ const ownerInstall = randomUUID()
+ await insertDynamicClientPending(db, KEY, {
+ ...baseCreds(name, { clientId: 'cid-owner', clientSecret: 'secret-owner' }),
+ installId: ownerInstall,
+ })
+ await expirePending(name)
+
+ const observed = await getDynamicClient(db, KEY, keyOf(name))
+ if (!observed) throw new Error('seeded row missing')
+ // No live CR yet at this point → the caller classifies it reclaimable.
+ expect(
+ classifyExistingDynamicClient(
+ {
+ installId: observed.installId ?? null,
+ crUid: observed.crUid ?? null,
+ ageMs: observed.ageMs,
+ },
+ { pendingTtlMs: PENDING_TTL_MS }
+ )
+ ).toBe('reclaimable')
+
+ // The owner's CR now exists and it binds its row.
+ const ownerUid = randomUUID()
+ expect((await bindDynamicClientToResource(db, keyOf(name), ownerInstall, ownerUid)).bound).toBe(
+ true
+ )
+ const bound = await getDynamicClient(db, KEY, keyOf(name))
+
+ const reclaim = await reclaimOrphanDynamicClient(
+ KEY,
+ {
+ key: keyOf(name),
+ newCredentials: baseCreds(name, { clientId: 'cid-thief', clientSecret: 'secret-thief' }),
+ newInstallId: randomUUID(),
+ observed,
+ },
+ runInTransaction
+ )
+ expect(reclaim.reclaimed).toBe(false)
+ expect(reclaim.oldHandle).toBeUndefined()
+ const after = await getDynamicClient(db, KEY, keyOf(name))
+ expect(after?.crUid).toBe(ownerUid)
+ expect(after?.clientSecret).toBe('secret-owner')
+ expect(rowState(after)).toEqual(rowState(bound))
+ })
+
+ // Two sagas observe the same reclaimable row; B′ reclaims first. B′'s reclaim
+ // leaves cr_uid NULL (as observed) and keeps the id, so only install_id tells B
+ // that the row is no longer the one it classified.
+ it('the second of two reclaims of the same observed row loses on install_id', async () => {
+ const name = `srv-tworeclaims-${randomUUID().slice(0, 8)}`
+ await insertDynamicClientPending(db, KEY, {
+ ...baseCreds(name, { clientId: 'cid-crashed' }),
+ installId: randomUUID(),
+ })
+ await expirePending(name)
+ const observed = await getDynamicClient(db, KEY, keyOf(name))
+ if (!observed) throw new Error('seeded row missing')
+
+ const winnerInstall = randomUUID()
+ const first = await reclaimOrphanDynamicClient(
+ KEY,
+ {
+ key: keyOf(name),
+ newCredentials: baseCreds(name, { clientId: 'cid-winner', clientSecret: 'secret-winner' }),
+ newInstallId: winnerInstall,
+ observed,
+ },
+ runInTransaction
+ )
+ expect(first.reclaimed).toBe(true)
+ const won = await getDynamicClient(db, KEY, keyOf(name))
- // Row is now fresh-pending (install2). A second reclaim classifies it
- // in-progress → loses. Models the concurrent second reclaim with a stale
- // old-install expectation.
- const install3 = randomUUID()
const second = await reclaimOrphanDynamicClient(
KEY,
{
- key: { serverNamespace: NS, serverName: name },
- newCredentials: baseCreds(name, { clientId: 'cid-loser' }),
- newInstallId: install3,
- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid: undefined },
+ key: keyOf(name),
+ newCredentials: baseCreds(name, { clientId: 'cid-loser', clientSecret: 'secret-loser' }),
+ newInstallId: randomUUID(),
+ observed,
},
runInTransaction
)
expect(second.reclaimed).toBe(false)
- const afterLoser = await getDynamicClient(db, KEY, { serverNamespace: NS, serverName: name })
- expect(afterLoser?.installId).toBe(install2)
- expect(afterLoser?.clientId).toBe('cid-new')
+ const after = await getDynamicClient(db, KEY, keyOf(name))
+ expect(after?.installId).toBe(winnerInstall)
+ expect(after?.clientId).toBe('cid-winner')
+ expect(rowState(after)).toEqual(rowState(won))
})
- it('reclaimOrphanDynamicClient leaves an in-progress (fresh pending) row alone', async () => {
- const name = `srv-noreclaim-${randomUUID().slice(0, 8)}`
+ it('reclaimOrphanDynamicClient loses when the observed row was deleted', async () => {
+ const name = `srv-reclaimgone-${randomUUID().slice(0, 8)}`
const install1 = randomUUID()
- await insertDynamicClientPending(db, KEY, {
- ...baseCreds(name, { clientId: 'cid-inflight' }),
- installId: install1,
- })
+ await insertDynamicClientPending(db, KEY, { ...baseCreds(name), installId: install1 })
+ const observed = await getDynamicClient(db, KEY, keyOf(name))
+ if (!observed) throw new Error('seeded row missing')
+ await deleteDynamicClientOwnedByInstall(db, keyOf(name), install1)
- const result = await reclaimOrphanDynamicClient(
+ const reclaim = await reclaimOrphanDynamicClient(
KEY,
{
- key: { serverNamespace: NS, serverName: name },
- newCredentials: baseCreds(name, { clientId: 'cid-would-steal' }),
+ key: keyOf(name),
+ newCredentials: baseCreds(name, { clientId: 'cid-late' }),
newInstallId: randomUUID(),
- ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid: undefined },
+ observed,
},
runInTransaction
)
- expect(result.reclaimed).toBe(false)
- const row = await getDynamicClient(db, KEY, { serverNamespace: NS, serverName: name })
- expect(row?.installId).toBe(install1)
- expect(row?.clientId).toBe('cid-inflight')
+ expect(reclaim.reclaimed).toBe(false)
+ expect(await getDynamicClient(db, KEY, keyOf(name))).toBeNull()
})
it('bindDynamicClientToResource binds our pending row and refuses a non-matching install', async () => {
diff --git a/control-api/test/routes.adminRemoteMcp.test.ts b/control-api/test/routes.adminRemoteMcp.test.ts
index adc3d6be9..6bd833579 100644
--- a/control-api/test/routes.adminRemoteMcp.test.ts
+++ b/control-api/test/routes.adminRemoteMcp.test.ts
@@ -3,7 +3,7 @@ import express from 'express'
import { randomUUID } from 'node:crypto'
import request from 'supertest'
import { config } from '../src/config.js'
-import type { DbTransactionClient } from '../src/db.js'
+import type { DbClient, DbTransactionClient } from '../src/db.js'
import type { PinnedTransport } from '../src/http/pinnedFetch.js'
import type { K8sGateway } from '../src/k8s.js'
import {
@@ -12,7 +12,9 @@ import {
discoverRemoteOAuth,
} from '../src/oauth/discovery.js'
import {
+ PENDING_TTL_MS,
bindDynamicClientToResource,
+ claimDeleteDynamicClientForResource,
getDynamicClient,
insertDynamicClientPending,
} from '../src/oauth/dynamicClientStore.js'
@@ -1157,6 +1159,13 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {
// row. Rows are seeded via the REAL producers (insertDynamicClientPending + bind),
// read through the REAL getDynamicClient (T1).
+ /** Every persisted column; `ageMs` is a read-time measurement, not row state. */
+ const rowState = (row: Awaited<ReturnType<typeof getDynamicClient>>) => {
+ if (!row) return row
+ const { ageMs: _ageMs, ...state } = row
+ return state
+ }
+
const LIVE_ROW = {
ownerKind: 'mcpserver' as const,
serverNamespace: NS,
@@ -1219,7 +1228,7 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {
serverNamespace: NS,
serverName: 'live-dcr',
})
- expect(after).toEqual(before)
+ expect(rowState(after)).toEqual(rowState(before))
expect(rows.size).toBe(1)
})
@@ -1281,7 +1290,7 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {
serverNamespace: NS,
serverName: 'toctou-dcr',
})
- expect(after).toEqual(before)
+ expect(rowState(after)).toEqual(rowState(before))
// The throwaway client minted before the conflict was revoked at the AS, and the
// seeded live client's handle was NOT touched.
const deleteCalls = calls.filter(c => c.method === 'DELETE')
@@ -1382,7 +1391,7 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {
serverNamespace: NS,
serverName: 'inprogress-dcr',
})
- expect(after).toEqual(before)
+ expect(rowState(after)).toEqual(rowState(before))
expect(rows.size).toBe(1)
// Our throwaway minted client was revoked at the AS.
expect(calls.some(c => c.method === 'DELETE')).toBe(true)
@@ -1440,4 +1449,241 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {
const deleteCalls = calls.filter(c => c.method === 'DELETE')
expect(deleteCalls.map(c => c.url)).toContain(LIVE_ROW.registrationClientUri)
})
+
+ // ── The live-uid read must follow the LAST row read ─────────────────────────
+ // Install A of the same name finishes (CR created, row bound) while B is resolving
+ // its INSERT conflict. The hook below plays A's steps with the REAL producers
+ // (gateway createResource + bindDynamicClientToResource) at the exact point in B's
+ // flow under test. Asserted on what is observable: A's client is never revoked at
+ // the AS and A's row is byte-identical. The status is 409 either way, so it does
+ // not discriminate.
+
+ const RACE_A_ROW = {
+ ...LIVE_ROW,
+ clientId: 'owner-a-client-id',
+ clientSecret: 'owner-a-secret',
+ registrationAccessToken: 'owner-a-reg-token',
+ registrationClientUri: 'https://as.example.com/register/owner-a',
+ }
+
+ /**
+ * Wrap the in-memory db so hooks run around B's Nth plain row read
+ * (`getDynamicClient`), counted only once armed: `before` effects are visible to
+ * that read, `after` effects land once its snapshot is taken.
+ */
+ function withRowReadHook(
+ inner: DbClient,
+ plan: Record<number, { before?: () => Promise<void>; after?: () => Promise<void> }>
+ ): { db: DbClient; arm: () => void } {
+ let armed = false
+ let reads = 0
+ const db = {
+ query: async (text: string, values?: unknown[]) => {
+ const isRowRead =
+ armed &&
+ text.includes('FROM dynamic_clients') &&
+ !text.includes('UPDATE') &&
+ !text.includes('DELETE') &&
+ !text.includes('FOR UPDATE')
+ if (!isRowRead) return inner.query(text, values)
+ reads += 1
+ const step = plan[reads]
+ await step?.before?.()
+ const result = await inner.query(text, values)
+ await step?.after?.()
+ return result
+ },
+ } as unknown as DbClient
+ return { db, arm: () => (armed = true) }
+ }
+
+ const rowKey = (serverName: string) => `mcpserver/${NS}/${serverName}`
+
+ /**
+ * Age a real row past the pending TTL. Only `updated_at` moves (what the store's
+ * age column measures); every other column stays as the producer wrote it.
+ */
+ function expirePending(rows: Map<string, Record<string, unknown>>, serverName: string): void {
+ const row = rows.get(rowKey(serverName))
+ if (!row) throw new Error(`no row for ${serverName}`)
+ row.updated_at = new Date(Date.now() - PENDING_TTL_MS - 60_000)
+ }
+
+ /**
+ * A's CR appears (real gateway create) and A binds its row to that CR's uid.
+ * Returns A's row as persisted right after the bind — the state that must survive.
+ */
+ async function ownerAFinishes(
+ gw: MockGateway,
+ inner: DbClient,
+ serverName: string,
+ installA: string
+ ) {
+ const cr = (await gw.createResource(
+ 'mcpservers',
+ { metadata: { name: serverName }, spec: {} },
+ NS
+ )) as { metadata: { uid: string } }
+ const bound = await bindDynamicClientToResource(
+ inner,
+ { serverNamespace: NS, serverName },
+ installA,
+ cr.metadata.uid
+ )
+ if (!bound.bound) throw new Error('owner A failed to bind its row')
+ return rowState(await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName }))
+ }
+
+ async function installB(gw: MockGateway, db: DbClient, serverName: string) {
+ const { transport, calls } = makeDcrTransport({
+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),
+ })
+ const res = await request(
+ makeAppWithDeps(gw, {
+ db,
+ dcr: { transport, resolveDns: PUBLIC_IP },
+ runInTransaction: work => work(db as unknown as DbTransactionClient),
+ })
+ )
+ .post('/admin/mcp-servers/remote')
+ .send({
+ serverName,
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'dcr',
+ })
+ return { res, deleteUrls: calls.filter(c => c.method === 'DELETE').map(c => c.url) }
+ }
+
+ it("A binds its row between B's row read and B's live-CR read → A's client is never revoked, A's row untouched", async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db: inner, rows } = makeInMemoryDynamicClientsDb()
+ const serverName = 'race-bind-dcr'
+ const installA = randomUUID()
+ await insertDynamicClientPending(inner, ENC_KEY, {
+ ...RACE_A_ROW,
+ serverName,
+ installId: installA,
+ })
+ // A's saga has been slow: B reads a pending row already past the TTL.
+ expirePending(rows, serverName)
+
+ const gw = gatewayWithContext('ctx-a')
+ let aState: Awaited<ReturnType<typeof ownerAFinishes>> | undefined
+ const { db, arm } = withRowReadHook(inner, {
+ 1: {
+ after: async () => {
+ aState = await ownerAFinishes(gw, inner, serverName, installA)
+ },
+ },
+ })
+ arm()
+ const { res, deleteUrls } = await installB(gw, db, serverName)
+
+ expect(res.status).toBe(409)
+ expect(aState).toBeDefined()
+ expect(deleteUrls).not.toContain(RACE_A_ROW.registrationClientUri)
+ // B still revokes the throwaway client it minted.
+ expect(deleteUrls).toContain(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri)
+ const afterA = await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName })
+ expect(rowState(afterA)).toEqual(aState)
+ })
+
+ // A finishes (CR created, row bound) before B even reads the row. The row B reads
+ // is already bound, so only a live-CR read taken AFTER that row read can tell it
+ // is in use; with the K8s read first, B sees no CR, classifies the bound row as an
+ // orphan, and its reclaim guard matches the row it just read.
+ it("A binds its row before B's row read → B's live-CR read follows it, A's client is never revoked", async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db: inner, rows } = makeInMemoryDynamicClientsDb()
+ const serverName = 'race-prebind-dcr'
+ const installA = randomUUID()
+ await insertDynamicClientPending(inner, ENC_KEY, {
+ ...RACE_A_ROW,
+ serverName,
+ installId: installA,
+ })
+ expirePending(rows, serverName)
+
+ const gw = gatewayWithContext('ctx-a')
+ let aState: Awaited<ReturnType<typeof ownerAFinishes>> | undefined
+ const { db, arm } = withRowReadHook(inner, {
+ 1: {
+ before: async () => {
+ aState = await ownerAFinishes(gw, inner, serverName, installA)
+ },
+ },
+ })
+ arm()
+ const { res, deleteUrls } = await installB(gw, db, serverName)
+
+ expect(res.status).toBe(409)
+ expect(aState).toBeDefined()
+ expect(deleteUrls).not.toContain(RACE_A_ROW.registrationClientUri)
+ expect(deleteUrls).toContain(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri)
+ const afterA = await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName })
+ expect(rowState(afterA)).toEqual(aState)
+ })
+
+ // Same race through the retry branch: B's first row read finds the row gone (a
+ // teardown), A reinstalls the name before B's retried INSERT, and A creates its CR
+ // and binds before B's second row read.
+ it("retry branch: A reinserts and binds before B's re-read → A's client is never revoked, A's row untouched", async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db: inner } = makeInMemoryDynamicClientsDb()
+ const serverName = 'race-retry-dcr'
+ // The row B's INSERT first conflicts with: an orphan of an earlier install.
+ const orphanInstall = randomUUID()
+ await insertDynamicClientPending(inner, ENC_KEY, {
+ ...LIVE_ROW,
+ serverName,
+ installId: orphanInstall,
+ })
+ await bindDynamicClientToResource(
+ inner,
+ { serverNamespace: NS, serverName },
+ orphanInstall,
+ 'uid-orphan-gone'
+ )
+
+ const gw = gatewayWithContext('ctx-a')
+ const installA = randomUUID()
+ let aState: Awaited<ReturnType<typeof ownerAFinishes>> | undefined
+ const { db, arm } = withRowReadHook(inner, {
+ // B's first read finds the orphan torn down; A then claims the name again,
+ // ahead of B's retried INSERT.
+ 1: {
+ before: async () => {
+ await claimDeleteDynamicClientForResource(
+ inner,
+ ENC_KEY,
+ { serverNamespace: NS, serverName },
+ 'uid-orphan-gone'
+ )
+ },
+ after: async () => {
+ await insertDynamicClientPending(inner, ENC_KEY, {
+ ...RACE_A_ROW,
+ serverName,
+ installId: installA,
+ })
+ },
+ },
+ // A creates its CR and binds before B re-reads the row.
+ 2: {
+ before: async () => {
+ aState = await ownerAFinishes(gw, inner, serverName, installA)
+ },
+ },
+ })
+ arm()
+ const { res, deleteUrls } = await installB(gw, db, serverName)
+
+ expect(res.status).toBe(409)
+ expect(aState).toBeDefined()
+ expect(deleteUrls).not.toContain(RACE_A_ROW.registrationClientUri)
+ expect(deleteUrls).toContain(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri)
+ const afterA = await getDynamicClient(inner, ENC_KEY, { serverNamespace: NS, serverName })
+ expect(rowState(afterA)).toEqual(aState)
+ })
})