Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): fence the remote install saga dynamic client writes by install identity (R3-H1)

Diego Zacarias90041bbf2b68
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 7988b7a1c..4c29ba6c0 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.664",
+ "version": "0.1.665",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.664",
+ "version": "0.1.665",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 7edfdcb20..63e5abaa8 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.664",
+ "version": "0.1.665",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/oauth/dynamicClientStore.ts b/control-api/src/oauth/dynamicClientStore.ts
index 1ece1d4e9..8be4832d5 100644
--- a/control-api/src/oauth/dynamicClientStore.ts
+++ b/control-api/src/oauth/dynamicClientStore.ts
@@ -115,10 +115,15 @@ function encodeCredentialColumns(
/**
* Insert (or replace) the dynamic client for a server CR. Re-registration
- * overwrites in place (INSERT … ON CONFLICT DO UPDATE), which is also the only
- * rotation path in C2 — there is no dedicated rotate fn (no caller; avoids dead
- * code, DEC-18). `client_secret` and `registration_access_token` are encrypted
- * at rest; a public client stores NULL for the secret.
+ * overwrites in place (INSERT … ON CONFLICT DO UPDATE). `client_secret` and
+ * `registration_access_token` are encrypted at rest; a public client stores NULL
+ * for the secret.
+ *
+ * NO production caller: the remote-install saga now claims the name with
+ * `insertDynamicClientPending` (ON CONFLICT DO NOTHING) so an install can never
+ * clobber a live server's row. This clobbering upsert is retained ONLY as a store
+ * primitive exercised by its own unit tests and as a row-seeder for other suites;
+ * it must NOT be reintroduced into any install/reclaim path.
*/
export async function upsertDynamicClient(
db: DbClient,
diff --git a/control-api/src/routes/admin/remoteMcp.ts b/control-api/src/routes/admin/remoteMcp.ts
index e8ced5ea6..d613baeb7 100644
--- a/control-api/src/routes/admin/remoteMcp.ts
+++ b/control-api/src/routes/admin/remoteMcp.ts
@@ -1,7 +1,8 @@
import { Router } from 'express'
+import { randomUUID } from 'node:crypto'
import { z } from 'zod'
import { config } from '../../config.js'
-import { type DbClient, pool } from '../../db.js'
+import { type DbClient, type DbTransactionClient, pool, withTransaction } from '../../db.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import { extractK8sError } from '../../http/k8sError.js'
import { enforceNamespace } from '../../http/namespaceAudit.js'
@@ -21,8 +22,14 @@ import { type DiscoveryResult, discoverRemoteOAuth } from '../../oauth/discovery
import { discoveryHttpStatus } from '../../oauth/discoveryHttpStatus.js'
import {
type DynamicClientKey,
- deleteDynamicClient,
- upsertDynamicClient,
+ PENDING_TTL_MS,
+ type UpsertDynamicClientInput,
+ bindDynamicClientToResource,
+ classifyExistingDynamicClient,
+ deleteDynamicClientOwnedByInstall,
+ getDynamicClient,
+ insertDynamicClientPending,
+ reclaimOrphanDynamicClient,
} from '../../oauth/dynamicClientStore.js'
import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'
import { probeMcpTransport } from '../../oauth/mcpTransportProbe.js'
@@ -110,6 +117,31 @@ function resourcePreconditionsFrom(resource: unknown): ResourcePreconditions | u
return { uid, resourceVersion }
}
+/** The server-assigned `metadata.uid`, or undefined when the response carried none. */
+function resourceUidFrom(resource: unknown): string | undefined {
+ const uid = (resource as { metadata?: { uid?: unknown } } | null)?.metadata?.uid
+ return typeof uid === 'string' && uid ? uid : undefined
+}
+
+/**
+ * The uid of a live McpServer with this name, or undefined when none exists. A live
+ * CR owns the name: the reclaim/classify path treats a conflict against a live uid as
+ * `in-use` regardless of the persisted row's state. A 404 is "no live CR" (undefined);
+ * any other read error propagates (fail closed, not "assume free").
+ */
+async function readLiveMcpServerUid(
+ gateway: K8sGateway,
+ name: string,
+ namespace: string
+): Promise<string | undefined> {
+ try {
+ return resourceUidFrom(await gateway.getResource('mcpservers', name, namespace))
+ } catch (err) {
+ if (err instanceof K8sNotFoundError || extractK8sError(err)?.status === 404) return undefined
+ throw err
+ }
+}
+
/**
* The `spec.oauth` remote shape written to the CR — the C3 contract. Uses a distinct
* `source: 'remote'` discriminator instead of the closed 8-provider `provider` enum
@@ -221,6 +253,15 @@ const REMOTE_MCP_PROXY_PORT = 3000
const DELETE_SETTLE_TIMEOUT_MS = 10_000
const DELETE_SETTLE_POLL_MS = 250
+/**
+ * Production default for the DCR orphan-reclaim transaction. Wraps `withTransaction`
+ * so the binding is only accessed when a reclaim actually runs — not at router
+ * creation, which suites that partial-mock db.js would break.
+ */
+function defaultRunInTransaction<T>(work: (tx: DbTransactionClient) => Promise<T>): Promise<T> {
+ return withTransaction(work)
+}
+
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
/** Poll until a best-effort delete settles to a 404, so a re-install cannot 409. */
@@ -260,6 +301,13 @@ export interface AdminRemoteMcpDeps {
* inject one derived from the real producer to drive a specific probe outcome offline.
*/
probe?: typeof probeMcpTransport
+ /**
+ * Injectable transaction runner (test seam only). Production leaves it undefined and
+ * `withTransaction` (module pool) is used. The DCR orphan reclaim opens a transaction
+ * for its `SELECT … FOR UPDATE` + CAS; a test injects a runner bound to its in-memory
+ * db so the reclaim runs against the same store as the rest of the saga.
+ */
+ runInTransaction?: <T>(work: (tx: DbTransactionClient) => Promise<T>) => Promise<T>
}
export function createAdminRemoteMcpRouter(
@@ -272,6 +320,10 @@ export function createAdminRemoteMcpRouter(
const dcrDeps: DcrDeps = deps.dcr ?? { logger: log }
const discover = deps.discover ?? discoverRemoteOAuth
const probe = deps.probe ?? probeMcpTransport
+ // Defer the `withTransaction` binding to CALL time (only the DCR reclaim uses it).
+ // Referencing it here at router-creation would break the many suites that
+ // partial-mock db.js without a `withTransaction` export.
+ const runInTransaction = deps.runInTransaction ?? defaultRunInTransaction
// ── POST /admin/mcp-servers/remote/discover — dry-run, no writes ──────────
router.post(
@@ -386,6 +438,24 @@ export function createAdminRemoteMcpRouter(
return
}
+ // Pre-check: a name already taken by a LIVE McpServer is rejected before any
+ // discovery or AS mint, so we never register a throwaway client at the AS for a
+ // name we cannot use. This is NOT the fence — a CR could still appear between here
+ // and the persist (TOCTOU); the pending INSERT's ON CONFLICT below is the fence.
+ const nameInUseUid = await readLiveMcpServerUid(
+ gateway,
+ body.serverName,
+ config.mcpServersNamespace
+ )
+ if (nameInUseUid !== undefined) {
+ log.warn(
+ { event: 'remote_install_server_name_in_use', serverName: body.serverName },
+ 'remote install rejected: an McpServer with this name already exists'
+ )
+ res.status(409).json({ error: 'server_name_in_use' })
+ return
+ }
+
// D-4: re-run discovery SERVER-SIDE at install and pin the result — never trust
// the client-passed prefill as authoritative. Discovery kernel-guards baseUrl +
// every discovered endpoint internally before returning them.
@@ -545,6 +615,10 @@ export function createAdminRemoteMcpRouter(
serverNamespace: targetNs,
serverName,
}
+ // Install-ownership token minted at the start of DCR step 0. Every compensation
+ // deletes ONLY the row carrying this install_id, so a concurrent install/reinstall
+ // of the same name that reclaimed the row is never destroyed (R3-H1).
+ const installId = randomUUID()
let dcrRegistered = false
let dcrClientId: string | undefined
let dcrRegistrationClientUri: string | undefined
@@ -552,7 +626,7 @@ export function createAdminRemoteMcpRouter(
const rollbackDynamicClient = async (): Promise<void> => {
if (!dcrRegistered) return
try {
- await deleteDynamicClient(db, dynamicClientKey)
+ await deleteDynamicClientOwnedByInstall(db, dynamicClientKey, installId)
} catch {
// Best-effort local revocation; preserve the original saga error.
}
@@ -664,30 +738,9 @@ export function createAdminRemoteMcpRouter(
// AS. Compensate here explicitly (DEC-18).
const mintedRegistrationClientUri = registration.registration_client_uri
const mintedRegistrationAccessToken = registration.registration_access_token
- try {
- await upsertDynamicClient(db, encryptionKey, {
- ...dynamicClientKey,
- issuer: discovery.issuer,
- clientId: registration.client_id,
- clientMode: effectiveClientMode,
- // A public client carries no secret; never persist one the AS may have
- // echoed alongside a `none` downgrade.
- clientSecret:
- effectiveClientMode === 'confidential' ? registration.client_secret : undefined,
- registrationAccessToken: registration.registration_access_token,
- registrationClientUri: registration.registration_client_uri,
- clientIdIssuedAtSec: registration.client_id_issued_at,
- clientSecretExpiresAtSec: registration.client_secret_expires_at,
- })
- } catch (err) {
- // Local delete is idempotent (the row may never have landed); the pinned
- // RFC 7592 DELETE is the courtesy revocation at the AS. Both best-effort —
- // neither must mask the persist failure we report.
- try {
- await deleteDynamicClient(db, dynamicClientKey)
- } catch {
- // Best-effort local revocation; the row may not exist.
- }
+ // Best-effort RFC 7592 revocation of the client WE just minted (handle in
+ // memory). Used on every path that abandons this install after the mint.
+ const revokeMintedClient = async (): Promise<void> => {
if (mintedRegistrationClientUri && mintedRegistrationAccessToken) {
await bestEffortRfc7592Delete(
dcrDeps,
@@ -695,6 +748,32 @@ export function createAdminRemoteMcpRouter(
mintedRegistrationAccessToken
)
}
+ }
+ // A public client carries no secret; never persist one the AS may have echoed
+ // alongside a `none` downgrade.
+ const dcrCredentials: UpsertDynamicClientInput = {
+ ...dynamicClientKey,
+ issuer: discovery.issuer,
+ clientId: registration.client_id,
+ clientMode: effectiveClientMode,
+ clientSecret:
+ effectiveClientMode === 'confidential' ? registration.client_secret : undefined,
+ registrationAccessToken: registration.registration_access_token,
+ registrationClientUri: registration.registration_client_uri,
+ clientIdIssuedAtSec: registration.client_id_issued_at,
+ clientSecretExpiresAtSec: registration.client_secret_expires_at,
+ }
+
+ const persistFailed503 = async (err: unknown): Promise<void> => {
+ // Local delete removes ONLY our row (idempotent; it may never have landed);
+ // the pinned RFC 7592 DELETE is the courtesy revocation at the AS. Both
+ // best-effort — neither must mask the persist failure we report.
+ try {
+ await deleteDynamicClientOwnedByInstall(db, dynamicClientKey, installId)
+ } catch {
+ // Best-effort local revocation; the row may not exist.
+ }
+ await revokeMintedClient()
// Names-only: never echo the persist error body (may carry secret material).
log.error(
{
@@ -707,8 +786,108 @@ export function createAdminRemoteMcpRouter(
)
// 503: transient/server-side (DB), not a client error.
res.status(503).json({ error: 'dcr_persist_failed' })
+ }
+
+ // Claim the name with a PENDING row. ON CONFLICT DO NOTHING NEVER clobbers a
+ // live server's credentials (the R3-H1 fix, replacing the old upsert) — a
+ // conflict means the name is already owned, and we classify who owns it.
+ let owned: boolean
+ try {
+ owned = (
+ await insertDynamicClientPending(db, encryptionKey, { ...dcrCredentials, installId })
+ ).inserted
+ } catch (err) {
+ await persistFailed503(err)
return
}
+
+ if (!owned) {
+ // Any throw inside this conflict-resolution block (a transient k8s error from
+ // the live-uid re-read, a DB error on the row re-read, or a failure of the
+ // reclaim tx) must still revoke the client we just minted — otherwise a
+ // confidential client is orphaned at the AS. Mirrors the revoke-on-abort
+ // discipline of persistFailed503 and every explicit return below.
+ try {
+ // Someone already holds the name. A live CR owns it unconditionally; else the
+ // persisted row's lifetime decides (in-progress = another saga; reclaimable =
+ // orphan/legacy). Re-read the live uid HERE (a CR may have appeared since the
+ // pre-check — the TOCTOU the fence closes).
+ const liveCrUid = await readLiveMcpServerUid(gateway, serverName, targetNs)
+ let existing = await getDynamicClient(db, encryptionKey, dynamicClientKey)
+ if (!existing) {
+ // The row was deleted between our INSERT-conflict and this read (a teardown
+ // in the gap). Retry the claim once.
+ try {
+ owned = (
+ await insertDynamicClientPending(db, encryptionKey, {
+ ...dcrCredentials,
+ installId,
+ })
+ ).inserted
+ } catch (err) {
+ await persistFailed503(err)
+ return
+ }
+ if (!owned) existing = await getDynamicClient(db, encryptionKey, dynamicClientKey)
+ }
+
+ if (!owned) {
+ if (!existing) {
+ // Still conflicting yet unreadable — cannot classify; fail safe as taken.
+ await revokeMintedClient()
+ res.status(409).json({ error: 'server_name_in_use' })
+ return
+ }
+ const klass = classifyExistingDynamicClient(
+ {
+ installId: existing.installId ?? null,
+ crUid: existing.crUid ?? null,
+ updatedAt: existing.updatedAt,
+ },
+ { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid }
+ )
+ if (klass === 'in-use' || klass === 'in-progress') {
+ await revokeMintedClient()
+ log.warn(
+ { event: 'remote_install_dcr_name_conflict', serverName, conflict: klass },
+ 'remote dcr install rejected: name already owned'
+ )
+ res.status(409).json({
+ error: klass === 'in-use' ? 'server_name_in_use' : 'install_in_progress',
+ })
+ return
+ }
+ // reclaimable: take over the orphan/legacy row atomically (CAS in a tx).
+ const reclaim = await reclaimOrphanDynamicClient(
+ encryptionKey,
+ {
+ key: dynamicClientKey,
+ newCredentials: dcrCredentials,
+ newInstallId: installId,
+ ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid },
+ },
+ runInTransaction
+ )
+ if (!reclaim.reclaimed) {
+ // Raced: another saga won the reclaim (or re-claimed the freed name).
+ await revokeMintedClient()
+ res.status(409).json({ error: 'install_in_progress' })
+ return
+ }
+ // We own the row now. Best-effort revoke the OLD client we superseded.
+ if (reclaim.oldHandle) {
+ await bestEffortRfc7592Delete(
+ dcrDeps,
+ reclaim.oldHandle.registrationClientUri,
+ reclaim.oldHandle.registrationAccessToken
+ )
+ }
+ }
+ } catch (err) {
+ await revokeMintedClient()
+ throw err
+ }
+ }
dcrRegistered = true
dcrClientId = registration.client_id
dcrRegistrationClientUri = registration.registration_client_uri
@@ -798,6 +977,7 @@ export function createAdminRemoteMcpRouter(
// the same serverName recreated between the create and the rollback.
let createdClientSecretSnapshot: SecretSnapshot | null = null
let createdServerPreconditions: ResourcePreconditions | undefined
+ let createdServerUid: string | undefined
if (effectiveClientMode === 'confidential' && clientSecretName) {
try {
createdClientSecretSnapshot = await gateway.createSecret({
@@ -837,6 +1017,7 @@ export function createAdminRemoteMcpRouter(
targetNs
)
createdServerPreconditions = resourcePreconditionsFrom(createdServer)
+ createdServerUid = resourceUidFrom(createdServer)
} catch (err) {
if (createdClientSecretSnapshot && clientSecretName) {
try {
@@ -858,6 +1039,71 @@ export function createAdminRemoteMcpRouter(
throw err
}
+ // ── Saga step 2b (DCR only): bind the pending row to the created CR's uid ──
+ // Only DCR wrote a dynamic_clients row; other modes skip this. Binding seals the
+ // row to the McpServer's metadata.uid so a later teardown/reclaim can fence by the
+ // exact installation. A missing uid or a lost bind means the row is no longer ours
+ // — compensate the CR (fenced) and this install rather than leave a dangling row.
+ if (body.mode === 'dcr' && dcrRegistered) {
+ const rollbackCreatedServer = async (): Promise<void> => {
+ try {
+ await gateway.deleteResource(
+ 'mcpservers',
+ serverName,
+ targetNs,
+ createdServerPreconditions
+ )
+ await waitForDeletion(
+ () => gateway.getResource('mcpservers', serverName, targetNs),
+ `McpServer/${serverName}`
+ )
+ } catch {
+ // Best-effort rollback; preserve the reported failure.
+ }
+ }
+ if (createdServerUid === undefined) {
+ // Without a uid the row can never be bound (it would strand as pending and be
+ // reclaimed by the TTL). Compensate now instead of leaving that residue.
+ await rollbackCreatedServer()
+ await rollbackDynamicClient()
+ log.error(
+ { event: 'remote_oauth_dcr_bind_no_uid', serverName, namespace: targetNs },
+ 'dcr bind aborted: created McpServer carried no metadata.uid'
+ )
+ res.status(503).json({ error: 'dcr_bind_failed' })
+ return
+ }
+ let bound: boolean
+ try {
+ ;({ bound } = await bindDynamicClientToResource(
+ db,
+ dynamicClientKey,
+ installId,
+ createdServerUid
+ ))
+ } catch (err) {
+ // A thrown bind (DB error) would otherwise strand the created CR, the pending
+ // row and the minted AS client at once. Compensate all three, like steps 2/3.
+ await rollbackCreatedServer()
+ await rollbackDynamicClient()
+ throw err
+ }
+ if (!bound) {
+ // The pending row was reclaimed out from under us (TTL expired mid-saga): the
+ // reclaimer now owns the name. Roll back OUR CR; `rollbackDynamicClient` deletes
+ // only rows with OUR install_id (0 now — the reclaim rewrote it) and revokes our
+ // minted client, so the reclaimer's row is untouched.
+ await rollbackCreatedServer()
+ await rollbackDynamicClient()
+ log.warn(
+ { event: 'remote_oauth_dcr_install_superseded', serverName, namespace: targetNs },
+ 'dcr install superseded: the pending row was reclaimed before bind'
+ )
+ res.status(503).json({ error: 'install_superseded' })
+ return
+ }
+ }
+
// ── Saga step 3: attach to the Context allowlist (rollback CR+Secret) ──
try {
const ctx = (await gateway.getResource('contexts', contextRef)) as {
diff --git a/control-api/test/fixtures/remoteOAuthDiscovery.ts b/control-api/test/fixtures/remoteOAuthDiscovery.ts
index 7985c5172..c36ae42d1 100644
--- a/control-api/test/fixtures/remoteOAuthDiscovery.ts
+++ b/control-api/test/fixtures/remoteOAuthDiscovery.ts
@@ -377,10 +377,18 @@ export function makeDcrTransport(opts: {
/**
* A minimal in-memory `dynamic_clients` DbClient for the store + saga tests. It is
* a TEST HARNESS (not a cross-layer fixture, T1): it mirrors the exact parameter
- * order of `dynamicClientStore.ts`'s own INSERT/SELECT/DELETE, so a store round-trip
- * (upsert → get) and the saga's rollback (delete) run end-to-end with zero real
- * Postgres. The encrypted envelope it stores is produced by the real
+ * order of `dynamicClientStore.ts`'s own statements so the saga runs end-to-end with
+ * zero real Postgres. The encrypted envelope it stores is produced by the real
* `encryptOAuthSecret` inside the store, never hand-written.
+ *
+ * It models the install-identity statements the route now drives: the pending INSERT
+ * (`ON CONFLICT DO NOTHING RETURNING id`), the legacy upsert (`ON CONFLICT DO UPDATE`),
+ * the bind UPDATE, the reclaim CAS UPDATE, and both DELETE shapes (by-install and
+ * by-name). The AUTHORITATIVE SQL semantics — real ON CONFLICT, the FOR UPDATE + CAS
+ * race — are certified by the real-Postgres suite
+ * (`oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts`); this
+ * harness only lets the route-level wiring (classify → HTTP code, which 7592 DELETE
+ * fires) be asserted offline.
*/
export function makeInMemoryDynamicClientsDb(): {
db: DbClient
@@ -388,6 +396,7 @@ export function makeInMemoryDynamicClientsDb(): {
} {
const rows = new Map<string, Record<string, unknown>>()
const keyOf = (owner: unknown, ns: unknown, name: unknown) => `${owner}/${ns}/${name}`
+ let idSequence = 0
const db = {
query: async (text: string, values: unknown[] = []) => {
if (text.includes('INSERT INTO dynamic_clients')) {
@@ -403,8 +412,40 @@ export function makeInMemoryDynamicClientsDb(): {
registration_client_uri,
client_id_issued_at,
client_secret_expires_at,
+ // Present only on the pending INSERT (12th param); undefined for the upsert.
+ install_id,
] = values
- rows.set(keyOf(owner_kind, server_namespace, server_name), {
+ const key = keyOf(owner_kind, server_namespace, server_name)
+ const existing = rows.get(key)
+ // Pending INSERT: ON CONFLICT DO NOTHING — never clobber, signal via rowCount.
+ if (text.includes('DO NOTHING')) {
+ if (existing) return { rows: [], rowCount: 0 }
+ const id = ++idSequence
+ rows.set(key, {
+ id,
+ owner_kind,
+ server_namespace,
+ server_name,
+ issuer,
+ client_id,
+ client_mode,
+ client_secret_encrypted,
+ registration_access_token_encrypted,
+ registration_client_uri,
+ client_id_issued_at,
+ client_secret_expires_at,
+ install_id: install_id ?? null,
+ cr_uid: null,
+ created_at: new Date(),
+ updated_at: new Date(),
+ })
+ return { rows: [{ id }], rowCount: 1 }
+ }
+ // Legacy upsert: ON CONFLICT DO UPDATE overwrites the creds in place; it does
+ // NOT touch install_id/cr_uid, so those survive a conflict (default NULL on a
+ // fresh insert).
+ rows.set(key, {
+ id: existing?.id ?? ++idSequence,
owner_kind,
server_namespace,
server_name,
@@ -416,20 +457,85 @@ export function makeInMemoryDynamicClientsDb(): {
registration_client_uri,
client_id_issued_at,
client_secret_expires_at,
- created_at: new Date(),
+ install_id: existing?.install_id ?? null,
+ cr_uid: existing?.cr_uid ?? null,
+ created_at: existing?.created_at ?? new Date(),
updated_at: new Date(),
})
return { rows: [], rowCount: 1 }
}
if (text.includes('DELETE FROM dynamic_clients')) {
- const [owner_kind, server_namespace, server_name] = values
- const existed = rows.delete(keyOf(owner_kind, server_namespace, server_name))
+ const [owner_kind, server_namespace, server_name, fourth] = values
+ const key = keyOf(owner_kind, server_namespace, server_name)
+ const row = rows.get(key)
+ // By-install DELETE (compensation): remove only when the install_id matches.
+ if (values.length === 4) {
+ if (row && row.install_id === fourth) {
+ rows.delete(key)
+ return { rows: [], rowCount: 1 }
+ }
+ return { rows: [], rowCount: 0 }
+ }
+ // By-name DELETE (legacy deleteDynamicClient / dcrCleanup).
+ const existed = rows.delete(key)
return { rows: [], rowCount: existed ? 1 : 0 }
}
+ if (text.includes('UPDATE dynamic_clients')) {
+ const key = keyOf(values[0], values[1], values[2])
+ const row = rows.get(key)
+ if (!row) return { rows: [], rowCount: 0 }
+ // Reclaim CAS: WHERE id = $13 AND install_id IS NOT DISTINCT FROM $14.
+ if (text.includes('IS NOT DISTINCT FROM')) {
+ const [
+ ,
+ ,
+ ,
+ issuer,
+ client_id,
+ client_mode,
+ client_secret_encrypted,
+ registration_access_token_encrypted,
+ registration_client_uri,
+ client_id_issued_at,
+ client_secret_expires_at,
+ newInstallId,
+ lockedId,
+ lockedInstallId,
+ ] = values
+ if (row.id !== lockedId || row.install_id !== lockedInstallId) {
+ return { rows: [], rowCount: 0 }
+ }
+ Object.assign(row, {
+ issuer,
+ client_id,
+ client_mode,
+ client_secret_encrypted,
+ registration_access_token_encrypted,
+ registration_client_uri,
+ client_id_issued_at,
+ client_secret_expires_at,
+ install_id: newInstallId,
+ cr_uid: null,
+ updated_at: new Date(),
+ })
+ return { rows: [], rowCount: 1 }
+ }
+ // Bind: WHERE install_id = $4 AND cr_uid IS NULL, SET cr_uid = $5.
+ const [, , , installId, crUid] = values
+ if (row.install_id === installId && row.cr_uid == null) {
+ row.cr_uid = crUid
+ row.updated_at = new Date()
+ return { rows: [], rowCount: 1 }
+ }
+ return { rows: [], rowCount: 0 }
+ }
if (text.includes('FROM dynamic_clients')) {
const [owner_kind, server_namespace, server_name] = values
const row = rows.get(keyOf(owner_kind, server_namespace, server_name))
- return { rows: row ? [row] : [], rowCount: row ? 1 : 0 }
+ // Return a SNAPSHOT (shallow copy), as a real SELECT does: a later UPDATE must
+ // not retroactively mutate a row already read (e.g. the reclaim reads the OLD
+ // handle from its FOR UPDATE snapshot AFTER issuing the overwriting UPDATE).
+ return { rows: row ? [{ ...row }] : [], rowCount: row ? 1 : 0 }
}
return { rows: [], rowCount: 0 }
},
diff --git a/control-api/test/routes.adminRemoteMcp.test.ts b/control-api/test/routes.adminRemoteMcp.test.ts
index f8d788995..c0d4e4ed9 100644
--- a/control-api/test/routes.adminRemoteMcp.test.ts
+++ b/control-api/test/routes.adminRemoteMcp.test.ts
@@ -1,7 +1,9 @@
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
+import { randomUUID } from 'node:crypto'
import request from 'supertest'
import { config } from '../src/config.js'
+import type { DbTransactionClient } from '../src/db.js'
import type { PinnedTransport } from '../src/http/pinnedFetch.js'
import type { K8sGateway } from '../src/k8s.js'
import {
@@ -9,12 +11,18 @@ import {
type DiscoveryResult,
discoverRemoteOAuth,
} from '../src/oauth/discovery.js'
+import {
+ bindDynamicClientToResource,
+ getDynamicClient,
+ insertDynamicClientPending,
+} from '../src/oauth/dynamicClientStore.js'
import { decryptOAuthSecret, deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import { probeMcpTransport } from '../src/oauth/mcpTransportProbe.js'
import {
type AdminRemoteMcpDeps,
createAdminRemoteMcpRouter,
} from '../src/routes/admin/remoteMcp.js'
+import { K8sNotFoundError } from '../src/services/resourceService.js'
import {
DCR_BASIC_REGISTRATION_RESPONSE,
DCR_CONFIDENTIAL_REGISTRATION_RESPONSE,
@@ -1143,4 +1151,293 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {
expect(calls).toHaveLength(0)
expect(rows.size).toBe(0)
})
+
+ // ── R3-H1: install identity fences the DCR row writes ──────────────────────
+ // The install saga must NEVER clobber or delete a live server's dynamic_clients
+ // row. Rows are seeded via the REAL producers (insertDynamicClientPending + bind),
+ // read through the REAL getDynamicClient (T1).
+
+ const LIVE_ROW = {
+ ownerKind: 'mcpserver' as const,
+ serverNamespace: NS,
+ issuer: 'https://as.example.com',
+ clientId: 'live-client-id',
+ clientMode: 'confidential' as const,
+ clientSecret: 'live-secret',
+ registrationAccessToken: 'live-reg-token',
+ registrationClientUri: 'https://as.example.com/register/live',
+ }
+
+ // T4/invariant-1 (R3-H1): installing a name already held by a LIVE McpServer must
+ // leave that install's row byte-identical and mint NOTHING at the AS. Fails at
+ // parent 6938afbd3: with no pre-check and the old clobbering upsert, the CR-create
+ // 409 rolls back and DELETEs the (now overwritten) row → the row is gone and the
+ // error is the create 409, not `server_name_in_use`.
+ it('install over a live McpServer of the same name → 409 server_name_in_use, live row untouched, nothing minted', async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db, rows } = makeInMemoryDynamicClientsDb()
+ const seededInstall = randomUUID()
+ await insertDynamicClientPending(db, ENC_KEY, {
+ ...LIVE_ROW,
+ serverName: 'live-dcr',
+ installId: seededInstall,
+ })
+ await bindDynamicClientToResource(
+ db,
+ { serverNamespace: NS, serverName: 'live-dcr' },
+ seededInstall,
+ 'uid-live-cr'
+ )
+ const before = await getDynamicClient(db, ENC_KEY, {
+ serverNamespace: NS,
+ serverName: 'live-dcr',
+ })
+
+ const gw = gatewayWithContext('ctx-a')
+ await gw.createResource('mcpservers', { metadata: { name: 'live-dcr' }, spec: {} }, NS)
+
+ const { transport, calls } = makeDcrTransport({
+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),
+ })
+ const res = await request(
+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })
+ )
+ .post('/admin/mcp-servers/remote')
+ .send({
+ serverName: 'live-dcr',
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'dcr',
+ })
+
+ expect(res.status).toBe(409)
+ expect(res.body.error).toBe('server_name_in_use')
+ // Pre-check short-circuits BEFORE discovery/mint: no AS interaction at all.
+ expect(calls).toHaveLength(0)
+ // The live row is byte-identical — never clobbered, never deleted (R3-H1).
+ const after = await getDynamicClient(db, ENC_KEY, {
+ serverNamespace: NS,
+ serverName: 'live-dcr',
+ })
+ expect(after).toEqual(before)
+ expect(rows.size).toBe(1)
+ })
+
+ // The INSERT fence itself (not the pre-check): a live CR that appears AFTER the
+ // pre-check but before the persist (TOCTOU) is caught by the ON CONFLICT + classify
+ // `in-use`. The row stays byte-identical; the throwaway client we minted is revoked.
+ it('TOCTOU: a live CR appears after the pre-check → 409 server_name_in_use via the INSERT fence, row untouched', async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db, rows } = makeInMemoryDynamicClientsDb()
+ const seededInstall = randomUUID()
+ await insertDynamicClientPending(db, ENC_KEY, {
+ ...LIVE_ROW,
+ serverName: 'toctou-dcr',
+ installId: seededInstall,
+ })
+ await bindDynamicClientToResource(
+ db,
+ { serverNamespace: NS, serverName: 'toctou-dcr' },
+ seededInstall,
+ 'uid-toctou-cr'
+ )
+ const before = await getDynamicClient(db, ENC_KEY, {
+ serverNamespace: NS,
+ serverName: 'toctou-dcr',
+ })
+
+ const gw = gatewayWithContext('ctx-a')
+ const realGet = gw.getResource.bind(gw)
+ let mcpReads = 0
+ vi.spyOn(gw, 'getResource').mockImplementation(async (plural, name, ns) => {
+ if (plural === 'mcpservers' && name === 'toctou-dcr') {
+ mcpReads += 1
+ // First read is the pre-check (name still free); the CR appears by the time
+ // the INSERT conflicts and we re-read the live uid for classification.
+ if (mcpReads === 1) throw new K8sNotFoundError('mcpservers/toctou-dcr not found')
+ return { metadata: { uid: 'uid-toctou-cr' } }
+ }
+ return realGet(plural, name, ns)
+ })
+
+ const { transport, calls } = makeDcrTransport({
+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),
+ })
+ const res = await request(
+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })
+ )
+ .post('/admin/mcp-servers/remote')
+ .send({
+ serverName: 'toctou-dcr',
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'dcr',
+ })
+
+ expect(res.status).toBe(409)
+ expect(res.body.error).toBe('server_name_in_use')
+ // The row we seeded is untouched (the INSERT never clobbered it).
+ const after = await getDynamicClient(db, ENC_KEY, {
+ serverNamespace: NS,
+ serverName: 'toctou-dcr',
+ })
+ expect(after).toEqual(before)
+ // The throwaway client minted before the conflict was revoked at the AS, and the
+ // seeded live client's handle was NOT touched.
+ const deleteCalls = calls.filter(c => c.method === 'DELETE')
+ expect(deleteCalls.map(c => c.url)).toContain(
+ DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri
+ )
+ expect(deleteCalls.map(c => c.url)).not.toContain(LIVE_ROW.registrationClientUri)
+ })
+
+ // Regression guard for the class fix: a transient throw in the conflict-resolution
+ // block (here the live-uid re-read) must STILL revoke the client we just minted —
+ // otherwise a confidential client is orphaned at the AS. Before the fix this path
+ // propagated to a 500 with zero 7592 DELETEs (reproduced by review).
+ it('a transient error while classifying a name conflict still revokes the minted client', async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db } = makeInMemoryDynamicClientsDb()
+ // Seed a fresh pending row so the install's INSERT conflicts and we enter the
+ // classify path (which re-reads the live uid).
+ await insertDynamicClientPending(db, ENC_KEY, {
+ ...LIVE_ROW,
+ serverName: 'throwrevoke-dcr',
+ installId: randomUUID(),
+ })
+
+ const gw = gatewayWithContext('ctx-a')
+ const realGet = gw.getResource.bind(gw)
+ let mcpReads = 0
+ vi.spyOn(gw, 'getResource').mockImplementation(async (plural, name, ns) => {
+ if (plural === 'mcpservers' && name === 'throwrevoke-dcr') {
+ mcpReads += 1
+ // 1st read = pre-check (name free → proceed to mint + INSERT conflict);
+ // 2nd read = the conflict re-read of the live uid → transient non-404 error.
+ if (mcpReads === 1) throw new K8sNotFoundError('mcpservers/throwrevoke-dcr not found')
+ throw Object.assign(new Error('apiserver unavailable'), { code: 500 })
+ }
+ return realGet(plural, name, ns)
+ })
+
+ const { transport, calls } = makeDcrTransport({
+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),
+ })
+ const res = await request(
+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })
+ )
+ .post('/admin/mcp-servers/remote')
+ .send({
+ serverName: 'throwrevoke-dcr',
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'dcr',
+ })
+
+ // The transient error surfaces (not swallowed), but the minted client is revoked.
+ expect(res.status).toBe(500)
+ const deleteCalls = calls.filter(c => c.method === 'DELETE')
+ expect(deleteCalls.map(c => c.url)).toContain(
+ DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri
+ )
+ })
+
+ // A fresh pending row (another saga of the same name in flight) → 409
+ // install_in_progress; the in-flight row is untouched and our throwaway client is
+ // revoked.
+ it('install over a fresh pending row → 409 install_in_progress, in-flight row untouched', async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db, rows } = makeInMemoryDynamicClientsDb()
+ const inflightInstall = randomUUID()
+ await insertDynamicClientPending(db, ENC_KEY, {
+ ...LIVE_ROW,
+ serverName: 'inprogress-dcr',
+ clientId: 'inflight-client-id',
+ installId: inflightInstall,
+ })
+ const before = await getDynamicClient(db, ENC_KEY, {
+ serverNamespace: NS,
+ serverName: 'inprogress-dcr',
+ })
+
+ const gw = gatewayWithContext('ctx-a') // no McpServer seeded → pre-check passes
+ const { transport, calls } = makeDcrTransport({
+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),
+ })
+ const res = await request(
+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })
+ )
+ .post('/admin/mcp-servers/remote')
+ .send({
+ serverName: 'inprogress-dcr',
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'dcr',
+ })
+
+ expect(res.status).toBe(409)
+ expect(res.body.error).toBe('install_in_progress')
+ // The in-flight row is untouched (its install still owns it).
+ const after = await getDynamicClient(db, ENC_KEY, {
+ serverNamespace: NS,
+ serverName: 'inprogress-dcr',
+ })
+ expect(after).toEqual(before)
+ expect(rows.size).toBe(1)
+ // Our throwaway minted client was revoked at the AS.
+ expect(calls.some(c => c.method === 'DELETE')).toBe(true)
+ })
+
+ // An orphan row (bound to a uid with no live CR) does NOT block a reinstall: it is
+ // reclaimed (CAS in a tx), the OLD client is revoked, and the install proceeds to 201.
+ it('install over a reclaimable orphan → reclaims the row, revokes the old client, 201', async () => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })
+ const { db, rows } = makeInMemoryDynamicClientsDb()
+ const orphanInstall = randomUUID()
+ await insertDynamicClientPending(db, ENC_KEY, {
+ ...LIVE_ROW,
+ serverName: 'reclaim-dcr',
+ clientId: 'orphan-client-id',
+ installId: orphanInstall,
+ })
+ // Bound to a uid whose CR no longer exists → orphan (reclaimable, no live CR).
+ await bindDynamicClientToResource(
+ db,
+ { serverNamespace: NS, serverName: 'reclaim-dcr' },
+ orphanInstall,
+ 'uid-orphan-gone'
+ )
+
+ const gw = gatewayWithContext('ctx-a') // no live McpServer → orphan is reclaimable
+ const { transport, calls } = makeDcrTransport({
+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),
+ })
+ const res = await request(
+ makeAppWithDeps(gw, {
+ db,
+ dcr: { transport, resolveDns: PUBLIC_IP },
+ // The reclaim opens a tx; bind it to the same in-memory db.
+ runInTransaction: work => work(db as unknown as DbTransactionClient),
+ })
+ )
+ .post('/admin/mcp-servers/remote')
+ .send({
+ serverName: 'reclaim-dcr',
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'dcr',
+ })
+
+ expect(res.status).toBe(201)
+ // Still exactly one row, now owned by the new install (minted client_id).
+ expect(rows.size).toBe(1)
+ const owned = await getDynamicClient(db, ENC_KEY, {
+ serverNamespace: NS,
+ serverName: 'reclaim-dcr',
+ })
+ expect(owned?.clientId).toBe(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.client_id)
+ // The OLD (orphan) client was revoked at the AS via its handle read under the lock.
+ const deleteCalls = calls.filter(c => c.method === 'DELETE')
+ expect(deleteCalls.map(c => c.url)).toContain(LIVE_ROW.registrationClientUri)
+ })
})