Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): fence the remote install saga dynamic client writes by install identity (R3-H1)
Diego Zacarias90041bbf2b68
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 7988b7a1c..4c29ba6c0 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.664",+ "version": "0.1.665","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.664",+ "version": "0.1.665","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 7edfdcb20..63e5abaa8 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.664",+ "version": "0.1.665","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/oauth/dynamicClientStore.ts b/control-api/src/oauth/dynamicClientStore.tsindex 1ece1d4e9..8be4832d5 100644--- a/control-api/src/oauth/dynamicClientStore.ts+++ b/control-api/src/oauth/dynamicClientStore.ts@@ -115,10 +115,15 @@ function encodeCredentialColumns(/*** Insert (or replace) the dynamic client for a server CR. Re-registration- * overwrites in place (INSERT … ON CONFLICT DO UPDATE), which is also the only- * rotation path in C2 — there is no dedicated rotate fn (no caller; avoids dead- * code, DEC-18). `client_secret` and `registration_access_token` are encrypted- * at rest; a public client stores NULL for the secret.+ * overwrites in place (INSERT … ON CONFLICT DO UPDATE). `client_secret` and+ * `registration_access_token` are encrypted at rest; a public client stores NULL+ * for the secret.+ *+ * NO production caller: the remote-install saga now claims the name with+ * `insertDynamicClientPending` (ON CONFLICT DO NOTHING) so an install can never+ * clobber a live server's row. This clobbering upsert is retained ONLY as a store+ * primitive exercised by its own unit tests and as a row-seeder for other suites;+ * it must NOT be reintroduced into any install/reclaim path.*/export async function upsertDynamicClient(db: DbClient,diff --git a/control-api/src/routes/admin/remoteMcp.ts b/control-api/src/routes/admin/remoteMcp.tsindex e8ced5ea6..d613baeb7 100644--- a/control-api/src/routes/admin/remoteMcp.ts+++ b/control-api/src/routes/admin/remoteMcp.ts@@ -1,7 +1,8 @@import { Router } from 'express'+import { randomUUID } from 'node:crypto'import { z } from 'zod'import { config } from '../../config.js'-import { type DbClient, pool } from '../../db.js'+import { type DbClient, type DbTransactionClient, pool, withTransaction } from '../../db.js'import { asyncHandler } from '../../http/asyncHandler.js'import { extractK8sError } from '../../http/k8sError.js'import { enforceNamespace } from '../../http/namespaceAudit.js'@@ -21,8 +22,14 @@ import { type DiscoveryResult, discoverRemoteOAuth } from '../../oauth/discoveryimport { discoveryHttpStatus } from '../../oauth/discoveryHttpStatus.js'import {type DynamicClientKey,- deleteDynamicClient,- upsertDynamicClient,+ PENDING_TTL_MS,+ type UpsertDynamicClientInput,+ bindDynamicClientToResource,+ classifyExistingDynamicClient,+ deleteDynamicClientOwnedByInstall,+ getDynamicClient,+ insertDynamicClientPending,+ reclaimOrphanDynamicClient,} from '../../oauth/dynamicClientStore.js'import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'import { probeMcpTransport } from '../../oauth/mcpTransportProbe.js'@@ -110,6 +117,31 @@ function resourcePreconditionsFrom(resource: unknown): ResourcePreconditions | ureturn { uid, resourceVersion }}+/** The server-assigned `metadata.uid`, or undefined when the response carried none. */+function resourceUidFrom(resource: unknown): string | undefined {+ const uid = (resource as { metadata?: { uid?: unknown } } | null)?.metadata?.uid+ return typeof uid === 'string' && uid ? uid : undefined+}++/**+ * The uid of a live McpServer with this name, or undefined when none exists. A live+ * CR owns the name: the reclaim/classify path treats a conflict against a live uid as+ * `in-use` regardless of the persisted row's state. A 404 is "no live CR" (undefined);+ * any other read error propagates (fail closed, not "assume free").+ */+async function readLiveMcpServerUid(+ gateway: K8sGateway,+ name: string,+ namespace: string+): Promise<string | undefined> {+ try {+ return resourceUidFrom(await gateway.getResource('mcpservers', name, namespace))+ } catch (err) {+ if (err instanceof K8sNotFoundError || extractK8sError(err)?.status === 404) return undefined+ throw err+ }+}+/*** The `spec.oauth` remote shape written to the CR — the C3 contract. Uses a distinct* `source: 'remote'` discriminator instead of the closed 8-provider `provider` enum@@ -221,6 +253,15 @@ const REMOTE_MCP_PROXY_PORT = 3000const DELETE_SETTLE_TIMEOUT_MS = 10_000const DELETE_SETTLE_POLL_MS = 250+/**+ * Production default for the DCR orphan-reclaim transaction. Wraps `withTransaction`+ * so the binding is only accessed when a reclaim actually runs — not at router+ * creation, which suites that partial-mock db.js would break.+ */+function defaultRunInTransaction<T>(work: (tx: DbTransactionClient) => Promise<T>): Promise<T> {+ return withTransaction(work)+}+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))/** Poll until a best-effort delete settles to a 404, so a re-install cannot 409. */@@ -260,6 +301,13 @@ export interface AdminRemoteMcpDeps {* inject one derived from the real producer to drive a specific probe outcome offline.*/probe?: typeof probeMcpTransport+ /**+ * Injectable transaction runner (test seam only). Production leaves it undefined and+ * `withTransaction` (module pool) is used. The DCR orphan reclaim opens a transaction+ * for its `SELECT … FOR UPDATE` + CAS; a test injects a runner bound to its in-memory+ * db so the reclaim runs against the same store as the rest of the saga.+ */+ runInTransaction?: <T>(work: (tx: DbTransactionClient) => Promise<T>) => Promise<T>}export function createAdminRemoteMcpRouter(@@ -272,6 +320,10 @@ export function createAdminRemoteMcpRouter(const dcrDeps: DcrDeps = deps.dcr ?? { logger: log }const discover = deps.discover ?? discoverRemoteOAuthconst probe = deps.probe ?? probeMcpTransport+ // Defer the `withTransaction` binding to CALL time (only the DCR reclaim uses it).+ // Referencing it here at router-creation would break the many suites that+ // partial-mock db.js without a `withTransaction` export.+ const runInTransaction = deps.runInTransaction ?? defaultRunInTransaction// ── POST /admin/mcp-servers/remote/discover — dry-run, no writes ──────────router.post(@@ -386,6 +438,24 @@ export function createAdminRemoteMcpRouter(return}+ // Pre-check: a name already taken by a LIVE McpServer is rejected before any+ // discovery or AS mint, so we never register a throwaway client at the AS for a+ // name we cannot use. This is NOT the fence — a CR could still appear between here+ // and the persist (TOCTOU); the pending INSERT's ON CONFLICT below is the fence.+ const nameInUseUid = await readLiveMcpServerUid(+ gateway,+ body.serverName,+ config.mcpServersNamespace+ )+ if (nameInUseUid !== undefined) {+ log.warn(+ { event: 'remote_install_server_name_in_use', serverName: body.serverName },+ 'remote install rejected: an McpServer with this name already exists'+ )+ res.status(409).json({ error: 'server_name_in_use' })+ return+ }+// D-4: re-run discovery SERVER-SIDE at install and pin the result — never trust// the client-passed prefill as authoritative. Discovery kernel-guards baseUrl +// every discovered endpoint internally before returning them.@@ -545,6 +615,10 @@ export function createAdminRemoteMcpRouter(serverNamespace: targetNs,serverName,}+ // Install-ownership token minted at the start of DCR step 0. Every compensation+ // deletes ONLY the row carrying this install_id, so a concurrent install/reinstall+ // of the same name that reclaimed the row is never destroyed (R3-H1).+ const installId = randomUUID()let dcrRegistered = falselet dcrClientId: string | undefinedlet dcrRegistrationClientUri: string | undefined@@ -552,7 +626,7 @@ export function createAdminRemoteMcpRouter(const rollbackDynamicClient = async (): Promise<void> => {if (!dcrRegistered) returntry {- await deleteDynamicClient(db, dynamicClientKey)+ await deleteDynamicClientOwnedByInstall(db, dynamicClientKey, installId)} catch {// Best-effort local revocation; preserve the original saga error.}@@ -664,30 +738,9 @@ export function createAdminRemoteMcpRouter(// AS. Compensate here explicitly (DEC-18).const mintedRegistrationClientUri = registration.registration_client_uriconst mintedRegistrationAccessToken = registration.registration_access_token- try {- await upsertDynamicClient(db, encryptionKey, {- ...dynamicClientKey,- issuer: discovery.issuer,- clientId: registration.client_id,- clientMode: effectiveClientMode,- // A public client carries no secret; never persist one the AS may have- // echoed alongside a `none` downgrade.- clientSecret:- effectiveClientMode === 'confidential' ? registration.client_secret : undefined,- registrationAccessToken: registration.registration_access_token,- registrationClientUri: registration.registration_client_uri,- clientIdIssuedAtSec: registration.client_id_issued_at,- clientSecretExpiresAtSec: registration.client_secret_expires_at,- })- } catch (err) {- // Local delete is idempotent (the row may never have landed); the pinned- // RFC 7592 DELETE is the courtesy revocation at the AS. Both best-effort —- // neither must mask the persist failure we report.- try {- await deleteDynamicClient(db, dynamicClientKey)- } catch {- // Best-effort local revocation; the row may not exist.- }+ // Best-effort RFC 7592 revocation of the client WE just minted (handle in+ // memory). Used on every path that abandons this install after the mint.+ const revokeMintedClient = async (): Promise<void> => {if (mintedRegistrationClientUri && mintedRegistrationAccessToken) {await bestEffortRfc7592Delete(dcrDeps,@@ -695,6 +748,32 @@ export function createAdminRemoteMcpRouter(mintedRegistrationAccessToken)}+ }+ // A public client carries no secret; never persist one the AS may have echoed+ // alongside a `none` downgrade.+ const dcrCredentials: UpsertDynamicClientInput = {+ ...dynamicClientKey,+ issuer: discovery.issuer,+ clientId: registration.client_id,+ clientMode: effectiveClientMode,+ clientSecret:+ effectiveClientMode === 'confidential' ? registration.client_secret : undefined,+ registrationAccessToken: registration.registration_access_token,+ registrationClientUri: registration.registration_client_uri,+ clientIdIssuedAtSec: registration.client_id_issued_at,+ clientSecretExpiresAtSec: registration.client_secret_expires_at,+ }++ const persistFailed503 = async (err: unknown): Promise<void> => {+ // Local delete removes ONLY our row (idempotent; it may never have landed);+ // the pinned RFC 7592 DELETE is the courtesy revocation at the AS. Both+ // best-effort — neither must mask the persist failure we report.+ try {+ await deleteDynamicClientOwnedByInstall(db, dynamicClientKey, installId)+ } catch {+ // Best-effort local revocation; the row may not exist.+ }+ await revokeMintedClient()// Names-only: never echo the persist error body (may carry secret material).log.error({@@ -707,8 +786,108 @@ export function createAdminRemoteMcpRouter()// 503: transient/server-side (DB), not a client error.res.status(503).json({ error: 'dcr_persist_failed' })+ }++ // Claim the name with a PENDING row. ON CONFLICT DO NOTHING NEVER clobbers a+ // live server's credentials (the R3-H1 fix, replacing the old upsert) — a+ // conflict means the name is already owned, and we classify who owns it.+ let owned: boolean+ try {+ owned = (+ await insertDynamicClientPending(db, encryptionKey, { ...dcrCredentials, installId })+ ).inserted+ } catch (err) {+ await persistFailed503(err)return}++ if (!owned) {+ // Any throw inside this conflict-resolution block (a transient k8s error from+ // the live-uid re-read, a DB error on the row re-read, or a failure of the+ // reclaim tx) must still revoke the client we just minted — otherwise a+ // confidential client is orphaned at the AS. Mirrors the revoke-on-abort+ // discipline of persistFailed503 and every explicit return below.+ try {+ // Someone already holds the name. A live CR owns it unconditionally; else the+ // persisted row's lifetime decides (in-progress = another saga; reclaimable =+ // orphan/legacy). Re-read the live uid HERE (a CR may have appeared since the+ // pre-check — the TOCTOU the fence closes).+ const liveCrUid = await readLiveMcpServerUid(gateway, serverName, targetNs)+ let existing = await getDynamicClient(db, encryptionKey, dynamicClientKey)+ if (!existing) {+ // The row was deleted between our INSERT-conflict and this read (a teardown+ // in the gap). Retry the claim once.+ try {+ owned = (+ await insertDynamicClientPending(db, encryptionKey, {+ ...dcrCredentials,+ installId,+ })+ ).inserted+ } catch (err) {+ await persistFailed503(err)+ return+ }+ if (!owned) existing = await getDynamicClient(db, encryptionKey, dynamicClientKey)+ }++ if (!owned) {+ if (!existing) {+ // Still conflicting yet unreadable — cannot classify; fail safe as taken.+ await revokeMintedClient()+ res.status(409).json({ error: 'server_name_in_use' })+ return+ }+ const klass = classifyExistingDynamicClient(+ {+ installId: existing.installId ?? null,+ crUid: existing.crUid ?? null,+ updatedAt: existing.updatedAt,+ },+ { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid }+ )+ if (klass === 'in-use' || klass === 'in-progress') {+ await revokeMintedClient()+ log.warn(+ { event: 'remote_install_dcr_name_conflict', serverName, conflict: klass },+ 'remote dcr install rejected: name already owned'+ )+ res.status(409).json({+ error: klass === 'in-use' ? 'server_name_in_use' : 'install_in_progress',+ })+ return+ }+ // reclaimable: take over the orphan/legacy row atomically (CAS in a tx).+ const reclaim = await reclaimOrphanDynamicClient(+ encryptionKey,+ {+ key: dynamicClientKey,+ newCredentials: dcrCredentials,+ newInstallId: installId,+ ctx: { now: new Date(), pendingTtlMs: PENDING_TTL_MS, liveCrUid },+ },+ runInTransaction+ )+ if (!reclaim.reclaimed) {+ // Raced: another saga won the reclaim (or re-claimed the freed name).+ await revokeMintedClient()+ res.status(409).json({ error: 'install_in_progress' })+ return+ }+ // We own the row now. Best-effort revoke the OLD client we superseded.+ if (reclaim.oldHandle) {+ await bestEffortRfc7592Delete(+ dcrDeps,+ reclaim.oldHandle.registrationClientUri,+ reclaim.oldHandle.registrationAccessToken+ )+ }+ }+ } catch (err) {+ await revokeMintedClient()+ throw err+ }+ }dcrRegistered = truedcrClientId = registration.client_iddcrRegistrationClientUri = registration.registration_client_uri@@ -798,6 +977,7 @@ export function createAdminRemoteMcpRouter(// the same serverName recreated between the create and the rollback.let createdClientSecretSnapshot: SecretSnapshot | null = nulllet createdServerPreconditions: ResourcePreconditions | undefined+ let createdServerUid: string | undefinedif (effectiveClientMode === 'confidential' && clientSecretName) {try {createdClientSecretSnapshot = await gateway.createSecret({@@ -837,6 +1017,7 @@ export function createAdminRemoteMcpRouter(targetNs)createdServerPreconditions = resourcePreconditionsFrom(createdServer)+ createdServerUid = resourceUidFrom(createdServer)} catch (err) {if (createdClientSecretSnapshot && clientSecretName) {try {@@ -858,6 +1039,71 @@ export function createAdminRemoteMcpRouter(throw err}+ // ── Saga step 2b (DCR only): bind the pending row to the created CR's uid ──+ // Only DCR wrote a dynamic_clients row; other modes skip this. Binding seals the+ // row to the McpServer's metadata.uid so a later teardown/reclaim can fence by the+ // exact installation. A missing uid or a lost bind means the row is no longer ours+ // — compensate the CR (fenced) and this install rather than leave a dangling row.+ if (body.mode === 'dcr' && dcrRegistered) {+ const rollbackCreatedServer = async (): Promise<void> => {+ try {+ await gateway.deleteResource(+ 'mcpservers',+ serverName,+ targetNs,+ createdServerPreconditions+ )+ await waitForDeletion(+ () => gateway.getResource('mcpservers', serverName, targetNs),+ `McpServer/${serverName}`+ )+ } catch {+ // Best-effort rollback; preserve the reported failure.+ }+ }+ if (createdServerUid === undefined) {+ // Without a uid the row can never be bound (it would strand as pending and be+ // reclaimed by the TTL). Compensate now instead of leaving that residue.+ await rollbackCreatedServer()+ await rollbackDynamicClient()+ log.error(+ { event: 'remote_oauth_dcr_bind_no_uid', serverName, namespace: targetNs },+ 'dcr bind aborted: created McpServer carried no metadata.uid'+ )+ res.status(503).json({ error: 'dcr_bind_failed' })+ return+ }+ let bound: boolean+ try {+ ;({ bound } = await bindDynamicClientToResource(+ db,+ dynamicClientKey,+ installId,+ createdServerUid+ ))+ } catch (err) {+ // A thrown bind (DB error) would otherwise strand the created CR, the pending+ // row and the minted AS client at once. Compensate all three, like steps 2/3.+ await rollbackCreatedServer()+ await rollbackDynamicClient()+ throw err+ }+ if (!bound) {+ // The pending row was reclaimed out from under us (TTL expired mid-saga): the+ // reclaimer now owns the name. Roll back OUR CR; `rollbackDynamicClient` deletes+ // only rows with OUR install_id (0 now — the reclaim rewrote it) and revokes our+ // minted client, so the reclaimer's row is untouched.+ await rollbackCreatedServer()+ await rollbackDynamicClient()+ log.warn(+ { event: 'remote_oauth_dcr_install_superseded', serverName, namespace: targetNs },+ 'dcr install superseded: the pending row was reclaimed before bind'+ )+ res.status(503).json({ error: 'install_superseded' })+ return+ }+ }+// ── Saga step 3: attach to the Context allowlist (rollback CR+Secret) ──try {const ctx = (await gateway.getResource('contexts', contextRef)) as {diff --git a/control-api/test/fixtures/remoteOAuthDiscovery.ts b/control-api/test/fixtures/remoteOAuthDiscovery.tsindex 7985c5172..c36ae42d1 100644--- a/control-api/test/fixtures/remoteOAuthDiscovery.ts+++ b/control-api/test/fixtures/remoteOAuthDiscovery.ts@@ -377,10 +377,18 @@ export function makeDcrTransport(opts: {/*** A minimal in-memory `dynamic_clients` DbClient for the store + saga tests. It is* a TEST HARNESS (not a cross-layer fixture, T1): it mirrors the exact parameter- * order of `dynamicClientStore.ts`'s own INSERT/SELECT/DELETE, so a store round-trip- * (upsert → get) and the saga's rollback (delete) run end-to-end with zero real- * Postgres. The encrypted envelope it stores is produced by the real+ * order of `dynamicClientStore.ts`'s own statements so the saga runs end-to-end with+ * zero real Postgres. The encrypted envelope it stores is produced by the real* `encryptOAuthSecret` inside the store, never hand-written.+ *+ * It models the install-identity statements the route now drives: the pending INSERT+ * (`ON CONFLICT DO NOTHING RETURNING id`), the legacy upsert (`ON CONFLICT DO UPDATE`),+ * the bind UPDATE, the reclaim CAS UPDATE, and both DELETE shapes (by-install and+ * by-name). The AUTHORITATIVE SQL semantics — real ON CONFLICT, the FOR UPDATE + CAS+ * race — are certified by the real-Postgres suite+ * (`oauth.dynamicClientStore.installIdentity.realPostgres.integration.test.ts`); this+ * harness only lets the route-level wiring (classify → HTTP code, which 7592 DELETE+ * fires) be asserted offline.*/export function makeInMemoryDynamicClientsDb(): {db: DbClient@@ -388,6 +396,7 @@ export function makeInMemoryDynamicClientsDb(): {} {const rows = new Map<string, Record<string, unknown>>()const keyOf = (owner: unknown, ns: unknown, name: unknown) => `${owner}/${ns}/${name}`+ let idSequence = 0const db = {query: async (text: string, values: unknown[] = []) => {if (text.includes('INSERT INTO dynamic_clients')) {@@ -403,8 +412,40 @@ export function makeInMemoryDynamicClientsDb(): {registration_client_uri,client_id_issued_at,client_secret_expires_at,+ // Present only on the pending INSERT (12th param); undefined for the upsert.+ install_id,] = values- rows.set(keyOf(owner_kind, server_namespace, server_name), {+ const key = keyOf(owner_kind, server_namespace, server_name)+ const existing = rows.get(key)+ // Pending INSERT: ON CONFLICT DO NOTHING — never clobber, signal via rowCount.+ if (text.includes('DO NOTHING')) {+ if (existing) return { rows: [], rowCount: 0 }+ const id = ++idSequence+ rows.set(key, {+ id,+ owner_kind,+ server_namespace,+ server_name,+ issuer,+ client_id,+ client_mode,+ client_secret_encrypted,+ registration_access_token_encrypted,+ registration_client_uri,+ client_id_issued_at,+ client_secret_expires_at,+ install_id: install_id ?? null,+ cr_uid: null,+ created_at: new Date(),+ updated_at: new Date(),+ })+ return { rows: [{ id }], rowCount: 1 }+ }+ // Legacy upsert: ON CONFLICT DO UPDATE overwrites the creds in place; it does+ // NOT touch install_id/cr_uid, so those survive a conflict (default NULL on a+ // fresh insert).+ rows.set(key, {+ id: existing?.id ?? ++idSequence,owner_kind,server_namespace,server_name,@@ -416,20 +457,85 @@ export function makeInMemoryDynamicClientsDb(): {registration_client_uri,client_id_issued_at,client_secret_expires_at,- created_at: new Date(),+ install_id: existing?.install_id ?? null,+ cr_uid: existing?.cr_uid ?? null,+ created_at: existing?.created_at ?? new Date(),updated_at: new Date(),})return { rows: [], rowCount: 1 }}if (text.includes('DELETE FROM dynamic_clients')) {- const [owner_kind, server_namespace, server_name] = values- const existed = rows.delete(keyOf(owner_kind, server_namespace, server_name))+ const [owner_kind, server_namespace, server_name, fourth] = values+ const key = keyOf(owner_kind, server_namespace, server_name)+ const row = rows.get(key)+ // By-install DELETE (compensation): remove only when the install_id matches.+ if (values.length === 4) {+ if (row && row.install_id === fourth) {+ rows.delete(key)+ return { rows: [], rowCount: 1 }+ }+ return { rows: [], rowCount: 0 }+ }+ // By-name DELETE (legacy deleteDynamicClient / dcrCleanup).+ const existed = rows.delete(key)return { rows: [], rowCount: existed ? 1 : 0 }}+ if (text.includes('UPDATE dynamic_clients')) {+ const key = keyOf(values[0], values[1], values[2])+ const row = rows.get(key)+ if (!row) return { rows: [], rowCount: 0 }+ // Reclaim CAS: WHERE id = $13 AND install_id IS NOT DISTINCT FROM $14.+ if (text.includes('IS NOT DISTINCT FROM')) {+ const [+ ,+ ,+ ,+ issuer,+ client_id,+ client_mode,+ client_secret_encrypted,+ registration_access_token_encrypted,+ registration_client_uri,+ client_id_issued_at,+ client_secret_expires_at,+ newInstallId,+ lockedId,+ lockedInstallId,+ ] = values+ if (row.id !== lockedId || row.install_id !== lockedInstallId) {+ return { rows: [], rowCount: 0 }+ }+ Object.assign(row, {+ issuer,+ client_id,+ client_mode,+ client_secret_encrypted,+ registration_access_token_encrypted,+ registration_client_uri,+ client_id_issued_at,+ client_secret_expires_at,+ install_id: newInstallId,+ cr_uid: null,+ updated_at: new Date(),+ })+ return { rows: [], rowCount: 1 }+ }+ // Bind: WHERE install_id = $4 AND cr_uid IS NULL, SET cr_uid = $5.+ const [, , , installId, crUid] = values+ if (row.install_id === installId && row.cr_uid == null) {+ row.cr_uid = crUid+ row.updated_at = new Date()+ return { rows: [], rowCount: 1 }+ }+ return { rows: [], rowCount: 0 }+ }if (text.includes('FROM dynamic_clients')) {const [owner_kind, server_namespace, server_name] = valuesconst row = rows.get(keyOf(owner_kind, server_namespace, server_name))- return { rows: row ? [row] : [], rowCount: row ? 1 : 0 }+ // Return a SNAPSHOT (shallow copy), as a real SELECT does: a later UPDATE must+ // not retroactively mutate a row already read (e.g. the reclaim reads the OLD+ // handle from its FOR UPDATE snapshot AFTER issuing the overwriting UPDATE).+ return { rows: row ? [{ ...row }] : [], rowCount: row ? 1 : 0 }}return { rows: [], rowCount: 0 }},diff --git a/control-api/test/routes.adminRemoteMcp.test.ts b/control-api/test/routes.adminRemoteMcp.test.tsindex f8d788995..c0d4e4ed9 100644--- a/control-api/test/routes.adminRemoteMcp.test.ts+++ b/control-api/test/routes.adminRemoteMcp.test.ts@@ -1,7 +1,9 @@import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'+import { randomUUID } from 'node:crypto'import request from 'supertest'import { config } from '../src/config.js'+import type { DbTransactionClient } from '../src/db.js'import type { PinnedTransport } from '../src/http/pinnedFetch.js'import type { K8sGateway } from '../src/k8s.js'import {@@ -9,12 +11,18 @@ import {type DiscoveryResult,discoverRemoteOAuth,} from '../src/oauth/discovery.js'+import {+ bindDynamicClientToResource,+ getDynamicClient,+ insertDynamicClientPending,+} from '../src/oauth/dynamicClientStore.js'import { decryptOAuthSecret, deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'import { probeMcpTransport } from '../src/oauth/mcpTransportProbe.js'import {type AdminRemoteMcpDeps,createAdminRemoteMcpRouter,} from '../src/routes/admin/remoteMcp.js'+import { K8sNotFoundError } from '../src/services/resourceService.js'import {DCR_BASIC_REGISTRATION_RESPONSE,DCR_CONFIDENTIAL_REGISTRATION_RESPONSE,@@ -1143,4 +1151,293 @@ describe('POST /admin/mcp-servers/remote — DCR install saga (C2)', () => {expect(calls).toHaveLength(0)expect(rows.size).toBe(0)})++ // ── R3-H1: install identity fences the DCR row writes ──────────────────────+ // The install saga must NEVER clobber or delete a live server's dynamic_clients+ // row. Rows are seeded via the REAL producers (insertDynamicClientPending + bind),+ // read through the REAL getDynamicClient (T1).++ const LIVE_ROW = {+ ownerKind: 'mcpserver' as const,+ serverNamespace: NS,+ issuer: 'https://as.example.com',+ clientId: 'live-client-id',+ clientMode: 'confidential' as const,+ clientSecret: 'live-secret',+ registrationAccessToken: 'live-reg-token',+ registrationClientUri: 'https://as.example.com/register/live',+ }++ // T4/invariant-1 (R3-H1): installing a name already held by a LIVE McpServer must+ // leave that install's row byte-identical and mint NOTHING at the AS. Fails at+ // parent 6938afbd3: with no pre-check and the old clobbering upsert, the CR-create+ // 409 rolls back and DELETEs the (now overwritten) row → the row is gone and the+ // error is the create 409, not `server_name_in_use`.+ it('install over a live McpServer of the same name → 409 server_name_in_use, live row untouched, nothing minted', async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db, rows } = makeInMemoryDynamicClientsDb()+ const seededInstall = randomUUID()+ await insertDynamicClientPending(db, ENC_KEY, {+ ...LIVE_ROW,+ serverName: 'live-dcr',+ installId: seededInstall,+ })+ await bindDynamicClientToResource(+ db,+ { serverNamespace: NS, serverName: 'live-dcr' },+ seededInstall,+ 'uid-live-cr'+ )+ const before = await getDynamicClient(db, ENC_KEY, {+ serverNamespace: NS,+ serverName: 'live-dcr',+ })++ const gw = gatewayWithContext('ctx-a')+ await gw.createResource('mcpservers', { metadata: { name: 'live-dcr' }, spec: {} }, NS)++ const { transport, calls } = makeDcrTransport({+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),+ })+ const res = await request(+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })+ )+ .post('/admin/mcp-servers/remote')+ .send({+ serverName: 'live-dcr',+ contextRef: 'ctx-a',+ baseUrl: 'https://mcp.notion.com/mcp',+ mode: 'dcr',+ })++ expect(res.status).toBe(409)+ expect(res.body.error).toBe('server_name_in_use')+ // Pre-check short-circuits BEFORE discovery/mint: no AS interaction at all.+ expect(calls).toHaveLength(0)+ // The live row is byte-identical — never clobbered, never deleted (R3-H1).+ const after = await getDynamicClient(db, ENC_KEY, {+ serverNamespace: NS,+ serverName: 'live-dcr',+ })+ expect(after).toEqual(before)+ expect(rows.size).toBe(1)+ })++ // The INSERT fence itself (not the pre-check): a live CR that appears AFTER the+ // pre-check but before the persist (TOCTOU) is caught by the ON CONFLICT + classify+ // `in-use`. The row stays byte-identical; the throwaway client we minted is revoked.+ it('TOCTOU: a live CR appears after the pre-check → 409 server_name_in_use via the INSERT fence, row untouched', async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db, rows } = makeInMemoryDynamicClientsDb()+ const seededInstall = randomUUID()+ await insertDynamicClientPending(db, ENC_KEY, {+ ...LIVE_ROW,+ serverName: 'toctou-dcr',+ installId: seededInstall,+ })+ await bindDynamicClientToResource(+ db,+ { serverNamespace: NS, serverName: 'toctou-dcr' },+ seededInstall,+ 'uid-toctou-cr'+ )+ const before = await getDynamicClient(db, ENC_KEY, {+ serverNamespace: NS,+ serverName: 'toctou-dcr',+ })++ const gw = gatewayWithContext('ctx-a')+ const realGet = gw.getResource.bind(gw)+ let mcpReads = 0+ vi.spyOn(gw, 'getResource').mockImplementation(async (plural, name, ns) => {+ if (plural === 'mcpservers' && name === 'toctou-dcr') {+ mcpReads += 1+ // First read is the pre-check (name still free); the CR appears by the time+ // the INSERT conflicts and we re-read the live uid for classification.+ if (mcpReads === 1) throw new K8sNotFoundError('mcpservers/toctou-dcr not found')+ return { metadata: { uid: 'uid-toctou-cr' } }+ }+ return realGet(plural, name, ns)+ })++ const { transport, calls } = makeDcrTransport({+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),+ })+ const res = await request(+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })+ )+ .post('/admin/mcp-servers/remote')+ .send({+ serverName: 'toctou-dcr',+ contextRef: 'ctx-a',+ baseUrl: 'https://mcp.notion.com/mcp',+ mode: 'dcr',+ })++ expect(res.status).toBe(409)+ expect(res.body.error).toBe('server_name_in_use')+ // The row we seeded is untouched (the INSERT never clobbered it).+ const after = await getDynamicClient(db, ENC_KEY, {+ serverNamespace: NS,+ serverName: 'toctou-dcr',+ })+ expect(after).toEqual(before)+ // The throwaway client minted before the conflict was revoked at the AS, and the+ // seeded live client's handle was NOT touched.+ const deleteCalls = calls.filter(c => c.method === 'DELETE')+ expect(deleteCalls.map(c => c.url)).toContain(+ DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri+ )+ expect(deleteCalls.map(c => c.url)).not.toContain(LIVE_ROW.registrationClientUri)+ })++ // Regression guard for the class fix: a transient throw in the conflict-resolution+ // block (here the live-uid re-read) must STILL revoke the client we just minted —+ // otherwise a confidential client is orphaned at the AS. Before the fix this path+ // propagated to a 500 with zero 7592 DELETEs (reproduced by review).+ it('a transient error while classifying a name conflict still revokes the minted client', async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db } = makeInMemoryDynamicClientsDb()+ // Seed a fresh pending row so the install's INSERT conflicts and we enter the+ // classify path (which re-reads the live uid).+ await insertDynamicClientPending(db, ENC_KEY, {+ ...LIVE_ROW,+ serverName: 'throwrevoke-dcr',+ installId: randomUUID(),+ })++ const gw = gatewayWithContext('ctx-a')+ const realGet = gw.getResource.bind(gw)+ let mcpReads = 0+ vi.spyOn(gw, 'getResource').mockImplementation(async (plural, name, ns) => {+ if (plural === 'mcpservers' && name === 'throwrevoke-dcr') {+ mcpReads += 1+ // 1st read = pre-check (name free → proceed to mint + INSERT conflict);+ // 2nd read = the conflict re-read of the live uid → transient non-404 error.+ if (mcpReads === 1) throw new K8sNotFoundError('mcpservers/throwrevoke-dcr not found')+ throw Object.assign(new Error('apiserver unavailable'), { code: 500 })+ }+ return realGet(plural, name, ns)+ })++ const { transport, calls } = makeDcrTransport({+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),+ })+ const res = await request(+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })+ )+ .post('/admin/mcp-servers/remote')+ .send({+ serverName: 'throwrevoke-dcr',+ contextRef: 'ctx-a',+ baseUrl: 'https://mcp.notion.com/mcp',+ mode: 'dcr',+ })++ // The transient error surfaces (not swallowed), but the minted client is revoked.+ expect(res.status).toBe(500)+ const deleteCalls = calls.filter(c => c.method === 'DELETE')+ expect(deleteCalls.map(c => c.url)).toContain(+ DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.registration_client_uri+ )+ })++ // A fresh pending row (another saga of the same name in flight) → 409+ // install_in_progress; the in-flight row is untouched and our throwaway client is+ // revoked.+ it('install over a fresh pending row → 409 install_in_progress, in-flight row untouched', async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db, rows } = makeInMemoryDynamicClientsDb()+ const inflightInstall = randomUUID()+ await insertDynamicClientPending(db, ENC_KEY, {+ ...LIVE_ROW,+ serverName: 'inprogress-dcr',+ clientId: 'inflight-client-id',+ installId: inflightInstall,+ })+ const before = await getDynamicClient(db, ENC_KEY, {+ serverNamespace: NS,+ serverName: 'inprogress-dcr',+ })++ const gw = gatewayWithContext('ctx-a') // no McpServer seeded → pre-check passes+ const { transport, calls } = makeDcrTransport({+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),+ })+ const res = await request(+ makeAppWithDeps(gw, { db, dcr: { transport, resolveDns: PUBLIC_IP } })+ )+ .post('/admin/mcp-servers/remote')+ .send({+ serverName: 'inprogress-dcr',+ contextRef: 'ctx-a',+ baseUrl: 'https://mcp.notion.com/mcp',+ mode: 'dcr',+ })++ expect(res.status).toBe(409)+ expect(res.body.error).toBe('install_in_progress')+ // The in-flight row is untouched (its install still owns it).+ const after = await getDynamicClient(db, ENC_KEY, {+ serverNamespace: NS,+ serverName: 'inprogress-dcr',+ })+ expect(after).toEqual(before)+ expect(rows.size).toBe(1)+ // Our throwaway minted client was revoked at the AS.+ expect(calls.some(c => c.method === 'DELETE')).toBe(true)+ })++ // An orphan row (bound to a uid with no live CR) does NOT block a reinstall: it is+ // reclaimed (CAS in a tx), the OLD client is revoked, and the install proceeds to 201.+ it('install over a reclaimable orphan → reclaims the row, revokes the old client, 201', async () => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({ ok: true, result: dcrConfidentialResult })+ const { db, rows } = makeInMemoryDynamicClientsDb()+ const orphanInstall = randomUUID()+ await insertDynamicClientPending(db, ENC_KEY, {+ ...LIVE_ROW,+ serverName: 'reclaim-dcr',+ clientId: 'orphan-client-id',+ installId: orphanInstall,+ })+ // Bound to a uid whose CR no longer exists → orphan (reclaimable, no live CR).+ await bindDynamicClientToResource(+ db,+ { serverNamespace: NS, serverName: 'reclaim-dcr' },+ orphanInstall,+ 'uid-orphan-gone'+ )++ const gw = gatewayWithContext('ctx-a') // no live McpServer → orphan is reclaimable+ const { transport, calls } = makeDcrTransport({+ responseJson: JSON.stringify(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE),+ })+ const res = await request(+ makeAppWithDeps(gw, {+ db,+ dcr: { transport, resolveDns: PUBLIC_IP },+ // The reclaim opens a tx; bind it to the same in-memory db.+ runInTransaction: work => work(db as unknown as DbTransactionClient),+ })+ )+ .post('/admin/mcp-servers/remote')+ .send({+ serverName: 'reclaim-dcr',+ contextRef: 'ctx-a',+ baseUrl: 'https://mcp.notion.com/mcp',+ mode: 'dcr',+ })++ expect(res.status).toBe(201)+ // Still exactly one row, now owned by the new install (minted client_id).+ expect(rows.size).toBe(1)+ const owned = await getDynamicClient(db, ENC_KEY, {+ serverNamespace: NS,+ serverName: 'reclaim-dcr',+ })+ expect(owned?.clientId).toBe(DCR_CONFIDENTIAL_REGISTRATION_RESPONSE.client_id)+ // The OLD (orphan) client was revoked at the AS via its handle read under the lock.+ const deleteCalls = calls.filter(c => c.method === 'DELETE')+ expect(deleteCalls.map(c => c.url)).toContain(LIVE_ROW.registrationClientUri)+ })})