Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

refactor(control-api): clean up before deleting an mcp-server CR and fence context attach (R3-H7)

Diego Zacarias178985b4087a
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 0a6faf059..1fea030c8 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.668",
+ "version": "0.1.669",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.668",
+ "version": "0.1.669",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index b552ba4c6..afd6d4008 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.668",
+ "version": "0.1.669",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/oauth/mcpServerOAuthTeardown.ts b/control-api/src/oauth/mcpServerOAuthTeardown.ts
index 11e45e728..e49ae9201 100644
--- a/control-api/src/oauth/mcpServerOAuthTeardown.ts
+++ b/control-api/src/oauth/mcpServerOAuthTeardown.ts
@@ -20,9 +20,10 @@ import { deleteOAuthGrantsForServer } from './store.js'
* - C: the old read→await(7592)→delete gap let a reinstall slip a new row in between.
* Here there is no gap: the claim is one statement.
*
- * Runs AFTER the CR delete; never throws. Each stage is best-effort — a throw is
- * caught, counted (metric, existing stage labels) and logged (names-only), and the
- * uninstall keeps its 200. Changing that response policy is R3-H7, out of scope.
+ * Runs while the CR still exists (see mcpServerUninstall.ts) and never throws: a
+ * throw is caught, counted (metric, per stage) and logged (names-only), and reported
+ * as `'failed'` so the caller can keep the CR and answer "repair required" — the
+ * retry reuses the same uid.
*/
export type TeardownStageResult = 'done' | 'none' | 'failed'
@@ -70,10 +71,7 @@ export async function teardownMcpServerOAuthState(
} catch (err) {
result.dynamicClient = 'failed'
mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'dynamic_client' })
- logger.error(
- { serverName: name, namespace, err },
- 'Dynamic client cleanup failed on uninstall (CR already deleted)'
- )
+ logger.error({ serverName: name, namespace, err }, 'Dynamic client cleanup failed on uninstall')
}
// Grants: purge this installation's rows (fenced by cr_uid + legacy). A reinstall's
@@ -94,10 +92,7 @@ export async function teardownMcpServerOAuthState(
} catch (err) {
result.grants = 'failed'
mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_grants' })
- logger.error(
- { serverName: name, namespace, err },
- 'OAuth grants purge failed on uninstall (CR already deleted)'
- )
+ logger.error({ serverName: name, namespace, err }, 'OAuth grants purge failed on uninstall')
}
return result
diff --git a/control-api/src/oauth/mcpServerUninstall.ts b/control-api/src/oauth/mcpServerUninstall.ts
new file mode 100644
index 000000000..3fe438624
--- /dev/null
+++ b/control-api/src/oauth/mcpServerUninstall.ts
@@ -0,0 +1,258 @@
+import type { DbClient } from '../db.js'
+import { extractK8sError } from '../http/k8sError.js'
+import type { K8sGateway } from '../k8s.js'
+import type { Logger } from '../observability/logger.js'
+import { mcpServerUninstallTeardownFailuresTotal } from '../observability/metrics.js'
+import { isNotFound, stripServerFromContexts } from '../services/contextAllowlist.js'
+import { type SecretCleanupCapture, captureSecretForCleanup } from '../services/secretCleanup.js'
+import type { DcrDeps } from './dcr.js'
+import { teardownMcpServerOAuthState } from './mcpServerOAuthTeardown.js'
+import { deleteOAuthGrantsForServer } from './store.js'
+
+/**
+ * Uninstall of one McpServer with every cleanup BEFORE the CR delete.
+ *
+ * Why this order: the CR's `metadata.uid` is the only identity that fences the OAuth
+ * teardown and the Secret cleanup against a same-name reinstall. While the CR is
+ * alive a failed step can be retried with the same uid, and a reinstall through an
+ * install route cannot create its CR (the name is taken), so it cannot race the
+ * Context strip either. Deleting the CR first — the previous order — made every
+ * cleanup failure permanent: the retry found no CR, hence no uid, hence no teardown.
+ *
+ * 1. read the CR → uid + Secret snapshots (404 → `not_found`, nothing touched)
+ * 2. strip the name from every Context allowlist (→ `contexts`)
+ * 3. OAuth teardown fenced by the uid (→ `dynamic_client` | `oauth_grants`)
+ * 4. delete the snapshotted Secrets (→ `secrets`)
+ * 5. delete the CR fenced by the uid (→ `mcp_server`)
+ * 6. second grants purge by the uid — sweeps a grant a consent callback sealed with
+ * this uid between 3 and 5; a failure only counts and logs, because readers
+ * fenced on the live CR uid already treat such a row as inert.
+ *
+ * The first failing step stops the run and is reported as `incomplete` with the CR
+ * still present, so repeating the same request completes it. The result is transport
+ * neutral: each route maps it to its own HTTP contract.
+ */
+
+export type McpServerUninstallStage =
+ | 'contexts'
+ | 'dynamic_client'
+ | 'oauth_grants'
+ | 'secrets'
+ | 'mcp_server'
+
+export type McpServerUninstallResult =
+ | { status: 'not_found' }
+ | { status: 'completed'; crUid: string; deleted: string[]; crDeleteResponse: unknown }
+ | { status: 'incomplete'; pending: McpServerUninstallStage[]; deleted: string[] }
+
+export type UninstallGateway = Pick<
+ K8sGateway,
+ | 'getResource'
+ | 'listResource'
+ | 'updateResource'
+ | 'deleteResource'
+ | 'getSecret'
+ | 'deleteSecret'
+>
+
+export interface McpServerUninstallDeps {
+ gateway: UninstallGateway
+ db: DbClient
+ encryptionKey: Buffer
+ dcrDeps: DcrDeps
+ /** Contexts live in their own namespace, never in the McpServer's. */
+ contextsNamespace: string
+ logger: Logger
+ /**
+ * Awaited after each accepted K8s delete, before the step counts as done. The
+ * apiserver accepts a delete before finalizers run; a caller that must not report
+ * success until the object is gone (registry) blocks here.
+ */
+ awaitDeletion?: (target: {
+ kind: 'McpServer' | 'Secret'
+ name: string
+ namespace: string
+ }) => Promise<void>
+}
+
+/** Both dependent Secrets of an McpServer, in their fixed cleanup order. */
+export async function captureMcpServerSecrets(
+ gateway: Pick<K8sGateway, 'getSecret'>,
+ name: string,
+ namespace: string,
+ logger: Logger
+): Promise<SecretCleanupCapture[]> {
+ return [
+ await captureSecretForCleanup(gateway, `${name}-credentials`, namespace, logger),
+ // Pre-registered confidential remote installs create this Secret before the CR,
+ // with no ownerReferences, so K8s GC never reaps it.
+ await captureSecretForCleanup(gateway, `${name}-oauth-client`, namespace, logger, {
+ requireManagedOwnership: true,
+ }),
+ ]
+}
+
+/**
+ * Deletes every `ready` capture, fenced on its snapshot. A 404 is already clean.
+ * Returns the names deleted and whether any delete failed; never throws.
+ */
+export async function deleteCapturedSecrets(
+ gateway: Pick<K8sGateway, 'deleteSecret'>,
+ captures: SecretCleanupCapture[],
+ namespace: string,
+ logger: Logger,
+ awaitDeletion?: McpServerUninstallDeps['awaitDeletion']
+): Promise<{ deleted: string[]; failed: boolean }> {
+ const deleted: string[] = []
+ let failed = false
+ for (const capture of captures) {
+ if (capture.status !== 'ready') {
+ if (capture.status !== 'absent') {
+ logger.warn(
+ { secretName: capture.name, namespace, reason: capture.status },
+ 'Skipped McpServer Secret cleanup'
+ )
+ }
+ continue
+ }
+ try {
+ await gateway.deleteSecret(capture.name, namespace, capture.precondition)
+ await awaitDeletion?.({ kind: 'Secret', name: capture.name, namespace })
+ deleted.push(`Secret/${capture.name}`)
+ logger.info({ secretName: capture.name, namespace }, 'Deleted McpServer Secret')
+ } catch (err) {
+ if (extractK8sError(err)?.status === 404) {
+ logger.info({ secretName: capture.name, namespace }, 'McpServer Secret already gone')
+ continue
+ }
+ failed = true
+ mcpServerUninstallTeardownFailuresTotal.inc({
+ stage: capture.name.endsWith('-oauth-client') ? 'oauth_client_secret' : 'secrets',
+ })
+ logger.error(
+ { secretName: capture.name, namespace, err },
+ 'McpServer Secret cleanup failed on uninstall'
+ )
+ }
+ }
+ return { deleted, failed }
+}
+
+function readCrUid(cr: unknown): string | undefined {
+ const uid = (cr as { metadata?: { uid?: unknown } } | null)?.metadata?.uid
+ return typeof uid === 'string' && uid ? uid : undefined
+}
+
+export async function uninstallMcpServer(
+ deps: McpServerUninstallDeps,
+ target: { name: string; namespace: string }
+): Promise<McpServerUninstallResult> {
+ const { gateway, logger } = deps
+ const { name, namespace } = target
+ const deleted: string[] = []
+ const incomplete = (stage: McpServerUninstallStage[]): McpServerUninstallResult => ({
+ status: 'incomplete',
+ pending: stage,
+ deleted,
+ })
+
+ // 1 · CR identity + Secret snapshots.
+ let cr: unknown
+ try {
+ cr = await gateway.getResource('mcpservers', name, namespace)
+ } catch (err) {
+ if (isNotFound(err)) return { status: 'not_found' }
+ throw err
+ }
+ const crUid = readCrUid(cr)
+ if (!crUid) {
+ // Every cleanup below is fenced by this uid, so without it nothing can run safely.
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'mcp_server' })
+ logger.error(
+ { serverName: name, namespace, reason: 'cr_uid_missing' },
+ 'McpServer read returned no metadata.uid; a real apiserver always sets it, so this ' +
+ 'points at a non-apiserver gateway or a broken proxy — retrying will not resolve it'
+ )
+ return incomplete(['mcp_server'])
+ }
+ const secretCaptures = await captureMcpServerSecrets(gateway, name, namespace, logger)
+ // An unreadable Secret now would be orphaned once the CR is gone; stop before any
+ // mutation so the retry re-captures it.
+ if (secretCaptures.some(c => c.status === 'read-failed')) {
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'secrets' })
+ return incomplete(['secrets'])
+ }
+
+ // 2 · Context allowlists.
+ try {
+ await stripServerFromContexts(gateway, deps.contextsNamespace, name, contextName => {
+ deleted.push(`Context/${contextName} (removed from allowlist)`)
+ logger.info({ serverName: name, contextName }, 'Removed MCP server from Context allowlist')
+ })
+ } catch (err) {
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'contexts' })
+ logger.error({ serverName: name, err }, 'Context allowlist cleanup failed on uninstall')
+ return incomplete(['contexts'])
+ }
+
+ // 3 · OAuth state of THIS installation.
+ const teardown = await teardownMcpServerOAuthState(
+ deps.db,
+ deps.encryptionKey,
+ deps.dcrDeps,
+ { namespace, name, crUid },
+ logger
+ )
+ if (teardown.dynamicClient === 'done') deleted.push(`DynamicClient/${name}`)
+ if (teardown.grants === 'done') deleted.push(`OAuthGrants/${name}`)
+ const oauthPending: McpServerUninstallStage[] = []
+ if (teardown.dynamicClient === 'failed') oauthPending.push('dynamic_client')
+ if (teardown.grants === 'failed') oauthPending.push('oauth_grants')
+ if (oauthPending.length > 0) return incomplete(oauthPending)
+
+ // 4 · Secrets, fenced on the step-1 snapshots.
+ const secrets = await deleteCapturedSecrets(
+ gateway,
+ secretCaptures,
+ namespace,
+ logger,
+ deps.awaitDeletion
+ )
+ deleted.push(...secrets.deleted)
+ if (secrets.failed) return incomplete(['secrets'])
+
+ // 5 · The CR itself, fenced on the uid every cleanup above was bound to.
+ let crDeleteResponse: unknown
+ try {
+ crDeleteResponse = await gateway.deleteResource('mcpservers', name, namespace, { uid: crUid })
+ await deps.awaitDeletion?.({ kind: 'McpServer', name, namespace })
+ } catch (err) {
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'mcp_server' })
+ logger.error({ serverName: name, namespace, err }, 'McpServer delete failed on uninstall')
+ return incomplete(['mcp_server'])
+ }
+ deleted.push(`McpServer/${name}`)
+
+ // 6 · Late-sealed grants of this uid.
+ try {
+ const purged = await deleteOAuthGrantsForServer(deps.db, {
+ recipeNamespace: namespace,
+ recipeName: name,
+ crUid,
+ })
+ if (purged > 0) {
+ logger.info(
+ { serverName: name, namespace, count: purged },
+ 'Purged OAuth grants sealed after the uninstall teardown'
+ )
+ }
+ } catch (err) {
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_grants_post_delete' })
+ logger.error(
+ { serverName: name, namespace, err },
+ 'Post-delete OAuth grants purge failed on uninstall'
+ )
+ }
+
+ return { status: 'completed', crUid, deleted, crDeleteResponse }
+}
diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.ts
index 0563c285e..cbfebc4b2 100644
--- a/control-api/src/observability/metrics.ts
+++ b/control-api/src/observability/metrics.ts
@@ -189,14 +189,16 @@ export const mcpHostJwtReissueTotal = getOrCreateCounter({
labelNames: ['result'] as const as Array<'result'>,
})
-// MCP server uninstall teardown: a best-effort cleanup step failed AFTER the CR was
-// already deleted (orphan oauth-client Secret, dynamic client row, or oauth_grants).
-// Nothing retries once the CR is gone, so the structured error log is the only trace
-// today — this counter makes the residual reconcilable/alertable. Rate > 0 ⇒ orphaned
-// state to sweep. Stages: oauth_client_secret | dynamic_client | oauth_grants.
+// MCP server uninstall: a cleanup step failed. Every stage except
+// oauth_grants_post_delete runs before the CR delete and leaves the CR in place for a
+// retry (the request answers "repair required"); a sustained rate means uninstalls
+// that nobody repaired. oauth_grants_post_delete is the only residual after a 200 —
+// rows it leaves are inert to the uid-fenced readers. Stages: contexts |
+// dynamic_client | oauth_grants | secrets | oauth_client_secret | mcp_server |
+// oauth_grants_post_delete.
export const mcpServerUninstallTeardownFailuresTotal = getOrCreateCounter({
name: 'mcp_server_uninstall_teardown_failures_total',
- help: 'Count of best-effort MCP server uninstall teardown steps that failed after CR deletion, by stage.',
+ help: 'Count of MCP server uninstall cleanup steps that failed, by stage.',
labelNames: ['stage'] as const as Array<'stage'>,
})
diff --git a/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts
index 54c554344..74676de36 100644
--- a/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts
+++ b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts
@@ -3,10 +3,10 @@ import express from 'express'
import request from 'supertest'
/**
- * R2-L2 (T3/T4): a best-effort uninstall teardown step that fails AFTER the CR is
- * deleted increments mcp_server_uninstall_teardown_failures_total{stage} — the
- * signal that makes orphaned Secret / dynamic-client / grants state reconcilable
- * and alertable. The uninstall still returns 200: teardown never blocks the delete.
+ * R2-L2 (T3/T4): a failed uninstall teardown step increments
+ * mcp_server_uninstall_teardown_failures_total{stage}. The teardown now runs BEFORE
+ * the CR delete, so the failure also stops the uninstall: 503 repair_required with
+ * the CR still present (a 200 here used to mean "CR gone, grants orphaned for good").
*
* Seam: the same pool.query swap as mcpServerUninstallPurgesGrants.test.ts, but the
* oauth_grants DELETE throws so the H-3 catch runs. The counter is read from the
@@ -33,6 +33,8 @@ vi.mock('../../../db.js', async importActual => {
// Import AFTER vi.mock so the swapped `pool` wires into the router.
const { createAdminResourcesRouter } = await import('../resources.js')
const { registry } = await import('../../../observability/metrics.js')
+const { config } = await import('../../../config.js')
+const { MockGateway } = await import('../../../../test/mockGateway.js')
async function teardownFailures(stage: string): Promise<number> {
const all = await registry.getMetricsAsJSON()
@@ -46,37 +48,33 @@ async function teardownFailures(stage: string): Promise<number> {
describe('DELETE /admin/mcp-servers/:name — teardown failure increments the counter (R2-L2)', () => {
const SERVER_NAME = 'gdrive'
- function buildApp() {
- const gateway = {
- // The uninstall reads the CR before deleting to capture metadata.uid (R3-H5).
- getResource: vi.fn(async () => ({
- metadata: { name: SERVER_NAME, uid: 'uid-gdrive-live', resourceVersion: '1' },
- })),
- deleteResource: vi.fn(async () => ({ metadata: { name: SERVER_NAME } })),
- deleteSecret: vi.fn(async () => ({})),
- getSecret: vi.fn(async () => ({})),
- listResource: vi.fn(async () => []),
- updateResource: vi.fn(async () => ({})),
- }
+ async function buildApp() {
+ const NS = config.mcpServersNamespace
+ const gateway = new MockGateway(NS)
+ await gateway.createResource('mcpservers', { metadata: { name: SERVER_NAME }, spec: {} }, NS)
const app = express()
app.use(express.json({ limit: '1mb' }))
app.use('/api/v1', createAdminResourcesRouter(gateway as never))
- return { app }
+ return { app, gateway, NS }
}
beforeEach(() => {
dbMock.query.mockClear()
})
- it('counts a failed oauth_grants purge under stage=oauth_grants and still returns 200', async () => {
+ it('counts a failed oauth_grants purge under stage=oauth_grants and keeps the CR for a retry', async () => {
const before = await teardownFailures('oauth_grants')
- const { app } = buildApp()
+ const { app, gateway, NS } = await buildApp()
const res = await request(app).delete(`/api/v1/admin/mcp-servers/${SERVER_NAME}`)
- // Teardown is best-effort — a purge failure must never block the uninstall.
- expect(res.status).toBe(200)
- // The failed stage is now an observable metric, not just a log line.
+ expect(res.status).toBe(503)
+ expect(res.body).toMatchObject({
+ error: 'mcp_server_uninstall_incomplete',
+ outcome: 'repair_required',
+ pending: ['oauth_grants'],
+ })
+ await expect(gateway.getResource('mcpservers', SERVER_NAME, NS)).resolves.toBeTruthy()
const after = await teardownFailures('oauth_grants')
expect(after - before).toBeGreaterThanOrEqual(1)
})
diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.ts
index b754cae78..bb636e37a 100644
--- a/control-api/src/routes/admin/registry.ts
+++ b/control-api/src/routes/admin/registry.ts
@@ -15,6 +15,7 @@ import { GenericConfigSuggestionSchema } from '../../oauth/genericKnobs.js'
import { getOAuthProviderAdapter, isKnownOAuthProvider } from '../../oauth/providers.js'
import { rootLogger } from '../../observability/logger.js'
import { type AdminUserRecord, findAdminById } from '../../services/adminAuthService.js'
+import { attachServerToContext } from '../../services/contextAllowlist.js'
import {
addHookRefToHost,
listHostsReferencingHook,
@@ -2726,42 +2727,18 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {
const contextsNs = config.contextsNamespace
let contextBefore: RegistryResourceSnapshot | null = null
try {
- const ctx = (await gateway.getResource('contexts', contextRef, contextsNs)) as {
- metadata?: { uid?: string; resourceVersion?: string }
- spec?: Record<string, unknown> & {
- contextId?: string
- description?: string
- mcpServers?: string[]
- }
- }
- contextBefore = normalizeRegistryResourceSnapshot(ctx, contextRef, contextsNs)
- if (!contextBefore.metadata?.uid || !contextBefore.metadata.resourceVersion) {
- throw Object.assign(new Error('Context identity is unavailable'), {
- statusCode: 503,
- code: 'context_identity_unavailable',
- })
- }
- const existing: string[] = ctx.spec?.mcpServers ?? []
- if (!existing.includes(serverName)) {
- await gateway.updateResource(
- 'contexts',
- contextRef,
- {
- metadata: {
- uid: contextBefore.metadata.uid,
- ...(ctx.metadata?.resourceVersion
- ? { resourceVersion: ctx.metadata.resourceVersion }
- : {}),
- },
- spec: {
- ...ctx.spec,
- contextId: ctx.spec?.contextId ?? contextRef,
- mcpServers: [...existing, serverName],
- } as Record<string, unknown>,
+ // The before-image for the ambiguous-outcome readback below is the snapshot
+ // the LAST attempt edited: an earlier one is stale after a 409 retry.
+ await attachServerToContext(
+ gateway,
+ { name: contextRef, namespace: contextsNs },
+ serverName,
+ {
+ onRead: ctx => {
+ contextBefore = normalizeRegistryResourceSnapshot(ctx, contextRef, contextsNs)
},
- contextsNs
- )
- }
+ }
+ )
} catch (err) {
let associationOutcome: RegistryMutationOutcome = isDeterministicRegistryNoCommit(err)
? 'not-committed'
diff --git a/control-api/src/routes/admin/remoteMcp.ts b/control-api/src/routes/admin/remoteMcp.ts
index 81d99900a..06307fb27 100644
--- a/control-api/src/routes/admin/remoteMcp.ts
+++ b/control-api/src/routes/admin/remoteMcp.ts
@@ -34,6 +34,7 @@ import {
import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'
import { probeMcpTransport } from '../../oauth/mcpTransportProbe.js'
import { rootLogger } from '../../observability/logger.js'
+import { attachServerToContext } from '../../services/contextAllowlist.js'
import { K8sNotFoundError } from '../../services/resourceService.js'
import type { SecretSnapshot } from '../../services/secretRepository.js'
import type { ResourcePreconditions } from '../../types.js'
@@ -1111,19 +1112,7 @@ export function createAdminRemoteMcpRouter(
// ── Saga step 3: attach to the Context allowlist (rollback CR+Secret) ──
try {
- const ctx = (await gateway.getResource('contexts', contextRef)) as {
- spec?: Record<string, unknown> & { contextId?: string; mcpServers?: string[] }
- }
- const existing: string[] = ctx.spec?.mcpServers ?? []
- if (!existing.includes(serverName)) {
- await gateway.updateResource('contexts', contextRef, {
- spec: {
- ...ctx.spec,
- contextId: ctx.spec?.contextId ?? contextRef,
- mcpServers: [...existing, serverName],
- } as Record<string, unknown>,
- })
- }
+ await attachServerToContext(gateway, { name: contextRef }, serverName)
} catch (err) {
try {
await gateway.deleteResource(
diff --git a/control-api/src/routes/admin/resources.ts b/control-api/src/routes/admin/resources.ts
index 9cc8aa66f..a888a0397 100644
--- a/control-api/src/routes/admin/resources.ts
+++ b/control-api/src/routes/admin/resources.ts
@@ -13,21 +13,21 @@ import { validateCommunicationChannelSpec } from '../../http/validateCommunicati
import { validateMcpServerSpecPreflight } from '../../http/validateMcpServerSpec.js'
import { K8sGateway } from '../../k8s.js'
import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'
-import { teardownMcpServerOAuthState } from '../../oauth/mcpServerOAuthTeardown.js'
+import {
+ captureMcpServerSecrets,
+ deleteCapturedSecrets,
+ uninstallMcpServer,
+} from '../../oauth/mcpServerUninstall.js'
import { rootLogger } from '../../observability/logger.js'
-import { mcpServerUninstallTeardownFailuresTotal } from '../../observability/metrics.js'
+import { stripServerFromContexts } from '../../services/contextAllowlist.js'
import { stripHookRefFromHosts } from '../../services/hostGuardrailRefs.js'
import {
K8sConflictError,
K8sNotFoundError,
type MutableResourceSnapshot,
} from '../../services/resourceService.js'
+import { captureSecretForCleanup } from '../../services/secretCleanup.js'
import { secretKeyNames } from '../../services/secretKeyNames.js'
-import {
- isControlApiManagedSecret,
- isRecipeOwnedSecret,
- secretIdentityPreconditions,
-} from '../../services/secretRepository.js'
import {
ClerumResourceType,
type ResourcePreconditions,
@@ -1079,6 +1079,47 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {
const { plural, ns } = resolveResource(req.params.resource)!
const name = req.params.name
+ if (plural === 'mcpservers') {
+ const outcome = await uninstallMcpServer(
+ {
+ gateway,
+ db: dcrDb,
+ encryptionKey: oauthEncryptionKey,
+ dcrDeps: { logger: log },
+ contextsNamespace: resourceNamespace('contexts'),
+ logger: log,
+ },
+ { name, namespace: ns }
+ )
+ if (outcome.status === 'completed') {
+ res.status(200).json(outcome.crDeleteResponse)
+ return
+ }
+ if (outcome.status === 'incomplete') {
+ res.status(503).json({
+ error: 'mcp_server_uninstall_incomplete',
+ outcome: 'repair_required',
+ pending: outcome.pending,
+ deleted: outcome.deleted,
+ })
+ return
+ }
+ // The CR read found nothing, so there is no uid to fence an OAuth teardown by:
+ // only the name-addressed K8s cleanup runs, and OAuth state is left alone.
+ const secretCaptures = await captureMcpServerSecrets(gateway, name, ns, log)
+ const deleted = await gateway.deleteResource(plural, name, ns)
+ await deleteCapturedSecrets(gateway, secretCaptures, ns, log)
+ try {
+ await stripServerFromContexts(gateway, resourceNamespace('contexts'), name, contextName =>
+ log.info({ serverName: name, contextName }, 'Removed MCP server from Context allowlist')
+ )
+ } catch (err) {
+ log.error({ serverName: name, err }, 'Failed to clean up Context allowlists')
+ }
+ res.status(200).json(deleted)
+ return
+ }
+
// CommunicationChannel: read the credentialsSecretRef name FIRST (we
// can't read the CC after it's deleted), then delete the CC, then
// delete the Secret. The CC-first order means HCC's SecretInformer
@@ -1102,105 +1143,15 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {
}
}
- // Capture the credential Secret identity BEFORE deleting the parent CR.
- // A same-name Secret can be created in the gap, and a name-addressed
- // delete afterwards would remove that replacement — a different owner's
- // object — on the strength of a decision made about something that no
- // longer exists. Every cleanup below is bound to the snapshot taken here.
- // Same reasoning, and same shape, as the registry uninstall path.
- const captureSecretForCleanup = async (
- secretName: string,
- { requireManagedOwnership = false }: { requireManagedOwnership?: boolean } = {}
- ): Promise<
- | { status: 'ready'; precondition: SecretPreconditions }
- | {
- status:
- | 'absent'
- | 'recipe-owned'
- | 'not-managed'
- | 'identity-unavailable'
- | 'read-failed'
- }
- > => {
- let raw: unknown
- try {
- raw = await gateway.getSecret(secretName, ns)
- } catch (err) {
- if (extractK8sStatusCode(err) === 404) return { status: 'absent' }
- // Deleting the CR is the request's actual intent; the Secret is a
- // cascade. A failed read means we cannot fence the cascade delete, so
- // it is skipped and logged — the same policy this PR already applies
- // to the rollback paths ("rollback skipped: identity unavailable").
- // Failing the whole request here would leave the CR undeleted because
- // its dependent Secret could not be read.
- log.error(
- { secretName, namespace: ns, err },
- 'Secret cleanup capture failed; cascade delete will be skipped'
- )
- return { status: 'read-failed' }
- }
- // Recipe-owned Secrets belong to /admin/recipe-secrets. The mcp-secret
- // route refuses them with 409; deleting one here would route around
- // that guard.
- if (isRecipeOwnedSecret(raw)) return { status: 'recipe-owned' }
- // Derived-name Secrets (`${name}-oauth-client`) share a name with any
- // operator-owned Secret referenced in OAuth reference mode. Only delete
- // when the managed-by label proves control-api's install saga created it;
- // UID/RV fencing guarantees same-object, not same-owner.
- if (requireManagedOwnership && !isControlApiManagedSecret(raw)) {
- return { status: 'not-managed' }
- }
- const precondition = secretIdentityPreconditions(raw)
- if (!precondition) return { status: 'identity-unavailable' }
- return { status: 'ready', precondition }
- }
-
+ // Capture the credential Secret identity BEFORE deleting the parent CR: a
+ // same-name Secret created in the gap belongs to a different owner, and a
+ // name-addressed delete afterwards would remove it.
const ccSecretCleanup =
plural === 'communicationchannels' && ccSecretRefName
- ? await captureSecretForCleanup(ccSecretRefName)
- : null
- const mcpCredentialsSecretName = `${name}-credentials`
- const mcpSecretCleanup =
- plural === 'mcpservers' ? await captureSecretForCleanup(mcpCredentialsSecretName) : null
- // Pre-registered confidential remote installs create a `${name}-oauth-client`
- // Secret in the install saga with NO ownerReferences (it predates the CR), so
- // K8s GC never reaps it — control-api must delete it here. Fence it on the
- // identity captured BEFORE the CR delete, exactly like the credentials Secret:
- // a name-addressed delete could otherwise clobber a fresh same-name Secret
- // written by a concurrent reinstall in the gap (the bug this capture shape
- // exists to prevent). 'absent' (404) is the norm for non-pre-registered servers.
- const mcpOAuthClientSecretName = `${name}-oauth-client`
- const mcpOAuthClientSecretCleanup =
- plural === 'mcpservers'
- ? await captureSecretForCleanup(mcpOAuthClientSecretName, {
- requireManagedOwnership: true,
- })
+ ? await captureSecretForCleanup(gateway, ccSecretRefName, ns, log)
: null
- // mcpservers only: read the CR before deleting so its metadata.uid fences BOTH
- // the delete (a same-name reinstall's CR has a different uid → the delete
- // precondition 409s and we never tear down its state) AND the OAuth teardown
- // (only this installation's dynamic_clients row + grants are removed, R3-H5). A
- // 404 means the CR is already gone → no uid, so the teardown is skipped entirely.
- let mcpServerCrUid: string | undefined
- if (plural === 'mcpservers') {
- try {
- const cr = await gateway.getResource(plural, name, ns)
- mcpServerCrUid = resourcePreconditionsFromResource(cr)?.uid
- } catch (err) {
- // getResource wraps a namespaced 404 as K8sNotFoundError (httpStatus, not
- // statusCode), so match both shapes — as the update handler does above.
- if (!(err instanceof K8sNotFoundError) && extractK8sStatusCode(err) !== 404) throw err
- // CR already gone: leave uid undefined so the teardown below is skipped.
- }
- }
-
- // Only the mcpservers delete is uid-fenced; every other plural keeps the exact
- // 3-arg call it had (a trailing undefined would change the observed call shape).
- const deleted =
- mcpServerCrUid !== undefined
- ? await gateway.deleteResource(plural, name, ns, { uid: mcpServerCrUid })
- : await gateway.deleteResource(plural, name, ns)
+ const deleted = await gateway.deleteResource(plural, name, ns)
if (plural === 'communicationchannels' && ccSecretRefName && ccSecretCleanup) {
if (ccSecretCleanup.status !== 'ready') {
@@ -1233,153 +1184,6 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {
}
}
- if (plural === 'mcpservers') {
- const contextsNs = resourceNamespace('contexts')
- if (mcpSecretCleanup && mcpSecretCleanup.status !== 'ready') {
- log.warn(
- {
- secretName: mcpCredentialsSecretName,
- namespace: ns,
- reason: mcpSecretCleanup.status,
- },
- 'Skipped MCP credentials Secret cleanup'
- )
- } else if (mcpSecretCleanup) {
- try {
- await gateway.deleteSecret(mcpCredentialsSecretName, ns, mcpSecretCleanup.precondition)
- log.info(
- { secretName: mcpCredentialsSecretName, namespace: ns },
- 'Deleted MCP credentials Secret'
- )
- } catch (err) {
- if (extractK8sStatusCode(err) === 404) {
- log.info(
- { secretName: mcpCredentialsSecretName, namespace: ns },
- 'MCP credentials Secret already gone'
- )
- } else {
- // The McpServer is already gone, so a 200 is still the honest
- // outcome — but this is a cleanup FAILURE, not an absence. The
- // previous catch reported every error class, 403 included, as
- // "no Secret to delete", telling the operator a credential had
- // been removed while it was still live.
- log.error(
- { secretName: mcpCredentialsSecretName, namespace: ns, err },
- 'McpServer delete succeeded but credentials cleanup failed'
- )
- }
- }
- }
-
- // Pre-registered confidential remote installs leave a `${name}-oauth-client`
- // Secret with no ownerReferences (see the capture above). Delete it fenced on
- // the identity captured before the CR delete; 'absent' (404) is expected for
- // every non-pre-registered server and never fails the uninstall.
- if (mcpOAuthClientSecretCleanup) {
- if (mcpOAuthClientSecretCleanup.status === 'ready') {
- try {
- await gateway.deleteSecret(
- mcpOAuthClientSecretName,
- ns,
- mcpOAuthClientSecretCleanup.precondition
- )
- log.info(
- { secretName: mcpOAuthClientSecretName, namespace: ns },
- 'Deleted MCP OAuth client Secret'
- )
- } catch (err) {
- if (extractK8sStatusCode(err) === 404) {
- log.info(
- { secretName: mcpOAuthClientSecretName, namespace: ns },
- 'MCP OAuth client Secret already gone'
- )
- } else {
- mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_client_secret' })
- log.error(
- { secretName: mcpOAuthClientSecretName, namespace: ns, err },
- 'McpServer delete succeeded but OAuth client Secret cleanup failed'
- )
- }
- }
- } else if (mcpOAuthClientSecretCleanup.status === 'absent') {
- // Expected for every non-pre-registered server: no OAuth client Secret exists.
- log.info(
- { secretName: mcpOAuthClientSecretName, namespace: ns },
- 'No MCP OAuth client Secret to delete'
- )
- } else {
- log.warn(
- {
- secretName: mcpOAuthClientSecretName,
- namespace: ns,
- reason: mcpOAuthClientSecretCleanup.status,
- },
- 'Skipped MCP OAuth client Secret cleanup'
- )
- }
- }
-
- // OAuth teardown fenced by the CR uid (R3-H5): revoke this installation's
- // encrypted `dynamic_clients` row + best-effort RFC 7592 delete at the AS, and
- // purge its `oauth_grants` rows — never a same-name reinstall's state. Skipped
- // when the uid is unknown (the CR read 404'd): a name-only teardown here would
- // reopen the very race this fences. The module never throws; it counts/logs per
- // stage and the uninstall keeps its 200 (the response policy is R3-H7).
- if (mcpServerCrUid !== undefined) {
- const teardown = await teardownMcpServerOAuthState(
- dcrDb,
- oauthEncryptionKey,
- { logger: log },
- { namespace: ns, name, crUid: mcpServerCrUid },
- log
- )
- log.info(
- {
- serverName: name,
- namespace: ns,
- dynamicClient: teardown.dynamicClient,
- grants: teardown.grants,
- },
- 'MCP server OAuth teardown completed'
- )
- }
-
- try {
- const ctxList = (await gateway.listResource('contexts', contextsNs)) as Array<{
- metadata?: { name?: string }
- spec?: Record<string, unknown> & {
- contextId?: string
- description?: string
- mcpServers?: string[]
- }
- }>
- for (const ctx of ctxList) {
- const ctxName = ctx.metadata?.name
- const servers = ctx.spec?.mcpServers ?? []
- if (ctxName && servers.includes(name)) {
- await gateway.updateResource(
- 'contexts',
- ctxName,
- {
- spec: {
- ...ctx.spec,
- contextId: ctx.spec?.contextId ?? ctxName,
- mcpServers: servers.filter(s => s !== name),
- } as Record<string, unknown>,
- },
- contextsNs
- )
- log.info(
- { serverName: name, contextName: ctxName },
- 'Removed MCP server from Context allowlist'
- )
- }
- }
- } catch (err) {
- log.error({ serverName: name, err }, 'Failed to clean up Context allowlists')
- }
- }
-
res.status(200).json(deleted)
})
)
diff --git a/control-api/src/services/contextAllowlist.ts b/control-api/src/services/contextAllowlist.ts
new file mode 100644
index 000000000..4920126a0
--- /dev/null
+++ b/control-api/src/services/contextAllowlist.ts
@@ -0,0 +1,156 @@
+import { extractK8sError } from '../http/k8sError.js'
+import type { K8sGateway } from '../k8s.js'
+import { K8sNotFoundError } from './resourceService.js'
+
+/** Shape of a Context CR as the allowlist writers read it. */
+export type ContextAllowlistSnapshot = {
+ metadata?: { name?: string; uid?: string; resourceVersion?: string }
+ spec?: Record<string, unknown> & { contextId?: string; mcpServers?: string[] }
+}
+
+/**
+ * Pure edit of `spec.mcpServers`. Returns the new list, or `null` when the current
+ * list already satisfies the caller (no write is issued).
+ */
+export type ContextAllowlistEdit = (servers: readonly string[]) => string[] | null
+
+/**
+ * Same shape as the registry's historical inline error so its outcome classifier
+ * (`statusCode` 503, string `code`) keeps treating it identically.
+ */
+export class ContextIdentityUnavailableError extends Error {
+ readonly statusCode = 503
+ readonly code = 'context_identity_unavailable'
+ constructor(contextName: string) {
+ super(`Context/${contextName} identity unavailable; refusing stale update`)
+ this.name = 'ContextIdentityUnavailableError'
+ }
+}
+
+/**
+ * The Context status subresource moves the resourceVersion on every controller status
+ * write, so a first-attempt 409 is routine and not evidence of a competing allowlist
+ * writer. Three attempts absorb that churn while still surfacing a genuinely hot object.
+ */
+export const CONTEXT_ALLOWLIST_MAX_ATTEMPTS = 3
+
+type AllowlistGateway = Pick<K8sGateway, 'getResource' | 'updateResource'>
+
+function isConflict(err: unknown): boolean {
+ return extractK8sError(err)?.status === 409
+}
+
+export function isNotFound(err: unknown): boolean {
+ return err instanceof K8sNotFoundError || extractK8sError(err)?.status === 404
+}
+
+/**
+ * Read-edit-write of one Context's `spec.mcpServers`, fenced on the uid AND
+ * resourceVersion of the read it edited. A write without the resourceVersion replays
+ * a stale list: an attach that read `[X]` before X's uninstall stripped it would put
+ * X back. On 409 the Context is re-read and the edit recomputed from the fresh list,
+ * never replayed.
+ *
+ * `initial` lets a caller that already listed the Context skip the first GET.
+ * `onRead` observes every snapshot the edit is computed from (registry keeps the last
+ * one as the before-image for its ambiguous-outcome readback).
+ */
+export async function editContextAllowlist(
+ gateway: AllowlistGateway,
+ target: { name: string; namespace?: string },
+ edit: ContextAllowlistEdit,
+ options: {
+ initial?: ContextAllowlistSnapshot
+ onRead?: (ctx: ContextAllowlistSnapshot) => void
+ maxAttempts?: number
+ } = {}
+): Promise<{ changed: boolean }> {
+ const { name, namespace } = target
+ const maxAttempts = options.maxAttempts ?? CONTEXT_ALLOWLIST_MAX_ATTEMPTS
+ for (let attempt = 1; ; attempt += 1) {
+ const ctx =
+ attempt === 1 && options.initial
+ ? options.initial
+ : ((namespace === undefined
+ ? await gateway.getResource('contexts', name)
+ : await gateway.getResource('contexts', name, namespace)) as ContextAllowlistSnapshot)
+ options.onRead?.(ctx)
+ const uid = ctx.metadata?.uid
+ const resourceVersion = ctx.metadata?.resourceVersion
+ if (
+ typeof uid !== 'string' ||
+ !uid ||
+ typeof resourceVersion !== 'string' ||
+ !resourceVersion
+ ) {
+ throw new ContextIdentityUnavailableError(name)
+ }
+ const next = edit(ctx.spec?.mcpServers ?? [])
+ if (next === null) return { changed: false }
+ const body = {
+ metadata: { uid, resourceVersion },
+ spec: {
+ ...ctx.spec,
+ contextId: ctx.spec?.contextId ?? name,
+ mcpServers: next,
+ } as Record<string, unknown>,
+ }
+ try {
+ if (namespace === undefined) {
+ await gateway.updateResource('contexts', name, body)
+ } else {
+ await gateway.updateResource('contexts', name, body, namespace)
+ }
+ return { changed: true }
+ } catch (err) {
+ if (isConflict(err) && attempt < maxAttempts) continue
+ throw err
+ }
+ }
+}
+
+/** Adds `serverName` to the Context allowlist unless it is already there. */
+export function attachServerToContext(
+ gateway: AllowlistGateway,
+ target: { name: string; namespace?: string },
+ serverName: string,
+ options: { onRead?: (ctx: ContextAllowlistSnapshot) => void } = {}
+): Promise<{ changed: boolean }> {
+ return editContextAllowlist(
+ gateway,
+ target,
+ servers => (servers.includes(serverName) ? null : [...servers, serverName]),
+ options
+ )
+}
+
+/**
+ * Removes `serverName` from every Context in `namespace` that allowlists it. A Context
+ * deleted between the list and its write is already clean. Throws on the first Context
+ * that cannot be written; the ones already stripped are returned through `onStripped`
+ * so a partial run is reportable.
+ */
+export async function stripServerFromContexts(
+ gateway: Pick<K8sGateway, 'listResource' | 'getResource' | 'updateResource'>,
+ namespace: string,
+ serverName: string,
+ onStripped: (contextName: string) => void
+): Promise<void> {
+ const contexts = (await gateway.listResource('contexts', namespace)) as ContextAllowlistSnapshot[]
+ for (const ctx of contexts) {
+ const contextName = ctx.metadata?.name
+ if (!contextName || !(ctx.spec?.mcpServers ?? []).includes(serverName)) continue
+ try {
+ const { changed } = await editContextAllowlist(
+ gateway,
+ { name: contextName, namespace },
+ servers => (servers.includes(serverName) ? servers.filter(s => s !== serverName) : null),
+ { initial: ctx }
+ )
+ if (changed) onStripped(contextName)
+ } catch (err) {
+ if (isNotFound(err)) continue
+ throw err
+ }
+ }
+}
diff --git a/control-api/src/services/secretCleanup.ts b/control-api/src/services/secretCleanup.ts
new file mode 100644
index 000000000..5746ace1d
--- /dev/null
+++ b/control-api/src/services/secretCleanup.ts
@@ -0,0 +1,49 @@
+import { extractK8sError } from '../http/k8sError.js'
+import type { K8sGateway } from '../k8s.js'
+import type { Logger } from '../observability/logger.js'
+import type { SecretPreconditions } from '../types.js'
+import {
+ isControlApiManagedSecret,
+ isRecipeOwnedSecret,
+ secretIdentityPreconditions,
+} from './secretRepository.js'
+
+export type SecretCleanupCapture =
+ | { status: 'ready'; name: string; precondition: SecretPreconditions }
+ | {
+ status: 'absent' | 'recipe-owned' | 'not-managed' | 'identity-unavailable' | 'read-failed'
+ name: string
+ }
+
+/**
+ * Snapshot a dependent Secret's identity so its later delete is bound to this exact
+ * object: a same-name Secret written afterwards is a different owner's.
+ * `requireManagedOwnership` guards derived names (`${name}-oauth-client`) that OAuth
+ * reference mode lets an operator Secret share — uid/RV proves same object, not same
+ * owner, so only a Secret labelled as created by control-api's install saga qualifies.
+ */
+export async function captureSecretForCleanup(
+ gateway: Pick<K8sGateway, 'getSecret'>,
+ secretName: string,
+ namespace: string,
+ logger: Logger,
+ { requireManagedOwnership = false }: { requireManagedOwnership?: boolean } = {}
+): Promise<SecretCleanupCapture> {
+ let raw: unknown
+ try {
+ raw = await gateway.getSecret(secretName, namespace)
+ } catch (err) {
+ if (extractK8sError(err)?.status === 404) return { status: 'absent', name: secretName }
+ logger.error({ secretName, namespace, err }, 'Secret cleanup capture failed')
+ return { status: 'read-failed', name: secretName }
+ }
+ // Recipe-owned Secrets belong to /admin/recipe-secrets, whose guard a cascade
+ // delete here would route around.
+ if (isRecipeOwnedSecret(raw)) return { status: 'recipe-owned', name: secretName }
+ if (requireManagedOwnership && !isControlApiManagedSecret(raw)) {
+ return { status: 'not-managed', name: secretName }
+ }
+ const precondition = secretIdentityPreconditions(raw)
+ if (!precondition) return { status: 'identity-unavailable', name: secretName }
+ return { status: 'ready', name: secretName, precondition }
+}
diff --git a/control-api/test/routes.adminRemoteMcp.test.ts b/control-api/test/routes.adminRemoteMcp.test.ts
index 6bd833579..2c9814674 100644
--- a/control-api/test/routes.adminRemoteMcp.test.ts
+++ b/control-api/test/routes.adminRemoteMcp.test.ts
@@ -580,7 +580,10 @@ describe('POST /admin/mcp-servers/remote (install saga)', () => {
NS
)) as { metadata: { uid: string } }
recreatedCrUid = recreated.metadata.uid
- throw Object.assign(new Error('conflict'), { code: 409 })
+ // Not a 409: the attach re-reads and retries on a conflict (the Context status
+ // subresource moves its resourceVersion), so only a non-retryable failure
+ // reaches the compensation this test is about.
+ throw Object.assign(new Error('context write failed'), { code: 500 })
})
const res = await request(makeApp(gw)).post('/admin/mcp-servers/remote').send({
diff --git a/control-api/test/routes.mcpServerUninstall.contextAllowlist.test.ts b/control-api/test/routes.mcpServerUninstall.contextAllowlist.test.ts
new file mode 100644
index 000000000..d6ef16f46
--- /dev/null
+++ b/control-api/test/routes.mcpServerUninstall.contextAllowlist.test.ts
@@ -0,0 +1,190 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+import type { PinnedTransport } from '../src/http/pinnedFetch.js'
+import type { K8sGateway } from '../src/k8s.js'
+import {
+ type DiscoveryOutcome,
+ type DiscoveryResult,
+ discoverRemoteOAuth,
+} from '../src/oauth/discovery.js'
+import { probeMcpTransport } from '../src/oauth/mcpTransportProbe.js'
+import { PILOTS, makeDiscoveryTransport } from './fixtures/remoteOAuthDiscovery.js'
+import { MockGateway } from './mockGateway.js'
+
+/**
+ * The McpServer uninstall vs. the Context allowlist writers of the install routes
+ * (R3-H5b + R3-S6), observed as the FINAL allowlist (T4).
+ *
+ * Both concurrent actors are the real routes mounted on one app over one stateful
+ * MockGateway: the generic uninstall (`DELETE /admin/mcp-servers/:name`) and the
+ * remote install saga (`POST /admin/mcp-servers/remote`, CIMD mode). The race is
+ * forced from a gateway hook that runs one route inside the other's K8s call — no
+ * allowlist write is hand-emulated (T1).
+ *
+ * The OAuth teardown's DB is a benign empty stub: these tests are about K8s order.
+ */
+
+vi.mock('node:dns/promises', () => ({
+ lookup: vi.fn(async () => [{ address: '93.184.216.34', family: 4 }]),
+}))
+
+vi.mock('../src/oauth/discovery.js', async importActual => {
+ const actual = await importActual<typeof import('../src/oauth/discovery.js')>()
+ return { ...actual, discoverRemoteOAuth: vi.fn() }
+})
+
+vi.mock('../src/db.js', async importActual => {
+ const actual = await importActual<typeof import('../src/db.js')>()
+ return { ...actual, pool: { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) } }
+})
+
+const { config } = await import('../src/config.js')
+const { createAdminResourcesRouter } = await import('../src/routes/admin/resources.js')
+const { createAdminRemoteMcpRouter } = await import('../src/routes/admin/remoteMcp.js')
+
+const NS = config.mcpServersNamespace
+
+const PROBE_DNS = async () => ['93.184.216.34']
+const ALIVE_TRANSPORT: PinnedTransport = async ({ method }) =>
+ method === 'POST'
+ ? { status: 401, headers: { 'www-authenticate': 'Bearer realm="OAuth"' }, bodyText: '' }
+ : { status: 404, headers: {}, bodyText: '' }
+const probe: typeof probeMcpTransport = (baseUrl, deps, opts) =>
+ probeMcpTransport(baseUrl, { ...deps, transport: ALIVE_TRANSPORT, resolveDns: PROBE_DNS }, opts)
+
+// Real Notion discovery with RFC 9207 advertised, derived from the real producer (T1)
+// by the same documented substitution the remote-install route tests use.
+let rfc9207Result: DiscoveryResult
+beforeAll(async () => {
+ const actual = await vi.importActual<typeof import('../src/oauth/discovery.js')>(
+ '../src/oauth/discovery.js'
+ )
+ const as = JSON.parse(PILOTS.notion.as.json)
+ as.authorization_response_iss_parameter_supported = true
+ const pilot = { ...PILOTS.notion, as: { ...PILOTS.notion.as, json: JSON.stringify(as) } }
+ const outcome = await actual.discoverRemoteOAuth(pilot.mcpUrl, {
+ transport: makeDiscoveryTransport(pilot),
+ resolveDns: PROBE_DNS,
+ })
+ if (!outcome.ok) throw new Error(`fixture discovery failed: ${outcome.error.kind}`)
+ rfc9207Result = outcome.result
+})
+
+let prevContextsNs = ''
+let prevCallbackBaseUrl = ''
+beforeEach(() => {
+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({
+ ok: true,
+ result: rfc9207Result,
+ } as DiscoveryOutcome)
+ // The saga reads the Context in the gateway default namespace and the uninstall
+ // in `contextsNamespace`; pinning them together makes both address one object, as
+ // they do in production.
+ prevContextsNs = config.contextsNamespace
+ config.contextsNamespace = NS
+ prevCallbackBaseUrl = config.oauthCallbackBaseUrl
+ config.oauthCallbackBaseUrl = 'https://control.example.com'
+})
+afterEach(() => {
+ config.contextsNamespace = prevContextsNs
+ config.oauthCallbackBaseUrl = prevCallbackBaseUrl
+ vi.restoreAllMocks()
+})
+
+function makeApp(gw: MockGateway) {
+ const app = express()
+ app.use(express.json())
+ app.use(createAdminResourcesRouter(gw as unknown as K8sGateway))
+ app.use(createAdminRemoteMcpRouter(gw as unknown as K8sGateway, { probe }))
+ return app
+}
+
+async function seed(gw: MockGateway, servers: string[]): Promise<void> {
+ await gw.createResource(
+ 'contexts',
+ { metadata: { name: 'ctx-a' }, spec: { contextId: 'ctx-a', mcpServers: servers } },
+ NS
+ )
+ for (const name of servers) {
+ await gw.createResource('mcpservers', { metadata: { name }, spec: {} }, NS)
+ }
+}
+
+const installBody = (serverName: string) => ({
+ serverName,
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'cimd',
+})
+
+async function allowlist(gw: MockGateway): Promise<string[]> {
+ const ctx = (await gw.getResource('contexts', 'ctx-a', NS)) as {
+ spec: { mcpServers?: string[] }
+ }
+ return ctx.spec.mcpServers ?? []
+}
+
+describe('McpServer uninstall vs. Context allowlist writers', () => {
+ // §7 row 7 (R3-H5b): a same-name reinstall can only create its CR once the old one
+ // is gone. Stripping the allowlist AFTER that delete removed the reinstall's entry.
+ it('a same-name reinstall that attaches right after the CR delete keeps its allowlist entry', async () => {
+ const gw = new MockGateway(NS)
+ await seed(gw, ['srv', 'other'])
+ const app = makeApp(gw)
+
+ let reinstallStatus: number | undefined
+ let armed = true
+ vi.spyOn(gw, 'deleteResource').mockImplementation(async (plural, name, ns, pre) => {
+ const result = await MockGateway.prototype.deleteResource.call(gw, plural, name, ns, pre)
+ if (armed && plural === 'mcpservers' && name === 'srv') {
+ armed = false
+ const reinstall = await request(app)
+ .post('/admin/mcp-servers/remote')
+ .send(installBody('srv'))
+ reinstallStatus = reinstall.status
+ }
+ return result
+ })
+
+ const res = await request(app).delete('/admin/mcp-servers/srv')
+
+ expect(res.status).toBe(200)
+ expect(reinstallStatus).toBe(201)
+ // The reinstalled server is live AND reachable through the Context.
+ await expect(gw.getResource('mcpservers', 'srv', NS)).resolves.toBeTruthy()
+ expect(await allowlist(gw)).toEqual(['other', 'srv'])
+ })
+
+ // §6.7 (R3-S6): an install of ANOTHER name read the Context before the uninstall's
+ // strip and writes after it. Without a resourceVersion fence its write replays the
+ // stale list and puts the uninstalled name back.
+ it('an install of another name racing the strip does not reintroduce the uninstalled name', async () => {
+ const gw = new MockGateway(NS)
+ await seed(gw, ['old-srv'])
+ const app = makeApp(gw)
+
+ let uninstallStatus: number | undefined
+ let armed = true
+ vi.spyOn(gw, 'getResource').mockImplementation(async (plural, name, ns) => {
+ const current = await MockGateway.prototype.getResource.call(gw, plural, name, ns)
+ if (armed && plural === 'contexts' && name === 'ctx-a') {
+ armed = false
+ // The install's read is taken; the uninstall of old-srv then runs to
+ // completion before the install gets to write.
+ const stale = structuredClone(current)
+ const uninstall = await request(app).delete('/admin/mcp-servers/old-srv')
+ uninstallStatus = uninstall.status
+ return stale
+ }
+ return current
+ })
+
+ const res = await request(app).post('/admin/mcp-servers/remote').send(installBody('new-srv'))
+
+ expect(uninstallStatus).toBe(200)
+ expect(res.status).toBe(201)
+ await expect(gw.getResource('mcpservers', 'old-srv', NS)).rejects.toThrow()
+ expect(await allowlist(gw)).toEqual(['new-srv'])
+ })
+})
diff --git a/control-api/test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.ts b/control-api/test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.ts
new file mode 100644
index 000000000..451a8c932
--- /dev/null
+++ b/control-api/test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.ts
@@ -0,0 +1,164 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import { randomUUID } from 'node:crypto'
+import { Pool } from 'pg'
+import request from 'supertest'
+import type { DbClient } from '../src/db.js'
+import type { K8sGateway } from '../src/k8s.js'
+import { MockGateway } from './mockGateway.js'
+
+/**
+ * R3-H7 (§7 row 5, T1/T3/T4) — a failed grants purge on the generic McpServer
+ * uninstall answers 503 repair_required with the CR still present, and repeating the
+ * DELETE once the DB is healthy completes it: 200, CR gone, and the installation's
+ * grant no longer readable.
+ *
+ * The grant is written by the REAL producer (`upsertOAuthGrant`, sealed with the CR
+ * uid) and read back by the REAL fenced reader (`getOAuthGrant`) on a real Postgres.
+ * The route's module-global `pool` is pointed at the test database; the only fault is
+ * the purge statement throwing on the first request.
+ *
+ * CONTROL_API_REAL_PG_ADMIN_URL=postgres://<u>:<p>@<host>:5432/<db> npm test -- \
+ * test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.ts
+ */
+
+const dbHolder = vi.hoisted(() => ({
+ target: null as null | { query: (text: string, values?: unknown[]) => Promise<unknown> },
+ failGrantsPurge: false,
+ // Evaluated per purge statement; lets a test fail only the post-delete purge.
+ failGrantsPurgeWhen: null as null | (() => Promise<boolean>),
+}))
+vi.mock('../src/db.js', async importActual => {
+ const actual = await importActual<typeof import('../src/db.js')>()
+ return {
+ ...actual,
+ pool: {
+ query: async (text: string, values?: unknown[]) => {
+ if (/DELETE\s+FROM\s+oauth_grants/i.test(text)) {
+ if (dbHolder.failGrantsPurge || (await dbHolder.failGrantsPurgeWhen?.())) {
+ throw new Error('simulated oauth_grants purge failure')
+ }
+ }
+ if (!dbHolder.target) throw new Error('test database not ready')
+ return dbHolder.target.query(text, values)
+ },
+ },
+ }
+})
+
+const { config } = await import('../src/config.js')
+const { initDb } = await import('../src/db.js')
+const { deriveOAuthEncryptionKey } = await import('../src/oauth/encryption.js')
+const { getOAuthGrant, upsertOAuthGrant } = await import('../src/oauth/store.js')
+const { createAdminResourcesRouter } = await import('../src/routes/admin/resources.js')
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+const KEY = deriveOAuthEncryptionKey(config.oauthEncryptionKey)
+const NS = config.mcpServersNamespace
+
+describeRealPostgres('generic McpServer uninstall — repair_required (real Postgres)', () => {
+ const database = `control_api_uninstall_${randomUUID().replace(/-/g, '')}`
+ let adminPool: Pool
+ let dbPool: Pool
+ let db: DbClient
+
+ beforeAll(async () => {
+ if (!adminUrl) throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required')
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)
+ dbPool = new Pool({ connectionString: databaseUrl(adminUrl, database) })
+ await initDb({ connect: () => dbPool.connect() })
+ db = { query: (text, values) => dbPool.query(text, values) }
+ dbHolder.target = db
+ })
+
+ afterAll(async () => {
+ dbHolder.target = null
+ await dbPool?.end()
+ if (adminPool) {
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)
+ await adminPool.end()
+ }
+ })
+
+ it('a failed purge keeps the CR (503); the repeated DELETE completes and no grant is visible', async () => {
+ const name = `srv-rr-${randomUUID().slice(0, 8)}`
+ const gw = new MockGateway(NS)
+ const cr = (await gw.createResource('mcpservers', { metadata: { name }, spec: {} }, NS)) as {
+ metadata: { uid: string }
+ }
+ const U = cr.metadata.uid
+ const grantKey = {
+ grantKind: 'user' as const,
+ ownerKind: 'mcpserver' as const,
+ recipeNamespace: NS,
+ recipeName: name,
+ userId: 'user-a',
+ oauthClientId: 'client',
+ }
+ await upsertOAuthGrant(db, KEY, {
+ ...grantKey,
+ provider: 'google',
+ accessToken: 'at-a',
+ crUid: U,
+ })
+ const app = express()
+ app.use(express.json())
+ app.use(createAdminResourcesRouter(gw as unknown as K8sGateway))
+
+ dbHolder.failGrantsPurge = true
+ let first: request.Response
+ try {
+ first = await request(app).delete(`/admin/mcp-servers/${name}`)
+ } finally {
+ dbHolder.failGrantsPurge = false
+ }
+
+ expect(first.status).toBe(503)
+ expect(first.body).toMatchObject({
+ error: 'mcp_server_uninstall_incomplete',
+ outcome: 'repair_required',
+ pending: ['oauth_grants'],
+ })
+ // Same installation, still there: the retry has the uid it needs.
+ await expect(gw.getResource('mcpservers', name, NS)).resolves.toMatchObject({
+ metadata: { uid: U },
+ })
+
+ // Only the post-delete purge (issued once the CR is gone) fails on the retry: the
+ // grant must already be gone from the pre-delete teardown, and the post-delete
+ // purge is hygiene that never changes the 200.
+ const crGone = async () => {
+ try {
+ await gw.getResource('mcpservers', name, NS)
+ return false
+ } catch {
+ return true
+ }
+ }
+ dbHolder.failGrantsPurgeWhen = crGone
+ let retry: request.Response
+ try {
+ retry = await request(app).delete(`/admin/mcp-servers/${name}`)
+ } finally {
+ dbHolder.failGrantsPurgeWhen = null
+ }
+
+ expect(retry.status).toBe(200)
+ await expect(gw.getResource('mcpservers', name, NS)).rejects.toThrow()
+ expect(await getOAuthGrant(db, KEY, { ...grantKey, crUid: U })).toBeNull()
+ })
+})
diff --git a/control-api/test/routes.mcpServerUninstall.repairRequired.test.ts b/control-api/test/routes.mcpServerUninstall.repairRequired.test.ts
new file mode 100644
index 000000000..0356f3584
--- /dev/null
+++ b/control-api/test/routes.mcpServerUninstall.repairRequired.test.ts
@@ -0,0 +1,171 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+import type { K8sGateway } from '../src/k8s.js'
+import { MockGateway } from './mockGateway.js'
+
+/**
+ * Generic McpServer uninstall (`DELETE /admin/mcp-servers/:name`): every cleanup runs
+ * before the CR delete, so a failed step answers 503 repair_required naming the step,
+ * leaves the CR in place, and the same DELETE completes the job (R3-H7).
+ *
+ * Observed on a stateful MockGateway (T4: what survives, not which calls were made).
+ * The OAuth teardown's DB is a benign empty stub; the grants path against a real
+ * Postgres lives in routes.mcpServerUninstall.repairRequired.realPostgres.
+ */
+
+const dbMock = vi.hoisted(() => ({
+ query: vi.fn(async (_text: string, _values?: unknown[]) => ({ rows: [], rowCount: 0 })),
+}))
+vi.mock('../src/db.js', async importActual => {
+ const actual = await importActual<typeof import('../src/db.js')>()
+ return { ...actual, pool: { query: dbMock.query } }
+})
+
+const { config } = await import('../src/config.js')
+const { createAdminResourcesRouter } = await import('../src/routes/admin/resources.js')
+
+const NS = config.mcpServersNamespace
+const SERVER = 'srv'
+const CREDENTIALS = `${SERVER}-credentials`
+
+let prevContextsNs = ''
+beforeEach(() => {
+ prevContextsNs = config.contextsNamespace
+ config.contextsNamespace = 'contexts-ns'
+ dbMock.query.mockClear()
+})
+afterEach(() => {
+ config.contextsNamespace = prevContextsNs
+ vi.restoreAllMocks()
+})
+
+async function setup() {
+ const gw = new MockGateway(NS)
+ await gw.createResource('mcpservers', { metadata: { name: SERVER }, spec: {} }, NS)
+ await gw.createResource(
+ 'contexts',
+ { metadata: { name: 'ctx-a' }, spec: { contextId: 'ctx-a', mcpServers: [SERVER, 'b'] } },
+ 'contexts-ns'
+ )
+ gw.seedSecret(CREDENTIALS, NS, { stringData: { TOKEN: 'x' } })
+ const app = express()
+ app.use(express.json())
+ app.use(createAdminResourcesRouter(gw as unknown as K8sGateway))
+ return { gw, app }
+}
+
+async function allowlist(gw: MockGateway): Promise<string[]> {
+ const ctx = (await gw.getResource('contexts', 'ctx-a', 'contexts-ns')) as {
+ spec: { mcpServers?: string[] }
+ }
+ return ctx.spec.mcpServers ?? []
+}
+
+const grantsPurges = () =>
+ dbMock.query.mock.calls.filter(([text]) => /DELETE\s+FROM\s+oauth_grants/i.test(text)).length
+
+const conflict = () =>
+ Object.assign(new Error('the object has been modified'), { statusCode: 409, code: 409 })
+const serverError = () =>
+ Object.assign(new Error('etcd unavailable'), { statusCode: 500, code: 500 })
+
+describe('DELETE /admin/mcp-servers/:name — cleanup before delete, 503 repair_required', () => {
+ it('a Context strip failure stops before any OAuth, Secret or CR change', async () => {
+ const { gw, app } = await setup()
+ vi.spyOn(gw, 'updateResource').mockRejectedValue(serverError())
+
+ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)
+
+ expect(res.status).toBe(503)
+ expect(res.body).toEqual({
+ error: 'mcp_server_uninstall_incomplete',
+ outcome: 'repair_required',
+ pending: ['contexts'],
+ deleted: [],
+ })
+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()
+ await expect(gw.getSecret(CREDENTIALS, NS)).resolves.toBeTruthy()
+ expect(grantsPurges()).toBe(0)
+ })
+
+ it('retries a Context 409 a bounded number of times, then reports contexts pending', async () => {
+ const { gw, app } = await setup()
+ const update = vi.spyOn(gw, 'updateResource').mockRejectedValue(conflict())
+
+ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)
+
+ expect(res.status).toBe(503)
+ expect(res.body.pending).toEqual(['contexts'])
+ expect(update).toHaveBeenCalledTimes(3)
+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()
+ })
+
+ it('absorbs a Context 409 from a concurrent status write and completes', async () => {
+ const { gw, app } = await setup()
+ const realUpdate = MockGateway.prototype.updateResource
+ let first = true
+ vi.spyOn(gw, 'updateResource').mockImplementation(async (plural, name, body, ns) => {
+ if (first && plural === 'contexts') {
+ first = false
+ throw conflict()
+ }
+ return realUpdate.call(gw, plural, name, body, ns)
+ })
+
+ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)
+
+ expect(res.status).toBe(200)
+ expect(await allowlist(gw)).toEqual(['b'])
+ })
+
+ it('a Secret delete failure keeps the CR and names secrets; the retry completes', async () => {
+ const { gw, app } = await setup()
+ const secretDelete = vi.spyOn(gw, 'deleteSecret').mockRejectedValueOnce(serverError())
+
+ const first = await request(app).delete(`/admin/mcp-servers/${SERVER}`)
+
+ expect(first.status).toBe(503)
+ expect(first.body.pending).toEqual(['secrets'])
+ expect(first.body.deleted).toEqual(['Context/ctx-a (removed from allowlist)'])
+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()
+ await expect(gw.getSecret(CREDENTIALS, NS)).resolves.toBeTruthy()
+
+ secretDelete.mockRestore()
+ const retry = await request(app).delete(`/admin/mcp-servers/${SERVER}`)
+
+ expect(retry.status).toBe(200)
+ await expect(gw.getResource('mcpservers', SERVER, NS)).rejects.toThrow()
+ await expect(gw.getSecret(CREDENTIALS, NS)).rejects.toThrow()
+ expect(await allowlist(gw)).toEqual(['b'])
+ })
+
+ it('an unreadable Secret stops the uninstall before any mutation', async () => {
+ const { gw, app } = await setup()
+ vi.spyOn(gw, 'getSecret').mockRejectedValueOnce(serverError())
+
+ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)
+
+ expect(res.status).toBe(503)
+ expect(res.body.pending).toEqual(['secrets'])
+ expect(await allowlist(gw)).toEqual([SERVER, 'b'])
+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()
+ expect(grantsPurges()).toBe(0)
+ })
+
+ it('a CR delete failure reports mcp_server with every dependency already cleaned', async () => {
+ const { gw, app } = await setup()
+ vi.spyOn(gw, 'deleteResource').mockRejectedValueOnce(serverError())
+
+ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)
+
+ expect(res.status).toBe(503)
+ expect(res.body.pending).toEqual(['mcp_server'])
+ expect(res.body.deleted).toEqual([
+ 'Context/ctx-a (removed from allowlist)',
+ `Secret/${CREDENTIALS}`,
+ ])
+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()
+ await expect(gw.getSecret(CREDENTIALS, NS)).rejects.toThrow()
+ })
+})
diff --git a/control-api/test/routes.registryInstall.test.ts b/control-api/test/routes.registryInstall.test.ts
index c329b2971..8749e51d6 100644
--- a/control-api/test/routes.registryInstall.test.ts
+++ b/control-api/test/routes.registryInstall.test.ts
@@ -1450,9 +1450,14 @@ describe('POST /admin/registry/install', () => {
})
})
- it('preserves dependencies after an ambiguous concurrent Context change', async () => {
+ // A concurrent Context writer moves the resourceVersion between the attach's read
+ // and its write. The attach re-reads and re-applies its edit to the fresh list
+ // instead of replaying the stale one (which would drop the other writer's change)
+ // or failing the install.
+ it('merges into a concurrently changed Context allowlist instead of replaying a stale one', async () => {
vi.mocked(getEntryVersion).mockResolvedValueOnce(MOCK_ENTRY)
vi.mocked(getCredentialSchema).mockResolvedValueOnce(MOCK_SCHEMA_REQUIRED)
+ vi.mocked(reportInstall).mockResolvedValueOnce({ acknowledged: true, stored: true })
const gw = new MockGateway('mcp-server')
await gw.createResource('contexts', {
@@ -1491,14 +1496,79 @@ describe('POST /admin/registry/install', () => {
registryEntryVersion: '1.0.0',
credentials: { AIRTABLE_API_KEY: ['context', 'race'].join('-') },
})
+ .expect(201)
+
+ expect(res.body.contextUpdated).toBe(true)
+ await expect(gw.getResource('contexts', 'default-context')).resolves.toMatchObject({
+ spec: { mcpServers: ['concurrent-writer', 'my-airtable'] },
+ })
+ await expect(gw.getResource('mcpservers', 'my-airtable', 'mcp-server')).resolves.toMatchObject({
+ metadata: { uid: expect.any(String), resourceVersion: expect.any(String) },
+ })
+ await expect(gw.getSecret('my-airtable-credentials', 'mcp-server')).resolves.toMatchObject({
+ metadata: { uid: expect.any(String), resourceVersion: expect.any(String) },
+ })
+ })
+
+ // Sibling of the merge above: the Context is rewritten before EVERY attach write, so
+ // the bounded retry runs out. The install must not guess — the CR and its Secret stay
+ // for repair and the other writers' entries are not clobbered.
+ it('gives up after the bounded attach retries when the Context keeps changing', async () => {
+ vi.mocked(getEntryVersion).mockResolvedValueOnce(MOCK_ENTRY)
+ vi.mocked(getCredentialSchema).mockResolvedValueOnce(MOCK_SCHEMA_REQUIRED)
+
+ const gw = new MockGateway('mcp-server')
+ await gw.createResource('contexts', {
+ metadata: { name: 'default-context' },
+ spec: { contextId: 'default-context', mcpServers: [] },
+ })
+ const originalUpdate = gw.updateResource.bind(gw)
+ let attachWrites = 0
+ vi.spyOn(gw, 'updateResource').mockImplementation(async (plural, name, body, ns) => {
+ // Only the attach writes carry the new server; the ambiguous-outcome fence
+ // rewrites the pre-attach snapshot and must reach the gateway untouched.
+ const servers = (body.spec as { mcpServers?: string[] }).mcpServers ?? []
+ if (plural === 'contexts' && servers.includes('my-airtable')) {
+ attachWrites += 1
+ const current = (await gw.getResource('contexts', name, ns)) as {
+ metadata: { resourceVersion: string }
+ spec: { mcpServers?: string[] }
+ }
+ await originalUpdate(
+ 'contexts',
+ name,
+ {
+ metadata: { resourceVersion: current.metadata.resourceVersion },
+ spec: {
+ ...current.spec,
+ mcpServers: [...(current.spec.mcpServers ?? []), `writer-${attachWrites}`],
+ },
+ },
+ ns
+ )
+ }
+ return originalUpdate(plural, name, body, ns)
+ })
+
+ const app = makeApp(gw as unknown as import('../src/k8s.js').K8sGateway)
+ const res = await request(app)
+ .post('/admin/registry/install')
+ .send({
+ serverName: 'my-airtable',
+ contextRef: 'default-context',
+ registryEntryName: 'airtable-mcp',
+ registryEntryVersion: '1.0.0',
+ credentials: { AIRTABLE_API_KEY: ['context', 'storm'].join('-') },
+ })
.expect(503)
expect(res.body).toMatchObject({
error: 'registry_install_outcome_ambiguous',
outcome: 'repair_required',
})
+ expect(attachWrites).toBe(3)
await expect(gw.getResource('contexts', 'default-context')).resolves.toMatchObject({
- spec: { mcpServers: ['concurrent-writer'] },
+ spec: { mcpServers: ['writer-1', 'writer-2', 'writer-3'] },
})
await expect(gw.getResource('mcpservers', 'my-airtable', 'mcp-server')).resolves.toMatchObject({
metadata: { uid: expect.any(String), resourceVersion: expect.any(String) },
diff --git a/control-api/test/routes.resources.test.ts b/control-api/test/routes.resources.test.ts
index f832d3e67..086e6393b 100644
--- a/control-api/test/routes.resources.test.ts
+++ b/control-api/test/routes.resources.test.ts
@@ -35,6 +35,9 @@ vi.mock('../src/db.js', async () => {
advisoryLockModelName: async () => {},
advisoryLockModelNames: async () => {},
boundCarrierTransactionIdleTimeout: async () => {},
+ // The McpServer uninstall runs its OAuth teardown before the CR delete and answers
+ // 503 when it fails, so it needs a DB that answers (empty: nothing to tear down).
+ pool: { query: async () => ({ rows: [], rowCount: 0 }) },
}
})
@@ -1243,21 +1246,24 @@ describe('routes/resources', () => {
config.contextsNamespace = 'contexts-ns'
config.mcpServersNamespace = 'mcpservers-ns'
- const gateway = {
- // The uninstall reads the CR before deleting to capture metadata.uid (R3-H5).
- getResource: vi
- .fn()
- .mockResolvedValue({ metadata: { name: 'mcp-a', uid: 'uid-mcp-a', resourceVersion: '1' } }),
- deleteResource: vi.fn().mockResolvedValue({ deleted: true }),
- deleteSecret: vi.fn().mockResolvedValue({ deleted: true }),
- listResource: vi.fn().mockResolvedValue([
- {
- metadata: { name: 'ctx-a' },
- spec: { contextId: 'ctx-a', description: 'desc', mcpServers: ['mcp-a', 'mcp-b'] },
- },
- ]),
- updateResource: vi.fn().mockResolvedValue({}),
- }
+ // Stateful gateway: the Context and the CR carry the uid/resourceVersion the
+ // apiserver always returns, which the fenced strip and delete depend on.
+ const gateway = new MockGateway('mcpservers-ns')
+ const cr = (await gateway.createResource(
+ 'mcpservers',
+ { metadata: { name: 'mcp-a' }, spec: {} },
+ 'mcpservers-ns'
+ )) as { metadata: { uid: string } }
+ await gateway.createResource(
+ 'contexts',
+ {
+ metadata: { name: 'ctx-a' },
+ spec: { contextId: 'ctx-a', description: 'desc', mcpServers: ['mcp-a', 'mcp-b'] },
+ },
+ 'contexts-ns'
+ )
+ const deleteSpy = vi.spyOn(gateway, 'deleteResource')
+ const updateSpy = vi.spyOn(gateway, 'updateResource')
try {
const app = express()
@@ -1266,21 +1272,20 @@ describe('routes/resources', () => {
await request(app).delete('/admin/mcp-servers/mcp-a').expect(200)
- // The CR delete is now fenced on the uid read back before deletion (R3-H5).
- expect(gateway.deleteResource).toHaveBeenCalledWith('mcpservers', 'mcp-a', 'mcpservers-ns', {
- uid: 'uid-mcp-a',
+ expect(deleteSpy).toHaveBeenCalledWith('mcpservers', 'mcp-a', 'mcpservers-ns', {
+ uid: cr.metadata.uid,
+ })
+ await expect(gateway.getResource('mcpservers', 'mcp-a', 'mcpservers-ns')).rejects.toThrow()
+ await expect(gateway.getResource('contexts', 'ctx-a', 'contexts-ns')).resolves.toMatchObject({
+ spec: { contextId: 'ctx-a', description: 'desc', mcpServers: ['mcp-b'] },
})
- expect(gateway.listResource).toHaveBeenCalledWith('contexts', 'contexts-ns')
- expect(gateway.updateResource).toHaveBeenCalledWith(
+ // The strip is fenced on the Context identity it read.
+ expect(updateSpy).toHaveBeenCalledWith(
'contexts',
'ctx-a',
- {
- spec: {
- contextId: 'ctx-a',
- description: 'desc',
- mcpServers: ['mcp-b'],
- },
- },
+ expect.objectContaining({
+ metadata: { uid: expect.any(String), resourceVersion: '1' },
+ }),
'contexts-ns'
)
} finally {